diff --git a/README.md b/README.md index 61333d2..ec9d0fa 100644 --- a/README.md +++ b/README.md @@ -319,6 +319,8 @@ brightdata scraper create [options] | `--name ` | Scraper template name (default: `cli-scraper-`) | | `--deliver-webhook ` | Webhook URL for the deliver stub (default: `https://example.com/webhook`) | | `--timeout ` | Polling timeout in seconds (default: `600`) | +| `--max-retries ` | Max retries on the AI-Flow concurrent-job cap 429 (default: `4`). See below. | +| `--no-retry` | Fail immediately on 429 instead of waiting. Same as `--max-retries 0`. | | `-o, --output ` | Write output to file | | `--json` / `--pretty` | JSON output (raw / indented) | | `--timing` | Show request timing | @@ -326,6 +328,21 @@ brightdata scraper create [options] > **Note:** The scraper is created with a placeholder webhook delivery target (`https://example.com/webhook`). You can reconfigure the actual delivery endpoint in the [Bright Data web UI](https://brightdata.com/cp/scrapers) after creation. +#### Concurrent-job cap & auto-backoff + +The Bright Data AI Flow caps concurrent `scraper create` generations per account (currently 3). If you exceed it, the API returns `429 Cannot run more than N jobs in parallel`. The CLI handles this automatically: it waits with exponential backoff + jitter and retries up to `--max-retries` times (default 4). During the wait the CLI prints status lines so you know it isn't hung: + +``` +Triggering AI generation... +Hit AI-Flow concurrent-job cap (429). Waiting 32s before retry 1/4... +Hit AI-Flow concurrent-job cap (429). Waiting 67s before retry 2/4... +Generating scraper... +``` + +If the cap is still hit after all retries, the CLI exits with a stderr note pointing at the half-built collector's dashboard URL so you can inspect or delete it manually (Bright Data does not yet expose programmatic collector deletion). + +Use `--no-retry` if you want the old fail-fast behavior — typically for scripts that prefer to handle backoff themselves. + **Examples** ```bash @@ -338,6 +355,17 @@ brightdata scraper create https://example.com/product/1 \ "Extract title, price, and image URL from this product page" \ --name my-product-scraper --pretty -o scraper-output.json +# Fan out 10 parallel creates — the CLI serialises automatically via 429 backoff +for url in $(cat urls.txt); do + brightdata scraper create "$url" "Extract title, price, ..." \ + --name "scraper-$(basename $url)" & +done; wait + +# Disable the auto-backoff (fail fast on 429) +brightdata scraper create https://example.com/product/1 \ + "Extract title, price, and image URL from this product page" \ + --no-retry + # Use a custom webhook delivery URL brightdata scraper create https://example.com/product/1 \ "Extract title, price, and image URL from this product page" \ diff --git a/src/__tests__/commands/scraper.test.ts b/src/__tests__/commands/scraper.test.ts index 39e7e0c..e142d56 100644 --- a/src/__tests__/commands/scraper.test.ts +++ b/src/__tests__/commands/scraper.test.ts @@ -58,6 +58,12 @@ import { parse_sync_timeout, is_realtime_page_limit_error, classify_dataset, + parse_max_retries, + build_ai_trigger_retry, + print_stub_recovery_note, + AI_TRIGGER_DEFAULT_RETRIES, + AI_TRIGGER_RETRY_BASE_MS, + AI_TRIGGER_RETRY_MAX_MS, } from '../../commands/scraper'; describe('commands/scraper', ()=>{ @@ -176,7 +182,7 @@ describe('commands/scraper', ()=>{ expect.objectContaining({ deliver: expect.objectContaining({type: 'webhook'}), }), - {timing: undefined} + expect.objectContaining({timing: undefined}) ); expect(mocks.post).toHaveBeenNthCalledWith( 2, @@ -184,7 +190,7 @@ describe('commands/scraper', ()=>{ '/dca/collectors/c_abc/automate_template', {description: 'extract title', urls: ['https://example.com/p/1']}, - {timing: undefined} + expect.objectContaining({timing: undefined}) ); expect(mocks.poll_until).toHaveBeenCalledTimes(1); expect(mocks.poll_until).toHaveBeenCalledWith( @@ -624,4 +630,224 @@ describe('commands/scraper', ()=>{ ); }); }); + + // PR-11: AI-Flow concurrent-job cap (429) auto-backoff. The + // mechanism in client.ts is generic; these tests assert the + // scraper command wires it with the right config + opts. + describe('parse_max_retries (PR-11)', ()=>{ + it('defaults to the AI-trigger default when undefined', ()=>{ + expect(parse_max_retries(undefined)) + .toBe(AI_TRIGGER_DEFAULT_RETRIES); + }); + + it('parses a non-negative integer', ()=>{ + expect(parse_max_retries('0')).toBe(0); + expect(parse_max_retries('1')).toBe(1); + expect(parse_max_retries('8')).toBe(8); + }); + + it('rejects negatives, floats, and non-numeric', ()=>{ + expect(()=>parse_max_retries('-1')).toThrow(/non-negative/); + expect(()=>parse_max_retries('1.5')).toThrow(/non-negative/); + expect(()=>parse_max_retries('abc')).toThrow(/non-negative/); + }); + }); + + describe('build_ai_trigger_retry (PR-11)', ()=>{ + it('returns the default schedule when no flags are set', ()=>{ + const cfg = build_ai_trigger_retry({}); + expect(cfg).toBeDefined(); + expect(cfg!.max_attempts).toBe(AI_TRIGGER_DEFAULT_RETRIES); + expect(cfg!.base_ms).toBe(AI_TRIGGER_RETRY_BASE_MS); + expect(cfg!.max_ms).toBe(AI_TRIGGER_RETRY_MAX_MS); + expect(typeof cfg!.on_retry).toBe('function'); + }); + + it('--max-retries overrides max_attempts', ()=>{ + const cfg = build_ai_trigger_retry({maxRetries: '8'}); + expect(cfg!.max_attempts).toBe(8); + // Base/ceiling stay at the AI-Flow values. + expect(cfg!.base_ms).toBe(AI_TRIGGER_RETRY_BASE_MS); + }); + + it('--no-retry disables retries entirely', ()=>{ + const cfg = build_ai_trigger_retry({noRetry: true}); + expect(cfg!.max_attempts).toBe(0); + }); + + it('on_retry emits a 429-specific stderr line when status=429', + ()=>{ + const error = vi.spyOn(console, 'error') + .mockImplementation(()=>{}); + const cfg = build_ai_trigger_retry({}); + cfg!.on_retry!({ + attempt: 1, max_attempts: 4, + delay_ms: 32_000, status: 429, + }); + const msg = error.mock.calls.map(c=>String(c[0])).join('\n'); + expect(msg).toMatch(/AI-Flow concurrent-job cap/); + expect(msg).toMatch(/32s/); + expect(msg).toMatch(/retry 1\/4/); + error.mockRestore(); + }); + + it('on_retry emits a generic transient line for non-429', ()=>{ + const error = vi.spyOn(console, 'error') + .mockImplementation(()=>{}); + const cfg = build_ai_trigger_retry({}); + cfg!.on_retry!({ + attempt: 2, max_attempts: 4, + delay_ms: 60_000, status: 503, + }); + const msg = error.mock.calls.map(c=>String(c[0])).join('\n'); + expect(msg).toMatch(/Transient error/); + expect(msg).toMatch(/status 503/); + error.mockRestore(); + }); + + it('on_retry handles network-error case (status=0)', ()=>{ + const error = vi.spyOn(console, 'error') + .mockImplementation(()=>{}); + const cfg = build_ai_trigger_retry({}); + cfg!.on_retry!({ + attempt: 1, max_attempts: 4, + delay_ms: 30_000, status: 0, + }); + const msg = error.mock.calls.map(c=>String(c[0])).join('\n'); + expect(msg).toMatch(/status network/); + error.mockRestore(); + }); + }); + + describe('print_stub_recovery_note (PR-11)', ()=>{ + it('prints a dashboard URL and a manual-deletion notice', ()=>{ + const error = vi.spyOn(console, 'error') + .mockImplementation(()=>{}); + print_stub_recovery_note('c_xyz'); + const msg = error.mock.calls.map(c=>String(c[0])).join('\n'); + expect(msg).toContain('c_xyz'); + expect(msg).toMatch( + /https:\/\/brightdata\.com\/cp\/scrapers\/c_xyz/); + expect(msg).toMatch(/inspect or delete it manually/); + expect(msg).toMatch(/does not yet expose programmatic/); + error.mockRestore(); + }); + + it('does nothing when collector_id is empty', ()=>{ + const error = vi.spyOn(console, 'error') + .mockImplementation(()=>{}); + print_stub_recovery_note(''); + expect(error).not.toHaveBeenCalled(); + error.mockRestore(); + }); + }); + + describe('handle_create_scraper retry + stub-nudge wiring (PR-11)', + ()=>{ + it('passes the AI-trigger retry config to the second post call', + async()=>{ + mocks.post + .mockResolvedValueOnce({id: 'c_abc'}) + .mockResolvedValueOnce({id: 'ia_xyz', queued: false}); + mocks.poll_until.mockResolvedValue({ + result: {status: 'done', completed_steps: []}, + attempts: 1, + }); + await handle_create_scraper('https://x.com', 'd', {}); + const ai_call_opts = mocks.post.mock.calls[1][3] as + {retry?: {max_attempts: number; base_ms: number}}; + expect(ai_call_opts.retry).toBeDefined(); + expect(ai_call_opts.retry!.max_attempts) + .toBe(AI_TRIGGER_DEFAULT_RETRIES); + expect(ai_call_opts.retry!.base_ms) + .toBe(AI_TRIGGER_RETRY_BASE_MS); + }); + + it('honors --max-retries on the AI-trigger', async()=>{ + mocks.post + .mockResolvedValueOnce({id: 'c_abc'}) + .mockResolvedValueOnce({id: 'ia_xyz', queued: false}); + mocks.poll_until.mockResolvedValue({ + result: {status: 'done', completed_steps: []}, + attempts: 1, + }); + await handle_create_scraper('https://x.com', 'd', + {maxRetries: '10'}); + const ai_call_opts = mocks.post.mock.calls[1][3] as + {retry?: {max_attempts: number}}; + expect(ai_call_opts.retry!.max_attempts).toBe(10); + }); + + it('honors --no-retry (max_attempts = 0)', async()=>{ + mocks.post + .mockResolvedValueOnce({id: 'c_abc'}) + .mockResolvedValueOnce({id: 'ia_xyz', queued: false}); + mocks.poll_until.mockResolvedValue({ + result: {status: 'done', completed_steps: []}, + attempts: 1, + }); + await handle_create_scraper('https://x.com', 'd', + {noRetry: true}); + const ai_call_opts = mocks.post.mock.calls[1][3] as + {retry?: {max_attempts: number}}; + expect(ai_call_opts.retry!.max_attempts).toBe(0); + }); + + it('emits the stub-recovery note when AI-trigger ultimately fails', + async()=>{ + mocks.post + .mockResolvedValueOnce({id: 'c_stub'}) + .mockRejectedValueOnce( + new Error('Cannot run more than 3 jobs in parallel')); + const exit = vi.spyOn(process, 'exit') + .mockImplementation(()=>undefined as never); + const error = vi.spyOn(console, 'error') + .mockImplementation(()=>{}); + await handle_create_scraper('https://x.com', 'd', {}); + const msg = error.mock.calls.map(c=>String(c[0])).join('\n'); + expect(msg).toContain('c_stub'); + expect(msg).toMatch(/inspect or delete it manually/); + exit.mockRestore(); + error.mockRestore(); + }); + + it('emits the stub-recovery note when poll status != done', + async()=>{ + mocks.post + .mockResolvedValueOnce({id: 'c_abc'}) + .mockResolvedValueOnce({id: 'ia_xyz', queued: false}); + mocks.poll_until.mockResolvedValue({ + result: {status: 'failed', completed_steps: []}, + attempts: 2, + }); + const exit = vi.spyOn(process, 'exit') + .mockImplementation(()=>undefined as never); + const error = vi.spyOn(console, 'error') + .mockImplementation(()=>{}); + await handle_create_scraper('https://x.com', 'd', {}); + const msg = error.mock.calls.map(c=>String(c[0])).join('\n'); + expect(msg).toMatch(/inspect or delete it manually/); + expect(msg).toContain('c_abc'); + exit.mockRestore(); + error.mockRestore(); + }); + + it('emits the stub-recovery note when polling itself throws', + async()=>{ + mocks.post + .mockResolvedValueOnce({id: 'c_abc'}) + .mockResolvedValueOnce({id: 'ia_xyz', queued: false}); + mocks.poll_until.mockRejectedValue( + new Error('Timeout after 600 seconds')); + const exit = vi.spyOn(process, 'exit') + .mockImplementation(()=>undefined as never); + const error = vi.spyOn(console, 'error') + .mockImplementation(()=>{}); + await handle_create_scraper('https://x.com', 'd', {}); + const msg = error.mock.calls.map(c=>String(c[0])).join('\n'); + expect(msg).toMatch(/inspect or delete it manually/); + exit.mockRestore(); + error.mockRestore(); + }); + }); }); diff --git a/src/__tests__/utils/client.retry.test.ts b/src/__tests__/utils/client.retry.test.ts new file mode 100644 index 0000000..30eced4 --- /dev/null +++ b/src/__tests__/utils/client.retry.test.ts @@ -0,0 +1,61 @@ +import {describe, it, expect} from 'vitest'; +import { + compute_backoff, + RETRY_BASE_MS, + RETRY_MAX_MS_DEFAULT, +} from '../../utils/client'; + +// PR-11: the per-request retry config + jitter is the mechanism that +// makes the AI-Flow concurrent-job cap (429) recoverable client-side. +// These tests cover the pure backoff math; the integration tests for +// the AI-trigger wiring live in commands/scraper.test.ts. + +describe('utils/client.compute_backoff', ()=>{ + it('grows exponentially with attempt', ()=>{ + // With base=1000, max=large: attempt 0 → exp=1000 → range [500,1000] + // attempt 1 → exp=2000 → range [1000,2000]; etc. Take min of many + // samples so jitter doesn't flake the test. + const sample_min = (attempt: number)=>{ + let m = Infinity; + for (let i = 0; i < 50; i++) + { + const d = compute_backoff(attempt, 1000, 1_000_000); + if (d < m) m = d; + } + return m; + }; + expect(sample_min(0)).toBeGreaterThanOrEqual(500); + expect(sample_min(1)).toBeGreaterThanOrEqual(1000); + expect(sample_min(2)).toBeGreaterThanOrEqual(2000); + }); + + it('honors the max_ms ceiling', ()=>{ + // At attempt 20, base * 2^20 would be huge; max_ms should cap it. + for (let i = 0; i < 50; i++) + { + const d = compute_backoff(20, 1000, 5000); + expect(d).toBeLessThanOrEqual(5000); + } + }); + + it('uses full-jitter (delay falls in [exp/2, exp])', ()=>{ + // Sample many to ensure both halves of the range are reachable. + const samples: number[] = []; + for (let i = 0; i < 200; i++) + samples.push(compute_backoff(0, 1000, 1_000_000)); + // exp = 1000, so range is [500, 1000]. + expect(Math.min(...samples)).toBeGreaterThanOrEqual(500); + expect(Math.max(...samples)).toBeLessThanOrEqual(1000); + // With 200 samples we'd expect both halves to be hit. + const below_mid = samples.filter(d=>d < 750).length; + const above_mid = samples.filter(d=>d >= 750).length; + expect(below_mid).toBeGreaterThan(20); + expect(above_mid).toBeGreaterThan(20); + }); + + it('exported defaults match the documented short schedule', ()=>{ + // Sanity: don't accidentally make every command wait minutes. + expect(RETRY_BASE_MS).toBe(500); + expect(RETRY_MAX_MS_DEFAULT).toBe(16_000); + }); +}); diff --git a/src/commands/scraper.ts b/src/commands/scraper.ts index 9ab4400..8cc9167 100644 --- a/src/commands/scraper.ts +++ b/src/commands/scraper.ts @@ -1,5 +1,5 @@ import {Command} from 'commander'; -import {post, get} from '../utils/client'; +import {post, get, type Retry_config, type Retry_event} from '../utils/client'; import {load as load_config} from '../utils/config'; import {ensure_authenticated} from '../utils/auth'; import {start as start_spinner} from '../utils/spinner'; @@ -37,6 +37,73 @@ const BATCH_POLL_INTERVAL_MS = 10_000; const BATCH_TIMEOUT_DEFAULT = 3600; const REALTIME_LIMIT_MARKER = 'realtime job limit'; +// PR-11: AI-Flow generation is undocumentedly capped at 3 concurrent +// jobs per account. The API surfaces this as a 429 on the AI-trigger +// POST. The default client retry schedule (500ms × 2^N up to 16s) is +// way too short for this — generation takes 2–11 minutes to free a +// slot. These constants size the retry schedule for that reality. +const AI_TRIGGER_RETRY_BASE_MS = 30_000; // first wait ~15–30s +const AI_TRIGGER_RETRY_MAX_MS = 240_000; // ceiling ~120–240s +const AI_TRIGGER_DEFAULT_RETRIES = 4; // ~7.5 min total max wait + +const parse_max_retries = (raw: string|undefined): number=>{ + if (raw == null) return AI_TRIGGER_DEFAULT_RETRIES; + const n = +raw; + if (!Number.isFinite(n) || n < 0 || !Number.isInteger(n)) + throw new Error( + `Invalid --max-retries "${raw}". ` + +'Must be a non-negative integer.'); + return n; +}; + +const build_ai_trigger_retry = ( + opts: Pick +): Retry_config|undefined=>{ + if (opts.noRetry) + return {max_attempts: 0}; // no retries at all + const max_attempts = parse_max_retries(opts.maxRetries); + return { + max_attempts, + base_ms: AI_TRIGGER_RETRY_BASE_MS, + max_ms: AI_TRIGGER_RETRY_MAX_MS, + on_retry: (e: Retry_event)=>{ + const seconds = Math.round(e.delay_ms / 1000); + if (e.status === 429) + { + console.error(dim( + `Hit AI-Flow concurrent-job cap (429). Waiting ` + +`${seconds}s before retry ${e.attempt}/` + +`${e.max_attempts}...`)); + } + else + { + console.error(dim( + `Transient error (status ${e.status || 'network'}). ` + +`Waiting ${seconds}s before retry ${e.attempt}/` + +`${e.max_attempts}...`)); + } + }, + }; +}; + +// PR-11: shown once after every terminal failure that leaves a stub +// collector in the dashboard. Composes with PR-2's envelope, which +// also surfaces the view_url in the -o file. Repeated at the TTY so +// the user sees it without parsing the file. +// +// Why this exists: Bright Data does not yet expose a programmatic +// DELETE for collectors, so we can't auto-clean. The best we can do +// is loudly point the user at the dashboard URL. +const print_stub_recovery_note = (collector_id: string): void=>{ + if (!collector_id) return; + console.error(dim( + `Note: a half-built collector was created at ${collector_id}.\n` + +`Open https://brightdata.com/cp/scrapers/${collector_id} ` + +'to inspect or delete it manually in the web UI.\n' + +'(Bright Data does not yet expose programmatic deletion.)' + )); +}; + const build_template_request = ( opts: Scraper_create_opts ): Create_template_request=>({ @@ -124,12 +191,20 @@ const handle_create_scraper = async( return; } const trigger_spinner = start_spinner('Triggering AI generation...'); + let ai_retry: Retry_config|undefined; + try { + ai_retry = build_ai_trigger_retry(opts); + } catch(e) { + trigger_spinner.stop(); + fail((e as Error).message); + return; + } try { await post( api_key, `/dca/collectors/${collector_id}/${AI_TRIGGER_PATH}`, build_ai_request(url, description), - {timing: opts.timing} + {timing: opts.timing, retry: ai_retry} ); trigger_spinner.stop(); } catch(e) { @@ -138,6 +213,7 @@ const handle_create_scraper = async( `Failed to start AI generation for collector ` +`${collector_id}: ${(e as Error).message}` ); + print_stub_recovery_note(collector_id); process.exit(1); return; } @@ -171,6 +247,7 @@ const handle_create_scraper = async( `AI generation failed (collector ${collector_id}, ` +`status: ${progress.status}).` ); + print_stub_recovery_note(collector_id); process.exit(1); return; } @@ -189,6 +266,7 @@ const handle_create_scraper = async( const suffix = msg.includes(collector_id) ? '' : ` (collector ${collector_id})`; console.error(`${msg}${suffix}`); + print_stub_recovery_note(collector_id); process.exit(1); return; } @@ -544,6 +622,13 @@ const create_subcommand = new Command('create') +'(default: https://example.com/webhook)') .option('--timeout ', 'Polling timeout in seconds (default: 600)') + .option('--max-retries ', + 'Max retries on the AI-Flow concurrent-job cap 429 ' + +`(default: ${AI_TRIGGER_DEFAULT_RETRIES}). Each wait grows ` + +'exponentially with jitter, up to ~4 min between attempts.') + .option('--no-retry', + 'Fail immediately on 429 instead of waiting through the cap. ' + +'Equivalent to --max-retries 0.') .option('-o, --output ', 'Write output to file') .option('--json', 'Force JSON output') .option('--pretty', 'Pretty-print JSON output') @@ -592,4 +677,10 @@ export { parse_sync_timeout, is_realtime_page_limit_error, classify_dataset, + parse_max_retries, + build_ai_trigger_retry, + print_stub_recovery_note, + AI_TRIGGER_DEFAULT_RETRIES, + AI_TRIGGER_RETRY_BASE_MS, + AI_TRIGGER_RETRY_MAX_MS, }; diff --git a/src/types/scraper.ts b/src/types/scraper.ts index 74dbbcf..7551a22 100644 --- a/src/types/scraper.ts +++ b/src/types/scraper.ts @@ -45,6 +45,9 @@ type Scraper_create_opts = { pretty?: boolean; timing?: boolean; apiKey?: string; + // PR-11: AI-Flow concurrent-job cap (429) handling. + maxRetries?: string; + noRetry?: boolean; }; type Run_request = { diff --git a/src/utils/client.ts b/src/utils/client.ts index 6930232..d3be12c 100644 --- a/src/utils/client.ts +++ b/src/utils/client.ts @@ -3,6 +3,7 @@ import {load as load_config} from './config'; const TRANSIENT_STATUSES = [429, 500, 502, 503, 504]; const MAX_RETRIES = 3; const RETRY_BASE_MS = 500; +const RETRY_MAX_MS_DEFAULT = 16_000; // short-schedule ceiling const ERROR_HINTS: Record = { 401: 'Invalid or expired API key. Run \'brightdata login\' to re-authenticate.', @@ -11,12 +12,30 @@ const ERROR_HINTS: Record = { 429: 'Rate limit exceeded. Wait a moment and try again.', }; +// PR-11: per-request retry config. Lets callers (e.g. `scraper create`'s +// AI-trigger POST, which hits an undocumented 3-job concurrent cap) use +// a longer backoff than the generic transient-error schedule. +type Retry_event = { + attempt: number; // 1-based: the retry # about to be slept + max_attempts: number; + delay_ms: number; + status: number; // the HTTP status that triggered the retry +}; + +type Retry_config = { + max_attempts?: number; // total retries after the first attempt + base_ms?: number; // multiplied by 2^(attempt-1) + max_ms?: number; // cap on the exponential + on_retry?: (e: Retry_event)=>void; +}; + type Request_opts = { method?: string; body?: unknown; headers?: Record; timing?: boolean; raw_buffer?: boolean; + retry?: Retry_config; }; type Api_error = { @@ -33,6 +52,19 @@ const format_error = (status: number, detail: string): Api_error=>({ hint: ERROR_HINTS[status], }); +// PR-11: exponential delay with full-jitter (delay ∈ [exp/2, exp]). +// Full-jitter spreads herds of concurrent processes effectively even +// when they all 429 on the same tick. +const compute_backoff = ( + attempt: number, + base_ms: number, + max_ms: number +): number=>{ + const exp = Math.min(base_ms * 2 ** attempt, max_ms); + const jitter = exp * 0.5 * Math.random(); + return Math.floor(exp / 2 + jitter); +}; + const request = async( api_key: string, endpoint: string, @@ -54,15 +86,18 @@ const request = async( }; if (opts.body !== undefined) fetch_opts.body = JSON.stringify(opts.body); + const max_attempts = opts.retry?.max_attempts ?? MAX_RETRIES; + const base_ms = opts.retry?.base_ms ?? RETRY_BASE_MS; + const max_ms = opts.retry?.max_ms ?? RETRY_MAX_MS_DEFAULT; let attempt = 0; let start = opts.timing ? Date.now() : 0; - while (attempt <= MAX_RETRIES) + while (attempt <= max_attempts) { try { const res = await fetch(url, fetch_opts); if (opts.timing) { - console.error(`Timing: ${Date.now()-start}ms + console.error(`Timing: ${Date.now()-start}ms (attempt ${attempt+1})`); } const brd_error = res.headers.get('x-brd-error') @@ -79,10 +114,16 @@ const request = async( return await res.json() as T; return await res.text() as unknown as T; } - if (TRANSIENT_STATUSES.includes(res.status) && - attempt < MAX_RETRIES) + if (TRANSIENT_STATUSES.includes(res.status) && + attempt < max_attempts) { - const delay = RETRY_BASE_MS * 2**attempt; + const delay = compute_backoff(attempt, base_ms, max_ms); + opts.retry?.on_retry?.({ + attempt: attempt + 1, + max_attempts, + delay_ms: delay, + status: res.status, + }); await sleep(delay); attempt++; continue; @@ -104,9 +145,15 @@ const request = async( } catch(e) { if (e instanceof Error && e.message.startsWith('Error:')) throw e; - if (attempt < MAX_RETRIES) + if (attempt < max_attempts) { - const delay = RETRY_BASE_MS * 2**attempt; + const delay = compute_backoff(attempt, base_ms, max_ms); + opts.retry?.on_retry?.({ + attempt: attempt + 1, + max_attempts, + delay_ms: delay, + status: 0, // 0 = pre-response (network) error + }); await sleep(delay); attempt++; continue; @@ -133,5 +180,6 @@ const get = ( opts: Omit = {} ): Promise=>request(api_key, endpoint, {method: 'GET', ...opts}); -export {request, post, get}; -export type {Request_opts, Api_error}; +export {request, post, get, compute_backoff, + RETRY_BASE_MS, RETRY_MAX_MS_DEFAULT, MAX_RETRIES}; +export type {Request_opts, Api_error, Retry_config, Retry_event};