Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,10 @@ console.log(`Saved to: ${filePath}`);

## Configuration

### Runtime support

Node.js (>= 20) is the supported and tested runtime. The SDK also runs under Bun, with one caveat: Bun's bundled `undici` omits the optional `dns` interceptor, so DNS-caching is disabled there (requests fall back to the platform resolver and work normally). The SDK detects this automatically and skips the missing interceptor rather than failing.

### API Token

Get your API token from [Bright Data Control Panel](https://brightdata.com/cp/setting/users?=).
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
"format": "prettier . --write",
"typecheck": "tsc --noEmit --project tsconfig.json",
"sanity": "npm run lint && npm run typecheck",
"build": "npm run build:clean && rollup -c && npm run build:dts",
"build": "npm run build:clean && rollup -c && npm run build:dts && node scripts/smoke-dist.mjs",
"build:dts": "tsc --project tsconfig.dts.json",
"build:clean": "rm -rf ./dist",
"build:dev": "npm run build:clean && rollup --watch -c",
Expand Down
118 changes: 118 additions & 0 deletions scripts/add-js-extensions.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
/**
* One-time codemod: add explicit `.js` extensions to relative import/export
* specifiers across src/, so the emitted .d.ts resolve under node16/nodenext.
*
* AST-based (TypeScript compiler API) — covers ImportDeclaration,
* ExportDeclaration (incl. `export *` / `export type *`), inline `import('...')`
* type-nodes, and dynamic `import()` calls. Resolves each relative specifier to
* file (`./x` -> `./x.js`) or barrel directory (`./x` -> `./x/index.js`).
*
* Usage: node scripts/add-js-extensions.mjs [--dry]
* Not a runtime/published dependency — delete after the fix lands if desired.
*/
import ts from 'typescript';
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

const DRY = process.argv.includes('--dry');
const ROOT = path.resolve(fileURLToPath(import.meta.url), '../../src');

const RELATIVE = /^\.\.?\//;
const HAS_EXT = /\.(js|mjs|cjs|json|node)$/;

function walk(dir) {
const out = [];
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
const p = path.join(dir, e.name);
if (e.isDirectory()) out.push(...walk(p));
else if (e.name.endsWith('.ts') && !e.name.endsWith('.d.ts'))
out.push(p);
}
return out;
}

/** new specifier, or null to leave as-is */
function resolveSpecifier(fileDir, spec) {
if (!RELATIVE.test(spec)) return null; // bare package / node:
if (HAS_EXT.test(spec)) return null; // already extensioned
const base = spec.replace(/\/$/, '');
if (
fs.existsSync(path.join(fileDir, base + '.ts')) ||
fs.existsSync(path.join(fileDir, base + '.tsx'))
)
return base + '.js';
if (fs.existsSync(path.join(fileDir, base, 'index.ts')))
return base + '/index.js';
return null; // unresolved — report
}

const files = walk(ROOT);
let filesChanged = 0;
let specsChanged = 0;
const unresolved = [];

for (const file of files) {
const text = fs.readFileSync(file, 'utf8');
const sf = ts.createSourceFile(
file,
text,
ts.ScriptTarget.Latest,
/* setParentNodes */ true,
ts.ScriptKind.TS,
);
const fileDir = path.dirname(file);
const edits = [];

const consider = (lit) => {
if (!lit || !ts.isStringLiteral(lit)) return;
const spec = lit.text;
const next = resolveSpecifier(fileDir, spec);
if (next === null) {
if (RELATIVE.test(spec) && !HAS_EXT.test(spec))
unresolved.push(`${path.relative(ROOT, file)}: ${spec}`);
return;
}
const start = lit.getStart(sf);
const quote = text[start];
edits.push({ start, end: lit.getEnd(), newText: quote + next + quote });
};

const visit = (node) => {
if (ts.isImportDeclaration(node) || ts.isExportDeclaration(node)) {
consider(node.moduleSpecifier);
} else if (ts.isImportTypeNode(node)) {
const arg = node.argument;
if (arg && ts.isLiteralTypeNode(arg)) consider(arg.literal);
} else if (
ts.isCallExpression(node) &&
node.expression.kind === ts.SyntaxKind.ImportKeyword
) {
consider(node.arguments[0]);
}
ts.forEachChild(node, visit);
};
visit(sf);

if (edits.length) {
specsChanged += edits.length;
filesChanged++;
if (!DRY) {
edits.sort((a, b) => b.start - a.start);
let out = text;
for (const e of edits)
out = out.slice(0, e.start) + e.newText + out.slice(e.end);
fs.writeFileSync(file, out);
}
}
}

console.log(
`${DRY ? '[dry-run] ' : ''}files changed: ${filesChanged}, specifiers changed: ${specsChanged}`,
);
if (unresolved.length) {
console.log(`UNRESOLVED (${unresolved.length}) — left as-is, review:`);
for (const u of unresolved) console.log(' ' + u);
} else {
console.log('no unresolved relative specifiers');
}
113 changes: 113 additions & 0 deletions scripts/smoke-dist.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
#!/usr/bin/env node
// Built-package runtime smoke.
//
// Loads every published entry point against the *actual* dist/ output, in
// BOTH module formats (ESM via import(), CJS via require()), and constructs
// the client. A broken inter-module specifier surfaces here as a load-time
// throw (ERR_MODULE_NOT_FOUND / "Cannot find module") instead of in a
// consumer's app on the next release.
//
// Why this exists: the vitest suite runs src/ through the bundler, never the
// emitted dist/. After authoring `.js` extensions across every relative
// import in src/, "rollup exited 0" was the only thing standing behind the
// runtime output. This converts that into a real load of the shipped files.
// It is wired into `npm run build` (post-emit) rather than vitest, because CI
// runs the test suite before the build — the artifacts don't exist yet at
// test time.

import { createRequire } from 'node:module';
import { fileURLToPath } from 'node:url';
import path from 'node:path';

const require = createRequire(import.meta.url);
const here = path.dirname(fileURLToPath(import.meta.url)); // scripts/
const dist = path.join(here, '..', 'dist');

// Mirror the key the unit tests construct with — long enough to pass apiKey
// validation, no network performed on construction.
const API_KEY = 'test-key-1234567890';

// One per `exports` entry in package.json.
const entries = [
{ name: '.', esm: 'esm/index.mjs', cjs: 'cjs/index.cjs', hasClient: true },
{
name: './scrapers',
esm: 'esm/scrapers.mjs',
cjs: 'cjs/scrapers.cjs',
hasClient: false,
},
{
name: './search',
esm: 'esm/search.mjs',
cjs: 'cjs/search.cjs',
hasClient: false,
},
{
name: './datasets',
esm: 'esm/datasets.mjs',
cjs: 'cjs/datasets.cjs',
hasClient: false,
},
];

function assertNonEmpty(mod, label) {
const named = Object.keys(mod).filter((k) => k !== 'default');
const total = named.length + (mod.default ? 1 : 0);
if (total === 0) {
throw new Error(`${label}: module loaded but exported nothing`);
}
}

async function constructClient(mod, label) {
const Ctor = mod.bdclient ?? mod.default?.bdclient;
if (typeof Ctor !== 'function') {
throw new Error(
`${label}: bdclient export is not a constructor (got ${typeof Ctor})`,
);
}
const client = new Ctor({ apiKey: API_KEY });
if (!client) {
throw new Error(`${label}: client construction returned falsy`);
}
// Best-effort cleanup so the undici agent doesn't keep the process alive.
if (typeof client.close === 'function') {
await client.close();
}
}

let failures = 0;

for (const e of entries) {
// ESM — exercises the import() resolution path.
try {
const mod = await import(path.join(dist, e.esm));
assertNonEmpty(mod, `esm ${e.name}`);
if (e.hasClient) await constructClient(mod, `esm ${e.name}`);
console.log(` ✓ esm ${e.name}`);
} catch (err) {
failures++;
console.error(` ✗ esm ${e.name}: ${err.message}`);
}

// CJS — exercises the require() resolution path.
try {
const mod = require(path.join(dist, e.cjs));
assertNonEmpty(mod, `cjs ${e.name}`);
if (e.hasClient) await constructClient(mod, `cjs ${e.name}`);
console.log(` ✓ cjs ${e.name}`);
} catch (err) {
failures++;
console.error(` ✗ cjs ${e.name}: ${err.message}`);
}
}

if (failures > 0) {
console.error(
`\nsmoke-dist: ${failures} entry/format combination(s) failed to load from dist/.`,
);
process.exit(1);
}

console.log(
'smoke-dist: all 4 entries load in both ESM and CJS; client constructs.',
);
2 changes: 1 addition & 1 deletion src/api/browser/index.ts
Original file line number Diff line number Diff line change
@@ -1 +1 @@
export { BrowserService } from './service';
export { BrowserService } from './service.js';
6 changes: 3 additions & 3 deletions src/api/browser/service.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { assertSchema } from '../../schemas/utils';
import { BrowserConnectOptionsSchema } from '../../schemas/browser';
import type { BrowserConnectOptions } from '../../schemas/browser';
import { assertSchema } from '../../schemas/utils.js';
import { BrowserConnectOptionsSchema } from '../../schemas/browser.js';
import type { BrowserConnectOptions } from '../../schemas/browser.js';

export class BrowserService {
static readonly DEFAULT_HOST = 'brd.superproxy.io';
Expand Down
8 changes: 4 additions & 4 deletions src/api/crawler/index.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
export { CrawlerService } from './service';
export { CrawlResult } from './result';
export type { CrawlRecord, CrawlResultFields } from './result';
export { CrawlerService } from './service.js';
export { CrawlResult } from './result.js';
export type { CrawlRecord, CrawlResultFields } from './result.js';

// CrawlJob is an alias for ScrapeJob — the snapshot-job wrapper is generic.
// Re-exported under the crawler name so porters from Python keep the same vocabulary.
export { ScrapeJob as CrawlJob } from '../scrape/job';
export { ScrapeJob as CrawlJob } from '../scrape/job.js';
2 changes: 1 addition & 1 deletion src/api/crawler/result.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { BaseResult, type BaseResultFields } from '../../models/result';
import { BaseResult, type BaseResultFields } from '../../models/result.js';

export interface CrawlRecord {
url?: string;
Expand Down
20 changes: 10 additions & 10 deletions src/api/crawler/service.ts
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
import { API_ENDPOINT } from '../../utils/constants';
import { Transport, assertResponse } from '../../core/transport';
import { parseResponse } from '../../utils/misc';
import { getLogger } from '../../utils/logger';
import { assertSchema } from '../../schemas/utils';
import { API_ENDPOINT } from '../../utils/constants.js';
import { Transport, assertResponse } from '../../core/transport.js';
import { parseResponse } from '../../utils/misc.js';
import { getLogger } from '../../utils/logger.js';
import { assertSchema } from '../../schemas/utils.js';
import {
CrawlInputSchema,
CrawlOptionsSchema,
CrawlDownloadOptionsSchema,
type CrawlOptions,
type CrawlDownloadOptions,
} from '../../schemas/crawler';
import { SnapshotMetaResponseSchema } from '../../schemas/responses';
import { ScrapeJob } from '../scrape/job';
import { CrawlResult, type CrawlRecord } from './result';
import type { SnapshotOperations } from '../../types/datasets';
} from '../../schemas/crawler.js';
import { SnapshotMetaResponseSchema } from '../../schemas/responses.js';
import { ScrapeJob } from '../scrape/job.js';
import { CrawlResult, type CrawlRecord } from './result.js';
import type { SnapshotOperations } from '../../types/datasets.js';

const DATASET_ID = 'gd_m6gjtfmeh43we6cqc';
const PLATFORM = 'crawler';
Expand Down
12 changes: 6 additions & 6 deletions src/api/datasets/base.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
import { Transport, assertResponse } from '../../core/transport';
import { API_ENDPOINT } from '../../utils/constants';
import { parseJSON } from '../../utils/misc';
import { getLogger } from '../../utils/logger';
import { pollUntilReady } from '../../utils/polling';
import { Transport, assertResponse } from '../../core/transport.js';
import { API_ENDPOINT } from '../../utils/constants.js';
import { parseJSON } from '../../utils/misc.js';
import { getLogger } from '../../utils/logger.js';
import { pollUntilReady } from '../../utils/polling.js';
import type {
DatasetMetadata,
DatasetSnapshotStatus,
DatasetDownloadOptions,
DatasetQueryOptions,
} from './types';
} from './types.js';

export abstract class BaseDataset {
abstract readonly datasetId: string;
Expand Down
Loading