Skip to content

Commit d4139b7

Browse files
B4nanclaude
B4nan
and
claude
committed
chore: bump zod to v4
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent 8d6e3d9 commit d4139b7

6 files changed

Lines changed: 157 additions & 159 deletions

File tree

‎packages/core/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,6 @@
6868
"tough-cookie": "^6.0.0",
6969
"tslib": "^2.8.1",
7070
"type-fest": "^4.41.0",
71-
"zod": "^3.24.0"
71+
"zod": "^4.0.0"
7272
}
7373
}

‎packages/core/src/crawlers/crawler_commons.ts‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,12 @@ export type WithRequired<T, K extends keyof T> = T & { [P in K]-?: T[P] };
1919
export type LoadedRequest<R extends Request> = WithRequired<R, 'id' | 'loadedUrl'>;
2020

2121
/** @internal */
22-
export type LoadedContext<Context extends RestrictedCrawlingContext> = IsAny<Context> extends true
23-
? Context
24-
: {
25-
request: LoadedRequest<Context['request']>;
26-
} & Omit<Context, 'request'>;
22+
export type LoadedContext<Context extends RestrictedCrawlingContext> =
23+
IsAny<Context> extends true
24+
? Context
25+
: {
26+
request: LoadedRequest<Context['request']>;
27+
} & Omit<Context, 'request'>;
2728

2829
export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> {
2930
id: string;

‎test/browser-pool/anonymize-proxy-sugar.test.ts‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,10 @@ describe('anonymizeProxySugar', () => {
2020
['http://username:password@proxy:1000/', 'http://username:password@proxy:1000'],
2121
['socks://username:password@proxy:1000', 'socks://username:password@proxy:1000'],
2222
['socks://username:password@proxy:1000/', 'socks://username:password@proxy:1000'],
23-
])(
24-
'should call anonymizeProxy from proxy-chain with correctly pre-processed URL: %s',
25-
async (input, expectedOutput) => {
26-
const [anonymized] = await anonymizeProxySugar(input);
23+
])('should call anonymizeProxy from proxy-chain with correctly pre-processed URL: %s', async (input, expectedOutput) => {
24+
const [anonymized] = await anonymizeProxySugar(input);
2725

28-
expect(anonymizeProxy).toHaveBeenCalledWith(expectedOutput);
29-
expect(anonymized).toBeTypeOf('string');
30-
},
31-
);
26+
expect(anonymizeProxy).toHaveBeenCalledWith(expectedOutput);
27+
expect(anonymized).toBeTypeOf('string');
28+
});
3229
});

‎test/core/crawlers/adaptive_playwright_crawler.test.ts‎

Lines changed: 59 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -234,34 +234,34 @@ describe('AdaptivePlaywrightCrawler', () => {
234234
expect(resultChecker).toHaveBeenCalledTimes(1);
235235
});
236236

237-
test.each([['static'], ['clientOnly']] as const)(
238-
'crawlingContext.addRequests() should add requests correctly (%s)',
239-
async (renderingType) => {
240-
const renderingTypePredictor = makeRiggedRenderingTypePredictor({
241-
detectionProbabilityRecommendation: 0,
242-
renderingType,
243-
});
244-
const url = new URL(`http://${HOSTNAME}:${port}`).toString();
237+
test.each([
238+
['static'],
239+
['clientOnly'],
240+
] as const)('crawlingContext.addRequests() should add requests correctly (%s)', async (renderingType) => {
241+
const renderingTypePredictor = makeRiggedRenderingTypePredictor({
242+
detectionProbabilityRecommendation: 0,
243+
renderingType,
244+
});
245+
const url = new URL(`http://${HOSTNAME}:${port}`).toString();
245246

246-
let requestContext: LoadedContext<AdaptivePlaywrightCrawlerContext> | undefined;
247-
const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = async (context) => {
248-
const isStartUrl = context.request.url === url;
247+
let requestContext: LoadedContext<AdaptivePlaywrightCrawlerContext> | undefined;
248+
const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = async (context) => {
249+
const isStartUrl = context.request.url === url;
249250

250-
if (isStartUrl) await context.addRequests([`${url}/1`]);
251-
else requestContext = context;
252-
};
251+
if (isStartUrl) await context.addRequests([`${url}/1`]);
252+
else requestContext = context;
253+
};
253254

254-
const crawler = await makeOneshotCrawler(
255-
{ requestHandler, renderingTypePredictor, maxRequestsPerCrawl: 10 },
256-
[],
257-
);
255+
const crawler = await makeOneshotCrawler(
256+
{ requestHandler, renderingTypePredictor, maxRequestsPerCrawl: 10 },
257+
[],
258+
);
258259

259-
await crawler.run([{ url, crawlDepth: 2 }]);
260+
await crawler.run([{ url, crawlDepth: 2 }]);
260261

261-
assert(requestContext);
262-
expect(requestContext.request).toMatchObject({ url: `${url}/1`, crawlDepth: 3 });
263-
},
264-
);
262+
assert(requestContext);
263+
expect(requestContext.request).toMatchObject({ url: `${url}/1`, crawlDepth: 3 });
264+
});
265265

266266
describe('should enqueue links correctly', () => {
267267
test.each([
@@ -315,49 +315,49 @@ describe('AdaptivePlaywrightCrawler', () => {
315315
});
316316
});
317317

318-
test.each([['static'], ['clientOnly']] as const)(
319-
'should respect the strategy option for enqueueLinks (%s)',
320-
async (renderingType) => {
321-
const renderingTypePredictor = makeRiggedRenderingTypePredictor({
322-
detectionProbabilityRecommendation: 0,
323-
renderingType,
324-
});
325-
const url = new URL(`http://${HOSTNAME}:${port}/external-links`);
326-
const enqueuedUrls = new Set<string>();
327-
const visitedUrls = new Set<string>();
318+
test.each([
319+
['static'],
320+
['clientOnly'],
321+
] as const)('should respect the strategy option for enqueueLinks (%s)', async (renderingType) => {
322+
const renderingTypePredictor = makeRiggedRenderingTypePredictor({
323+
detectionProbabilityRecommendation: 0,
324+
renderingType,
325+
});
326+
const url = new URL(`http://${HOSTNAME}:${port}/external-links`);
327+
const enqueuedUrls = new Set<string>();
328+
const visitedUrls = new Set<string>();
328329

329-
const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(
330-
async ({ enqueueLinks, request }) => {
331-
visitedUrls.add(request.loadedUrl);
330+
const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(
331+
async ({ enqueueLinks, request }) => {
332+
visitedUrls.add(request.loadedUrl);
332333

333-
if (!request.label) {
334-
const result = await enqueueLinks({
335-
label: 'enqueued-url',
336-
strategy: 'same-hostname',
337-
});
334+
if (!request.label) {
335+
const result = await enqueueLinks({
336+
label: 'enqueued-url',
337+
strategy: 'same-hostname',
338+
});
338339

339-
for (const processedRequest of result.processedRequests) {
340-
enqueuedUrls.add(processedRequest.uniqueKey);
341-
}
340+
for (const processedRequest of result.processedRequests) {
341+
enqueuedUrls.add(processedRequest.uniqueKey);
342342
}
343-
},
344-
);
343+
}
344+
},
345+
);
345346

346-
const crawler = await makeOneshotCrawler(
347-
{
348-
requestHandler,
349-
renderingTypePredictor,
350-
maxRequestsPerCrawl: 10,
351-
},
352-
[url.toString()],
353-
);
347+
const crawler = await makeOneshotCrawler(
348+
{
349+
requestHandler,
350+
renderingTypePredictor,
351+
maxRequestsPerCrawl: 10,
352+
},
353+
[url.toString()],
354+
);
354355

355-
await crawler.run();
356+
await crawler.run();
356357

357-
expect(new Set(visitedUrls)).toEqual(new Set([`http://${HOSTNAME}:${port}/external-links`]));
358-
expect(new Set(enqueuedUrls)).toEqual(new Set([`http://${HOSTNAME}:${port}/external-redirect`]));
359-
},
360-
);
358+
expect(new Set(visitedUrls)).toEqual(new Set([`http://${HOSTNAME}:${port}/external-links`]));
359+
expect(new Set(enqueuedUrls)).toEqual(new Set([`http://${HOSTNAME}:${port}/external-redirect`]));
360+
});
361361

362362
test('should persist crawler state', async () => {
363363
const renderingTypePredictor = makeRiggedRenderingTypePredictor({

‎test/core/crawlers/basic_crawler.test.ts‎

Lines changed: 47 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -469,61 +469,61 @@ describe('BasicCrawler', () => {
469469
expect(state2.urls).toContain(`http://${HOSTNAME}:${port}/?page=2`);
470470
});
471471

472-
test.each([EventType.MIGRATING, EventType.ABORTING])(
473-
'should pause on %s event and persist RequestList state',
474-
async (event) => {
475-
const sources = [...Array(500).keys()].map((index) => ({ url: `https://example.com/${index + 1}` }));
476-
477-
let persistResolve!: (value?: unknown) => void;
478-
const persistPromise = new Promise((res) => {
479-
persistResolve = res;
480-
});
472+
test.each([
473+
EventType.MIGRATING,
474+
EventType.ABORTING,
475+
])('should pause on %s event and persist RequestList state', async (event) => {
476+
const sources = [...Array(500).keys()].map((index) => ({ url: `https://example.com/${index + 1}` }));
477+
478+
let persistResolve!: (value?: unknown) => void;
479+
const persistPromise = new Promise((res) => {
480+
persistResolve = res;
481+
});
481482

482-
// Mock the calls to persist sources.
483-
const getValueSpy = vitest.spyOn(KeyValueStore.prototype, 'getValue');
484-
const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue');
485-
getValueSpy.mockResolvedValue(null);
483+
// Mock the calls to persist sources.
484+
const getValueSpy = vitest.spyOn(KeyValueStore.prototype, 'getValue');
485+
const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue');
486+
getValueSpy.mockResolvedValue(null);
486487

487-
const processed: { url: string }[] = [];
488-
const requestList = await RequestList.open('reqList', sources);
489-
const requestHandler: RequestHandler = async ({ request }) => {
490-
if (request.url.endsWith('200')) events.emit(event);
491-
processed.push({ url: request.url });
492-
};
488+
const processed: { url: string }[] = [];
489+
const requestList = await RequestList.open('reqList', sources);
490+
const requestHandler: RequestHandler = async ({ request }) => {
491+
if (request.url.endsWith('200')) events.emit(event);
492+
processed.push({ url: request.url });
493+
};
493494

494-
const basicCrawler = new BasicCrawler({
495-
requestList,
496-
minConcurrency: 25,
497-
maxConcurrency: 25,
498-
requestHandler,
499-
});
495+
const basicCrawler = new BasicCrawler({
496+
requestList,
497+
minConcurrency: 25,
498+
maxConcurrency: 25,
499+
requestHandler,
500+
});
500501

501-
let finished = false;
502-
// Mock the call to persist state.
503-
setValueSpy.mockImplementationOnce(persistResolve as any);
504-
// The crawler will pause after 200 requests
505-
const runPromise = basicCrawler.run();
506-
void runPromise.then(() => {
507-
finished = true;
508-
});
502+
let finished = false;
503+
// Mock the call to persist state.
504+
setValueSpy.mockImplementationOnce(persistResolve as any);
505+
// The crawler will pause after 200 requests
506+
const runPromise = basicCrawler.run();
507+
void runPromise.then(() => {
508+
finished = true;
509+
});
509510

510-
// need to monkeypatch the stats class, otherwise it will never finish
511-
basicCrawler.stats.persistState = async () => Promise.resolve();
512-
await persistPromise;
511+
// need to monkeypatch the stats class, otherwise it will never finish
512+
basicCrawler.stats.persistState = async () => Promise.resolve();
513+
await persistPromise;
513514

514-
expect(finished).toBe(false);
515-
expect(await requestList.isFinished()).toBe(false);
516-
expect(await requestList.isEmpty()).toBe(false);
517-
expect(processed.length).toBe(200);
515+
expect(finished).toBe(false);
516+
expect(await requestList.isFinished()).toBe(false);
517+
expect(await requestList.isEmpty()).toBe(false);
518+
expect(processed.length).toBe(200);
518519

519-
expect(getValueSpy).toBeCalled();
520-
expect(setValueSpy).toBeCalled();
520+
expect(getValueSpy).toBeCalled();
521+
expect(setValueSpy).toBeCalled();
521522

522-
// clean up
523-
// @ts-expect-error Accessing private method
524-
await basicCrawler.autoscaledPool!._destroy();
525-
},
526-
);
523+
// clean up
524+
// @ts-expect-error Accessing private method
525+
await basicCrawler.autoscaledPool!._destroy();
526+
});
527527

528528
test('should retry failed requests', async () => {
529529
const sources = [

‎test/core/crawlers/playwright_crawler.test.ts‎

Lines changed: 39 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -149,47 +149,47 @@ describe('PlaywrightCrawler', () => {
149149
expect(Object.keys(options.browserPoolOptions).length).toBe(0);
150150
});
151151

152-
test.each([{ useIncognitoPages: true }, { useIncognitoPages: false }])(
153-
'should apply launchOptions with useIncognitoPages: $useIncognitoPages',
154-
async ({ useIncognitoPages }) => {
155-
// Some launch options apply to the browser, while some apply to the context.
156-
// Here we use some context options to verify that those are actually applied.
157-
const launchOptions = {
158-
locale: 'cz-CZ',
159-
reducedMotion: 'reduce' as const,
160-
timezoneId: 'Pacific/Tahiti',
161-
};
152+
test.each([
153+
{ useIncognitoPages: true },
154+
{ useIncognitoPages: false },
155+
])('should apply launchOptions with useIncognitoPages: $useIncognitoPages', async ({ useIncognitoPages }) => {
156+
// Some launch options apply to the browser, while some apply to the context.
157+
// Here we use some context options to verify that those are actually applied.
158+
const launchOptions = {
159+
locale: 'cz-CZ',
160+
reducedMotion: 'reduce' as const,
161+
timezoneId: 'Pacific/Tahiti',
162+
};
162163

163-
let [timezone, locale, reducedMotion] = ['', '', ''];
164+
let [timezone, locale, reducedMotion] = ['', '', ''];
164165

165-
const playwrightCrawler = new PlaywrightCrawler({
166-
maxConcurrency: 1,
167-
launchContext: {
168-
useIncognitoPages,
169-
launchOptions,
170-
},
171-
browserPoolOptions: {
172-
// don't overwrite locale with fingerprint's locale
173-
useFingerprints: false,
174-
},
175-
requestHandler: async ({ page }) => {
176-
[timezone, locale, reducedMotion] = await Promise.all([
177-
page.evaluate(() => Intl.DateTimeFormat().resolvedOptions().timeZone),
178-
page.evaluate(() => navigator.language),
179-
page.evaluate(() => {
180-
return window.matchMedia('(prefers-reduced-motion: reduce)').matches
181-
? 'reduce'
182-
: 'no-preference';
183-
}),
184-
]);
185-
},
186-
});
166+
const playwrightCrawler = new PlaywrightCrawler({
167+
maxConcurrency: 1,
168+
launchContext: {
169+
useIncognitoPages,
170+
launchOptions,
171+
},
172+
browserPoolOptions: {
173+
// don't overwrite locale with fingerprint's locale
174+
useFingerprints: false,
175+
},
176+
requestHandler: async ({ page }) => {
177+
[timezone, locale, reducedMotion] = await Promise.all([
178+
page.evaluate(() => Intl.DateTimeFormat().resolvedOptions().timeZone),
179+
page.evaluate(() => navigator.language),
180+
page.evaluate(() => {
181+
return window.matchMedia('(prefers-reduced-motion: reduce)').matches
182+
? 'reduce'
183+
: 'no-preference';
184+
}),
185+
]);
186+
},
187+
});
187188

188-
await playwrightCrawler.run([`http://${HOSTNAME}:${port}/`]);
189+
await playwrightCrawler.run([`http://${HOSTNAME}:${port}/`]);
189190

190-
expect(timezone).toBe(launchOptions.timezoneId);
191-
expect(locale).toBe(launchOptions.locale);
192-
expect(reducedMotion).toBe(launchOptions.reducedMotion);
193-
},
194-
);
191+
expect(timezone).toBe(launchOptions.timezoneId);
192+
expect(locale).toBe(launchOptions.locale);
193+
expect(reducedMotion).toBe(launchOptions.reducedMotion);
194+
});
195195
});

0 commit comments

Comments
 (0)