From 370cb02201b5b8a2732c14c253a4437bf2b69667 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 20 Feb 2026 13:16:46 +0000 Subject: [PATCH 1/9] Initial plan From 544376cb79fb9fe19c6ed175902750b94e513aa8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 20 Feb 2026 13:23:29 +0000 Subject: [PATCH 2/9] fix: treat all CookieJar methods as potentially throwing Co-authored-by: barjin <61918049+barjin@users.noreply.github.com> --- packages/core/src/session_pool/session.ts | 6 +++++- packages/http-client/src/base-http-client.ts | 20 ++++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/packages/core/src/session_pool/session.ts b/packages/core/src/session_pool/session.ts index 95c5cd7378f2..a80c7552632b 100644 --- a/packages/core/src/session_pool/session.ts +++ b/packages/core/src/session_pool/session.ts @@ -364,7 +364,11 @@ export class Session implements ISession { * Sets a cookie within this session for the specific URL. */ setCookie(rawCookie: string, url: string): void { - this.cookieJar.setCookieSync(rawCookie, url); + try { + this.cookieJar.setCookieSync(rawCookie, url); + } catch (e) { + this.log.debug('Could not set cookie.', { url, error: (e as Error).message }); + } } /** diff --git a/packages/http-client/src/base-http-client.ts b/packages/http-client/src/base-http-client.ts index f2831f0af598..249f035e4c60 100644 --- a/packages/http-client/src/base-http-client.ts +++ b/packages/http-client/src/base-http-client.ts @@ -18,10 +18,15 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface { protected abstract fetch(input: Request, init?: RequestInit & CustomFetchOptions): Promise; private async applyCookies(request: Request, cookieJar: CookieJar): Promise { - const cookies = (await cookieJar.getCookies(request.url)).map((x) => x.cookieString().trim()).filter(Boolean); + try { + const cookies = (await cookieJar.getCookies(request.url)).map((x) => x.cookieString().trim()).filter(Boolean); - if (cookies?.length > 0) { - request.headers.set('cookie', cookies.join('; ')); + if (cookies?.length > 0) { + request.headers.set('cookie', cookies.join('; ')); + } + } catch (e) { + // eslint-disable-next-line no-console + console.warn(`Failed to get cookies for URL "${request.url}": ${(e as Error).message}`); } return request; } @@ -29,7 +34,14 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface { private async setCookies(response: Response, cookieJar: CookieJar): Promise { const setCookieHeaders = response.headers.getSetCookie(); - await Promise.all(setCookieHeaders.map((header) => cookieJar.setCookie(header, response.url))); + for (const header of setCookieHeaders) { + try { + await cookieJar.setCookie(header, response.url); + } catch (e) { + // eslint-disable-next-line no-console + console.warn(`Failed to set cookie for URL "${response.url}": ${(e as Error).message}`); + } + } } private resolveRequestContext(options?: SendRequestOptions): { From d4cf37a6cb585f0564db67b20620bfcbb747fdb5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 20 Feb 2026 13:23:57 +0000 Subject: [PATCH 3/9] fix: treat all CookieJar methods as potentially throwing Co-authored-by: barjin <61918049+barjin@users.noreply.github.com> --- packages/http-client/src/base-http-client.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/http-client/src/base-http-client.ts b/packages/http-client/src/base-http-client.ts index 249f035e4c60..3f391dec056c 100644 --- a/packages/http-client/src/base-http-client.ts +++ b/packages/http-client/src/base-http-client.ts @@ -19,7 +19,9 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface { private async applyCookies(request: Request, cookieJar: CookieJar): Promise { try { - const cookies = (await cookieJar.getCookies(request.url)).map((x) => x.cookieString().trim()).filter(Boolean); + const cookies = (await cookieJar.getCookies(request.url)) + .map((x) => x.cookieString().trim()) + .filter(Boolean); if (cookies?.length > 0) { request.headers.set('cookie', cookies.join('; ')); From 6b9e2462f0c2a19101c6d743417aa642916d306f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Fri, 20 Feb 2026 16:03:50 +0100 Subject: [PATCH 4/9] chore: log warnings, guard more methods --- packages/core/src/session_pool/session.ts | 20 +++++++++++++++----- packages/http-client/package.json | 1 + packages/http-client/src/base-http-client.ts | 14 ++++++++++---- test/core/session_pool/session.test.ts | 16 +++++++++++++++- yarn.lock | 18 ++++++++++++++++++ 5 files changed, 59 insertions(+), 10 deletions(-) diff --git a/packages/core/src/session_pool/session.ts b/packages/core/src/session_pool/session.ts index a80c7552632b..172c6c84e310 100644 --- a/packages/core/src/session_pool/session.ts +++ b/packages/core/src/session_pool/session.ts @@ -346,8 +346,13 @@ export class Session implements ISession { * @param url website url. Only cookies stored for this url will be returned */ getCookies(url: string): CookieObject[] { - const cookies = this.cookieJar.getCookiesSync(url); - return cookies.map((c) => toughCookieToBrowserPoolCookie(c)); + try { + const cookies = this.cookieJar.getCookiesSync(url); + return cookies.map((c) => toughCookieToBrowserPoolCookie(c)); + } catch (e) { + this.log.warning('Could not get cookies from cookie jar.', { url, error: (e as Error).message }); + return []; + } } /** @@ -357,7 +362,12 @@ export class Session implements ISession { * @returns Represents `Cookie` header. */ getCookieString(url: string): string { - return this.cookieJar.getCookieStringSync(url, {}); + try { + return this.cookieJar.getCookieStringSync(url, {}); + } catch (e) { + this.log.warning('Could not get cookie string.', { url, error: (e as Error).message }); + return ''; + } } /** @@ -367,7 +377,7 @@ export class Session implements ISession { try { this.cookieJar.setCookieSync(rawCookie, url); } catch (e) { - this.log.debug('Could not set cookie.', { url, error: (e as Error).message }); + this.log.warning('Could not set cookie.', { url, error: (e as Error).message }); } } @@ -388,7 +398,7 @@ export class Session implements ISession { // if invalid cookies are provided just log the exception. No need to retry the request automatically. if (errorMessages.length) { - this.log.debug('Could not set cookies.', { errorMessages }); + this.log.warning('Could not set cookies.', { errorMessages }); } } diff --git a/packages/http-client/package.json b/packages/http-client/package.json index e7d4c5b4f3f0..f94278cc9e54 100644 --- a/packages/http-client/package.json +++ b/packages/http-client/package.json @@ -47,6 +47,7 @@ "access": "public" }, "dependencies": { + "@apify/log": "^2.5.32", "@crawlee/types": "4.0.0", "tough-cookie": "^6.0.0" } diff --git a/packages/http-client/src/base-http-client.ts b/packages/http-client/src/base-http-client.ts index 3f391dec056c..f37c63bd9d6b 100644 --- a/packages/http-client/src/base-http-client.ts +++ b/packages/http-client/src/base-http-client.ts @@ -1,6 +1,9 @@ import type { BaseHttpClient as BaseHttpClientInterface, SendRequestOptions } from '@crawlee/types'; import { CookieJar } from 'tough-cookie'; +import type { Log } from '@apify/log'; +import defaultLog from '@apify/log'; + export interface CustomFetchOptions { proxyUrl?: string; } @@ -11,6 +14,11 @@ export interface CustomFetchOptions { * implement only the low-level network call in `fetch`. */ export abstract class BaseHttpClient implements BaseHttpClientInterface { + protected log: Log; + + constructor(log?: Log) { + this.log = log ?? defaultLog; + } /** * Perform the raw network request and return a single Response without any * automatic redirect following or special error handling. @@ -27,8 +35,7 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface { request.headers.set('cookie', cookies.join('; ')); } } catch (e) { - // eslint-disable-next-line no-console - console.warn(`Failed to get cookies for URL "${request.url}": ${(e as Error).message}`); + this.log.warning(`Failed to get cookies for URL "${request.url}": ${(e as Error).message}`); } return request; } @@ -40,8 +47,7 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface { try { await cookieJar.setCookie(header, response.url); } catch (e) { - // eslint-disable-next-line no-console - console.warn(`Failed to set cookie for URL "${response.url}": ${(e as Error).message}`); + this.log.warning(`Failed to set cookie for URL "${response.url}": ${(e as Error).message}`); } } } diff --git a/test/core/session_pool/session.test.ts b/test/core/session_pool/session.test.ts index 03bb260b916e..ec1947e92747 100644 --- a/test/core/session_pool/session.test.ts +++ b/test/core/session_pool/session.test.ts @@ -1,4 +1,4 @@ -import { EVENT_SESSION_RETIRED, Session, SessionPool } from '@crawlee/core'; +import { EVENT_SESSION_RETIRED, log, Session, SessionPool } from '@crawlee/core'; import { ResponseWithUrl } from '@crawlee/http-client'; import { entries, sleep } from '@crawlee/utils'; import { CookieJar } from 'tough-cookie'; @@ -207,6 +207,20 @@ describe('Session - testing session behaviour ', () => { expect(session.getCookieString(url)).toBe('cookie2=your-cookie'); }); + test('setCookies will log warning (not throw) on invalid cookies', () => { + const url = 'https://www.example.com'; + const cookies = [{ name: 'cookie1', value: 'my-cookie', domain: 'abc.different.domain' }]; + + const mockedLog = vitest.mockObject(log, { + spy: true, + }); + + session = new Session({ sessionPool, log: mockedLog } as any); + session.setCookies(cookies, url); + expect(session.getCookieString(url)).toBe(''); + expect(mockedLog.warning).toHaveBeenCalledOnce(); + }); + test('setCookies works with hostOnly cookies', () => { const url = 'https://www.example.com'; const cookies = [ diff --git a/yarn.lock b/yarn.lock index a017feaf987d..12f158b9903c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19,6 +19,13 @@ __metadata: languageName: node linkType: hard +"@apify/consts@npm:^2.51.0": + version: 2.51.0 + resolution: "@apify/consts@npm:2.51.0" + checksum: 10c0/06902272f1ca99ed515d45aca546ba41130dc8719413e1dd3ceed8c5dd7eeea30a9215227ac393403646b7fbef04bd26ef4f8215c5f04e5d42fa71834304071b + languageName: node + linkType: hard + "@apify/datastructures@npm:^2.0.0, @apify/datastructures@npm:^2.0.3": version: 2.0.3 resolution: "@apify/datastructures@npm:2.0.3" @@ -79,6 +86,16 @@ __metadata: languageName: node linkType: hard +"@apify/log@npm:^2.5.32": + version: 2.5.32 + resolution: "@apify/log@npm:2.5.32" + dependencies: + "@apify/consts": "npm:^2.51.0" + ansi-colors: "npm:^4.1.1" + checksum: 10c0/dbe716561ee9df72a5f1b26881837780f3d7d8afb11676fd20f4e529dfce7997a4cc6e240c5902c27efa2b73beb66a8563d72a911dc4b6e641c92e8507dfdefb + languageName: node + linkType: hard + "@apify/ps-tree@npm:^1.2.0": version: 1.2.0 resolution: "@apify/ps-tree@npm:1.2.0" @@ -668,6 +685,7 @@ __metadata: version: 0.0.0-use.local resolution: "@crawlee/http-client@workspace:packages/http-client" dependencies: + "@apify/log": "npm:^2.5.32" "@crawlee/types": "npm:4.0.0" tough-cookie: "npm:^6.0.0" languageName: unknown From 098b7e49aaa2e298bc0c76705ffee88761048123 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Fri, 20 Feb 2026 16:08:55 +0100 Subject: [PATCH 5/9] chore: run formatter --- packages/http-client/src/base-http-client.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/http-client/src/base-http-client.ts b/packages/http-client/src/base-http-client.ts index f37c63bd9d6b..94f9e2fc83c3 100644 --- a/packages/http-client/src/base-http-client.ts +++ b/packages/http-client/src/base-http-client.ts @@ -19,6 +19,7 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface { constructor(log?: Log) { this.log = log ?? defaultLog; } + /** * Perform the raw network request and return a single Response without any * automatic redirect following or special error handling. From 36d82f7d8e968450ffe8d4f9239e7a428584c188 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 23 Feb 2026 13:44:59 +0000 Subject: [PATCH 6/9] test: add comment and malformed cookie test for setCookie Co-authored-by: barjin <61918049+barjin@users.noreply.github.com> --- test/core/session_pool/session.test.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/core/session_pool/session.test.ts b/test/core/session_pool/session.test.ts index ec1947e92747..dbad8c232cf5 100644 --- a/test/core/session_pool/session.test.ts +++ b/test/core/session_pool/session.test.ts @@ -209,6 +209,7 @@ describe('Session - testing session behaviour ', () => { test('setCookies will log warning (not throw) on invalid cookies', () => { const url = 'https://www.example.com'; + // domain 'abc.different.domain' does not match the request URL, so tough-cookie rejects it const cookies = [{ name: 'cookie1', value: 'my-cookie', domain: 'abc.different.domain' }]; const mockedLog = vitest.mockObject(log, { @@ -221,6 +222,14 @@ describe('Session - testing session behaviour ', () => { expect(mockedLog.warning).toHaveBeenCalledOnce(); }); + test('setCookie does not throw on malformed raw cookie string', () => { + session = new Session({ sessionPool }); + vitest.spyOn(session.cookieJar, 'setCookieSync').mockImplementation(() => { + throw new Error('malformed cookie'); + }); + expect(() => session.setCookie('malformed=🍪; domain=evil.com', 'https://www.example.com')).not.toThrow(); + }); + test('setCookies works with hostOnly cookies', () => { const url = 'https://www.example.com'; const cookies = [ From fd55f27fa332633aff1772e0b773a001cc276612 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 23 Feb 2026 13:47:52 +0000 Subject: [PATCH 7/9] fix: revert try-catch from getCookies and getCookieString in session Co-authored-by: barjin <61918049+barjin@users.noreply.github.com> --- packages/core/src/session_pool/session.ts | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/packages/core/src/session_pool/session.ts b/packages/core/src/session_pool/session.ts index 172c6c84e310..df3e9e7c836d 100644 --- a/packages/core/src/session_pool/session.ts +++ b/packages/core/src/session_pool/session.ts @@ -346,13 +346,8 @@ export class Session implements ISession { * @param url website url. Only cookies stored for this url will be returned */ getCookies(url: string): CookieObject[] { - try { - const cookies = this.cookieJar.getCookiesSync(url); - return cookies.map((c) => toughCookieToBrowserPoolCookie(c)); - } catch (e) { - this.log.warning('Could not get cookies from cookie jar.', { url, error: (e as Error).message }); - return []; - } + const cookies = this.cookieJar.getCookiesSync(url); + return cookies.map((c) => toughCookieToBrowserPoolCookie(c)); } /** @@ -362,12 +357,7 @@ export class Session implements ISession { * @returns Represents `Cookie` header. */ getCookieString(url: string): string { - try { - return this.cookieJar.getCookieStringSync(url, {}); - } catch (e) { - this.log.warning('Could not get cookie string.', { url, error: (e as Error).message }); - return ''; - } + return this.cookieJar.getCookieStringSync(url, {}); } /** From b81fca2ad5b3455cb314ebcd97b7c6fc98a992f9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 23 Feb 2026 14:13:47 +0000 Subject: [PATCH 8/9] test: use real setCookieSync throw instead of mock Co-authored-by: barjin <61918049+barjin@users.noreply.github.com> --- test/core/session_pool/session.test.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/core/session_pool/session.test.ts b/test/core/session_pool/session.test.ts index dbad8c232cf5..5d94e66d3a93 100644 --- a/test/core/session_pool/session.test.ts +++ b/test/core/session_pool/session.test.ts @@ -224,10 +224,8 @@ describe('Session - testing session behaviour ', () => { test('setCookie does not throw on malformed raw cookie string', () => { session = new Session({ sessionPool }); - vitest.spyOn(session.cookieJar, 'setCookieSync').mockImplementation(() => { - throw new Error('malformed cookie'); - }); - expect(() => session.setCookie('malformed=🍪; domain=evil.com', 'https://www.example.com')).not.toThrow(); + // domain mismatch causes setCookieSync to throw + expect(() => session.setCookie('name=value; domain=evil.com', 'https://www.example.com')).not.toThrow(); }); test('setCookies works with hostOnly cookies', () => { From 636757c28062c31eccc67e95ec60df33c7fcdcc4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 23 Feb 2026 14:25:19 +0000 Subject: [PATCH 9/9] test: use garbled string to trigger real cookie parse failure Co-authored-by: barjin <61918049+barjin@users.noreply.github.com> --- test/core/session_pool/session.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/core/session_pool/session.test.ts b/test/core/session_pool/session.test.ts index 5d94e66d3a93..7cbd32da0a45 100644 --- a/test/core/session_pool/session.test.ts +++ b/test/core/session_pool/session.test.ts @@ -224,8 +224,7 @@ describe('Session - testing session behaviour ', () => { test('setCookie does not throw on malformed raw cookie string', () => { session = new Session({ sessionPool }); - // domain mismatch causes setCookieSync to throw - expect(() => session.setCookie('name=value; domain=evil.com', 'https://www.example.com')).not.toThrow(); + expect(() => session.setCookie('garbled!!!@#$%nonsense', 'https://www.example.com')).not.toThrow(); }); test('setCookies works with hostOnly cookies', () => {