diff --git a/packages/core/src/session_pool/session.ts b/packages/core/src/session_pool/session.ts index 95c5cd7378f2..df3e9e7c836d 100644 --- a/packages/core/src/session_pool/session.ts +++ b/packages/core/src/session_pool/session.ts @@ -364,7 +364,11 @@ export class Session implements ISession { * Sets a cookie within this session for the specific URL. */ setCookie(rawCookie: string, url: string): void { - this.cookieJar.setCookieSync(rawCookie, url); + try { + this.cookieJar.setCookieSync(rawCookie, url); + } catch (e) { + this.log.warning('Could not set cookie.', { url, error: (e as Error).message }); + } } /** @@ -384,7 +388,7 @@ export class Session implements ISession { // if invalid cookies are provided just log the exception. No need to retry the request automatically. if (errorMessages.length) { - this.log.debug('Could not set cookies.', { errorMessages }); + this.log.warning('Could not set cookies.', { errorMessages }); } } diff --git a/packages/http-client/package.json b/packages/http-client/package.json index e7d4c5b4f3f0..f94278cc9e54 100644 --- a/packages/http-client/package.json +++ b/packages/http-client/package.json @@ -47,6 +47,7 @@ "access": "public" }, "dependencies": { + "@apify/log": "^2.5.32", "@crawlee/types": "4.0.0", "tough-cookie": "^6.0.0" } diff --git a/packages/http-client/src/base-http-client.ts b/packages/http-client/src/base-http-client.ts index f2831f0af598..94f9e2fc83c3 100644 --- a/packages/http-client/src/base-http-client.ts +++ b/packages/http-client/src/base-http-client.ts @@ -1,6 +1,9 @@ import type { BaseHttpClient as BaseHttpClientInterface, SendRequestOptions } from '@crawlee/types'; import { CookieJar } from 'tough-cookie'; +import type { Log } from '@apify/log'; +import defaultLog from '@apify/log'; + export interface CustomFetchOptions { proxyUrl?: string; } @@ -11,6 +14,12 @@ export interface CustomFetchOptions { * implement only the low-level network call in `fetch`. */ export abstract class BaseHttpClient implements BaseHttpClientInterface { + protected log: Log; + + constructor(log?: Log) { + this.log = log ?? defaultLog; + } + /** * Perform the raw network request and return a single Response without any * automatic redirect following or special error handling. @@ -18,10 +27,16 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface { protected abstract fetch(input: Request, init?: RequestInit & CustomFetchOptions): Promise; private async applyCookies(request: Request, cookieJar: CookieJar): Promise { - const cookies = (await cookieJar.getCookies(request.url)).map((x) => x.cookieString().trim()).filter(Boolean); + try { + const cookies = (await cookieJar.getCookies(request.url)) + .map((x) => x.cookieString().trim()) + .filter(Boolean); - if (cookies?.length > 0) { - request.headers.set('cookie', cookies.join('; ')); + if (cookies?.length > 0) { + request.headers.set('cookie', cookies.join('; ')); + } + } catch (e) { + this.log.warning(`Failed to get cookies for URL "${request.url}": ${(e as Error).message}`); } return request; } @@ -29,7 +44,13 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface { private async setCookies(response: Response, cookieJar: CookieJar): Promise { const setCookieHeaders = response.headers.getSetCookie(); - await Promise.all(setCookieHeaders.map((header) => cookieJar.setCookie(header, response.url))); + for (const header of setCookieHeaders) { + try { + await cookieJar.setCookie(header, response.url); + } catch (e) { + this.log.warning(`Failed to set cookie for URL "${response.url}": ${(e as Error).message}`); + } + } } private resolveRequestContext(options?: SendRequestOptions): { diff --git a/test/core/session_pool/session.test.ts b/test/core/session_pool/session.test.ts index 03bb260b916e..7cbd32da0a45 100644 --- a/test/core/session_pool/session.test.ts +++ b/test/core/session_pool/session.test.ts @@ -1,4 +1,4 @@ -import { EVENT_SESSION_RETIRED, Session, SessionPool } from '@crawlee/core'; +import { EVENT_SESSION_RETIRED, log, Session, SessionPool } from '@crawlee/core'; import { ResponseWithUrl } from '@crawlee/http-client'; import { entries, sleep } from '@crawlee/utils'; import { CookieJar } from 'tough-cookie'; @@ -207,6 +207,26 @@ describe('Session - testing session behaviour ', () => { expect(session.getCookieString(url)).toBe('cookie2=your-cookie'); }); + test('setCookies will log warning (not throw) on invalid cookies', () => { + const url = 'https://www.example.com'; + // domain 'abc.different.domain' does not match the request URL, so tough-cookie rejects it + const cookies = [{ name: 'cookie1', value: 'my-cookie', domain: 'abc.different.domain' }]; + + const mockedLog = vitest.mockObject(log, { + spy: true, + }); + + session = new Session({ sessionPool, log: mockedLog } as any); + session.setCookies(cookies, url); + expect(session.getCookieString(url)).toBe(''); + expect(mockedLog.warning).toHaveBeenCalledOnce(); + }); + + test('setCookie does not throw on malformed raw cookie string', () => { + session = new Session({ sessionPool }); + expect(() => session.setCookie('garbled!!!@#$%nonsense', 'https://www.example.com')).not.toThrow(); + }); + test('setCookies works with hostOnly cookies', () => { const url = 'https://www.example.com'; const cookies = [ diff --git a/yarn.lock b/yarn.lock index a017feaf987d..12f158b9903c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19,6 +19,13 @@ __metadata: languageName: node linkType: hard +"@apify/consts@npm:^2.51.0": + version: 2.51.0 + resolution: "@apify/consts@npm:2.51.0" + checksum: 10c0/06902272f1ca99ed515d45aca546ba41130dc8719413e1dd3ceed8c5dd7eeea30a9215227ac393403646b7fbef04bd26ef4f8215c5f04e5d42fa71834304071b + languageName: node + linkType: hard + "@apify/datastructures@npm:^2.0.0, @apify/datastructures@npm:^2.0.3": version: 2.0.3 resolution: "@apify/datastructures@npm:2.0.3" @@ -79,6 +86,16 @@ __metadata: languageName: node linkType: hard +"@apify/log@npm:^2.5.32": + version: 2.5.32 + resolution: "@apify/log@npm:2.5.32" + dependencies: + "@apify/consts": "npm:^2.51.0" + ansi-colors: "npm:^4.1.1" + checksum: 10c0/dbe716561ee9df72a5f1b26881837780f3d7d8afb11676fd20f4e529dfce7997a4cc6e240c5902c27efa2b73beb66a8563d72a911dc4b6e641c92e8507dfdefb + languageName: node + linkType: hard + "@apify/ps-tree@npm:^1.2.0": version: 1.2.0 resolution: "@apify/ps-tree@npm:1.2.0" @@ -668,6 +685,7 @@ __metadata: version: 0.0.0-use.local resolution: "@crawlee/http-client@workspace:packages/http-client" dependencies: + "@apify/log": "npm:^2.5.32" "@crawlee/types": "npm:4.0.0" tough-cookie: "npm:^6.0.0" languageName: unknown