From 86c0c26e43fca7904bf640ccb9396ef92cff11cb Mon Sep 17 00:00:00 2001 From: Ilango Rajagopal Date: Wed, 24 Jun 2026 14:21:46 +0530 Subject: [PATCH 1/2] fix: check deny list in webscrappers --- packages/components/nodes/documentloaders/Cheerio/Cheerio.ts | 2 ++ .../components/nodes/documentloaders/Playwright/Playwright.ts | 2 ++ .../components/nodes/documentloaders/Puppeteer/Puppeteer.ts | 2 ++ 3 files changed, 6 insertions(+) diff --git a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts index e062fb4c43d..52ff991605f 100644 --- a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts +++ b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts @@ -6,6 +6,7 @@ import { parse } from 'css-what' import { SelectorType } from 'cheerio' import { ICommonObject, INodeOutputsValue, IDocument, INode, INodeData, INodeParams } from '../../../src/Interface' import { handleEscapeCharacters, webCrawl, xmlScrape } from '../../../src/utils' +import { checkDenyList } from '../../../src/httpSecurity' class Cheerio_DocumentLoaders implements INode { label: string @@ -148,6 +149,7 @@ class Cheerio_DocumentLoaders implements INode { async function cheerioLoader(url: string): Promise { try { + await checkDenyList(url) let docs: IDocument[] = [] if (url.endsWith('.pdf')) { if (process.env.DEBUG === 'true') diff --git a/packages/components/nodes/documentloaders/Playwright/Playwright.ts b/packages/components/nodes/documentloaders/Playwright/Playwright.ts index fcef1faddf3..1a268b38844 100644 --- a/packages/components/nodes/documentloaders/Playwright/Playwright.ts +++ b/packages/components/nodes/documentloaders/Playwright/Playwright.ts @@ -10,6 +10,7 @@ import { test } from 'linkifyjs' import { omit } from 'lodash' import { handleEscapeCharacters, INodeOutputsValue, webCrawl, xmlScrape } from '../../../src' import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { checkDenyList } from '../../../src/httpSecurity' class Playwright_DocumentLoaders implements INode { label: string @@ -189,6 +190,7 @@ class Playwright_DocumentLoaders implements INode { async function playwrightLoader(url: string): Promise { try { + await checkDenyList(url) let docs = [] const executablePath = process.env.PLAYWRIGHT_EXECUTABLE_PATH diff --git a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts index c22b05de94f..2f12b7dece6 100644 --- a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts +++ b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts @@ -6,6 +6,7 @@ import { omit } from 'lodash' import { PuppeteerLifeCycleEvent } from 'puppeteer' import { handleEscapeCharacters, INodeOutputsValue, webCrawl, xmlScrape } from '../../../src' import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { checkDenyList } from '../../../src/httpSecurity' class Puppeteer_DocumentLoaders implements INode { label: string @@ -180,6 +181,7 @@ class Puppeteer_DocumentLoaders implements INode { async function puppeteerLoader(url: string): Promise { try { + await checkDenyList(url) let docs: Document[] = [] const executablePath = process.env.PUPPETEER_EXECUTABLE_PATH From d7744acc33fa1c99dac05fafb415bec7b81689a5 Mon Sep 17 00:00:00 2001 From: Ilango Rajagopal Date: Wed, 1 Jul 2026 13:48:35 +0530 Subject: [PATCH 2/2] fix: review feedback --- .../nodes/documentloaders/Cheerio/Cheerio.ts | 11 ++++++- .../documentloaders/Playwright/Playwright.ts | 26 +++++++++++++++++ .../documentloaders/Puppeteer/Puppeteer.ts | 29 +++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts index 52ff991605f..a7f2ccf1e0b 100644 --- a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts +++ b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts @@ -6,7 +6,7 @@ import { parse } from 'css-what' import { SelectorType } from 'cheerio' import { ICommonObject, INodeOutputsValue, IDocument, INode, INodeData, INodeParams } from '../../../src/Interface' import { handleEscapeCharacters, webCrawl, xmlScrape } from '../../../src/utils' -import { checkDenyList } from '../../../src/httpSecurity' +import { checkDenyList, secureFetch } from '../../../src/httpSecurity' class Cheerio_DocumentLoaders implements INode { label: string @@ -157,6 +157,15 @@ class Cheerio_DocumentLoaders implements INode { return docs } const loader = new CheerioWebBaseLoader(url, params) + loader.scrape = async () => { + const { load } = await CheerioWebBaseLoader.imports() + const response = await secureFetch(url, { + signal: loader.timeout ? AbortSignal.timeout(loader.timeout) : undefined, + headers: loader.headers + } as any) + const html = loader.textDecoder?.decode(await response.arrayBuffer()) ?? (await response.text()) + return load(html) + } if (textSplitter) { docs = await loader.load() docs = await textSplitter.splitDocuments(docs) diff --git a/packages/components/nodes/documentloaders/Playwright/Playwright.ts b/packages/components/nodes/documentloaders/Playwright/Playwright.ts index 1a268b38844..30e874b1b26 100644 --- a/packages/components/nodes/documentloaders/Playwright/Playwright.ts +++ b/packages/components/nodes/documentloaders/Playwright/Playwright.ts @@ -226,6 +226,32 @@ class Playwright_DocumentLoaders implements INode { } } const loader = new PlaywrightWebBaseLoader(url, config) + loader.scrape = async (): Promise => { + const { chromium } = await PlaywrightWebBaseLoader.imports() + const browser = await chromium.launch({ + headless: true, + ...config.launchOptions + }) + try { + const page = await browser.newPage() + await page.route('**', async (route, request) => { + try { + await checkDenyList(request.url()) + await route.continue() + } catch { + await route.abort('blockedbyclient') + } + }) + const response = await page.goto(url, { + timeout: 180000, + waitUntil: 'domcontentloaded', + ...config.gotoOptions + }) + return config.evaluate ? await config.evaluate(page, browser, response) : await page.content() + } finally { + await browser.close() + } + } if (textSplitter) { docs = await loader.load() docs = await textSplitter.splitDocuments(docs) diff --git a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts index 2f12b7dece6..5c363b3665d 100644 --- a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts +++ b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts @@ -217,6 +217,35 @@ class Puppeteer_DocumentLoaders implements INode { } } const loader = new PuppeteerWebBaseLoader(url, config) + loader.scrape = async (): Promise => { + const { launch } = await PuppeteerWebBaseLoader.imports() + const browser = await launch({ + headless: true, + defaultViewport: null, + ignoreDefaultArgs: ['--disable-extensions'], + ...config.launchOptions + }) + try { + const page = await browser.newPage() + await page.setRequestInterception(true) + page.on('request', async (interceptedRequest) => { + try { + await checkDenyList(interceptedRequest.url()) + await interceptedRequest.continue() + } catch { + await interceptedRequest.abort('blockedbyclient') + } + }) + await page.goto(url, { + timeout: 180000, + waitUntil: 'domcontentloaded', + ...config.gotoOptions + }) + return config.evaluate ? await config.evaluate(page, browser) : await page.evaluate(() => document.body.innerHTML) + } finally { + await browser.close() + } + } if (textSplitter) { docs = await loader.load() docs = await textSplitter.splitDocuments(docs)