diff --git a/packages/components/nodes/agentflow/ExecuteFlow/ExecuteFlow.ts b/packages/components/nodes/agentflow/ExecuteFlow/ExecuteFlow.ts index af92acd53dc..a9158a3187f 100644 --- a/packages/components/nodes/agentflow/ExecuteFlow/ExecuteFlow.ts +++ b/packages/components/nodes/agentflow/ExecuteFlow/ExecuteFlow.ts @@ -7,7 +7,8 @@ import { INodeParams, IServerSideEventStreamer } from '../../../src/Interface' -import axios, { AxiosRequestConfig } from 'axios' +import { AxiosRequestConfig } from 'axios' +import { secureAxiosRequest } from '../../../src/httpSecurity' import { getCredentialData, getCredentialParam, processTemplateVariables, parseJsonBody } from '../../../src/utils' import { DataSource } from 'typeorm' import { BaseMessageLike } from '@langchain/core/messages' @@ -201,7 +202,7 @@ class ExecuteFlow_Agentflow implements INode { } } - const response = await axios(requestConfig) + const response = await secureAxiosRequest(requestConfig) let resultText = '' if (response.data.text) resultText = response.data.text diff --git a/packages/components/nodes/documentloaders/API/APILoader.ts b/packages/components/nodes/documentloaders/API/APILoader.ts index 479ad2e9481..d343cb9b38b 100644 --- a/packages/components/nodes/documentloaders/API/APILoader.ts +++ b/packages/components/nodes/documentloaders/API/APILoader.ts @@ -1,6 +1,6 @@ import { Document } from '@langchain/core/documents' -import axios, { AxiosRequestConfig } from 'axios' -import * as https from 'https' +import { AxiosRequestConfig } from 'axios' +import { secureAxiosRequest } from '../../../src/httpSecurity' import { BaseDocumentLoader } from 'langchain/document_loaders/base' import { TextSplitter } from 'langchain/text_splitter' import { omit } from 'lodash' @@ -256,16 +256,9 @@ class ApiLoader extends BaseDocumentLoader { protected async executeGetRequest(url: string, headers?: ICommonObject, ca?: string): Promise { try { - const config: AxiosRequestConfig = {} - if (headers) { - config.headers = headers - } - if (ca) { - config.httpsAgent = new https.Agent({ - ca: ca - }) - } - const response = await axios.get(url, config) + const config: AxiosRequestConfig = { method: 'GET', url, headers: headers ?? {} } + const agentOptions = ca ? { ca } : undefined + const response = await secureAxiosRequest(config, 5, agentOptions) const responseJsonString = JSON.stringify(response.data, null, 2) const doc = new Document({ pageContent: responseJsonString, @@ -281,16 +274,9 @@ class ApiLoader extends BaseDocumentLoader { protected async executePostRequest(url: string, headers?: ICommonObject, body?: ICommonObject, ca?: string): Promise { try { - const config: AxiosRequestConfig = {} - if (headers) { - config.headers = headers - } - if (ca) { - config.httpsAgent = new https.Agent({ - ca: ca - }) - } - const response = await axios.post(url, body ?? {}, config) + const config: AxiosRequestConfig = { method: 'POST', url, data: body ?? {}, headers: headers ?? {} } + const agentOptions = ca ? { ca } : undefined + const response = await secureAxiosRequest(config, 5, agentOptions) const responseJsonString = JSON.stringify(response.data, null, 2) const doc = new Document({ pageContent: responseJsonString, diff --git a/packages/components/nodes/documentloaders/FireCrawl/FireCrawl.ts b/packages/components/nodes/documentloaders/FireCrawl/FireCrawl.ts index 27bc3c5b669..c8cf4a3dd04 100644 --- a/packages/components/nodes/documentloaders/FireCrawl/FireCrawl.ts +++ b/packages/components/nodes/documentloaders/FireCrawl/FireCrawl.ts @@ -3,7 +3,8 @@ import { Document, DocumentInterface } from '@langchain/core/documents' import { BaseDocumentLoader } from 'langchain/document_loaders/base' import { INode, INodeData, INodeParams, ICommonObject, INodeOutputsValue } from '../../../src/Interface' import { getCredentialData, getCredentialParam, handleEscapeCharacters } from '../../../src/utils' -import axios, { AxiosResponse, AxiosRequestHeaders } from 'axios' +import { AxiosResponse, AxiosRequestHeaders } from 'axios' +import { secureAxiosRequest } from '../../../src/httpSecurity' import { z } from 'zod' // FirecrawlApp interfaces @@ -466,12 +467,12 @@ class FirecrawlApp { } private async postRequest(url: string, data: Params, headers: AxiosRequestHeaders): Promise { - const result = await axios.post(url, data, { headers }) + const result = await secureAxiosRequest({ method: 'POST', url, data, headers }) return result } private getRequest(url: string, headers: AxiosRequestHeaders): Promise { - return axios.get(url, { headers }) + return secureAxiosRequest({ method: 'GET', url, headers }) } private async monitorJobStatus(jobId: string, headers: AxiosRequestHeaders, checkInterval: number): Promise { diff --git a/packages/components/nodes/documentloaders/Spider/SpiderApp.ts b/packages/components/nodes/documentloaders/Spider/SpiderApp.ts index e2bc1d5bf2f..001bad74d1f 100644 --- a/packages/components/nodes/documentloaders/Spider/SpiderApp.ts +++ b/packages/components/nodes/documentloaders/Spider/SpiderApp.ts @@ -1,4 +1,5 @@ -import axios, { AxiosResponse, AxiosRequestHeaders } from 'axios' +import { AxiosResponse, AxiosRequestHeaders } from 'axios' +import { secureAxiosRequest } from '../../../src/httpSecurity' interface SpiderAppConfig { apiKey?: string | null @@ -100,7 +101,7 @@ class SpiderApp { } private postRequest(url: string, data: Params, headers: AxiosRequestHeaders): Promise { - return axios.post(`${this.apiUrl}/${url}`, data, { headers }) + return secureAxiosRequest({ method: 'POST', url: `${this.apiUrl}/${url}`, data, headers }) } private handleError(response: AxiosResponse, action: string): void { diff --git a/packages/components/nodes/retrievers/AzureRerankRetriever/AzureRerank.ts b/packages/components/nodes/retrievers/AzureRerankRetriever/AzureRerank.ts index 91c6bf317e9..7115f799383 100644 --- a/packages/components/nodes/retrievers/AzureRerankRetriever/AzureRerank.ts +++ b/packages/components/nodes/retrievers/AzureRerankRetriever/AzureRerank.ts @@ -1,4 +1,4 @@ -import axios from 'axios' +import { secureAxiosRequest } from '../../../src/httpSecurity' import { Callbacks } from '@langchain/core/callbacks/manager' import { Document } from '@langchain/core/documents' import { BaseDocumentCompressor } from 'langchain/retrievers/document_compressors' @@ -42,7 +42,7 @@ export class AzureRerank extends BaseDocumentCompressor { documents: documents.map((doc) => doc.pageContent) } try { - let returnedDocs = await axios.post(this.azureApiUrl, data, config) + let returnedDocs = await secureAxiosRequest({ method: 'POST', url: this.azureApiUrl, data, ...config }) const finalResults: Document>[] = [] returnedDocs.data.results.forEach((result: any) => { const doc = documents[result.index] diff --git a/packages/components/nodes/tools/Jira/core.ts b/packages/components/nodes/tools/Jira/core.ts index 978c2b81614..f0d1507790a 100644 --- a/packages/components/nodes/tools/Jira/core.ts +++ b/packages/components/nodes/tools/Jira/core.ts @@ -1,8 +1,7 @@ import { z } from 'zod' -import fetch from 'node-fetch' -import * as https from 'https' import { DynamicStructuredTool } from '../OpenAPIToolkit/core' import { TOOL_ARGS_PREFIX, formatToolError } from '../../../src/agents' +import { secureFetch } from '../../../src/httpSecurity' export const desc = `Use this when you want to access Jira API for managing issues, comments, and users` @@ -147,7 +146,6 @@ class BaseJiraTool extends DynamicStructuredTool { protected accessToken: string = '' protected jiraHost: string = '' protected authConfig: JiraAuthConfig | undefined - protected httpsAgent: https.Agent | undefined protected apiVersion: string = '3' constructor(args: any) { @@ -157,13 +155,6 @@ class BaseJiraTool extends DynamicStructuredTool { this.jiraHost = args.jiraHost ?? '' this.authConfig = args.authConfig this.apiVersion = args.apiVersion ?? '3' - - // Create HTTPS agent if SSL certificate is provided - if (this.authConfig?.sslCertificate) { - this.httpsAgent = new https.Agent({ - ca: this.authConfig.sslCertificate - }) - } } async makeJiraRequest({ @@ -203,12 +194,8 @@ class BaseJiraTool extends DynamicStructuredTool { body: body ? JSON.stringify(body) : undefined } - // Use HTTPS agent created in constructor if available - if (this.httpsAgent) { - fetchOptions.agent = this.httpsAgent - } - - const response = await fetch(url, fetchOptions) + const agentOptions = this.authConfig?.sslCertificate ? { ca: this.authConfig.sslCertificate } : undefined + const response = await secureFetch(url, fetchOptions, 5, agentOptions) if (!response.ok) { const errorText = await response.text() diff --git a/packages/components/nodes/tools/MCP/core.ts b/packages/components/nodes/tools/MCP/core.ts index 0d2bb93ce6f..fd0a5e92bda 100644 --- a/packages/components/nodes/tools/MCP/core.ts +++ b/packages/components/nodes/tools/MCP/core.ts @@ -5,6 +5,7 @@ import { BaseToolkit, tool, Tool } from '@langchain/core/tools' import { z } from 'zod' import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js' import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js' +import { checkDenyList, secureFetch } from '../../../src/httpSecurity' export class MCPToolkit extends BaseToolkit { tools: Tool[] = [] @@ -52,6 +53,7 @@ export class MCPToolkit extends BaseToolkit { } const baseUrl = new URL(this.serverParams.url) + await checkDenyList(this.serverParams.url) try { if (this.serverParams.headers) { transport = new StreamableHTTPClientTransport(baseUrl, { @@ -70,11 +72,22 @@ export class MCPToolkit extends BaseToolkit { headers: this.serverParams.headers }, eventSourceInit: { - fetch: (url, init) => fetch(url, { ...init, headers: this.serverParams.headers }) + fetch: async (url, init) => { + return secureFetch(url.toString(), { + ...(init as any), + headers: this.serverParams.headers + }) as any + } } }) } else { - transport = new SSEClientTransport(baseUrl) + transport = new SSEClientTransport(baseUrl, { + eventSourceInit: { + fetch: async (url, init) => { + return secureFetch(url.toString(), init as any) as any + } + } + }) } await client.connect(transport) } diff --git a/packages/components/nodes/tools/OpenAPIToolkit/OpenAPIToolkit.ts b/packages/components/nodes/tools/OpenAPIToolkit/OpenAPIToolkit.ts index 5f0a8bc209e..fa039a6681d 100644 --- a/packages/components/nodes/tools/OpenAPIToolkit/OpenAPIToolkit.ts +++ b/packages/components/nodes/tools/OpenAPIToolkit/OpenAPIToolkit.ts @@ -5,7 +5,7 @@ import $RefParser from '@apidevtools/json-schema-ref-parser' import { z, ZodSchema, ZodTypeAny } from 'zod' import { defaultCode, DynamicStructuredTool, howToUseCode } from './core' import { DataSource } from 'typeorm' -import fetch from 'node-fetch' +import { secureFetch } from '../../../src/httpSecurity' class OpenAPIToolkit_Tools implements INode { label: string @@ -284,7 +284,7 @@ class OpenAPIToolkit_Tools implements INode { const { inputType = 'file', openApiFile = '', openApiLink = '' } = args try { if (inputType === 'link' && openApiLink) { - const res = await fetch(openApiLink) + const res = await secureFetch(openApiLink) const text = await res.text() // Auto-detect format from URL extension or content diff --git a/packages/components/nodes/tools/Searxng/Searxng.ts b/packages/components/nodes/tools/Searxng/Searxng.ts index e317e21fc2d..102ae632eae 100644 --- a/packages/components/nodes/tools/Searxng/Searxng.ts +++ b/packages/components/nodes/tools/Searxng/Searxng.ts @@ -1,6 +1,7 @@ import { Tool } from '@langchain/core/tools' import { INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses } from '../../../src/utils' +import { secureFetch } from '../../../src/httpSecurity' const defaultDesc = 'A meta search engine. Useful for when you need to answer questions about current events. Input should be a search query. Output is a JSON array of the query results' @@ -293,10 +294,10 @@ class SearxngSearch extends Tool { } const url = this.buildUrl('search', queryParams, this.apiBase as string) - const resp = await fetch(url, { + const resp = await secureFetch(url, { method: 'POST', headers: this.headers, - signal: AbortSignal.timeout(5 * 1000) // 5 seconds + signal: AbortSignal.timeout(5 * 1000) as any // node-fetch AbortSignal type predates native AbortSignal }) if (!resp.ok) { diff --git a/packages/components/src/httpSecurity.ts b/packages/components/src/httpSecurity.ts index a4a9b1d0091..476a373c229 100644 --- a/packages/components/src/httpSecurity.ts +++ b/packages/components/src/httpSecurity.ts @@ -96,25 +96,43 @@ export async function checkDenyList(url: string): Promise { } } +/** + * Optional TLS options for secureAxiosRequest (e.g. custom CA for mutual TLS or private CAs). + */ +export interface SecureRequestAgentOptions { + ca?: string | string[] | Buffer +} + /** * Makes a secure HTTP request that validates all URLs in redirect chains against the deny list - * @param config - Axios request configuration + * @param config - Axios request configuration (httpsAgent/httpAgent are ignored; use agentOptions for custom CA) * @param maxRedirects - Maximum number of redirects to follow (default: 5) + * @param agentOptions - Optional TLS options (e.g. { ca } for custom CA PEM) * @returns Promise * @throws Error if any URL in the redirect chain is denied */ -export async function secureAxiosRequest(config: AxiosRequestConfig, maxRedirects: number = 5): Promise { +export async function secureAxiosRequest( + config: AxiosRequestConfig, + maxRedirects: number = 5, + agentOptions?: SecureRequestAgentOptions +): Promise { let currentUrl = config.url if (!currentUrl) { throw new Error('secureAxiosRequest: url is required') } let redirects = 0 - let currentConfig = { ...config, maxRedirects: 0, validateStatus: () => true } // Disable automatic redirects, accept all status codes + let currentConfig: AxiosRequestConfig = { + ...config, + maxRedirects: 0, + validateStatus: () => true, + httpsAgent: undefined, + httpAgent: undefined + } // Disable automatic redirects; agents set per-request below while (redirects <= maxRedirects) { const target = await resolveAndValidate(currentUrl) - const agent = createPinnedAgent(target) + const agent = createPinnedAgent(target, agentOptions) currentConfig = { ...currentConfig, @@ -168,17 +186,23 @@ export async function secureAxiosRequest(config: AxiosRequestConfig, maxRedirect * @param url - URL to fetch * @param init - Fetch request options * @param maxRedirects - Maximum number of redirects to follow (default: 5) + * @param agentOptions - Optional TLS options (e.g. { ca } for custom CA PEM) * @returns Promise * @throws Error if any URL in the redirect chain is denied */ -export async function secureFetch(url: string, init?: RequestInit, maxRedirects: number = 5): Promise { +export async function secureFetch( + url: string, + init?: RequestInit, + maxRedirects: number = 5, + agentOptions?: SecureRequestAgentOptions +): Promise { let currentUrl = url let redirectCount = 0 let currentInit = { ...init, redirect: 'manual' as const } // Disable automatic redirects while (redirectCount <= maxRedirects) { const resolved = await resolveAndValidate(currentUrl) - const agent = createPinnedAgent(resolved) + const agent = createPinnedAgent(resolved, agentOptions) const response = await fetch(currentUrl, { ...currentInit, agent: () => agent }) @@ -263,12 +287,13 @@ async function resolveAndValidate(url: string): Promise { } } -function createPinnedAgent(target: ResolvedTarget): http.Agent | https.Agent { +function createPinnedAgent(target: ResolvedTarget, options?: { ca?: string | string[] | Buffer }): http.Agent | https.Agent { const Agent = target.protocol === 'https' ? https.Agent : http.Agent return new Agent({ lookup: (_host, _opts, cb) => { cb(null, target.ip, target.family) - } + }, + ...options }) }