Skip to content

Commit 8d13865

Browse files
HenryHengZJHenryHengZJ
committed
Merge branch 'main' into chore/Langchain-V1
# Conflicts: # packages/components/nodes/documentloaders/API/APILoader.ts # packages/components/nodes/documentloaders/FireCrawl/FireCrawl.ts # packages/components/nodes/tools/Jira/core.ts
2 parents 904e31c + c7f18f5 commit 8d13865

10 files changed

Lines changed: 76 additions & 61 deletions

File tree

‎packages/components/nodes/agentflow/ExecuteFlow/ExecuteFlow.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@ import {
77
INodeParams,
88
IServerSideEventStreamer
99
} from '../../../src/Interface'
10-
import axios, { AxiosRequestConfig } from 'axios'
10+
import { AxiosRequestConfig } from 'axios'
11+
import { secureAxiosRequest } from '../../../src/httpSecurity'
1112
import { getCredentialData, getCredentialParam, processTemplateVariables, parseJsonBody } from '../../../src/utils'
1213
import { DataSource } from 'typeorm'
1314
import { BaseMessageLike } from '@langchain/core/messages'
@@ -201,7 +202,7 @@ class ExecuteFlow_Agentflow implements INode {
201202
}
202203
}
203204

204-
const response = await axios(requestConfig)
205+
const response = await secureAxiosRequest(requestConfig)
205206

206207
let resultText = ''
207208
if (response.data.text) resultText = response.data.text

‎packages/components/nodes/documentloaders/API/APILoader.ts‎

Lines changed: 8 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { Document } from '@langchain/core/documents'
2-
import axios, { AxiosRequestConfig } from 'axios'
3-
import * as https from 'https'
2+
import { AxiosRequestConfig } from 'axios'
3+
import { secureAxiosRequest } from '../../../src/httpSecurity'
44
import { BaseDocumentLoader } from '@langchain/classic/document_loaders/base'
55
import { TextSplitter } from '@langchain/textsplitters'
66
import { omit } from 'lodash'
@@ -256,16 +256,9 @@ class ApiLoader extends BaseDocumentLoader {
256256

257257
protected async executeGetRequest(url: string, headers?: ICommonObject, ca?: string): Promise<IDocument[]> {
258258
try {
259-
const config: AxiosRequestConfig = {}
260-
if (headers) {
261-
config.headers = headers
262-
}
263-
if (ca) {
264-
config.httpsAgent = new https.Agent({
265-
ca: ca
266-
})
267-
}
268-
const response = await axios.get(url, config)
259+
const config: AxiosRequestConfig = { method: 'GET', url, headers: headers ?? {} }
260+
const agentOptions = ca ? { ca } : undefined
261+
const response = await secureAxiosRequest(config, 5, agentOptions)
269262
const responseJsonString = JSON.stringify(response.data, null, 2)
270263
const doc = new Document({
271264
pageContent: responseJsonString,
@@ -281,16 +274,9 @@ class ApiLoader extends BaseDocumentLoader {
281274

282275
protected async executePostRequest(url: string, headers?: ICommonObject, body?: ICommonObject, ca?: string): Promise<IDocument[]> {
283276
try {
284-
const config: AxiosRequestConfig = {}
285-
if (headers) {
286-
config.headers = headers
287-
}
288-
if (ca) {
289-
config.httpsAgent = new https.Agent({
290-
ca: ca
291-
})
292-
}
293-
const response = await axios.post(url, body ?? {}, config)
277+
const config: AxiosRequestConfig = { method: 'POST', url, data: body ?? {}, headers: headers ?? {} }
278+
const agentOptions = ca ? { ca } : undefined
279+
const response = await secureAxiosRequest(config, 5, agentOptions)
294280
const responseJsonString = JSON.stringify(response.data, null, 2)
295281
const doc = new Document({
296282
pageContent: responseJsonString,

‎packages/components/nodes/documentloaders/FireCrawl/FireCrawl.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@ import { Document, DocumentInterface } from '@langchain/core/documents'
33
import { BaseDocumentLoader } from '@langchain/classic/document_loaders/base'
44
import { INode, INodeData, INodeParams, ICommonObject, INodeOutputsValue } from '../../../src/Interface'
55
import { getCredentialData, getCredentialParam, handleEscapeCharacters } from '../../../src/utils'
6-
import axios, { AxiosResponse, AxiosRequestHeaders } from 'axios'
6+
import { AxiosResponse, AxiosRequestHeaders } from 'axios'
7+
import { secureAxiosRequest } from '../../../src/httpSecurity'
78
import { z } from 'zod/v3'
89

910
// FirecrawlApp interfaces
@@ -466,12 +467,12 @@ class FirecrawlApp {
466467
}
467468

468469
private async postRequest(url: string, data: Params, headers: AxiosRequestHeaders): Promise<AxiosResponse> {
469-
const result = await axios.post(url, data, { headers })
470+
const result = await secureAxiosRequest({ method: 'POST', url, data, headers })
470471
return result
471472
}
472473

473474
private getRequest(url: string, headers: AxiosRequestHeaders): Promise<AxiosResponse> {
474-
return axios.get(url, { headers })
475+
return secureAxiosRequest({ method: 'GET', url, headers })
475476
}
476477

477478
private async monitorJobStatus(jobId: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<CrawlStatusResponse> {

‎packages/components/nodes/documentloaders/Spider/SpiderApp.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
import axios, { AxiosResponse, AxiosRequestHeaders } from 'axios'
1+
import { AxiosResponse, AxiosRequestHeaders } from 'axios'
2+
import { secureAxiosRequest } from '../../../src/httpSecurity'
23

34
interface SpiderAppConfig {
45
apiKey?: string | null
@@ -100,7 +101,7 @@ class SpiderApp {
100101
}
101102

102103
private postRequest(url: string, data: Params, headers: AxiosRequestHeaders): Promise<AxiosResponse> {
103-
return axios.post(`${this.apiUrl}/${url}`, data, { headers })
104+
return secureAxiosRequest({ method: 'POST', url: `${this.apiUrl}/${url}`, data, headers })
104105
}
105106

106107
private handleError(response: AxiosResponse, action: string): void {

‎packages/components/nodes/retrievers/AzureRerankRetriever/AzureRerank.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import axios from 'axios'
1+
import { secureAxiosRequest } from '../../../src/httpSecurity'
22
import { Callbacks } from '@langchain/core/callbacks/manager'
33
import { Document } from '@langchain/core/documents'
44
import { BaseDocumentCompressor } from '@langchain/classic/retrievers/document_compressors'
@@ -42,7 +42,7 @@ export class AzureRerank extends BaseDocumentCompressor {
4242
documents: documents.map((doc) => doc.pageContent)
4343
}
4444
try {
45-
let returnedDocs = await axios.post(this.azureApiUrl, data, config)
45+
let returnedDocs = await secureAxiosRequest({ method: 'POST', url: this.azureApiUrl, data, ...config })
4646
const finalResults: Document<Record<string, any>>[] = []
4747
returnedDocs.data.results.forEach((result: any) => {
4848
const doc = documents[result.index]

‎packages/components/nodes/tools/Jira/core.ts‎

Lines changed: 3 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
import { z } from 'zod/v3'
2-
import fetch from 'node-fetch'
3-
import * as https from 'https'
42
import { DynamicStructuredTool } from '../OpenAPIToolkit/core'
53
import { TOOL_ARGS_PREFIX, formatToolError } from '../../../src/agents'
4+
import { secureFetch } from '../../../src/httpSecurity'
65

76
export const desc = `Use this when you want to access Jira API for managing issues, comments, and users`
87

@@ -147,7 +146,6 @@ class BaseJiraTool extends DynamicStructuredTool {
147146
protected accessToken: string = ''
148147
protected jiraHost: string = ''
149148
protected authConfig: JiraAuthConfig | undefined
150-
protected httpsAgent: https.Agent | undefined
151149
protected apiVersion: string = '3'
152150

153151
constructor(args: any) {
@@ -157,13 +155,6 @@ class BaseJiraTool extends DynamicStructuredTool {
157155
this.jiraHost = args.jiraHost ?? ''
158156
this.authConfig = args.authConfig
159157
this.apiVersion = args.apiVersion ?? '3'
160-
161-
// Create HTTPS agent if SSL certificate is provided
162-
if (this.authConfig?.sslCertificate) {
163-
this.httpsAgent = new https.Agent({
164-
ca: this.authConfig.sslCertificate
165-
})
166-
}
167158
}
168159

169160
async makeJiraRequest({
@@ -203,12 +194,8 @@ class BaseJiraTool extends DynamicStructuredTool {
203194
body: body ? JSON.stringify(body) : undefined
204195
}
205196

206-
// Use HTTPS agent created in constructor if available
207-
if (this.httpsAgent) {
208-
fetchOptions.agent = this.httpsAgent
209-
}
210-
211-
const response = await fetch(url, fetchOptions)
197+
const agentOptions = this.authConfig?.sslCertificate ? { ca: this.authConfig.sslCertificate } : undefined
198+
const response = await secureFetch(url, fetchOptions, 5, agentOptions)
212199

213200
if (!response.ok) {
214201
const errorText = await response.text()

‎packages/components/nodes/tools/MCP/core.ts‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { BaseToolkit, tool, Tool } from '@langchain/core/tools'
55
import { z, type ZodTypeAny } from 'zod/v3'
66
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'
77
import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js'
8+
import { checkDenyList, secureFetch } from '../../../src/httpSecurity'
89

910
export class MCPToolkit extends BaseToolkit {
1011
tools: Tool[] = []
@@ -52,6 +53,7 @@ export class MCPToolkit extends BaseToolkit {
5253
}
5354

5455
const baseUrl = new URL(this.serverParams.url)
56+
await checkDenyList(this.serverParams.url)
5557
try {
5658
if (this.serverParams.headers) {
5759
transport = new StreamableHTTPClientTransport(baseUrl, {
@@ -70,11 +72,22 @@ export class MCPToolkit extends BaseToolkit {
7072
headers: this.serverParams.headers
7173
},
7274
eventSourceInit: {
73-
fetch: (url, init) => fetch(url, { ...init, headers: this.serverParams.headers })
75+
fetch: async (url, init) => {
76+
return secureFetch(url.toString(), {
77+
...(init as any),
78+
headers: this.serverParams.headers
79+
}) as any
80+
}
7481
}
7582
})
7683
} else {
77-
transport = new SSEClientTransport(baseUrl)
84+
transport = new SSEClientTransport(baseUrl, {
85+
eventSourceInit: {
86+
fetch: async (url, init) => {
87+
return secureFetch(url.toString(), init as any) as any
88+
}
89+
}
90+
})
7891
}
7992
await client.connect(transport)
8093
}

‎packages/components/nodes/tools/OpenAPIToolkit/OpenAPIToolkit.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import $RefParser from '@apidevtools/json-schema-ref-parser'
55
import { z, ZodSchema, ZodTypeAny } from 'zod/v3'
66
import { defaultCode, DynamicStructuredTool, howToUseCode } from './core'
77
import { DataSource } from 'typeorm'
8-
import fetch from 'node-fetch'
8+
import { secureFetch } from '../../../src/httpSecurity'
99

1010
class OpenAPIToolkit_Tools implements INode {
1111
label: string
@@ -284,7 +284,7 @@ class OpenAPIToolkit_Tools implements INode {
284284
const { inputType = 'file', openApiFile = '', openApiLink = '' } = args
285285
try {
286286
if (inputType === 'link' && openApiLink) {
287-
const res = await fetch(openApiLink)
287+
const res = await secureFetch(openApiLink)
288288
const text = await res.text()
289289

290290
// Auto-detect format from URL extension or content

‎packages/components/nodes/tools/Searxng/Searxng.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { Tool } from '@langchain/core/tools'
22
import { INode, INodeData, INodeParams } from '../../../src/Interface'
33
import { getBaseClasses } from '../../../src/utils'
4+
import { secureFetch } from '../../../src/httpSecurity'
45

56
const defaultDesc =
67
'A meta search engine. Useful for when you need to answer questions about current events. Input should be a search query. Output is a JSON array of the query results'
@@ -293,10 +294,10 @@ class SearxngSearch extends Tool {
293294
}
294295
const url = this.buildUrl('search', queryParams, this.apiBase as string)
295296

296-
const resp = await fetch(url, {
297+
const resp = await secureFetch(url, {
297298
method: 'POST',
298299
headers: this.headers,
299-
signal: AbortSignal.timeout(5 * 1000) // 5 seconds
300+
signal: AbortSignal.timeout(5 * 1000) as any // node-fetch AbortSignal type predates native AbortSignal
300301
})
301302

302303
if (!resp.ok) {

‎packages/components/src/httpSecurity.ts‎

Lines changed: 33 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -96,25 +96,43 @@ export async function checkDenyList(url: string): Promise<void> {
9696
}
9797
}
9898

99+
/**
100+
* Optional TLS options for secureAxiosRequest (e.g. custom CA for mutual TLS or private CAs).
101+
*/
102+
export interface SecureRequestAgentOptions {
103+
ca?: string | string[] | Buffer
104+
}
105+
99106
/**
100107
* Makes a secure HTTP request that validates all URLs in redirect chains against the deny list
101-
* @param config - Axios request configuration
108+
* @param config - Axios request configuration (httpsAgent/httpAgent are ignored; use agentOptions for custom CA)
102109
* @param maxRedirects - Maximum number of redirects to follow (default: 5)
110+
* @param agentOptions - Optional TLS options (e.g. { ca } for custom CA PEM)
103111
* @returns Promise<AxiosResponse>
104112
* @throws Error if any URL in the redirect chain is denied
105113
*/
106-
export async function secureAxiosRequest(config: AxiosRequestConfig, maxRedirects: number = 5): Promise<AxiosResponse> {
114+
export async function secureAxiosRequest(
115+
config: AxiosRequestConfig,
116+
maxRedirects: number = 5,
117+
agentOptions?: SecureRequestAgentOptions
118+
): Promise<AxiosResponse> {
107119
let currentUrl = config.url
108120
if (!currentUrl) {
109121
throw new Error('secureAxiosRequest: url is required')
110122
}
111123

112124
let redirects = 0
113-
let currentConfig = { ...config, maxRedirects: 0, validateStatus: () => true } // Disable automatic redirects, accept all status codes
125+
let currentConfig: AxiosRequestConfig = {
126+
...config,
127+
maxRedirects: 0,
128+
validateStatus: () => true,
129+
httpsAgent: undefined,
130+
httpAgent: undefined
131+
} // Disable automatic redirects; agents set per-request below
114132

115133
while (redirects <= maxRedirects) {
116134
const target = await resolveAndValidate(currentUrl)
117-
const agent = createPinnedAgent(target)
135+
const agent = createPinnedAgent(target, agentOptions)
118136

119137
currentConfig = {
120138
...currentConfig,
@@ -168,17 +186,23 @@ export async function secureAxiosRequest(config: AxiosRequestConfig, maxRedirect
168186
* @param url - URL to fetch
169187
* @param init - Fetch request options
170188
* @param maxRedirects - Maximum number of redirects to follow (default: 5)
189+
* @param agentOptions - Optional TLS options (e.g. { ca } for custom CA PEM)
171190
* @returns Promise<Response>
172191
* @throws Error if any URL in the redirect chain is denied
173192
*/
174-
export async function secureFetch(url: string, init?: RequestInit, maxRedirects: number = 5): Promise<Response> {
193+
export async function secureFetch(
194+
url: string,
195+
init?: RequestInit,
196+
maxRedirects: number = 5,
197+
agentOptions?: SecureRequestAgentOptions
198+
): Promise<Response> {
175199
let currentUrl = url
176200
let redirectCount = 0
177201
let currentInit = { ...init, redirect: 'manual' as const } // Disable automatic redirects
178202

179203
while (redirectCount <= maxRedirects) {
180204
const resolved = await resolveAndValidate(currentUrl)
181-
const agent = createPinnedAgent(resolved)
205+
const agent = createPinnedAgent(resolved, agentOptions)
182206

183207
const response = await fetch(currentUrl, { ...currentInit, agent: () => agent })
184208

@@ -263,12 +287,13 @@ async function resolveAndValidate(url: string): Promise<ResolvedTarget> {
263287
}
264288
}
265289

266-
function createPinnedAgent(target: ResolvedTarget): http.Agent | https.Agent {
290+
function createPinnedAgent(target: ResolvedTarget, options?: { ca?: string | string[] | Buffer }): http.Agent | https.Agent {
267291
const Agent = target.protocol === 'https' ? https.Agent : http.Agent
268292

269293
return new Agent({
270294
lookup: (_host, _opts, cb) => {
271295
cb(null, target.ip, target.family)
272-
}
296+
},
297+
...options
273298
})
274299
}

0 commit comments

Comments
 (0)