Use secureFetch and secureAxiosRequest for more URLs (#5886)

* Use secureFetch and secureAxiosRequest for URL's controlled by user input

* Use secureFetch and secureAxiosRequest for more URLs

* Use secureFetch and secureAxiosRequest for more URLs

---------

Co-authored-by: christopherholland-workday <[email protected]>

packages/components/nodes/agentflow/ExecuteFlow/ExecuteFlow.ts

@@ -7,7 +7,8 @@ import {
     INodeParams,
     IServerSideEventStreamer
 } from '../../../src/Interface'
-import axios, { AxiosRequestConfig } from 'axios'
+import { AxiosRequestConfig } from 'axios'
+import { secureAxiosRequest } from '../../../src/httpSecurity'
 import { getCredentialData, getCredentialParam, processTemplateVariables, parseJsonBody } from '../../../src/utils'
 import { DataSource } from 'typeorm'
 import { BaseMessageLike } from '@langchain/core/messages'
@@ -201,7 +202,7 @@ class ExecuteFlow_Agentflow implements INode {
                 }
             }
 
-            const response = await axios(requestConfig)
+            const response = await secureAxiosRequest(requestConfig)
 
             let resultText = ''
             if (response.data.text) resultText = response.data.text

packages/components/nodes/documentloaders/API/APILoader.ts

@@ -1,6 +1,6 @@
 import { Document } from '@langchain/core/documents'
-import axios, { AxiosRequestConfig } from 'axios'
-import * as https from 'https'
+import { AxiosRequestConfig } from 'axios'
+import { secureAxiosRequest } from '../../../src/httpSecurity'
 import { BaseDocumentLoader } from 'langchain/document_loaders/base'
 import { TextSplitter } from 'langchain/text_splitter'
 import { omit } from 'lodash'
@@ -256,16 +256,9 @@ class ApiLoader extends BaseDocumentLoader {
 
     protected async executeGetRequest(url: string, headers?: ICommonObject, ca?: string): Promise<IDocument[]> {
         try {
-            const config: AxiosRequestConfig = {}
-            if (headers) {
-                config.headers = headers
-            }
-            if (ca) {
-                config.httpsAgent = new https.Agent({
-                    ca: ca
-                })
-            }
-            const response = await axios.get(url, config)
+            const config: AxiosRequestConfig = { method: 'GET', url, headers: headers ?? {} }
+            const agentOptions = ca ? { ca } : undefined
+            const response = await secureAxiosRequest(config, 5, agentOptions)
             const responseJsonString = JSON.stringify(response.data, null, 2)
             const doc = new Document({
                 pageContent: responseJsonString,
@@ -281,16 +274,9 @@ class ApiLoader extends BaseDocumentLoader {
 
     protected async executePostRequest(url: string, headers?: ICommonObject, body?: ICommonObject, ca?: string): Promise<IDocument[]> {
         try {
-            const config: AxiosRequestConfig = {}
-            if (headers) {
-                config.headers = headers
-            }
-            if (ca) {
-                config.httpsAgent = new https.Agent({
-                    ca: ca
-                })
-            }
-            const response = await axios.post(url, body ?? {}, config)
+            const config: AxiosRequestConfig = { method: 'POST', url, data: body ?? {}, headers: headers ?? {} }
+            const agentOptions = ca ? { ca } : undefined
+            const response = await secureAxiosRequest(config, 5, agentOptions)
             const responseJsonString = JSON.stringify(response.data, null, 2)
             const doc = new Document({
                 pageContent: responseJsonString,

packages/components/nodes/documentloaders/FireCrawl/FireCrawl.ts

@@ -3,7 +3,8 @@ import { Document, DocumentInterface } from '@langchain/core/documents'
 import { BaseDocumentLoader } from 'langchain/document_loaders/base'
 import { INode, INodeData, INodeParams, ICommonObject, INodeOutputsValue } from '../../../src/Interface'
 import { getCredentialData, getCredentialParam, handleEscapeCharacters } from '../../../src/utils'
-import axios, { AxiosResponse, AxiosRequestHeaders } from 'axios'
+import { AxiosResponse, AxiosRequestHeaders } from 'axios'
+import { secureAxiosRequest } from '../../../src/httpSecurity'
 import { z } from 'zod'
 
 // FirecrawlApp interfaces
@@ -466,12 +467,12 @@ class FirecrawlApp {
     }
 
     private async postRequest(url: string, data: Params, headers: AxiosRequestHeaders): Promise<AxiosResponse> {
-        const result = await axios.post(url, data, { headers })
+        const result = await secureAxiosRequest({ method: 'POST', url, data, headers })
         return result
     }
 
     private getRequest(url: string, headers: AxiosRequestHeaders): Promise<AxiosResponse> {
-        return axios.get(url, { headers })
+        return secureAxiosRequest({ method: 'GET', url, headers })
     }
 
     private async monitorJobStatus(jobId: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<CrawlStatusResponse> {

packages/components/nodes/documentloaders/Spider/SpiderApp.ts

@@ -1,4 +1,5 @@
-import axios, { AxiosResponse, AxiosRequestHeaders } from 'axios'
+import { AxiosResponse, AxiosRequestHeaders } from 'axios'
+import { secureAxiosRequest } from '../../../src/httpSecurity'
 
 interface SpiderAppConfig {
     apiKey?: string | null
@@ -100,7 +101,7 @@ class SpiderApp {
     }
 
     private postRequest(url: string, data: Params, headers: AxiosRequestHeaders): Promise<AxiosResponse> {
-        return axios.post(`${this.apiUrl}/${url}`, data, { headers })
+        return secureAxiosRequest({ method: 'POST', url: `${this.apiUrl}/${url}`, data, headers })
     }
 
     private handleError(response: AxiosResponse, action: string): void {

packages/components/nodes/retrievers/AzureRerankRetriever/AzureRerank.ts

@@ -1,4 +1,4 @@
-import axios from 'axios'
+import { secureAxiosRequest } from '../../../src/httpSecurity'
 import { Callbacks } from '@langchain/core/callbacks/manager'
 import { Document } from '@langchain/core/documents'
 import { BaseDocumentCompressor } from 'langchain/retrievers/document_compressors'
@@ -42,7 +42,7 @@ export class AzureRerank extends BaseDocumentCompressor {
             documents: documents.map((doc) => doc.pageContent)
         }
         try {
-            let returnedDocs = await axios.post(this.azureApiUrl, data, config)
+            let returnedDocs = await secureAxiosRequest({ method: 'POST', url: this.azureApiUrl, data, ...config })
             const finalResults: Document<Record<string, any>>[] = []
             returnedDocs.data.results.forEach((result: any) => {
                 const doc = documents[result.index]

packages/components/nodes/tools/Jira/core.ts

@@ -1,8 +1,7 @@
 import { z } from 'zod'
-import fetch from 'node-fetch'
-import * as https from 'https'
 import { DynamicStructuredTool } from '../OpenAPIToolkit/core'
 import { TOOL_ARGS_PREFIX, formatToolError } from '../../../src/agents'
+import { secureFetch } from '../../../src/httpSecurity'
 
 export const desc = `Use this when you want to access Jira API for managing issues, comments, and users`
 
@@ -147,7 +146,6 @@ class BaseJiraTool extends DynamicStructuredTool {
     protected accessToken: string = ''
     protected jiraHost: string = ''
     protected authConfig: JiraAuthConfig | undefined
-    protected httpsAgent: https.Agent | undefined
     protected apiVersion: string = '3'
 
     constructor(args: any) {
@@ -157,13 +155,6 @@ class BaseJiraTool extends DynamicStructuredTool {
         this.jiraHost = args.jiraHost ?? ''
         this.authConfig = args.authConfig
         this.apiVersion = args.apiVersion ?? '3'
-
-        // Create HTTPS agent if SSL certificate is provided
-        if (this.authConfig?.sslCertificate) {
-            this.httpsAgent = new https.Agent({
-                ca: this.authConfig.sslCertificate
-            })
-        }
     }
 
     async makeJiraRequest({
@@ -203,12 +194,8 @@ class BaseJiraTool extends DynamicStructuredTool {
             body: body ? JSON.stringify(body) : undefined
         }
 
-        // Use HTTPS agent created in constructor if available
-        if (this.httpsAgent) {
-            fetchOptions.agent = this.httpsAgent
-        }
-
-        const response = await fetch(url, fetchOptions)
+        const agentOptions = this.authConfig?.sslCertificate ? { ca: this.authConfig.sslCertificate } : undefined
+        const response = await secureFetch(url, fetchOptions, 5, agentOptions)
 
         if (!response.ok) {
             const errorText = await response.text()

packages/components/nodes/tools/MCP/core.ts

@@ -5,6 +5,7 @@ import { BaseToolkit, tool, Tool } from '@langchain/core/tools'
 import { z } from 'zod'
 import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'
 import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js'
+import { checkDenyList, secureFetch } from '../../../src/httpSecurity'
 
 export class MCPToolkit extends BaseToolkit {
     tools: Tool[] = []
@@ -52,6 +53,7 @@ export class MCPToolkit extends BaseToolkit {
             }
 
             const baseUrl = new URL(this.serverParams.url)
+            await checkDenyList(this.serverParams.url)
             try {
                 if (this.serverParams.headers) {
                     transport = new StreamableHTTPClientTransport(baseUrl, {
@@ -70,11 +72,22 @@ export class MCPToolkit extends BaseToolkit {
                             headers: this.serverParams.headers
                         },
                         eventSourceInit: {
-                            fetch: (url, init) => fetch(url, { ...init, headers: this.serverParams.headers })
+                            fetch: async (url, init) => {
+                                return secureFetch(url.toString(), {
+                                    ...(init as any),
+                                    headers: this.serverParams.headers
+                                }) as any
+                            }
                         }
                     })
                 } else {
-                    transport = new SSEClientTransport(baseUrl)
+                    transport = new SSEClientTransport(baseUrl, {
+                        eventSourceInit: {
+                            fetch: async (url, init) => {
+                                return secureFetch(url.toString(), init as any) as any
+                            }
+                        }
+                    })
                 }
                 await client.connect(transport)
             }

packages/components/nodes/tools/OpenAPIToolkit/OpenAPIToolkit.ts

@@ -5,7 +5,7 @@ import $RefParser from '@apidevtools/json-schema-ref-parser'
 import { z, ZodSchema, ZodTypeAny } from 'zod'
 import { defaultCode, DynamicStructuredTool, howToUseCode } from './core'
 import { DataSource } from 'typeorm'
-import fetch from 'node-fetch'
+import { secureFetch } from '../../../src/httpSecurity'
 
 class OpenAPIToolkit_Tools implements INode {
     label: string
@@ -284,7 +284,7 @@ class OpenAPIToolkit_Tools implements INode {
         const { inputType = 'file', openApiFile = '', openApiLink = '' } = args
         try {
             if (inputType === 'link' && openApiLink) {
-                const res = await fetch(openApiLink)
+                const res = await secureFetch(openApiLink)
                 const text = await res.text()
 
                 // Auto-detect format from URL extension or content

packages/components/nodes/tools/Searxng/Searxng.ts

@@ -1,6 +1,7 @@
 import { Tool } from '@langchain/core/tools'
 import { INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses } from '../../../src/utils'
+import { secureFetch } from '../../../src/httpSecurity'
 
 const defaultDesc =
     'A meta search engine. Useful for when you need to answer questions about current events. Input should be a search query. Output is a JSON array of the query results'
@@ -293,10 +294,10 @@ class SearxngSearch extends Tool {
         }
         const url = this.buildUrl('search', queryParams, this.apiBase as string)
 
-        const resp = await fetch(url, {
+        const resp = await secureFetch(url, {
             method: 'POST',
             headers: this.headers,
-            signal: AbortSignal.timeout(5 * 1000) // 5 seconds
+            signal: AbortSignal.timeout(5 * 1000) as any // node-fetch AbortSignal type predates native AbortSignal
         })
 
         if (!resp.ok) {

packages/components/src/httpSecurity.ts

@@ -96,25 +96,43 @@ export async function checkDenyList(url: string): Promise<void> {
     }
 }
 
+/**
+ * Optional TLS options for secureAxiosRequest (e.g. custom CA for mutual TLS or private CAs).
+ */
+export interface SecureRequestAgentOptions {
+    ca?: string | string[] | Buffer
+}
+
 /**
  * Makes a secure HTTP request that validates all URLs in redirect chains against the deny list
- * @param config - Axios request configuration
+ * @param config - Axios request configuration (httpsAgent/httpAgent are ignored; use agentOptions for custom CA)
  * @param maxRedirects - Maximum number of redirects to follow (default: 5)
+ * @param agentOptions - Optional TLS options (e.g. { ca } for custom CA PEM)
  * @returns Promise<AxiosResponse>
  * @throws Error if any URL in the redirect chain is denied
  */
-export async function secureAxiosRequest(config: AxiosRequestConfig, maxRedirects: number = 5): Promise<AxiosResponse> {
+export async function secureAxiosRequest(
+    config: AxiosRequestConfig,
+    maxRedirects: number = 5,
+    agentOptions?: SecureRequestAgentOptions
+): Promise<AxiosResponse> {
     let currentUrl = config.url
     if (!currentUrl) {
         throw new Error('secureAxiosRequest: url is required')
     }
 
     let redirects = 0
-    let currentConfig = { ...config, maxRedirects: 0, validateStatus: () => true } // Disable automatic redirects, accept all status codes
+    let currentConfig: AxiosRequestConfig = {
+        ...config,
+        maxRedirects: 0,
+        validateStatus: () => true,
+        httpsAgent: undefined,
+        httpAgent: undefined
+    } // Disable automatic redirects; agents set per-request below
 
     while (redirects <= maxRedirects) {
         const target = await resolveAndValidate(currentUrl)
-        const agent = createPinnedAgent(target)
+        const agent = createPinnedAgent(target, agentOptions)
 
         currentConfig = {
             ...currentConfig,
@@ -168,17 +186,23 @@ export async function secureAxiosRequest(config: AxiosRequestConfig, maxRedirect
  * @param url - URL to fetch
  * @param init - Fetch request options
  * @param maxRedirects - Maximum number of redirects to follow (default: 5)
+ * @param agentOptions - Optional TLS options (e.g. { ca } for custom CA PEM)
  * @returns Promise<Response>
  * @throws Error if any URL in the redirect chain is denied
  */
-export async function secureFetch(url: string, init?: RequestInit, maxRedirects: number = 5): Promise<Response> {
+export async function secureFetch(
+    url: string,
+    init?: RequestInit,
+    maxRedirects: number = 5,
+    agentOptions?: SecureRequestAgentOptions
+): Promise<Response> {
     let currentUrl = url
     let redirectCount = 0
     let currentInit = { ...init, redirect: 'manual' as const } // Disable automatic redirects
 
     while (redirectCount <= maxRedirects) {
         const resolved = await resolveAndValidate(currentUrl)
-        const agent = createPinnedAgent(resolved)
+        const agent = createPinnedAgent(resolved, agentOptions)
 
         const response = await fetch(currentUrl, { ...currentInit, agent: () => agent })
 
@@ -263,12 +287,13 @@ async function resolveAndValidate(url: string): Promise<ResolvedTarget> {
     }
 }
 
-function createPinnedAgent(target: ResolvedTarget): http.Agent | https.Agent {
+function createPinnedAgent(target: ResolvedTarget, options?: { ca?: string | string[] | Buffer }): http.Agent | https.Agent {
     const Agent = target.protocol === 'https' ? https.Agent : http.Agent
 
     return new Agent({
         lookup: (_host, _opts, cb) => {
             cb(null, target.ip, target.family)
-        }
+        },
+        ...options
     })
 }