FlowiseAI/Flowise
1
0
Fork 0
mirror of https://github.com/FlowiseAI/Flowise.git synced 2026-10-12 02:02:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: check deny list in webscrappers (#6559)

Browse Source 
* fix: check deny list in webscrappers

* fix: review feedback
This commit is contained in:
Ilango
2026-07-15 12:36:43 +05:30
committed by GitHub
parent 9fcb7ffbd5
commit 6c989ec860
3 changed files with 70 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/documentloaders/Cheerio/Cheerio.ts
+11
View File
@@ -6,6 +6,7 @@ import { parse } from 'css-what'
import { SelectorType } from 'cheerio'
import { ICommonObject, INodeOutputsValue, IDocument, INode, INodeData, INodeParams } from '../../../src/Interface'
import { handleEscapeCharacters, webCrawl, xmlScrape } from '../../../src/utils'
import { checkDenyList, secureFetch } from '../../../src/httpSecurity'
class Cheerio_DocumentLoaders implements INode {
label: string
@@ -148,6 +149,7 @@ class Cheerio_DocumentLoaders implements INode {
async function cheerioLoader(url: string): Promise<any> {
try {
await checkDenyList(url)
let docs: IDocument[] = []
if (url.endsWith('.pdf')) {
if (process.env.DEBUG === 'true')
@@ -155,6 +157,15 @@ class Cheerio_DocumentLoaders implements INode {
return docs
}
const loader = new CheerioWebBaseLoader(url, params)
loader.scrape = async () => {
const { load } = await CheerioWebBaseLoader.imports()
const response = await secureFetch(url, {
signal: loader.timeout ? AbortSignal.timeout(loader.timeout) : undefined,
headers: loader.headers
} as any)
const html = loader.textDecoder?.decode(await response.arrayBuffer()) ?? (await response.text())
return load(html)
}
if (textSplitter) {
docs = await loader.load()
docs = await textSplitter.splitDocuments(docs)
packages/components/nodes/documentloaders/Playwright/Playwright.ts
+28
View File
@@ -10,6 +10,7 @@ import { test } from 'linkifyjs'
import { omit } from 'lodash'
import { handleEscapeCharacters, INodeOutputsValue, webCrawl, xmlScrape } from '../../../src'
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { checkDenyList } from '../../../src/httpSecurity'
class Playwright_DocumentLoaders implements INode {
label: string
@@ -189,6 +190,7 @@ class Playwright_DocumentLoaders implements INode {
async function playwrightLoader(url: string): Promise<Document[] | undefined> {
try {
await checkDenyList(url)
let docs = []
const executablePath = process.env.PLAYWRIGHT_EXECUTABLE_PATH
@@ -224,6 +226,32 @@ class Playwright_DocumentLoaders implements INode {
}
}
const loader = new PlaywrightWebBaseLoader(url, config)
loader.scrape = async (): Promise<string> => {
const { chromium } = await PlaywrightWebBaseLoader.imports()
const browser = await chromium.launch({
headless: true,
...config.launchOptions
})
try {
const page = await browser.newPage()
await page.route('**', async (route, request) => {
try {
await checkDenyList(request.url())
await route.continue()
} catch {
await route.abort('blockedbyclient')
}
})
const response = await page.goto(url, {
timeout: 180000,
waitUntil: 'domcontentloaded',
...config.gotoOptions
})
return config.evaluate ? await config.evaluate(page, browser, response) : await page.content()
} finally {
await browser.close()
}
}
if (textSplitter) {
docs = await loader.load()
docs = await textSplitter.splitDocuments(docs)
packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts
+31
View File
@@ -6,6 +6,7 @@ import { omit } from 'lodash'
import { PuppeteerLifeCycleEvent } from 'puppeteer'
import { handleEscapeCharacters, INodeOutputsValue, webCrawl, xmlScrape } from '../../../src'
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { checkDenyList } from '../../../src/httpSecurity'
class Puppeteer_DocumentLoaders implements INode {
label: string
@@ -180,6 +181,7 @@ class Puppeteer_DocumentLoaders implements INode {
async function puppeteerLoader(url: string): Promise<Document[] | undefined> {
try {
await checkDenyList(url)
let docs: Document[] = []
const executablePath = process.env.PUPPETEER_EXECUTABLE_PATH
@@ -215,6 +217,35 @@ class Puppeteer_DocumentLoaders implements INode {
}
}
const loader = new PuppeteerWebBaseLoader(url, config)
loader.scrape = async (): Promise<string> => {
const { launch } = await PuppeteerWebBaseLoader.imports()
const browser = await launch({
headless: true,
defaultViewport: null,
ignoreDefaultArgs: ['--disable-extensions'],
...config.launchOptions
})
try {
const page = await browser.newPage()
await page.setRequestInterception(true)
page.on('request', async (interceptedRequest) => {
try {
await checkDenyList(interceptedRequest.url())
await interceptedRequest.continue()
} catch {
await interceptedRequest.abort('blockedbyclient')
}
})
await page.goto(url, {
timeout: 180000,
waitUntil: 'domcontentloaded',
...config.gotoOptions
})
return config.evaluate ? await config.evaluate(page, browser) : await page.evaluate(() => document.body.innerHTML)
} finally {
await browser.close()
}
}
if (textSplitter) {
docs = await loader.load()
docs = await textSplitter.splitDocuments(docs)