mirror of
https://github.com/FlowiseAI/Flowise.git
synced 2026-10-12 02:02:17 +00:00
Feat/Reasoning Support, Migrate LangChain to v1, Upgrade AI SDKs and Models (#5893)
* update langchain and all related scoped packages to v1, deprecate agents and llms, rename and upgrade chat models * add reasoning capability to Agent and LLM * add token cost to usageMetadata * update azure and chat openai * restore executable permission for husky pre-commit hook * refactor(agentflow): implement extractResponseContent utility for consistent response handling - Introduced extractResponseContent function to streamline response content extraction across various components. - Updated multiple files to utilize the new utility, enhancing code readability and maintainability. - Adjusted model configurations and added new parameters for Gemini models to improve functionality. * added a mock implementation for the logger in the URL security utilities test file to isolate tests and improve reliability. * chore: update package.json and pnpm-lock.yaml to add uuid dependency and mock multer-azure-blob-storage in Jest config - Added uuid dependency with version 10.0.0 and request>uuid with version 3.4.0 in package.json. - Updated pnpm-lock.yaml to reflect the new uuid dependency and its deprecation notice. - Modified Jest configuration to mock multer-azure-blob-storage to avoid module resolution issues with uuid/v4. * gpt 5.4 models * - Changed the stream conversion method in convertTextToSpeechStream to utilize Readable.fromWeb, enhancing compatibility with web streams. - Added '/api/v1/chatmessage/abort' to the WHITELIST_URLS array to allow for aborting chat messages in the API. * - Refactors how agentflow nodes (Agent, ConditionAgent, LLM) handle images in chat messages so that base64 is only used during model invocation - Storage and chat history keep file references only - Added comprehensive test cases to ensure correct handling of base64 image data and file references * fix unit test * implement filename sanitization to prevent path traversal vulnerabilities * fix(validator): normalize backslashes in file names for cross-platform compatibility * updated utils to process messages with images, converting stored-file references to base64 for model invocation, and implement interfaces * add sanitization to unwanted keys when processing images * add uuid to components package.json * move allow image uploads params on chat models * add encodingformat to openai embeddings * Refactor output parsing for reasoning models * Update ConditionAgent to version 2.0, refactor system prompt handling, and enhance message preparation logic * add claude settings local json to gitignore * remove claude settings local json --------- Co-authored-by: yau-wd <[email protected]>
This commit is contained in:
Henry Heng
@@ -114,3 +114,7 @@ extensions/
|
||||
# Only allow apps/oss-app, ignore all other apps
|
||||
apps/*/
|
||||
!apps/oss-app/
|
||||
|
||||
# Claude - session/user specific files
|
||||
.claude/plans/
|
||||
.claude/settings.local.json
|
||||
|
||||
+8
-4
@@ -86,7 +86,9 @@
|
||||
"unset-value": "2.0.1",
|
||||
"webpack-dev-middleware": "7.4.2",
|
||||
"ws": "8.18.3",
|
||||
"xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz"
|
||||
"xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz",
|
||||
"uuid": "^10.0.0",
|
||||
"request>uuid": "^3.4.0"
|
||||
}
|
||||
},
|
||||
"engines": {
|
||||
@@ -94,12 +96,14 @@
|
||||
"pnpm": "^10.26.0"
|
||||
},
|
||||
"resolutions": {
|
||||
"@anthropic-ai/sdk": "^0.73.0",
|
||||
"@google/generative-ai": "^0.24.0",
|
||||
"@grpc/grpc-js": "^1.10.10",
|
||||
"@langchain/core": "0.3.61",
|
||||
"@langchain/core": "1.1.20",
|
||||
"@qdrant/openapi-typescript-fetch": "1.2.6",
|
||||
"openai": "4.96.0",
|
||||
"protobufjs": "7.4.0"
|
||||
"openai": "6.19.0",
|
||||
"protobufjs": "7.4.0",
|
||||
"uuid": "^10.0.0"
|
||||
},
|
||||
"eslintIgnore": [
|
||||
"**/dist",
|
||||
|
||||
@@ -71,7 +71,7 @@
|
||||
"@tabler/icons-react": "^3.7.0",
|
||||
"axios": "^1.7.2",
|
||||
"lodash": "^4.17.21",
|
||||
"uuid": "^9.0.1"
|
||||
"uuid": "^10.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@testing-library/jest-dom": "^6.6.3",
|
||||
@@ -81,7 +81,6 @@
|
||||
"@types/lodash": "^4.14.195",
|
||||
"@types/react": "^18.2.0",
|
||||
"@types/react-dom": "^18.2.0",
|
||||
"@types/uuid": "^9.0.0",
|
||||
"@typescript-eslint/eslint-plugin": "^8.18.2",
|
||||
"@typescript-eslint/parser": "^8.18.2",
|
||||
"@vitejs/plugin-react": "^4.2.0",
|
||||
|
||||
@@ -36,9 +36,9 @@ class AzureOpenAIApi implements INodeCredential {
|
||||
label: 'Azure OpenAI Api Version',
|
||||
name: 'azureOpenAIApiVersion',
|
||||
type: 'string',
|
||||
placeholder: '2023-06-01-preview',
|
||||
placeholder: '2024-10-21',
|
||||
description:
|
||||
'Description of Supported API Versions. Please refer <a target="_blank" href="https://learn.microsoft.com/en-us/azure/cognitive-services/openai/reference#chat-completions">examples</a>'
|
||||
'Description of Supported API Versions. Please refer <a target="_blank" href="https://learn.microsoft.com/en-us/azure/ai-foundry/openai/api-version-lifecycle">API version lifecycle</a>'
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -17,6 +17,10 @@ module.exports = {
|
||||
// use these imports, so stubbing them out is safe.
|
||||
// Note: @langchain/core is NOT stubbed here because it ships CJS builds
|
||||
// (e.g. tools.cjs) that Jest can require() normally.
|
||||
'^@modelcontextprotocol/sdk/(.*)$': '<rootDir>/test/__mocks__/esm-stub.js'
|
||||
'^@modelcontextprotocol/sdk/(.*)$': '<rootDir>/test/__mocks__/esm-stub.js',
|
||||
// multer-azure-blob-storage transitively pulls in azure-storage -> [email protected] -> uuid/v4.
|
||||
// The uuid/v4 sub-path no longer exists in modern uuid versions, breaking module resolution.
|
||||
// Tests don't exercise Azure storage, so stubbing it out avoids the chain entirely.
|
||||
'^multer-azure-blob-storage$': '<rootDir>/test/__mocks__/esm-stub.js'
|
||||
}
|
||||
}
|
||||
|
||||
+118
-10
@@ -3,6 +3,20 @@
|
||||
{
|
||||
"name": "awsChatBedrock",
|
||||
"models": [
|
||||
{
|
||||
"label": "anthropic.claude-opus-4-6-v1",
|
||||
"name": "anthropic.claude-opus-4-6-v1",
|
||||
"description": "Claude 4.6 Opus",
|
||||
"input_cost": 0.000005,
|
||||
"output_cost": 0.000025
|
||||
},
|
||||
{
|
||||
"label": "anthropic.claude-sonnet-4-6",
|
||||
"name": "anthropic.claude-sonnet-4-6",
|
||||
"description": "Claude 4.6 Sonnet",
|
||||
"input_cost": 0.000003,
|
||||
"output_cost": 0.000015
|
||||
},
|
||||
{
|
||||
"label": "anthropic.claude-opus-4-5-20251101-v1:0",
|
||||
"name": "anthropic.claude-opus-4-5-20251101-v1:0",
|
||||
@@ -382,6 +396,30 @@
|
||||
"input_cost": 1.1e-6,
|
||||
"output_cost": 4.4e-6
|
||||
},
|
||||
{
|
||||
"label": "o3",
|
||||
"name": "o3",
|
||||
"input_cost": 2e-6,
|
||||
"output_cost": 8e-6
|
||||
},
|
||||
{
|
||||
"label": "o3-pro",
|
||||
"name": "o3-pro",
|
||||
"input_cost": 2e-5,
|
||||
"output_cost": 8e-5
|
||||
},
|
||||
{
|
||||
"label": "o4-mini",
|
||||
"name": "o4-mini",
|
||||
"input_cost": 1.1e-6,
|
||||
"output_cost": 4.4e-6
|
||||
},
|
||||
{
|
||||
"label": "codex-mini",
|
||||
"name": "codex-mini",
|
||||
"input_cost": 1.5e-6,
|
||||
"output_cost": 6e-6
|
||||
},
|
||||
{
|
||||
"label": "o1",
|
||||
"name": "o1",
|
||||
@@ -454,6 +492,18 @@
|
||||
"input_cost": 0.0000004,
|
||||
"output_cost": 0.0000016
|
||||
},
|
||||
{
|
||||
"label": "gpt-4.1-nano",
|
||||
"name": "gpt-4.1-nano",
|
||||
"input_cost": 1e-7,
|
||||
"output_cost": 4e-7
|
||||
},
|
||||
{
|
||||
"label": "gpt-5-pro",
|
||||
"name": "gpt-5-pro",
|
||||
"input_cost": 2.1e-5,
|
||||
"output_cost": 1.68e-4
|
||||
},
|
||||
{
|
||||
"label": "gpt-5-chat-latest",
|
||||
"name": "gpt-5-chat-latest",
|
||||
@@ -536,6 +586,20 @@
|
||||
{
|
||||
"name": "chatAnthropic",
|
||||
"models": [
|
||||
{
|
||||
"label": "claude-opus-4-6",
|
||||
"name": "claude-opus-4-6",
|
||||
"description": "Claude 4.6 Opus",
|
||||
"input_cost": 0.000005,
|
||||
"output_cost": 0.000025
|
||||
},
|
||||
{
|
||||
"label": "claude-sonnet-4-6",
|
||||
"name": "claude-sonnet-4-6",
|
||||
"description": "Claude 4.6 Sonnet",
|
||||
"input_cost": 0.000003,
|
||||
"output_cost": 0.000015
|
||||
},
|
||||
{
|
||||
"label": "claude-opus-4-5",
|
||||
"name": "claude-opus-4-5",
|
||||
@@ -666,11 +730,23 @@
|
||||
"name": "chatGoogleGenerativeAI",
|
||||
"models": [
|
||||
{
|
||||
"label": "gemini-3-pro-preview",
|
||||
"name": "gemini-3-pro-preview",
|
||||
"label": "gemini-3.1-pro-preview",
|
||||
"name": "gemini-3.1-pro-preview",
|
||||
"input_cost": 0.00002,
|
||||
"output_cost": 0.00012
|
||||
},
|
||||
{
|
||||
"label": "gemini-3.1-flash-lite-preview",
|
||||
"name": "gemini-3.1-flash-lite-preview",
|
||||
"input_cost": 0.00000025,
|
||||
"output_cost": 0.0000015
|
||||
},
|
||||
{
|
||||
"label": "gemini-3.1-flash-image-preview",
|
||||
"name": "gemini-3.1-flash-image-preview",
|
||||
"input_cost": 0.00000025,
|
||||
"output_cost": 0.000067
|
||||
},
|
||||
{
|
||||
"label": "gemini-3-flash-preview",
|
||||
"name": "gemini-3-flash-preview",
|
||||
@@ -754,11 +830,17 @@
|
||||
"name": "chatGoogleVertexAI",
|
||||
"models": [
|
||||
{
|
||||
"label": "gemini-3-pro-preview",
|
||||
"name": "gemini-3-pro-preview",
|
||||
"label": "gemini-3.1-pro-preview",
|
||||
"name": "gemini-3.1-pro-preview",
|
||||
"input_cost": 0.00002,
|
||||
"output_cost": 0.00012
|
||||
},
|
||||
{
|
||||
"label": "gemini-3.1-flash-lite-preview",
|
||||
"name": "gemini-3.1-flash-lite-preview",
|
||||
"input_cost": 0.00000025,
|
||||
"output_cost": 0.0000015
|
||||
},
|
||||
{
|
||||
"label": "gemini-3-flash-preview",
|
||||
"name": "gemini-3-flash-preview",
|
||||
@@ -831,6 +913,20 @@
|
||||
"input_cost": 1.25e-7,
|
||||
"output_cost": 3.75e-7
|
||||
},
|
||||
{
|
||||
"label": "claude-opus-4-6",
|
||||
"name": "claude-opus-4-6",
|
||||
"description": "Claude 4.6 Opus",
|
||||
"input_cost": 0.000005,
|
||||
"output_cost": 0.000025
|
||||
},
|
||||
{
|
||||
"label": "claude-sonnet-4-6",
|
||||
"name": "claude-sonnet-4-6",
|
||||
"description": "Claude 4.6 Sonnet",
|
||||
"input_cost": 0.000003,
|
||||
"output_cost": 0.000015
|
||||
},
|
||||
{
|
||||
"label": "claude-opus-4-5@20251101",
|
||||
"name": "claude-opus-4-5@20251101",
|
||||
@@ -1137,6 +1233,18 @@
|
||||
{
|
||||
"name": "chatOpenAI",
|
||||
"models": [
|
||||
{
|
||||
"label": "gpt-5.4",
|
||||
"name": "gpt-5.4",
|
||||
"input_cost": 0.0000025,
|
||||
"output_cost": 0.000015
|
||||
},
|
||||
{
|
||||
"label": "gpt-5.4-pro",
|
||||
"name": "gpt-5.4-pro",
|
||||
"input_cost": 0.00003,
|
||||
"output_cost": 0.00018
|
||||
},
|
||||
{
|
||||
"label": "gpt-5.2",
|
||||
"name": "gpt-5.2",
|
||||
@@ -1245,6 +1353,12 @@
|
||||
"input_cost": 2.5e-6,
|
||||
"output_cost": 0.00001
|
||||
},
|
||||
{
|
||||
"label": "o4-mini (latest)",
|
||||
"name": "o4-mini",
|
||||
"input_cost": 1.5e-7,
|
||||
"output_cost": 6e-7
|
||||
},
|
||||
{
|
||||
"label": "o3-mini (latest)",
|
||||
"name": "o3-mini",
|
||||
@@ -1376,12 +1490,6 @@
|
||||
"name": "gpt-3.5-turbo-16k-0613",
|
||||
"input_cost": 0.000003,
|
||||
"output_cost": 0.000004
|
||||
},
|
||||
{
|
||||
"label": "o4-mini",
|
||||
"name": "o4-mini",
|
||||
"input_cost": 1.5e-7,
|
||||
"output_cost": 6e-7
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -11,10 +11,11 @@ import {
|
||||
IServerSideEventStreamer,
|
||||
IUsedTool
|
||||
} from '../../../src/Interface'
|
||||
import { AIMessageChunk, BaseMessageLike, MessageContentText } from '@langchain/core/messages'
|
||||
import { ContentBlock } from 'langchain'
|
||||
import { AIMessageChunk, BaseMessageLike } from '@langchain/core/messages'
|
||||
import { AnalyticHandler } from '../../../src/handler'
|
||||
import { DEFAULT_SUMMARIZER_TEMPLATE } from '../prompt'
|
||||
import { ILLMMessage } from '../Interface.Agentflow'
|
||||
import { ILLMMessage, IResponseMetadata } from '../Interface.Agentflow'
|
||||
import { Tool } from '@langchain/core/tools'
|
||||
import { ARTIFACTS_PREFIX, SOURCE_DOCUMENTS_PREFIX, TOOL_ARGS_PREFIX } from '../../../src/agents'
|
||||
import { flatten } from 'lodash'
|
||||
@@ -28,11 +29,18 @@ import {
|
||||
getPastChatHistoryImageMessages,
|
||||
getUniqueImageMessages,
|
||||
processMessagesWithImages,
|
||||
replaceBase64ImagesWithFileReferences,
|
||||
revertBase64ImagesToFileRefs,
|
||||
replaceInlineDataWithFileReferences,
|
||||
updateFlowState
|
||||
} from '../utils'
|
||||
import { convertMultiOptionsToStringArray, processTemplateVariables, configureStructuredOutput } from '../../../src/utils'
|
||||
import {
|
||||
convertMultiOptionsToStringArray,
|
||||
processTemplateVariables,
|
||||
configureStructuredOutput,
|
||||
extractResponseContent
|
||||
} from '../../../src/utils'
|
||||
import { sanitizeFileName } from '../../../src/validator'
|
||||
import { getModelConfigByModelName, MODEL_TYPE } from '../../../src/modelLoader'
|
||||
|
||||
interface ITool {
|
||||
agentSelectedTool: string
|
||||
@@ -912,7 +920,7 @@ class Agent_Agentflow implements INode {
|
||||
}
|
||||
|
||||
const llmWithoutToolsBind = (await newLLMNodeInstance.init(newNodeData, '', options)) as BaseChatModel
|
||||
let llmNodeInstance = llmWithoutToolsBind
|
||||
let llmNodeInstance = llmWithoutToolsBind // save the original LLM instance for later use in withStructuredOutput, getNumTokens
|
||||
|
||||
const isStructuredOutput = _agentStructuredOutput && Array.isArray(_agentStructuredOutput) && _agentStructuredOutput.length > 0
|
||||
|
||||
@@ -1001,10 +1009,6 @@ class Agent_Agentflow implements INode {
|
||||
|
||||
// Prepare messages array
|
||||
const messages: BaseMessageLike[] = []
|
||||
// Use to store messages with image file references as we do not want to store the base64 data into database
|
||||
let runtimeImageMessagesWithFileRef: BaseMessageLike[] = []
|
||||
// Use to keep track of past messages with image file references
|
||||
let pastImageMessagesWithFileRef: BaseMessageLike[] = []
|
||||
|
||||
// Prepend history ONLY if it is the first node
|
||||
if (prependedChatHistory.length > 0 && !runtimeChatHistory.length) {
|
||||
@@ -1037,15 +1041,13 @@ class Agent_Agentflow implements INode {
|
||||
memoryType,
|
||||
pastChatHistory,
|
||||
runtimeChatHistory,
|
||||
llmNodeInstance,
|
||||
llmWithoutToolsBind,
|
||||
nodeData,
|
||||
userMessage,
|
||||
input,
|
||||
abortController,
|
||||
options,
|
||||
modelConfig,
|
||||
runtimeImageMessagesWithFileRef,
|
||||
pastImageMessagesWithFileRef
|
||||
modelConfig
|
||||
})
|
||||
} else if (!runtimeChatHistory.length) {
|
||||
/*
|
||||
@@ -1056,9 +1058,7 @@ class Agent_Agentflow implements INode {
|
||||
if (options.uploads) {
|
||||
const imageContents = await getUniqueImageMessages(options, messages, modelConfig)
|
||||
if (imageContents) {
|
||||
const { imageMessageWithBase64, imageMessageWithFileRef } = imageContents
|
||||
messages.push(imageMessageWithBase64)
|
||||
runtimeImageMessagesWithFileRef.push(imageMessageWithFileRef)
|
||||
messages.push(imageContents.imageMessageWithBase64)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1094,14 +1094,16 @@ class Agent_Agentflow implements INode {
|
||||
let fileAnnotations: any[] = []
|
||||
let additionalTokens = 0
|
||||
let isWaitingForHumanInput = false
|
||||
let reasonContent = ''
|
||||
let thinkingDuration: number | undefined
|
||||
|
||||
// Store the current messages length to track which messages are added during tool calls
|
||||
const messagesBeforeToolCalls = [...messages]
|
||||
let _toolCallMessages: BaseMessageLike[] = []
|
||||
|
||||
/**
|
||||
* Add image artifacts from previous assistant responses as user messages
|
||||
* Images are converted from FILE-STORAGE::<image_path> to base 64 image_url format
|
||||
* Add image artifacts from previous assistant responses as user messages.
|
||||
* Only the inserted temporary messages contain base64 — other messages are untouched.
|
||||
*/
|
||||
await addImageArtifactsToMessages(messages, options)
|
||||
|
||||
@@ -1144,6 +1146,12 @@ class Agent_Agentflow implements INode {
|
||||
artifacts = result.artifacts
|
||||
additionalTokens = result.totalTokens
|
||||
isWaitingForHumanInput = result.isWaitingForHumanInput || false
|
||||
if (result.accumulatedReasonContent !== undefined) {
|
||||
reasonContent = result.accumulatedReasonContent
|
||||
}
|
||||
if (result.accumulatedReasoningDuration !== undefined) {
|
||||
thinkingDuration = result.accumulatedReasoningDuration
|
||||
}
|
||||
|
||||
// Calculate which messages were added during tool calls
|
||||
_toolCallMessages = messages.slice(messagesBeforeToolCalls.length)
|
||||
@@ -1170,13 +1178,22 @@ class Agent_Agentflow implements INode {
|
||||
messages,
|
||||
chatId,
|
||||
abortController,
|
||||
isStructuredOutput
|
||||
isStructuredOutput,
|
||||
isLastNode
|
||||
)
|
||||
} else {
|
||||
response = await llmNodeInstance.invoke(messages, { signal: abortController?.signal })
|
||||
}
|
||||
}
|
||||
|
||||
// Capture reasoning and duration from first LLM response so they can be accumulated across tool-call turns
|
||||
if (response.additional_kwargs?.reasoning_content) {
|
||||
reasonContent = (response.additional_kwargs.reasoning_content as string) || ''
|
||||
}
|
||||
if (typeof response.additional_kwargs?.reasoning_duration === 'number') {
|
||||
thinkingDuration = response.additional_kwargs.reasoning_duration
|
||||
}
|
||||
|
||||
// Address built in tools (after artifacts are processed)
|
||||
const builtInUsedTools: IUsedTool[] = await this.extractBuiltInUsedTools(response, [])
|
||||
|
||||
@@ -1194,7 +1211,9 @@ class Agent_Agentflow implements INode {
|
||||
isStreamable,
|
||||
isLastNode,
|
||||
iterationContext,
|
||||
isStructuredOutput
|
||||
isStructuredOutput,
|
||||
accumulatedReasonContent: reasonContent,
|
||||
accumulatedReasoningDuration: thinkingDuration
|
||||
})
|
||||
|
||||
response = result.response
|
||||
@@ -1203,6 +1222,12 @@ class Agent_Agentflow implements INode {
|
||||
artifacts = result.artifacts
|
||||
additionalTokens = result.totalTokens
|
||||
isWaitingForHumanInput = result.isWaitingForHumanInput || false
|
||||
if (result.accumulatedReasonContent !== undefined) {
|
||||
reasonContent = result.accumulatedReasonContent
|
||||
}
|
||||
if (result.accumulatedReasoningDuration !== undefined) {
|
||||
thinkingDuration = result.accumulatedReasoningDuration
|
||||
}
|
||||
|
||||
// Calculate which messages were added during tool calls
|
||||
_toolCallMessages = messages.slice(messagesBeforeToolCalls.length)
|
||||
@@ -1224,23 +1249,27 @@ class Agent_Agentflow implements INode {
|
||||
} else if (!humanInput && !isStreamable && isLastNode && sseStreamer && !isStructuredOutput) {
|
||||
// Stream whole response back to UI if not streaming and no tool calls
|
||||
// Skip this if structured output is enabled - it will be streamed after conversion
|
||||
let finalResponse = ''
|
||||
if (response.content && Array.isArray(response.content)) {
|
||||
finalResponse = response.content
|
||||
.map((item: any) => {
|
||||
if ((item.text && !item.type) || (item.type === 'text' && item.text)) {
|
||||
return item.text
|
||||
}
|
||||
return ''
|
||||
})
|
||||
.filter((text: string) => text)
|
||||
.join('\n')
|
||||
} else if (response.content && typeof response.content === 'string') {
|
||||
finalResponse = response.content
|
||||
} else {
|
||||
finalResponse = JSON.stringify(response, null, 2)
|
||||
|
||||
// Stream thinking content if available
|
||||
if (response.contentBlocks?.length) {
|
||||
for (const block of response.contentBlocks) {
|
||||
if (block.type === 'reasoning' && (block as { reasoning?: string }).reasoning) {
|
||||
reasonContent += (block as { reasoning: string }).reasoning
|
||||
}
|
||||
if ((block as any).type === 'thinking' && block.thinking) {
|
||||
reasonContent += block.thinking
|
||||
}
|
||||
}
|
||||
|
||||
sseStreamer.streamThinkingEvent(chatId, reasonContent)
|
||||
// Send end of thinking event with duration from token details if available
|
||||
const reasoningTokens = response.usage_metadata?.output_token_details?.reasoning || 0
|
||||
// Estimate duration based on reasoning tokens (rough estimate: ~50 tokens/sec)
|
||||
thinkingDuration = reasoningTokens > 0 ? Math.round(reasoningTokens / 50) : 2
|
||||
sseStreamer.streamThinkingEvent(chatId, '', thinkingDuration)
|
||||
}
|
||||
sseStreamer.streamTokenEvent(chatId, finalResponse)
|
||||
|
||||
sseStreamer.streamTokenEvent(chatId, extractResponseContent(response))
|
||||
}
|
||||
|
||||
// Calculate execution time
|
||||
@@ -1331,7 +1360,7 @@ class Agent_Agentflow implements INode {
|
||||
artifacts: extractedArtifacts,
|
||||
fileAnnotations: extractedFileAnnotations,
|
||||
savedInlineImages
|
||||
} = await extractArtifactsFromResponse(response.response_metadata, newNodeData, options)
|
||||
} = await extractArtifactsFromResponse(response.response_metadata as IResponseMetadata, newNodeData, options)
|
||||
if (extractedArtifacts.length > 0) {
|
||||
artifacts = [...artifacts, ...extractedArtifacts]
|
||||
|
||||
@@ -1363,10 +1392,11 @@ class Agent_Agentflow implements INode {
|
||||
|
||||
// If is structured output, then invoke LLM again with structured output at the very end after all tool calls
|
||||
if (isStructuredOutput) {
|
||||
llmNodeInstance = configureStructuredOutput(llmNodeInstance, _agentStructuredOutput)
|
||||
const structuredllmNodeInstance = configureStructuredOutput(llmWithoutToolsBind, _agentStructuredOutput)
|
||||
const prompt = 'Convert the following response to the structured output format: ' + finalResponse
|
||||
response = await llmNodeInstance.invoke(prompt, { signal: abortController?.signal })
|
||||
response = await structuredllmNodeInstance.invoke(prompt, { signal: abortController?.signal })
|
||||
|
||||
// Prefix the response with ```json and suffix with ``` to render as a code block
|
||||
if (typeof response === 'object') {
|
||||
finalResponse = '```json\n' + JSON.stringify(response, null, 2) + '\n```'
|
||||
} else {
|
||||
@@ -1378,6 +1408,23 @@ class Agent_Agentflow implements INode {
|
||||
}
|
||||
}
|
||||
|
||||
// Add reasoning content
|
||||
if (!reasonContent && response.additional_kwargs?.reasoning_content) {
|
||||
reasonContent = response.additional_kwargs.reasoning_content as string
|
||||
}
|
||||
if (reasonContent && response.additional_kwargs?.reasoning_duration != null) {
|
||||
thinkingDuration = response.additional_kwargs.reasoning_duration as number
|
||||
}
|
||||
const reasonContentObj =
|
||||
reasonContent !== undefined && reasonContent !== '' ? { thinking: reasonContent, thinkingDuration } : undefined
|
||||
|
||||
const costMetadata = await this.calculateUsageCost(
|
||||
model,
|
||||
modelConfig?.modelName as string | undefined,
|
||||
response.usage_metadata,
|
||||
additionalTokens
|
||||
)
|
||||
|
||||
const output = this.prepareOutputObject(
|
||||
response,
|
||||
availableTools,
|
||||
@@ -1391,7 +1438,9 @@ class Agent_Agentflow implements INode {
|
||||
additionalTokens,
|
||||
isWaitingForHumanInput,
|
||||
fileAnnotations,
|
||||
isStructuredOutput
|
||||
isStructuredOutput,
|
||||
reasonContentObj,
|
||||
costMetadata
|
||||
)
|
||||
|
||||
// End analytics tracking
|
||||
@@ -1415,23 +1464,25 @@ class Agent_Agentflow implements INode {
|
||||
newState = processTemplateVariables(newState, outputForStateProcessing)
|
||||
|
||||
/**
|
||||
* Remove the temporarily added image artifact messages before storing
|
||||
* Remove temporary artifact image messages (they were only needed for the model invoke).
|
||||
* Then revert all remaining tagged base64 image_url items back to stored-file format.
|
||||
* This is to avoid storing the actual base64 data into database
|
||||
*/
|
||||
const messagesToStore = messages.filter((msg: any) => !msg._isTemporaryImageMessage)
|
||||
|
||||
// Replace the actual messages array with one that includes the file references for images instead of base64 data
|
||||
const messagesWithFileReferences = replaceBase64ImagesWithFileReferences(
|
||||
messagesToStore,
|
||||
runtimeImageMessagesWithFileRef,
|
||||
pastImageMessagesWithFileRef
|
||||
)
|
||||
const messagesWithFileReferences = revertBase64ImagesToFileRefs(messagesToStore)
|
||||
|
||||
// Only add to runtime chat history if this is the first node
|
||||
const inputMessages = []
|
||||
if (!runtimeChatHistory.length) {
|
||||
if (runtimeImageMessagesWithFileRef.length) {
|
||||
inputMessages.push(...runtimeImageMessagesWithFileRef)
|
||||
// Include any image file reference messages from uploads in the chat history
|
||||
const imageInputMessages = messagesWithFileReferences.filter(
|
||||
(msg: any) =>
|
||||
msg.role === 'user' &&
|
||||
Array.isArray(msg.content) &&
|
||||
msg.content.some((item: any) => item.type === 'stored-file' && item.mime?.startsWith('image/'))
|
||||
)
|
||||
if (imageInputMessages.length) {
|
||||
inputMessages.push(...imageInputMessages)
|
||||
}
|
||||
if (input && typeof input === 'string') {
|
||||
if (!enableMemory) {
|
||||
@@ -1509,7 +1560,12 @@ class Agent_Agentflow implements INode {
|
||||
return builtInUsedTools
|
||||
}
|
||||
|
||||
const { output, tools, groundingMetadata, urlContextMetadata } = response.response_metadata
|
||||
const { output, tools, groundingMetadata, urlContextMetadata } = response.response_metadata as {
|
||||
output?: any[]
|
||||
tools?: any[]
|
||||
groundingMetadata?: { webSearchQueries?: string[] }
|
||||
urlContextMetadata?: { urlMetadata?: any[] }
|
||||
}
|
||||
|
||||
// Handle OpenAI built-in tools
|
||||
if (output && Array.isArray(output) && output.length > 0 && tools && Array.isArray(tools) && tools.length > 0) {
|
||||
@@ -1602,8 +1658,9 @@ class Agent_Agentflow implements INode {
|
||||
const item = response.content[i]
|
||||
|
||||
if (item.type === 'executableCode' && item.executableCode) {
|
||||
const language = item.executableCode.language || 'PYTHON'
|
||||
const code = item.executableCode.code || ''
|
||||
const executableCode = item.executableCode as { language?: string; code?: string }
|
||||
const language = executableCode.language || 'PYTHON'
|
||||
const code = executableCode.code || ''
|
||||
let toolOutput = ''
|
||||
|
||||
// Check for duplicates
|
||||
@@ -1622,8 +1679,9 @@ class Agent_Agentflow implements INode {
|
||||
|
||||
if (nextItem) {
|
||||
if (nextItem.type === 'codeExecutionResult' && nextItem.codeExecutionResult) {
|
||||
const outcome = nextItem.codeExecutionResult.outcome
|
||||
const output = nextItem.codeExecutionResult.output || ''
|
||||
const codeExecutionResult = nextItem.codeExecutionResult as { outcome?: string; output?: string }
|
||||
const outcome = codeExecutionResult.outcome
|
||||
const output = codeExecutionResult.output || ''
|
||||
toolOutput = outcome === 'OUTCOME_OK' ? output : `Error: ${output}`
|
||||
} else if (nextItem.type === 'inlineData') {
|
||||
toolOutput = 'Generated image data'
|
||||
@@ -1653,33 +1711,28 @@ class Agent_Agentflow implements INode {
|
||||
memoryType,
|
||||
pastChatHistory,
|
||||
runtimeChatHistory,
|
||||
llmNodeInstance,
|
||||
llmWithoutToolsBind,
|
||||
nodeData,
|
||||
userMessage,
|
||||
input,
|
||||
abortController,
|
||||
options,
|
||||
modelConfig,
|
||||
runtimeImageMessagesWithFileRef,
|
||||
pastImageMessagesWithFileRef
|
||||
modelConfig
|
||||
}: {
|
||||
messages: BaseMessageLike[]
|
||||
memoryType: string
|
||||
pastChatHistory: BaseMessageLike[]
|
||||
runtimeChatHistory: BaseMessageLike[]
|
||||
llmNodeInstance: BaseChatModel
|
||||
llmWithoutToolsBind: BaseChatModel
|
||||
nodeData: INodeData
|
||||
userMessage: string
|
||||
input: string | Record<string, any>
|
||||
abortController: AbortController
|
||||
options: ICommonObject
|
||||
modelConfig: ICommonObject
|
||||
runtimeImageMessagesWithFileRef: BaseMessageLike[]
|
||||
pastImageMessagesWithFileRef: BaseMessageLike[]
|
||||
}): Promise<void> {
|
||||
const { updatedPastMessages, transformedPastMessages } = await getPastChatHistoryImageMessages(pastChatHistory, options)
|
||||
const { updatedPastMessages } = await getPastChatHistoryImageMessages(pastChatHistory, options)
|
||||
pastChatHistory = updatedPastMessages
|
||||
pastImageMessagesWithFileRef.push(...transformedPastMessages)
|
||||
|
||||
let pastMessages = [...pastChatHistory, ...runtimeChatHistory]
|
||||
if (!runtimeChatHistory.length && input && typeof input === 'string') {
|
||||
@@ -1691,9 +1744,7 @@ class Agent_Agentflow implements INode {
|
||||
if (options.uploads) {
|
||||
const imageContents = await getUniqueImageMessages(options, messages, modelConfig)
|
||||
if (imageContents) {
|
||||
const { imageMessageWithBase64, imageMessageWithFileRef } = imageContents
|
||||
pastMessages.push(imageMessageWithBase64)
|
||||
runtimeImageMessagesWithFileRef.push(imageMessageWithFileRef)
|
||||
pastMessages.push(imageContents.imageMessageWithBase64)
|
||||
}
|
||||
}
|
||||
pastMessages.push({
|
||||
@@ -1701,9 +1752,8 @@ class Agent_Agentflow implements INode {
|
||||
content: input
|
||||
})
|
||||
}
|
||||
const { updatedMessages, transformedMessages } = await processMessagesWithImages(pastMessages, options)
|
||||
const { updatedMessages } = await processMessagesWithImages(pastMessages, options)
|
||||
pastMessages = updatedMessages
|
||||
pastImageMessagesWithFileRef.push(...transformedMessages)
|
||||
|
||||
if (pastMessages.length > 0) {
|
||||
if (memoryType === 'windowSize') {
|
||||
@@ -1713,7 +1763,7 @@ class Agent_Agentflow implements INode {
|
||||
messages.push(...windowedMessages)
|
||||
} else if (memoryType === 'conversationSummary') {
|
||||
// Summary memory: Summarize all past messages
|
||||
const summary = await llmNodeInstance.invoke(
|
||||
const summary = await llmWithoutToolsBind.invoke(
|
||||
[
|
||||
{
|
||||
role: 'user',
|
||||
@@ -1725,10 +1775,16 @@ class Agent_Agentflow implements INode {
|
||||
],
|
||||
{ signal: abortController?.signal }
|
||||
)
|
||||
messages.push({ role: 'assistant', content: summary.content as string })
|
||||
messages.push({ role: 'assistant', content: extractResponseContent(summary) })
|
||||
if (!userMessage && input && typeof input === 'string') {
|
||||
messages.push({
|
||||
role: 'user',
|
||||
content: input
|
||||
})
|
||||
}
|
||||
} else if (memoryType === 'conversationSummaryBuffer') {
|
||||
// Summary buffer: Summarize messages that exceed token limit
|
||||
await this.handleSummaryBuffer(messages, pastMessages, llmNodeInstance, nodeData, abortController)
|
||||
await this.handleSummaryBuffer(messages, pastMessages, llmWithoutToolsBind, nodeData, abortController)
|
||||
} else {
|
||||
// Default: Use all messages
|
||||
messages.push(...pastMessages)
|
||||
@@ -1750,7 +1806,7 @@ class Agent_Agentflow implements INode {
|
||||
private async handleSummaryBuffer(
|
||||
messages: BaseMessageLike[],
|
||||
pastMessages: BaseMessageLike[],
|
||||
llmNodeInstance: BaseChatModel,
|
||||
llmWithoutToolsBind: BaseChatModel,
|
||||
nodeData: INodeData,
|
||||
abortController: AbortController
|
||||
): Promise<void> {
|
||||
@@ -1758,7 +1814,7 @@ class Agent_Agentflow implements INode {
|
||||
|
||||
// Convert past messages to a format suitable for token counting
|
||||
const messagesString = pastMessages.map((msg: any) => `${msg.role}: ${msg.content}`).join('\n')
|
||||
const tokenCount = await llmNodeInstance.getNumTokens(messagesString)
|
||||
const tokenCount = await llmWithoutToolsBind.getNumTokens(messagesString)
|
||||
|
||||
if (tokenCount > maxTokenLimit) {
|
||||
// Calculate how many messages to summarize (messages that exceed the token limit)
|
||||
@@ -1773,14 +1829,14 @@ class Agent_Agentflow implements INode {
|
||||
messagesToSummarize.push(poppedMessage)
|
||||
// Recalculate token count for remaining messages
|
||||
const remainingMessagesString = remainingMessages.map((msg: any) => `${msg.role}: ${msg.content}`).join('\n')
|
||||
currBufferLength = await llmNodeInstance.getNumTokens(remainingMessagesString)
|
||||
currBufferLength = await llmWithoutToolsBind.getNumTokens(remainingMessagesString)
|
||||
}
|
||||
}
|
||||
|
||||
// Summarize the messages that were removed
|
||||
const messagesToSummarizeString = messagesToSummarize.map((msg: any) => `${msg.role}: ${msg.content}`).join('\n')
|
||||
|
||||
const summary = await llmNodeInstance.invoke(
|
||||
const summary = await llmWithoutToolsBind.invoke(
|
||||
[
|
||||
{
|
||||
role: 'user',
|
||||
@@ -1791,7 +1847,11 @@ class Agent_Agentflow implements INode {
|
||||
)
|
||||
|
||||
// Add summary as a system message at the beginning, then add remaining messages
|
||||
messages.push({ role: 'system', content: `Previous conversation summary: ${summary.content}` })
|
||||
let summaryRole = 'system'
|
||||
if (messages.some((msg) => typeof msg === 'object' && !Array.isArray(msg) && 'role' in msg && msg.role === 'system')) {
|
||||
summaryRole = 'user' // some model doesn't allow multiple system messages
|
||||
}
|
||||
messages.push({ role: summaryRole, content: `Previous conversation summary: ${extractResponseContent(summary)}` })
|
||||
messages.push(...remainingMessages)
|
||||
} else {
|
||||
// If under token limit, use all messages
|
||||
@@ -1808,15 +1868,43 @@ class Agent_Agentflow implements INode {
|
||||
messages: BaseMessageLike[],
|
||||
chatId: string,
|
||||
abortController: AbortController,
|
||||
isStructuredOutput: boolean = false
|
||||
isStructuredOutput: boolean = false,
|
||||
isLastNode: boolean = false
|
||||
): Promise<AIMessageChunk> {
|
||||
let response = new AIMessageChunk('')
|
||||
let reasonContent = ''
|
||||
let thinkingDuration: number | undefined
|
||||
let thinkingStartTime: number | null = null
|
||||
let wasThinking = false
|
||||
let sentLastThinkingEvent = false
|
||||
|
||||
try {
|
||||
for await (const chunk of await llmNodeInstance.stream(messages, { signal: abortController?.signal })) {
|
||||
if (sseStreamer && !isStructuredOutput) {
|
||||
let content = ''
|
||||
|
||||
if (chunk.contentBlocks?.length) {
|
||||
for (const block of chunk.contentBlocks) {
|
||||
if (isLastNode) {
|
||||
// As soon as we see the first non-reasoning block, send last thinking event with duration
|
||||
if (block.type !== 'reasoning' && wasThinking && !sentLastThinkingEvent && thinkingStartTime != null) {
|
||||
thinkingDuration = Math.round((Date.now() - thinkingStartTime) / 1000)
|
||||
sseStreamer.streamThinkingEvent(chatId, '', thinkingDuration)
|
||||
sentLastThinkingEvent = true
|
||||
}
|
||||
if (block.type === 'reasoning' && (block as { reasoning?: string }).reasoning) {
|
||||
if (!thinkingStartTime) {
|
||||
thinkingStartTime = Date.now()
|
||||
}
|
||||
wasThinking = true
|
||||
const reasoningContent = (block as { reasoning: string }).reasoning
|
||||
sseStreamer.streamThinkingEvent(chatId, reasoningContent)
|
||||
reasonContent += reasoningContent
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof chunk === 'string') {
|
||||
content = chunk
|
||||
} else if (Array.isArray(chunk.content) && chunk.content.length > 0) {
|
||||
@@ -1860,13 +1948,65 @@ class Agent_Agentflow implements INode {
|
||||
(item: any) => item.type === 'inlineData' || item.type === 'executableCode' || item.type === 'codeExecutionResult'
|
||||
)
|
||||
if (!hasNonTextContent) {
|
||||
const responseContents = response.content as MessageContentText[]
|
||||
const responseContents = response.content as ContentBlock.Text[]
|
||||
response.content = responseContents.map((item) => item.text).join('')
|
||||
}
|
||||
}
|
||||
|
||||
if (reasonContent.length > 0) {
|
||||
response.additional_kwargs = {
|
||||
...response.additional_kwargs,
|
||||
reasoning_content: reasonContent,
|
||||
reasoning_duration: thinkingDuration
|
||||
}
|
||||
}
|
||||
|
||||
return response
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates input/output and total cost from usage metadata using model pricing from models.json.
|
||||
* Also returns the model's base (per-token) input and output costs.
|
||||
*/
|
||||
private async calculateUsageCost(
|
||||
provider: string | undefined,
|
||||
modelName: string | undefined,
|
||||
usageMetadata: Record<string, any> | undefined,
|
||||
additionalTokens: number = 0
|
||||
): Promise<
|
||||
| {
|
||||
input_cost: number
|
||||
output_cost: number
|
||||
total_cost: number
|
||||
base_input_cost: number
|
||||
base_output_cost: number
|
||||
}
|
||||
| undefined
|
||||
> {
|
||||
if (!provider || !modelName) return undefined
|
||||
const inputTokens = (usageMetadata?.input_tokens ?? 0) as number
|
||||
const outputTokens = ((usageMetadata?.output_tokens ?? 0) as number) + additionalTokens
|
||||
try {
|
||||
const modelConfig = await getModelConfigByModelName(MODEL_TYPE.CHAT, provider, modelName)
|
||||
if (!modelConfig) return undefined
|
||||
const baseInputCost = Number(modelConfig.input_cost) || 0
|
||||
const baseOutputCost = Number(modelConfig.output_cost) || 0
|
||||
const inputCost = inputTokens * baseInputCost
|
||||
const outputCost = outputTokens * baseOutputCost
|
||||
const totalCost = inputCost + outputCost
|
||||
if (inputCost === 0 && outputCost === 0) return undefined
|
||||
return {
|
||||
input_cost: inputCost,
|
||||
output_cost: outputCost,
|
||||
total_cost: totalCost,
|
||||
base_input_cost: baseInputCost,
|
||||
base_output_cost: baseOutputCost
|
||||
}
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepares the output object with response and metadata
|
||||
*/
|
||||
@@ -1883,7 +2023,15 @@ class Agent_Agentflow implements INode {
|
||||
additionalTokens: number = 0,
|
||||
isWaitingForHumanInput: boolean = false,
|
||||
fileAnnotations: any[] = [],
|
||||
isStructuredOutput: boolean = false
|
||||
isStructuredOutput: boolean = false,
|
||||
reasonContent?: { thinking: string; thinkingDuration?: number },
|
||||
costMetadata?: {
|
||||
input_cost: number
|
||||
output_cost: number
|
||||
total_cost: number
|
||||
base_input_cost: number
|
||||
base_output_cost: number
|
||||
}
|
||||
): any {
|
||||
const output: any = {
|
||||
content: finalResponse,
|
||||
@@ -1914,6 +2062,14 @@ class Agent_Agentflow implements INode {
|
||||
}
|
||||
}
|
||||
|
||||
if (costMetadata && output.usageMetadata) {
|
||||
output.usageMetadata.input_cost = costMetadata.input_cost
|
||||
output.usageMetadata.output_cost = costMetadata.output_cost
|
||||
output.usageMetadata.total_cost = costMetadata.total_cost
|
||||
output.usageMetadata.base_input_cost = costMetadata.base_input_cost
|
||||
output.usageMetadata.base_output_cost = costMetadata.base_output_cost
|
||||
}
|
||||
|
||||
if (response.response_metadata) {
|
||||
output.responseMetadata = response.response_metadata
|
||||
}
|
||||
@@ -1952,6 +2108,10 @@ class Agent_Agentflow implements INode {
|
||||
output.fileAnnotations = fileAnnotations
|
||||
}
|
||||
|
||||
if (reasonContent) {
|
||||
output.reasonContent = reasonContent
|
||||
}
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
@@ -1993,7 +2153,9 @@ class Agent_Agentflow implements INode {
|
||||
isStreamable,
|
||||
isLastNode,
|
||||
iterationContext,
|
||||
isStructuredOutput = false
|
||||
isStructuredOutput = false,
|
||||
accumulatedReasonContent: initialAccumulatedReasonContent,
|
||||
accumulatedReasoningDuration: initialAccumulatedReasoningDuration
|
||||
}: {
|
||||
response: AIMessageChunk
|
||||
messages: BaseMessageLike[]
|
||||
@@ -2008,6 +2170,8 @@ class Agent_Agentflow implements INode {
|
||||
isLastNode: boolean
|
||||
iterationContext: ICommonObject
|
||||
isStructuredOutput?: boolean
|
||||
accumulatedReasonContent?: string
|
||||
accumulatedReasoningDuration?: number
|
||||
}): Promise<{
|
||||
response: AIMessageChunk
|
||||
usedTools: IUsedTool[]
|
||||
@@ -2015,6 +2179,8 @@ class Agent_Agentflow implements INode {
|
||||
artifacts: any[]
|
||||
totalTokens: number
|
||||
isWaitingForHumanInput?: boolean
|
||||
accumulatedReasonContent?: string
|
||||
accumulatedReasoningDuration?: number
|
||||
}> {
|
||||
// Track total tokens used throughout this process
|
||||
let totalTokens = response.usage_metadata?.total_tokens || 0
|
||||
@@ -2022,9 +2188,20 @@ class Agent_Agentflow implements INode {
|
||||
let sourceDocuments: Array<any> = []
|
||||
let artifacts: any[] = []
|
||||
let isWaitingForHumanInput: boolean | undefined
|
||||
// Use reasoning from caller (first turn); subsequent turns are added when we get newResponse
|
||||
let accumulatedReasonContent = initialAccumulatedReasonContent ?? ''
|
||||
let accumulatedReasoningDuration = initialAccumulatedReasoningDuration ?? 0
|
||||
|
||||
if (!response.tool_calls || response.tool_calls.length === 0) {
|
||||
return { response, usedTools: [], sourceDocuments: [], artifacts: [], totalTokens }
|
||||
return {
|
||||
response,
|
||||
usedTools: [],
|
||||
sourceDocuments: [],
|
||||
artifacts: [],
|
||||
totalTokens,
|
||||
accumulatedReasonContent: accumulatedReasonContent || undefined,
|
||||
accumulatedReasoningDuration: accumulatedReasoningDuration || undefined
|
||||
}
|
||||
}
|
||||
|
||||
// Stream tool calls if available
|
||||
@@ -2090,7 +2267,16 @@ class Agent_Agentflow implements INode {
|
||||
if (!isStructuredOutput) {
|
||||
sseStreamer?.streamTokenEvent(chatId, responseContent)
|
||||
}
|
||||
return { response, usedTools, sourceDocuments, artifacts, totalTokens, isWaitingForHumanInput: true }
|
||||
return {
|
||||
response,
|
||||
usedTools,
|
||||
sourceDocuments,
|
||||
artifacts,
|
||||
totalTokens,
|
||||
isWaitingForHumanInput: true,
|
||||
accumulatedReasonContent: accumulatedReasonContent || undefined,
|
||||
accumulatedReasoningDuration: accumulatedReasoningDuration || undefined
|
||||
}
|
||||
}
|
||||
|
||||
let toolIds: ICommonObject | undefined
|
||||
@@ -2204,19 +2390,23 @@ class Agent_Agentflow implements INode {
|
||||
usedTools,
|
||||
sourceDocuments,
|
||||
artifacts,
|
||||
totalTokens
|
||||
totalTokens,
|
||||
accumulatedReasonContent: accumulatedReasonContent || undefined,
|
||||
accumulatedReasoningDuration: accumulatedReasoningDuration || undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (response.tool_calls.length === 0) {
|
||||
const responseContent = typeof response.content === 'string' ? response.content : JSON.stringify(response.content, null, 2)
|
||||
const responseContent = extractResponseContent(response)
|
||||
return {
|
||||
response: new AIMessageChunk(responseContent),
|
||||
usedTools,
|
||||
sourceDocuments,
|
||||
artifacts,
|
||||
totalTokens
|
||||
totalTokens,
|
||||
accumulatedReasonContent: accumulatedReasonContent || undefined,
|
||||
accumulatedReasoningDuration: accumulatedReasoningDuration || undefined
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2230,18 +2420,15 @@ class Agent_Agentflow implements INode {
|
||||
messages,
|
||||
chatId,
|
||||
abortController,
|
||||
isStructuredOutput
|
||||
isStructuredOutput,
|
||||
isLastNode
|
||||
)
|
||||
} else {
|
||||
newResponse = await llmNodeInstance.invoke(messages, { signal: abortController?.signal })
|
||||
|
||||
// Stream non-streaming response if this is the last node
|
||||
if (isLastNode && sseStreamer && !isStructuredOutput) {
|
||||
let responseContent = JSON.stringify(newResponse, null, 2)
|
||||
if (typeof newResponse.content === 'string') {
|
||||
responseContent = newResponse.content
|
||||
}
|
||||
sseStreamer.streamTokenEvent(chatId, responseContent)
|
||||
sseStreamer.streamTokenEvent(chatId, extractResponseContent(newResponse))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2250,6 +2437,15 @@ class Agent_Agentflow implements INode {
|
||||
totalTokens += newResponse.usage_metadata.total_tokens
|
||||
}
|
||||
|
||||
// Accumulate this turn's reasoning content and duration
|
||||
if (newResponse.additional_kwargs?.reasoning_content) {
|
||||
const chunkReason = newResponse.additional_kwargs.reasoning_content as string
|
||||
accumulatedReasonContent += (accumulatedReasonContent ? '\n\n' : '') + chunkReason
|
||||
}
|
||||
if (typeof newResponse.additional_kwargs?.reasoning_duration === 'number') {
|
||||
accumulatedReasoningDuration += newResponse.additional_kwargs.reasoning_duration
|
||||
}
|
||||
|
||||
// Check for recursive tool calls and handle them
|
||||
if (newResponse.tool_calls && newResponse.tool_calls.length > 0) {
|
||||
const {
|
||||
@@ -2258,7 +2454,9 @@ class Agent_Agentflow implements INode {
|
||||
sourceDocuments: recursiveSourceDocuments,
|
||||
artifacts: recursiveArtifacts,
|
||||
totalTokens: recursiveTokens,
|
||||
isWaitingForHumanInput: recursiveIsWaitingForHumanInput
|
||||
isWaitingForHumanInput: recursiveIsWaitingForHumanInput,
|
||||
accumulatedReasonContent: recursiveAccumulatedReasonContent,
|
||||
accumulatedReasoningDuration: recursiveAccumulatedReasoningDuration
|
||||
} = await this.handleToolCalls({
|
||||
response: newResponse,
|
||||
messages,
|
||||
@@ -2272,7 +2470,9 @@ class Agent_Agentflow implements INode {
|
||||
isStreamable,
|
||||
isLastNode,
|
||||
iterationContext,
|
||||
isStructuredOutput
|
||||
isStructuredOutput,
|
||||
accumulatedReasonContent,
|
||||
accumulatedReasoningDuration
|
||||
})
|
||||
|
||||
// Merge results from recursive tool calls
|
||||
@@ -2282,9 +2482,24 @@ class Agent_Agentflow implements INode {
|
||||
artifacts = [...artifacts, ...recursiveArtifacts]
|
||||
totalTokens += recursiveTokens
|
||||
isWaitingForHumanInput = recursiveIsWaitingForHumanInput
|
||||
if (recursiveAccumulatedReasonContent !== undefined) {
|
||||
accumulatedReasonContent = recursiveAccumulatedReasonContent
|
||||
}
|
||||
if (recursiveAccumulatedReasoningDuration !== undefined) {
|
||||
accumulatedReasoningDuration = recursiveAccumulatedReasoningDuration
|
||||
}
|
||||
}
|
||||
|
||||
return { response: newResponse, usedTools, sourceDocuments, artifacts, totalTokens, isWaitingForHumanInput }
|
||||
return {
|
||||
response: newResponse,
|
||||
usedTools,
|
||||
sourceDocuments,
|
||||
artifacts,
|
||||
totalTokens,
|
||||
isWaitingForHumanInput,
|
||||
accumulatedReasonContent: accumulatedReasonContent || undefined,
|
||||
accumulatedReasoningDuration: accumulatedReasoningDuration || undefined
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -2327,6 +2542,8 @@ class Agent_Agentflow implements INode {
|
||||
artifacts: any[]
|
||||
totalTokens: number
|
||||
isWaitingForHumanInput?: boolean
|
||||
accumulatedReasonContent?: string
|
||||
accumulatedReasoningDuration?: number
|
||||
}> {
|
||||
let llmNodeInstance = llmWithoutToolsBind
|
||||
const usedTools: IUsedTool[] = []
|
||||
@@ -2336,7 +2553,15 @@ class Agent_Agentflow implements INode {
|
||||
|
||||
const lastCheckpointMessages = humanInputAction?.data?.input?.messages ?? []
|
||||
if (!lastCheckpointMessages.length) {
|
||||
return { response: new AIMessageChunk(''), usedTools: [], sourceDocuments: [], artifacts: [], totalTokens: 0 }
|
||||
return {
|
||||
response: new AIMessageChunk(''),
|
||||
usedTools: [],
|
||||
sourceDocuments: [],
|
||||
artifacts: [],
|
||||
totalTokens: 0,
|
||||
accumulatedReasonContent: undefined,
|
||||
accumulatedReasoningDuration: undefined
|
||||
}
|
||||
}
|
||||
|
||||
// Use the last message as the response
|
||||
@@ -2350,7 +2575,20 @@ class Agent_Agentflow implements INode {
|
||||
let totalTokens = response.usage_metadata?.total_tokens || 0
|
||||
|
||||
if (!response.tool_calls || response.tool_calls.length === 0) {
|
||||
return { response, usedTools: [], sourceDocuments: [], artifacts: [], totalTokens }
|
||||
const acc = (response.additional_kwargs?.reasoning_content as string) || undefined
|
||||
const dur =
|
||||
typeof response.additional_kwargs?.reasoning_duration === 'number'
|
||||
? response.additional_kwargs.reasoning_duration
|
||||
: undefined
|
||||
return {
|
||||
response,
|
||||
usedTools: [],
|
||||
sourceDocuments: [],
|
||||
artifacts: [],
|
||||
totalTokens,
|
||||
accumulatedReasonContent: acc,
|
||||
accumulatedReasoningDuration: dur
|
||||
}
|
||||
}
|
||||
|
||||
// Stream tool calls if available
|
||||
@@ -2526,12 +2764,19 @@ class Agent_Agentflow implements INode {
|
||||
sseStreamer.streamTokenEvent(chatId, lastToolOutputString)
|
||||
}
|
||||
|
||||
const acc = (response.additional_kwargs?.reasoning_content as string) || undefined
|
||||
const dur =
|
||||
typeof response.additional_kwargs?.reasoning_duration === 'number'
|
||||
? response.additional_kwargs.reasoning_duration
|
||||
: undefined
|
||||
return {
|
||||
response: new AIMessageChunk(lastToolOutputString),
|
||||
usedTools,
|
||||
sourceDocuments,
|
||||
artifacts,
|
||||
totalTokens
|
||||
totalTokens,
|
||||
accumulatedReasonContent: acc,
|
||||
accumulatedReasoningDuration: dur
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2559,18 +2804,15 @@ class Agent_Agentflow implements INode {
|
||||
messages,
|
||||
chatId,
|
||||
abortController,
|
||||
isStructuredOutput
|
||||
isStructuredOutput,
|
||||
isLastNode
|
||||
)
|
||||
} else {
|
||||
newResponse = await llmNodeInstance.invoke(messages, { signal: abortController?.signal })
|
||||
|
||||
// Stream non-streaming response if this is the last node
|
||||
if (isLastNode && sseStreamer && !isStructuredOutput) {
|
||||
let responseContent = JSON.stringify(newResponse, null, 2)
|
||||
if (typeof newResponse.content === 'string') {
|
||||
responseContent = newResponse.content
|
||||
}
|
||||
sseStreamer.streamTokenEvent(chatId, responseContent)
|
||||
sseStreamer.streamTokenEvent(chatId, extractResponseContent(newResponse))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2579,6 +2821,16 @@ class Agent_Agentflow implements INode {
|
||||
totalTokens += newResponse.usage_metadata.total_tokens
|
||||
}
|
||||
|
||||
// Accumulate reasoning and duration from checkpoint response and this turn
|
||||
let accumulatedReasonContent = (response.additional_kwargs?.reasoning_content as string) || ''
|
||||
if (newResponse.additional_kwargs?.reasoning_content) {
|
||||
accumulatedReasonContent +=
|
||||
(accumulatedReasonContent ? '\n\n' : '') + (newResponse.additional_kwargs.reasoning_content as string)
|
||||
}
|
||||
let accumulatedReasoningDuration =
|
||||
(typeof response.additional_kwargs?.reasoning_duration === 'number' ? response.additional_kwargs.reasoning_duration : 0) +
|
||||
(typeof newResponse.additional_kwargs?.reasoning_duration === 'number' ? newResponse.additional_kwargs.reasoning_duration : 0)
|
||||
|
||||
// Check for recursive tool calls and handle them
|
||||
if (newResponse.tool_calls && newResponse.tool_calls.length > 0) {
|
||||
const {
|
||||
@@ -2587,7 +2839,9 @@ class Agent_Agentflow implements INode {
|
||||
sourceDocuments: recursiveSourceDocuments,
|
||||
artifacts: recursiveArtifacts,
|
||||
totalTokens: recursiveTokens,
|
||||
isWaitingForHumanInput: recursiveIsWaitingForHumanInput
|
||||
isWaitingForHumanInput: recursiveIsWaitingForHumanInput,
|
||||
accumulatedReasonContent: recursiveAccumulatedReasonContent,
|
||||
accumulatedReasoningDuration: recursiveAccumulatedReasoningDuration
|
||||
} = await this.handleToolCalls({
|
||||
response: newResponse,
|
||||
messages,
|
||||
@@ -2601,7 +2855,9 @@ class Agent_Agentflow implements INode {
|
||||
isStreamable,
|
||||
isLastNode,
|
||||
iterationContext,
|
||||
isStructuredOutput
|
||||
isStructuredOutput,
|
||||
accumulatedReasonContent,
|
||||
accumulatedReasoningDuration
|
||||
})
|
||||
|
||||
// Merge results from recursive tool calls
|
||||
@@ -2611,9 +2867,24 @@ class Agent_Agentflow implements INode {
|
||||
artifacts = [...artifacts, ...recursiveArtifacts]
|
||||
totalTokens += recursiveTokens
|
||||
isWaitingForHumanInput = recursiveIsWaitingForHumanInput
|
||||
if (recursiveAccumulatedReasonContent !== undefined) {
|
||||
accumulatedReasonContent = recursiveAccumulatedReasonContent
|
||||
}
|
||||
if (recursiveAccumulatedReasoningDuration !== undefined) {
|
||||
accumulatedReasoningDuration = recursiveAccumulatedReasoningDuration
|
||||
}
|
||||
}
|
||||
|
||||
return { response: newResponse, usedTools, sourceDocuments, artifacts, totalTokens, isWaitingForHumanInput }
|
||||
return {
|
||||
response: newResponse,
|
||||
usedTools,
|
||||
sourceDocuments,
|
||||
artifacts,
|
||||
totalTokens,
|
||||
isWaitingForHumanInput,
|
||||
accumulatedReasonContent: accumulatedReasonContent || undefined,
|
||||
accumulatedReasoningDuration: accumulatedReasoningDuration || undefined
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -2632,8 +2903,8 @@ class Agent_Agentflow implements INode {
|
||||
const filePath = match[2]
|
||||
|
||||
try {
|
||||
// Extract filename from the file path
|
||||
const fileName = filePath.split('/').pop() || filePath
|
||||
// Extract and sanitize filename from the file path (LLM-generated, untrusted)
|
||||
const fileName = sanitizeFileName(filePath)
|
||||
|
||||
// Replace sandbox link with proper download URL
|
||||
const downloadUrl = `${baseURL}/api/v1/get-upload-file?chatflowId=${chatflowId}&chatId=${chatId}&fileName=${fileName}&download=true`
|
||||
|
||||
@@ -1,15 +1,13 @@
|
||||
import { AnalyticHandler } from '../../../src/handler'
|
||||
import { ICommonObject, INode, INodeData, INodeOptionsValue, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { ICommonObject, IMessage, INode, INodeData, INodeOptionsValue, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { AIMessageChunk, BaseMessageLike } from '@langchain/core/messages'
|
||||
import {
|
||||
getPastChatHistoryImageMessages,
|
||||
getUniqueImageMessages,
|
||||
processMessagesWithImages,
|
||||
replaceBase64ImagesWithFileReferences
|
||||
} from '../utils'
|
||||
import { getPastChatHistoryImageMessages, getUniqueImageMessages, processMessagesWithImages, revertBase64ImagesToFileRefs } from '../utils'
|
||||
import { CONDITION_AGENT_SYSTEM_PROMPT, DEFAULT_SUMMARIZER_TEMPLATE } from '../prompt'
|
||||
import { BaseChatModel } from '@langchain/core/language_models/chat_models'
|
||||
import { findBestScenarioIndex } from './matchScenario'
|
||||
import { extractResponseContent } from '../../../src/utils'
|
||||
import { getModelConfigByModelName, MODEL_TYPE } from '../../../src/modelLoader'
|
||||
import { NodeHtmlMarkdown } from 'node-html-markdown'
|
||||
|
||||
class ConditionAgent_Agentflow implements INode {
|
||||
label: string
|
||||
@@ -28,7 +26,7 @@ class ConditionAgent_Agentflow implements INode {
|
||||
constructor() {
|
||||
this.label = 'Condition Agent'
|
||||
this.name = 'conditionAgentAgentflow'
|
||||
this.version = 1.1
|
||||
this.version = 2.0
|
||||
this.type = 'ConditionAgent'
|
||||
this.category = 'Agent Flows'
|
||||
this.description = `Utilize an agent to split flows based on dynamic conditions`
|
||||
@@ -84,26 +82,6 @@ class ConditionAgent_Agentflow implements INode {
|
||||
]
|
||||
},
|
||||
{
|
||||
label: 'Override System Prompt',
|
||||
name: 'conditionAgentOverrideSystemPrompt',
|
||||
type: 'boolean',
|
||||
description: 'Override initial system prompt for Condition Agent',
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Node System Prompt',
|
||||
name: 'conditionAgentSystemPrompt',
|
||||
type: 'string',
|
||||
rows: 4,
|
||||
optional: true,
|
||||
acceptVariable: true,
|
||||
default: CONDITION_AGENT_SYSTEM_PROMPT,
|
||||
description: 'Expert use only. Modifying this can significantly alter agent behavior. Leave default if unsure',
|
||||
show: {
|
||||
conditionAgentOverrideSystemPrompt: true
|
||||
}
|
||||
}
|
||||
/*{
|
||||
label: 'Enable Memory',
|
||||
name: 'conditionAgentEnableMemory',
|
||||
type: 'boolean',
|
||||
@@ -162,7 +140,27 @@ class ConditionAgent_Agentflow implements INode {
|
||||
show: {
|
||||
conditionAgentMemoryType: 'conversationSummaryBuffer'
|
||||
}
|
||||
}*/
|
||||
},
|
||||
{
|
||||
label: 'Override System Prompt',
|
||||
name: 'conditionAgentOverrideSystemPrompt',
|
||||
type: 'boolean',
|
||||
description: 'Override initial system prompt for Condition Agent',
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Condition Agent System Prompt',
|
||||
name: 'conditionAgentSystemPrompt',
|
||||
type: 'string',
|
||||
rows: 4,
|
||||
optional: true,
|
||||
acceptVariable: true,
|
||||
default: CONDITION_AGENT_SYSTEM_PROMPT,
|
||||
description: 'Expert use only. Modifying this can significantly alter agent behavior. Leave default if unsure',
|
||||
show: {
|
||||
conditionAgentOverrideSystemPrompt: true
|
||||
}
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
@@ -268,7 +266,7 @@ class ConditionAgent_Agentflow implements INode {
|
||||
const conditionAgentInstructions = nodeData.inputs?.conditionAgentInstructions as string
|
||||
const conditionAgentSystemPrompt = nodeData.inputs?.conditionAgentSystemPrompt as string
|
||||
const conditionAgentOverrideSystemPrompt = nodeData.inputs?.conditionAgentOverrideSystemPrompt as boolean
|
||||
let systemPrompt = CONDITION_AGENT_SYSTEM_PROMPT
|
||||
let systemPrompt = NodeHtmlMarkdown.translate(CONDITION_AGENT_SYSTEM_PROMPT)
|
||||
if (conditionAgentSystemPrompt && conditionAgentOverrideSystemPrompt) {
|
||||
systemPrompt = conditionAgentSystemPrompt
|
||||
}
|
||||
@@ -282,6 +280,7 @@ class ConditionAgent_Agentflow implements INode {
|
||||
const state = options.agentflowRuntime?.state as ICommonObject
|
||||
const pastChatHistory = (options.pastChatHistory as BaseMessageLike[]) ?? []
|
||||
const runtimeChatHistory = (options.agentflowRuntime?.chatHistory as BaseMessageLike[]) ?? []
|
||||
const prependedChatHistory = options.prependedChatHistory as IMessage[]
|
||||
|
||||
// Initialize the LLM model instance
|
||||
const nodeInstanceFilePath = options.componentNodes[model].filePath as string
|
||||
@@ -303,8 +302,8 @@ class ConditionAgent_Agentflow implements INode {
|
||||
throw new Error('Scenarios are required')
|
||||
}
|
||||
|
||||
// Prepare messages array
|
||||
const messages: BaseMessageLike[] = [
|
||||
// Prepare prefix messages (system prompt + few-shot examples) - needed for model invocation only
|
||||
const prefixMessages: BaseMessageLike[] = [
|
||||
{
|
||||
role: 'system',
|
||||
content: systemPrompt
|
||||
@@ -318,10 +317,24 @@ class ConditionAgent_Agentflow implements INode {
|
||||
content: `\`\`\`json\n{"output": "user is not asking about AI"}\n\`\`\``
|
||||
}
|
||||
]
|
||||
// Use to store messages with image file references as we do not want to store the base64 data into database
|
||||
let runtimeImageMessagesWithFileRef: BaseMessageLike[] = []
|
||||
// Use to keep track of past messages with image file references
|
||||
let pastImageMessagesWithFileRef: BaseMessageLike[] = []
|
||||
|
||||
// Prepare messages array (these get stored in output)
|
||||
const messages: BaseMessageLike[] = []
|
||||
|
||||
// Prepend history ONLY if it is the first node
|
||||
if (prependedChatHistory.length > 0 && !runtimeChatHistory.length) {
|
||||
for (const msg of prependedChatHistory) {
|
||||
const role: string = msg.role === 'apiMessage' ? 'assistant' : 'user'
|
||||
const content: string = msg.content ?? ''
|
||||
messages.push({
|
||||
role,
|
||||
content
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const scenariosList = _conditionAgentScenarios.map((scenario, index) => `${index + 1}. ${scenario.scenario}`).join('\n')
|
||||
const prettyInput = `### Input\n${input}\n\n### Scenarios\n${scenariosList}\n\n### Instruction\n${conditionAgentInstructions}`
|
||||
|
||||
input = `{"input": ${input}, "scenarios": ${JSON.stringify(
|
||||
_conditionAgentScenarios.map((scenario) => scenario.scenario)
|
||||
@@ -339,9 +352,7 @@ class ConditionAgent_Agentflow implements INode {
|
||||
input,
|
||||
abortController,
|
||||
options,
|
||||
modelConfig,
|
||||
runtimeImageMessagesWithFileRef,
|
||||
pastImageMessagesWithFileRef
|
||||
modelConfig
|
||||
})
|
||||
} else {
|
||||
/*
|
||||
@@ -351,9 +362,7 @@ class ConditionAgent_Agentflow implements INode {
|
||||
if (!runtimeChatHistory.length && options.uploads) {
|
||||
const imageContents = await getUniqueImageMessages(options, messages, modelConfig)
|
||||
if (imageContents) {
|
||||
const { imageMessageWithBase64, imageMessageWithFileRef } = imageContents
|
||||
messages.push(imageMessageWithBase64)
|
||||
runtimeImageMessagesWithFileRef.push(imageMessageWithFileRef)
|
||||
messages.push(imageContents.imageMessageWithBase64)
|
||||
}
|
||||
}
|
||||
messages.push({
|
||||
@@ -365,16 +374,19 @@ class ConditionAgent_Agentflow implements INode {
|
||||
// Initialize response and determine if streaming is possible
|
||||
let response: AIMessageChunk = new AIMessageChunk('')
|
||||
|
||||
// Combine prefix messages with regular messages for model invocation
|
||||
const allMessages = [...prefixMessages, ...messages]
|
||||
|
||||
// Start analytics
|
||||
if (analyticHandlers && options.parentTraceIds) {
|
||||
const llmLabel = options?.componentNodes?.[model]?.label || model
|
||||
llmIds = await analyticHandlers.onLLMStart(llmLabel, messages, options.parentTraceIds)
|
||||
llmIds = await analyticHandlers.onLLMStart(llmLabel, allMessages, options.parentTraceIds)
|
||||
}
|
||||
|
||||
// Track execution time
|
||||
const startTime = Date.now()
|
||||
|
||||
response = await llmNodeInstance.invoke(messages, { signal: abortController?.signal })
|
||||
response = await llmNodeInstance.invoke(allMessages, { signal: abortController?.signal })
|
||||
|
||||
// Calculate execution time
|
||||
const endTime = Date.now()
|
||||
@@ -383,7 +395,7 @@ class ConditionAgent_Agentflow implements INode {
|
||||
// End analytics tracking (pass structured output with usage metadata)
|
||||
if (analyticHandlers && llmIds) {
|
||||
const analyticsOutput: any = {
|
||||
content: typeof response.content === 'string' ? response.content : JSON.stringify(response.content)
|
||||
content: extractResponseContent(response)
|
||||
}
|
||||
// Include usage metadata if available
|
||||
if (response.usage_metadata) {
|
||||
@@ -427,37 +439,68 @@ class ConditionAgent_Agentflow implements INode {
|
||||
}
|
||||
})
|
||||
|
||||
// Replace the actual messages array with one that includes the file references for images instead of base64 data
|
||||
const messagesWithFileReferences = replaceBase64ImagesWithFileReferences(
|
||||
messages,
|
||||
runtimeImageMessagesWithFileRef,
|
||||
pastImageMessagesWithFileRef
|
||||
)
|
||||
// Revert all tagged base64 image_url items back to stored-file format
|
||||
const messagesWithFileReferences = revertBase64ImagesToFileRefs(messages)
|
||||
|
||||
// Replace the user input with prettified input for display purposes
|
||||
for (let i = messagesWithFileReferences.length - 1; i >= 0; i--) {
|
||||
const msg = messagesWithFileReferences[i] as any
|
||||
if (msg.role === 'user' && msg.content === input) {
|
||||
msg.content = prettyInput
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Only add to runtime chat history if this is the first node
|
||||
const inputMessages = []
|
||||
if (!runtimeChatHistory.length) {
|
||||
if (runtimeImageMessagesWithFileRef.length) {
|
||||
inputMessages.push(...runtimeImageMessagesWithFileRef)
|
||||
const imageInputMessages = messagesWithFileReferences.filter(
|
||||
(msg: any) =>
|
||||
msg.role === 'user' &&
|
||||
Array.isArray(msg.content) &&
|
||||
msg.content.some((item: any) => item.type === 'stored-file' && item.mime?.startsWith('image/'))
|
||||
)
|
||||
if (imageInputMessages.length) {
|
||||
inputMessages.push(...imageInputMessages)
|
||||
}
|
||||
if (input && typeof input === 'string') {
|
||||
inputMessages.push({ role: 'user', content: question })
|
||||
}
|
||||
}
|
||||
|
||||
const costMetadata = await this.calculateUsageCost(model, modelConfig?.modelName as string | undefined, response.usage_metadata)
|
||||
|
||||
const output: any = {
|
||||
conditions,
|
||||
content: extractResponseContent(response),
|
||||
timeMetadata: {
|
||||
start: startTime,
|
||||
end: endTime,
|
||||
delta: timeDelta
|
||||
}
|
||||
}
|
||||
|
||||
if (response.usage_metadata) {
|
||||
output.usageMetadata = { ...response.usage_metadata }
|
||||
}
|
||||
|
||||
if (costMetadata && output.usageMetadata) {
|
||||
output.usageMetadata.input_cost = costMetadata.input_cost
|
||||
output.usageMetadata.output_cost = costMetadata.output_cost
|
||||
output.usageMetadata.total_cost = costMetadata.total_cost
|
||||
output.usageMetadata.base_input_cost = costMetadata.base_input_cost
|
||||
output.usageMetadata.base_output_cost = costMetadata.base_output_cost
|
||||
}
|
||||
|
||||
if (response.response_metadata) {
|
||||
output.responseMetadata = response.response_metadata
|
||||
}
|
||||
|
||||
const returnOutput = {
|
||||
id: nodeData.id,
|
||||
name: this.name,
|
||||
input: { messages: messagesWithFileReferences },
|
||||
output: {
|
||||
conditions,
|
||||
content: typeof response.content === 'string' ? response.content : JSON.stringify(response.content),
|
||||
timeMetadata: {
|
||||
start: startTime,
|
||||
end: endTime,
|
||||
delta: timeDelta
|
||||
}
|
||||
},
|
||||
output,
|
||||
state,
|
||||
chatHistory: [...inputMessages]
|
||||
}
|
||||
@@ -488,9 +531,7 @@ class ConditionAgent_Agentflow implements INode {
|
||||
input,
|
||||
abortController,
|
||||
options,
|
||||
modelConfig,
|
||||
runtimeImageMessagesWithFileRef,
|
||||
pastImageMessagesWithFileRef
|
||||
modelConfig
|
||||
}: {
|
||||
messages: BaseMessageLike[]
|
||||
memoryType: string
|
||||
@@ -502,12 +543,9 @@ class ConditionAgent_Agentflow implements INode {
|
||||
abortController: AbortController
|
||||
options: ICommonObject
|
||||
modelConfig: ICommonObject
|
||||
runtimeImageMessagesWithFileRef: BaseMessageLike[]
|
||||
pastImageMessagesWithFileRef: BaseMessageLike[]
|
||||
}): Promise<void> {
|
||||
const { updatedPastMessages, transformedPastMessages } = await getPastChatHistoryImageMessages(pastChatHistory, options)
|
||||
const { updatedPastMessages } = await getPastChatHistoryImageMessages(pastChatHistory, options)
|
||||
pastChatHistory = updatedPastMessages
|
||||
pastImageMessagesWithFileRef.push(...transformedPastMessages)
|
||||
|
||||
let pastMessages = [...pastChatHistory, ...runtimeChatHistory]
|
||||
if (!runtimeChatHistory.length) {
|
||||
@@ -518,15 +556,12 @@ class ConditionAgent_Agentflow implements INode {
|
||||
if (options.uploads) {
|
||||
const imageContents = await getUniqueImageMessages(options, messages, modelConfig)
|
||||
if (imageContents) {
|
||||
const { imageMessageWithBase64, imageMessageWithFileRef } = imageContents
|
||||
pastMessages.push(imageMessageWithBase64)
|
||||
runtimeImageMessagesWithFileRef.push(imageMessageWithFileRef)
|
||||
pastMessages.push(imageContents.imageMessageWithBase64)
|
||||
}
|
||||
}
|
||||
}
|
||||
const { updatedMessages, transformedMessages } = await processMessagesWithImages(pastMessages, options)
|
||||
const { updatedMessages } = await processMessagesWithImages(pastMessages, options)
|
||||
pastMessages = updatedMessages
|
||||
pastImageMessagesWithFileRef.push(...transformedMessages)
|
||||
|
||||
if (pastMessages.length > 0) {
|
||||
if (memoryType === 'windowSize') {
|
||||
@@ -548,7 +583,7 @@ class ConditionAgent_Agentflow implements INode {
|
||||
],
|
||||
{ signal: abortController?.signal }
|
||||
)
|
||||
messages.push({ role: 'assistant', content: summary.content as string })
|
||||
messages.push({ role: 'assistant', content: extractResponseContent(summary) })
|
||||
} else if (memoryType === 'conversationSummaryBuffer') {
|
||||
// Summary buffer: Summarize messages that exceed token limit
|
||||
await this.handleSummaryBuffer(messages, pastMessages, llmNodeInstance, nodeData, abortController)
|
||||
@@ -611,13 +646,58 @@ class ConditionAgent_Agentflow implements INode {
|
||||
)
|
||||
|
||||
// Add summary as a system message at the beginning, then add remaining messages
|
||||
messages.push({ role: 'system', content: `Previous conversation summary: ${summary.content}` })
|
||||
let summaryRole = 'system'
|
||||
if (messages.some((msg) => typeof msg === 'object' && !Array.isArray(msg) && 'role' in msg && msg.role === 'system')) {
|
||||
summaryRole = 'user' // some model doesn't allow multiple system messages
|
||||
}
|
||||
messages.push({ role: summaryRole, content: `Previous conversation summary: ${extractResponseContent(summary)}` })
|
||||
messages.push(...remainingMessages)
|
||||
} else {
|
||||
// If under token limit, use all messages
|
||||
messages.push(...pastMessages)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates input/output and total cost from usage metadata using model pricing from models.json.
|
||||
*/
|
||||
private async calculateUsageCost(
|
||||
provider: string | undefined,
|
||||
modelName: string | undefined,
|
||||
usageMetadata: Record<string, any> | undefined
|
||||
): Promise<
|
||||
| {
|
||||
input_cost: number
|
||||
output_cost: number
|
||||
total_cost: number
|
||||
base_input_cost: number
|
||||
base_output_cost: number
|
||||
}
|
||||
| undefined
|
||||
> {
|
||||
if (!provider || !modelName) return undefined
|
||||
const inputTokens = (usageMetadata?.input_tokens ?? 0) as number
|
||||
const outputTokens = (usageMetadata?.output_tokens ?? 0) as number
|
||||
try {
|
||||
const modelConfig = await getModelConfigByModelName(MODEL_TYPE.CHAT, provider, modelName)
|
||||
if (!modelConfig) return undefined
|
||||
const baseInputCost = Number(modelConfig.input_cost) || 0
|
||||
const baseOutputCost = Number(modelConfig.output_cost) || 0
|
||||
const inputCost = inputTokens * baseInputCost
|
||||
const outputCost = outputTokens * baseOutputCost
|
||||
const totalCost = inputCost + outputCost
|
||||
if (inputCost === 0 && outputCost === 0) return undefined
|
||||
return {
|
||||
input_cost: inputCost,
|
||||
output_cost: outputCost,
|
||||
total_cost: totalCost,
|
||||
base_input_cost: baseInputCost,
|
||||
base_output_cost: baseOutputCost
|
||||
}
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: ConditionAgent_Agentflow }
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
} from '../../../src/Interface'
|
||||
import { AIMessageChunk, BaseMessageLike } from '@langchain/core/messages'
|
||||
import { DEFAULT_HUMAN_INPUT_DESCRIPTION, DEFAULT_HUMAN_INPUT_DESCRIPTION_HTML } from '../prompt'
|
||||
import { extractResponseContent } from '../../../src/utils'
|
||||
|
||||
class HumanInput_Agentflow implements INode {
|
||||
label: string
|
||||
@@ -250,7 +251,7 @@ class HumanInput_Agentflow implements INode {
|
||||
humanInputDescription = response.content as string
|
||||
} else {
|
||||
const response = await llmNodeInstance.invoke(messages)
|
||||
humanInputDescription = response.content as string
|
||||
humanInputDescription = extractResponseContent(response)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { MessageContent } from '@langchain/core/messages'
|
||||
import { IFileUpload } from '../../src/Interface'
|
||||
|
||||
export interface ILLMMessage {
|
||||
role: 'system' | 'assistant' | 'user' | 'tool' | 'developer'
|
||||
content: string
|
||||
@@ -15,3 +18,135 @@ export interface IFlowState {
|
||||
key: string
|
||||
value: string
|
||||
}
|
||||
|
||||
// ─── Image & artifact interfaces ─────────────────────────────────────────────
|
||||
|
||||
/** Tracks which content-array indices map back to stored files. */
|
||||
export interface IImageFileRef {
|
||||
index: number
|
||||
fileName: string
|
||||
mime: string
|
||||
}
|
||||
|
||||
/** An artifact produced by a model (image, file, etc.). */
|
||||
export interface IArtifact {
|
||||
type: string
|
||||
data: string
|
||||
}
|
||||
|
||||
/** A file annotation from container file citations. */
|
||||
export interface IFileAnnotation {
|
||||
filePath: string
|
||||
fileName: string
|
||||
}
|
||||
|
||||
/** Result of saving an image to storage. */
|
||||
export interface ISavedImageResult {
|
||||
filePath: string
|
||||
fileName: string
|
||||
totalSize: number
|
||||
}
|
||||
|
||||
/** Info about a saved inline image (for replacing base64 in content). */
|
||||
export interface ISavedInlineImage {
|
||||
filePath: string
|
||||
fileName: string
|
||||
mimeType: string
|
||||
}
|
||||
|
||||
// ─── Response metadata interfaces (LLM provider outputs) ─────────────────────
|
||||
|
||||
/** OpenAI image generation output item. */
|
||||
export interface IImageGenerationOutput {
|
||||
type: 'image_generation_call'
|
||||
id?: string
|
||||
result: string
|
||||
output_format?: string
|
||||
}
|
||||
|
||||
/** Gemini inline data item (image generation). */
|
||||
export interface IGeminiInlineData {
|
||||
type: 'gemini_inline_data'
|
||||
data: string
|
||||
mimeType: string
|
||||
}
|
||||
|
||||
/** OpenAI container file citation annotation. */
|
||||
export interface IContainerFileCitation {
|
||||
type: 'container_file_citation'
|
||||
container_id: string
|
||||
file_id: string
|
||||
filename: string
|
||||
}
|
||||
|
||||
/** A single content item within a response metadata output. */
|
||||
export interface IResponseOutputContent {
|
||||
annotations?: IContainerFileCitation[]
|
||||
}
|
||||
|
||||
/** A single output item from response metadata. */
|
||||
export interface IResponseMetadataOutput {
|
||||
type: string
|
||||
content?: IResponseOutputContent[]
|
||||
result?: string
|
||||
id?: string
|
||||
output_format?: string
|
||||
}
|
||||
|
||||
/** Top-level response metadata shape from LLM providers. */
|
||||
export interface IResponseMetadata {
|
||||
output?: IResponseMetadataOutput[]
|
||||
inlineData?: IGeminiInlineData[]
|
||||
}
|
||||
|
||||
// ─── Message interfaces ──────────────────────────────────────────────────────
|
||||
|
||||
/** Stored-file content item within a message content array. */
|
||||
export interface IStoredFileContent {
|
||||
type: 'stored-file'
|
||||
name: string
|
||||
mime: string
|
||||
path?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Additional kwargs attached to messages in agentflow.
|
||||
* Extends LangChain's `Record<string, unknown>` with known keys.
|
||||
*/
|
||||
export interface IMessageAdditionalKwargs {
|
||||
artifacts?: IArtifact[]
|
||||
fileAnnotations?: IFileAnnotation[]
|
||||
usedTools?: unknown[]
|
||||
fileUploads?: string | IFileUpload[]
|
||||
_imageFileRefs?: IImageFileRef[]
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
/**
|
||||
* Plain chat message as used in agentflow utils.
|
||||
* Compatible with LangChain's `MessageFieldWithRole` (which is `{role, content, name?} & Record<string, unknown>`).
|
||||
* The index signature ensures assignability to `BaseMessageLike`.
|
||||
*/
|
||||
export interface IChatMessage {
|
||||
role: string
|
||||
content: MessageContent
|
||||
additional_kwargs?: IMessageAdditionalKwargs
|
||||
_isTemporaryImageMessage?: boolean
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
/** Image artifact extracted from an assistant message. */
|
||||
export interface IImageArtifact {
|
||||
name: string
|
||||
mime: string
|
||||
}
|
||||
|
||||
/** Content item in a multimodal message (superset of stored-file and image_url). */
|
||||
export interface IMultimodalContentItem {
|
||||
type: string
|
||||
name?: string
|
||||
mime?: string
|
||||
data?: string
|
||||
image_url?: { url: string; detail?: string }
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
@@ -1,20 +1,22 @@
|
||||
import { BaseChatModel } from '@langchain/core/language_models/chat_models'
|
||||
import { ICommonObject, IMessage, INode, INodeData, INodeOptionsValue, INodeParams, IServerSideEventStreamer } from '../../../src/Interface'
|
||||
import { AIMessageChunk, BaseMessageLike, MessageContentText } from '@langchain/core/messages'
|
||||
import { ContentBlock } from 'langchain'
|
||||
import { AIMessageChunk, BaseMessageLike } from '@langchain/core/messages'
|
||||
import { DEFAULT_SUMMARIZER_TEMPLATE } from '../prompt'
|
||||
import { AnalyticHandler } from '../../../src/handler'
|
||||
import { ILLMMessage } from '../Interface.Agentflow'
|
||||
import { ILLMMessage, IResponseMetadata } from '../Interface.Agentflow'
|
||||
import {
|
||||
addImageArtifactsToMessages,
|
||||
extractArtifactsFromResponse,
|
||||
getPastChatHistoryImageMessages,
|
||||
getUniqueImageMessages,
|
||||
processMessagesWithImages,
|
||||
replaceBase64ImagesWithFileReferences,
|
||||
revertBase64ImagesToFileRefs,
|
||||
replaceInlineDataWithFileReferences,
|
||||
updateFlowState
|
||||
} from '../utils'
|
||||
import { processTemplateVariables, configureStructuredOutput } from '../../../src/utils'
|
||||
import { processTemplateVariables, configureStructuredOutput, extractResponseContent } from '../../../src/utils'
|
||||
import { getModelConfigByModelName, MODEL_TYPE } from '../../../src/modelLoader'
|
||||
import { flatten } from 'lodash'
|
||||
|
||||
class LLM_Agentflow implements INode {
|
||||
@@ -381,10 +383,6 @@ class LLM_Agentflow implements INode {
|
||||
|
||||
// Prepare messages array
|
||||
const messages: BaseMessageLike[] = []
|
||||
// Use to store messages with image file references as we do not want to store the base64 data into database
|
||||
let runtimeImageMessagesWithFileRef: BaseMessageLike[] = []
|
||||
// Use to keep track of past messages with image file references
|
||||
let pastImageMessagesWithFileRef: BaseMessageLike[] = []
|
||||
|
||||
// Prepend history ONLY if it is the first node
|
||||
if (prependedChatHistory.length > 0 && !runtimeChatHistory.length) {
|
||||
@@ -423,9 +421,7 @@ class LLM_Agentflow implements INode {
|
||||
input,
|
||||
abortController,
|
||||
options,
|
||||
modelConfig,
|
||||
runtimeImageMessagesWithFileRef,
|
||||
pastImageMessagesWithFileRef
|
||||
modelConfig
|
||||
})
|
||||
} else if (!runtimeChatHistory.length) {
|
||||
/*
|
||||
@@ -436,9 +432,7 @@ class LLM_Agentflow implements INode {
|
||||
if (options.uploads) {
|
||||
const imageContents = await getUniqueImageMessages(options, messages, modelConfig)
|
||||
if (imageContents) {
|
||||
const { imageMessageWithBase64, imageMessageWithFileRef } = imageContents
|
||||
messages.push(imageMessageWithBase64)
|
||||
runtimeImageMessagesWithFileRef.push(imageMessageWithFileRef)
|
||||
messages.push(imageContents.imageMessageWithBase64)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -452,8 +446,8 @@ class LLM_Agentflow implements INode {
|
||||
delete nodeData.inputs?.llmMessages
|
||||
|
||||
/**
|
||||
* Add image artifacts from previous assistant responses as user messages
|
||||
* Images are converted from FILE-STORAGE::<image_path> to base 64 image_url format
|
||||
* Add image artifacts from previous assistant responses as user messages.
|
||||
* Only the inserted temporary messages contain base64 — other messages are untouched.
|
||||
*/
|
||||
await addImageArtifactsToMessages(messages, options)
|
||||
|
||||
@@ -487,21 +481,22 @@ class LLM_Agentflow implements INode {
|
||||
* Invoke LLM
|
||||
*/
|
||||
if (isStreamable) {
|
||||
response = await this.handleStreamingResponse(sseStreamer, llmNodeInstance, messages, chatId, abortController)
|
||||
response = await this.handleStreamingResponse(
|
||||
sseStreamer,
|
||||
llmNodeInstance,
|
||||
messages,
|
||||
chatId,
|
||||
abortController,
|
||||
isStructuredOutput,
|
||||
isLastNode
|
||||
)
|
||||
} else {
|
||||
response = await llmNodeInstance.invoke(messages, { signal: abortController?.signal })
|
||||
|
||||
// Stream whole response back to UI if this is the last node
|
||||
if (isLastNode && options.sseStreamer) {
|
||||
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
|
||||
let finalResponse = ''
|
||||
if (response.content && Array.isArray(response.content)) {
|
||||
finalResponse = response.content.map((item: any) => item.text).join('\n')
|
||||
} else if (response.content && typeof response.content === 'string') {
|
||||
finalResponse = response.content
|
||||
} else {
|
||||
finalResponse = JSON.stringify(response, null, 2)
|
||||
}
|
||||
const finalResponse = extractResponseContent(response)
|
||||
sseStreamer.streamTokenEvent(chatId, finalResponse)
|
||||
}
|
||||
}
|
||||
@@ -518,7 +513,7 @@ class LLM_Agentflow implements INode {
|
||||
artifacts: extractedArtifacts,
|
||||
fileAnnotations: extractedFileAnnotations,
|
||||
savedInlineImages
|
||||
} = await extractArtifactsFromResponse(response.response_metadata, newNodeData, options)
|
||||
} = await extractArtifactsFromResponse(response.response_metadata as IResponseMetadata, newNodeData, options)
|
||||
|
||||
if (extractedArtifacts.length > 0) {
|
||||
artifacts = extractedArtifacts
|
||||
@@ -557,18 +552,36 @@ class LLM_Agentflow implements INode {
|
||||
}
|
||||
}
|
||||
|
||||
// Prepare final response and output object
|
||||
let finalResponse = ''
|
||||
if (response.content && Array.isArray(response.content)) {
|
||||
finalResponse = response.content.map((item: any) => item.text).join('\n')
|
||||
} else if (response.content && typeof response.content === 'string') {
|
||||
finalResponse = response.content
|
||||
} else if (response.content === '') {
|
||||
// Empty response content, this could happen when there is only image data
|
||||
finalResponse = ''
|
||||
} else {
|
||||
finalResponse = JSON.stringify(response, null, 2)
|
||||
// Extract reason content from response (reasoning_content/reasoning_duration or contentBlocks)
|
||||
let reasonContent = (response.additional_kwargs?.reasoning_content as string) || ''
|
||||
let thinkingDuration: number | undefined =
|
||||
typeof response.additional_kwargs?.reasoning_duration === 'number'
|
||||
? response.additional_kwargs.reasoning_duration
|
||||
: undefined
|
||||
if (!reasonContent && response.contentBlocks?.length && isLastNode && sseStreamer && !isStructuredOutput) {
|
||||
for (const block of response.contentBlocks) {
|
||||
if (block.type === 'reasoning' && (block as { reasoning?: string }).reasoning) {
|
||||
reasonContent += (block as { reasoning: string }).reasoning
|
||||
}
|
||||
if ((block as any).type === 'thinking' && (block as any).thinking) {
|
||||
reasonContent += (block as any).thinking
|
||||
}
|
||||
}
|
||||
if (reasonContent) {
|
||||
sseStreamer.streamThinkingEvent(chatId, reasonContent)
|
||||
const reasoningTokens = response.usage_metadata?.output_token_details?.reasoning || 0
|
||||
thinkingDuration = reasoningTokens > 0 ? Math.round(reasoningTokens / 50) : 2
|
||||
sseStreamer.streamThinkingEvent(chatId, '', thinkingDuration)
|
||||
}
|
||||
}
|
||||
const reasonContentObj =
|
||||
reasonContent !== undefined && reasonContent !== '' ? { thinking: reasonContent, thinkingDuration } : undefined
|
||||
|
||||
// Prepare final response and output object
|
||||
const finalResponse = extractResponseContent(response)
|
||||
|
||||
const costMetadata = await this.calculateUsageCost(model, modelConfig?.modelName as string | undefined, response.usage_metadata)
|
||||
|
||||
const output = this.prepareOutputObject(
|
||||
response,
|
||||
finalResponse,
|
||||
@@ -577,7 +590,9 @@ class LLM_Agentflow implements INode {
|
||||
timeDelta,
|
||||
isStructuredOutput,
|
||||
artifacts,
|
||||
fileAnnotations
|
||||
fileAnnotations,
|
||||
reasonContentObj,
|
||||
costMetadata
|
||||
)
|
||||
|
||||
// End analytics tracking
|
||||
@@ -599,23 +614,24 @@ class LLM_Agentflow implements INode {
|
||||
newState = processTemplateVariables(newState, finalResponse)
|
||||
|
||||
/**
|
||||
* Remove the temporarily added image artifact messages before storing
|
||||
* Remove temporary artifact image messages (only needed for model invoke).
|
||||
* Then revert all remaining tagged base64 image_url items back to stored-file format.
|
||||
* This is to avoid storing the actual base64 data into database
|
||||
*/
|
||||
const messagesToStore = messages.filter((msg: any) => !msg._isTemporaryImageMessage)
|
||||
|
||||
// Replace the actual messages array with one that includes the file references for images instead of base64 data
|
||||
const messagesWithFileReferences = replaceBase64ImagesWithFileReferences(
|
||||
messagesToStore,
|
||||
runtimeImageMessagesWithFileRef,
|
||||
pastImageMessagesWithFileRef
|
||||
)
|
||||
const messagesWithFileReferences = revertBase64ImagesToFileRefs(messagesToStore)
|
||||
|
||||
// Only add to runtime chat history if this is the first node
|
||||
const inputMessages = []
|
||||
if (!runtimeChatHistory.length) {
|
||||
if (runtimeImageMessagesWithFileRef.length) {
|
||||
inputMessages.push(...runtimeImageMessagesWithFileRef)
|
||||
const imageInputMessages = messagesWithFileReferences.filter(
|
||||
(msg: any) =>
|
||||
msg.role === 'user' &&
|
||||
Array.isArray(msg.content) &&
|
||||
msg.content.some((item: any) => item.type === 'stored-file' && item.mime?.startsWith('image/'))
|
||||
)
|
||||
if (imageInputMessages.length) {
|
||||
inputMessages.push(...imageInputMessages)
|
||||
}
|
||||
if (input && typeof input === 'string') {
|
||||
if (!enableMemory) {
|
||||
@@ -693,9 +709,7 @@ class LLM_Agentflow implements INode {
|
||||
input,
|
||||
abortController,
|
||||
options,
|
||||
modelConfig,
|
||||
runtimeImageMessagesWithFileRef,
|
||||
pastImageMessagesWithFileRef
|
||||
modelConfig
|
||||
}: {
|
||||
messages: BaseMessageLike[]
|
||||
memoryType: string
|
||||
@@ -708,12 +722,9 @@ class LLM_Agentflow implements INode {
|
||||
abortController: AbortController
|
||||
options: ICommonObject
|
||||
modelConfig: ICommonObject
|
||||
runtimeImageMessagesWithFileRef: BaseMessageLike[]
|
||||
pastImageMessagesWithFileRef: BaseMessageLike[]
|
||||
}): Promise<void> {
|
||||
const { updatedPastMessages, transformedPastMessages } = await getPastChatHistoryImageMessages(pastChatHistory, options)
|
||||
const { updatedPastMessages } = await getPastChatHistoryImageMessages(pastChatHistory, options)
|
||||
pastChatHistory = updatedPastMessages
|
||||
pastImageMessagesWithFileRef.push(...transformedPastMessages)
|
||||
|
||||
let pastMessages = [...pastChatHistory, ...runtimeChatHistory]
|
||||
if (!runtimeChatHistory.length && input && typeof input === 'string') {
|
||||
@@ -725,9 +736,7 @@ class LLM_Agentflow implements INode {
|
||||
if (options.uploads) {
|
||||
const imageContents = await getUniqueImageMessages(options, messages, modelConfig)
|
||||
if (imageContents) {
|
||||
const { imageMessageWithBase64, imageMessageWithFileRef } = imageContents
|
||||
pastMessages.push(imageMessageWithBase64)
|
||||
runtimeImageMessagesWithFileRef.push(imageMessageWithFileRef)
|
||||
pastMessages.push(imageContents.imageMessageWithBase64)
|
||||
}
|
||||
}
|
||||
pastMessages.push({
|
||||
@@ -735,9 +744,8 @@ class LLM_Agentflow implements INode {
|
||||
content: input
|
||||
})
|
||||
}
|
||||
const { updatedMessages, transformedMessages } = await processMessagesWithImages(pastMessages, options)
|
||||
const { updatedMessages } = await processMessagesWithImages(pastMessages, options)
|
||||
pastMessages = updatedMessages
|
||||
pastImageMessagesWithFileRef.push(...transformedMessages)
|
||||
|
||||
if (pastMessages.length > 0) {
|
||||
if (memoryType === 'windowSize') {
|
||||
@@ -759,7 +767,7 @@ class LLM_Agentflow implements INode {
|
||||
],
|
||||
{ signal: abortController?.signal }
|
||||
)
|
||||
messages.push({ role: 'assistant', content: summary.content as string })
|
||||
messages.push({ role: 'assistant', content: extractResponseContent(summary) })
|
||||
} else if (memoryType === 'conversationSummaryBuffer') {
|
||||
// Summary buffer: Summarize messages that exceed token limit
|
||||
await this.handleSummaryBuffer(messages, pastMessages, llmNodeInstance, nodeData, abortController)
|
||||
@@ -825,7 +833,11 @@ class LLM_Agentflow implements INode {
|
||||
)
|
||||
|
||||
// Add summary as a system message at the beginning, then add remaining messages
|
||||
messages.push({ role: 'system', content: `Previous conversation summary: ${summary.content}` })
|
||||
let summaryRole = 'system'
|
||||
if (messages.some((msg) => typeof msg === 'object' && !Array.isArray(msg) && 'role' in msg && msg.role === 'system')) {
|
||||
summaryRole = 'user' // some model doesn't allow multiple system messages
|
||||
}
|
||||
messages.push({ role: summaryRole, content: `Previous conversation summary: ${extractResponseContent(summary)}` })
|
||||
messages.push(...remainingMessages)
|
||||
} else {
|
||||
// If under token limit, use all messages
|
||||
@@ -841,19 +853,48 @@ class LLM_Agentflow implements INode {
|
||||
llmNodeInstance: BaseChatModel,
|
||||
messages: BaseMessageLike[],
|
||||
chatId: string,
|
||||
abortController: AbortController
|
||||
abortController: AbortController,
|
||||
isStructuredOutput: boolean = false,
|
||||
isLastNode: boolean = false
|
||||
): Promise<AIMessageChunk> {
|
||||
let response = new AIMessageChunk('')
|
||||
let reasonContent = ''
|
||||
let thinkingDuration: number | undefined
|
||||
let thinkingStartTime: number | null = null
|
||||
let wasThinking = false
|
||||
let sentLastThinkingEvent = false
|
||||
|
||||
try {
|
||||
for await (const chunk of await llmNodeInstance.stream(messages, { signal: abortController?.signal })) {
|
||||
if (sseStreamer) {
|
||||
if (sseStreamer && !isStructuredOutput) {
|
||||
let content = ''
|
||||
|
||||
if (chunk.contentBlocks?.length) {
|
||||
for (const block of chunk.contentBlocks) {
|
||||
if (isLastNode) {
|
||||
// As soon as we see the first non-reasoning block, send last thinking event with duration (only when isLastNode)
|
||||
if (block.type !== 'reasoning' && wasThinking && !sentLastThinkingEvent && thinkingStartTime != null) {
|
||||
thinkingDuration = Math.round((Date.now() - thinkingStartTime) / 1000)
|
||||
sseStreamer.streamThinkingEvent(chatId, '', thinkingDuration)
|
||||
sentLastThinkingEvent = true
|
||||
}
|
||||
if (block.type === 'reasoning' && (block as { reasoning?: string }).reasoning) {
|
||||
if (!thinkingStartTime) {
|
||||
thinkingStartTime = Date.now()
|
||||
}
|
||||
wasThinking = true
|
||||
const reasoningContent = (block as { reasoning: string }).reasoning
|
||||
sseStreamer.streamThinkingEvent(chatId, reasoningContent)
|
||||
reasonContent += reasoningContent
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof chunk === 'string') {
|
||||
content = chunk
|
||||
} else if (Array.isArray(chunk.content) && chunk.content.length > 0) {
|
||||
const contents = chunk.content as MessageContentText[]
|
||||
const contents = chunk.content as ContentBlock.Text[]
|
||||
content = contents.map((item) => item.text).join('')
|
||||
} else if (chunk.content) {
|
||||
content = chunk.content.toString()
|
||||
@@ -868,13 +909,71 @@ class LLM_Agentflow implements INode {
|
||||
console.error('Error during streaming:', error)
|
||||
throw error
|
||||
}
|
||||
|
||||
// Only convert to string if all content items are text (no inlineData or other special types)
|
||||
if (Array.isArray(response.content) && response.content.length > 0) {
|
||||
const responseContents = response.content as MessageContentText[]
|
||||
response.content = responseContents.map((item) => item.text).join('')
|
||||
const hasNonTextContent = response.content.some(
|
||||
(item: any) => item.type === 'inlineData' || item.type === 'executableCode' || item.type === 'codeExecutionResult'
|
||||
)
|
||||
if (!hasNonTextContent) {
|
||||
const responseContents = response.content as ContentBlock.Text[]
|
||||
response.content = responseContents.map((item) => item.text).join('')
|
||||
}
|
||||
}
|
||||
|
||||
if (reasonContent.length > 0) {
|
||||
response.additional_kwargs = {
|
||||
...response.additional_kwargs,
|
||||
reasoning_content: reasonContent,
|
||||
reasoning_duration: thinkingDuration
|
||||
}
|
||||
}
|
||||
|
||||
return response
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates input/output and total cost from usage metadata using model pricing from models.json.
|
||||
* Also returns the model's base (per-token) input and output costs.
|
||||
*/
|
||||
private async calculateUsageCost(
|
||||
provider: string | undefined,
|
||||
modelName: string | undefined,
|
||||
usageMetadata: Record<string, any> | undefined
|
||||
): Promise<
|
||||
| {
|
||||
input_cost: number
|
||||
output_cost: number
|
||||
total_cost: number
|
||||
base_input_cost: number
|
||||
base_output_cost: number
|
||||
}
|
||||
| undefined
|
||||
> {
|
||||
if (!provider || !modelName) return undefined
|
||||
const inputTokens = (usageMetadata?.input_tokens ?? 0) as number
|
||||
const outputTokens = (usageMetadata?.output_tokens ?? 0) as number
|
||||
try {
|
||||
const modelConfig = await getModelConfigByModelName(MODEL_TYPE.CHAT, provider, modelName)
|
||||
if (!modelConfig) return undefined
|
||||
const baseInputCost = Number(modelConfig.input_cost) || 0
|
||||
const baseOutputCost = Number(modelConfig.output_cost) || 0
|
||||
const inputCost = inputTokens * baseInputCost
|
||||
const outputCost = outputTokens * baseOutputCost
|
||||
const totalCost = inputCost + outputCost
|
||||
if (inputCost === 0 && outputCost === 0) return undefined
|
||||
return {
|
||||
input_cost: inputCost,
|
||||
output_cost: outputCost,
|
||||
total_cost: totalCost,
|
||||
base_input_cost: baseInputCost,
|
||||
base_output_cost: baseOutputCost
|
||||
}
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepares the output object with response and metadata
|
||||
*/
|
||||
@@ -886,7 +985,15 @@ class LLM_Agentflow implements INode {
|
||||
timeDelta: number,
|
||||
isStructuredOutput: boolean,
|
||||
artifacts: any[] = [],
|
||||
fileAnnotations: any[] = []
|
||||
fileAnnotations: any[] = [],
|
||||
reasonContent?: { thinking: string; thinkingDuration?: number },
|
||||
costMetadata?: {
|
||||
input_cost: number
|
||||
output_cost: number
|
||||
total_cost: number
|
||||
base_input_cost: number
|
||||
base_output_cost: number
|
||||
}
|
||||
): any {
|
||||
const output: any = {
|
||||
content: finalResponse,
|
||||
@@ -902,7 +1009,15 @@ class LLM_Agentflow implements INode {
|
||||
}
|
||||
|
||||
if (response.usage_metadata) {
|
||||
output.usageMetadata = response.usage_metadata
|
||||
output.usageMetadata = { ...response.usage_metadata }
|
||||
}
|
||||
|
||||
if (costMetadata && output.usageMetadata) {
|
||||
output.usageMetadata.input_cost = costMetadata.input_cost
|
||||
output.usageMetadata.output_cost = costMetadata.output_cost
|
||||
output.usageMetadata.total_cost = costMetadata.total_cost
|
||||
output.usageMetadata.base_input_cost = costMetadata.base_input_cost
|
||||
output.usageMetadata.base_output_cost = costMetadata.base_output_cost
|
||||
}
|
||||
|
||||
if (response.response_metadata) {
|
||||
@@ -926,6 +1041,10 @@ class LLM_Agentflow implements INode {
|
||||
output.fileAnnotations = fileAnnotations
|
||||
}
|
||||
|
||||
if (reasonContent) {
|
||||
output.reasonContent = reasonContent
|
||||
}
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,659 @@
|
||||
import { revertBase64ImagesToFileRefs, processMessagesWithImages, addImageArtifactsToMessages, getUniqueImageMessages } from './utils'
|
||||
import { sanitizeFileName } from '../../src/validator'
|
||||
import { IChatMessage, IMultimodalContentItem } from './Interface.Agentflow'
|
||||
import { IFileUpload } from '../../src/Interface'
|
||||
|
||||
// Mock storageUtils
|
||||
jest.mock('../../src/storageUtils', () => ({
|
||||
getFileFromStorage: jest.fn().mockResolvedValue(Buffer.from('fake-image-data')),
|
||||
addSingleFileToStorage: jest.fn().mockResolvedValue({ path: 'mock/path', totalSize: 100 })
|
||||
}))
|
||||
|
||||
// Mock multiModalUtils
|
||||
jest.mock('../../src/multiModalUtils', () => ({
|
||||
getImageUploads: jest.fn((uploads: IFileUpload[]) => uploads.filter((u) => u.mime?.startsWith('image/')))
|
||||
}))
|
||||
|
||||
// Mock node-fetch
|
||||
jest.mock('node-fetch', () => jest.fn())
|
||||
|
||||
// Mock ../../src/utils to avoid pulling in axios (ESM)
|
||||
jest.mock('../../src/utils', () => {
|
||||
return {
|
||||
getCredentialData: jest.fn(),
|
||||
getCredentialParam: jest.fn(),
|
||||
handleEscapeCharacters: jest.fn((str: string) => str),
|
||||
mapMimeTypeToInputField: jest.fn()
|
||||
}
|
||||
})
|
||||
|
||||
describe('revertBase64ImagesToFileRefs', () => {
|
||||
it('reverts tagged image_url items to stored-file format', () => {
|
||||
const messages: IChatMessage[] = [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: { url: 'data:image/jpeg;base64,abc123' }
|
||||
}
|
||||
],
|
||||
additional_kwargs: {
|
||||
_imageFileRefs: [{ index: 0, fileName: 'photo.jpg', mime: 'image/jpeg' }]
|
||||
}
|
||||
},
|
||||
{ role: 'user', content: 'what is this' }
|
||||
]
|
||||
|
||||
const result = revertBase64ImagesToFileRefs(messages) as IChatMessage[]
|
||||
|
||||
expect(result[0]).toEqual({
|
||||
role: 'user',
|
||||
content: [{ type: 'stored-file', name: 'photo.jpg', mime: 'image/jpeg' }]
|
||||
})
|
||||
expect(result[1]).toEqual({ role: 'user', content: 'what is this' })
|
||||
})
|
||||
|
||||
it('leaves untagged image_url items (external URLs) untouched', () => {
|
||||
const messages: IChatMessage[] = [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: { url: 'https://example.com/image.png' }
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
const result = revertBase64ImagesToFileRefs(messages) as IChatMessage[]
|
||||
|
||||
expect((result[0].content as IMultimodalContentItem[])[0]).toEqual({
|
||||
type: 'image_url',
|
||||
image_url: { url: 'https://example.com/image.png' }
|
||||
})
|
||||
})
|
||||
|
||||
it('handles mixed tagged and untagged items in the same message', () => {
|
||||
const messages: IChatMessage[] = [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: { url: 'data:image/png;base64,xyz' }
|
||||
},
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: { url: 'https://example.com/photo.jpg' }
|
||||
}
|
||||
],
|
||||
additional_kwargs: {
|
||||
_imageFileRefs: [{ index: 0, fileName: 'screenshot.png', mime: 'image/png' }]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
const result = revertBase64ImagesToFileRefs(messages) as IChatMessage[]
|
||||
const content = result[0].content as IMultimodalContentItem[]
|
||||
|
||||
expect(content[0]).toEqual({ type: 'stored-file', name: 'screenshot.png', mime: 'image/png' })
|
||||
expect(content[1]).toEqual({ type: 'image_url', image_url: { url: 'https://example.com/photo.jpg' } })
|
||||
})
|
||||
|
||||
it('handles multiple messages with multiple images each', () => {
|
||||
const messages: IChatMessage[] = [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'image_url', image_url: { url: 'data:image/jpeg;base64,a' } },
|
||||
{ type: 'image_url', image_url: { url: 'data:image/png;base64,b' } }
|
||||
],
|
||||
additional_kwargs: {
|
||||
_imageFileRefs: [
|
||||
{ index: 0, fileName: 'img1.jpg', mime: 'image/jpeg' },
|
||||
{ index: 1, fileName: 'img2.png', mime: 'image/png' }
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: [{ type: 'image_url', image_url: { url: 'data:image/gif;base64,c' } }],
|
||||
additional_kwargs: {
|
||||
_imageFileRefs: [{ index: 0, fileName: 'img3.gif', mime: 'image/gif' }]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
const result = revertBase64ImagesToFileRefs(messages) as IChatMessage[]
|
||||
const content0 = result[0].content as IMultimodalContentItem[]
|
||||
const content1 = result[1].content as IMultimodalContentItem[]
|
||||
|
||||
expect(content0[0]).toEqual({ type: 'stored-file', name: 'img1.jpg', mime: 'image/jpeg' })
|
||||
expect(content0[1]).toEqual({ type: 'stored-file', name: 'img2.png', mime: 'image/png' })
|
||||
expect(content1[0]).toEqual({ type: 'stored-file', name: 'img3.gif', mime: 'image/gif' })
|
||||
})
|
||||
|
||||
it('does not mutate the original messages', () => {
|
||||
const original: IChatMessage[] = [
|
||||
{
|
||||
role: 'user',
|
||||
content: [{ type: 'image_url', image_url: { url: 'data:image/jpeg;base64,abc' } }],
|
||||
additional_kwargs: {
|
||||
_imageFileRefs: [{ index: 0, fileName: 'test.jpg', mime: 'image/jpeg' }]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
const originalJson = JSON.stringify(original)
|
||||
revertBase64ImagesToFileRefs(original)
|
||||
|
||||
expect(JSON.stringify(original)).toBe(originalJson)
|
||||
})
|
||||
|
||||
it('handles messages with string content (no array)', () => {
|
||||
const messages: IChatMessage[] = [
|
||||
{ role: 'user', content: 'hello world' },
|
||||
{ role: 'assistant', content: 'hi there' }
|
||||
]
|
||||
|
||||
const result = revertBase64ImagesToFileRefs(messages) as IChatMessage[]
|
||||
|
||||
expect(result).toEqual(messages)
|
||||
})
|
||||
|
||||
it('handles empty messages array', () => {
|
||||
const result = revertBase64ImagesToFileRefs([])
|
||||
expect(result).toEqual([])
|
||||
})
|
||||
|
||||
it('skips non-image_url array content items', () => {
|
||||
const messages: IChatMessage[] = [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: 'describe this' },
|
||||
{ type: 'image_url', image_url: { url: 'data:image/png;base64,x' } }
|
||||
],
|
||||
additional_kwargs: {
|
||||
_imageFileRefs: [{ index: 1, fileName: 'pic.png', mime: 'image/png' }]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
const result = revertBase64ImagesToFileRefs(messages) as IChatMessage[]
|
||||
const content = result[0].content as IMultimodalContentItem[]
|
||||
|
||||
expect(content[0]).toEqual({ type: 'text', text: 'describe this' })
|
||||
expect(content[1]).toEqual({ type: 'stored-file', name: 'pic.png', mime: 'image/png' })
|
||||
})
|
||||
|
||||
it('cleans up additional_kwargs when _imageFileRefs is the only key', () => {
|
||||
const messages: IChatMessage[] = [
|
||||
{
|
||||
role: 'user',
|
||||
content: [{ type: 'image_url', image_url: { url: 'data:image/png;base64,x' } }],
|
||||
additional_kwargs: {
|
||||
_imageFileRefs: [{ index: 0, fileName: 'pic.png', mime: 'image/png' }]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
const result = revertBase64ImagesToFileRefs(messages) as IChatMessage[]
|
||||
|
||||
expect(result[0].additional_kwargs).toBeUndefined()
|
||||
})
|
||||
|
||||
it('preserves other additional_kwargs when removing _imageFileRefs', () => {
|
||||
const messages: IChatMessage[] = [
|
||||
{
|
||||
role: 'user',
|
||||
content: [{ type: 'image_url', image_url: { url: 'data:image/png;base64,x' } }],
|
||||
additional_kwargs: {
|
||||
_imageFileRefs: [{ index: 0, fileName: 'pic.png', mime: 'image/png' }],
|
||||
artifacts: [{ type: 'png', data: 'some/path' }]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
const result = revertBase64ImagesToFileRefs(messages) as IChatMessage[]
|
||||
|
||||
expect(result[0].additional_kwargs?._imageFileRefs).toBeUndefined()
|
||||
expect(result[0].additional_kwargs?.artifacts).toEqual([{ type: 'png', data: 'some/path' }])
|
||||
})
|
||||
})
|
||||
|
||||
describe('processMessagesWithImages', () => {
|
||||
const options = { chatflowid: 'flow1', chatId: 'chat1', orgId: 'org1' }
|
||||
|
||||
it('converts stored-file refs to base64 with file refs in additional_kwargs', async () => {
|
||||
const messages: IChatMessage[] = [{ role: 'user', content: [{ type: 'stored-file', name: 'photo.jpg', mime: 'image/jpeg' }] }]
|
||||
|
||||
const { updatedMessages } = await processMessagesWithImages(messages, options)
|
||||
const msg = updatedMessages[0] as IChatMessage
|
||||
const content = (msg.content as IMultimodalContentItem[])[0]
|
||||
|
||||
expect(content.type).toBe('image_url')
|
||||
expect(content.image_url?.url).toMatch(/^data:image\/jpeg;base64,/)
|
||||
expect(content._fileName).toBeUndefined()
|
||||
expect(content._mime).toBeUndefined()
|
||||
expect(msg.additional_kwargs?._imageFileRefs).toEqual([{ index: 0, fileName: 'photo.jpg', mime: 'image/jpeg' }])
|
||||
})
|
||||
|
||||
it('returns original messages when chatflowid or chatId is missing', async () => {
|
||||
const messages: IChatMessage[] = [{ role: 'user', content: [{ type: 'stored-file', name: 'a.png', mime: 'image/png' }] }]
|
||||
|
||||
const { updatedMessages, transformedMessages } = await processMessagesWithImages(messages, { chatflowid: '', chatId: '' })
|
||||
|
||||
expect(updatedMessages).toBe(messages)
|
||||
expect(transformedMessages).toEqual([])
|
||||
})
|
||||
|
||||
it('skips non-user messages', async () => {
|
||||
const messages: IChatMessage[] = [{ role: 'assistant', content: [{ type: 'stored-file', name: 'a.png', mime: 'image/png' }] }]
|
||||
|
||||
const { updatedMessages } = await processMessagesWithImages(messages, options)
|
||||
const content = ((updatedMessages[0] as IChatMessage).content as IMultimodalContentItem[])[0]
|
||||
|
||||
expect(content.type).toBe('stored-file')
|
||||
})
|
||||
|
||||
it('tracks transformed messages for revert', async () => {
|
||||
const messages: IChatMessage[] = [{ role: 'user', content: [{ type: 'stored-file', name: 'img.png', mime: 'image/png' }] }]
|
||||
|
||||
const { transformedMessages } = await processMessagesWithImages(messages, options)
|
||||
|
||||
expect(transformedMessages).toHaveLength(1)
|
||||
expect(transformedMessages[0]).toEqual({
|
||||
role: 'user',
|
||||
content: [{ type: 'stored-file', name: 'img.png', mime: 'image/png' }]
|
||||
})
|
||||
})
|
||||
|
||||
it('does not mutate the original messages', async () => {
|
||||
const messages: IChatMessage[] = [{ role: 'user', content: [{ type: 'stored-file', name: 'img.png', mime: 'image/png' }] }]
|
||||
|
||||
await processMessagesWithImages(messages, options)
|
||||
const content = (messages[0].content as IMultimodalContentItem[])[0]
|
||||
|
||||
expect(content.type).toBe('stored-file')
|
||||
})
|
||||
})
|
||||
|
||||
describe('addImageArtifactsToMessages', () => {
|
||||
const options = { chatflowid: 'flow1', chatId: 'chat1', orgId: 'org1' }
|
||||
|
||||
it('inserts temporary base64 user message after assistant message with image artifacts', async () => {
|
||||
const messages: IChatMessage[] = [
|
||||
{
|
||||
role: 'assistant',
|
||||
content: 'Here is the image',
|
||||
additional_kwargs: {
|
||||
artifacts: [{ type: 'png', data: 'uploads/generated_image.png' }]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
await addImageArtifactsToMessages(messages, options)
|
||||
|
||||
expect(messages).toHaveLength(2)
|
||||
const inserted = messages[1] as IChatMessage
|
||||
expect(inserted.role).toBe('user')
|
||||
expect(inserted._isTemporaryImageMessage).toBe(true)
|
||||
const content = (inserted.content as IMultimodalContentItem[])[0]
|
||||
expect(content.type).toBe('image_url')
|
||||
expect(content._fileName).toBeUndefined()
|
||||
expect(inserted.additional_kwargs?._imageFileRefs?.[0].fileName).toBe('generated_image.png')
|
||||
expect(inserted.additional_kwargs?._imageFileRefs?.[0].mime).toBe('image/png')
|
||||
})
|
||||
|
||||
it('does not insert duplicate if next message already has the image', async () => {
|
||||
const messages: IChatMessage[] = [
|
||||
{
|
||||
role: 'assistant',
|
||||
content: 'Image generated',
|
||||
additional_kwargs: {
|
||||
artifacts: [{ type: 'png', data: 'uploads/generated_image.png' }]
|
||||
}
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: [{ type: 'stored-file', name: 'generated_image.png', mime: 'image/png' }]
|
||||
}
|
||||
]
|
||||
|
||||
await addImageArtifactsToMessages(messages, options)
|
||||
|
||||
expect(messages).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('skips non-image artifacts', async () => {
|
||||
const messages: IChatMessage[] = [
|
||||
{
|
||||
role: 'assistant',
|
||||
content: 'Here is a document',
|
||||
additional_kwargs: {
|
||||
artifacts: [{ type: 'pdf', data: 'uploads/doc.pdf' }]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
await addImageArtifactsToMessages(messages, options)
|
||||
|
||||
expect(messages).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('does not modify messages without artifacts', async () => {
|
||||
const messages: IChatMessage[] = [
|
||||
{ role: 'user', content: 'hello' },
|
||||
{ role: 'assistant', content: 'hi' }
|
||||
]
|
||||
|
||||
await addImageArtifactsToMessages(messages, options)
|
||||
|
||||
expect(messages).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('handles multiple assistant messages with image artifacts', async () => {
|
||||
const messages: IChatMessage[] = [
|
||||
{
|
||||
role: 'assistant',
|
||||
content: 'First image',
|
||||
additional_kwargs: { artifacts: [{ type: 'png', data: 'uploads/img1.png' }] }
|
||||
},
|
||||
{ role: 'user', content: 'now another' },
|
||||
{
|
||||
role: 'assistant',
|
||||
content: 'Second image',
|
||||
additional_kwargs: { artifacts: [{ type: 'jpg', data: 'uploads/img2.jpg' }] }
|
||||
}
|
||||
]
|
||||
|
||||
await addImageArtifactsToMessages(messages, options)
|
||||
|
||||
// Should have 5 messages: assistant, temp_img, user, assistant, temp_img
|
||||
expect(messages).toHaveLength(5)
|
||||
const msg1 = messages[1] as IChatMessage
|
||||
const msg4 = messages[4] as IChatMessage
|
||||
expect(msg1._isTemporaryImageMessage).toBe(true)
|
||||
expect(msg1.additional_kwargs?._imageFileRefs?.[0].fileName).toBe('img1.png')
|
||||
expect(msg4._isTemporaryImageMessage).toBe(true)
|
||||
expect(msg4.additional_kwargs?._imageFileRefs?.[0].fileName).toBe('img2.jpg')
|
||||
})
|
||||
})
|
||||
|
||||
describe('getUniqueImageMessages', () => {
|
||||
it('returns base64 message with file refs in additional_kwargs', async () => {
|
||||
const options = {
|
||||
uploads: [{ type: 'stored-file', name: 'photo.jpg', mime: 'image/jpeg', data: '' }],
|
||||
chatflowid: 'flow1',
|
||||
chatId: 'chat1',
|
||||
orgId: 'org1'
|
||||
}
|
||||
const modelConfig = { allowImageUploads: true }
|
||||
|
||||
const result = await getUniqueImageMessages(options, [], modelConfig)
|
||||
|
||||
expect(result).toBeDefined()
|
||||
const msg = result!.imageMessageWithBase64 as IChatMessage
|
||||
const content = (msg.content as IMultimodalContentItem[])[0]
|
||||
expect(content.type).toBe('image_url')
|
||||
expect(content._fileName).toBeUndefined()
|
||||
expect(msg.additional_kwargs?._imageFileRefs?.[0].fileName).toBe('photo.jpg')
|
||||
expect(msg.additional_kwargs?._imageFileRefs?.[0].mime).toBe('image/jpeg')
|
||||
})
|
||||
|
||||
it('returns undefined when no uploads', async () => {
|
||||
const result = await getUniqueImageMessages({}, [], {})
|
||||
expect(result).toBeUndefined()
|
||||
})
|
||||
|
||||
it('filters out images already in messages', async () => {
|
||||
const existingBase64 = 'data:image/jpeg;base64,' + Buffer.from('fake-image-data').toString('base64')
|
||||
const options = {
|
||||
uploads: [{ type: 'stored-file', name: 'photo.jpg', mime: 'image/jpeg', data: '' }],
|
||||
chatflowid: 'flow1',
|
||||
chatId: 'chat1',
|
||||
orgId: 'org1'
|
||||
}
|
||||
const messages: IChatMessage[] = [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: { url: existingBase64, detail: 'low' }
|
||||
}
|
||||
],
|
||||
additional_kwargs: {
|
||||
_imageFileRefs: [{ index: 0, fileName: 'photo.jpg', mime: 'image/jpeg' }]
|
||||
}
|
||||
}
|
||||
]
|
||||
const modelConfig = { allowImageUploads: true }
|
||||
|
||||
const result = await getUniqueImageMessages(options, messages, modelConfig)
|
||||
|
||||
expect(result).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('end-to-end: base64 tagging and revert', () => {
|
||||
it('upload image → base64 for invoke → revert to file refs for storage', async () => {
|
||||
const options = {
|
||||
uploads: [{ type: 'stored-file', name: 'cat.png', mime: 'image/png', data: '' }],
|
||||
chatflowid: 'flow1',
|
||||
chatId: 'chat1',
|
||||
orgId: 'org1'
|
||||
}
|
||||
const modelConfig = { allowImageUploads: true }
|
||||
|
||||
// Step 1: Build messages with base64 (as the node does)
|
||||
const messages: IChatMessage[] = []
|
||||
const imageContents = await getUniqueImageMessages(options, messages, modelConfig)
|
||||
if (imageContents) {
|
||||
messages.push(imageContents.imageMessageWithBase64 as IChatMessage)
|
||||
}
|
||||
messages.push({ role: 'user', content: 'describe this image' })
|
||||
|
||||
// Verify messages have base64 for model invoke
|
||||
const content0 = (messages[0].content as IMultimodalContentItem[])[0]
|
||||
expect(content0.type).toBe('image_url')
|
||||
expect(content0.image_url?.url).toMatch(/^data:/)
|
||||
// No _fileName on content items
|
||||
expect(content0._fileName).toBeUndefined()
|
||||
|
||||
// Step 2: After invoke, revert to file refs for storage
|
||||
const reverted = revertBase64ImagesToFileRefs(messages) as IChatMessage[]
|
||||
|
||||
expect(reverted[0]).toEqual({
|
||||
role: 'user',
|
||||
content: [{ type: 'stored-file', name: 'cat.png', mime: 'image/png' }]
|
||||
})
|
||||
expect(reverted[1]).toEqual({ role: 'user', content: 'describe this image' })
|
||||
})
|
||||
|
||||
it('processMessagesWithImages: stored-file → base64 → revert', async () => {
|
||||
const options = { chatflowid: 'flow1', chatId: 'chat1', orgId: 'org1' }
|
||||
|
||||
const messages: IChatMessage[] = [
|
||||
{ role: 'user', content: [{ type: 'stored-file', name: 'diagram.png', mime: 'image/png' }] },
|
||||
{ role: 'user', content: 'explain this diagram' },
|
||||
{ role: 'assistant', content: 'This is a flowchart...' }
|
||||
]
|
||||
|
||||
// Convert stored-file → base64 (as handleMemory does)
|
||||
const { updatedMessages } = await processMessagesWithImages(messages, options)
|
||||
const msg0 = updatedMessages[0] as IChatMessage
|
||||
const content0 = (msg0.content as IMultimodalContentItem[])[0]
|
||||
|
||||
expect(content0.type).toBe('image_url')
|
||||
// File refs are in additional_kwargs, not on content items
|
||||
expect(content0._fileName).toBeUndefined()
|
||||
expect(msg0.additional_kwargs?._imageFileRefs?.[0].fileName).toBe('diagram.png')
|
||||
|
||||
// Revert for storage
|
||||
const reverted = revertBase64ImagesToFileRefs(updatedMessages) as IChatMessage[]
|
||||
|
||||
expect(reverted[0]).toEqual({
|
||||
role: 'user',
|
||||
content: [{ type: 'stored-file', name: 'diagram.png', mime: 'image/png' }]
|
||||
})
|
||||
expect(reverted[1]).toEqual({ role: 'user', content: 'explain this diagram' })
|
||||
expect(reverted[2]).toEqual({ role: 'assistant', content: 'This is a flowchart...' })
|
||||
})
|
||||
|
||||
it('artifact images: insert temp → filter → revert', async () => {
|
||||
const options = { chatflowid: 'flow1', chatId: 'chat1', orgId: 'org1' }
|
||||
|
||||
const messages: IChatMessage[] = [
|
||||
{ role: 'user', content: 'generate an image of a cat' },
|
||||
{
|
||||
role: 'assistant',
|
||||
content: 'Here is your cat image',
|
||||
additional_kwargs: { artifacts: [{ type: 'png', data: 'uploads/cat_gen.png' }] }
|
||||
},
|
||||
{ role: 'user', content: 'make it blue' }
|
||||
]
|
||||
|
||||
// Step 1: addImageArtifactsToMessages inserts temp base64 message
|
||||
await addImageArtifactsToMessages(messages, options)
|
||||
|
||||
expect(messages).toHaveLength(4)
|
||||
expect((messages[2] as IChatMessage)._isTemporaryImageMessage).toBe(true)
|
||||
|
||||
// Step 2: Filter temp messages (as Agent.ts does after invoke)
|
||||
const stored = messages.filter((m) => !(m as IChatMessage)._isTemporaryImageMessage)
|
||||
expect(stored).toHaveLength(3)
|
||||
|
||||
// Step 3: Revert remaining base64 — none should remain since temp was removed
|
||||
const reverted = revertBase64ImagesToFileRefs(stored) as IChatMessage[]
|
||||
|
||||
expect(reverted[0]).toEqual({ role: 'user', content: 'generate an image of a cat' })
|
||||
expect(reverted[2]).toEqual({ role: 'user', content: 'make it blue' })
|
||||
})
|
||||
})
|
||||
|
||||
describe('sanitizeFileName', () => {
|
||||
it('returns a plain filename as-is', () => {
|
||||
expect(sanitizeFileName('photo.jpg')).toBe('photo.jpg')
|
||||
})
|
||||
|
||||
it('strips FILE-STORAGE:: prefix', () => {
|
||||
expect(sanitizeFileName('FILE-STORAGE::photo.jpg')).toBe('photo.jpg')
|
||||
})
|
||||
|
||||
it('strips path traversal sequences', () => {
|
||||
expect(sanitizeFileName('../../../../etc/passwd')).toBe('passwd')
|
||||
expect(sanitizeFileName('../../../secret.txt')).toBe('secret.txt')
|
||||
})
|
||||
|
||||
it('strips absolute paths', () => {
|
||||
expect(sanitizeFileName('/etc/passwd')).toBe('passwd')
|
||||
expect(sanitizeFileName('C:\\Windows\\system32\\config.sys')).toBe('config.sys')
|
||||
})
|
||||
|
||||
it('strips FILE-STORAGE:: prefix combined with traversal', () => {
|
||||
expect(sanitizeFileName('FILE-STORAGE::../../etc/shadow')).toBe('shadow')
|
||||
})
|
||||
|
||||
it('throws on empty or dot-only names', () => {
|
||||
expect(() => sanitizeFileName('')).toThrow()
|
||||
expect(() => sanitizeFileName('..')).toThrow()
|
||||
expect(() => sanitizeFileName('FILE-STORAGE::')).toThrow()
|
||||
})
|
||||
|
||||
it('handles names with subdirectory components', () => {
|
||||
expect(sanitizeFileName('uploads/subfolder/image.png')).toBe('image.png')
|
||||
})
|
||||
|
||||
it('strips URL-encoded path traversal sequences', () => {
|
||||
expect(sanitizeFileName('%2e%2e%2fetc%2fpasswd')).toBe('passwd')
|
||||
})
|
||||
|
||||
it('rejects double-encoded traversal attempts', () => {
|
||||
expect(() => sanitizeFileName('%252e%252e%252fpasswd')).toThrow()
|
||||
})
|
||||
|
||||
it('strips backslash-based traversal (Windows)', () => {
|
||||
expect(sanitizeFileName('..\\..\\Windows\\system.ini')).toBe('system.ini')
|
||||
})
|
||||
})
|
||||
|
||||
describe('path traversal prevention in image processing', () => {
|
||||
it('_addImagesToMessages sanitizes filenames with traversal', async () => {
|
||||
const options = {
|
||||
uploads: [{ type: 'stored-file', name: '../../../../etc/passwd', mime: 'image/jpeg', data: '' }],
|
||||
chatflowid: 'flow1',
|
||||
chatId: 'chat1',
|
||||
orgId: 'org1'
|
||||
}
|
||||
const modelConfig = { allowImageUploads: true }
|
||||
|
||||
const result = await getUniqueImageMessages(options, [], modelConfig)
|
||||
|
||||
expect(result).toBeDefined()
|
||||
const msg = result!.imageMessageWithBase64 as IChatMessage
|
||||
const fileRef = msg.additional_kwargs?._imageFileRefs?.[0]
|
||||
expect(fileRef?.fileName).toBe('passwd')
|
||||
expect(fileRef?.fileName).not.toContain('..')
|
||||
})
|
||||
|
||||
it('processMessagesWithImages sanitizes stored-file names', async () => {
|
||||
const options = { chatflowid: 'flow1', chatId: 'chat1', orgId: 'org1' }
|
||||
const messages: IChatMessage[] = [
|
||||
{ role: 'user', content: [{ type: 'stored-file', name: '../../../secret.png', mime: 'image/png' }] }
|
||||
]
|
||||
|
||||
const { updatedMessages } = await processMessagesWithImages(messages, options)
|
||||
const msg = updatedMessages[0] as IChatMessage
|
||||
const fileRef = msg.additional_kwargs?._imageFileRefs?.[0]
|
||||
|
||||
expect(fileRef?.fileName).toBe('secret.png')
|
||||
expect(fileRef?.fileName).not.toContain('..')
|
||||
})
|
||||
|
||||
it('addImageArtifactsToMessages sanitizes LLM-controlled artifact paths (prompt injection)', async () => {
|
||||
const options = { chatflowid: 'flow1', chatId: 'chat1', orgId: 'org1' }
|
||||
const messages: IChatMessage[] = [
|
||||
{
|
||||
role: 'assistant',
|
||||
content: 'Here is your image',
|
||||
additional_kwargs: {
|
||||
artifacts: [{ type: 'png', data: '../../../../etc/shadow' }]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
await addImageArtifactsToMessages(messages, options)
|
||||
|
||||
expect(messages).toHaveLength(2)
|
||||
const inserted = messages[1] as IChatMessage
|
||||
expect(inserted._isTemporaryImageMessage).toBe(true)
|
||||
const fileRef = inserted.additional_kwargs?._imageFileRefs?.[0]
|
||||
expect(fileRef?.fileName).toBe('shadow')
|
||||
expect(fileRef?.fileName).not.toContain('..')
|
||||
expect(fileRef?.fileName).not.toContain('/')
|
||||
})
|
||||
|
||||
it('addImageArtifactsToMessages sanitizes URL-encoded artifact paths', async () => {
|
||||
const options = { chatflowid: 'flow1', chatId: 'chat1', orgId: 'org1' }
|
||||
const messages: IChatMessage[] = [
|
||||
{
|
||||
role: 'assistant',
|
||||
content: 'Image ready',
|
||||
additional_kwargs: {
|
||||
artifacts: [{ type: 'png', data: 'uploads%2f..%2f..%2f..%2fetc%2fpasswd' }]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
await addImageArtifactsToMessages(messages, options)
|
||||
|
||||
expect(messages).toHaveLength(2)
|
||||
const fileRef = (messages[1] as IChatMessage).additional_kwargs?._imageFileRefs?.[0]
|
||||
expect(fileRef?.fileName).toBe('passwd')
|
||||
expect(fileRef?.fileName).not.toContain('..')
|
||||
})
|
||||
})
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,7 +1,7 @@
|
||||
import axios from 'axios'
|
||||
import { BaseLanguageModel } from '@langchain/core/language_models/base'
|
||||
import { AgentExecutor } from 'langchain/agents'
|
||||
import { LLMChain } from 'langchain/chains'
|
||||
import { AgentExecutor } from '@langchain/classic/agents'
|
||||
import { LLMChain } from '@langchain/classic/chains'
|
||||
import { ICommonObject, INode, INodeData, INodeParams, IServerSideEventStreamer, PromptTemplate } from '../../../src/Interface'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
|
||||
@@ -21,6 +21,8 @@ class Airtable_Agents implements INode {
|
||||
baseClasses: string[]
|
||||
credential: INodeParams
|
||||
inputs: INodeParams[]
|
||||
deprecateMessage: string
|
||||
badge: string
|
||||
|
||||
constructor() {
|
||||
this.label = 'Airtable Agent'
|
||||
@@ -30,6 +32,8 @@ class Airtable_Agents implements INode {
|
||||
this.category = 'Agents'
|
||||
this.icon = 'airtable.svg'
|
||||
this.description = 'Agent used to answer queries on Airtable table'
|
||||
this.badge = 'DEPRECATING'
|
||||
this.deprecateMessage = 'Airtable Agent is deprecated and will be removed in a future release. Use Agent from AgentFlow instead.'
|
||||
this.baseClasses = [this.type, ...getBaseClasses(AgentExecutor)]
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
|
||||
@@ -1,226 +0,0 @@
|
||||
import { flatten } from 'lodash'
|
||||
import { Tool, StructuredTool } from '@langchain/core/tools'
|
||||
import { BaseChatModel } from '@langchain/core/language_models/chat_models'
|
||||
import { AIMessage, HumanMessage, SystemMessage } from '@langchain/core/messages'
|
||||
import { VectorStoreRetriever } from '@langchain/core/vectorstores'
|
||||
import { PromptTemplate } from '@langchain/core/prompts'
|
||||
import { AutoGPT } from 'langchain/experimental/autogpt'
|
||||
import { LLMChain } from 'langchain/chains'
|
||||
import { INode, INodeData, INodeParams } from '../../../src/Interface'
|
||||
import { checkInputs, Moderation } from '../../moderation/Moderation'
|
||||
import { formatResponse } from '../../outputparsers/OutputParserHelpers'
|
||||
|
||||
type ObjectTool = StructuredTool
|
||||
const FINISH_NAME = 'finish'
|
||||
|
||||
class AutoGPT_Agents implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
badge: string
|
||||
|
||||
constructor() {
|
||||
this.label = 'AutoGPT'
|
||||
this.name = 'autoGPT'
|
||||
this.version = 2.0
|
||||
this.type = 'AutoGPT'
|
||||
this.category = 'Agents'
|
||||
this.badge = 'DEPRECATING'
|
||||
this.icon = 'autogpt.svg'
|
||||
this.description = 'Autonomous agent with chain of thoughts for self-guided task completion'
|
||||
this.baseClasses = ['AutoGPT']
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Allowed Tools',
|
||||
name: 'tools',
|
||||
type: 'Tool',
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Chat Model',
|
||||
name: 'model',
|
||||
type: 'BaseChatModel'
|
||||
},
|
||||
{
|
||||
label: 'Vector Store Retriever',
|
||||
name: 'vectorStoreRetriever',
|
||||
type: 'BaseRetriever'
|
||||
},
|
||||
{
|
||||
label: 'AutoGPT Name',
|
||||
name: 'aiName',
|
||||
type: 'string',
|
||||
placeholder: 'Tom',
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'AutoGPT Role',
|
||||
name: 'aiRole',
|
||||
type: 'string',
|
||||
placeholder: 'Assistant',
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Maximum Loop',
|
||||
name: 'maxLoop',
|
||||
type: 'number',
|
||||
default: 5,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Input Moderation',
|
||||
description: 'Detect text that could generate harmful output and prevent it from being sent to the language model',
|
||||
name: 'inputModeration',
|
||||
type: 'Moderation',
|
||||
optional: true,
|
||||
list: true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
const model = nodeData.inputs?.model as BaseChatModel
|
||||
const vectorStoreRetriever = nodeData.inputs?.vectorStoreRetriever as VectorStoreRetriever
|
||||
let tools = nodeData.inputs?.tools as Tool[]
|
||||
tools = flatten(tools)
|
||||
const aiName = (nodeData.inputs?.aiName as string) || 'AutoGPT'
|
||||
const aiRole = (nodeData.inputs?.aiRole as string) || 'Assistant'
|
||||
const maxLoop = nodeData.inputs?.maxLoop as string
|
||||
|
||||
const autogpt = AutoGPT.fromLLMAndTools(model, tools, {
|
||||
memory: vectorStoreRetriever,
|
||||
aiName,
|
||||
aiRole
|
||||
})
|
||||
|
||||
autogpt.maxIterations = parseInt(maxLoop, 10)
|
||||
|
||||
return autogpt
|
||||
}
|
||||
|
||||
async run(nodeData: INodeData, input: string): Promise<string | object> {
|
||||
const executor = nodeData.instance as AutoGPT
|
||||
const model = nodeData.inputs?.model as BaseChatModel
|
||||
const moderations = nodeData.inputs?.inputModeration as Moderation[]
|
||||
|
||||
if (moderations && moderations.length > 0) {
|
||||
try {
|
||||
// Use the output of the moderation chain as input for the AutoGPT agent
|
||||
input = await checkInputs(moderations, input)
|
||||
} catch (e) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500))
|
||||
// if (options.shouldStreamResponse) {
|
||||
// streamResponse(options.sseStreamer, options.chatId, e.message)
|
||||
// }
|
||||
return formatResponse(e.message)
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
let totalAssistantReply = ''
|
||||
executor.run = async (goals: string[]): Promise<string | undefined> => {
|
||||
const user_input = 'Determine which next command to use, and respond using the format specified above:'
|
||||
let loopCount = 0
|
||||
while (loopCount < executor.maxIterations) {
|
||||
loopCount += 1
|
||||
|
||||
const { text: assistantReply } = await executor.chain.call({
|
||||
goals,
|
||||
user_input,
|
||||
memory: executor.memory,
|
||||
messages: executor.fullMessageHistory
|
||||
})
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('\x1b[92m\x1b[1m\n*****AutoGPT*****\n\x1b[0m\x1b[0m')
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(assistantReply)
|
||||
totalAssistantReply += assistantReply + '\n'
|
||||
executor.fullMessageHistory.push(new HumanMessage(user_input))
|
||||
executor.fullMessageHistory.push(new AIMessage(assistantReply))
|
||||
|
||||
const action = await executor.outputParser.parse(assistantReply)
|
||||
const tools = executor.tools.reduce((acc, tool) => ({ ...acc, [tool.name]: tool }), {} as { [key: string]: ObjectTool })
|
||||
if (action.name === FINISH_NAME) {
|
||||
return action.args.response
|
||||
}
|
||||
let result: string
|
||||
if (action.name in tools) {
|
||||
const tool = tools[action.name]
|
||||
let observation
|
||||
try {
|
||||
observation = await tool.call(action.args)
|
||||
} catch (e) {
|
||||
observation = `Error in args: ${e}`
|
||||
}
|
||||
result = `Command ${tool.name} returned: ${observation}`
|
||||
} else if (action.name === 'ERROR') {
|
||||
result = `Error: ${action.args}. `
|
||||
} else {
|
||||
result = `Unknown command '${action.name}'. Please refer to the 'COMMANDS' list for available commands and only respond in the specified JSON format.`
|
||||
}
|
||||
|
||||
let memoryToAdd = `Assistant Reply: ${assistantReply}\nResult: ${result} `
|
||||
if (executor.feedbackTool) {
|
||||
const feedback = `\n${await executor.feedbackTool.call('Input: ')}`
|
||||
if (feedback === 'q' || feedback === 'stop') {
|
||||
return 'EXITING'
|
||||
}
|
||||
memoryToAdd += feedback
|
||||
}
|
||||
|
||||
const documents = await executor.textSplitter.createDocuments([memoryToAdd])
|
||||
await executor.memory.addDocuments(documents)
|
||||
executor.fullMessageHistory.push(new SystemMessage(result))
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
const res = await executor.run([input])
|
||||
|
||||
if (!res) {
|
||||
const sentence = `Unfortunately I was not able to complete all the task. Here is the chain of thoughts:`
|
||||
return `${await rephraseString(sentence, model)}\n\`\`\`javascript\n${totalAssistantReply}\n\`\`\`\n`
|
||||
}
|
||||
|
||||
const sentence = `I have completed all my tasks. Here is the chain of thoughts:`
|
||||
let writeFilePath = ''
|
||||
const writeTool = executor.tools.find((tool) => tool.name === 'write_file')
|
||||
if (executor.tools.length && writeTool) {
|
||||
writeFilePath = (writeTool as any).store.basePath
|
||||
}
|
||||
return `${await rephraseString(
|
||||
sentence,
|
||||
model
|
||||
)}\n\`\`\`javascript\n${totalAssistantReply}\n\`\`\`\nAnd the final result:\n\`\`\`javascript\n${res}\n\`\`\`\n${
|
||||
writeFilePath
|
||||
? await rephraseString(
|
||||
`You can download the final result displayed above, or see if a new file has been successfully written to \`${writeFilePath}\``,
|
||||
model
|
||||
)
|
||||
: ''
|
||||
}`
|
||||
} catch (e) {
|
||||
throw new Error(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const rephraseString = async (sentence: string, model: BaseChatModel) => {
|
||||
const promptTemplate = new PromptTemplate({
|
||||
template: 'You are a helpful Assistant that rephrase a sentence: {sentence}',
|
||||
inputVariables: ['sentence']
|
||||
})
|
||||
const chain = new LLMChain({ llm: model, prompt: promptTemplate })
|
||||
const res = await chain.call({ sentence })
|
||||
return res?.text
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: AutoGPT_Agents }
|
||||
@@ -1,11 +0,0 @@
|
||||
<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M16 15V27" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<path d="M19 15V18.4545C19 19.5591 19.8954 20.4545 21 20.4545H22C23.1046 20.4545 24 21.35 24 22.4545V27" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<path d="M13 15V18.4545C13 19.5591 12.1046 20.4545 11 20.4545H10C8.89543 20.4545 8 21.35 8 22.4545V27" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<path d="M21 14V16C21 17.1046 21.8954 18 23 18H24C25.1046 18 26 17.1046 26 16V12.5C26 11.9477 26.4477 11.5 27 11.5V11.5" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<path d="M11 14V16C11 17.1046 10.1046 18 9 18H8C6.89543 18 6 17.1046 6 16V12C6 11.4477 5.55228 11 5 11V11" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<path d="M10 6C10 5.44772 10.4477 5 11 5H21C21.5523 5 22 5.44772 22 6V11C22 13.2091 20.2091 15 18 15H14C11.7909 15 10 13.2091 10 11V6Z" stroke="black" stroke-width="2" stroke-linejoin="round"/>
|
||||
<path d="M16 5V3" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<circle cx="13.5" cy="11" r="1.5" fill="black"/>
|
||||
<circle cx="18.5" cy="11" r="1.5" fill="black"/>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 1.3 KiB |
@@ -1,92 +0,0 @@
|
||||
import { BaseChatModel } from '@langchain/core/language_models/chat_models'
|
||||
import { VectorStore } from '@langchain/core/vectorstores'
|
||||
import { INode, INodeData, INodeParams } from '../../../src/Interface'
|
||||
import { BabyAGI } from './core'
|
||||
import { checkInputs, Moderation } from '../../moderation/Moderation'
|
||||
import { formatResponse } from '../../outputparsers/OutputParserHelpers'
|
||||
|
||||
class BabyAGI_Agents implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
badge: string
|
||||
|
||||
constructor() {
|
||||
this.label = 'BabyAGI'
|
||||
this.name = 'babyAGI'
|
||||
this.version = 2.0
|
||||
this.type = 'BabyAGI'
|
||||
this.category = 'Agents'
|
||||
this.icon = 'babyagi.svg'
|
||||
this.badge = 'DEPRECATING'
|
||||
this.description = 'Task Driven Autonomous Agent which creates new task and reprioritizes task list based on objective'
|
||||
this.baseClasses = ['BabyAGI']
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Chat Model',
|
||||
name: 'model',
|
||||
type: 'BaseChatModel'
|
||||
},
|
||||
{
|
||||
label: 'Vector Store',
|
||||
name: 'vectorStore',
|
||||
type: 'VectorStore'
|
||||
},
|
||||
{
|
||||
label: 'Task Loop',
|
||||
name: 'taskLoop',
|
||||
type: 'number',
|
||||
default: 3
|
||||
},
|
||||
{
|
||||
label: 'Input Moderation',
|
||||
description: 'Detect text that could generate harmful output and prevent it from being sent to the language model',
|
||||
name: 'inputModeration',
|
||||
type: 'Moderation',
|
||||
optional: true,
|
||||
list: true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
const model = nodeData.inputs?.model as BaseChatModel
|
||||
const vectorStore = nodeData.inputs?.vectorStore as VectorStore
|
||||
const taskLoop = nodeData.inputs?.taskLoop as string
|
||||
const k = (vectorStore as any)?.k ?? 4
|
||||
|
||||
const babyAgi = BabyAGI.fromLLM(model, vectorStore, parseInt(taskLoop, 10), k)
|
||||
return babyAgi
|
||||
}
|
||||
|
||||
async run(nodeData: INodeData, input: string): Promise<string | object> {
|
||||
const executor = nodeData.instance as BabyAGI
|
||||
const moderations = nodeData.inputs?.inputModeration as Moderation[]
|
||||
|
||||
if (moderations && moderations.length > 0) {
|
||||
try {
|
||||
// Use the output of the moderation chain as input for the BabyAGI agent
|
||||
input = await checkInputs(moderations, input)
|
||||
} catch (e) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500))
|
||||
// if (options.shouldStreamResponse) {
|
||||
// streamResponse(options.sseStreamer, options.chatId, e.message)
|
||||
// }
|
||||
return formatResponse(e.message)
|
||||
}
|
||||
}
|
||||
|
||||
const objective = input
|
||||
|
||||
const res = await executor.call({ objective })
|
||||
return res
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: BabyAGI_Agents }
|
||||
@@ -1,7 +0,0 @@
|
||||
<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<circle cx="16" cy="16" r="13" stroke="black" stroke-width="2"/>
|
||||
<path d="M16 3C17 3.12455 19 3.97145 19 6.36272C19 8.75399 17 9.10271 16 8.97817" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<circle cx="11" cy="16" r="1.5" fill="black"/>
|
||||
<circle cx="21" cy="16" r="1.5" fill="black"/>
|
||||
<path d="M13 22C13 22 14 23 16 23C18 23 19 22 19 22" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 559 B |
@@ -1,270 +0,0 @@
|
||||
import { BaseChatModel } from '@langchain/core/language_models/chat_models'
|
||||
import { VectorStore } from '@langchain/core/vectorstores'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { PromptTemplate } from '@langchain/core/prompts'
|
||||
import { LLMChain } from 'langchain/chains'
|
||||
|
||||
class TaskCreationChain extends LLMChain {
|
||||
constructor(prompt: PromptTemplate, llm: BaseChatModel) {
|
||||
super({ prompt, llm })
|
||||
}
|
||||
|
||||
static from_llm(llm: BaseChatModel): LLMChain {
|
||||
const taskCreationTemplate: string =
|
||||
'You are a task creation AI that uses the result of an execution agent' +
|
||||
' to create new tasks with the following objective: {objective},' +
|
||||
' The last completed task has the result: {result}.' +
|
||||
' This result was based on this task description: {task_description}.' +
|
||||
' These are incomplete tasks list: {incomplete_tasks}.' +
|
||||
' Based on the result, create new tasks to be completed' +
|
||||
' by the AI system that do not overlap with incomplete tasks.' +
|
||||
' Return the tasks as an array.'
|
||||
|
||||
const prompt = new PromptTemplate({
|
||||
template: taskCreationTemplate,
|
||||
inputVariables: ['result', 'task_description', 'incomplete_tasks', 'objective']
|
||||
})
|
||||
|
||||
return new TaskCreationChain(prompt, llm)
|
||||
}
|
||||
}
|
||||
|
||||
class TaskPrioritizationChain extends LLMChain {
|
||||
constructor(prompt: PromptTemplate, llm: BaseChatModel) {
|
||||
super({ prompt, llm })
|
||||
}
|
||||
|
||||
static from_llm(llm: BaseChatModel): TaskPrioritizationChain {
|
||||
const taskPrioritizationTemplate: string =
|
||||
'You are a task prioritization AI tasked with cleaning the formatting of and reprioritizing' +
|
||||
' the following task list: {task_names}.' +
|
||||
' Consider the ultimate objective of your team: {objective}.' +
|
||||
' Do not remove any tasks. Return the result as a numbered list, like:' +
|
||||
' #. First task' +
|
||||
' #. Second task' +
|
||||
' Start the task list with number {next_task_id}.'
|
||||
const prompt = new PromptTemplate({
|
||||
template: taskPrioritizationTemplate,
|
||||
inputVariables: ['task_names', 'next_task_id', 'objective']
|
||||
})
|
||||
return new TaskPrioritizationChain(prompt, llm)
|
||||
}
|
||||
}
|
||||
|
||||
class ExecutionChain extends LLMChain {
|
||||
constructor(prompt: PromptTemplate, llm: BaseChatModel) {
|
||||
super({ prompt, llm })
|
||||
}
|
||||
|
||||
static from_llm(llm: BaseChatModel): LLMChain {
|
||||
const executionTemplate: string =
|
||||
'You are an AI who performs one task based on the following objective: {objective}.' +
|
||||
' Take into account these previously completed tasks: {context}.' +
|
||||
' Your task: {task}.' +
|
||||
' Response:'
|
||||
|
||||
const prompt = new PromptTemplate({
|
||||
template: executionTemplate,
|
||||
inputVariables: ['objective', 'context', 'task']
|
||||
})
|
||||
|
||||
return new ExecutionChain(prompt, llm)
|
||||
}
|
||||
}
|
||||
|
||||
async function getNextTask(
|
||||
taskCreationChain: LLMChain,
|
||||
result: string,
|
||||
taskDescription: string,
|
||||
taskList: string[],
|
||||
objective: string
|
||||
): Promise<any[]> {
|
||||
const incompleteTasks: string = taskList.join(', ')
|
||||
const response: string = await taskCreationChain.predict({
|
||||
result,
|
||||
task_description: taskDescription,
|
||||
incomplete_tasks: incompleteTasks,
|
||||
objective
|
||||
})
|
||||
|
||||
const newTasks: string[] = response.split('\n')
|
||||
|
||||
return newTasks.filter((taskName) => taskName.trim()).map((taskName) => ({ task_name: taskName }))
|
||||
}
|
||||
|
||||
interface Task {
|
||||
task_id: number
|
||||
task_name: string
|
||||
}
|
||||
|
||||
async function prioritizeTasks(
|
||||
taskPrioritizationChain: LLMChain,
|
||||
thisTaskId: number,
|
||||
taskList: Task[],
|
||||
objective: string
|
||||
): Promise<Task[]> {
|
||||
const next_task_id = thisTaskId + 1
|
||||
const task_names = taskList.map((t) => t.task_name).join(', ')
|
||||
const response = await taskPrioritizationChain.predict({ task_names, next_task_id, objective })
|
||||
const newTasks = response.split('\n')
|
||||
const prioritizedTaskList: Task[] = []
|
||||
|
||||
for (const taskString of newTasks) {
|
||||
if (!taskString.trim()) {
|
||||
// eslint-disable-next-line no-continue
|
||||
continue
|
||||
}
|
||||
const taskParts = taskString.trim().split('. ', 2)
|
||||
if (taskParts.length === 2) {
|
||||
const task_id = parseInt(taskParts[0].trim(), 10)
|
||||
const task_name = taskParts[1].trim()
|
||||
prioritizedTaskList.push({ task_id, task_name })
|
||||
}
|
||||
}
|
||||
|
||||
return prioritizedTaskList
|
||||
}
|
||||
|
||||
export async function get_top_tasks(vectorStore: VectorStore, query: string, k: number): Promise<string[]> {
|
||||
const docs = await vectorStore.similaritySearch(query, k)
|
||||
let returnDocs: string[] = []
|
||||
for (const doc of docs) {
|
||||
returnDocs.push(doc.metadata.task)
|
||||
}
|
||||
return returnDocs
|
||||
}
|
||||
|
||||
async function executeTask(vectorStore: VectorStore, executionChain: LLMChain, objective: string, task: string, k = 5): Promise<string> {
|
||||
const context = await get_top_tasks(vectorStore, objective, k)
|
||||
return executionChain.predict({ objective, context, task })
|
||||
}
|
||||
|
||||
export class BabyAGI {
|
||||
taskList: Array<Task> = []
|
||||
|
||||
taskCreationChain: TaskCreationChain
|
||||
|
||||
taskPrioritizationChain: TaskPrioritizationChain
|
||||
|
||||
executionChain: ExecutionChain
|
||||
|
||||
taskIdCounter = 1
|
||||
|
||||
vectorStore: VectorStore
|
||||
|
||||
maxIterations = 3
|
||||
|
||||
topK = 4
|
||||
|
||||
constructor(
|
||||
taskCreationChain: TaskCreationChain,
|
||||
taskPrioritizationChain: TaskPrioritizationChain,
|
||||
executionChain: ExecutionChain,
|
||||
vectorStore: VectorStore,
|
||||
maxIterations: number,
|
||||
topK: number
|
||||
) {
|
||||
this.taskCreationChain = taskCreationChain
|
||||
this.taskPrioritizationChain = taskPrioritizationChain
|
||||
this.executionChain = executionChain
|
||||
this.vectorStore = vectorStore
|
||||
this.maxIterations = maxIterations
|
||||
this.topK = topK
|
||||
}
|
||||
|
||||
addTask(task: Task) {
|
||||
this.taskList.push(task)
|
||||
}
|
||||
|
||||
printTaskList() {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('\x1b[95m\x1b[1m\n*****TASK LIST*****\n\x1b[0m\x1b[0m')
|
||||
// eslint-disable-next-line no-console
|
||||
this.taskList.forEach((t) => console.log(`${t.task_id}: ${t.task_name}`))
|
||||
}
|
||||
|
||||
printNextTask(task: Task) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('\x1b[92m\x1b[1m\n*****NEXT TASK*****\n\x1b[0m\x1b[0m')
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`${task.task_id}: ${task.task_name}`)
|
||||
}
|
||||
|
||||
printTaskResult(result: string) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('\x1b[93m\x1b[1m\n*****TASK RESULT*****\n\x1b[0m\x1b[0m')
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(result)
|
||||
}
|
||||
|
||||
getInputKeys(): string[] {
|
||||
return ['objective']
|
||||
}
|
||||
|
||||
getOutputKeys(): string[] {
|
||||
return []
|
||||
}
|
||||
|
||||
async call(inputs: Record<string, any>): Promise<string> {
|
||||
const { objective } = inputs
|
||||
const firstTask = inputs.first_task || 'Make a todo list'
|
||||
this.addTask({ task_id: 1, task_name: firstTask })
|
||||
let numIters = 0
|
||||
let loop = true
|
||||
let finalResult = ''
|
||||
|
||||
while (loop) {
|
||||
if (this.taskList.length) {
|
||||
this.printTaskList()
|
||||
|
||||
// Step 1: Pull the first task
|
||||
const task = this.taskList.shift()
|
||||
if (!task) break
|
||||
this.printNextTask(task)
|
||||
|
||||
// Step 2: Execute the task
|
||||
const result = await executeTask(this.vectorStore, this.executionChain, objective, task.task_name, this.topK)
|
||||
const thisTaskId = task.task_id
|
||||
finalResult = result
|
||||
this.printTaskResult(result)
|
||||
|
||||
// Step 3: Store the result in Pinecone
|
||||
const docs = new Document({ pageContent: result, metadata: { task: task.task_name } })
|
||||
this.vectorStore.addDocuments([docs])
|
||||
|
||||
// Step 4: Create new tasks and reprioritize task list
|
||||
const newTasks = await getNextTask(
|
||||
this.taskCreationChain,
|
||||
result,
|
||||
task.task_name,
|
||||
this.taskList.map((t) => t.task_name),
|
||||
objective
|
||||
)
|
||||
newTasks.forEach((newTask) => {
|
||||
this.taskIdCounter += 1
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
newTask.task_id = this.taskIdCounter
|
||||
this.addTask(newTask)
|
||||
})
|
||||
this.taskList = await prioritizeTasks(this.taskPrioritizationChain, thisTaskId, this.taskList, objective)
|
||||
}
|
||||
|
||||
numIters += 1
|
||||
if (this.maxIterations !== null && numIters === this.maxIterations) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('\x1b[91m\x1b[1m\n*****TASK ENDING*****\n\x1b[0m\x1b[0m')
|
||||
loop = false
|
||||
this.taskList = []
|
||||
}
|
||||
}
|
||||
|
||||
return finalResult
|
||||
}
|
||||
|
||||
static fromLLM(llm: BaseChatModel, vectorstore: VectorStore, maxIterations = 3, topK = 4): BabyAGI {
|
||||
const taskCreationChain = TaskCreationChain.from_llm(llm)
|
||||
const taskPrioritizationChain = TaskPrioritizationChain.from_llm(llm)
|
||||
const executionChain = ExecutionChain.from_llm(llm)
|
||||
return new BabyAGI(taskCreationChain, taskPrioritizationChain, executionChain, vectorstore, maxIterations, topK)
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { BaseLanguageModel } from '@langchain/core/language_models/base'
|
||||
import { AgentExecutor } from 'langchain/agents'
|
||||
import { LLMChain } from 'langchain/chains'
|
||||
import { AgentExecutor } from '@langchain/classic/agents'
|
||||
import { LLMChain } from '@langchain/classic/chains'
|
||||
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
|
||||
import { ICommonObject, INode, INodeData, INodeParams, IServerSideEventStreamer, PromptTemplate } from '../../../src/Interface'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { flatten } from 'lodash'
|
||||
import type { BaseLanguageModel } from '@langchain/core/language_models/base'
|
||||
import { Tool } from '@langchain/core/tools'
|
||||
import { BaseChatModel } from '@langchain/core/language_models/chat_models'
|
||||
import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages'
|
||||
@@ -6,7 +7,7 @@ import { ChainValues } from '@langchain/core/utils/types'
|
||||
import { AgentStep } from '@langchain/core/agents'
|
||||
import { renderTemplate, MessagesPlaceholder, HumanMessagePromptTemplate, PromptTemplate } from '@langchain/core/prompts'
|
||||
import { RunnableSequence } from '@langchain/core/runnables'
|
||||
import { ChatConversationalAgent } from 'langchain/agents'
|
||||
import { ChatConversationalAgent } from '@langchain/classic/agents'
|
||||
import { getBaseClasses, transformBracesWithColon } from '../../../src/utils'
|
||||
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
|
||||
import {
|
||||
@@ -52,6 +53,8 @@ class ConversationalAgent_Agents implements INode {
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
sessionId?: string
|
||||
deprecateMessage: string
|
||||
badge: string
|
||||
|
||||
constructor(fields?: { sessionId?: string }) {
|
||||
this.label = 'Conversational Agent'
|
||||
@@ -61,6 +64,9 @@ class ConversationalAgent_Agents implements INode {
|
||||
this.category = 'Agents'
|
||||
this.icon = 'agent.svg'
|
||||
this.description = 'Conversational agent for a chat model. It will utilize chat specific prompts'
|
||||
this.badge = 'DEPRECATING'
|
||||
this.deprecateMessage =
|
||||
'Conversational agent is deprecated and will be removed in a future release. Use Agent from AgentFlow instead.'
|
||||
this.baseClasses = [this.type, ...getBaseClasses(AgentExecutor)]
|
||||
this.inputs = [
|
||||
{
|
||||
@@ -264,7 +270,7 @@ const prepareAgent = async (
|
||||
}
|
||||
|
||||
/** Bind a stop token to the model */
|
||||
const modelWithStop = model.bind({
|
||||
const modelWithStop = (model as BaseLanguageModel).withConfig({
|
||||
stop: ['\nObservation']
|
||||
})
|
||||
|
||||
|
||||
+10
-5
@@ -4,9 +4,15 @@ import { ChainValues } from '@langchain/core/utils/types'
|
||||
import { RunnableSequence } from '@langchain/core/runnables'
|
||||
import { BaseChatModel } from '@langchain/core/language_models/chat_models'
|
||||
import { ChatPromptTemplate, MessagesPlaceholder, HumanMessagePromptTemplate, PromptTemplate } from '@langchain/core/prompts'
|
||||
import { formatToOpenAIToolMessages } from 'langchain/agents/format_scratchpad/openai_tools'
|
||||
import { getBaseClasses, transformBracesWithColon, convertChatHistoryToText, convertBaseMessagetoIMessage } from '../../../src/utils'
|
||||
import { type ToolsAgentStep } from 'langchain/agents/openai/output_parser'
|
||||
import { formatToOpenAIToolMessages } from '@langchain/classic/agents/format_scratchpad/openai_tools'
|
||||
import {
|
||||
getBaseClasses,
|
||||
transformBracesWithColon,
|
||||
convertChatHistoryToText,
|
||||
convertBaseMessagetoIMessage,
|
||||
createTextOnlyOutputParser
|
||||
} from '../../../src/utils'
|
||||
import { type ToolsAgentStep } from '@langchain/classic/agents/openai/output_parser'
|
||||
import {
|
||||
FlowiseMemory,
|
||||
ICommonObject,
|
||||
@@ -25,7 +31,6 @@ import type { Document } from '@langchain/core/documents'
|
||||
import { BaseRetriever } from '@langchain/core/retrievers'
|
||||
import { RESPONSE_TEMPLATE, REPHRASE_TEMPLATE } from '../../chains/ConversationalRetrievalQAChain/prompts'
|
||||
import { addImagesToMessages, llmSupportsVision } from '../../../src/multiModalUtils'
|
||||
import { StringOutputParser } from '@langchain/core/output_parsers'
|
||||
import { Tool } from '@langchain/core/tools'
|
||||
|
||||
class ConversationalRetrievalToolAgent_Agents implements INode {
|
||||
@@ -323,7 +328,7 @@ const prepareAgent = async (
|
||||
// Always rephrase to normalize/expand user queries for better retrieval
|
||||
try {
|
||||
const CONDENSE_QUESTION_PROMPT = PromptTemplate.fromTemplate(rephrasePrompt)
|
||||
const condenseQuestionChain = RunnableSequence.from([CONDENSE_QUESTION_PROMPT, rephraseModel, new StringOutputParser()])
|
||||
const condenseQuestionChain = RunnableSequence.from([CONDENSE_QUESTION_PROMPT, rephraseModel, createTextOnlyOutputParser()])
|
||||
const res = await condenseQuestionChain.invoke({
|
||||
question: input,
|
||||
chat_history: chatHistoryString
|
||||
|
||||
@@ -33,6 +33,8 @@ class OpenAIAssistant_Agents implements INode {
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
badge: string
|
||||
deprecateMessage: string
|
||||
|
||||
constructor() {
|
||||
this.label = 'OpenAI Assistant'
|
||||
@@ -42,6 +44,8 @@ class OpenAIAssistant_Agents implements INode {
|
||||
this.category = 'Agents'
|
||||
this.icon = 'assistant.svg'
|
||||
this.description = `An agent that uses OpenAI Assistant API to pick the tool and args to call`
|
||||
this.badge = 'DEPRECATING'
|
||||
this.deprecateMessage = 'OpenAI Assistant is deprecated and will be removed in a future release. Use Custom Assistant instead.'
|
||||
this.baseClasses = [this.type]
|
||||
this.inputs = [
|
||||
{
|
||||
@@ -173,7 +177,7 @@ class OpenAIAssistant_Agents implements INode {
|
||||
options.logger.info(`[${orgId}]: Clearing OpenAI Thread ${sessionId}`)
|
||||
try {
|
||||
if (sessionId && sessionId.startsWith('thread_')) {
|
||||
await openai.beta.threads.del(sessionId)
|
||||
await openai.beta.threads.delete(sessionId)
|
||||
options.logger.info(`[${orgId}]: Successfully cleared OpenAI Thread ${sessionId}`)
|
||||
} else {
|
||||
options.logger.error(`[${orgId}]: Error clearing OpenAI Thread ${sessionId}`)
|
||||
@@ -587,7 +591,7 @@ class OpenAIAssistant_Agents implements INode {
|
||||
}
|
||||
|
||||
try {
|
||||
await handleToolSubmission({
|
||||
const result = await handleToolSubmission({
|
||||
openai,
|
||||
threadId,
|
||||
runThreadId,
|
||||
@@ -604,9 +608,11 @@ class OpenAIAssistant_Agents implements INode {
|
||||
text,
|
||||
isStreamingStarted
|
||||
})
|
||||
text = result.text
|
||||
isStreamingStarted = result.isStreamingStarted
|
||||
} catch (error) {
|
||||
console.error('Error submitting tool outputs:', error)
|
||||
await openai.beta.threads.runs.cancel(threadId, runThreadId)
|
||||
await openai.beta.threads.runs.cancel(runThreadId, { thread_id: threadId })
|
||||
|
||||
const errMsg = `Error submitting tool outputs. Thread ID: ${threadId}. Run ID: ${runThreadId}`
|
||||
|
||||
@@ -631,7 +637,6 @@ class OpenAIAssistant_Agents implements INode {
|
||||
|
||||
await analyticHandlers.onLLMEnd(llmIds, llmOutput)
|
||||
await analyticHandlers.onChainEnd(parentIds, messageData, true)
|
||||
|
||||
return {
|
||||
text,
|
||||
usedTools,
|
||||
@@ -651,7 +656,7 @@ class OpenAIAssistant_Agents implements INode {
|
||||
|
||||
const timeout = setInterval(async () => {
|
||||
try {
|
||||
const run = await openai.beta.threads.runs.retrieve(threadId, runId)
|
||||
const run = await openai.beta.threads.runs.retrieve(runId, { thread_id: threadId })
|
||||
const state = run.status
|
||||
|
||||
if (state === 'completed') {
|
||||
@@ -715,17 +720,18 @@ class OpenAIAssistant_Agents implements INode {
|
||||
}
|
||||
}
|
||||
|
||||
const newRun = await openai.beta.threads.runs.retrieve(threadId, runId)
|
||||
const newRun = await openai.beta.threads.runs.retrieve(runId, { thread_id: threadId })
|
||||
const newStatus = newRun?.status
|
||||
|
||||
try {
|
||||
if (submitToolOutputs.length && newStatus === 'requires_action') {
|
||||
await openai.beta.threads.runs.submitToolOutputs(threadId, runId, {
|
||||
tool_outputs: submitToolOutputs
|
||||
await openai.beta.threads.runs.submitToolOutputs(runId, {
|
||||
tool_outputs: submitToolOutputs,
|
||||
thread_id: threadId
|
||||
})
|
||||
resolve(state)
|
||||
} else {
|
||||
await openai.beta.threads.runs.cancel(threadId, runId)
|
||||
await openai.beta.threads.runs.cancel(runId, { thread_id: threadId })
|
||||
resolve('requires_action_retry')
|
||||
}
|
||||
} catch (e) {
|
||||
@@ -1033,8 +1039,9 @@ async function handleToolSubmission(params: ToolSubmissionParams): Promise<ToolS
|
||||
let updatedText = params.text
|
||||
let updatedIsStreamingStarted = params.isStreamingStarted
|
||||
|
||||
const stream = openai.beta.threads.runs.submitToolOutputsStream(threadId, runThreadId, {
|
||||
tool_outputs: submitToolOutputs
|
||||
const stream = openai.beta.threads.runs.submitToolOutputsStream(runThreadId, {
|
||||
tool_outputs: submitToolOutputs,
|
||||
thread_id: threadId
|
||||
})
|
||||
|
||||
try {
|
||||
@@ -1136,7 +1143,7 @@ async function handleToolSubmission(params: ToolSubmissionParams): Promise<ToolS
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error submitting tool outputs:', error)
|
||||
await openai.beta.threads.runs.cancel(threadId, runThreadId)
|
||||
await openai.beta.threads.runs.cancel(runThreadId, { thread_id: threadId })
|
||||
|
||||
const errMsg = `Error submitting tool outputs. Thread ID: ${threadId}. Run ID: ${runThreadId}`
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { flatten } from 'lodash'
|
||||
import { AgentExecutor } from 'langchain/agents'
|
||||
import { AgentExecutor } from '@langchain/classic/agents'
|
||||
import { ChatPromptTemplate, HumanMessagePromptTemplate } from '@langchain/core/prompts'
|
||||
import { Tool } from '@langchain/core/tools'
|
||||
import type { PromptTemplate } from '@langchain/core/prompts'
|
||||
@@ -24,6 +24,8 @@ class ReActAgentChat_Agents implements INode {
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
sessionId?: string
|
||||
badge: string
|
||||
deprecateMessage: string
|
||||
|
||||
constructor(fields?: { sessionId?: string }) {
|
||||
this.label = 'ReAct Agent for Chat Models'
|
||||
@@ -33,6 +35,9 @@ class ReActAgentChat_Agents implements INode {
|
||||
this.category = 'Agents'
|
||||
this.icon = 'agent.svg'
|
||||
this.description = 'Agent that uses the ReAct logic to decide what action to take, optimized to be used with Chat Models'
|
||||
this.badge = 'DEPRECATING'
|
||||
this.deprecateMessage =
|
||||
'ReAct Agent for Chat Models is deprecated and will be removed in a future release. Use Agent from AgentFlow instead.'
|
||||
this.baseClasses = [this.type, ...getBaseClasses(AgentExecutor)]
|
||||
this.inputs = [
|
||||
{
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { flatten } from 'lodash'
|
||||
import { AgentExecutor } from 'langchain/agents'
|
||||
import { AgentExecutor } from '@langchain/classic/agents'
|
||||
import { pull } from 'langchain/hub'
|
||||
import { Tool } from '@langchain/core/tools'
|
||||
import type { PromptTemplate } from '@langchain/core/prompts'
|
||||
@@ -21,6 +21,8 @@ class ReActAgentLLM_Agents implements INode {
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
badge: string
|
||||
deprecateMessage: string
|
||||
|
||||
constructor() {
|
||||
this.label = 'ReAct Agent for LLMs'
|
||||
@@ -30,6 +32,9 @@ class ReActAgentLLM_Agents implements INode {
|
||||
this.category = 'Agents'
|
||||
this.icon = 'agent.svg'
|
||||
this.description = 'Agent that uses the ReAct logic to decide what action to take, optimized to be used with LLMs'
|
||||
this.badge = 'DEPRECATING'
|
||||
this.deprecateMessage =
|
||||
'ReAct Agent for LLMs is deprecated and will be removed in a future release. Use Agent from AgentFlow instead.'
|
||||
this.baseClasses = [this.type, ...getBaseClasses(AgentExecutor)]
|
||||
this.inputs = [
|
||||
{
|
||||
|
||||
@@ -5,8 +5,8 @@ import { ChainValues } from '@langchain/core/utils/types'
|
||||
import { RunnableSequence } from '@langchain/core/runnables'
|
||||
import { BaseChatModel } from '@langchain/core/language_models/chat_models'
|
||||
import { ChatPromptTemplate, MessagesPlaceholder, HumanMessagePromptTemplate, PromptTemplate } from '@langchain/core/prompts'
|
||||
import { formatToOpenAIToolMessages } from 'langchain/agents/format_scratchpad/openai_tools'
|
||||
import { type ToolsAgentStep } from 'langchain/agents/openai/output_parser'
|
||||
import { formatToOpenAIToolMessages } from '@langchain/classic/agents/format_scratchpad/openai_tools'
|
||||
import { type ToolsAgentStep } from '@langchain/classic/agents/openai/output_parser'
|
||||
import {
|
||||
extractOutputFromArray,
|
||||
getBaseClasses,
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import { flatten } from 'lodash'
|
||||
import type { BaseLanguageModel } from '@langchain/core/language_models/base'
|
||||
import { ChainValues } from '@langchain/core/utils/types'
|
||||
import { AgentStep } from '@langchain/core/agents'
|
||||
import { BaseChatModel } from '@langchain/core/language_models/chat_models'
|
||||
import { RunnableSequence } from '@langchain/core/runnables'
|
||||
import { Tool } from '@langchain/core/tools'
|
||||
import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder } from '@langchain/core/prompts'
|
||||
import { formatLogToMessage } from 'langchain/agents/format_scratchpad/log_to_message'
|
||||
import { formatLogToMessage } from '@langchain/classic/agents/format_scratchpad/log_to_message'
|
||||
import { getBaseClasses, transformBracesWithColon } from '../../../src/utils'
|
||||
import {
|
||||
FlowiseMemory,
|
||||
@@ -246,7 +247,9 @@ const prepareAgent = async (
|
||||
throw new Error(`Provided prompt is missing required input variables: ${JSON.stringify(missingVariables)}`)
|
||||
}
|
||||
|
||||
const llmWithStop = model.bind({ stop: ['</tool_input>', '</final_answer>'] })
|
||||
const llmWithStop = (model as BaseLanguageModel).withConfig({
|
||||
stop: ['</tool_input>', '</final_answer>']
|
||||
})
|
||||
|
||||
const messages = (await memory.getChatMessages(flowObj.sessionId, false, prependMessages)) as IMessage[]
|
||||
let chatHistoryMsgTxt = ''
|
||||
|
||||
Vendored
-44
@@ -1,44 +0,0 @@
|
||||
import type { CachedContentBase, CachedContent, Content } from '@google/generative-ai'
|
||||
import { GoogleAICacheManager as GoogleAICacheManagerBase } from '@google/generative-ai/server'
|
||||
import hash from 'object-hash'
|
||||
|
||||
type CacheContentOptions = Omit<CachedContentBase, 'contents'> & { contents?: Content[] }
|
||||
|
||||
export class GoogleAICacheManager extends GoogleAICacheManagerBase {
|
||||
private ttlSeconds: number
|
||||
private cachedContents: Map<string, CachedContent> = new Map()
|
||||
|
||||
setTtlSeconds(ttlSeconds: number) {
|
||||
this.ttlSeconds = ttlSeconds
|
||||
}
|
||||
|
||||
async lookup(options: CacheContentOptions): Promise<CachedContent | undefined> {
|
||||
const { model, tools, contents } = options
|
||||
if (!contents?.length) {
|
||||
return undefined
|
||||
}
|
||||
const hashKey = hash({
|
||||
model,
|
||||
tools,
|
||||
contents
|
||||
})
|
||||
if (this.cachedContents.has(hashKey)) {
|
||||
return this.cachedContents.get(hashKey)
|
||||
}
|
||||
const { cachedContents } = await this.list()
|
||||
const cachedContent = (cachedContents ?? []).find((cache) => cache.displayName === hashKey)
|
||||
if (cachedContent) {
|
||||
this.cachedContents.set(hashKey, cachedContent)
|
||||
return cachedContent
|
||||
}
|
||||
const res = await this.create({
|
||||
...(options as CachedContentBase),
|
||||
displayName: hashKey,
|
||||
ttlSeconds: this.ttlSeconds
|
||||
})
|
||||
this.cachedContents.set(hashKey, res)
|
||||
return res
|
||||
}
|
||||
}
|
||||
|
||||
export default GoogleAICacheManager
|
||||
-34
@@ -1,34 +0,0 @@
|
||||
<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<mask id="mask0_42_15021" style="mask-type:alpha" maskUnits="userSpaceOnUse" x="4" y="4" width="24" height="24">
|
||||
<path d="M16.9976 4.93059C16.9611 4.40651 16.5253 4 16 4C15.4747 4 15.0389 4.40651 15.0024 4.93059L14.951 5.66926C14.6048 10.645 10.645 14.6048 5.66926 14.951L4.93059 15.0024C4.40651 15.0389 4 15.4747 4 16C4 16.5253 4.40651 16.9611 4.93059 16.9976L5.66926 17.049C10.645 17.3952 14.6048 21.355 14.951 26.3307L15.0024 27.0694C15.0389 27.5935 15.4747 28 16 28C16.5253 28 16.9611 27.5935 16.9976 27.0694L17.049 26.3307C17.3952 21.355 21.355 17.3952 26.3307 17.049L27.0694 16.9976C27.5935 16.9611 28 16.5253 28 16C28 15.4747 27.5935 15.0389 27.0694 15.0024L26.3307 14.951C21.355 14.6048 17.3952 10.645 17.049 5.66926L16.9976 4.93059Z" fill="black"/>
|
||||
</mask>
|
||||
<g mask="url(#mask0_42_15021)">
|
||||
<path d="M16.9976 4.93059C16.9611 4.40651 16.5253 4 16 4C15.4747 4 15.0389 4.40651 15.0024 4.93059L14.951 5.66926C14.6048 10.645 10.645 14.6048 5.66926 14.951L4.93059 15.0024C4.40651 15.0389 4 15.4747 4 16C4 16.5253 4.40651 16.9611 4.93059 16.9976L5.66926 17.049C10.645 17.3952 14.6048 21.355 14.951 26.3307L15.0024 27.0694C15.0389 27.5935 15.4747 28 16 28C16.5253 28 16.9611 27.5935 16.9976 27.0694L17.049 26.3307C17.3952 21.355 21.355 17.3952 26.3307 17.049L27.0694 16.9976C27.5935 16.9611 28 16.5253 28 16C28 15.4747 27.5935 15.0389 27.0694 15.0024L26.3307 14.951C21.355 14.6048 17.3952 10.645 17.049 5.66926L16.9976 4.93059Z" fill="white"/>
|
||||
<g filter="url(#filter0_f_42_15021)">
|
||||
<circle cx="10.4616" cy="13.2307" r="8.30769" fill="#77B6E5"/>
|
||||
</g>
|
||||
<g filter="url(#filter1_f_42_15021)">
|
||||
<circle cx="16" cy="22.4615" r="8.30769" fill="#1E90C9"/>
|
||||
</g>
|
||||
<g filter="url(#filter2_f_42_15021)">
|
||||
<ellipse cx="21.5385" cy="10.4615" rx="10.1538" ry="8.30769" fill="#E9E5DF"/>
|
||||
</g>
|
||||
</g>
|
||||
<defs>
|
||||
<filter id="filter0_f_42_15021" x="-7.84613" y="-5.07697" width="36.6154" height="36.6154" filterUnits="userSpaceOnUse" color-interpolation-filters="sRGB">
|
||||
<feFlood flood-opacity="0" result="BackgroundImageFix"/>
|
||||
<feBlend mode="normal" in="SourceGraphic" in2="BackgroundImageFix" result="shape"/>
|
||||
<feGaussianBlur stdDeviation="5" result="effect1_foregroundBlur_42_15021"/>
|
||||
</filter>
|
||||
<filter id="filter1_f_42_15021" x="-0.307678" y="6.15381" width="32.6154" height="32.6154" filterUnits="userSpaceOnUse" color-interpolation-filters="sRGB">
|
||||
<feFlood flood-opacity="0" result="BackgroundImageFix"/>
|
||||
<feBlend mode="normal" in="SourceGraphic" in2="BackgroundImageFix" result="shape"/>
|
||||
<feGaussianBlur stdDeviation="4" result="effect1_foregroundBlur_42_15021"/>
|
||||
</filter>
|
||||
<filter id="filter2_f_42_15021" x="3.38464" y="-5.84619" width="36.3077" height="32.6154" filterUnits="userSpaceOnUse" color-interpolation-filters="sRGB">
|
||||
<feFlood flood-opacity="0" result="BackgroundImageFix"/>
|
||||
<feBlend mode="normal" in="SourceGraphic" in2="BackgroundImageFix" result="shape"/>
|
||||
<feGaussianBlur stdDeviation="4" result="effect1_foregroundBlur_42_15021"/>
|
||||
</filter>
|
||||
</defs>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 3.0 KiB |
Vendored
-53
@@ -1,53 +0,0 @@
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam, ICommonObject, INode, INodeData, INodeParams } from '../../../src'
|
||||
import FlowiseGoogleAICacheManager from './FlowiseGoogleAICacheManager'
|
||||
|
||||
class GoogleGenerativeAIContextCache implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
|
||||
constructor() {
|
||||
this.label = 'Google GenAI Context Cache'
|
||||
this.name = 'googleGenerativeAIContextCache'
|
||||
this.version = 1.0
|
||||
this.type = 'GoogleAICacheManager'
|
||||
this.description = 'Large context cache for Google Gemini large language models'
|
||||
this.icon = 'GoogleGemini.svg'
|
||||
this.category = 'Cache'
|
||||
this.baseClasses = [this.type, ...getBaseClasses(FlowiseGoogleAICacheManager)]
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'TTL',
|
||||
name: 'ttl',
|
||||
type: 'number',
|
||||
default: 60 * 60 * 24 * 30
|
||||
}
|
||||
]
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['googleGenerativeAI'],
|
||||
optional: false,
|
||||
description: 'Google Generative AI credential.'
|
||||
}
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const ttl = nodeData.inputs?.ttl as number
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const apiKey = getCredentialParam('googleGenerativeAPIKey', credentialData, nodeData)
|
||||
const manager = new FlowiseGoogleAICacheManager(apiKey)
|
||||
manager.setTtlSeconds(ttl)
|
||||
return manager
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: GoogleGenerativeAIContextCache }
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
import { BaseStore } from '@langchain/core/stores'
|
||||
import { CacheBackedEmbeddings } from 'langchain/embeddings/cache_backed'
|
||||
import { CacheBackedEmbeddings } from '@langchain/classic/embeddings/cache_backed'
|
||||
import { getBaseClasses, ICommonObject, INode, INodeData, INodeParams } from '../../../src'
|
||||
|
||||
class InMemoryEmbeddingCache implements INode {
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import { Redis } from 'ioredis'
|
||||
import { RedisByteStore } from '@langchain/community/storage/ioredis'
|
||||
import { Embeddings, EmbeddingsInterface } from '@langchain/core/embeddings'
|
||||
import { CacheBackedEmbeddingsFields } from 'langchain/embeddings/cache_backed'
|
||||
import { CacheBackedEmbeddingsFields } from '@langchain/classic/embeddings/cache_backed'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam, ICommonObject, INode, INodeData, INodeParams } from '../../../src'
|
||||
import { BaseStore } from '@langchain/core/stores'
|
||||
import { insecureHash } from '@langchain/core/utils/hash'
|
||||
import { sha256 } from '@langchain/core/utils/hash'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
|
||||
class RedisEmbeddingsCache implements INode {
|
||||
@@ -168,7 +168,7 @@ class CacheBackedEmbeddings extends Embeddings {
|
||||
const decoder = new TextDecoder()
|
||||
const encoderBackedStore = new EncoderBackedStore<string, number[], Uint8Array>({
|
||||
store: documentEmbeddingStore,
|
||||
keyEncoder: (key) => (options?.namespace ?? '') + insecureHash(key),
|
||||
keyEncoder: (key) => (options?.namespace ?? '') + sha256(key),
|
||||
valueSerializer: (value) => encoder.encode(JSON.stringify(value)),
|
||||
valueDeserializer: (serializedValue) => JSON.parse(decoder.decode(serializedValue))
|
||||
})
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { BaseLanguageModel } from '@langchain/core/language_models/base'
|
||||
import { PromptTemplate } from '@langchain/core/prompts'
|
||||
import { APIChain } from 'langchain/chains'
|
||||
import { APIChain } from '@langchain/classic/chains'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { ICommonObject, INode, INodeData, INodeParams, IServerSideEventStreamer } from '../../../src/Interface'
|
||||
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { APIChain, createOpenAPIChain } from 'langchain/chains'
|
||||
import { APIChain, createOpenAPIChain } from '@langchain/classic/chains'
|
||||
import { ICommonObject, INode, INodeData, INodeParams, IServerSideEventStreamer } from '../../../src/Interface'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
|
||||
|
||||
@@ -2,7 +2,7 @@ import { CallbackManagerForChainRun } from '@langchain/core/callbacks/manager'
|
||||
import { BaseLanguageModel } from '@langchain/core/language_models/base'
|
||||
import { BasePromptTemplate, PromptTemplate } from '@langchain/core/prompts'
|
||||
import { ChainValues } from '@langchain/core/utils/types'
|
||||
import { BaseChain, ChainInputs, LLMChain, SerializedAPIChain } from 'langchain/chains'
|
||||
import { BaseChain, ChainInputs, LLMChain, SerializedAPIChain } from '@langchain/classic/chains'
|
||||
import { secureFetch } from '../../../src'
|
||||
|
||||
export const API_URL_RAW_PROMPT_TEMPLATE = `You are given the below API Documentation:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { ConversationChain } from 'langchain/chains'
|
||||
import { ConversationChain } from '@langchain/classic/chains'
|
||||
import {
|
||||
ChatPromptTemplate,
|
||||
HumanMessagePromptTemplate,
|
||||
@@ -8,7 +8,6 @@ import {
|
||||
PromptTemplate
|
||||
} from '@langchain/core/prompts'
|
||||
import { RunnableSequence } from '@langchain/core/runnables'
|
||||
import { StringOutputParser } from '@langchain/core/output_parsers'
|
||||
import { BaseChatModel } from '@langchain/core/language_models/chat_models'
|
||||
import { HumanMessage } from '@langchain/core/messages'
|
||||
import { ConsoleCallbackHandler as LCConsoleCallbackHandler } from '@langchain/core/tracers/console'
|
||||
@@ -27,7 +26,7 @@ import {
|
||||
IServerSideEventStreamer
|
||||
} from '../../../src/Interface'
|
||||
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
|
||||
import { getBaseClasses, handleEscapeCharacters, transformBracesWithColon } from '../../../src/utils'
|
||||
import { getBaseClasses, handleEscapeCharacters, transformBracesWithColon, createTextOnlyOutputParser } from '../../../src/utils'
|
||||
|
||||
let systemMessage = `The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.`
|
||||
const inputKey = 'input'
|
||||
@@ -268,7 +267,7 @@ const prepareChain = async (nodeData: INodeData, options: ICommonObject, session
|
||||
},
|
||||
prepareChatPrompt(nodeData, messageContent),
|
||||
model,
|
||||
new StringOutputParser()
|
||||
createTextOnlyOutputParser()
|
||||
])
|
||||
|
||||
return conversationChain
|
||||
|
||||
+5
-6
@@ -8,11 +8,10 @@ import { BaseMessage, HumanMessage, AIMessage } from '@langchain/core/messages'
|
||||
import { ConsoleCallbackHandler as LCConsoleCallbackHandler } from '@langchain/core/tracers/console'
|
||||
import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
|
||||
import { formatResponse } from '../../outputparsers/OutputParserHelpers'
|
||||
import { StringOutputParser } from '@langchain/core/output_parsers'
|
||||
import type { Document } from '@langchain/core/documents'
|
||||
import { BufferMemoryInput } from 'langchain/memory'
|
||||
import { ConversationalRetrievalQAChain } from 'langchain/chains'
|
||||
import { getBaseClasses, mapChatMessageToBaseMessage } from '../../../src/utils'
|
||||
import { BufferMemoryInput } from '@langchain/classic/memory'
|
||||
import { ConversationalRetrievalQAChain } from '@langchain/classic/chains'
|
||||
import { getBaseClasses, mapChatMessageToBaseMessage, createTextOnlyOutputParser } from '../../../src/utils'
|
||||
import { ConsoleCallbackHandler, additionalCallbacks } from '../../../src/handler'
|
||||
import {
|
||||
FlowiseMemory,
|
||||
@@ -308,7 +307,7 @@ const createRetrieverChain = (llm: BaseLanguageModel, retriever: Runnable, rephr
|
||||
// Small speed/accuracy optimization: no need to rephrase the first question
|
||||
// since there shouldn't be any meta-references to prior chat history
|
||||
const CONDENSE_QUESTION_PROMPT = PromptTemplate.fromTemplate(rephrasePrompt)
|
||||
const condenseQuestionChain = RunnableSequence.from([CONDENSE_QUESTION_PROMPT, llm, new StringOutputParser()]).withConfig({
|
||||
const condenseQuestionChain = RunnableSequence.from([CONDENSE_QUESTION_PROMPT, llm, createTextOnlyOutputParser()]).withConfig({
|
||||
runName: 'CondenseQuestion'
|
||||
})
|
||||
|
||||
@@ -385,7 +384,7 @@ const createChain = (
|
||||
['human', `{question}`]
|
||||
])
|
||||
|
||||
const responseSynthesizerChain = RunnableSequence.from([prompt, llm, new StringOutputParser()]).withConfig({
|
||||
const responseSynthesizerChain = RunnableSequence.from([prompt, llm, createTextOnlyOutputParser()]).withConfig({
|
||||
tags: ['GenerateResponse']
|
||||
})
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@ import { BaseLanguageModel, BaseLanguageModelCallOptions } from '@langchain/core
|
||||
import { BaseLLMOutputParser, BaseOutputParser } from '@langchain/core/output_parsers'
|
||||
import { HumanMessage } from '@langchain/core/messages'
|
||||
import { ChatPromptTemplate, FewShotPromptTemplate, HumanMessagePromptTemplate, PromptTemplate } from '@langchain/core/prompts'
|
||||
import { OutputFixingParser } from 'langchain/output_parsers'
|
||||
import { LLMChain } from 'langchain/chains'
|
||||
import { OutputFixingParser } from '@langchain/classic/output_parsers'
|
||||
import { LLMChain } from '@langchain/classic/chains'
|
||||
import {
|
||||
IVisionChatModal,
|
||||
ICommonObject,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { BaseLanguageModel } from '@langchain/core/language_models/base'
|
||||
import { MultiPromptChain } from 'langchain/chains'
|
||||
import { MultiPromptChain } from '@langchain/classic/chains'
|
||||
import { ICommonObject, INode, INodeData, INodeParams, IServerSideEventStreamer, PromptRetriever } from '../../../src/Interface'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { BaseLanguageModel } from '@langchain/core/language_models/base'
|
||||
import { MultiRetrievalQAChain } from 'langchain/chains'
|
||||
import { MultiRetrievalQAChain } from '@langchain/classic/chains'
|
||||
import { ICommonObject, INode, INodeData, INodeParams, IServerSideEventStreamer, VectorStoreRetriever } from '../../../src/Interface'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { BaseRetriever } from '@langchain/core/retrievers'
|
||||
import { BaseLanguageModel } from '@langchain/core/language_models/base'
|
||||
import { RetrievalQAChain } from 'langchain/chains'
|
||||
import { RetrievalQAChain } from '@langchain/classic/chains'
|
||||
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
|
||||
import { ICommonObject, INode, INodeData, INodeParams, IServerSideEventStreamer } from '../../../src/Interface'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
|
||||
@@ -2,8 +2,8 @@ import { DataSourceOptions } from 'typeorm/data-source'
|
||||
import { DataSource } from 'typeorm'
|
||||
import { BaseLanguageModel } from '@langchain/core/language_models/base'
|
||||
import { PromptTemplate, PromptTemplateInput } from '@langchain/core/prompts'
|
||||
import { SqlDatabaseChain, SqlDatabaseChainInput, DEFAULT_SQL_DATABASE_PROMPT } from 'langchain/chains/sql_db'
|
||||
import { SqlDatabase } from 'langchain/sql_db'
|
||||
import { SqlDatabaseChain, SqlDatabaseChainInput, DEFAULT_SQL_DATABASE_PROMPT } from '@langchain/classic/chains/sql_db'
|
||||
import { SqlDatabase } from '@langchain/classic/sql_db'
|
||||
import { ICommonObject, INode, INodeData, INodeParams, IServerSideEventStreamer } from '../../../src/Interface'
|
||||
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
|
||||
import { getBaseClasses, getInputVariables, transformBracesWithColon } from '../../../src/utils'
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import fetch from 'node-fetch'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { VectaraStore } from '@langchain/community/vectorstores/vectara'
|
||||
import { VectorDBQAChain } from 'langchain/chains'
|
||||
import { VectorDBQAChain } from '@langchain/classic/chains'
|
||||
import { INode, INodeData, INodeParams } from '../../../src/Interface'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { checkInputs, Moderation } from '../../moderation/Moderation'
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { BaseLanguageModel } from '@langchain/core/language_models/base'
|
||||
import { VectorStore } from '@langchain/core/vectorstores'
|
||||
import { VectorDBQAChain } from 'langchain/chains'
|
||||
import { VectorDBQAChain } from '@langchain/classic/chains'
|
||||
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
|
||||
import { ICommonObject, INode, INodeData, INodeParams, IServerSideEventStreamer } from '../../../src/Interface'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
|
||||
@@ -6,9 +6,6 @@ import { getAWSCredentialConfig } from '../../../src/awsToolsUtils'
|
||||
import { ChatBedrockConverseInput, ChatBedrockConverse } from '@langchain/aws'
|
||||
import { BedrockChat } from './FlowiseAWSChatBedrock'
|
||||
|
||||
/**
|
||||
* @author Michael Connor <[email protected]>
|
||||
*/
|
||||
class AWSChatBedrock_ChatModels implements INode {
|
||||
label: string
|
||||
name: string
|
||||
@@ -22,7 +19,7 @@ class AWSChatBedrock_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'AWS ChatBedrock'
|
||||
this.label = 'AWS Bedrock'
|
||||
this.name = 'awsChatBedrock'
|
||||
this.version = 6.1
|
||||
this.type = 'AWSChatBedrock'
|
||||
@@ -80,6 +77,15 @@ class AWSChatBedrock_ChatModels implements INode {
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Allow Image Uploads',
|
||||
name: 'allowImageUploads',
|
||||
type: 'boolean',
|
||||
description:
|
||||
'Allow image input. Refer to the <a href="https://docs.flowiseai.com/using-flowise/uploads#image" target="_blank">docs</a> for more details.',
|
||||
default: false,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Temperature',
|
||||
name: 'temperature',
|
||||
@@ -100,15 +106,6 @@ class AWSChatBedrock_ChatModels implements INode {
|
||||
additionalParams: true,
|
||||
default: 200
|
||||
},
|
||||
{
|
||||
label: 'Allow Image Uploads',
|
||||
name: 'allowImageUploads',
|
||||
type: 'boolean',
|
||||
description:
|
||||
'Allow image input. Refer to the <a href="https://docs.flowiseai.com/using-flowise/uploads#image" target="_blank">docs</a> for more details.',
|
||||
default: false,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Latency Optimized',
|
||||
name: 'latencyOptimized',
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { AzureOpenAIInput, AzureChatOpenAI as LangchainAzureChatOpenAI, ChatOpenAIFields } from '@langchain/openai'
|
||||
import { BaseCache } from '@langchain/core/caches'
|
||||
import { ICommonObject, IMultiModalOption, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam, isReasoningModelOpenAI } from '../../../src/utils'
|
||||
import { getModels, MODEL_TYPE } from '../../../src/modelLoader'
|
||||
import { AzureChatOpenAI } from './FlowiseAzureChatOpenAI'
|
||||
import { OpenAI as OpenAIClient } from 'openai'
|
||||
@@ -25,7 +25,7 @@ class AzureChatOpenAI_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Azure ChatOpenAI'
|
||||
this.label = 'Azure OpenAI'
|
||||
this.name = 'azureChatOpenAI'
|
||||
this.version = 7.1
|
||||
this.type = 'AzureChatOpenAI'
|
||||
@@ -51,7 +51,8 @@ class AzureChatOpenAI_ChatModels implements INode {
|
||||
label: 'Model Name',
|
||||
name: 'modelName',
|
||||
type: 'asyncOptions',
|
||||
loadMethod: 'listModels'
|
||||
loadMethod: 'listModels',
|
||||
freeSolo: true
|
||||
},
|
||||
{
|
||||
label: 'Temperature',
|
||||
@@ -77,52 +78,6 @@ class AzureChatOpenAI_ChatModels implements INode {
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Top Probability',
|
||||
name: 'topP',
|
||||
type: 'number',
|
||||
step: 0.1,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Frequency Penalty',
|
||||
name: 'frequencyPenalty',
|
||||
type: 'number',
|
||||
step: 0.1,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Presence Penalty',
|
||||
name: 'presencePenalty',
|
||||
type: 'number',
|
||||
step: 0.1,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Timeout',
|
||||
name: 'timeout',
|
||||
type: 'number',
|
||||
step: 1,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'BasePath',
|
||||
name: 'basepath',
|
||||
type: 'string',
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'BaseOptions',
|
||||
name: 'baseOptions',
|
||||
type: 'json',
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Allow Image Uploads',
|
||||
name: 'allowImageUploads',
|
||||
@@ -132,32 +87,9 @@ class AzureChatOpenAI_ChatModels implements INode {
|
||||
default: false,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Image Resolution',
|
||||
description: 'This parameter controls the resolution in which the model views the image.',
|
||||
name: 'imageResolution',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
label: 'Low',
|
||||
name: 'low'
|
||||
},
|
||||
{
|
||||
label: 'High',
|
||||
name: 'high'
|
||||
},
|
||||
{
|
||||
label: 'Auto',
|
||||
name: 'auto'
|
||||
}
|
||||
],
|
||||
default: 'low',
|
||||
optional: false,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Reasoning',
|
||||
description: 'Whether the model supports reasoning. Only applicable for reasoning models.',
|
||||
description: 'Whether the model supports reasoning. Only applicable for reasoning models (gpt-5 and o-series models only)',
|
||||
name: 'reasoning',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
@@ -166,7 +98,7 @@ class AzureChatOpenAI_ChatModels implements INode {
|
||||
},
|
||||
{
|
||||
label: 'Reasoning Effort',
|
||||
description: 'Constrains effort on reasoning for reasoning models. Only applicable for o1 and o3 models.',
|
||||
description: 'Constrains effort on reasoning. Only applicable for reasoning models (gpt-5 and o-series models only)',
|
||||
name: 'reasoningEffort',
|
||||
type: 'options',
|
||||
options: [
|
||||
@@ -211,6 +143,54 @@ class AzureChatOpenAI_ChatModels implements INode {
|
||||
show: {
|
||||
reasoning: true
|
||||
}
|
||||
},
|
||||
{
|
||||
label: 'Top Probability',
|
||||
name: 'topP',
|
||||
type: 'number',
|
||||
step: 0.1,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Frequency Penalty',
|
||||
name: 'frequencyPenalty',
|
||||
type: 'number',
|
||||
step: 0.1,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Presence Penalty',
|
||||
name: 'presencePenalty',
|
||||
type: 'number',
|
||||
step: 0.1,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Timeout',
|
||||
name: 'timeout',
|
||||
type: 'number',
|
||||
step: 1,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Base Path',
|
||||
name: 'basepath',
|
||||
type: 'string',
|
||||
optional: true,
|
||||
description: 'Override the default base URL for the API, e.g., "https://api.example.com/v2/',
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Base Options',
|
||||
name: 'baseOptions',
|
||||
type: 'json',
|
||||
optional: true,
|
||||
description: 'Default headers to include with every request to the API.',
|
||||
additionalParams: true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -244,7 +224,6 @@ class AzureChatOpenAI_ChatModels implements INode {
|
||||
const azureOpenAIApiVersion = getCredentialParam('azureOpenAIApiVersion', credentialData, nodeData)
|
||||
|
||||
const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean
|
||||
const imageResolution = nodeData.inputs?.imageResolution as string
|
||||
|
||||
const obj: ChatOpenAIFields & Partial<AzureOpenAIInput> = {
|
||||
temperature: parseFloat(temperature),
|
||||
@@ -256,7 +235,7 @@ class AzureChatOpenAI_ChatModels implements INode {
|
||||
streaming: streaming ?? true
|
||||
}
|
||||
|
||||
if (maxTokens) obj.maxTokens = parseInt(maxTokens, 10)
|
||||
if (maxTokens) obj.maxCompletionTokens = parseInt(maxTokens, 10)
|
||||
if (frequencyPenalty) obj.frequencyPenalty = parseFloat(frequencyPenalty)
|
||||
if (presencePenalty) obj.presencePenalty = parseFloat(presencePenalty)
|
||||
if (timeout) obj.timeout = parseInt(timeout, 10)
|
||||
@@ -273,7 +252,7 @@ class AzureChatOpenAI_ChatModels implements INode {
|
||||
console.error('Error parsing base options', exception)
|
||||
}
|
||||
}
|
||||
if (modelName.includes('o1') || modelName.includes('o3') || modelName.includes('gpt-5')) {
|
||||
if (isReasoningModelOpenAI(modelName)) {
|
||||
delete obj.temperature
|
||||
delete obj.stop
|
||||
const reasoning: OpenAIClient.Reasoning = {}
|
||||
@@ -284,17 +263,11 @@ class AzureChatOpenAI_ChatModels implements INode {
|
||||
reasoning.summary = reasoningSummary
|
||||
}
|
||||
obj.reasoning = reasoning
|
||||
|
||||
if (maxTokens) {
|
||||
delete obj.maxTokens
|
||||
obj.maxCompletionTokens = parseInt(maxTokens, 10)
|
||||
}
|
||||
}
|
||||
|
||||
const multiModalOption: IMultiModalOption = {
|
||||
image: {
|
||||
allowImageUploads: allowImageUploads ?? false,
|
||||
imageResolution
|
||||
allowImageUploads: allowImageUploads ?? false
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { AzureChatOpenAI as LangchainAzureChatOpenAI, OpenAIChatInput, AzureOpenAIInput, ClientOptions } from '@langchain/openai'
|
||||
import { AzureChatOpenAI as LangchainAzureChatOpenAI } from '@langchain/openai'
|
||||
import { IMultiModalOption, IVisionChatModal } from '../../../src'
|
||||
import { BaseChatModelParams } from '@langchain/core/language_models/chat_models'
|
||||
|
||||
export type AzureChatOpenAIConstructorFields = ConstructorParameters<typeof LangchainAzureChatOpenAI>[0]
|
||||
|
||||
export class AzureChatOpenAI extends LangchainAzureChatOpenAI implements IVisionChatModal {
|
||||
configuredModel: string
|
||||
@@ -9,18 +10,7 @@ export class AzureChatOpenAI extends LangchainAzureChatOpenAI implements IVision
|
||||
builtInTools: Record<string, any>[] = []
|
||||
id: string
|
||||
|
||||
constructor(
|
||||
id: string,
|
||||
fields?: Partial<OpenAIChatInput> &
|
||||
Partial<AzureOpenAIInput> & {
|
||||
openAIApiKey?: string
|
||||
openAIApiVersion?: string
|
||||
openAIBasePath?: string
|
||||
deploymentName?: string
|
||||
} & BaseChatModelParams & {
|
||||
configuration?: ClientOptions
|
||||
}
|
||||
) {
|
||||
constructor(id: string, fields?: AzureChatOpenAIConstructorFields) {
|
||||
super(fields)
|
||||
this.id = id
|
||||
this.configuredModel = fields?.modelName ?? ''
|
||||
|
||||
@@ -17,7 +17,7 @@ class ChatAlibabaTongyi_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatAlibabaTongyi'
|
||||
this.label = 'Alibaba Tongyi'
|
||||
this.name = 'chatAlibabaTongyi'
|
||||
this.version = 2.0
|
||||
this.type = 'ChatAlibabaTongyi'
|
||||
|
||||
@@ -19,7 +19,7 @@ class ChatAnthropic_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatAnthropic'
|
||||
this.label = 'Anthropic Claude'
|
||||
this.name = 'chatAnthropic'
|
||||
this.version = 8.0
|
||||
this.type = 'ChatAnthropic'
|
||||
@@ -45,7 +45,7 @@ class ChatAnthropic_ChatModels implements INode {
|
||||
name: 'modelName',
|
||||
type: 'asyncOptions',
|
||||
loadMethod: 'listModels',
|
||||
default: 'claude-3-haiku'
|
||||
default: 'claude-haiku-4-5'
|
||||
},
|
||||
{
|
||||
label: 'Temperature',
|
||||
@@ -63,6 +63,87 @@ class ChatAnthropic_ChatModels implements INode {
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Allow Image Uploads',
|
||||
name: 'allowImageUploads',
|
||||
type: 'boolean',
|
||||
description:
|
||||
'Allow image input. Refer to the <a href="https://docs.flowiseai.com/using-flowise/uploads#image" target="_blank">docs</a> for more details.',
|
||||
default: false,
|
||||
optional: true
|
||||
},
|
||||
/* The manual thinking: {type: "enabled", budget_tokens: N} configuration is deprecated on Opus 4.6 and will be removed in a future model release */
|
||||
{
|
||||
label: 'Extended Thinking',
|
||||
name: 'extendedThinking',
|
||||
type: 'boolean',
|
||||
description: 'Enable extended thinking for reasoning model such as Claude Sonnet 3.7 and Claude 4',
|
||||
optional: true,
|
||||
additionalParams: true,
|
||||
hide: {
|
||||
modelName: ['claude-opus-4-6', 'claude-sonnet-4-6']
|
||||
}
|
||||
},
|
||||
{
|
||||
label: 'Budget Tokens',
|
||||
name: 'budgetTokens',
|
||||
type: 'number',
|
||||
step: 1,
|
||||
default: '1024',
|
||||
description: 'Maximum number of tokens Claude is allowed use for its internal reasoning process',
|
||||
optional: true,
|
||||
additionalParams: true,
|
||||
show: {
|
||||
extendedThinking: true
|
||||
},
|
||||
hide: {
|
||||
modelName: ['claude-opus-4-6', 'claude-sonnet-4-6']
|
||||
}
|
||||
},
|
||||
{
|
||||
label: 'Adaptive Thinking',
|
||||
description:
|
||||
'Claude evaluates the complexity of each request and determines whether and how much to use extended thinking.',
|
||||
name: 'adaptiveThinking',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
optional: true,
|
||||
additionalParams: true,
|
||||
show: {
|
||||
modelName: ['claude-opus-4-6', 'claude-sonnet-4-6']
|
||||
}
|
||||
},
|
||||
{
|
||||
label: 'Thinking Effort',
|
||||
description: 'Control how eager Claude is about spending tokens when responding to requests',
|
||||
name: 'thinkingEffort',
|
||||
type: 'options',
|
||||
optional: true,
|
||||
options: [
|
||||
{
|
||||
label: 'Low',
|
||||
name: 'low'
|
||||
},
|
||||
{
|
||||
label: 'Medium',
|
||||
name: 'medium'
|
||||
},
|
||||
{
|
||||
label: 'High',
|
||||
name: 'high'
|
||||
},
|
||||
{
|
||||
label: 'Max',
|
||||
name: 'max',
|
||||
description: 'Absolute maximum capability with no constraints on token spending. Opus 4.6 only'
|
||||
}
|
||||
],
|
||||
additionalParams: true,
|
||||
show: {
|
||||
adaptiveThinking: true,
|
||||
modelName: ['claude-opus-4-6', 'claude-sonnet-4-6']
|
||||
}
|
||||
},
|
||||
{
|
||||
label: 'Max Tokens',
|
||||
name: 'maxTokensToSample',
|
||||
@@ -86,33 +167,6 @@ class ChatAnthropic_ChatModels implements INode {
|
||||
step: 0.1,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Extended Thinking',
|
||||
name: 'extendedThinking',
|
||||
type: 'boolean',
|
||||
description: 'Enable extended thinking for reasoning model such as Claude Sonnet 3.7 and Claude 4',
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Budget Tokens',
|
||||
name: 'budgetTokens',
|
||||
type: 'number',
|
||||
step: 1,
|
||||
default: 1024,
|
||||
description: 'Maximum number of tokens Claude is allowed use for its internal reasoning process',
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Allow Image Uploads',
|
||||
name: 'allowImageUploads',
|
||||
type: 'boolean',
|
||||
description:
|
||||
'Allow image input. Refer to the <a href="https://docs.flowiseai.com/using-flowise/uploads#image" target="_blank">docs</a> for more details.',
|
||||
default: false,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -134,6 +188,8 @@ class ChatAnthropic_ChatModels implements INode {
|
||||
const cache = nodeData.inputs?.cache as BaseCache
|
||||
const extendedThinking = nodeData.inputs?.extendedThinking as boolean
|
||||
const budgetTokens = nodeData.inputs?.budgetTokens as string
|
||||
const adaptiveThinking = nodeData.inputs?.adaptiveThinking as boolean
|
||||
const thinkingEffort = nodeData.inputs?.thinkingEffort as 'low' | 'medium' | 'high' | 'max'
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const anthropicApiKey = getCredentialParam('anthropicApiKey', credentialData, nodeData)
|
||||
@@ -151,11 +207,24 @@ class ChatAnthropic_ChatModels implements INode {
|
||||
if (topP) obj.topP = parseFloat(topP)
|
||||
if (topK) obj.topK = parseFloat(topK)
|
||||
if (cache) obj.cache = cache
|
||||
if (extendedThinking) {
|
||||
|
||||
if (adaptiveThinking) {
|
||||
obj.thinking = {
|
||||
type: 'adaptive'
|
||||
}
|
||||
if (thinkingEffort) {
|
||||
obj.outputConfig = {
|
||||
effort: thinkingEffort
|
||||
}
|
||||
}
|
||||
|
||||
delete obj.temperature
|
||||
} else if (extendedThinking) {
|
||||
obj.thinking = {
|
||||
type: 'enabled',
|
||||
budget_tokens: parseInt(budgetTokens, 10)
|
||||
budget_tokens: parseInt(budgetTokens ?? '1024', 10)
|
||||
}
|
||||
|
||||
delete obj.temperature
|
||||
}
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ class ChatBaiduWenxin_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatBaiduWenxin'
|
||||
this.label = 'Baidu Wenxin'
|
||||
this.name = 'chatBaiduWenxin'
|
||||
this.version = 2.0
|
||||
this.type = 'ChatBaiduWenxin'
|
||||
|
||||
@@ -17,7 +17,7 @@ class ChatCerebras_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatCerebras'
|
||||
this.label = 'Cerebras'
|
||||
this.name = 'chatCerebras'
|
||||
this.version = 3.0
|
||||
this.type = 'ChatCerebras'
|
||||
@@ -103,18 +103,20 @@ class ChatCerebras_ChatModels implements INode {
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'BasePath',
|
||||
label: 'Base Path',
|
||||
name: 'basepath',
|
||||
type: 'string',
|
||||
optional: true,
|
||||
default: 'https://api.cerebras.ai/v1',
|
||||
description: 'Override the default base URL for the API, e.g., "https://api.example.com/v2/',
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'BaseOptions',
|
||||
label: 'Base Options',
|
||||
name: 'baseOptions',
|
||||
type: 'json',
|
||||
optional: true,
|
||||
description: 'Default headers to include with every request to the API.',
|
||||
additionalParams: true
|
||||
}
|
||||
]
|
||||
|
||||
+1
-1
@@ -15,7 +15,7 @@ class ChatCloudflareWorkersAI_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatCloudflareWorkersAI'
|
||||
this.label = 'Cloudflare Workers AI'
|
||||
this.name = 'chatCloudflareWorkersAI'
|
||||
this.version = 1.0
|
||||
this.type = 'ChatCloudflareWorkersAI'
|
||||
|
||||
@@ -17,7 +17,7 @@ class ChatCohere_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatCohere'
|
||||
this.label = 'Cohere'
|
||||
this.name = 'chatCohere'
|
||||
this.version = 3.0
|
||||
this.type = 'ChatCohere'
|
||||
|
||||
@@ -17,7 +17,7 @@ class ChatCometAPI_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatCometAPI'
|
||||
this.label = 'Comet'
|
||||
this.name = 'chatCometAPI'
|
||||
this.version = 1.0
|
||||
this.type = 'ChatCometAPI'
|
||||
|
||||
@@ -16,7 +16,7 @@ class ChatFireworks_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatFireworks'
|
||||
this.label = 'Fireworks AI'
|
||||
this.name = 'chatFireworks'
|
||||
this.version = 2.0
|
||||
this.type = 'ChatFireworks'
|
||||
|
||||
+62
-24
@@ -4,7 +4,8 @@ import { BaseCache } from '@langchain/core/caches'
|
||||
import { ICommonObject, IMultiModalOption, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { getModels, MODEL_TYPE } from '../../../src/modelLoader'
|
||||
import { ChatGoogleGenerativeAI, GoogleGenerativeAIChatInput } from './FlowiseChatGoogleGenerativeAI'
|
||||
import { GoogleGenerativeAIChatInput } from '@langchain/google-genai'
|
||||
import { ChatGoogleGenerativeAI } from './FlowiseChatGoogleGenerativeAI'
|
||||
|
||||
class GoogleGenerativeAI_ChatModels implements INode {
|
||||
label: string
|
||||
@@ -19,7 +20,7 @@ class GoogleGenerativeAI_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatGoogleGenerativeAI'
|
||||
this.label = 'Google Gemini'
|
||||
this.name = 'chatGoogleGenerativeAI'
|
||||
this.version = 3.1
|
||||
this.type = 'ChatGoogleGenerativeAI'
|
||||
@@ -74,6 +75,53 @@ class GoogleGenerativeAI_ChatModels implements INode {
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Allow Image Uploads',
|
||||
name: 'allowImageUploads',
|
||||
type: 'boolean',
|
||||
description:
|
||||
'Allow image input. Refer to the <a href="https://docs.flowiseai.com/using-flowise/uploads#image" target="_blank">docs</a> for more details.',
|
||||
default: false,
|
||||
optional: true
|
||||
},
|
||||
/** The thinkingLevel parameter, recommended for Gemini 3 models and onwards. */
|
||||
{
|
||||
label: 'Thinking Budget',
|
||||
name: 'thinkingBudget',
|
||||
type: 'number',
|
||||
description: 'Guides the number of thinking tokens. -1 for dynamic, 0 to disable, or positive integer (Gemini 2.5 models).',
|
||||
step: 1,
|
||||
optional: true,
|
||||
additionalParams: true,
|
||||
show: {
|
||||
modelName: ['gemini-2.5-pro', 'gemini-2.5-flash', 'gemini-2.5-flash-lite']
|
||||
}
|
||||
},
|
||||
{
|
||||
label: 'Thinking Level',
|
||||
name: 'thinkingLevel',
|
||||
type: 'options',
|
||||
description: 'Adjust the amount of reasoning effort based on the complexity of the user request',
|
||||
options: [
|
||||
{
|
||||
label: 'Low',
|
||||
name: 'LOW'
|
||||
},
|
||||
{
|
||||
label: 'Medium',
|
||||
name: 'MEDIUM'
|
||||
},
|
||||
{
|
||||
label: 'High',
|
||||
name: 'HIGH'
|
||||
}
|
||||
],
|
||||
optional: true,
|
||||
additionalParams: true,
|
||||
show: {
|
||||
modelName: ['gemini-3.1-pro-preview', 'gemini-3.1-flash-lite-preview', 'gemini-3-flash-preview']
|
||||
}
|
||||
},
|
||||
{
|
||||
label: 'Max Output Tokens',
|
||||
name: 'maxOutputTokens',
|
||||
@@ -174,18 +222,6 @@ class GoogleGenerativeAI_ChatModels implements INode {
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Thinking Budget',
|
||||
name: 'thinkingBudget',
|
||||
type: 'number',
|
||||
description: 'Guides the number of thinking tokens. -1 for dynamic, 0 to disable, or positive integer (Gemini 2.5 models).',
|
||||
step: 1,
|
||||
optional: true,
|
||||
additionalParams: true,
|
||||
show: {
|
||||
modelName: ['gemini-2.5-pro', 'gemini-2.5-flash', 'gemini-2.5-flash-lite']
|
||||
}
|
||||
},
|
||||
{
|
||||
label: 'Base URL',
|
||||
name: 'baseUrl',
|
||||
@@ -193,15 +229,6 @@ class GoogleGenerativeAI_ChatModels implements INode {
|
||||
description: 'Base URL for the API. Leave empty to use the default.',
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Allow Image Uploads',
|
||||
name: 'allowImageUploads',
|
||||
type: 'boolean',
|
||||
description:
|
||||
'Allow image input. Refer to the <a href="https://docs.flowiseai.com/using-flowise/uploads#image" target="_blank">docs</a> for more details.',
|
||||
default: false,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -229,6 +256,7 @@ class GoogleGenerativeAI_ChatModels implements INode {
|
||||
const streaming = nodeData.inputs?.streaming as boolean
|
||||
const baseUrl = nodeData.inputs?.baseUrl as string | undefined
|
||||
const thinkingBudget = nodeData.inputs?.thinkingBudget as string
|
||||
const thinkingLevel = nodeData.inputs?.thinkingLevel as 'LOW' | 'MEDIUM' | 'HIGH'
|
||||
|
||||
const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean
|
||||
|
||||
@@ -248,7 +276,17 @@ class GoogleGenerativeAI_ChatModels implements INode {
|
||||
if (cache) obj.cache = cache
|
||||
if (temperature) obj.temperature = parseFloat(temperature)
|
||||
if (baseUrl) obj.baseUrl = baseUrl
|
||||
if (thinkingBudget) obj.thinkingBudget = parseInt(thinkingBudget, 10)
|
||||
if (thinkingLevel) {
|
||||
obj.thinkingConfig = {
|
||||
thinkingLevel: thinkingLevel,
|
||||
includeThoughts: true
|
||||
}
|
||||
} else if (thinkingBudget) {
|
||||
obj.thinkingConfig = {
|
||||
thinkingBudget: parseInt(thinkingBudget, 10),
|
||||
includeThoughts: true
|
||||
}
|
||||
}
|
||||
|
||||
let safetySettings: SafetySetting[] = []
|
||||
if (_safetySettings) {
|
||||
|
||||
+798
-926
File diff suppressed because it is too large
Load Diff
@@ -1,673 +0,0 @@
|
||||
import {
|
||||
EnhancedGenerateContentResponse,
|
||||
Content,
|
||||
Part,
|
||||
type FunctionDeclarationsTool as GoogleGenerativeAIFunctionDeclarationsTool,
|
||||
type FunctionDeclaration as GenerativeAIFunctionDeclaration,
|
||||
POSSIBLE_ROLES,
|
||||
FunctionCallPart,
|
||||
TextPart,
|
||||
FileDataPart,
|
||||
InlineDataPart
|
||||
} from '@google/generative-ai'
|
||||
import {
|
||||
AIMessage,
|
||||
AIMessageChunk,
|
||||
BaseMessage,
|
||||
ChatMessage,
|
||||
ToolMessage,
|
||||
ToolMessageChunk,
|
||||
MessageContent,
|
||||
MessageContentComplex,
|
||||
UsageMetadata,
|
||||
isAIMessage,
|
||||
isBaseMessage,
|
||||
isToolMessage,
|
||||
StandardContentBlockConverter,
|
||||
parseBase64DataUrl,
|
||||
convertToProviderContentBlock,
|
||||
isDataContentBlock
|
||||
} from '@langchain/core/messages'
|
||||
import { ChatGeneration, ChatGenerationChunk, ChatResult } from '@langchain/core/outputs'
|
||||
import { isLangChainTool } from '@langchain/core/utils/function_calling'
|
||||
import { isOpenAITool } from '@langchain/core/language_models/base'
|
||||
import { ToolCallChunk } from '@langchain/core/messages/tool'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import { jsonSchemaToGeminiParameters, schemaToGenerativeAIParameters } from './zod_to_genai_parameters.js'
|
||||
import { GoogleGenerativeAIToolType } from './types.js'
|
||||
|
||||
export function getMessageAuthor(message: BaseMessage) {
|
||||
const type = message._getType()
|
||||
if (ChatMessage.isInstance(message)) {
|
||||
return message.role
|
||||
}
|
||||
if (type === 'tool') {
|
||||
return type
|
||||
}
|
||||
return message.name ?? type
|
||||
}
|
||||
|
||||
/**
|
||||
* !!! IMPORTANT: Must return 'user' as default instead of throwing error
|
||||
* https://github.com/FlowiseAI/Flowise/issues/4743
|
||||
* Maps a message type to a Google Generative AI chat author.
|
||||
* @param message The message to map.
|
||||
* @param model The model to use for mapping.
|
||||
* @returns The message type mapped to a Google Generative AI chat author.
|
||||
*/
|
||||
export function convertAuthorToRole(author: string): (typeof POSSIBLE_ROLES)[number] {
|
||||
switch (author) {
|
||||
/**
|
||||
* Note: Gemini currently is not supporting system messages
|
||||
* we will convert them to human messages and merge with following
|
||||
* */
|
||||
case 'supervisor':
|
||||
case 'ai':
|
||||
case 'model': // getMessageAuthor returns message.name. code ex.: return message.name ?? type;
|
||||
return 'model'
|
||||
case 'system':
|
||||
return 'system'
|
||||
case 'human':
|
||||
return 'user'
|
||||
case 'tool':
|
||||
case 'function':
|
||||
return 'function'
|
||||
default:
|
||||
return 'user' // return user as default instead of throwing error
|
||||
}
|
||||
}
|
||||
|
||||
function messageContentMedia(content: MessageContentComplex): Part {
|
||||
if ('mimeType' in content && 'data' in content) {
|
||||
return {
|
||||
inlineData: {
|
||||
mimeType: content.mimeType,
|
||||
data: content.data
|
||||
}
|
||||
}
|
||||
}
|
||||
if ('mimeType' in content && 'fileUri' in content) {
|
||||
return {
|
||||
fileData: {
|
||||
mimeType: content.mimeType,
|
||||
fileUri: content.fileUri
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('Invalid media content')
|
||||
}
|
||||
|
||||
function inferToolNameFromPreviousMessages(message: ToolMessage | ToolMessageChunk, previousMessages: BaseMessage[]): string | undefined {
|
||||
return previousMessages
|
||||
.map((msg) => {
|
||||
if (isAIMessage(msg)) {
|
||||
return msg.tool_calls ?? []
|
||||
}
|
||||
return []
|
||||
})
|
||||
.flat()
|
||||
.find((toolCall) => {
|
||||
return toolCall.id === message.tool_call_id
|
||||
})?.name
|
||||
}
|
||||
|
||||
function _getStandardContentBlockConverter(isMultimodalModel: boolean) {
|
||||
const standardContentBlockConverter: StandardContentBlockConverter<{
|
||||
text: TextPart
|
||||
image: FileDataPart | InlineDataPart
|
||||
audio: FileDataPart | InlineDataPart
|
||||
file: FileDataPart | InlineDataPart | TextPart
|
||||
}> = {
|
||||
providerName: 'Google Gemini',
|
||||
|
||||
fromStandardTextBlock(block) {
|
||||
return {
|
||||
text: block.text
|
||||
}
|
||||
},
|
||||
|
||||
fromStandardImageBlock(block): FileDataPart | InlineDataPart {
|
||||
if (!isMultimodalModel) {
|
||||
throw new Error('This model does not support images')
|
||||
}
|
||||
if (block.source_type === 'url') {
|
||||
const data = parseBase64DataUrl({ dataUrl: block.url })
|
||||
if (data) {
|
||||
return {
|
||||
inlineData: {
|
||||
mimeType: data.mime_type,
|
||||
data: data.data
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return {
|
||||
fileData: {
|
||||
mimeType: block.mime_type ?? '',
|
||||
fileUri: block.url
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (block.source_type === 'base64') {
|
||||
return {
|
||||
inlineData: {
|
||||
mimeType: block.mime_type ?? '',
|
||||
data: block.data
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported source type: ${block.source_type}`)
|
||||
},
|
||||
|
||||
fromStandardAudioBlock(block): FileDataPart | InlineDataPart {
|
||||
if (!isMultimodalModel) {
|
||||
throw new Error('This model does not support audio')
|
||||
}
|
||||
if (block.source_type === 'url') {
|
||||
const data = parseBase64DataUrl({ dataUrl: block.url })
|
||||
if (data) {
|
||||
return {
|
||||
inlineData: {
|
||||
mimeType: data.mime_type,
|
||||
data: data.data
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return {
|
||||
fileData: {
|
||||
mimeType: block.mime_type ?? '',
|
||||
fileUri: block.url
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (block.source_type === 'base64') {
|
||||
return {
|
||||
inlineData: {
|
||||
mimeType: block.mime_type ?? '',
|
||||
data: block.data
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported source type: ${block.source_type}`)
|
||||
},
|
||||
|
||||
fromStandardFileBlock(block): FileDataPart | InlineDataPart | TextPart {
|
||||
if (!isMultimodalModel) {
|
||||
throw new Error('This model does not support files')
|
||||
}
|
||||
if (block.source_type === 'text') {
|
||||
return {
|
||||
text: block.text
|
||||
}
|
||||
}
|
||||
if (block.source_type === 'url') {
|
||||
const data = parseBase64DataUrl({ dataUrl: block.url })
|
||||
if (data) {
|
||||
return {
|
||||
inlineData: {
|
||||
mimeType: data.mime_type,
|
||||
data: data.data
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return {
|
||||
fileData: {
|
||||
mimeType: block.mime_type ?? '',
|
||||
fileUri: block.url
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (block.source_type === 'base64') {
|
||||
return {
|
||||
inlineData: {
|
||||
mimeType: block.mime_type ?? '',
|
||||
data: block.data
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new Error(`Unsupported source type: ${block.source_type}`)
|
||||
}
|
||||
}
|
||||
return standardContentBlockConverter
|
||||
}
|
||||
|
||||
function _convertLangChainContentToPart(content: MessageContentComplex, isMultimodalModel: boolean): Part | undefined {
|
||||
if (isDataContentBlock(content)) {
|
||||
return convertToProviderContentBlock(content, _getStandardContentBlockConverter(isMultimodalModel))
|
||||
}
|
||||
|
||||
if (content.type === 'text') {
|
||||
return { text: content.text }
|
||||
} else if (content.type === 'executableCode') {
|
||||
return { executableCode: content.executableCode }
|
||||
} else if (content.type === 'codeExecutionResult') {
|
||||
return { codeExecutionResult: content.codeExecutionResult }
|
||||
} else if (content.type === 'image_url') {
|
||||
if (!isMultimodalModel) {
|
||||
throw new Error(`This model does not support images`)
|
||||
}
|
||||
let source
|
||||
if (typeof content.image_url === 'string') {
|
||||
source = content.image_url
|
||||
} else if (typeof content.image_url === 'object' && 'url' in content.image_url) {
|
||||
source = content.image_url.url
|
||||
} else {
|
||||
throw new Error('Please provide image as base64 encoded data URL')
|
||||
}
|
||||
const [dm, data] = source.split(',')
|
||||
if (!dm.startsWith('data:')) {
|
||||
throw new Error('Please provide image as base64 encoded data URL')
|
||||
}
|
||||
|
||||
const [mimeType, encoding] = dm.replace(/^data:/, '').split(';')
|
||||
if (encoding !== 'base64') {
|
||||
throw new Error('Please provide image as base64 encoded data URL')
|
||||
}
|
||||
|
||||
return {
|
||||
inlineData: {
|
||||
data,
|
||||
mimeType
|
||||
}
|
||||
}
|
||||
} else if (content.type === 'media') {
|
||||
return messageContentMedia(content)
|
||||
} else if (content.type === 'tool_use') {
|
||||
return {
|
||||
functionCall: {
|
||||
name: content.name,
|
||||
args: content.input
|
||||
}
|
||||
}
|
||||
} else if (
|
||||
content.type?.includes('/') &&
|
||||
// Ensure it's a single slash.
|
||||
content.type.split('/').length === 2 &&
|
||||
'data' in content &&
|
||||
typeof content.data === 'string'
|
||||
) {
|
||||
return {
|
||||
inlineData: {
|
||||
mimeType: content.type,
|
||||
data: content.data
|
||||
}
|
||||
}
|
||||
} else if ('functionCall' in content) {
|
||||
// No action needed here — function calls will be added later from message.tool_calls
|
||||
return undefined
|
||||
} else {
|
||||
if ('type' in content) {
|
||||
throw new Error(`Unknown content type ${content.type}`)
|
||||
} else {
|
||||
throw new Error(`Unknown content ${JSON.stringify(content)}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function convertMessageContentToParts(message: BaseMessage, isMultimodalModel: boolean, previousMessages: BaseMessage[]): Part[] {
|
||||
if (isToolMessage(message)) {
|
||||
const messageName = message.name ?? inferToolNameFromPreviousMessages(message, previousMessages)
|
||||
if (messageName === undefined) {
|
||||
throw new Error(
|
||||
`Google requires a tool name for each tool call response, and we could not infer a called tool name for ToolMessage "${message.id}" from your passed messages. Please populate a "name" field on that ToolMessage explicitly.`
|
||||
)
|
||||
}
|
||||
|
||||
const result = Array.isArray(message.content)
|
||||
? (message.content.map((c) => _convertLangChainContentToPart(c, isMultimodalModel)).filter((p) => p !== undefined) as Part[])
|
||||
: message.content
|
||||
|
||||
if (message.status === 'error') {
|
||||
return [
|
||||
{
|
||||
functionResponse: {
|
||||
name: messageName,
|
||||
// The API expects an object with an `error` field if the function call fails.
|
||||
// `error` must be a valid object (not a string or array), so we wrap `message.content` here
|
||||
response: { error: { details: result } }
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
functionResponse: {
|
||||
name: messageName,
|
||||
// again, can't have a string or array value for `response`, so we wrap it as an object here
|
||||
response: { result }
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
let functionCalls: FunctionCallPart[] = []
|
||||
const messageParts: Part[] = []
|
||||
|
||||
if (typeof message.content === 'string' && message.content) {
|
||||
messageParts.push({ text: message.content })
|
||||
}
|
||||
|
||||
if (Array.isArray(message.content)) {
|
||||
messageParts.push(
|
||||
...(message.content.map((c) => _convertLangChainContentToPart(c, isMultimodalModel)).filter((p) => p !== undefined) as Part[])
|
||||
)
|
||||
}
|
||||
|
||||
if (isAIMessage(message) && message.tool_calls?.length) {
|
||||
functionCalls = message.tool_calls.map((tc) => {
|
||||
return {
|
||||
functionCall: {
|
||||
name: tc.name,
|
||||
args: tc.args
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
return [...messageParts, ...functionCalls]
|
||||
}
|
||||
|
||||
export function convertBaseMessagesToContent(
|
||||
messages: BaseMessage[],
|
||||
isMultimodalModel: boolean,
|
||||
convertSystemMessageToHumanContent: boolean = false
|
||||
) {
|
||||
return messages.reduce<{
|
||||
content: Content[]
|
||||
mergeWithPreviousContent: boolean
|
||||
}>(
|
||||
(acc, message, index) => {
|
||||
if (!isBaseMessage(message)) {
|
||||
throw new Error('Unsupported message input')
|
||||
}
|
||||
const author = getMessageAuthor(message)
|
||||
if (author === 'system' && index !== 0) {
|
||||
throw new Error('System message should be the first one')
|
||||
}
|
||||
const role = convertAuthorToRole(author)
|
||||
|
||||
const prevContent = acc.content[acc.content.length]
|
||||
if (!acc.mergeWithPreviousContent && prevContent && prevContent.role === role) {
|
||||
throw new Error('Google Generative AI requires alternate messages between authors')
|
||||
}
|
||||
|
||||
const parts = convertMessageContentToParts(message, isMultimodalModel, messages.slice(0, index))
|
||||
|
||||
if (acc.mergeWithPreviousContent) {
|
||||
const prevContent = acc.content[acc.content.length - 1]
|
||||
if (!prevContent) {
|
||||
throw new Error('There was a problem parsing your system message. Please try a prompt without one.')
|
||||
}
|
||||
prevContent.parts.push(...parts)
|
||||
|
||||
return {
|
||||
mergeWithPreviousContent: false,
|
||||
content: acc.content
|
||||
}
|
||||
}
|
||||
let actualRole = role
|
||||
if (actualRole === 'function' || (actualRole === 'system' && !convertSystemMessageToHumanContent)) {
|
||||
// GenerativeAI API will throw an error if the role is not "user" or "model."
|
||||
actualRole = 'user'
|
||||
}
|
||||
const content: Content = {
|
||||
role: actualRole,
|
||||
parts
|
||||
}
|
||||
return {
|
||||
mergeWithPreviousContent: author === 'system' && !convertSystemMessageToHumanContent,
|
||||
content: [...acc.content, content]
|
||||
}
|
||||
},
|
||||
{ content: [], mergeWithPreviousContent: false }
|
||||
).content
|
||||
}
|
||||
|
||||
export function mapGenerateContentResultToChatResult(
|
||||
response: EnhancedGenerateContentResponse,
|
||||
extra?: {
|
||||
usageMetadata: UsageMetadata | undefined
|
||||
}
|
||||
): ChatResult {
|
||||
// if rejected or error, return empty generations with reason in filters
|
||||
if (!response.candidates || response.candidates.length === 0 || !response.candidates[0]) {
|
||||
return {
|
||||
generations: [],
|
||||
llmOutput: {
|
||||
filters: response.promptFeedback
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const functionCalls = response.functionCalls()
|
||||
const [candidate] = response.candidates
|
||||
const { content: candidateContent, ...generationInfo } = candidate
|
||||
let content: MessageContent | undefined
|
||||
const inlineDataItems: any[] = []
|
||||
|
||||
if (Array.isArray(candidateContent?.parts) && candidateContent.parts.length === 1 && candidateContent.parts[0].text) {
|
||||
content = candidateContent.parts[0].text
|
||||
} else if (Array.isArray(candidateContent?.parts) && candidateContent.parts.length > 0) {
|
||||
content = candidateContent.parts.map((p) => {
|
||||
if ('text' in p) {
|
||||
return {
|
||||
type: 'text',
|
||||
text: p.text
|
||||
}
|
||||
} else if ('executableCode' in p) {
|
||||
return {
|
||||
type: 'executableCode',
|
||||
executableCode: p.executableCode
|
||||
}
|
||||
} else if ('codeExecutionResult' in p) {
|
||||
return {
|
||||
type: 'codeExecutionResult',
|
||||
codeExecutionResult: p.codeExecutionResult
|
||||
}
|
||||
} else if ('inlineData' in p && p.inlineData) {
|
||||
// Extract inline image data for processing by Agent
|
||||
inlineDataItems.push({
|
||||
type: 'gemini_inline_data',
|
||||
mimeType: p.inlineData.mimeType,
|
||||
data: p.inlineData.data
|
||||
})
|
||||
// Return the inline data as part of the content structure
|
||||
return {
|
||||
type: 'inlineData',
|
||||
inlineData: p.inlineData
|
||||
}
|
||||
}
|
||||
return p
|
||||
})
|
||||
} else {
|
||||
// no content returned - likely due to abnormal stop reason, e.g. malformed function call
|
||||
content = []
|
||||
}
|
||||
|
||||
let text = ''
|
||||
if (typeof content === 'string') {
|
||||
text = content
|
||||
} else if (Array.isArray(content) && content.length > 0) {
|
||||
const block = content.find((b) => 'text' in b) as { text: string } | undefined
|
||||
text = block?.text ?? text
|
||||
}
|
||||
|
||||
// Build response_metadata with inline data if present
|
||||
const response_metadata: any = {}
|
||||
if (inlineDataItems.length > 0) {
|
||||
response_metadata.inlineData = inlineDataItems
|
||||
}
|
||||
|
||||
const generation: ChatGeneration = {
|
||||
text,
|
||||
message: new AIMessage({
|
||||
content: content ?? '',
|
||||
tool_calls: functionCalls?.map((fc) => {
|
||||
return {
|
||||
...fc,
|
||||
type: 'tool_call',
|
||||
id: 'id' in fc && typeof fc.id === 'string' ? fc.id : uuidv4()
|
||||
}
|
||||
}),
|
||||
additional_kwargs: {
|
||||
...generationInfo
|
||||
},
|
||||
usage_metadata: extra?.usageMetadata,
|
||||
response_metadata: Object.keys(response_metadata).length > 0 ? response_metadata : undefined
|
||||
}),
|
||||
generationInfo
|
||||
}
|
||||
|
||||
return {
|
||||
generations: [generation],
|
||||
llmOutput: {
|
||||
tokenUsage: {
|
||||
promptTokens: extra?.usageMetadata?.input_tokens,
|
||||
completionTokens: extra?.usageMetadata?.output_tokens,
|
||||
totalTokens: extra?.usageMetadata?.total_tokens
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function convertResponseContentToChatGenerationChunk(
|
||||
response: EnhancedGenerateContentResponse,
|
||||
extra: {
|
||||
usageMetadata?: UsageMetadata | undefined
|
||||
index: number
|
||||
}
|
||||
): ChatGenerationChunk | null {
|
||||
if (!response.candidates || response.candidates.length === 0) {
|
||||
return null
|
||||
}
|
||||
const functionCalls = response.functionCalls()
|
||||
const [candidate] = response.candidates
|
||||
const { content: candidateContent, ...generationInfo } = candidate
|
||||
let content: MessageContent | undefined
|
||||
const inlineDataItems: any[] = []
|
||||
|
||||
// Checks if some parts do not have text. If false, it means that the content is a string.
|
||||
if (Array.isArray(candidateContent?.parts) && candidateContent.parts.every((p) => 'text' in p)) {
|
||||
content = candidateContent.parts.map((p) => p.text).join('')
|
||||
} else if (Array.isArray(candidateContent?.parts)) {
|
||||
content = candidateContent.parts.map((p) => {
|
||||
if ('text' in p) {
|
||||
return {
|
||||
type: 'text',
|
||||
text: p.text
|
||||
}
|
||||
} else if ('executableCode' in p) {
|
||||
return {
|
||||
type: 'executableCode',
|
||||
executableCode: p.executableCode
|
||||
}
|
||||
} else if ('codeExecutionResult' in p) {
|
||||
return {
|
||||
type: 'codeExecutionResult',
|
||||
codeExecutionResult: p.codeExecutionResult
|
||||
}
|
||||
} else if ('inlineData' in p && p.inlineData) {
|
||||
// Extract inline image data for processing by Agent
|
||||
inlineDataItems.push({
|
||||
type: 'gemini_inline_data',
|
||||
mimeType: p.inlineData.mimeType,
|
||||
data: p.inlineData.data
|
||||
})
|
||||
// Return the inline data as part of the content structure
|
||||
return {
|
||||
type: 'inlineData',
|
||||
inlineData: p.inlineData
|
||||
}
|
||||
}
|
||||
return p
|
||||
})
|
||||
} else {
|
||||
// no content returned - likely due to abnormal stop reason, e.g. malformed function call
|
||||
content = []
|
||||
}
|
||||
|
||||
let text = ''
|
||||
if (content && typeof content === 'string') {
|
||||
text = content
|
||||
} else if (Array.isArray(content)) {
|
||||
const block = content.find((b) => 'text' in b) as { text: string } | undefined
|
||||
text = block?.text ?? ''
|
||||
}
|
||||
|
||||
const toolCallChunks: ToolCallChunk[] = []
|
||||
if (functionCalls) {
|
||||
toolCallChunks.push(
|
||||
...functionCalls.map((fc) => ({
|
||||
...fc,
|
||||
args: JSON.stringify(fc.args),
|
||||
index: extra.index,
|
||||
type: 'tool_call_chunk' as const,
|
||||
id: 'id' in fc && typeof fc.id === 'string' ? fc.id : uuidv4()
|
||||
}))
|
||||
)
|
||||
}
|
||||
|
||||
// Build response_metadata with inline data if present
|
||||
const response_metadata: any = {}
|
||||
if (inlineDataItems.length > 0) {
|
||||
response_metadata.inlineData = inlineDataItems
|
||||
}
|
||||
|
||||
return new ChatGenerationChunk({
|
||||
text,
|
||||
message: new AIMessageChunk({
|
||||
content: content || '',
|
||||
name: !candidateContent ? undefined : candidateContent.role,
|
||||
tool_call_chunks: toolCallChunks,
|
||||
// Each chunk can have unique "generationInfo", and merging strategy is unclear,
|
||||
// so leave blank for now.
|
||||
additional_kwargs: {},
|
||||
usage_metadata: extra.usageMetadata,
|
||||
response_metadata: Object.keys(response_metadata).length > 0 ? response_metadata : undefined
|
||||
}),
|
||||
generationInfo
|
||||
})
|
||||
}
|
||||
|
||||
export function convertToGenerativeAITools(tools: GoogleGenerativeAIToolType[]): GoogleGenerativeAIFunctionDeclarationsTool[] {
|
||||
if (tools.every((tool) => 'functionDeclarations' in tool && Array.isArray(tool.functionDeclarations))) {
|
||||
return tools as GoogleGenerativeAIFunctionDeclarationsTool[]
|
||||
}
|
||||
return [
|
||||
{
|
||||
functionDeclarations: tools.map((tool): GenerativeAIFunctionDeclaration => {
|
||||
if (isLangChainTool(tool)) {
|
||||
const jsonSchema = schemaToGenerativeAIParameters(tool.schema)
|
||||
if (jsonSchema.type === 'object' && 'properties' in jsonSchema && Object.keys(jsonSchema.properties).length === 0) {
|
||||
return {
|
||||
name: tool.name,
|
||||
description: tool.description
|
||||
}
|
||||
}
|
||||
return {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: jsonSchema
|
||||
}
|
||||
}
|
||||
if (isOpenAITool(tool)) {
|
||||
return {
|
||||
name: tool.function.name,
|
||||
description: tool.function.description ?? `A function available to call.`,
|
||||
parameters: jsonSchemaToGeminiParameters(tool.function.parameters)
|
||||
}
|
||||
}
|
||||
return tool as unknown as GenerativeAIFunctionDeclaration
|
||||
})
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
import { BaseLLMOutputParser, OutputParserException } from '@langchain/core/output_parsers'
|
||||
import { ChatGeneration } from '@langchain/core/outputs'
|
||||
import { ToolCall } from '@langchain/core/messages/tool'
|
||||
import { InteropZodType, interopSafeParseAsync } from '@langchain/core/utils/types'
|
||||
import { JsonOutputKeyToolsParserParamsInterop } from '@langchain/core/output_parsers/openai_tools'
|
||||
|
||||
interface GoogleGenerativeAIToolsOutputParserParams<T extends Record<string, any>> extends JsonOutputKeyToolsParserParamsInterop<T> {}
|
||||
|
||||
export class GoogleGenerativeAIToolsOutputParser<T extends Record<string, any> = Record<string, any>> extends BaseLLMOutputParser<T> {
|
||||
static lc_name() {
|
||||
return 'GoogleGenerativeAIToolsOutputParser'
|
||||
}
|
||||
|
||||
lc_namespace = ['langchain', 'google_genai', 'output_parsers']
|
||||
|
||||
returnId = false
|
||||
|
||||
/** The type of tool calls to return. */
|
||||
keyName: string
|
||||
|
||||
/** Whether to return only the first tool call. */
|
||||
returnSingle = false
|
||||
|
||||
zodSchema?: InteropZodType<T>
|
||||
|
||||
constructor(params: GoogleGenerativeAIToolsOutputParserParams<T>) {
|
||||
super(params)
|
||||
this.keyName = params.keyName
|
||||
this.returnSingle = params.returnSingle ?? this.returnSingle
|
||||
this.zodSchema = params.zodSchema
|
||||
}
|
||||
|
||||
protected async _validateResult(result: unknown): Promise<T> {
|
||||
if (this.zodSchema === undefined) {
|
||||
return result as T
|
||||
}
|
||||
const zodParsedResult = await interopSafeParseAsync(this.zodSchema, result)
|
||||
if (zodParsedResult.success) {
|
||||
return zodParsedResult.data
|
||||
} else {
|
||||
throw new OutputParserException(
|
||||
`Failed to parse. Text: "${JSON.stringify(result, null, 2)}". Error: ${JSON.stringify(zodParsedResult.error.issues)}`,
|
||||
JSON.stringify(result, null, 2)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async parseResult(generations: ChatGeneration[]): Promise<T> {
|
||||
const tools = generations.flatMap((generation) => {
|
||||
const { message } = generation
|
||||
if (!('tool_calls' in message) || !Array.isArray(message.tool_calls)) {
|
||||
return []
|
||||
}
|
||||
return message.tool_calls as ToolCall[]
|
||||
})
|
||||
if (tools[0] === undefined) {
|
||||
throw new Error('No parseable tool calls provided to GoogleGenerativeAIToolsOutputParser.')
|
||||
}
|
||||
const [tool] = tools
|
||||
const validatedResult = await this._validateResult(tool.args)
|
||||
return validatedResult
|
||||
}
|
||||
}
|
||||
@@ -1,136 +0,0 @@
|
||||
import {
|
||||
Tool as GenerativeAITool,
|
||||
ToolConfig,
|
||||
FunctionCallingMode,
|
||||
FunctionDeclaration,
|
||||
FunctionDeclarationsTool,
|
||||
FunctionDeclarationSchema
|
||||
} from '@google/generative-ai'
|
||||
import { ToolChoice } from '@langchain/core/language_models/chat_models'
|
||||
import { StructuredToolInterface } from '@langchain/core/tools'
|
||||
import { isLangChainTool } from '@langchain/core/utils/function_calling'
|
||||
import { isOpenAITool, ToolDefinition } from '@langchain/core/language_models/base'
|
||||
import { convertToGenerativeAITools } from './common.js'
|
||||
import { GoogleGenerativeAIToolType } from './types.js'
|
||||
import { removeAdditionalProperties } from './zod_to_genai_parameters.js'
|
||||
|
||||
export function convertToolsToGenAI(
|
||||
tools: GoogleGenerativeAIToolType[],
|
||||
extra?: {
|
||||
toolChoice?: ToolChoice
|
||||
allowedFunctionNames?: string[]
|
||||
}
|
||||
): {
|
||||
tools: GenerativeAITool[]
|
||||
toolConfig?: ToolConfig
|
||||
} {
|
||||
// Extract function declaration processing to a separate function
|
||||
const genAITools = processTools(tools)
|
||||
|
||||
// Simplify tool config creation
|
||||
const toolConfig = createToolConfig(genAITools, extra)
|
||||
|
||||
return { tools: genAITools, toolConfig }
|
||||
}
|
||||
|
||||
function processTools(tools: GoogleGenerativeAIToolType[]): GenerativeAITool[] {
|
||||
let functionDeclarationTools: FunctionDeclaration[] = []
|
||||
const genAITools: GenerativeAITool[] = []
|
||||
|
||||
tools.forEach((tool) => {
|
||||
if (isLangChainTool(tool)) {
|
||||
const [convertedTool] = convertToGenerativeAITools([tool as StructuredToolInterface])
|
||||
if (convertedTool.functionDeclarations) {
|
||||
functionDeclarationTools.push(...convertedTool.functionDeclarations)
|
||||
}
|
||||
} else if (isOpenAITool(tool)) {
|
||||
const { functionDeclarations } = convertOpenAIToolToGenAI(tool)
|
||||
if (functionDeclarations) {
|
||||
functionDeclarationTools.push(...functionDeclarations)
|
||||
} else {
|
||||
throw new Error('Failed to convert OpenAI structured tool to GenerativeAI tool')
|
||||
}
|
||||
} else {
|
||||
genAITools.push(tool as GenerativeAITool)
|
||||
}
|
||||
})
|
||||
|
||||
const genAIFunctionDeclaration = genAITools.find((t) => 'functionDeclarations' in t)
|
||||
if (genAIFunctionDeclaration) {
|
||||
return genAITools.map((tool) => {
|
||||
if (functionDeclarationTools?.length > 0 && 'functionDeclarations' in tool) {
|
||||
const newTool = {
|
||||
functionDeclarations: [...(tool.functionDeclarations || []), ...functionDeclarationTools]
|
||||
}
|
||||
// Clear the functionDeclarationTools array so it is not passed again
|
||||
functionDeclarationTools = []
|
||||
return newTool
|
||||
}
|
||||
return tool
|
||||
})
|
||||
}
|
||||
|
||||
return [
|
||||
...genAITools,
|
||||
...(functionDeclarationTools.length > 0
|
||||
? [
|
||||
{
|
||||
functionDeclarations: functionDeclarationTools
|
||||
}
|
||||
]
|
||||
: [])
|
||||
]
|
||||
}
|
||||
|
||||
function convertOpenAIToolToGenAI(tool: ToolDefinition): FunctionDeclarationsTool {
|
||||
return {
|
||||
functionDeclarations: [
|
||||
{
|
||||
name: tool.function.name,
|
||||
description: tool.function.description,
|
||||
parameters: removeAdditionalProperties(tool.function.parameters) as FunctionDeclarationSchema
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
function createToolConfig(
|
||||
genAITools: GenerativeAITool[],
|
||||
extra?: {
|
||||
toolChoice?: ToolChoice
|
||||
allowedFunctionNames?: string[]
|
||||
}
|
||||
): ToolConfig | undefined {
|
||||
if (!genAITools.length || !extra) return undefined
|
||||
|
||||
const { toolChoice, allowedFunctionNames } = extra
|
||||
|
||||
const modeMap: Record<string, FunctionCallingMode> = {
|
||||
any: FunctionCallingMode.ANY,
|
||||
auto: FunctionCallingMode.AUTO,
|
||||
none: FunctionCallingMode.NONE
|
||||
}
|
||||
|
||||
if (toolChoice && ['any', 'auto', 'none'].includes(toolChoice as string)) {
|
||||
return {
|
||||
functionCallingConfig: {
|
||||
mode: modeMap[toolChoice as keyof typeof modeMap] ?? 'MODE_UNSPECIFIED',
|
||||
allowedFunctionNames
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof toolChoice === 'string' || allowedFunctionNames) {
|
||||
return {
|
||||
functionCallingConfig: {
|
||||
mode: FunctionCallingMode.ANY,
|
||||
allowedFunctionNames: [
|
||||
...(allowedFunctionNames ?? []),
|
||||
...(toolChoice && typeof toolChoice === 'string' ? [toolChoice] : [])
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
import {
|
||||
CodeExecutionTool,
|
||||
FunctionDeclarationsTool as GoogleGenerativeAIFunctionDeclarationsTool,
|
||||
GoogleSearchRetrievalTool
|
||||
} from '@google/generative-ai'
|
||||
import { BindToolsInput } from '@langchain/core/language_models/chat_models'
|
||||
|
||||
export type GoogleGenerativeAIToolType =
|
||||
| BindToolsInput
|
||||
| GoogleGenerativeAIFunctionDeclarationsTool
|
||||
| CodeExecutionTool
|
||||
| GoogleSearchRetrievalTool
|
||||
-67
@@ -1,67 +0,0 @@
|
||||
import {
|
||||
type FunctionDeclarationSchema as GenerativeAIFunctionDeclarationSchema,
|
||||
type SchemaType as FunctionDeclarationSchemaType
|
||||
} from '@google/generative-ai'
|
||||
import { InteropZodType, isInteropZodSchema } from '@langchain/core/utils/types'
|
||||
import { type JsonSchema7Type, toJsonSchema } from '@langchain/core/utils/json_schema'
|
||||
|
||||
export interface GenerativeAIJsonSchema extends Record<string, unknown> {
|
||||
properties?: Record<string, GenerativeAIJsonSchema>
|
||||
type: FunctionDeclarationSchemaType
|
||||
}
|
||||
|
||||
export interface GenerativeAIJsonSchemaDirty extends GenerativeAIJsonSchema {
|
||||
properties?: Record<string, GenerativeAIJsonSchemaDirty>
|
||||
additionalProperties?: boolean
|
||||
}
|
||||
|
||||
export function removeAdditionalProperties(obj: Record<string, any>): GenerativeAIJsonSchema {
|
||||
if (typeof obj === 'object' && obj !== null) {
|
||||
const newObj = { ...obj }
|
||||
|
||||
if ('additionalProperties' in newObj) {
|
||||
delete newObj.additionalProperties
|
||||
}
|
||||
if ('$schema' in newObj) {
|
||||
delete newObj.$schema
|
||||
}
|
||||
if ('strict' in newObj) {
|
||||
delete newObj.strict
|
||||
}
|
||||
|
||||
for (const key in newObj) {
|
||||
if (key in newObj) {
|
||||
if (Array.isArray(newObj[key])) {
|
||||
newObj[key] = newObj[key].map(removeAdditionalProperties)
|
||||
} else if (typeof newObj[key] === 'object' && newObj[key] !== null) {
|
||||
newObj[key] = removeAdditionalProperties(newObj[key])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return newObj as GenerativeAIJsonSchema
|
||||
}
|
||||
|
||||
return obj as GenerativeAIJsonSchema
|
||||
}
|
||||
|
||||
export function schemaToGenerativeAIParameters<RunOutput extends Record<string, any> = Record<string, any>>(
|
||||
schema: InteropZodType<RunOutput> | JsonSchema7Type
|
||||
): GenerativeAIFunctionDeclarationSchema {
|
||||
// GenerativeAI doesn't accept either the $schema or additionalProperties
|
||||
// attributes, so we need to explicitly remove them.
|
||||
const jsonSchema = removeAdditionalProperties(isInteropZodSchema(schema) ? toJsonSchema(schema) : schema)
|
||||
const { _schema, ...rest } = jsonSchema
|
||||
|
||||
return rest as GenerativeAIFunctionDeclarationSchema
|
||||
}
|
||||
|
||||
export function jsonSchemaToGeminiParameters(schema: Record<string, any>): GenerativeAIFunctionDeclarationSchema {
|
||||
// Gemini doesn't accept either the $schema or additionalProperties
|
||||
// attributes, so we need to explicitly remove them.
|
||||
|
||||
const jsonSchema = removeAdditionalProperties(schema as GenerativeAIJsonSchemaDirty)
|
||||
const { _schema, ...rest } = jsonSchema
|
||||
|
||||
return rest as GenerativeAIFunctionDeclarationSchema
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { BaseCache } from '@langchain/core/caches'
|
||||
import { ChatVertexAIInput, ChatVertexAI as LcChatVertexAI } from '@langchain/google-vertexai'
|
||||
import { GoogleGenerativeAIChatInput } from '@langchain/google-genai'
|
||||
import { buildGoogleCredentials } from '../../../src/google-utils'
|
||||
import {
|
||||
ICommonObject,
|
||||
@@ -64,7 +65,7 @@ class GoogleVertexAI_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatGoogleVertexAI'
|
||||
this.label = 'Google VertexAI'
|
||||
this.name = 'chatGoogleVertexAI'
|
||||
this.version = 5.3
|
||||
this.type = 'ChatGoogleVertexAI'
|
||||
@@ -119,6 +120,14 @@ class GoogleVertexAI_ChatModels implements INode {
|
||||
default: 0.9,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Streaming',
|
||||
name: 'streaming',
|
||||
type: 'boolean',
|
||||
default: true,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Allow Image Uploads',
|
||||
name: 'allowImageUploads',
|
||||
@@ -128,13 +137,43 @@ class GoogleVertexAI_ChatModels implements INode {
|
||||
default: false,
|
||||
optional: true
|
||||
},
|
||||
/** The thinkingLevel parameter, recommended for Gemini 3 models and onwards. */
|
||||
{
|
||||
label: 'Streaming',
|
||||
name: 'streaming',
|
||||
type: 'boolean',
|
||||
default: true,
|
||||
label: 'Thinking Budget',
|
||||
name: 'thinkingBudget',
|
||||
type: 'number',
|
||||
description: 'Guides the number of thinking tokens. -1 for dynamic, 0 to disable, or positive integer (Gemini 2.5 models).',
|
||||
step: 1,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
additionalParams: true,
|
||||
show: {
|
||||
modelName: ['gemini-2.5-pro', 'gemini-2.5-flash', 'gemini-2.5-flash-lite']
|
||||
}
|
||||
},
|
||||
{
|
||||
label: 'Thinking Level',
|
||||
name: 'thinkingLevel',
|
||||
type: 'options',
|
||||
description: 'Adjust the amount of reasoning effort based on the complexity of the user request',
|
||||
options: [
|
||||
{
|
||||
label: 'Low',
|
||||
name: 'LOW'
|
||||
},
|
||||
{
|
||||
label: 'Medium',
|
||||
name: 'MEDIUM'
|
||||
},
|
||||
{
|
||||
label: 'High',
|
||||
name: 'HIGH'
|
||||
}
|
||||
],
|
||||
optional: true,
|
||||
additionalParams: true,
|
||||
show: {
|
||||
modelName: ['gemini-3.1-pro-preview', 'gemini-3.1-flash-lite-preview', 'gemini-3-flash-preview']
|
||||
}
|
||||
},
|
||||
{
|
||||
label: 'Max Output Tokens',
|
||||
@@ -193,8 +232,9 @@ class GoogleVertexAI_ChatModels implements INode {
|
||||
const cache = nodeData.inputs?.cache as BaseCache
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const streaming = nodeData.inputs?.streaming as boolean
|
||||
const thinkingBudget = nodeData.inputs?.thinkingBudget as string
|
||||
const region = nodeData.inputs?.region as string
|
||||
const thinkingBudget = nodeData.inputs?.thinkingBudget as string
|
||||
const thinkingLevel = nodeData.inputs?.thinkingLevel as 'LOW' | 'MEDIUM' | 'HIGH'
|
||||
|
||||
const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean
|
||||
|
||||
@@ -204,7 +244,7 @@ class GoogleVertexAI_ChatModels implements INode {
|
||||
}
|
||||
}
|
||||
|
||||
const obj: ChatVertexAIInput = {
|
||||
const obj: ChatVertexAIInput & Partial<GoogleGenerativeAIChatInput> = {
|
||||
temperature: parseFloat(temperature),
|
||||
modelName: customModelName || modelName,
|
||||
streaming: streaming ?? true
|
||||
@@ -217,8 +257,18 @@ class GoogleVertexAI_ChatModels implements INode {
|
||||
if (topP) obj.topP = parseFloat(topP)
|
||||
if (cache) obj.cache = cache
|
||||
if (topK) obj.topK = parseFloat(topK)
|
||||
if (thinkingBudget) obj.thinkingBudget = parseInt(thinkingBudget, 10)
|
||||
if (region) obj.location = region
|
||||
if (thinkingLevel) {
|
||||
obj.thinkingConfig = {
|
||||
thinkingLevel: thinkingLevel,
|
||||
includeThoughts: true
|
||||
}
|
||||
} else if (thinkingBudget) {
|
||||
obj.thinkingConfig = {
|
||||
thinkingBudget: parseInt(thinkingBudget, 10),
|
||||
includeThoughts: true
|
||||
}
|
||||
}
|
||||
|
||||
const model = new ChatVertexAI(nodeData.id, obj)
|
||||
model.setMultiModalOption(multiModalOption)
|
||||
|
||||
@@ -16,7 +16,7 @@ class ChatHuggingFace_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatHuggingFace'
|
||||
this.label = 'HuggingFace'
|
||||
this.name = 'chatHuggingFace'
|
||||
this.version = 3.0
|
||||
this.type = 'ChatHuggingFace'
|
||||
|
||||
@@ -25,7 +25,7 @@ class ChatIBMWatsonx_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatIBMWatsonx'
|
||||
this.label = 'IBM Watsonx'
|
||||
this.name = 'chatIBMWatsonx'
|
||||
this.version = 2.0
|
||||
this.type = 'ChatIBMWatsonx'
|
||||
|
||||
@@ -18,7 +18,7 @@ class ChatLitellm_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatLitellm'
|
||||
this.label = 'LiteLLM'
|
||||
this.name = 'chatLitellm'
|
||||
this.version = 2.0
|
||||
this.type = 'ChatLitellm'
|
||||
@@ -68,6 +68,15 @@ class ChatLitellm_ChatModels implements INode {
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Allow Image Uploads',
|
||||
name: 'allowImageUploads',
|
||||
type: 'boolean',
|
||||
description:
|
||||
'Allow image input. Image uploads need a model marked supports_vision=true in LiteLLM. Refer to the <a href="https://docs.flowiseai.com/using-flowise/uploads#image" target="_blank">docs</a> for more details.',
|
||||
default: false,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Max Tokens',
|
||||
name: 'maxTokens',
|
||||
@@ -91,40 +100,6 @@ class ChatLitellm_ChatModels implements INode {
|
||||
step: 1,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Allow Image Uploads',
|
||||
name: 'allowImageUploads',
|
||||
type: 'boolean',
|
||||
description:
|
||||
'Allow image input. Image uploads need a model marked supports_vision=true in LiteLLM. Refer to the <a href="https://docs.flowiseai.com/using-flowise/uploads#image" target="_blank">docs</a> for more details.',
|
||||
default: false,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Image Resolution',
|
||||
description: 'This parameter controls the resolution in which the model views the image.',
|
||||
name: 'imageResolution',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
label: 'Low',
|
||||
name: 'low'
|
||||
},
|
||||
{
|
||||
label: 'High',
|
||||
name: 'high'
|
||||
},
|
||||
{
|
||||
label: 'Auto',
|
||||
name: 'auto'
|
||||
}
|
||||
],
|
||||
default: 'low',
|
||||
optional: false,
|
||||
show: {
|
||||
allowImageUploads: true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -139,7 +114,6 @@ class ChatLitellm_ChatModels implements INode {
|
||||
const topP = nodeData.inputs?.topP as string
|
||||
const timeout = nodeData.inputs?.timeout as string
|
||||
const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean
|
||||
const imageResolution = nodeData.inputs?.imageResolution as string
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const apiKey = getCredentialParam('litellmApiKey', credentialData, nodeData)
|
||||
@@ -168,8 +142,7 @@ class ChatLitellm_ChatModels implements INode {
|
||||
|
||||
const multiModalOption: IMultiModalOption = {
|
||||
image: {
|
||||
allowImageUploads: allowImageUploads ?? false,
|
||||
imageResolution
|
||||
allowImageUploads: allowImageUploads ?? false
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ class ChatLocalAI_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatLocalAI'
|
||||
this.label = 'LocalAI'
|
||||
this.name = 'chatLocalAI'
|
||||
this.version = 3.0
|
||||
this.type = 'ChatLocalAI'
|
||||
|
||||
@@ -17,7 +17,7 @@ class ChatMistral_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatMistralAI'
|
||||
this.label = 'MistralAI'
|
||||
this.name = 'chatMistralAI'
|
||||
this.version = 4.0
|
||||
this.type = 'ChatMistralAI'
|
||||
|
||||
@@ -82,7 +82,7 @@ class ChatNemoGuardrailsChatModel implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Chat Nemo Guardrails'
|
||||
this.label = 'Nemo Guardrails'
|
||||
this.name = 'chatNemoGuardrails'
|
||||
this.version = 1.0
|
||||
this.type = 'ChatNemoGuardrails'
|
||||
|
||||
@@ -16,7 +16,7 @@ class ChatNvdiaNIM_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Chat NVIDIA NIM'
|
||||
this.label = 'Nvidia NIM'
|
||||
this.name = 'chatNvidiaNIM'
|
||||
this.version = 1.1
|
||||
this.type = 'ChatNvidiaNIM'
|
||||
|
||||
@@ -18,7 +18,7 @@ class ChatOllama_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatOllama'
|
||||
this.label = 'Ollama'
|
||||
this.name = 'chatOllama'
|
||||
this.version = 5.0
|
||||
this.type = 'ChatOllama'
|
||||
@@ -62,6 +62,14 @@ class ChatOllama_ChatModels implements INode {
|
||||
default: 0.9,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Streaming',
|
||||
name: 'streaming',
|
||||
type: 'boolean',
|
||||
default: true,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Allow Image Uploads',
|
||||
name: 'allowImageUploads',
|
||||
@@ -71,14 +79,6 @@ class ChatOllama_ChatModels implements INode {
|
||||
default: false,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Streaming',
|
||||
name: 'streaming',
|
||||
type: 'boolean',
|
||||
default: true,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'JSON Mode',
|
||||
name: 'jsonMode',
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
import { ChatOpenAI as LangchainChatOpenAI, ChatOpenAIFields } from '@langchain/openai'
|
||||
import { BaseCache } from '@langchain/core/caches'
|
||||
import { ICommonObject, IMultiModalOption, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam, isReasoningModelOpenAI } from '../../../src/utils'
|
||||
import { ChatOpenAI } from './FlowiseChatOpenAI'
|
||||
import { getModels, MODEL_TYPE } from '../../../src/modelLoader'
|
||||
import { HttpsProxyAgent } from 'https-proxy-agent'
|
||||
import { OpenAI as OpenAIClient } from 'openai'
|
||||
|
||||
class ChatOpenAI_ChatModels implements INode {
|
||||
@@ -20,7 +19,7 @@ class ChatOpenAI_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatOpenAI'
|
||||
this.label = 'OpenAI'
|
||||
this.name = 'chatOpenAI'
|
||||
this.version = 8.3
|
||||
this.type = 'ChatOpenAI'
|
||||
@@ -64,6 +63,77 @@ class ChatOpenAI_ChatModels implements INode {
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Allow Image Uploads',
|
||||
name: 'allowImageUploads',
|
||||
type: 'boolean',
|
||||
description:
|
||||
'Allow image input. Refer to the <a href="https://docs.flowiseai.com/using-flowise/uploads#image" target="_blank">docs</a> for more details.',
|
||||
default: false,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Reasoning',
|
||||
description: 'Whether the model supports reasoning. Only applicable for reasoning models (gpt-5 and o-series models only)',
|
||||
name: 'reasoning',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Reasoning Effort',
|
||||
description: 'Constrains effort on reasoning. Only applicable for reasoning models (gpt-5 and o-series models only)',
|
||||
name: 'reasoningEffort',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
label: 'Low',
|
||||
name: 'low'
|
||||
},
|
||||
{
|
||||
label: 'Medium',
|
||||
name: 'medium'
|
||||
},
|
||||
{
|
||||
label: 'High',
|
||||
name: 'high'
|
||||
},
|
||||
{
|
||||
label: 'X-High',
|
||||
name: 'xhigh',
|
||||
description: 'X-High is supported for all models after gpt-5.1-codex-max'
|
||||
}
|
||||
],
|
||||
additionalParams: true,
|
||||
show: {
|
||||
reasoning: true
|
||||
}
|
||||
},
|
||||
{
|
||||
label: 'Reasoning Summary',
|
||||
description: `A summary of the reasoning performed by the model. This can be useful for debugging and understanding the model's reasoning process`,
|
||||
name: 'reasoningSummary',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
label: 'Auto',
|
||||
name: 'auto'
|
||||
},
|
||||
{
|
||||
label: 'Concise',
|
||||
name: 'concise'
|
||||
},
|
||||
{
|
||||
label: 'Detailed',
|
||||
name: 'detailed'
|
||||
}
|
||||
],
|
||||
additionalParams: true,
|
||||
show: {
|
||||
reasoning: true
|
||||
}
|
||||
},
|
||||
{
|
||||
label: 'Max Tokens',
|
||||
name: 'maxTokens',
|
||||
@@ -123,116 +193,20 @@ class ChatOpenAI_ChatModels implements INode {
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'BasePath',
|
||||
label: 'Base Path',
|
||||
name: 'basepath',
|
||||
type: 'string',
|
||||
optional: true,
|
||||
description: 'Override the default base URL for the API, e.g., "https://api.example.com/v2/',
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Proxy Url',
|
||||
name: 'proxyUrl',
|
||||
type: 'string',
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'BaseOptions',
|
||||
label: 'Base Options',
|
||||
name: 'baseOptions',
|
||||
type: 'json',
|
||||
optional: true,
|
||||
description: 'Default headers to include with every request to the API.',
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Allow Image Uploads',
|
||||
name: 'allowImageUploads',
|
||||
type: 'boolean',
|
||||
description:
|
||||
'Allow image input. Refer to the <a href="https://docs.flowiseai.com/using-flowise/uploads#image" target="_blank">docs</a> for more details.',
|
||||
default: false,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Image Resolution',
|
||||
description: 'This parameter controls the resolution in which the model views the image.',
|
||||
name: 'imageResolution',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
label: 'Low',
|
||||
name: 'low'
|
||||
},
|
||||
{
|
||||
label: 'High',
|
||||
name: 'high'
|
||||
},
|
||||
{
|
||||
label: 'Auto',
|
||||
name: 'auto'
|
||||
}
|
||||
],
|
||||
default: 'low',
|
||||
optional: false,
|
||||
show: {
|
||||
allowImageUploads: true
|
||||
}
|
||||
},
|
||||
{
|
||||
label: 'Reasoning',
|
||||
description: 'Whether the model supports reasoning. Only applicable for reasoning models.',
|
||||
name: 'reasoning',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Reasoning Effort',
|
||||
description: 'Constrains effort on reasoning for reasoning models',
|
||||
name: 'reasoningEffort',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
label: 'Low',
|
||||
name: 'low'
|
||||
},
|
||||
{
|
||||
label: 'Medium',
|
||||
name: 'medium'
|
||||
},
|
||||
{
|
||||
label: 'High',
|
||||
name: 'high'
|
||||
}
|
||||
],
|
||||
additionalParams: true,
|
||||
show: {
|
||||
reasoning: true
|
||||
}
|
||||
},
|
||||
{
|
||||
label: 'Reasoning Summary',
|
||||
description: `A summary of the reasoning performed by the model. This can be useful for debugging and understanding the model's reasoning process`,
|
||||
name: 'reasoningSummary',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
label: 'Auto',
|
||||
name: 'auto'
|
||||
},
|
||||
{
|
||||
label: 'Concise',
|
||||
name: 'concise'
|
||||
},
|
||||
{
|
||||
label: 'Detailed',
|
||||
name: 'detailed'
|
||||
}
|
||||
],
|
||||
additionalParams: true,
|
||||
show: {
|
||||
reasoning: true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -256,13 +230,10 @@ class ChatOpenAI_ChatModels implements INode {
|
||||
const streaming = nodeData.inputs?.streaming as boolean
|
||||
const strictToolCalling = nodeData.inputs?.strictToolCalling as boolean
|
||||
const basePath = nodeData.inputs?.basepath as string
|
||||
const proxyUrl = nodeData.inputs?.proxyUrl as string
|
||||
const baseOptions = nodeData.inputs?.baseOptions
|
||||
const reasoningEffort = nodeData.inputs?.reasoningEffort as OpenAIClient.ReasoningEffort | null
|
||||
const reasoningSummary = nodeData.inputs?.reasoningSummary as 'auto' | 'concise' | 'detailed' | null
|
||||
|
||||
const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean
|
||||
const imageResolution = nodeData.inputs?.imageResolution as string
|
||||
|
||||
if (nodeData.inputs?.credentialId) {
|
||||
nodeData.credential = nodeData.inputs?.credentialId
|
||||
@@ -280,7 +251,7 @@ class ChatOpenAI_ChatModels implements INode {
|
||||
streaming: streaming ?? true
|
||||
}
|
||||
|
||||
if (maxTokens) obj.maxTokens = parseInt(maxTokens, 10)
|
||||
if (maxTokens) obj.maxCompletionTokens = parseInt(maxTokens, 10)
|
||||
if (topP) obj.topP = parseFloat(topP)
|
||||
if (frequencyPenalty) obj.frequencyPenalty = parseFloat(frequencyPenalty)
|
||||
if (presencePenalty) obj.presencePenalty = parseFloat(presencePenalty)
|
||||
@@ -292,7 +263,7 @@ class ChatOpenAI_ChatModels implements INode {
|
||||
}
|
||||
if (strictToolCalling) obj.supportsStrictToolCalling = strictToolCalling
|
||||
|
||||
if (modelName.includes('o1') || modelName.includes('o3') || modelName.includes('gpt-5')) {
|
||||
if (isReasoningModelOpenAI(modelName)) {
|
||||
delete obj.temperature
|
||||
delete obj.stop
|
||||
const reasoning: OpenAIClient.Reasoning = {}
|
||||
@@ -322,22 +293,15 @@ class ChatOpenAI_ChatModels implements INode {
|
||||
}
|
||||
}
|
||||
|
||||
if (proxyUrl) {
|
||||
obj.configuration = {
|
||||
...obj?.configuration,
|
||||
httpAgent: new HttpsProxyAgent(proxyUrl)
|
||||
}
|
||||
}
|
||||
|
||||
const multiModalOption: IMultiModalOption = {
|
||||
image: {
|
||||
allowImageUploads: allowImageUploads ?? false,
|
||||
imageResolution
|
||||
allowImageUploads: allowImageUploads ?? false
|
||||
}
|
||||
}
|
||||
|
||||
const model = new ChatOpenAI(nodeData.id, obj)
|
||||
model.setMultiModalOption(multiModalOption)
|
||||
|
||||
return model
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ class ChatOpenAICustom_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatOpenAI Custom'
|
||||
this.label = 'OpenAI Custom Model'
|
||||
this.name = 'chatOpenAICustom'
|
||||
this.version = 4.0
|
||||
this.type = 'ChatOpenAI-Custom'
|
||||
@@ -101,17 +101,19 @@ class ChatOpenAICustom_ChatModels implements INode {
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'BasePath',
|
||||
label: 'Base Path',
|
||||
name: 'basepath',
|
||||
type: 'string',
|
||||
optional: true,
|
||||
description: 'Override the default base URL for the API, e.g., "https://api.example.com/v2/',
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'BaseOptions',
|
||||
label: 'Base Options',
|
||||
name: 'baseOptions',
|
||||
type: 'json',
|
||||
optional: true,
|
||||
description: 'Default headers to include with every request to the API.',
|
||||
additionalParams: true
|
||||
}
|
||||
]
|
||||
|
||||
@@ -17,7 +17,7 @@ class ChatOpenRouter_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatOpenRouter'
|
||||
this.label = 'OpenRouter'
|
||||
this.name = 'chatOpenRouter'
|
||||
this.version = 1.0
|
||||
this.type = 'ChatOpenRouter'
|
||||
@@ -61,6 +61,15 @@ class ChatOpenRouter_ChatModels implements INode {
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Allow Image Uploads',
|
||||
name: 'allowImageUploads',
|
||||
type: 'boolean',
|
||||
description:
|
||||
'Allow image input. Refer to the <a href="https://docs.flowiseai.com/using-flowise/uploads#image" target="_blank">docs</a> for more details.',
|
||||
default: false,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Max Tokens',
|
||||
name: 'maxTokens',
|
||||
@@ -102,53 +111,21 @@ class ChatOpenRouter_ChatModels implements INode {
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'BasePath',
|
||||
label: 'Base Path',
|
||||
name: 'basepath',
|
||||
type: 'string',
|
||||
optional: true,
|
||||
default: 'https://openrouter.ai/api/v1',
|
||||
description: 'Override the default base URL for the API, e.g., "https://api.example.com/v2/',
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'BaseOptions',
|
||||
label: 'Base Options',
|
||||
name: 'baseOptions',
|
||||
type: 'json',
|
||||
optional: true,
|
||||
description: 'Default headers to include with every request to the API.',
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Allow Image Uploads',
|
||||
name: 'allowImageUploads',
|
||||
type: 'boolean',
|
||||
description:
|
||||
'Allow image input. Refer to the <a href="https://docs.flowiseai.com/using-flowise/uploads#image" target="_blank">docs</a> for more details.',
|
||||
default: false,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Image Resolution',
|
||||
description: 'This parameter controls the resolution in which the model views the image.',
|
||||
name: 'imageResolution',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
label: 'Low',
|
||||
name: 'low'
|
||||
},
|
||||
{
|
||||
label: 'High',
|
||||
name: 'high'
|
||||
},
|
||||
{
|
||||
label: 'Auto',
|
||||
name: 'auto'
|
||||
}
|
||||
],
|
||||
default: 'low',
|
||||
optional: false,
|
||||
show: {
|
||||
allowImageUploads: true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -166,7 +143,6 @@ class ChatOpenRouter_ChatModels implements INode {
|
||||
const baseOptions = nodeData.inputs?.baseOptions
|
||||
const cache = nodeData.inputs?.cache as BaseCache
|
||||
const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean
|
||||
const imageResolution = nodeData.inputs?.imageResolution as string
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const openRouterApiKey = getCredentialParam('openRouterApiKey', credentialData, nodeData)
|
||||
@@ -205,8 +181,7 @@ class ChatOpenRouter_ChatModels implements INode {
|
||||
|
||||
const multiModalOption: IMultiModalOption = {
|
||||
image: {
|
||||
allowImageUploads: allowImageUploads ?? false,
|
||||
imageResolution
|
||||
allowImageUploads: allowImageUploads ?? false
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ class ChatPerplexity_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatPerplexity'
|
||||
this.label = 'Perplexity'
|
||||
this.name = 'chatPerplexity'
|
||||
this.version = 0.1
|
||||
this.type = 'ChatPerplexity'
|
||||
|
||||
@@ -16,7 +16,7 @@ class ChatSambanova_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatSambanova'
|
||||
this.label = 'SambaNova'
|
||||
this.name = 'chatSambanova'
|
||||
this.version = 1.0
|
||||
this.type = 'ChatSambanova'
|
||||
@@ -60,18 +60,20 @@ class ChatSambanova_ChatModels implements INode {
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'BasePath',
|
||||
label: 'Base Path',
|
||||
name: 'basepath',
|
||||
type: 'string',
|
||||
optional: true,
|
||||
default: 'htps://api.sambanova.ai/v1',
|
||||
description: 'Override the default base URL for the API, e.g., "https://api.example.com/v2/',
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'BaseOptions',
|
||||
label: 'Base Options',
|
||||
name: 'baseOptions',
|
||||
type: 'json',
|
||||
optional: true,
|
||||
description: 'Default headers to include with every request to the API.',
|
||||
additionalParams: true
|
||||
}
|
||||
]
|
||||
|
||||
@@ -16,7 +16,7 @@ class ChatTogetherAI_ChatModels implements INode {
|
||||
credential: INodeParams
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatTogetherAI'
|
||||
this.label = 'TogetherAI'
|
||||
this.name = 'chatTogetherAI'
|
||||
this.version = 2.0
|
||||
this.type = 'ChatTogetherAI'
|
||||
|
||||
@@ -17,7 +17,7 @@ class ChatXAI_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatXAI'
|
||||
this.label = 'xAI Grok'
|
||||
this.name = 'chatXAI'
|
||||
this.version = 2.0
|
||||
this.type = 'ChatXAI'
|
||||
@@ -60,22 +60,6 @@ class ChatXAI_ChatModels implements INode {
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Max Tokens',
|
||||
name: 'maxTokens',
|
||||
type: 'number',
|
||||
step: 1,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Max Tokens',
|
||||
name: 'maxTokens',
|
||||
type: 'number',
|
||||
step: 1,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Allow Image Uploads',
|
||||
name: 'allowImageUploads',
|
||||
@@ -84,6 +68,22 @@ class ChatXAI_ChatModels implements INode {
|
||||
'Allow image input. Refer to the <a href="https://docs.flowiseai.com/using-flowise/uploads#image" target="_blank">docs</a> for more details.',
|
||||
default: false,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Max Tokens',
|
||||
name: 'maxTokens',
|
||||
type: 'number',
|
||||
step: 1,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Max Tokens',
|
||||
name: 'maxTokens',
|
||||
type: 'number',
|
||||
step: 1,
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ export class ChatXAI extends LCChatXAI implements IVisionChatModal {
|
||||
}
|
||||
|
||||
revertToOriginalModel(): void {
|
||||
this.modelName = this.configuredModel
|
||||
this.model = this.configuredModel
|
||||
this.maxTokens = this.configuredMaxToken
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
import { BaseCache } from '@langchain/core/caches'
|
||||
import { ChatOpenAI, ChatOpenAIFields } from '@langchain/openai'
|
||||
import { ChatDeepSeek, ChatDeepSeekInput } from '@langchain/deepseek'
|
||||
import { ICommonObject, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface'
|
||||
import { getModels, MODEL_TYPE } from '../../../src/modelLoader'
|
||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
|
||||
class Deepseek_ChatModels implements INode {
|
||||
readonly baseURL: string = 'https://api.deepseek.com'
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
@@ -18,14 +17,14 @@ class Deepseek_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'ChatDeepseek'
|
||||
this.label = 'Deepseek'
|
||||
this.name = 'chatDeepseek'
|
||||
this.version = 1.0
|
||||
this.type = 'chatDeepseek'
|
||||
this.icon = 'deepseek.svg'
|
||||
this.category = 'Chat Models'
|
||||
this.description = 'Wrapper around Deepseek large language models that use the Chat endpoint'
|
||||
this.baseClasses = [this.type, ...getBaseClasses(ChatOpenAI)]
|
||||
this.baseClasses = [this.type, ...getBaseClasses(ChatDeepSeek)]
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
@@ -111,6 +110,16 @@ class Deepseek_ChatModels implements INode {
|
||||
description: 'List of stop words to use when generating. Use comma to separate multiple stop words.',
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Thinking',
|
||||
name: 'thinking',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
optional: true,
|
||||
additionalParams: true,
|
||||
description:
|
||||
'Enable deep thinking mode for complex reasoning tasks. When enabled, the model will use extended thinking before responding.'
|
||||
},
|
||||
{
|
||||
label: 'Base Options',
|
||||
name: 'baseOptions',
|
||||
@@ -139,6 +148,7 @@ class Deepseek_ChatModels implements INode {
|
||||
const timeout = nodeData.inputs?.timeout as string
|
||||
const stopSequence = nodeData.inputs?.stopSequence as string
|
||||
const streaming = nodeData.inputs?.streaming as boolean
|
||||
const thinking = nodeData.inputs?.thinking as boolean
|
||||
const baseOptions = nodeData.inputs?.baseOptions
|
||||
|
||||
if (nodeData.inputs?.credentialId) {
|
||||
@@ -149,7 +159,7 @@ class Deepseek_ChatModels implements INode {
|
||||
|
||||
const cache = nodeData.inputs?.cache as BaseCache
|
||||
|
||||
const obj: ChatOpenAIFields = {
|
||||
const obj: ChatDeepSeekInput = {
|
||||
temperature: parseFloat(temperature),
|
||||
modelName,
|
||||
openAIApiKey,
|
||||
@@ -167,28 +177,25 @@ class Deepseek_ChatModels implements INode {
|
||||
const stopSequenceArray = stopSequence.split(',').map((item) => item.trim())
|
||||
obj.stop = stopSequenceArray
|
||||
}
|
||||
|
||||
let parsedBaseOptions: any | undefined = undefined
|
||||
if (thinking) {
|
||||
obj.modelKwargs = {
|
||||
...obj.modelKwargs,
|
||||
thinking: { type: 'enabled' }
|
||||
}
|
||||
}
|
||||
|
||||
if (baseOptions) {
|
||||
try {
|
||||
parsedBaseOptions = typeof baseOptions === 'object' ? baseOptions : JSON.parse(baseOptions)
|
||||
if (parsedBaseOptions.baseURL) {
|
||||
console.warn("The 'baseURL' parameter is not allowed when using the ChatDeepseek node.")
|
||||
parsedBaseOptions.baseURL = undefined
|
||||
const parsedBaseOptions = typeof baseOptions === 'object' ? baseOptions : JSON.parse(baseOptions)
|
||||
obj.configuration = {
|
||||
defaultHeaders: parsedBaseOptions
|
||||
}
|
||||
} catch (exception) {
|
||||
throw new Error('Invalid JSON in the BaseOptions: ' + exception)
|
||||
}
|
||||
}
|
||||
|
||||
const model = new ChatOpenAI({
|
||||
...obj,
|
||||
configuration: {
|
||||
baseURL: this.baseURL,
|
||||
...parsedBaseOptions
|
||||
}
|
||||
})
|
||||
const model = new ChatDeepSeek(obj)
|
||||
return model
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ class Groq_ChatModels implements INode {
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'GroqChat'
|
||||
this.label = 'Groq'
|
||||
this.name = 'groqChat'
|
||||
this.version = 4.0
|
||||
this.type = 'GroqChat'
|
||||
@@ -90,7 +90,7 @@ class Groq_ChatModels implements INode {
|
||||
const groqApiKey = getCredentialParam('groqApiKey', credentialData, nodeData)
|
||||
|
||||
const obj: ChatGroqInput = {
|
||||
modelName,
|
||||
model: modelName,
|
||||
temperature: parseFloat(temperature),
|
||||
apiKey: groqApiKey,
|
||||
streaming: streaming ?? true
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { AxiosRequestConfig } from 'axios'
|
||||
import { secureAxiosRequest } from '../../../src/httpSecurity'
|
||||
import { BaseDocumentLoader } from 'langchain/document_loaders/base'
|
||||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { BaseDocumentLoader } from '@langchain/classic/document_loaders/base'
|
||||
import { TextSplitter } from '@langchain/textsplitters'
|
||||
import { omit } from 'lodash'
|
||||
import { getFileFromStorage } from '../../../src'
|
||||
import { ICommonObject, IDocument, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import axios from 'axios'
|
||||
import { omit } from 'lodash'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { BaseDocumentLoader } from 'langchain/document_loaders/base'
|
||||
import { TextSplitter } from '@langchain/textsplitters'
|
||||
import { BaseDocumentLoader } from '@langchain/classic/document_loaders/base'
|
||||
import { getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { IDocument, ICommonObject, INode, INodeData, INodeParams, INodeOutputsValue } from '../../../src/Interface'
|
||||
import { handleEscapeCharacters } from '../../../src'
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
import { omit } from 'lodash'
|
||||
import { INode, INodeData, INodeParams, ICommonObject, INodeOutputsValue } from '../../../src/Interface'
|
||||
import { getCredentialData, getCredentialParam, handleEscapeCharacters } from '../../../src/utils'
|
||||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { TextSplitter } from '@langchain/textsplitters'
|
||||
import { ApifyDatasetLoader } from '@langchain/community/document_loaders/web/apify_dataset'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { omit } from 'lodash'
|
||||
import { ICommonObject, IDocument, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { TextSplitter } from '@langchain/textsplitters'
|
||||
import { BraveSearch } from '@langchain/community/tools/brave_search'
|
||||
import { getCredentialData, getCredentialParam, handleEscapeCharacters } from '../../../src/utils'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { TextSplitter } from '@langchain/textsplitters'
|
||||
import { omit } from 'lodash'
|
||||
import { CheerioWebBaseLoader, WebBaseLoaderParams } from '@langchain/community/document_loaders/web/cheerio'
|
||||
import { CheerioWebBaseLoader, CheerioWebBaseLoaderParams } from '@langchain/community/document_loaders/web/cheerio'
|
||||
import { test } from 'linkifyjs'
|
||||
import { parse } from 'css-what'
|
||||
import { SelectorType } from 'cheerio'
|
||||
@@ -140,7 +140,7 @@ class Cheerio_DocumentLoaders implements INode {
|
||||
|
||||
const selector: SelectorType = nodeData.inputs?.selector as SelectorType
|
||||
|
||||
let params: WebBaseLoaderParams = {}
|
||||
let params: CheerioWebBaseLoaderParams = {}
|
||||
if (selector) {
|
||||
parse(selector) // comes with cheerio - will throw error if invalid
|
||||
params['selector'] = selector
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { omit } from 'lodash'
|
||||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { TextSplitter } from '@langchain/textsplitters'
|
||||
import { ConfluencePagesLoader, ConfluencePagesLoaderParams } from '@langchain/community/document_loaders/web/confluence'
|
||||
import { getCredentialData, getCredentialParam, handleEscapeCharacters } from '../../../src/utils'
|
||||
import { ICommonObject, INode, INodeData, INodeParams, INodeOutputsValue } from '../../../src/Interface'
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { TextSplitter } from '@langchain/textsplitters'
|
||||
import { CSVLoader } from './CsvLoader'
|
||||
import { getFileFromStorage, handleDocumentLoaderDocuments, handleDocumentLoaderMetadata, handleDocumentLoaderOutput } from '../../../src'
|
||||
import { ICommonObject, IDocument, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
@@ -135,7 +135,7 @@ class Csv_DocumentLoaders implements INode {
|
||||
if (!file) continue
|
||||
|
||||
const fileData = await this.getFileData(file, { orgId, chatflowid }, fromStorage)
|
||||
const blob = new Blob([fileData])
|
||||
const blob = new Blob([new Uint8Array(fileData)])
|
||||
const loader = new CSVLoader(blob, columnName.trim().length === 0 ? undefined : columnName.trim())
|
||||
|
||||
// use spread instead of push, because it raises RangeError: Maximum call stack size exceeded when too many docs
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { TextLoader } from 'langchain/document_loaders/fs/text'
|
||||
import { TextLoader } from '@langchain/classic/document_loaders/fs/text'
|
||||
import Papa from 'papaparse'
|
||||
|
||||
type CSVLoaderOptions = {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { omit } from 'lodash'
|
||||
import { ICommonObject, IDocument, INode, INodeData, INodeParams, INodeOutputsValue } from '../../../src/Interface'
|
||||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { TextSplitter } from '@langchain/textsplitters'
|
||||
import { DocxLoader } from '@langchain/community/document_loaders/fs/docx'
|
||||
import { getFileFromStorage, handleEscapeCharacters } from '../../../src'
|
||||
|
||||
@@ -102,7 +102,7 @@ class Docx_DocumentLoaders implements INode {
|
||||
for (const file of files) {
|
||||
if (!file) continue
|
||||
const fileData = await getFileFromStorage(file, orgId, chatflowid)
|
||||
const blob = new Blob([fileData])
|
||||
const blob = new Blob([new Uint8Array(fileData)])
|
||||
const loader = new DocxLoader(blob)
|
||||
|
||||
if (textSplitter) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { omit } from 'lodash'
|
||||
import { IDocument, ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
|
||||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { TextSplitter } from '@langchain/textsplitters'
|
||||
import { getFileFromStorage, handleEscapeCharacters, INodeOutputsValue } from '../../../src'
|
||||
import { EPubLoader } from '@langchain/community/document_loaders/fs/epub'
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import { omit } from 'lodash'
|
||||
import { getCredentialData, getCredentialParam, handleEscapeCharacters } from '../../../src'
|
||||
import { ICommonObject, IDocument, INode, INodeData, INodeParams, INodeOutputsValue } from '../../../src/Interface'
|
||||
import { FigmaFileLoader, FigmaLoaderParams } from '@langchain/community/document_loaders/web/figma'
|
||||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { TextSplitter } from '@langchain/textsplitters'
|
||||
|
||||
class Figma_DocumentLoaders implements INode {
|
||||
label: string
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
import { omit } from 'lodash'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { TextLoader } from 'langchain/document_loaders/fs/text'
|
||||
import { JSONLinesLoader, JSONLoader } from 'langchain/document_loaders/fs/json'
|
||||
import { TextSplitter } from '@langchain/textsplitters'
|
||||
import { TextLoader } from '@langchain/classic/document_loaders/fs/text'
|
||||
import { JSONLinesLoader, JSONLoader } from '@langchain/classic/document_loaders/fs/json'
|
||||
import { CSVLoader } from '@langchain/community/document_loaders/fs/csv'
|
||||
import { PDFLoader } from '@langchain/community/document_loaders/fs/pdf'
|
||||
import { DocxLoader } from '@langchain/community/document_loaders/fs/docx'
|
||||
import { BaseDocumentLoader } from 'langchain/document_loaders/base'
|
||||
import { BaseDocumentLoader } from '@langchain/classic/document_loaders/base'
|
||||
import { LoadOfSheet } from '../MicrosoftExcel/ExcelLoader'
|
||||
import { PowerpointLoader } from '../MicrosoftPowerpoint/PowerpointLoader'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
@@ -156,14 +156,14 @@ class File_DocumentLoaders implements INode {
|
||||
for (const file of files) {
|
||||
if (!file) continue
|
||||
const fileData = await getFileFromStorage(file, orgId, chatflowid, options.chatId)
|
||||
const blob = new Blob([fileData])
|
||||
const blob = new Blob([new Uint8Array(fileData)])
|
||||
fileBlobs.push({ blob, ext: file.split('.').pop() || '' })
|
||||
}
|
||||
} else {
|
||||
for (const file of files) {
|
||||
if (!file) continue
|
||||
const fileData = await getFileFromStorage(file, orgId, chatflowid)
|
||||
const blob = new Blob([fileData])
|
||||
const blob = new Blob([new Uint8Array(fileData)])
|
||||
fileBlobs.push({ blob, ext: file.split('.').pop() || '' })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { TextSplitter } from '@langchain/textsplitters'
|
||||
import { Document, DocumentInterface } from '@langchain/core/documents'
|
||||
import { BaseDocumentLoader } from 'langchain/document_loaders/base'
|
||||
import { BaseDocumentLoader } from '@langchain/classic/document_loaders/base'
|
||||
import { INode, INodeData, INodeParams, ICommonObject, INodeOutputsValue } from '../../../src/Interface'
|
||||
import { getCredentialData, getCredentialParam, handleEscapeCharacters } from '../../../src/utils'
|
||||
import { AxiosResponse, AxiosRequestHeaders } from 'axios'
|
||||
import { secureAxiosRequest } from '../../../src/httpSecurity'
|
||||
import { z } from 'zod'
|
||||
import { z } from 'zod/v3'
|
||||
|
||||
// FirecrawlApp interfaces
|
||||
interface FirecrawlAppConfig {
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import { omit } from 'lodash'
|
||||
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { TextLoader } from 'langchain/document_loaders/fs/text'
|
||||
import { DirectoryLoader } from 'langchain/document_loaders/fs/directory'
|
||||
import { JSONLinesLoader, JSONLoader } from 'langchain/document_loaders/fs/json'
|
||||
import { TextSplitter } from '@langchain/textsplitters'
|
||||
import { TextLoader } from '@langchain/classic/document_loaders/fs/text'
|
||||
import { DirectoryLoader } from '@langchain/classic/document_loaders/fs/directory'
|
||||
import { JSONLinesLoader, JSONLoader } from '@langchain/classic/document_loaders/fs/json'
|
||||
import { CSVLoader } from '@langchain/community/document_loaders/fs/csv'
|
||||
import { PDFLoader } from '@langchain/community/document_loaders/fs/pdf'
|
||||
import { DocxLoader } from '@langchain/community/document_loaders/fs/docx'
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { omit } from 'lodash'
|
||||
import { IDocument, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { TextSplitter } from '@langchain/textsplitters'
|
||||
import { GitbookLoader } from '@langchain/community/document_loaders/web/gitbook'
|
||||
import { handleEscapeCharacters } from '../../../src/utils'
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { omit } from 'lodash'
|
||||
import { ICommonObject, IDocument, INode, INodeData, INodeParams } from '../../../src/Interface'
|
||||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { TextSplitter } from '@langchain/textsplitters'
|
||||
import { GithubRepoLoader, GithubRepoLoaderParams } from '@langchain/community/document_loaders/web/github'
|
||||
import { getCredentialData, getCredentialParam, handleEscapeCharacters, INodeOutputsValue } from '../../../src'
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { omit } from 'lodash'
|
||||
import { ICommonObject, IDocument, INode, INodeData, INodeParams, INodeOptionsValue } from '../../../src/Interface'
|
||||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { TextSplitter } from '@langchain/textsplitters'
|
||||
import {
|
||||
convertMultiOptionsToStringArray,
|
||||
getCredentialData,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { omit } from 'lodash'
|
||||
import { ICommonObject, IDocument, INode, INodeData, INodeParams, INodeOptionsValue } from '../../../src/Interface'
|
||||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { TextSplitter } from '@langchain/textsplitters'
|
||||
import {
|
||||
convertMultiOptionsToStringArray,
|
||||
getCredentialData,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { omit } from 'lodash'
|
||||
import { ICommonObject, IDocument, INode, INodeData, INodeParams } from '../../../src/Interface'
|
||||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { TextSplitter } from '@langchain/textsplitters'
|
||||
import { JiraProjectLoaderParams, JiraProjectLoader } from '@langchain/community/document_loaders/web/jira'
|
||||
import { getCredentialData, getCredentialParam, handleEscapeCharacters, INodeOutputsValue } from '../../../src'
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user