mirror of
https://github.com/FlowiseAI/Flowise.git
synced 2026-10-12 02:02:17 +00:00
Fix: Disclosures (#5562)
* fix: remove hostname from log filename * fix: remove metrics endpoint from whitelist and require authentication * update: example prometheus configuration * fix: require authentication for bullmq dashboard * remove: get-upload-path route * update: move export chat messages logic to backend * update: auth method for bullmq dashboard * fix: hard-coded api key in example prometheus config * fix: unused imports * fix: duplicated try-catch blocks * fix: add rate-limiting to bullmq dashboard * fix: add rate-limiting to bullmq dashboard * update: use existing json parse method
This commit is contained in:
@@ -6,4 +6,8 @@ scrape_configs:
|
||||
- targets: ["localhost:8080","localhost:3000"]
|
||||
|
||||
metrics_path: /api/v1/metrics/
|
||||
scheme: http
|
||||
scheme: http
|
||||
|
||||
authorization:
|
||||
type: Bearer
|
||||
credentials_file: '/etc/prometheus/api_key.txt'
|
||||
|
||||
@@ -169,7 +169,6 @@ JWT_REFRESH_TOKEN_EXPIRY_IN_MINUTES=43200
|
||||
# REDIS_KEEP_ALIVE=
|
||||
# ENABLE_BULLMQ_DASHBOARD=
|
||||
|
||||
|
||||
############################################################################################################
|
||||
############################################## SECURITY ####################################################
|
||||
############################################################################################################
|
||||
|
||||
@@ -49,7 +49,45 @@ const importData = async (req: Request, res: Response, next: NextFunction) => {
|
||||
}
|
||||
}
|
||||
|
||||
const exportChatflowMessages = async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const workspaceId = req.user?.activeWorkspaceId
|
||||
if (!workspaceId) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.NOT_FOUND,
|
||||
`Error: exportImportController.exportChatflowMessages - workspace ${workspaceId} not found!`
|
||||
)
|
||||
}
|
||||
|
||||
const { chatflowId, chatType, feedbackType, startDate, endDate } = req.body
|
||||
if (!chatflowId) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.BAD_REQUEST,
|
||||
'Error: exportImportController.exportChatflowMessages - chatflowId is required!'
|
||||
)
|
||||
}
|
||||
|
||||
const apiResponse = await exportImportService.exportChatflowMessages(
|
||||
chatflowId,
|
||||
chatType,
|
||||
feedbackType,
|
||||
startDate,
|
||||
endDate,
|
||||
workspaceId
|
||||
)
|
||||
|
||||
// Set headers for file download
|
||||
res.setHeader('Content-Type', 'application/json')
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${chatflowId}-Message.json"`)
|
||||
|
||||
return res.json(apiResponse)
|
||||
} catch (error) {
|
||||
next(error)
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
exportData,
|
||||
importData
|
||||
importData,
|
||||
exportChatflowMessages
|
||||
}
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
import { Request, Response, NextFunction } from 'express'
|
||||
import { getStoragePath } from 'flowise-components'
|
||||
|
||||
const getPathForUploads = async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const apiResponse = {
|
||||
storagePath: getStoragePath()
|
||||
}
|
||||
return res.json(apiResponse)
|
||||
} catch (error) {
|
||||
next(error)
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
getPathForUploads
|
||||
}
|
||||
@@ -429,6 +429,34 @@ export const verifyToken = (req: Request, res: Response, next: NextFunction) =>
|
||||
})(req, res, next)
|
||||
}
|
||||
|
||||
export const verifyTokenForBullMQDashboard = (req: Request, res: Response, next: NextFunction) => {
|
||||
passport.authenticate('jwt', { session: true }, (err: any, user: LoggedInUser, info: object) => {
|
||||
if (err) {
|
||||
return next(err)
|
||||
}
|
||||
|
||||
// @ts-ignore
|
||||
if (info && info.name === 'TokenExpiredError') {
|
||||
if (req.cookies && req.cookies.refreshToken) {
|
||||
return res.redirect('/signin?retry=true')
|
||||
}
|
||||
return res.redirect('/signin')
|
||||
}
|
||||
|
||||
if (!user) {
|
||||
return res.redirect('/signin')
|
||||
}
|
||||
|
||||
const identityManager = getRunningExpressApp().identityManager
|
||||
if (identityManager.isEnterprise() && !identityManager.isLicenseValid()) {
|
||||
return res.redirect('/license-expired')
|
||||
}
|
||||
|
||||
req.user = user
|
||||
next()
|
||||
})(req, res, next)
|
||||
}
|
||||
|
||||
const storeSSOUserPayload = (ssoToken: string, returnUser: any) => {
|
||||
const app = getRunningExpressApp()
|
||||
app.cachePool.addSSOTokenCache(ssoToken, returnUser)
|
||||
|
||||
@@ -18,7 +18,7 @@ import { Telemetry } from './utils/telemetry'
|
||||
import flowiseApiV1Router from './routes'
|
||||
import errorHandlerMiddleware from './middlewares/errors'
|
||||
import { WHITELIST_URLS } from './utils/constants'
|
||||
import { initializeJwtCookieMiddleware, verifyToken } from './enterprise/middleware/passport'
|
||||
import { initializeJwtCookieMiddleware, verifyToken, verifyTokenForBullMQDashboard } from './enterprise/middleware/passport'
|
||||
import { IdentityManager } from './IdentityManager'
|
||||
import { SSEStreamer } from './utils/SSEStreamer'
|
||||
import { validateAPIKey } from './utils/validateKey'
|
||||
@@ -331,7 +331,17 @@ export class App {
|
||||
})
|
||||
|
||||
if (process.env.MODE === MODE.QUEUE && process.env.ENABLE_BULLMQ_DASHBOARD === 'true' && !this.identityManager.isCloud()) {
|
||||
this.app.use('/admin/queues', this.queueManager.getBullBoardRouter())
|
||||
// Initialize admin queues rate limiter
|
||||
const id = 'bullmq_admin_dashboard'
|
||||
await this.rateLimiterManager.addRateLimiter(
|
||||
id,
|
||||
60,
|
||||
100,
|
||||
process.env.ADMIN_RATE_LIMIT_MESSAGE || 'Too many requests to admin dashboard, please try again later.'
|
||||
)
|
||||
|
||||
const rateLimiter = this.rateLimiterManager.getRateLimiterById(id)
|
||||
this.app.use('/admin/queues', rateLimiter, verifyTokenForBullMQDashboard, this.queueManager.getBullBoardRouter())
|
||||
}
|
||||
|
||||
// ----------------------------------------
|
||||
|
||||
@@ -5,6 +5,8 @@ const router = express.Router()
|
||||
|
||||
router.post('/export', checkPermission('workspace:export'), exportImportController.exportData)
|
||||
|
||||
router.post('/chatflow-messages', checkPermission('workspace:export'), exportImportController.exportChatflowMessages)
|
||||
|
||||
router.post('/import', checkPermission('workspace:import'), exportImportController.importData)
|
||||
|
||||
export default router
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
import express from 'express'
|
||||
import getUploadPathController from '../../controllers/get-upload-path'
|
||||
const router = express.Router()
|
||||
|
||||
// READ
|
||||
router.get('/', getUploadPathController.getPathForUploads)
|
||||
|
||||
export default router
|
||||
@@ -19,7 +19,6 @@ import fetchLinksRouter from './fetch-links'
|
||||
import filesRouter from './files'
|
||||
import flowConfigRouter from './flow-config'
|
||||
import getUploadFileRouter from './get-upload-file'
|
||||
import getUploadPathRouter from './get-upload-path'
|
||||
import internalChatmessagesRouter from './internal-chat-messages'
|
||||
import internalPredictionRouter from './internal-predictions'
|
||||
import leadsRouter from './leads'
|
||||
@@ -93,7 +92,6 @@ router.use('/flow-config', flowConfigRouter)
|
||||
router.use('/internal-chatmessage', internalChatmessagesRouter)
|
||||
router.use('/internal-prediction', internalPredictionRouter)
|
||||
router.use('/get-upload-file', getUploadFileRouter)
|
||||
router.use('/get-upload-path', getUploadPathRouter)
|
||||
router.use('/leads', leadsRouter)
|
||||
router.use('/load-prompt', loadPromptRouter)
|
||||
router.use('/marketplaces', marketplacesRouter)
|
||||
|
||||
@@ -13,20 +13,21 @@ import { Tool } from '../../database/entities/Tool'
|
||||
import { Variable } from '../../database/entities/Variable'
|
||||
import { InternalFlowiseError } from '../../errors/internalFlowiseError'
|
||||
import { getErrorMessage } from '../../errors/utils'
|
||||
import { Platform } from '../../Interface'
|
||||
import assistantsService from '../../services/assistants'
|
||||
import chatflowsService from '../../services/chatflows'
|
||||
import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
|
||||
import { checkUsageLimit } from '../../utils/quotaUsage'
|
||||
import { sanitizeNullBytes } from '../../utils/sanitize.util'
|
||||
import assistantService from '../assistants'
|
||||
import chatMessagesService from '../chat-messages'
|
||||
import assistantsService from '../assistants'
|
||||
import chatflowService from '../chatflows'
|
||||
import chatMessagesService from '../chat-messages'
|
||||
import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
|
||||
import { utilGetChatMessage } from '../../utils/getChatMessage'
|
||||
import { getStoragePath, parseJsonBody } from 'flowise-components'
|
||||
import path from 'path'
|
||||
import { checkUsageLimit } from '../../utils/quotaUsage'
|
||||
import documenStoreService from '../documentstore'
|
||||
import executionService, { ExecutionFilters } from '../executions'
|
||||
import marketplacesService from '../marketplaces'
|
||||
import toolsService from '../tools'
|
||||
import variableService from '../variables'
|
||||
import { ChatMessageRatingType, ChatType, Platform } from '../../Interface'
|
||||
import { sanitizeNullBytes } from '../../utils/sanitize.util'
|
||||
|
||||
type ExportInput = {
|
||||
agentflow: boolean
|
||||
@@ -101,17 +102,17 @@ const exportData = async (exportInput: ExportInput, activeWorkspaceId: string):
|
||||
AgentFlowV2 = 'data' in AgentFlowV2 ? AgentFlowV2.data : AgentFlowV2
|
||||
|
||||
let AssistantCustom: Assistant[] =
|
||||
exportInput.assistantCustom === true ? await assistantService.getAllAssistants(activeWorkspaceId, 'CUSTOM') : []
|
||||
exportInput.assistantCustom === true ? await assistantsService.getAllAssistants(activeWorkspaceId, 'CUSTOM') : []
|
||||
|
||||
let AssistantFlow: ChatFlow[] | { data: ChatFlow[]; total: number } =
|
||||
exportInput.assistantCustom === true ? await chatflowService.getAllChatflows('ASSISTANT', activeWorkspaceId) : []
|
||||
AssistantFlow = 'data' in AssistantFlow ? AssistantFlow.data : AssistantFlow
|
||||
|
||||
let AssistantOpenAI: Assistant[] =
|
||||
exportInput.assistantOpenAI === true ? await assistantService.getAllAssistants(activeWorkspaceId, 'OPENAI') : []
|
||||
exportInput.assistantOpenAI === true ? await assistantsService.getAllAssistants(activeWorkspaceId, 'OPENAI') : []
|
||||
|
||||
let AssistantAzure: Assistant[] =
|
||||
exportInput.assistantAzure === true ? await assistantService.getAllAssistants(activeWorkspaceId, 'AZURE') : []
|
||||
exportInput.assistantAzure === true ? await assistantsService.getAllAssistants(activeWorkspaceId, 'AZURE') : []
|
||||
|
||||
let ChatFlow: ChatFlow[] | { data: ChatFlow[]; total: number } =
|
||||
exportInput.chatflow === true ? await chatflowService.getAllChatflows('CHATFLOW', activeWorkspaceId) : []
|
||||
@@ -635,7 +636,7 @@ const importData = async (importData: ExportData, orgId: string, activeWorkspace
|
||||
importData.Tool = importData.Tool || []
|
||||
importData.Variable = importData.Variable || []
|
||||
|
||||
let queryRunner
|
||||
let queryRunner: QueryRunner
|
||||
try {
|
||||
queryRunner = getRunningExpressApp().AppDataSource.createQueryRunner()
|
||||
await queryRunner.connect()
|
||||
@@ -644,7 +645,7 @@ const importData = async (importData: ExportData, orgId: string, activeWorkspace
|
||||
if (importData.AgentFlow.length > 0) {
|
||||
importData.AgentFlow = reduceSpaceForChatflowFlowData(importData.AgentFlow)
|
||||
importData.AgentFlow = insertWorkspaceId(importData.AgentFlow, activeWorkspaceId)
|
||||
const existingChatflowCount = await chatflowsService.getAllChatflowsCountByOrganization('MULTIAGENT', orgId)
|
||||
const existingChatflowCount = await chatflowService.getAllChatflowsCountByOrganization('MULTIAGENT', orgId)
|
||||
const newChatflowCount = importData.AgentFlow.length
|
||||
await checkUsageLimit(
|
||||
'flows',
|
||||
@@ -657,7 +658,7 @@ const importData = async (importData: ExportData, orgId: string, activeWorkspace
|
||||
if (importData.AgentFlowV2.length > 0) {
|
||||
importData.AgentFlowV2 = reduceSpaceForChatflowFlowData(importData.AgentFlowV2)
|
||||
importData.AgentFlowV2 = insertWorkspaceId(importData.AgentFlowV2, activeWorkspaceId)
|
||||
const existingChatflowCount = await chatflowsService.getAllChatflowsCountByOrganization('AGENTFLOW', orgId)
|
||||
const existingChatflowCount = await chatflowService.getAllChatflowsCountByOrganization('AGENTFLOW', orgId)
|
||||
const newChatflowCount = importData.AgentFlowV2.length
|
||||
await checkUsageLimit(
|
||||
'flows',
|
||||
@@ -682,7 +683,7 @@ const importData = async (importData: ExportData, orgId: string, activeWorkspace
|
||||
if (importData.AssistantFlow.length > 0) {
|
||||
importData.AssistantFlow = reduceSpaceForChatflowFlowData(importData.AssistantFlow)
|
||||
importData.AssistantFlow = insertWorkspaceId(importData.AssistantFlow, activeWorkspaceId)
|
||||
const existingChatflowCount = await chatflowsService.getAllChatflowsCountByOrganization('ASSISTANT', orgId)
|
||||
const existingChatflowCount = await chatflowService.getAllChatflowsCountByOrganization('ASSISTANT', orgId)
|
||||
const newChatflowCount = importData.AssistantFlow.length
|
||||
await checkUsageLimit(
|
||||
'flows',
|
||||
@@ -719,7 +720,7 @@ const importData = async (importData: ExportData, orgId: string, activeWorkspace
|
||||
if (importData.ChatFlow.length > 0) {
|
||||
importData.ChatFlow = reduceSpaceForChatflowFlowData(importData.ChatFlow)
|
||||
importData.ChatFlow = insertWorkspaceId(importData.ChatFlow, activeWorkspaceId)
|
||||
const existingChatflowCount = await chatflowsService.getAllChatflowsCountByOrganization('CHATFLOW', orgId)
|
||||
const existingChatflowCount = await chatflowService.getAllChatflowsCountByOrganization('CHATFLOW', orgId)
|
||||
const newChatflowCount = importData.ChatFlow.length
|
||||
await checkUsageLimit(
|
||||
'flows',
|
||||
@@ -800,8 +801,160 @@ const importData = async (importData: ExportData, orgId: string, activeWorkspace
|
||||
}
|
||||
}
|
||||
|
||||
// Export chatflow messages
|
||||
const exportChatflowMessages = async (
|
||||
chatflowId: string,
|
||||
chatType?: ChatType[] | string,
|
||||
feedbackType?: ChatMessageRatingType[] | string,
|
||||
startDate?: string,
|
||||
endDate?: string,
|
||||
workspaceId?: string
|
||||
) => {
|
||||
try {
|
||||
// Parse chatType if it's a string
|
||||
let parsedChatTypes: ChatType[] | undefined
|
||||
if (chatType) {
|
||||
if (typeof chatType === 'string') {
|
||||
const parsed = parseJsonBody(chatType)
|
||||
parsedChatTypes = Array.isArray(parsed) ? parsed : [chatType as ChatType]
|
||||
} else if (Array.isArray(chatType)) {
|
||||
parsedChatTypes = chatType
|
||||
}
|
||||
}
|
||||
|
||||
// Parse feedbackType if it's a string
|
||||
let parsedFeedbackTypes: ChatMessageRatingType[] | undefined
|
||||
if (feedbackType) {
|
||||
if (typeof feedbackType === 'string') {
|
||||
const parsed = parseJsonBody(feedbackType)
|
||||
parsedFeedbackTypes = Array.isArray(parsed) ? parsed : [feedbackType as ChatMessageRatingType]
|
||||
} else if (Array.isArray(feedbackType)) {
|
||||
parsedFeedbackTypes = feedbackType
|
||||
}
|
||||
}
|
||||
|
||||
// Get all chat messages for the chatflow with feedback
|
||||
const chatMessages = await utilGetChatMessage({
|
||||
chatflowid: chatflowId,
|
||||
chatTypes: parsedChatTypes,
|
||||
feedbackTypes: parsedFeedbackTypes,
|
||||
startDate,
|
||||
endDate,
|
||||
sortOrder: 'DESC',
|
||||
feedback: true,
|
||||
activeWorkspaceId: workspaceId
|
||||
})
|
||||
|
||||
const storagePath = getStoragePath()
|
||||
const exportObj: { [key: string]: any } = {}
|
||||
|
||||
// Process each chat message
|
||||
for (const chatmsg of chatMessages) {
|
||||
const chatPK = getChatPK(chatmsg)
|
||||
const filePaths: string[] = []
|
||||
|
||||
// Handle file uploads
|
||||
if (chatmsg.fileUploads) {
|
||||
const uploads = parseJsonBody(chatmsg.fileUploads)
|
||||
if (Array.isArray(uploads)) {
|
||||
uploads.forEach((file: any) => {
|
||||
if (file.type === 'stored-file') {
|
||||
filePaths.push(path.join(storagePath, chatmsg.chatflowid, chatmsg.chatId, file.name))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Create message object
|
||||
const msg: any = {
|
||||
content: chatmsg.content,
|
||||
role: chatmsg.role === 'apiMessage' ? 'bot' : 'user',
|
||||
time: chatmsg.createdDate
|
||||
}
|
||||
|
||||
// Add optional properties
|
||||
if (filePaths.length) msg.filePaths = filePaths
|
||||
if (chatmsg.sourceDocuments) msg.sourceDocuments = parseJsonBody(chatmsg.sourceDocuments)
|
||||
if (chatmsg.usedTools) msg.usedTools = parseJsonBody(chatmsg.usedTools)
|
||||
if (chatmsg.fileAnnotations) msg.fileAnnotations = parseJsonBody(chatmsg.fileAnnotations)
|
||||
if ((chatmsg as any).feedback) msg.feedback = (chatmsg as any).feedback.content
|
||||
if (chatmsg.agentReasoning) msg.agentReasoning = parseJsonBody(chatmsg.agentReasoning)
|
||||
|
||||
// Handle artifacts
|
||||
if (chatmsg.artifacts) {
|
||||
const artifacts = parseJsonBody(chatmsg.artifacts)
|
||||
msg.artifacts = artifacts
|
||||
if (Array.isArray(artifacts)) {
|
||||
artifacts.forEach((artifact: any) => {
|
||||
if (artifact.type === 'png' || artifact.type === 'jpeg') {
|
||||
const baseURL = process.env.BASE_URL || `http://localhost:${process.env.PORT || 3000}`
|
||||
artifact.data = `${baseURL}/api/v1/get-upload-file?chatflowId=${chatmsg.chatflowid}&chatId=${
|
||||
chatmsg.chatId
|
||||
}&fileName=${artifact.data.replace('FILE-STORAGE::', '')}`
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Group messages by chat session
|
||||
if (!exportObj[chatPK]) {
|
||||
exportObj[chatPK] = {
|
||||
id: chatmsg.chatId,
|
||||
source: getChatType(chatmsg.chatType as ChatType),
|
||||
sessionId: chatmsg.sessionId ?? null,
|
||||
memoryType: chatmsg.memoryType ?? null,
|
||||
email: (chatmsg as any).leadEmail ?? null,
|
||||
messages: [msg]
|
||||
}
|
||||
} else {
|
||||
exportObj[chatPK].messages.push(msg)
|
||||
}
|
||||
}
|
||||
|
||||
// Convert to array and reverse message order within each conversation
|
||||
const exportMessages = Object.values(exportObj).map((conversation: any) => ({
|
||||
...conversation,
|
||||
messages: conversation.messages.reverse()
|
||||
}))
|
||||
|
||||
return exportMessages
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
`Error: exportImportService.exportChatflowMessages - ${getErrorMessage(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to get chat primary key
|
||||
const getChatPK = (chatmsg: ChatMessage): string => {
|
||||
const chatId = chatmsg.chatId
|
||||
const memoryType = chatmsg.memoryType
|
||||
const sessionId = chatmsg.sessionId
|
||||
|
||||
if (memoryType && sessionId) {
|
||||
return `${chatId}_${memoryType}_${sessionId}`
|
||||
}
|
||||
return chatId
|
||||
}
|
||||
|
||||
// Helper function to get chat type display name
|
||||
const getChatType = (chatType?: ChatType): string => {
|
||||
if (!chatType) return 'Unknown'
|
||||
|
||||
switch (chatType) {
|
||||
case ChatType.EVALUATION:
|
||||
return 'Evaluation'
|
||||
case ChatType.INTERNAL:
|
||||
return 'UI'
|
||||
case ChatType.EXTERNAL:
|
||||
return 'API/Embed'
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
convertExportInput,
|
||||
exportData,
|
||||
importData
|
||||
importData,
|
||||
exportChatflowMessages
|
||||
}
|
||||
|
||||
@@ -23,7 +23,6 @@ export const WHITELIST_URLS = [
|
||||
'/api/v1/ping',
|
||||
'/api/v1/version',
|
||||
'/api/v1/attachments',
|
||||
'/api/v1/metrics',
|
||||
'/api/v1/nvidia-nim',
|
||||
'/api/v1/auth/resolve',
|
||||
'/api/v1/auth/login',
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import * as path from 'path'
|
||||
import * as fs from 'fs'
|
||||
import { hostname } from 'node:os'
|
||||
import config from './config' // should be replaced by node-config or similar
|
||||
import { createLogger, transports, format } from 'winston'
|
||||
import { NextFunction, Request, Response } from 'express'
|
||||
@@ -54,21 +53,21 @@ if (process.env.STORAGE_TYPE === 's3') {
|
||||
s3ServerStream = new S3StreamLogger({
|
||||
bucket: s3Bucket,
|
||||
folder: 'logs/server',
|
||||
name_format: `server-%Y-%m-%d-%H-%M-%S-%L-${hostname()}.log`,
|
||||
name_format: `server-%Y-%m-%d-%H-%M-%S-%L.log`,
|
||||
config: s3Config
|
||||
})
|
||||
|
||||
s3ErrorStream = new S3StreamLogger({
|
||||
bucket: s3Bucket,
|
||||
folder: 'logs/error',
|
||||
name_format: `server-error-%Y-%m-%d-%H-%M-%S-%L-${hostname()}.log`,
|
||||
name_format: `server-error-%Y-%m-%d-%H-%M-%S-%L.log`,
|
||||
config: s3Config
|
||||
})
|
||||
|
||||
s3ServerReqStream = new S3StreamLogger({
|
||||
bucket: s3Bucket,
|
||||
folder: 'logs/requests',
|
||||
name_format: `server-requests-%Y-%m-%d-%H-%M-%S-%L-${hostname()}.log.jsonl`,
|
||||
name_format: `server-requests-%Y-%m-%d-%H-%M-%S-%L.log.jsonl`,
|
||||
config: s3Config
|
||||
})
|
||||
}
|
||||
|
||||
@@ -134,6 +134,14 @@ export class RateLimiterManager {
|
||||
}
|
||||
}
|
||||
|
||||
public getRateLimiterById(id: string): (req: Request, res: Response, next: NextFunction) => void {
|
||||
return (req: Request, res: Response, next: NextFunction) => {
|
||||
if (!this.rateLimiters[id]) return next()
|
||||
const idRateLimiter = this.rateLimiters[id]
|
||||
return idRateLimiter(req, res, next)
|
||||
}
|
||||
}
|
||||
|
||||
public async updateRateLimiter(chatFlow: IChatFlow, isInitialized?: boolean): Promise<void> {
|
||||
if (!chatFlow.apiConfig) return
|
||||
const apiConfig = JSON.parse(chatFlow.apiConfig)
|
||||
|
||||
@@ -6,7 +6,6 @@ const getAllChatmessageFromChatflow = (id, params = {}) =>
|
||||
client.get(`/chatmessage/${id}`, { params: { order: 'DESC', feedback: true, ...params } })
|
||||
const getChatmessageFromPK = (id, params = {}) => client.get(`/chatmessage/${id}`, { params: { order: 'ASC', feedback: true, ...params } })
|
||||
const deleteChatmessage = (id, params = {}) => client.delete(`/chatmessage/${id}`, { params: { ...params } })
|
||||
const getStoragePath = () => client.get(`/get-upload-path`)
|
||||
const abortMessage = (chatflowid, chatid) => client.put(`/chatmessage/abort/${chatflowid}/${chatid}`)
|
||||
|
||||
export default {
|
||||
@@ -14,6 +13,5 @@ export default {
|
||||
getAllChatmessageFromChatflow,
|
||||
getChatmessageFromPK,
|
||||
deleteChatmessage,
|
||||
getStoragePath,
|
||||
abortMessage
|
||||
}
|
||||
|
||||
@@ -2,8 +2,10 @@ import client from './client'
|
||||
|
||||
const exportData = (body) => client.post('/export-import/export', body)
|
||||
const importData = (body) => client.post('/export-import/import', body)
|
||||
const exportChatflowMessages = (body) => client.post('/export-import/chatflow-messages', body)
|
||||
|
||||
export default {
|
||||
exportData,
|
||||
importData
|
||||
importData,
|
||||
exportChatflowMessages
|
||||
}
|
||||
|
||||
@@ -57,11 +57,12 @@ import { HIDE_CANVAS_DIALOG, SHOW_CANVAS_DIALOG } from '@/store/actions'
|
||||
// API
|
||||
import chatmessageApi from '@/api/chatmessage'
|
||||
import feedbackApi from '@/api/feedback'
|
||||
import exportImportApi from '@/api/exportimport'
|
||||
import useApi from '@/hooks/useApi'
|
||||
import useConfirm from '@/hooks/useConfirm'
|
||||
|
||||
// Utils
|
||||
import { getOS, isValidURL, removeDuplicateURL } from '@/utils/genericHelper'
|
||||
import { isValidURL, removeDuplicateURL } from '@/utils/genericHelper'
|
||||
import useNotifier from '@/utils/useNotifier'
|
||||
import { baseURL } from '@/store/constant'
|
||||
|
||||
@@ -204,8 +205,6 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
|
||||
const getChatmessageApi = useApi(chatmessageApi.getAllChatmessageFromChatflow)
|
||||
const getChatmessageFromPKApi = useApi(chatmessageApi.getChatmessageFromPK)
|
||||
const getStatsApi = useApi(feedbackApi.getStatsFromChatflow)
|
||||
const getStoragePathFromServer = useApi(chatmessageApi.getStoragePath)
|
||||
let storagePath = ''
|
||||
|
||||
/* Table Pagination */
|
||||
const [currentPage, setCurrentPage] = useState(1)
|
||||
@@ -352,94 +351,55 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
|
||||
}
|
||||
|
||||
const exportMessages = async () => {
|
||||
if (!storagePath && getStoragePathFromServer.data) {
|
||||
storagePath = getStoragePathFromServer.data.storagePath
|
||||
}
|
||||
const obj = {}
|
||||
let fileSeparator = '/'
|
||||
if ('windows' === getOS()) {
|
||||
fileSeparator = '\\'
|
||||
}
|
||||
try {
|
||||
const response = await exportImportApi.exportChatflowMessages({
|
||||
chatflowId: dialogProps.chatflow.id,
|
||||
chatType: chatTypeFilter.length ? chatTypeFilter : undefined,
|
||||
feedbackType: feedbackTypeFilter.length ? feedbackTypeFilter : undefined,
|
||||
startDate: startDate,
|
||||
endDate: endDate
|
||||
})
|
||||
|
||||
const resp = await chatmessageApi.getAllChatmessageFromChatflow(dialogProps.chatflow.id, {
|
||||
chatType: chatTypeFilter.length ? chatTypeFilter : undefined,
|
||||
feedbackType: feedbackTypeFilter.length ? feedbackTypeFilter : undefined,
|
||||
startDate: startDate,
|
||||
endDate: endDate,
|
||||
order: 'DESC'
|
||||
})
|
||||
const exportMessages = response.data
|
||||
const dataStr = JSON.stringify(exportMessages, null, 2)
|
||||
const blob = new Blob([dataStr], { type: 'application/json' })
|
||||
const dataUri = URL.createObjectURL(blob)
|
||||
|
||||
const allChatlogs = resp.data ?? []
|
||||
for (let i = 0; i < allChatlogs.length; i += 1) {
|
||||
const chatmsg = allChatlogs[i]
|
||||
const chatPK = getChatPK(chatmsg)
|
||||
let filePaths = []
|
||||
if (chatmsg.fileUploads && Array.isArray(chatmsg.fileUploads)) {
|
||||
chatmsg.fileUploads.forEach((file) => {
|
||||
if (file.type === 'stored-file') {
|
||||
filePaths.push(
|
||||
`${storagePath}${fileSeparator}${chatmsg.chatflowid}${fileSeparator}${chatmsg.chatId}${fileSeparator}${file.name}`
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
const msg = {
|
||||
content: chatmsg.content,
|
||||
role: chatmsg.role === 'apiMessage' ? 'bot' : 'user',
|
||||
time: chatmsg.createdDate
|
||||
}
|
||||
if (filePaths.length) msg.filePaths = filePaths
|
||||
if (chatmsg.sourceDocuments) msg.sourceDocuments = chatmsg.sourceDocuments
|
||||
if (chatmsg.usedTools) msg.usedTools = chatmsg.usedTools
|
||||
if (chatmsg.fileAnnotations) msg.fileAnnotations = chatmsg.fileAnnotations
|
||||
if (chatmsg.feedback) msg.feedback = chatmsg.feedback?.content
|
||||
if (chatmsg.agentReasoning) msg.agentReasoning = chatmsg.agentReasoning
|
||||
if (chatmsg.artifacts) {
|
||||
msg.artifacts = chatmsg.artifacts
|
||||
msg.artifacts.forEach((artifact) => {
|
||||
if (artifact.type === 'png' || artifact.type === 'jpeg') {
|
||||
artifact.data = `${baseURL}/api/v1/get-upload-file?chatflowId=${chatmsg.chatflowid}&chatId=${
|
||||
chatmsg.chatId
|
||||
}&fileName=${artifact.data.replace('FILE-STORAGE::', '')}`
|
||||
}
|
||||
})
|
||||
}
|
||||
if (!Object.prototype.hasOwnProperty.call(obj, chatPK)) {
|
||||
obj[chatPK] = {
|
||||
id: chatmsg.chatId,
|
||||
source: getChatType(chatmsg.chatType),
|
||||
sessionId: chatmsg.sessionId ?? null,
|
||||
memoryType: chatmsg.memoryType ?? null,
|
||||
email: chatmsg.leadEmail ?? null,
|
||||
messages: [msg]
|
||||
const exportFileDefaultName = `${dialogProps.chatflow.id}-Message.json`
|
||||
|
||||
let linkElement = document.createElement('a')
|
||||
linkElement.setAttribute('href', dataUri)
|
||||
linkElement.setAttribute('download', exportFileDefaultName)
|
||||
linkElement.click()
|
||||
|
||||
enqueueSnackbar({
|
||||
message: 'Messages exported successfully',
|
||||
options: {
|
||||
key: new Date().getTime() + Math.random(),
|
||||
variant: 'success',
|
||||
action: (key) => (
|
||||
<Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
|
||||
<IconX />
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Error exporting messages:', error)
|
||||
enqueueSnackbar({
|
||||
message: 'Failed to export messages',
|
||||
options: {
|
||||
key: new Date().getTime() + Math.random(),
|
||||
variant: 'error',
|
||||
persist: true,
|
||||
action: (key) => (
|
||||
<Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
|
||||
<IconX />
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
} else if (Object.prototype.hasOwnProperty.call(obj, chatPK)) {
|
||||
obj[chatPK].messages = [...obj[chatPK].messages, msg]
|
||||
}
|
||||
}
|
||||
|
||||
const exportMessages = []
|
||||
for (const key in obj) {
|
||||
exportMessages.push({
|
||||
...obj[key]
|
||||
})
|
||||
}
|
||||
|
||||
for (let i = 0; i < exportMessages.length; i += 1) {
|
||||
exportMessages[i].messages = exportMessages[i].messages.reverse()
|
||||
}
|
||||
|
||||
const dataStr = JSON.stringify(exportMessages, null, 2)
|
||||
//const dataUri = 'data:application/json;charset=utf-8,' + encodeURIComponent(dataStr)
|
||||
const blob = new Blob([dataStr], { type: 'application/json' })
|
||||
const dataUri = URL.createObjectURL(blob)
|
||||
|
||||
const exportFileDefaultName = `${dialogProps.chatflow.id}-Message.json`
|
||||
|
||||
let linkElement = document.createElement('a')
|
||||
linkElement.setAttribute('href', dataUri)
|
||||
linkElement.setAttribute('download', exportFileDefaultName)
|
||||
linkElement.click()
|
||||
}
|
||||
|
||||
const clearChat = async (chatmsg) => {
|
||||
@@ -755,8 +715,6 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
|
||||
|
||||
useEffect(() => {
|
||||
if (getChatmessageApi.data) {
|
||||
getStoragePathFromServer.request()
|
||||
|
||||
const chatPK = processChatLogs(getChatmessageApi.data)
|
||||
setSelectedMessageIndex(0)
|
||||
if (chatPK) {
|
||||
|
||||
Reference in New Issue
Block a user