diff --git a/core/agent/agent.go b/core/agent/agent.go index 2824f4a..5e2f997 100644 --- a/core/agent/agent.go +++ b/core/agent/agent.go @@ -444,12 +444,13 @@ func (a *Agent) describeImage(ctx context.Context, model, imageURL string) (stri return resp.Choices[0].Message.Content, nil } -func extractImageContent(message openai.ChatCompletionMessage) (imageURL, text string, e error) { +// extractAllImageContent extracts all images from a message +func extractAllImageContent(message openai.ChatCompletionMessage) (images []string, text string, e error) { e = fmt.Errorf("no image found") if message.MultiContent != nil { for _, content := range message.MultiContent { if content.Type == openai.ChatMessagePartTypeImageURL { - imageURL = content.ImageURL.URL + images = append(images, content.ImageURL.URL) e = nil } if content.Type == openai.ChatMessagePartTypeText { @@ -463,45 +464,75 @@ func extractImageContent(message openai.ChatCompletionMessage) (imageURL, text s func (a *Agent) processUserInputs(job *types.Job, role string, conv Messages) Messages { - // walk conversation history, and check if last message from user contains image. - // If it does, we need to describe the image first with a model that supports image understanding (if the current model doesn't support it) - // and add it to the conversation context + // walk conversation history, and check if any message contains images. + // If they do, we need to describe the images first with a model that supports image understanding (if the current model doesn't support it) + // and add them to the conversation context if !a.options.SeparatedMultimodalModel() { return conv } xlog.Debug("Processing user inputs", "agent", a.Character.Name, "conversation", conv) - lastUserMessage := conv.GetLatestUserMessage() - xlog.Debug("Last user message", "lastUserMessage", lastUserMessage) - if lastUserMessage != nil && conv.IsLastMessageFromRole(UserRole) { - imageURL, text, err := extractImageContent(*lastUserMessage) - if err == nil { - xlog.Debug("Found image in user input", "image", imageURL) - // We have an image, we need to describe it first - // and add it to the conversation context - imageDescription, err := a.describeImage(a.context.Context, a.options.LLMAPI.MultimodalModel, imageURL) - if err != nil { - xlog.Error("Error describing image", "error", err) - } else { - // We replace the user message with the image description - // and add the user text to the conversation - explainerMessage := openai.ChatCompletionMessage{ - Role: "system", - Content: fmt.Sprintf("The user shared an image which can be described as: %s", imageDescription), - } + // Process all messages in the conversation to extract and describe images + var processedMessages Messages + var messagesToRemove []int - // remove lastUserMessage from the conversation - conv = conv.RemoveLastUserMessage() - conv = append(conv, explainerMessage) - conv = append(conv, openai.ChatCompletionMessage{ - Role: role, - Content: text, - }) + for i, message := range conv { + images, text, err := extractAllImageContent(message) + if err == nil && len(images) > 0 { + xlog.Debug("Found images in message", "messageIndex", i, "imageCount", len(images), "role", message.Role) + + // Mark this message for removal + messagesToRemove = append(messagesToRemove, i) + + // Process each image in the message + var imageDescriptions []string + for j, image := range images { + imageDescription, err := a.describeImage(a.context.Context, a.options.LLMAPI.MultimodalModel, image) + if err != nil { + xlog.Error("Error describing image", "error", err, "messageIndex", i, "imageIndex", j) + imageDescriptions = append(imageDescriptions, fmt.Sprintf("Image %d: [Error describing image: %v]", j+1, err)) + } else { + imageDescriptions = append(imageDescriptions, fmt.Sprintf("Image %d: %s", j+1, imageDescription)) + } } + + // Add the text content as a new message with the same role first + if text != "" { + textMessage := openai.ChatCompletionMessage{ + Role: message.Role, + Content: text, + } + processedMessages = append(processedMessages, textMessage) + + // Add the image descriptions as a system message after the text + explainerMessage := openai.ChatCompletionMessage{ + Role: "system", + Content: fmt.Sprintf("The above message also contains %d image(s) which can be described as: %s", + len(images), strings.Join(imageDescriptions, "; ")), + } + processedMessages = append(processedMessages, explainerMessage) + } else { + // If there's no text, just add the image descriptions as a system message + explainerMessage := openai.ChatCompletionMessage{ + Role: "system", + Content: fmt.Sprintf("Message contains %d image(s) which can be described as: %s", + len(images), strings.Join(imageDescriptions, "; ")), + } + processedMessages = append(processedMessages, explainerMessage) + } + } else { + // No image found, keep the original message + processedMessages = append(processedMessages, message) } } + // If we found and processed any images, replace the conversation + if len(messagesToRemove) > 0 { + xlog.Info("Processed images in conversation", "messagesWithImages", len(messagesToRemove), "agent", a.Character.Name) + return processedMessages + } + return conv } @@ -578,30 +609,30 @@ func (a *Agent) validateBuiltinTools(job *types.Job) { if len(builtinTools) == 0 { return } - + // Get available actions availableActions := a.mcpActions - + for _, tool := range builtinTools { functionName := tool.Name - + // Check if this is a web search builtin tool if strings.HasPrefix(string(functionName), "web_search_") { // Look for a search action searchAction := availableActions.Find("search") if searchAction == nil { - xlog.Warn("Web search builtin tool specified but no 'search' action available", - "function_name", functionName, + xlog.Warn("Web search builtin tool specified but no 'search' action available", + "function_name", functionName, "agent", a.Character.Name) } else { - xlog.Debug("Web search builtin tool matched to search action", - "function_name", functionName, + xlog.Debug("Web search builtin tool matched to search action", + "function_name", functionName, "agent", a.Character.Name) } } else { // For future builtin tools, add more matching logic here - xlog.Warn("Unknown builtin tool specified", - "function_name", functionName, + xlog.Warn("Unknown builtin tool specified", + "function_name", functionName, "agent", a.Character.Name) } } @@ -621,10 +652,10 @@ func (a *Agent) replyWithToolCall(job *types.Job, conv []openai.ChatCompletionMe Result: reasoning, // The reasoning/message to show to user }, } - + // Add the action state to the job result job.Result.SetResult(stateResult) - + // Set conversation but leave Response empty // The webui will detect the user-defined action and generate the proper tool call response job.Result.Conversation = conv @@ -912,7 +943,7 @@ func (a *Agent) consumeJob(job *types.Job, role string, retries int) { a.replyWithToolCall(job, conv, actionParams, chosenAction, reasoning) return } - + result, err := a.runAction(job, chosenAction, actionParams) if err != nil { result.Result = fmt.Sprintf("Error running tool: %v", err) diff --git a/services/connectors/slack.go b/services/connectors/slack.go index e40beb4..9682c21 100644 --- a/services/connectors/slack.go +++ b/services/connectors/slack.go @@ -208,9 +208,15 @@ func generateAttachmentsFromJobResponse(j *types.JobResult, api *slack.Client, c return } -func scanImagesInMessages(api *slack.Client, ev *slackevents.MessageEvent) (*bytes.Buffer, string) { - imageBytes := new(bytes.Buffer) - mimeType := "image/jpeg" +// ImageData represents a single image with its metadata +type ImageData struct { + Data []byte + MimeType string +} + +// scanImagesInMessages scans for all images in a message and returns them as a slice +func scanImagesInMessages(api *slack.Client, ev *slackevents.MessageEvent) []ImageData { + var images []ImageData // Fetch the message using the API messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{ @@ -220,28 +226,142 @@ func scanImagesInMessages(api *slack.Client, ev *slackevents.MessageEvent) (*byt if err != nil { xlog.Error(fmt.Sprintf("Error fetching messages: %v", err)) - } else { - xlog.Debug("Scanning images in messages", "messages", messages) - for _, msg := range messages { - if len(msg.Files) == 0 { - xlog.Debug("No files in message", "message", msg.Text) - continue - } - xlog.Debug("Files in message", "files", msg.Files) - for _, attachment := range msg.Files { - if attachment.URLPrivate != "" { - xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment)) - // download image with slack api - mimeType = attachment.Mimetype - if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil { - xlog.Error(fmt.Sprintf("Error downloading image: %v", err)) - } + return images + } + + xlog.Debug("Scanning images in messages", "messages", messages) + for _, msg := range messages { + if len(msg.Files) == 0 { + xlog.Debug("No files in message", "message", msg.Text) + continue + } + xlog.Debug("Files in message", "files", msg.Files) + for _, attachment := range msg.Files { + if attachment.URLPrivate != "" { + xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment)) + // download image with slack api + imageBytes := new(bytes.Buffer) + if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil { + xlog.Error(fmt.Sprintf("Error downloading image: %v", err)) + continue } + + images = append(images, ImageData{ + Data: imageBytes.Bytes(), + MimeType: attachment.Mimetype, + }) } } } - return imageBytes, mimeType + return images +} + +// scanImagesInAppMentionEvent scans for all images in an app mention event +func scanImagesInAppMentionEvent(api *slack.Client, ev *slackevents.AppMentionEvent) []ImageData { + var images []ImageData + + // Fetch the message using the API + messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{ + ChannelID: ev.Channel, + Timestamp: ev.TimeStamp, + }) + + if err != nil { + xlog.Error(fmt.Sprintf("Error fetching messages: %v", err)) + return images + } + + xlog.Debug("Scanning images in app mention event", "messages", messages) + for _, msg := range messages { + if len(msg.Files) == 0 { + xlog.Debug("No files in message", "message", msg.Text) + continue + } + xlog.Debug("Files in message", "files", msg.Files) + for _, attachment := range msg.Files { + if attachment.URLPrivate != "" { + xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment)) + // download image with slack api + imageBytes := new(bytes.Buffer) + if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil { + xlog.Error(fmt.Sprintf("Error downloading image: %v", err)) + continue + } + + images = append(images, ImageData{ + Data: imageBytes.Bytes(), + MimeType: attachment.Mimetype, + }) + } + } + } + + return images +} + +// scanImagesInThreadMessage scans for all images in a single thread message +func scanImagesInThreadMessage(api *slack.Client, msg slack.Message) []ImageData { + var images []ImageData + + if len(msg.Files) == 0 { + return images + } + + xlog.Debug("found files in the message", "files", len(msg.Files)) + for _, attachment := range msg.Files { + if attachment.URLPrivate != "" { + xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment)) + // download image with slack api + imageBytes := new(bytes.Buffer) + if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil { + xlog.Error(fmt.Sprintf("Error downloading image: %v", err)) + continue + } + + images = append(images, ImageData{ + Data: imageBytes.Bytes(), + MimeType: attachment.Mimetype, + }) + } + } + + return images +} + +// createMultiContentMessage creates a ChatCompletionMessage with text and multiple images +func createMultiContentMessage(role, text string, images []ImageData) openai.ChatCompletionMessage { + multiContent := []openai.ChatMessagePart{ + { + Text: text, + Type: openai.ChatMessagePartTypeText, + }, + } + + for _, img := range images { + imgBase64, err := encodeImageFromBytes(img.Data) + if err != nil { + xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err)) + continue + } + + multiContent = append(multiContent, openai.ChatMessagePart{ + Type: openai.ChatMessagePartTypeImageURL, + ImageURL: &openai.ChatMessageImageURL{ + URL: fmt.Sprintf("data:%s;base64,%s", img.MimeType, imgBase64), + }, + }) + } + + return openai.ChatCompletionMessage{ + Role: role, + MultiContent: multiContent, + } +} + +// encodeImageFromBytes encodes image bytes to base64 +func encodeImageFromBytes(imageData []byte) (string, error) { + return base64.StdEncoding.EncodeToString(imageData), nil } func (t *Slack) handleChannelMessage( @@ -269,37 +389,15 @@ func (t *Slack) handleChannelMessage( go func() { - imageBytes, mimeType := scanImagesInMessages(api, ev) + images := scanImagesInMessages(api, ev) agentOptions := []types.JobOption{ types.WithUUID(ev.ThreadTimeStamp), } // If the last message has an image, we send it as a multi content message - if len(imageBytes.Bytes()) > 0 { - // // Encode the image to base64 - imgBase64, err := encodeImageFromURL(*imageBytes) - if err != nil { - xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err)) - } else { - currentConv = append(currentConv, - openai.ChatCompletionMessage{ - Role: "user", - MultiContent: []openai.ChatMessagePart{ - { - Text: message, - Type: openai.ChatMessagePartTypeText, - }, - { - Type: openai.ChatMessagePartTypeImageURL, - ImageURL: &openai.ChatMessageImageURL{ - URL: fmt.Sprintf("data:%s;base64,%s", mimeType, imgBase64), - }, - }, - }, - }, - ) - } + if len(images) > 0 { + currentConv = append(currentConv, createMultiContentMessage("user", message, images)) } else { currentConv = append(currentConv, openai.ChatCompletionMessage{ Role: "user", @@ -370,14 +468,6 @@ func (t *Slack) handleChannelMessage( }() } -// Function to download the image from a URL and encode it to base64 -func encodeImageFromURL(imageBytes bytes.Buffer) (string, error) { - - // Encode the image data to base64 - base64Image := base64.StdEncoding.EncodeToString(imageBytes.Bytes()) - return base64Image, nil -} - func replyWithPostMessage(finalResponse string, api *slack.Client, ev *slackevents.MessageEvent, postMessageParams slack.PostMessageParameters, res *types.JobResult) { if len(finalResponse) > 4000 { // split response in multiple messages, and update the first @@ -542,55 +632,16 @@ func (t *Slack) handleMention( role = "user" } - imageBytes := new(bytes.Buffer) - mimeType := "image/jpeg" + images := scanImagesInThreadMessage(api, msg) - xlog.Debug(fmt.Sprintf("Message: %+v", msg)) - if len(msg.Files) > 0 { - xlog.Debug("found files in the message", "files", len(msg.Files)) - for _, attachment := range msg.Files { - - if attachment.URLPrivate != "" { - xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment)) - mimeType = attachment.Mimetype - // download image with slack api - if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil { - xlog.Error(fmt.Sprintf("Error downloading image: %v", err)) - } - } - } - } // If the last message has an image, we send it as a multi content message - if len(imageBytes.Bytes()) > 0 { + if len(images) > 0 { - xlog.Debug("found image in an existing thread", "image", len(imageBytes.Bytes())) - - // // Encode the image to base64 - imgBase64, err := encodeImageFromURL(*imageBytes) - if err != nil { - xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err)) - } - - xlog.Debug("Image", "sending encoded image") + xlog.Debug("found image in an existing thread", "image", len(images)) threadMessages = append( threadMessages, - openai.ChatCompletionMessage{ - Role: role, - MultiContent: []openai.ChatMessagePart{ - { - Text: replaceUserIDsWithNamesInMessage(api, cleanUpUsernameFromMessage(msg.Text, b)), - Type: openai.ChatMessagePartTypeText, - }, - { - Type: openai.ChatMessagePartTypeImageURL, - ImageURL: &openai.ChatMessageImageURL{ - URL: fmt.Sprintf("data:%s;base64,%s", mimeType, imgBase64), - // URL: imgUrl, - }, - }, - }, - }, + createMultiContentMessage(role, replaceUserIDsWithNamesInMessage(api, cleanUpUsernameFromMessage(msg.Text, b)), images), ) } else { xlog.Debug("no image in the last message of the thread", "message", msg.Text) @@ -606,65 +657,15 @@ func (t *Slack) handleMention( } } else { - imageBytes := new(bytes.Buffer) - mimeType := "image/jpeg" - - // Fetch the message using the API - messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{ - ChannelID: ev.Channel, - Timestamp: ev.TimeStamp, - }) - - if err != nil { - xlog.Error(fmt.Sprintf("Error fetching messages: %v", err)) - } else { - for _, msg := range messages { - if len(msg.Files) == 0 { - xlog.Debug("no files in the message", "message", msg.Text) - continue - } - for _, attachment := range msg.Files { - if attachment.URLPrivate != "" { - xlog.Debug("found image in the message of the thread", "image", attachment.URLPrivate) - xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment)) - // download image with slack api - mimeType = attachment.Mimetype - if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil { - xlog.Error(fmt.Sprintf("Error downloading image: %v", err)) - } - } - } - } - } + images := scanImagesInAppMentionEvent(api, ev) // If the last message has an image, we send it as a multi content message - if len(imageBytes.Bytes()) > 0 { - - xlog.Debug("found image in the last message of the thread", "image", len(imageBytes.Bytes())) - // // Encode the image to base64 - imgBase64, err := encodeImageFromURL(*imageBytes) - if err != nil { - xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err)) - } + if len(images) > 0 { + xlog.Debug("found image in the last message of the thread", "image", len(images)) threadMessages = append( threadMessages, - openai.ChatCompletionMessage{ - Role: "user", - MultiContent: []openai.ChatMessagePart{ - { - Text: replaceUserIDsWithNamesInMessage(api, cleanUpUsernameFromMessage(message, b)), - Type: openai.ChatMessagePartTypeText, - }, - { - Type: openai.ChatMessagePartTypeImageURL, - ImageURL: &openai.ChatMessageImageURL{ - // URL: imgURL, - URL: fmt.Sprintf("data:%s;base64,%s", mimeType, imgBase64), - }, - }, - }, - }, + createMultiContentMessage("user", replaceUserIDsWithNamesInMessage(api, cleanUpUsernameFromMessage(message, b)), images), ) } else { threadMessages = append(threadMessages, openai.ChatCompletionMessage{