From 9160ca598e991fbf7b3d942b8ca0bcff533bb165 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 24 Jun 2025 23:00:45 +0200 Subject: [PATCH] fix(multimodal): fix multimodal input in telegram and slack (#219) Signed-off-by: Ettore Di Giacinto --- core/agent/actions.go | 1 + core/agent/agent.go | 7 +- services/connectors/slack.go | 16 +++- services/connectors/telegram.go | 134 ++++++++++++++++++-------------- 4 files changed, 97 insertions(+), 61 deletions(-) diff --git a/core/agent/actions.go b/core/agent/actions.go index a37448e..f1123ae 100644 --- a/core/agent/actions.go +++ b/core/agent/actions.go @@ -217,6 +217,7 @@ func (m Messages) Save(path string) error { } func (m Messages) GetLatestUserMessage() *openai.ChatCompletionMessage { + xlog.Debug("Getting latest user message", "messages", m) for i := len(m) - 1; i >= 0; i-- { msg := m[i] if msg.Role == UserRole { diff --git a/core/agent/agent.go b/core/agent/agent.go index 8f9520e..9f8bd52 100644 --- a/core/agent/agent.go +++ b/core/agent/agent.go @@ -410,7 +410,7 @@ func (a *Agent) processPrompts(conversation Messages) Messages { } func (a *Agent) describeImage(ctx context.Context, model, imageURL string) (string, error) { - xlog.Debug("Describing image", "model", model, "image", imageURL) + xlog.Debug("Describing image", "model", model) resp, err := a.client.CreateChatCompletion(ctx, openai.ChatCompletionRequest{ Model: model, @@ -469,10 +469,15 @@ func (a *Agent) processUserInputs(job *types.Job, role string, conv Messages) Me if !a.options.SeparatedMultimodalModel() { return conv } + + xlog.Debug("Processing user inputs", "agent", a.Character.Name, "conversation", conv) + lastUserMessage := conv.GetLatestUserMessage() + xlog.Debug("Last user message", "lastUserMessage", lastUserMessage) if lastUserMessage != nil && conv.IsLastMessageFromRole(UserRole) { imageURL, text, err := extractImageContent(*lastUserMessage) if err == nil { + xlog.Debug("Found image in user input", "image", imageURL) // We have an image, we need to describe it first // and add it to the conversation context imageDescription, err := a.describeImage(a.context.Context, a.options.LLMAPI.MultimodalModel, imageURL) diff --git a/services/connectors/slack.go b/services/connectors/slack.go index 71a2d21..e40beb4 100644 --- a/services/connectors/slack.go +++ b/services/connectors/slack.go @@ -221,10 +221,13 @@ func scanImagesInMessages(api *slack.Client, ev *slackevents.MessageEvent) (*byt if err != nil { xlog.Error(fmt.Sprintf("Error fetching messages: %v", err)) } else { + xlog.Debug("Scanning images in messages", "messages", messages) for _, msg := range messages { if len(msg.Files) == 0 { + xlog.Debug("No files in message", "message", msg.Text) continue } + xlog.Debug("Files in message", "files", msg.Files) for _, attachment := range msg.Files { if attachment.URLPrivate != "" { xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment)) @@ -528,7 +531,7 @@ func (t *Slack) handleMention( if err != nil { xlog.Error(fmt.Sprintf("Error fetching thread messages: %v", err)) } else { - for i, msg := range messages { + for _, msg := range messages { // Skip our placeholder message if msg.Timestamp == msgTs { continue @@ -544,6 +547,7 @@ func (t *Slack) handleMention( xlog.Debug(fmt.Sprintf("Message: %+v", msg)) if len(msg.Files) > 0 { + xlog.Debug("found files in the message", "files", len(msg.Files)) for _, attachment := range msg.Files { if attachment.URLPrivate != "" { @@ -557,7 +561,9 @@ func (t *Slack) handleMention( } } // If the last message has an image, we send it as a multi content message - if len(imageBytes.Bytes()) > 0 && i == len(messages)-1 { + if len(imageBytes.Bytes()) > 0 { + + xlog.Debug("found image in an existing thread", "image", len(imageBytes.Bytes())) // // Encode the image to base64 imgBase64, err := encodeImageFromURL(*imageBytes) @@ -565,6 +571,8 @@ func (t *Slack) handleMention( xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err)) } + xlog.Debug("Image", "sending encoded image") + threadMessages = append( threadMessages, openai.ChatCompletionMessage{ @@ -585,6 +593,7 @@ func (t *Slack) handleMention( }, ) } else { + xlog.Debug("no image in the last message of the thread", "message", msg.Text) threadMessages = append( threadMessages, openai.ChatCompletionMessage{ @@ -611,10 +620,12 @@ func (t *Slack) handleMention( } else { for _, msg := range messages { if len(msg.Files) == 0 { + xlog.Debug("no files in the message", "message", msg.Text) continue } for _, attachment := range msg.Files { if attachment.URLPrivate != "" { + xlog.Debug("found image in the message of the thread", "image", attachment.URLPrivate) xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment)) // download image with slack api mimeType = attachment.Mimetype @@ -629,6 +640,7 @@ func (t *Slack) handleMention( // If the last message has an image, we send it as a multi content message if len(imageBytes.Bytes()) > 0 { + xlog.Debug("found image in the last message of the thread", "image", len(imageBytes.Bytes())) // // Encode the image to base64 imgBase64, err := encodeImageFromURL(*imageBytes) if err != nil { diff --git a/services/connectors/telegram.go b/services/connectors/telegram.go index 8d9e406..e0b3076 100644 --- a/services/connectors/telegram.go +++ b/services/connectors/telegram.go @@ -13,6 +13,7 @@ import ( "slices" "strings" "sync" + "time" "github.com/go-telegram/bot" "github.com/go-telegram/bot/models" @@ -54,6 +55,68 @@ func (t *Telegram) isBotMentioned(message string, botUsername string) bool { return strings.Contains(message, "@"+botUsername) } +func (t *Telegram) chatFromMessage(update *models.Update) (openai.ChatCompletionMessage, error) { + + if len(update.Message.Photo) == 0 { + return openai.ChatCompletionMessage{ + Content: update.Message.Text, + Role: "user", + }, nil + } + + xlog.Debug("Image", "found image") + // Get the largest photo + photo := update.Message.Photo[len(update.Message.Photo)-1] + + ctx := context.Background() + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + // Download the photo + file, err := t.bot.GetFile(ctx, &bot.GetFileParams{ + FileID: photo.FileID, + }) + if err != nil { + xlog.Error("Error getting file", "error", err) + } else { + // Construct the full URL for downloading the file + fileURL := fmt.Sprintf("https://api.telegram.org/file/bot%s/%s", t.Token, file.FilePath) + + // Download the file content + resp, err := http.Get(fileURL) + if err != nil { + xlog.Error("Error downloading file", "error", err) + } else { + defer resp.Body.Close() + imageBytes, err := io.ReadAll(resp.Body) + if err != nil { + xlog.Error("Error reading image", "error", err) + } else { + // Encode to base64 + imgBase64 := base64.StdEncoding.EncodeToString(imageBytes) + xlog.Debug("Image", "sending encoded image") + // Add to conversation as multi-content message + return openai.ChatCompletionMessage{ + Role: "user", + MultiContent: []openai.ChatMessagePart{ + { + Text: update.Message.Caption, + Type: openai.ChatMessagePartTypeText, + }, + { + Type: openai.ChatMessagePartTypeImageURL, + ImageURL: &openai.ChatMessageImageURL{ + URL: fmt.Sprintf("data:image/jpeg;base64,%s", imgBase64), + }, + }, + }, + }, nil + } + } + } + + return openai.ChatCompletionMessage{}, errors.New("no image found") +} + // handleGroupMessage handles messages in group chats func (t *Telegram) handleGroupMessage(ctx context.Context, b *bot.Bot, a *agent.Agent, update *models.Update) { xlog.Debug("Handling group message", "update", update) @@ -115,60 +178,14 @@ func (t *Telegram) handleGroupMessage(ctx context.Context, b *bot.Bot, a *agent. "chatID": update.Message.Chat.ID, } - // Handle images if present - if len(update.Message.Photo) > 0 { - // Get the largest photo - photo := update.Message.Photo[len(update.Message.Photo)-1] - - // Download the photo - file, err := b.GetFile(ctx, &bot.GetFileParams{ - FileID: photo.FileID, - }) - if err != nil { - xlog.Error("Error getting file", "error", err) - } else { - // Download the file content - resp, err := http.Get(file.FilePath) - if err != nil { - xlog.Error("Error downloading file", "error", err) - } else { - defer resp.Body.Close() - imageBytes, err := io.ReadAll(resp.Body) - if err != nil { - xlog.Error("Error reading image", "error", err) - } else { - // Encode to base64 - imgBase64 := base64.StdEncoding.EncodeToString(imageBytes) - - // Add to conversation as multi-content message - currentConv = append(currentConv, openai.ChatCompletionMessage{ - Role: "user", - MultiContent: []openai.ChatMessagePart{ - { - Text: message, - Type: openai.ChatMessagePartTypeText, - }, - { - Type: openai.ChatMessagePartTypeImageURL, - ImageURL: &openai.ChatMessageImageURL{ - URL: fmt.Sprintf("data:image/jpeg;base64,%s", imgBase64), - }, - }, - }, - }) - } - } - } - } else { - currentConv = append(currentConv, openai.ChatCompletionMessage{ - Content: message, - Role: "user", - }) + chatMessage, err := t.chatFromMessage(update) + if err != nil { + xlog.Error("Error extracting chat message", "error", err) } a.SharedState().ConversationTracker.AddMessage( fmt.Sprintf("telegram:%d", update.Message.Chat.ID), - currentConv[len(currentConv)-1], + chatMessage, ) // Create a new job with the conversation history and metadata @@ -486,17 +503,18 @@ func (t *Telegram) handleUpdate(ctx context.Context, b *bot.Bot, a *agent.Agent, t.cancelActiveJobForChat(update.Message.Chat.ID) currentConv := a.SharedState().ConversationTracker.GetConversation(fmt.Sprintf("telegram:%d", update.Message.From.ID)) - currentConv = append(currentConv, openai.ChatCompletionMessage{ - Content: update.Message.Text, - Role: "user", - }) + + message, err := t.chatFromMessage(update) + if err != nil { + xlog.Error("Error extracting chat message", "error", err) + return + } + + currentConv = append(currentConv, message) a.SharedState().ConversationTracker.AddMessage( fmt.Sprintf("telegram:%d", update.Message.From.ID), - openai.ChatCompletionMessage{ - Content: update.Message.Text, - Role: "user", - }, + message, ) // Send initial placeholder message