LocalAI/LocalAGI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAGI.git synced 2026-07-24 03:10:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix(multimodal): fix multimodal input in telegram and slack (#219)

Browse Source 
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto
2025-06-24 23:00:45 +02:00
committed by GitHub
parent 7a8b3d93cf
commit 9160ca598e
4 changed files with 97 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/agent/actions.go
+1
View File
@@ -217,6 +217,7 @@ func (m Messages) Save(path string) error {
}
func (m Messages) GetLatestUserMessage() *openai.ChatCompletionMessage {
xlog.Debug("Getting latest user message", "messages", m)
for i := len(m) - 1; i >= 0; i-- {
msg := m[i]
if msg.Role == UserRole {
core/agent/agent.go
+6 -1
View File
@@ -410,7 +410,7 @@ func (a *Agent) processPrompts(conversation Messages) Messages {
}
func (a *Agent) describeImage(ctx context.Context, model, imageURL string) (string, error) {
xlog.Debug("Describing image", "model", model, "image", imageURL)
xlog.Debug("Describing image", "model", model)
resp, err := a.client.CreateChatCompletion(ctx,
openai.ChatCompletionRequest{
Model: model,
@@ -469,10 +469,15 @@ func (a *Agent) processUserInputs(job *types.Job, role string, conv Messages) Me
if !a.options.SeparatedMultimodalModel() {
return conv
}
xlog.Debug("Processing user inputs", "agent", a.Character.Name, "conversation", conv)
lastUserMessage := conv.GetLatestUserMessage()
xlog.Debug("Last user message", "lastUserMessage", lastUserMessage)
if lastUserMessage != nil && conv.IsLastMessageFromRole(UserRole) {
imageURL, text, err := extractImageContent(*lastUserMessage)
if err == nil {
xlog.Debug("Found image in user input", "image", imageURL)
// We have an image, we need to describe it first
// and add it to the conversation context
imageDescription, err := a.describeImage(a.context.Context, a.options.LLMAPI.MultimodalModel, imageURL)
services/connectors/slack.go
+14 -2
View File
@@ -221,10 +221,13 @@ func scanImagesInMessages(api *slack.Client, ev *slackevents.MessageEvent) (*byt
if err != nil {
xlog.Error(fmt.Sprintf("Error fetching messages: %v", err))
} else {
xlog.Debug("Scanning images in messages", "messages", messages)
for _, msg := range messages {
if len(msg.Files) == 0 {
xlog.Debug("No files in message", "message", msg.Text)
continue
}
xlog.Debug("Files in message", "files", msg.Files)
for _, attachment := range msg.Files {
if attachment.URLPrivate != "" {
xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment))
@@ -528,7 +531,7 @@ func (t *Slack) handleMention(
if err != nil {
xlog.Error(fmt.Sprintf("Error fetching thread messages: %v", err))
} else {
for i, msg := range messages {
for _, msg := range messages {
// Skip our placeholder message
if msg.Timestamp == msgTs {
continue
@@ -544,6 +547,7 @@ func (t *Slack) handleMention(
xlog.Debug(fmt.Sprintf("Message: %+v", msg))
if len(msg.Files) > 0 {
xlog.Debug("found files in the message", "files", len(msg.Files))
for _, attachment := range msg.Files {
if attachment.URLPrivate != "" {
@@ -557,7 +561,9 @@ func (t *Slack) handleMention(
}
}
// If the last message has an image, we send it as a multi content message
if len(imageBytes.Bytes()) > 0 && i == len(messages)-1 {
if len(imageBytes.Bytes()) > 0 {
xlog.Debug("found image in an existing thread", "image", len(imageBytes.Bytes()))
// // Encode the image to base64
imgBase64, err := encodeImageFromURL(*imageBytes)
@@ -565,6 +571,8 @@ func (t *Slack) handleMention(
xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err))
}
xlog.Debug("Image", "sending encoded image")
threadMessages = append(
threadMessages,
openai.ChatCompletionMessage{
@@ -585,6 +593,7 @@ func (t *Slack) handleMention(
},
)
} else {
xlog.Debug("no image in the last message of the thread", "message", msg.Text)
threadMessages = append(
threadMessages,
openai.ChatCompletionMessage{
@@ -611,10 +620,12 @@ func (t *Slack) handleMention(
} else {
for _, msg := range messages {
if len(msg.Files) == 0 {
xlog.Debug("no files in the message", "message", msg.Text)
continue
}
for _, attachment := range msg.Files {
if attachment.URLPrivate != "" {
xlog.Debug("found image in the message of the thread", "image", attachment.URLPrivate)
xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment))
// download image with slack api
mimeType = attachment.Mimetype
@@ -629,6 +640,7 @@ func (t *Slack) handleMention(
// If the last message has an image, we send it as a multi content message
if len(imageBytes.Bytes()) > 0 {
xlog.Debug("found image in the last message of the thread", "image", len(imageBytes.Bytes()))
// // Encode the image to base64
imgBase64, err := encodeImageFromURL(*imageBytes)
if err != nil {
services/connectors/telegram.go
+76 -58
View File
@@ -13,6 +13,7 @@ import (
"slices"
"strings"
"sync"
"time"
"github.com/go-telegram/bot"
"github.com/go-telegram/bot/models"
@@ -54,6 +55,68 @@ func (t *Telegram) isBotMentioned(message string, botUsername string) bool {
return strings.Contains(message, "@"+botUsername)
}
func (t *Telegram) chatFromMessage(update *models.Update) (openai.ChatCompletionMessage, error) {
if len(update.Message.Photo) == 0 {
return openai.ChatCompletionMessage{
Content: update.Message.Text,
Role: "user",
}, nil
}
xlog.Debug("Image", "found image")
// Get the largest photo
photo := update.Message.Photo[len(update.Message.Photo)-1]
ctx := context.Background()
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
// Download the photo
file, err := t.bot.GetFile(ctx, &bot.GetFileParams{
FileID: photo.FileID,
})
if err != nil {
xlog.Error("Error getting file", "error", err)
} else {
// Construct the full URL for downloading the file
fileURL := fmt.Sprintf("https://api.telegram.org/file/bot%s/%s", t.Token, file.FilePath)
// Download the file content
resp, err := http.Get(fileURL)
if err != nil {
xlog.Error("Error downloading file", "error", err)
} else {
defer resp.Body.Close()
imageBytes, err := io.ReadAll(resp.Body)
if err != nil {
xlog.Error("Error reading image", "error", err)
} else {
// Encode to base64
imgBase64 := base64.StdEncoding.EncodeToString(imageBytes)
xlog.Debug("Image", "sending encoded image")
// Add to conversation as multi-content message
return openai.ChatCompletionMessage{
Role: "user",
MultiContent: []openai.ChatMessagePart{
{
Text: update.Message.Caption,
Type: openai.ChatMessagePartTypeText,
},
{
Type: openai.ChatMessagePartTypeImageURL,
ImageURL: &openai.ChatMessageImageURL{
URL: fmt.Sprintf("data:image/jpeg;base64,%s", imgBase64),
},
},
},
}, nil
}
}
}
return openai.ChatCompletionMessage{}, errors.New("no image found")
}
// handleGroupMessage handles messages in group chats
func (t *Telegram) handleGroupMessage(ctx context.Context, b *bot.Bot, a *agent.Agent, update *models.Update) {
xlog.Debug("Handling group message", "update", update)
@@ -115,60 +178,14 @@ func (t *Telegram) handleGroupMessage(ctx context.Context, b *bot.Bot, a *agent.
"chatID": update.Message.Chat.ID,
}
// Handle images if present
if len(update.Message.Photo) > 0 {
// Get the largest photo
photo := update.Message.Photo[len(update.Message.Photo)-1]
// Download the photo
file, err := b.GetFile(ctx, &bot.GetFileParams{
FileID: photo.FileID,
})
if err != nil {
xlog.Error("Error getting file", "error", err)
} else {
// Download the file content
resp, err := http.Get(file.FilePath)
if err != nil {
xlog.Error("Error downloading file", "error", err)
} else {
defer resp.Body.Close()
imageBytes, err := io.ReadAll(resp.Body)
if err != nil {
xlog.Error("Error reading image", "error", err)
} else {
// Encode to base64
imgBase64 := base64.StdEncoding.EncodeToString(imageBytes)
// Add to conversation as multi-content message
currentConv = append(currentConv, openai.ChatCompletionMessage{
Role: "user",
MultiContent: []openai.ChatMessagePart{
{
Text: message,
Type: openai.ChatMessagePartTypeText,
},
{
Type: openai.ChatMessagePartTypeImageURL,
ImageURL: &openai.ChatMessageImageURL{
URL: fmt.Sprintf("data:image/jpeg;base64,%s", imgBase64),
},
},
},
})
}
}
}
} else {
currentConv = append(currentConv, openai.ChatCompletionMessage{
Content: message,
Role: "user",
})
chatMessage, err := t.chatFromMessage(update)
if err != nil {
xlog.Error("Error extracting chat message", "error", err)
}
a.SharedState().ConversationTracker.AddMessage(
fmt.Sprintf("telegram:%d", update.Message.Chat.ID),
currentConv[len(currentConv)-1],
chatMessage,
)
// Create a new job with the conversation history and metadata
@@ -486,17 +503,18 @@ func (t *Telegram) handleUpdate(ctx context.Context, b *bot.Bot, a *agent.Agent,
t.cancelActiveJobForChat(update.Message.Chat.ID)
currentConv := a.SharedState().ConversationTracker.GetConversation(fmt.Sprintf("telegram:%d", update.Message.From.ID))
currentConv = append(currentConv, openai.ChatCompletionMessage{
Content: update.Message.Text,
Role: "user",
})
message, err := t.chatFromMessage(update)
if err != nil {
xlog.Error("Error extracting chat message", "error", err)
return
}
currentConv = append(currentConv, message)
a.SharedState().ConversationTracker.AddMessage(
fmt.Sprintf("telegram:%d", update.Message.From.ID),
openai.ChatCompletionMessage{
Content: update.Message.Text,
Role: "user",
},
message,
)
// Send initial placeholder message