LocalAI/LocalAGI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAGI.git synced 2026-07-23 10:45:41 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(mutlimodal): do parse all images shared in the conversation (#221)

Browse Source 
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto
2025-06-30 22:56:30 +02:00
committed by GitHub
parent 1fb7f8bc75
commit 9500ec7af0
2 changed files with 224 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/agent/agent.go
+73 -42
View File
@@ -444,12 +444,13 @@ func (a *Agent) describeImage(ctx context.Context, model, imageURL string) (stri
return resp.Choices[0].Message.Content, nil
}
func extractImageContent(message openai.ChatCompletionMessage) (imageURL, text string, e error) {
// extractAllImageContent extracts all images from a message
func extractAllImageContent(message openai.ChatCompletionMessage) (images []string, text string, e error) {
e = fmt.Errorf("no image found")
if message.MultiContent != nil {
for _, content := range message.MultiContent {
if content.Type == openai.ChatMessagePartTypeImageURL {
imageURL = content.ImageURL.URL
images = append(images, content.ImageURL.URL)
e = nil
}
if content.Type == openai.ChatMessagePartTypeText {
@@ -463,45 +464,75 @@ func extractImageContent(message openai.ChatCompletionMessage) (imageURL, text s
func (a *Agent) processUserInputs(job *types.Job, role string, conv Messages) Messages {
// walk conversation history, and check if last message from user contains image.
// If it does, we need to describe the image first with a model that supports image understanding (if the current model doesn't support it)
// and add it to the conversation context
// walk conversation history, and check if any message contains images.
// If they do, we need to describe the images first with a model that supports image understanding (if the current model doesn't support it)
// and add them to the conversation context
if !a.options.SeparatedMultimodalModel() {
return conv
}
xlog.Debug("Processing user inputs", "agent", a.Character.Name, "conversation", conv)
lastUserMessage := conv.GetLatestUserMessage()
xlog.Debug("Last user message", "lastUserMessage", lastUserMessage)
if lastUserMessage != nil && conv.IsLastMessageFromRole(UserRole) {
imageURL, text, err := extractImageContent(*lastUserMessage)
if err == nil {
xlog.Debug("Found image in user input", "image", imageURL)
// We have an image, we need to describe it first
// and add it to the conversation context
imageDescription, err := a.describeImage(a.context.Context, a.options.LLMAPI.MultimodalModel, imageURL)
if err != nil {
xlog.Error("Error describing image", "error", err)
} else {
// We replace the user message with the image description
// and add the user text to the conversation
explainerMessage := openai.ChatCompletionMessage{
Role: "system",
Content: fmt.Sprintf("The user shared an image which can be described as: %s", imageDescription),
}
// Process all messages in the conversation to extract and describe images
var processedMessages Messages
var messagesToRemove []int
// remove lastUserMessage from the conversation
conv = conv.RemoveLastUserMessage()
conv = append(conv, explainerMessage)
conv = append(conv, openai.ChatCompletionMessage{
Role: role,
Content: text,
})
for i, message := range conv {
images, text, err := extractAllImageContent(message)
if err == nil && len(images) > 0 {
xlog.Debug("Found images in message", "messageIndex", i, "imageCount", len(images), "role", message.Role)
// Mark this message for removal
messagesToRemove = append(messagesToRemove, i)
// Process each image in the message
var imageDescriptions []string
for j, image := range images {
imageDescription, err := a.describeImage(a.context.Context, a.options.LLMAPI.MultimodalModel, image)
if err != nil {
xlog.Error("Error describing image", "error", err, "messageIndex", i, "imageIndex", j)
imageDescriptions = append(imageDescriptions, fmt.Sprintf("Image %d: [Error describing image: %v]", j+1, err))
} else {
imageDescriptions = append(imageDescriptions, fmt.Sprintf("Image %d: %s", j+1, imageDescription))
}
}
// Add the text content as a new message with the same role first
if text != "" {
textMessage := openai.ChatCompletionMessage{
Role: message.Role,
Content: text,
}
processedMessages = append(processedMessages, textMessage)
// Add the image descriptions as a system message after the text
explainerMessage := openai.ChatCompletionMessage{
Role: "system",
Content: fmt.Sprintf("The above message also contains %d image(s) which can be described as: %s",
len(images), strings.Join(imageDescriptions, "; ")),
}
processedMessages = append(processedMessages, explainerMessage)
} else {
// If there's no text, just add the image descriptions as a system message
explainerMessage := openai.ChatCompletionMessage{
Role: "system",
Content: fmt.Sprintf("Message contains %d image(s) which can be described as: %s",
len(images), strings.Join(imageDescriptions, "; ")),
}
processedMessages = append(processedMessages, explainerMessage)
}
} else {
// No image found, keep the original message
processedMessages = append(processedMessages, message)
}
}
// If we found and processed any images, replace the conversation
if len(messagesToRemove) > 0 {
xlog.Info("Processed images in conversation", "messagesWithImages", len(messagesToRemove), "agent", a.Character.Name)
return processedMessages
}
return conv
}
@@ -578,30 +609,30 @@ func (a *Agent) validateBuiltinTools(job *types.Job) {
if len(builtinTools) == 0 {
return
}
// Get available actions
availableActions := a.mcpActions
for _, tool := range builtinTools {
functionName := tool.Name
// Check if this is a web search builtin tool
if strings.HasPrefix(string(functionName), "web_search_") {
// Look for a search action
searchAction := availableActions.Find("search")
if searchAction == nil {
xlog.Warn("Web search builtin tool specified but no 'search' action available",
"function_name", functionName,
xlog.Warn("Web search builtin tool specified but no 'search' action available",
"function_name", functionName,
"agent", a.Character.Name)
} else {
xlog.Debug("Web search builtin tool matched to search action",
"function_name", functionName,
xlog.Debug("Web search builtin tool matched to search action",
"function_name", functionName,
"agent", a.Character.Name)
}
} else {
// For future builtin tools, add more matching logic here
xlog.Warn("Unknown builtin tool specified",
"function_name", functionName,
xlog.Warn("Unknown builtin tool specified",
"function_name", functionName,
"agent", a.Character.Name)
}
}
@@ -621,10 +652,10 @@ func (a *Agent) replyWithToolCall(job *types.Job, conv []openai.ChatCompletionMe
Result: reasoning, // The reasoning/message to show to user
},
}
// Add the action state to the job result
job.Result.SetResult(stateResult)
// Set conversation but leave Response empty
// The webui will detect the user-defined action and generate the proper tool call response
job.Result.Conversation = conv
@@ -912,7 +943,7 @@ func (a *Agent) consumeJob(job *types.Job, role string, retries int) {
a.replyWithToolCall(job, conv, actionParams, chosenAction, reasoning)
return
}
result, err := a.runAction(job, chosenAction, actionParams)
if err != nil {
result.Result = fmt.Sprintf("Error running tool: %v", err)
services/connectors/slack.go
+151 -150
View File
@@ -208,9 +208,15 @@ func generateAttachmentsFromJobResponse(j *types.JobResult, api *slack.Client, c
return
}
func scanImagesInMessages(api *slack.Client, ev *slackevents.MessageEvent) (*bytes.Buffer, string) {
imageBytes := new(bytes.Buffer)
mimeType := "image/jpeg"
// ImageData represents a single image with its metadata
type ImageData struct {
Data []byte
MimeType string
}
// scanImagesInMessages scans for all images in a message and returns them as a slice
func scanImagesInMessages(api *slack.Client, ev *slackevents.MessageEvent) []ImageData {
var images []ImageData
// Fetch the message using the API
messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{
@@ -220,28 +226,142 @@ func scanImagesInMessages(api *slack.Client, ev *slackevents.MessageEvent) (*byt
if err != nil {
xlog.Error(fmt.Sprintf("Error fetching messages: %v", err))
} else {
xlog.Debug("Scanning images in messages", "messages", messages)
for _, msg := range messages {
if len(msg.Files) == 0 {
xlog.Debug("No files in message", "message", msg.Text)
continue
}
xlog.Debug("Files in message", "files", msg.Files)
for _, attachment := range msg.Files {
if attachment.URLPrivate != "" {
xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment))
// download image with slack api
mimeType = attachment.Mimetype
if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil {
xlog.Error(fmt.Sprintf("Error downloading image: %v", err))
}
return images
}
xlog.Debug("Scanning images in messages", "messages", messages)
for _, msg := range messages {
if len(msg.Files) == 0 {
xlog.Debug("No files in message", "message", msg.Text)
continue
}
xlog.Debug("Files in message", "files", msg.Files)
for _, attachment := range msg.Files {
if attachment.URLPrivate != "" {
xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment))
// download image with slack api
imageBytes := new(bytes.Buffer)
if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil {
xlog.Error(fmt.Sprintf("Error downloading image: %v", err))
continue
}
images = append(images, ImageData{
Data: imageBytes.Bytes(),
MimeType: attachment.Mimetype,
})
}
}
}
return imageBytes, mimeType
return images
}
// scanImagesInAppMentionEvent scans for all images in an app mention event
func scanImagesInAppMentionEvent(api *slack.Client, ev *slackevents.AppMentionEvent) []ImageData {
var images []ImageData
// Fetch the message using the API
messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{
ChannelID: ev.Channel,
Timestamp: ev.TimeStamp,
})
if err != nil {
xlog.Error(fmt.Sprintf("Error fetching messages: %v", err))
return images
}
xlog.Debug("Scanning images in app mention event", "messages", messages)
for _, msg := range messages {
if len(msg.Files) == 0 {
xlog.Debug("No files in message", "message", msg.Text)
continue
}
xlog.Debug("Files in message", "files", msg.Files)
for _, attachment := range msg.Files {
if attachment.URLPrivate != "" {
xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment))
// download image with slack api
imageBytes := new(bytes.Buffer)
if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil {
xlog.Error(fmt.Sprintf("Error downloading image: %v", err))
continue
}
images = append(images, ImageData{
Data: imageBytes.Bytes(),
MimeType: attachment.Mimetype,
})
}
}
}
return images
}
// scanImagesInThreadMessage scans for all images in a single thread message
func scanImagesInThreadMessage(api *slack.Client, msg slack.Message) []ImageData {
var images []ImageData
if len(msg.Files) == 0 {
return images
}
xlog.Debug("found files in the message", "files", len(msg.Files))
for _, attachment := range msg.Files {
if attachment.URLPrivate != "" {
xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment))
// download image with slack api
imageBytes := new(bytes.Buffer)
if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil {
xlog.Error(fmt.Sprintf("Error downloading image: %v", err))
continue
}
images = append(images, ImageData{
Data: imageBytes.Bytes(),
MimeType: attachment.Mimetype,
})
}
}
return images
}
// createMultiContentMessage creates a ChatCompletionMessage with text and multiple images
func createMultiContentMessage(role, text string, images []ImageData) openai.ChatCompletionMessage {
multiContent := []openai.ChatMessagePart{
{
Text: text,
Type: openai.ChatMessagePartTypeText,
},
}
for _, img := range images {
imgBase64, err := encodeImageFromBytes(img.Data)
if err != nil {
xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err))
continue
}
multiContent = append(multiContent, openai.ChatMessagePart{
Type: openai.ChatMessagePartTypeImageURL,
ImageURL: &openai.ChatMessageImageURL{
URL: fmt.Sprintf("data:%s;base64,%s", img.MimeType, imgBase64),
},
})
}
return openai.ChatCompletionMessage{
Role: role,
MultiContent: multiContent,
}
}
// encodeImageFromBytes encodes image bytes to base64
func encodeImageFromBytes(imageData []byte) (string, error) {
return base64.StdEncoding.EncodeToString(imageData), nil
}
func (t *Slack) handleChannelMessage(
@@ -269,37 +389,15 @@ func (t *Slack) handleChannelMessage(
go func() {
imageBytes, mimeType := scanImagesInMessages(api, ev)
images := scanImagesInMessages(api, ev)
agentOptions := []types.JobOption{
types.WithUUID(ev.ThreadTimeStamp),
}
// If the last message has an image, we send it as a multi content message
if len(imageBytes.Bytes()) > 0 {
// // Encode the image to base64
imgBase64, err := encodeImageFromURL(*imageBytes)
if err != nil {
xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err))
} else {
currentConv = append(currentConv,
openai.ChatCompletionMessage{
Role: "user",
MultiContent: []openai.ChatMessagePart{
{
Text: message,
Type: openai.ChatMessagePartTypeText,
},
{
Type: openai.ChatMessagePartTypeImageURL,
ImageURL: &openai.ChatMessageImageURL{
URL: fmt.Sprintf("data:%s;base64,%s", mimeType, imgBase64),
},
},
},
},
)
}
if len(images) > 0 {
currentConv = append(currentConv, createMultiContentMessage("user", message, images))
} else {
currentConv = append(currentConv, openai.ChatCompletionMessage{
Role: "user",
@@ -370,14 +468,6 @@ func (t *Slack) handleChannelMessage(
}()
}
// Function to download the image from a URL and encode it to base64
func encodeImageFromURL(imageBytes bytes.Buffer) (string, error) {
// Encode the image data to base64
base64Image := base64.StdEncoding.EncodeToString(imageBytes.Bytes())
return base64Image, nil
}
func replyWithPostMessage(finalResponse string, api *slack.Client, ev *slackevents.MessageEvent, postMessageParams slack.PostMessageParameters, res *types.JobResult) {
if len(finalResponse) > 4000 {
// split response in multiple messages, and update the first
@@ -542,55 +632,16 @@ func (t *Slack) handleMention(
role = "user"
}
imageBytes := new(bytes.Buffer)
mimeType := "image/jpeg"
images := scanImagesInThreadMessage(api, msg)
xlog.Debug(fmt.Sprintf("Message: %+v", msg))
if len(msg.Files) > 0 {
xlog.Debug("found files in the message", "files", len(msg.Files))
for _, attachment := range msg.Files {
if attachment.URLPrivate != "" {
xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment))
mimeType = attachment.Mimetype
// download image with slack api
if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil {
xlog.Error(fmt.Sprintf("Error downloading image: %v", err))
}
}
}
}
// If the last message has an image, we send it as a multi content message
if len(imageBytes.Bytes()) > 0 {
if len(images) > 0 {
xlog.Debug("found image in an existing thread", "image", len(imageBytes.Bytes()))
// // Encode the image to base64
imgBase64, err := encodeImageFromURL(*imageBytes)
if err != nil {
xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err))
}
xlog.Debug("Image", "sending encoded image")
xlog.Debug("found image in an existing thread", "image", len(images))
threadMessages = append(
threadMessages,
openai.ChatCompletionMessage{
Role: role,
MultiContent: []openai.ChatMessagePart{
{
Text: replaceUserIDsWithNamesInMessage(api, cleanUpUsernameFromMessage(msg.Text, b)),
Type: openai.ChatMessagePartTypeText,
},
{
Type: openai.ChatMessagePartTypeImageURL,
ImageURL: &openai.ChatMessageImageURL{
URL: fmt.Sprintf("data:%s;base64,%s", mimeType, imgBase64),
// URL: imgUrl,
},
},
},
},
createMultiContentMessage(role, replaceUserIDsWithNamesInMessage(api, cleanUpUsernameFromMessage(msg.Text, b)), images),
)
} else {
xlog.Debug("no image in the last message of the thread", "message", msg.Text)
@@ -606,65 +657,15 @@ func (t *Slack) handleMention(
}
} else {
imageBytes := new(bytes.Buffer)
mimeType := "image/jpeg"
// Fetch the message using the API
messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{
ChannelID: ev.Channel,
Timestamp: ev.TimeStamp,
})
if err != nil {
xlog.Error(fmt.Sprintf("Error fetching messages: %v", err))
} else {
for _, msg := range messages {
if len(msg.Files) == 0 {
xlog.Debug("no files in the message", "message", msg.Text)
continue
}
for _, attachment := range msg.Files {
if attachment.URLPrivate != "" {
xlog.Debug("found image in the message of the thread", "image", attachment.URLPrivate)
xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment))
// download image with slack api
mimeType = attachment.Mimetype
if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil {
xlog.Error(fmt.Sprintf("Error downloading image: %v", err))
}
}
}
}
}
images := scanImagesInAppMentionEvent(api, ev)
// If the last message has an image, we send it as a multi content message
if len(imageBytes.Bytes()) > 0 {
xlog.Debug("found image in the last message of the thread", "image", len(imageBytes.Bytes()))
// // Encode the image to base64
imgBase64, err := encodeImageFromURL(*imageBytes)
if err != nil {
xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err))
}
if len(images) > 0 {
xlog.Debug("found image in the last message of the thread", "image", len(images))
threadMessages = append(
threadMessages,
openai.ChatCompletionMessage{
Role: "user",
MultiContent: []openai.ChatMessagePart{
{
Text: replaceUserIDsWithNamesInMessage(api, cleanUpUsernameFromMessage(message, b)),
Type: openai.ChatMessagePartTypeText,
},
{
Type: openai.ChatMessagePartTypeImageURL,
ImageURL: &openai.ChatMessageImageURL{
// URL: imgURL,
URL: fmt.Sprintf("data:%s;base64,%s", mimeType, imgBase64),
},
},
},
},
createMultiContentMessage("user", replaceUserIDsWithNamesInMessage(api, cleanUpUsernameFromMessage(message, b)), images),
)
} else {
threadMessages = append(threadMessages, openai.ChatCompletionMessage{