From 42716d5e9ae1caad57bda62f30da9681bacc2fca Mon Sep 17 00:00:00 2001 From: roldengarm <37638588+roldengarm@users.noreply.github.com> Date: Tue, 2 Dec 2025 05:13:11 +1200 Subject: [PATCH] Add IncludeThoughts parameter to Google Connector for accessing Gemini reasoning content (#13383) ### Motivation and Context This implements the IncludeThoughts parameter for the Google connector. See [related issue here](https://github.com/microsoft/semantic-kernel/issues/13360). ### Description Adds `IncludeThoughts` parameter to `GeminiThinkingConfig` to enable access to Gemini's reasoning process. When enabled, thinking content is exposed via the `Items` collection as `ReasoningContent` objects, allowing developers to access both the model's internal reasoning and final response separately. ### Contribution Checklist - [X] The code builds clean without any errors or warnings - [X] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [X] All unit tests pass, and I have added new tests where possible - [X] I didn't break anyone :smile: --------- Co-authored-by: Roger Barreto <19890735+rogerbarreto@users.noreply.github.com> Co-authored-by: Mark Wallace <127216156+markwallace-microsoft@users.noreply.github.com> --- .../Clients/GeminiChatGenerationTests.cs | 150 ++++++++++++++++++ .../Clients/GeminiChatStreamingTests.cs | 41 +++++ .../GeminiPromptExecutionSettingsTests.cs | 63 ++++++++ .../Clients/GeminiChatCompletionClient.cs | 83 ++++++++-- .../Core/Gemini/Models/GeminiPart.cs | 7 + .../Core/Gemini/Models/GeminiRequest.cs | 5 + .../Connectors.Google/GeminiThinkingConfig.cs | 7 + 7 files changed, 342 insertions(+), 14 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/Clients/GeminiChatGenerationTests.cs b/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/Clients/GeminiChatGenerationTests.cs index 82d804d500..027195a648 100644 --- a/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/Clients/GeminiChatGenerationTests.cs +++ b/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/Clients/GeminiChatGenerationTests.cs @@ -9,6 +9,7 @@ using System.Text; using System.Text.Json; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.ChatCompletion; using Microsoft.SemanticKernel.Connectors.Google; using Microsoft.SemanticKernel.Connectors.Google.Core; @@ -597,6 +598,155 @@ public sealed class GeminiChatGenerationTests : IDisposable public ValueTask GetBearerToken() => ValueTask.FromResult(this.BearerKeys[this._index++]); } + [Fact] + public async Task ShouldHandleThoughtAndTextPartsAsync() + { + // Arrange + var responseContent = """ + { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "Let me think about this...", + "thought": true + }, + { + "text": "The answer is 42." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ] + } + """; + + this._messageHandlerStub.ResponseToReturn.Content = new StringContent(responseContent); + var client = this.CreateChatCompletionClient(); + var chatHistory = CreateSampleChatHistory(); + + // Act + var result = await client.GenerateChatMessageAsync(chatHistory); + + // Assert + Assert.NotNull(result); + var message = result[0]; + Assert.Equal("The answer is 42.", message.Content); + Assert.Equal(2, message.Items.Count); + +#pragma warning disable SKEXP0110 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + var reasoningContent = message.Items.OfType().FirstOrDefault(); + Assert.NotNull(reasoningContent); + Assert.Equal("Let me think about this...", reasoningContent.Text); + + var textContent = message.Items.OfType().FirstOrDefault(); + Assert.NotNull(textContent); + Assert.Equal("The answer is 42.", textContent.Text); +#pragma warning restore SKEXP0110 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + } + + [Fact] + public async Task ShouldHandleOnlyThoughtPartsAsync() + { + // Arrange + var responseContent = """ + { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "This is just thinking content...", + "thought": true + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ] + } + """; + + this._messageHandlerStub.ResponseToReturn.Content = new StringContent(responseContent); + var client = this.CreateChatCompletionClient(); + var chatHistory = CreateSampleChatHistory(); + + // Act + var result = await client.GenerateChatMessageAsync(chatHistory); + + // Assert + Assert.NotNull(result); + var message = result[0]; + Assert.True(string.IsNullOrEmpty(message.Content)); + Assert.Single(message.Items); + +#pragma warning disable SKEXP0110 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + var reasoningContent = message.Items.OfType().Single(); + Assert.Equal("This is just thinking content...", reasoningContent.Text); +#pragma warning restore SKEXP0110 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + } + + [Fact] + public async Task ShouldHandleMultipleThoughtPartsAsync() + { + // Arrange + var responseContent = """ + { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "First thought...", + "thought": true + }, + { + "text": "Second thought...", + "thought": true + }, + { + "text": "Final answer." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ] + } + """; + + this._messageHandlerStub.ResponseToReturn.Content = new StringContent(responseContent); + var client = this.CreateChatCompletionClient(); + var chatHistory = CreateSampleChatHistory(); + + // Act + var result = await client.GenerateChatMessageAsync(chatHistory); + + // Assert + Assert.NotNull(result); + var message = result[0]; + Assert.Equal("Final answer.", message.Content); + Assert.Equal(3, message.Items.Count); + +#pragma warning disable SKEXP0110 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + var reasoningContents = message.Items.OfType().ToList(); + Assert.Equal(2, reasoningContents.Count); + Assert.Equal("First thought...", reasoningContents[0].Text); + Assert.Equal("Second thought...", reasoningContents[1].Text); + + var textContent = message.Items.OfType().Single(); + Assert.Equal("Final answer.", textContent.Text); +#pragma warning restore SKEXP0110 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + } + private static ChatHistory CreateSampleChatHistory() { var chatHistory = new ChatHistory(); diff --git a/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/Clients/GeminiChatStreamingTests.cs b/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/Clients/GeminiChatStreamingTests.cs index 808ce2eb6f..90588f80d6 100644 --- a/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/Clients/GeminiChatStreamingTests.cs +++ b/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/Clients/GeminiChatStreamingTests.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Net.Http; using System.Text.Json; using System.Threading.Tasks; +using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.ChatCompletion; using Microsoft.SemanticKernel.Connectors.Google; using Microsoft.SemanticKernel.Connectors.Google.Core; @@ -428,6 +429,46 @@ public sealed class GeminiChatStreamingTests : IDisposable Assert.DoesNotContain("key=", this._messageHandlerStub.RequestUri.ToString()); } + [Fact] + public async Task ShouldHandleStreamingThoughtPartsAsync() + { + // Arrange + var streamingResponse = """ + data: {"candidates": [{"content": {"parts": [{"text": "Let me think...", "thought": true}], "role": "model"}, "index": 0}]} + + data: {"candidates": [{"content": {"parts": [{"text": "The answer is"}], "role": "model"}, "index": 0}]} + + data: {"candidates": [{"content": {"parts": [{"text": " 42."}], "role": "model"}, "finishReason": "STOP", "index": 0}]} + + """; + + this._messageHandlerStub.ResponseToReturn.Content = new StringContent(streamingResponse); + var client = this.CreateChatCompletionClient(); + var chatHistory = CreateSampleChatHistory(); + + // Act + var messages = await client.StreamGenerateChatMessageAsync(chatHistory).ToListAsync(); + + // Assert + Assert.Equal(3, messages.Count); + + // First message should contain thought + var firstMessage = messages[0]; + Assert.True(string.IsNullOrEmpty(firstMessage.Content)); + Assert.Single(firstMessage.Items); +#pragma warning disable SKEXP0110 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + var thoughtItem = firstMessage.Items.OfType().Single(); +#pragma warning restore SKEXP0110 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + Assert.Equal("Let me think...", thoughtItem.InnerContent); + + // Second and third messages contain regular text + var secondMessage = messages[1]; + Assert.Equal("The answer is", secondMessage.Content); + + var thirdMessage = messages[2]; + Assert.Equal(" 42.", thirdMessage.Content); + } + private static ChatHistory CreateSampleChatHistory() { var chatHistory = new ChatHistory(); diff --git a/dotnet/src/Connectors/Connectors.Google.UnitTests/GeminiPromptExecutionSettingsTests.cs b/dotnet/src/Connectors/Connectors.Google.UnitTests/GeminiPromptExecutionSettingsTests.cs index 38d7ca3a2d..2eb2500b1a 100644 --- a/dotnet/src/Connectors/Connectors.Google.UnitTests/GeminiPromptExecutionSettingsTests.cs +++ b/dotnet/src/Connectors/Connectors.Google.UnitTests/GeminiPromptExecutionSettingsTests.cs @@ -268,4 +268,67 @@ public sealed class GeminiPromptExecutionSettingsTests Assert.Throws(() => executionSettings.SafetySettings!.Add(new GeminiSafetySetting(GeminiSafetyCategory.Toxicity, GeminiSafetyThreshold.Unspecified))); Assert.Throws(() => executionSettings.ThinkingConfig = new GeminiThinkingConfig { ThinkingBudget = 1 }); } + + [Fact] + public void ItCreatesThinkingConfigWithIncludeThoughts() + { + // Arrange & Act + var thinkingConfig = new GeminiThinkingConfig + { + ThinkingBudget = 2000, + IncludeThoughts = true + }; + + var executionSettings = new GeminiPromptExecutionSettings + { + ThinkingConfig = thinkingConfig + }; + + // Assert + Assert.NotNull(executionSettings.ThinkingConfig); + Assert.Equal(2000, executionSettings.ThinkingConfig.ThinkingBudget); + Assert.True(executionSettings.ThinkingConfig.IncludeThoughts); + } + + [Fact] + public void ItSerializesThinkingConfigWithIncludeThoughts() + { + // Arrange + var executionSettings = new GeminiPromptExecutionSettings + { + ThinkingConfig = new GeminiThinkingConfig + { + ThinkingBudget = 1500, + IncludeThoughts = true + } + }; + + // Act + var json = JsonSerializer.Serialize(executionSettings); + + // Assert + Assert.Contains("thinking_budget", json); + Assert.Contains("1500", json); + Assert.Contains("include_thoughts", json); + Assert.Contains("true", json); + } + + [Fact] + public void ItClonesThinkingConfigWithIncludeThoughts() + { + // Arrange + var original = new GeminiThinkingConfig + { + ThinkingBudget = 3000, + IncludeThoughts = true + }; + + // Act + var cloned = original.Clone(); + + // Assert + Assert.NotSame(original, cloned); + Assert.Equal(original.ThinkingBudget, cloned.ThinkingBudget); + Assert.Equal(original.IncludeThoughts, cloned.IncludeThoughts); + } } diff --git a/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Clients/GeminiChatCompletionClient.cs b/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Clients/GeminiChatCompletionClient.cs index 5226e1a246..2d093c2207 100644 --- a/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Clients/GeminiChatCompletionClient.cs +++ b/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Clients/GeminiChatCompletionClient.cs @@ -748,20 +748,53 @@ internal sealed class GeminiChatCompletionClient : ClientBase private GeminiChatMessageContent GetChatMessageContentFromCandidate(GeminiResponse geminiResponse, GeminiResponseCandidate candidate) { - // Join text parts - string text = string.Concat(candidate.Content?.Parts?.Select(part => part.Text) ?? []); + var items = new List(); + + // Process parts to separate regular text from thinking content + var regularTextParts = new List(); + + if (candidate.Content?.Parts != null) + { + foreach (var part in candidate.Content.Parts) + { + if (part.Thought == true && !string.IsNullOrEmpty(part.Text)) + { + // This is thinking content +#pragma warning disable SKEXP0110 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + items.Add(new ReasoningContent(part.Text)); +#pragma warning restore SKEXP0110 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + } + else if (!string.IsNullOrEmpty(part.Text)) + { + // This is regular text content + regularTextParts.Add(part.Text); + } + } + } + + // Regular text goes into message.Content, not as a separate item + var regularText = string.Concat(regularTextParts); // Gemini sometimes returns function calls with text parts, so collect them var toolCalls = candidate.Content?.Parts? .Select(part => part.FunctionCall!) .Where(toolCall => toolCall is not null).ToArray(); - return new GeminiChatMessageContent( + // Pass null if there's no regular (non-thinking) text to avoid creating an empty TextContent item + var chatMessage = new GeminiChatMessageContent( role: candidate.Content?.Role ?? AuthorRole.Assistant, - content: text, + content: string.IsNullOrEmpty(regularText) ? null : regularText, modelId: this._modelId, functionsToolCalls: toolCalls, metadata: GetResponseMetadata(geminiResponse, candidate)); + + // Add items to the message + foreach (var item in items) + { + chatMessage.Items.Add(item); + } + + return chatMessage; } private static GeminiRequest CreateRequest( @@ -776,9 +809,11 @@ internal sealed class GeminiChatCompletionClient : ClientBase private GeminiStreamingChatMessageContent GetStreamingChatContentFromChatContent(GeminiChatMessageContent message) { + GeminiStreamingChatMessageContent streamingMessage; + if (message.CalledToolResult is not null) { - return new GeminiStreamingChatMessageContent( + streamingMessage = new GeminiStreamingChatMessageContent( role: message.Role, content: message.Content, modelId: this._modelId, @@ -786,10 +821,9 @@ internal sealed class GeminiChatCompletionClient : ClientBase metadata: message.Metadata, choiceIndex: message.Metadata?.Index ?? 0); } - - if (message.ToolCalls is not null) + else if (message.ToolCalls is not null) { - return new GeminiStreamingChatMessageContent( + streamingMessage = new GeminiStreamingChatMessageContent( role: message.Role, content: message.Content, modelId: this._modelId, @@ -797,13 +831,34 @@ internal sealed class GeminiChatCompletionClient : ClientBase metadata: message.Metadata, choiceIndex: message.Metadata?.Index ?? 0); } + else + { + streamingMessage = new GeminiStreamingChatMessageContent( + role: message.Role, + content: message.Content, + modelId: this._modelId, + choiceIndex: message.Metadata?.Index ?? 0, + metadata: message.Metadata); + } - return new GeminiStreamingChatMessageContent( - role: message.Role, - content: message.Content, - modelId: this._modelId, - choiceIndex: message.Metadata?.Index ?? 0, - metadata: message.Metadata); + // Copy ReasoningContent items to streaming message, converting to StreamingReasoningContent + foreach (var item in message.Items) + { +#pragma warning disable SKEXP0110 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + if (item is ReasoningContent reasoningContent) + { + var streamingReasoning = new StreamingReasoningContent(reasoningContent.Text) + { + InnerContent = reasoningContent.Text + }; + streamingMessage.Items.Add(streamingReasoning); + } + // Note: Other item types like TextContent are not copied since the main content + // is already in streamingMessage.Content +#pragma warning restore SKEXP0110 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + } + + return streamingMessage; } private static void ValidateAutoInvoke(bool autoInvoke, int resultsPerPrompt) diff --git a/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiPart.cs b/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiPart.cs index 7a3b22803d..725e6ae54f 100644 --- a/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiPart.cs +++ b/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiPart.cs @@ -47,6 +47,13 @@ internal sealed class GeminiPart : IJsonOnDeserialized [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public FunctionResponsePart? FunctionResponse { get; set; } + /// + /// Gets or sets a value indicating whether this part contains thinking content. + /// + [JsonPropertyName("thought")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? Thought { get; set; } + /// /// Checks whether only one property of the GeminiPart instance is not null. /// Returns true if only one property among Text, InlineData, FileData, FunctionCall, and FunctionResponse is not null, diff --git a/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiRequest.cs b/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiRequest.cs index 835b0a4933..3bba31805c 100644 --- a/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiRequest.cs +++ b/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiRequest.cs @@ -501,6 +501,7 @@ internal sealed class GeminiRequest request.Configuration.ThinkingConfig = new GeminiRequestThinkingConfig { ThinkingBudget = executionSettings.ThinkingConfig.ThinkingBudget, + IncludeThoughts = executionSettings.ThinkingConfig.IncludeThoughts, ThinkingLevel = executionSettings.ThinkingConfig.ThinkingLevel }; } @@ -555,6 +556,10 @@ internal sealed class GeminiRequest [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public int? ThinkingBudget { get; set; } + [JsonPropertyName("includeThoughts")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? IncludeThoughts { get; set; } + [JsonPropertyName("thinkingLevel")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? ThinkingLevel { get; set; } diff --git a/dotnet/src/Connectors/Connectors.Google/GeminiThinkingConfig.cs b/dotnet/src/Connectors/Connectors.Google/GeminiThinkingConfig.cs index ad4fd84563..da30d54bab 100644 --- a/dotnet/src/Connectors/Connectors.Google/GeminiThinkingConfig.cs +++ b/dotnet/src/Connectors/Connectors.Google/GeminiThinkingConfig.cs @@ -31,6 +31,13 @@ public class GeminiThinkingConfig [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? ThinkingLevel { get; set; } + /// + /// Whether to include the thinking content in the response. + /// + [JsonPropertyName("include_thoughts")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? IncludeThoughts { get; set; } + /// /// Clones this instance. ///