Java: Add support for response_format on chat completion (#5782)

### Motivation and Context

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->

### Description

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [ ] The code builds clean without any errors or warnings
- [ ] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [ ] All unit tests pass, and I have added new tests where possible
- [ ] I didn't break anyone 😄

java/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java

@@ -6,7 +6,9 @@ import com.azure.ai.openai.models.ChatChoice;
 import com.azure.ai.openai.models.ChatCompletions;
 import com.azure.ai.openai.models.ChatCompletionsFunctionToolCall;
 import com.azure.ai.openai.models.ChatCompletionsFunctionToolDefinition;
+import com.azure.ai.openai.models.ChatCompletionsJsonResponseFormat;
 import com.azure.ai.openai.models.ChatCompletionsOptions;
+import com.azure.ai.openai.models.ChatCompletionsTextResponseFormat;
 import com.azure.ai.openai.models.ChatCompletionsToolCall;
 import com.azure.ai.openai.models.ChatCompletionsToolDefinition;
 import com.azure.ai.openai.models.ChatRequestAssistantMessage;
@@ -407,6 +409,20 @@ public class OpenAIChatCompletion implements ChatCompletionService {
             .setUser(promptExecutionSettings.getUser())
             .setLogitBias(logit);
 
+        if (promptExecutionSettings.getResponseFormat() != null) {
+            switch (promptExecutionSettings.getResponseFormat()) {
+                case JSON_OBJECT:
+                    options.setResponseFormat(new ChatCompletionsJsonResponseFormat());
+                    break;
+                case TEXT:
+                    options.setResponseFormat(new ChatCompletionsTextResponseFormat());
+                    break;
+                default:
+                    throw new SKException(
+                        "Unknown response format: " + promptExecutionSettings.getResponseFormat());
+            }
+        }
+
         return options;
     }
 

java/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/InvocationContext.java

@@ -97,6 +97,20 @@ public class InvocationContext {
         }
     }
 
+    /**
+     * Create a new instance of InvocationContext by copying the values from another instance.
+     *
+     * @param context The context to copy.
+     * @return The new instance of InvocationContext.
+     */
+    public static Builder copy(InvocationContext context) {
+        return new Builder()
+            .withKernelHooks(context.getKernelHooks())
+            .withContextVariableConverter(context.contextVariableTypes)
+            .withPromptExecutionSettings(context.getPromptExecutionSettings())
+            .withToolCallBehavior(context.getToolCallBehavior());
+    }
+
     /**
      * Get the hooks to use for the invocation.
      *
@@ -198,6 +212,17 @@ public class InvocationContext {
             return this;
         }
 
+        /**
+         * Add a context variable type converter to the builder.
+         *
+         * @param contextVariableTypes the context variable types to add.
+         * @return this {@link Builder}
+         */
+        public Builder withContextVariableConverter(ContextVariableTypes contextVariableTypes) {
+            this.contextVariableTypes.putConverters(contextVariableTypes);
+            return this;
+        }
+
         @Override
         public InvocationContext build() {
             return new InvocationContext(hooks, promptExecutionSettings, toolCallBehavior,

java/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/PromptExecutionSettings.java

@@ -7,6 +7,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import javax.annotation.Nullable;
@@ -17,58 +18,53 @@ import javax.annotation.Nullable;
 public class PromptExecutionSettings {
 
     /**
-     * The default for {@link #getServiceId()} if a 
-     * {@link Builder#withServiceId(String) service id} is not provided.
-     * Defaults to {@code "default"}
+     * The default for {@link #getServiceId()} if a {@link Builder#withServiceId(String) service id}
+     * is not provided. Defaults to {@code "default"}
      */
     public static final String DEFAULT_SERVICE_ID = "default";
 
     /**
-     * The default for {@link #getMaxTokens()} if 
-     * {@link Builder#withMaxTokens(int) max_tokens} is not provided.
-     * Defaults to {@code 256}
+     * The default for {@link #getMaxTokens()} if {@link Builder#withMaxTokens(int) max_tokens} is
+     * not provided. Defaults to {@code 256}
      */
     public static final int DEFAULT_MAX_TOKENS = 256;
 
     /**
-     * The default for {@link #getTemperature()} if 
-     * {@link Builder#withTemperature(double) temperature} is not provided.
-     * Defaults to {@code 1.0}
+     * The default for {@link #getTemperature()} if
+     * {@link Builder#withTemperature(double) temperature} is not provided. Defaults to {@code 1.0}
      */
     public static final double DEFAULT_TEMPERATURE = 1.0;
 
     /**
-     * The default for {@link #getTopP()} if 
-     * {@link Builder#withTopP(double) top_p} is not provided.
+     * The default for {@link #getTopP()} if {@link Builder#withTopP(double) top_p} is not provided.
      * Defaults to {@code 1.0}
      */
     public static final double DEFAULT_TOP_P = 1.0;
 
     /**
-     * The default for {@link #getPresencePenalty()} if 
-     * {@link Builder#withPresencePenalty(double) presence_penalty} is not provided.
-     * Defaults to {@code 0.0}
+     * The default for {@link #getPresencePenalty()} if
+     * {@link Builder#withPresencePenalty(double) presence_penalty} is not provided. Defaults to
+     * {@code 0.0}
      */
     public static final double DEFAULT_PRESENCE_PENALTY = 0.0;
 
     /**
-     * The default for {@link #getFrequencyPenalty()} if 
-     * {@link Builder#withMaxTokens(doulbe) frequency_penalty} is not provided.
-     * Defaults to {@code 0.0}
+     * The default for {@link #getFrequencyPenalty()} if
+     * {@link Builder#withMaxTokens(doulbe) frequency_penalty} is not provided. Defaults to
+     * {@code 0.0}
      */
     public static final double DEFAULT_FREQUENCY_PENALTY = 0.0;
 
     /**
-     * The default for {@link #getBestOf()} if 
-     * {@link Builder#withBestOf(int) best_of} is not provided.
-     * Defaults to {@code 1}
+     * The default for {@link #getBestOf()} if {@link Builder#withBestOf(int) best_of} is not
+     * provided. Defaults to {@code 1}
      */
     public static final int DEFAULT_BEST_OF = 1;
 
     /**
-     * The default for {@link #getResultsPerPrompt()} if 
-     * {@link Builder#withResultsPerPrompt(int) results per prompt (n)} is not provided.
-     * Defaults to {@code 1}
+     * The default for {@link #getResultsPerPrompt()} if
+     * {@link Builder#withResultsPerPrompt(int) results per prompt (n)} is not provided. Defaults to
+     * {@code 1}
      */
     public static final int DEFAULT_RESULTS_PER_PROMPT = 1;
 
@@ -87,6 +83,7 @@ public class PromptExecutionSettings {
     private static final String STOP_SEQUENCES = "stop_sequences";
     private static final String RESULTS_PER_PROMPT = "results_per_prompt";
     private static final String TOKEN_SELECTION_BIASES = "token_selection_biases";
+    private static final String RESPONSE_FORMAT = "response_format";
 
     private final String serviceId;
     private final String modelId;
@@ -100,6 +97,7 @@ public class PromptExecutionSettings {
     private final String user;
     private final List<String> stopSequences;
     private final Map<Integer, Integer> tokenSelectionBiases;
+    private final ResponseFormat responseFormat;
 
     /**
      * Create a new instance of PromptExecutionSettings.
@@ -116,6 +114,8 @@ public class PromptExecutionSettings {
      * @param user                 The user to associate with the prompt execution.
      * @param stopSequences        The stop sequences to use for prompt execution.
      * @param tokenSelectionBiases The token selection biases to use for prompt execution.
+     * @param responseFormat       The response format to use for prompt execution @{link
+     *                             ResponseFormat}.
      */
     @JsonCreator
     public PromptExecutionSettings(
@@ -130,7 +130,8 @@ public class PromptExecutionSettings {
         @JsonProperty(BEST_OF) Integer bestOf,
         @JsonProperty(USER) String user,
         @Nullable @JsonProperty(STOP_SEQUENCES) List<String> stopSequences,
-        @Nullable @JsonProperty(TOKEN_SELECTION_BIASES) Map<Integer, Integer> tokenSelectionBiases) {
+        @Nullable @JsonProperty(TOKEN_SELECTION_BIASES) Map<Integer, Integer> tokenSelectionBiases,
+        @Nullable @JsonProperty(RESPONSE_FORMAT) String responseFormat) {
         this.serviceId = serviceId != null ? serviceId : DEFAULT_SERVICE_ID;
         this.modelId = modelId != null ? modelId : "";
         this.temperature = clamp(temperature, 0d, 2d, DEFAULT_TEMPERATURE);
@@ -148,6 +149,12 @@ public class PromptExecutionSettings {
             ? new HashMap<>(tokenSelectionBiases)
             : Collections.emptyMap();
         this.tokenSelectionBiases.replaceAll((k, v) -> clamp(v, -100, 100, 0));
+
+        if (responseFormat != null && !responseFormat.isEmpty()) {
+            this.responseFormat = ResponseFormat.valueOf(responseFormat.toUpperCase(Locale.ROOT));
+        } else {
+            this.responseFormat = null;
+        }
     }
 
     /**
@@ -307,43 +314,71 @@ public class PromptExecutionSettings {
     public int hashCode() {
         return Objects.hash(serviceId, modelId, temperature, topP, presencePenalty,
             frequencyPenalty,
-            maxTokens, bestOf, resultsPerPrompt, user, stopSequences, tokenSelectionBiases);
+            maxTokens, bestOf, resultsPerPrompt, user, stopSequences, tokenSelectionBiases,
+            responseFormat);
     }
 
     @Override
     public boolean equals(Object obj) {
-        if (this == obj)
+        if (this == obj) {
             return true;
-        if (!getClass().isInstance(obj))
+        }
+        if (!getClass().isInstance(obj)) {
             return false;
+        }
 
         PromptExecutionSettings other = (PromptExecutionSettings) obj;
 
-        if (!Objects.equals(serviceId, other.serviceId))
+        if (!Objects.equals(serviceId, other.serviceId)) {
             return false;
-        if (!Objects.equals(modelId, other.modelId))
+        }
+        if (!Objects.equals(modelId, other.modelId)) {
             return false;
-        if (Double.compare(temperature, other.temperature) != 0)
+        }
+        if (Double.compare(temperature, other.temperature) != 0) {
             return false;
-        if (Double.compare(topP, other.topP) != 0)
+        }
+        if (Double.compare(topP, other.topP) != 0) {
             return false;
-        if (Double.compare(presencePenalty, other.presencePenalty) != 0)
+        }
+        if (Double.compare(presencePenalty, other.presencePenalty) != 0) {
             return false;
-        if (Double.compare(frequencyPenalty, other.frequencyPenalty) != 0)
+        }
+        if (Double.compare(frequencyPenalty, other.frequencyPenalty) != 0) {
             return false;
-        if (maxTokens != other.maxTokens)
+        }
+        if (maxTokens != other.maxTokens) {
             return false;
-        if (bestOf != other.bestOf)
+        }
+        if (bestOf != other.bestOf) {
             return false;
-        if (resultsPerPrompt != other.resultsPerPrompt)
+        }
+        if (resultsPerPrompt != other.resultsPerPrompt) {
             return false;
-        if (!Objects.equals(user, other.user))
+        }
+        if (!Objects.equals(user, other.user)) {
             return false;
-        if (!Objects.equals(stopSequences, other.stopSequences))
+        }
+        if (!Objects.equals(stopSequences, other.stopSequences)) {
             return false;
+        }
+        if (!Objects.equals(responseFormat, other.responseFormat)) {
+            return false;
+        }
         return Objects.equals(tokenSelectionBiases, other.tokenSelectionBiases);
     }
 
+    /**
+     * The response format to use for prompt execution. Currently this only applies to chat
+     * completions.
+     *
+     * @return The response format to use for prompt execution.
+     */
+    @Nullable
+    public ResponseFormat getResponseFormat() {
+        return responseFormat;
+    }
+
     /**
      * Builder for PromptExecutionSettings.
      */
@@ -507,6 +542,19 @@ public class PromptExecutionSettings {
             return this;
         }
 
+        /**
+         * Set the response format to use for prompt execution.
+         *
+         * @param responseFormat The response format to use for prompt execution.
+         * @return This builder.
+         */
+        public Builder withResponseFormat(ResponseFormat responseFormat) {
+            if (responseFormat != null) {
+                settings.put(RESPONSE_FORMAT, responseFormat);
+            }
+            return this;
+        }
+
         /**
          * Build the PromptExecutionSettings.
          *
@@ -527,7 +575,8 @@ public class PromptExecutionSettings {
                 (String) settings.getOrDefault(USER, ""),
                 (List<String>) settings.getOrDefault(STOP_SEQUENCES, Collections.emptyList()),
                 (Map<Integer, Integer>) settings.getOrDefault(TOKEN_SELECTION_BIASES,
-                    Collections.emptyMap()));
+                    Collections.emptyMap()),
+                (String) settings.getOrDefault(RESPONSE_FORMAT, null));
         }
     }
 }

java/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/ResponseFormat.java

@@ -0,0 +1,10 @@
+// Copyright (c) Microsoft. All rights reserved.
+package com.microsoft.semantickernel.orchestration;
+
+public enum ResponseFormat {
+
+    /**
+     * Only valid for openai chat completion, with GPT-4 and gpt-3.5-turbo-1106+ models.
+     */
+    JSON_OBJECT, TEXT;
+}

java/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java

@@ -152,11 +152,18 @@ public class KernelFunctionFromPrompt<T> extends KernelFunction<T> {
                 PromptExecutionSettings executionSettings = aiServiceSelection.getSettings();
 
                 if (client instanceof ChatCompletionService) {
+                    InvocationContext contextWithExecutionSettings = context;
+                    if (context.getPromptExecutionSettings() == null) {
+                        contextWithExecutionSettings = InvocationContext.copy(context)
+                            .withPromptExecutionSettings(executionSettings)
+                            .build();
+                    }
+
                     result = ((ChatCompletionService) client)
                         .getChatMessageContentsAsync(
                             prompt,
                             kernel,
-                            context)
+                            contextWithExecutionSettings)
                         .flatMapMany(Flux::fromIterable)
                         .concatMap(chatMessageContent -> {
                             if (chatMessageContent.getAuthorRole() == AuthorRole.ASSISTANT) {