microsoft/semantic-kernel
1
0
Fork 0
mirror of https://github.com/microsoft/semantic-kernel.git synced 2026-10-12 02:19:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Java: Add OpenAI Audio to/from Text services (#5916)

Browse Source 
### Motivation and Context

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->

### Description

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [ ] The code builds clean without any errors or warnings
- [ ] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [ ] All unit tests pass, and I have added new tests where possible
- [ ] I didn't break anyone 😄
This commit is contained in:
John Oliver
2024-04-19 15:12:54 +01:00
committed by GitHub
parent 0a7f96ae4b
commit fcc56cd909
27 changed files with 1188 additions and 301 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/OpenAiService.java
+40
View File
@@ -0,0 +1,40 @@
// Copyright (c) Microsoft. All rights reserved.
package com.microsoft.semantickernel.aiservices.openai;
import com.azure.ai.openai.OpenAIAsyncClient;
import com.microsoft.semantickernel.services.AIService;
import javax.annotation.Nullable;
/**
* Provides OpenAI service.
*/
public abstract class OpenAiService implements AIService {
private final OpenAIAsyncClient client;
@Nullable
private final String serviceId;
private final String modelId;
protected OpenAiService(OpenAIAsyncClient client, @Nullable String serviceId, String modelId) {
this.client = client;
this.serviceId = serviceId;
this.modelId = modelId;
}
@Nullable
@Override
public String getModelId() {
return modelId;
}
@Override
@Nullable
public String getServiceId() {
return serviceId;
}
protected OpenAIAsyncClient getClient() {
return client;
}
}
java/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiAudioToTextService.java
+111
View File
@@ -0,0 +1,111 @@
// Copyright (c) Microsoft. All rights reserved.
package com.microsoft.semantickernel.aiservices.openai.audio;
import com.azure.ai.openai.OpenAIAsyncClient;
import com.azure.ai.openai.models.AudioTranscription;
import com.azure.ai.openai.models.AudioTranscriptionFormat;
import com.azure.ai.openai.models.AudioTranscriptionOptions;
import com.microsoft.semantickernel.aiservices.openai.OpenAiService;
import com.microsoft.semantickernel.exceptions.SKException;
import com.microsoft.semantickernel.services.audio.AudioContent;
import com.microsoft.semantickernel.services.audio.AudioToTextExecutionSettings;
import com.microsoft.semantickernel.services.audio.AudioToTextService;
import javax.annotation.Nullable;
import reactor.core.publisher.Mono;
/**
* Provides OpenAi implementation of audio to text service.
*/
public class OpenAiAudioToTextService extends OpenAiService implements AudioToTextService {
/**
* Creates an instance of OpenAi audio to text service.
*
* @param client OpenAI client.
* @param modelId The model ID.
*/
public OpenAiAudioToTextService(
OpenAIAsyncClient client,
String modelId) {
super(client, null, modelId);
}
@Override
public Mono<String> getTextContentsAsync(
AudioContent content,
@Nullable AudioToTextExecutionSettings executionSettings) {
AudioTranscriptionOptions options = convertOptions(content, executionSettings);
// TODO: Should use getAudioTranscriptionTextWithResponse, and OpenAIRequestSettings.getRequestOptions()
// however currently this breaks the request
return getClient()
.getAudioTranscription(
getModelId(),
options.getFilename(),
options)
.map(AudioTranscription::getText);
}
private AudioTranscriptionOptions convertOptions(
AudioContent content,
@Nullable AudioToTextExecutionSettings executionSettings) {
AudioTranscriptionOptions options = new AudioTranscriptionOptions(content.getData());
options.setModel(getModelId());
if (executionSettings == null) {
return options;
}
if (executionSettings.getResponseFormat() != null) {
options.setResponseFormat(
AudioTranscriptionFormat.fromString(executionSettings.getResponseFormat()));
}
if (executionSettings.getFilename() != null) {
options.setFilename(executionSettings.getFilename());
}
if (executionSettings.getLanguage() != null) {
options.setLanguage(executionSettings.getLanguage());
}
if (executionSettings.getPrompt() != null) {
options.setPrompt(executionSettings.getPrompt());
}
if (executionSettings.getTemperature() != null) {
options.setTemperature(executionSettings.getTemperature());
}
return options;
}
/**
* Builder for OpenAiAudioToTextService.
*
* @return The builder.
*/
public static Builder builder() {
return new Builder();
}
/**
* Represents a builder for OpenAiAudioToTextService.
*/
public static class Builder extends AudioToTextService.Builder {
/**
* builds the OpenAiAudioToTextService.
*/
@Override
public AudioToTextService build() {
if (client == null) {
throw new SKException("OpenAI client is required");
}
if (modelId == null) {
throw new SKException("Model id is required");
}
return new OpenAiAudioToTextService(client, modelId);
}
}
}
java/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiTextToAudioService.java
+95
View File
@@ -0,0 +1,95 @@
// Copyright (c) Microsoft. All rights reserved.
package com.microsoft.semantickernel.aiservices.openai.audio;
import com.azure.ai.openai.OpenAIAsyncClient;
import com.azure.ai.openai.models.SpeechGenerationOptions;
import com.azure.ai.openai.models.SpeechGenerationResponseFormat;
import com.azure.ai.openai.models.SpeechVoice;
import com.microsoft.semantickernel.aiservices.openai.OpenAiService;
import com.microsoft.semantickernel.exceptions.SKException;
import com.microsoft.semantickernel.services.audio.AudioContent;
import com.microsoft.semantickernel.services.audio.TextToAudioExecutionSettings;
import com.microsoft.semantickernel.services.audio.TextToAudioService;
import reactor.core.publisher.Mono;
/**
* Provides OpenAi implementation of text to audio service.
*/
public class OpenAiTextToAudioService extends OpenAiService implements TextToAudioService {
/**
* Creates an instance of OpenAi text to audio service.
*
* @param client OpenAI client.
* @param modelId The model ID.
*/
public OpenAiTextToAudioService(
OpenAIAsyncClient client,
String modelId) {
super(client, null, modelId);
}
@Override
public Mono<AudioContent> getAudioContentAsync(
String text,
TextToAudioExecutionSettings executionSettings) {
SpeechGenerationOptions options = convertOptions(text, executionSettings);
return getClient().generateSpeechFromText(getModelId(), options)
.map(response -> new AudioContent(response.toBytes(), getModelId()));
}
private SpeechGenerationOptions convertOptions(
String text,
TextToAudioExecutionSettings executionSettings) {
SpeechGenerationOptions options = new SpeechGenerationOptions(
text,
SpeechVoice.fromString(executionSettings.getVoice()));
options.setModel(getModelId());
if (executionSettings.getResponseFormat() != null) {
options.setResponseFormat(
SpeechGenerationResponseFormat.fromString(executionSettings.getResponseFormat()));
}
if (executionSettings.getSpeed() != null) {
options.setSpeed(executionSettings.getSpeed());
}
return options;
}
/**
* Creates a new builder.
*
* @return The builder.
*/
public static Builder builder() {
return new Builder();
}
/**
* Represents a builder for OpenAi text to audio service.
*/
public static class Builder extends TextToAudioService.Builder {
/**
* Builds the OpenAi text to audio service.
*
* @return The OpenAi text to audio service.
*/
@Override
public TextToAudioService build() {
if (client == null) {
throw new SKException("OpenAI client is required");
}
if (modelId == null) {
throw new SKException("Model id is required");
}
return new OpenAiTextToAudioService(client, modelId);
}
}
}
java/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java
+32 -33
View File
@@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ContainerNode;
import com.microsoft.semantickernel.Kernel;
import com.microsoft.semantickernel.aiservices.openai.OpenAiService;
import com.microsoft.semantickernel.aiservices.openai.implementation.OpenAIRequestSettings;
import com.microsoft.semantickernel.contextvariables.ContextVariable;
import com.microsoft.semantickernel.contextvariables.ContextVariableTypes;
@@ -60,22 +61,15 @@ import reactor.core.publisher.Mono;
/**
* OpenAI chat completion service.
*/
public class OpenAIChatCompletion implements ChatCompletionService {
public class OpenAIChatCompletion extends OpenAiService implements ChatCompletionService {
private static final Logger LOGGER = LoggerFactory.getLogger(OpenAIChatCompletion.class);
private final OpenAIAsyncClient client;
@Nullable
private final String serviceId;
private final String modelId;
protected OpenAIChatCompletion(
OpenAIAsyncClient client,
String modelId,
@Nullable String serviceId) {
this.serviceId = serviceId;
this.client = client;
this.modelId = modelId;
super(client, serviceId, modelId);
}
/**
@@ -87,12 +81,6 @@ public class OpenAIChatCompletion implements ChatCompletionService {
return new OpenAIChatCompletion.Builder();
}
@Override
@Nullable
public String getServiceId() {
return serviceId;
}
@Override
public Mono<List<ChatMessageContent<?>>> getChatMessageContentsAsync(
ChatHistory chatHistory,
@@ -161,7 +149,7 @@ public class OpenAIChatCompletion implements ChatCompletionService {
autoInvokeAttempts)))
.getOptions();
Mono<List<? extends ChatMessageContent>> result = client
Mono<List<? extends ChatMessageContent>> result = getClient()
.getChatCompletionsWithResponse(getModelId(), options,
OpenAIRequestSettings.getRequestOptions())
.flatMap(completionsResult -> {
@@ -216,6 +204,11 @@ public class OpenAIChatCompletion implements ChatCompletionService {
})
.flatMap(it -> it)
.flatMap(msgs -> {
ChatRequestMessage m = msgs.get(msgs.size() - 1);
if (m instanceof ChatRequestAssistantMessage &&
((ChatRequestAssistantMessage) m).getToolCalls().size() > 0) {
LOGGER.error("No response");
}
return internalChatMessageContentsAsync(msgs, kernel, functions,
invocationContext, autoInvokeAttempts - 1);
})
@@ -607,8 +600,9 @@ public class OpenAIChatCompletion implements ChatCompletionService {
switch (authorRole) {
case ASSISTANT:
// TODO: break this out into a separate method and handle tools other than function calls
ChatRequestAssistantMessage asstMessage = new ChatRequestAssistantMessage(content);
List<OpenAIFunctionToolCall> toolCalls = ((OpenAIChatMessageContent<?>) message).getToolCall();
ChatRequestAssistantMessage asstMessage = new ChatRequestAssistantMessage(content);
List<OpenAIFunctionToolCall> toolCalls = ((OpenAIChatMessageContent<?>) message)
.getToolCall();
if (toolCalls != null) {
asstMessage.setToolCalls(
toolCalls.stream()
@@ -616,22 +610,33 @@ public class OpenAIChatCompletion implements ChatCompletionService {
KernelFunctionArguments arguments = toolCall.getArguments();
String args = arguments != null && !arguments.isEmpty()
? arguments.entrySet().stream()
.map(entry -> String.format("\"%s\": \"%s\"", entry.getKey(), entry.getValue()))
.collect(Collectors.joining("{","}",","))
.map(entry -> String.format("\"%s\": \"%s\"",
entry.getKey(), entry.getValue()))
.collect(Collectors.joining("{", "}", ","))
: "{}";
FunctionCall fnCall = new FunctionCall(toolCall.getFunctionName(), args);
return new ChatCompletionsFunctionToolCall(toolCall.getId(), fnCall);
FunctionCall fnCall = new FunctionCall(toolCall.getFunctionName(),
args);
return new ChatCompletionsFunctionToolCall(toolCall.getId(),
fnCall);
})
.collect(Collectors.toList())
);
return asstMessage;
.collect(Collectors.toList()));
}
return asstMessage;
case SYSTEM:
return new ChatRequestSystemMessage(content);
case USER:
return new ChatRequestUserMessage(content);
case TOOL:
String id = message.getMetadata().getId();
String id = null;
if (message.getMetadata() != null) {
id = message.getMetadata().getId();
}
if (id == null) {
throw new SKException(
"Require to create a tool call message, but not tool call id is available");
}
return new ChatRequestToolMessage(content, id);
default:
LOGGER.debug("Unexpected author role: " + authorRole);
@@ -641,7 +646,7 @@ public class OpenAIChatCompletion implements ChatCompletionService {
}
static ChatRequestMessage getChatRequestMessage(
AuthorRole authorRole,
AuthorRole authorRole,
String content) {
switch (authorRole) {
@@ -660,12 +665,6 @@ public class OpenAIChatCompletion implements ChatCompletionService {
}
@Nullable
@Override
public String getModelId() {
return modelId;
}
/**
* Builder for creating a new instance of {@link OpenAIChatCompletion}.
*/
java/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAITextGenerationService.java
+4 -22
View File
@@ -4,6 +4,7 @@ package com.microsoft.semantickernel.aiservices.openai.textcompletion;
import com.azure.ai.openai.OpenAIAsyncClient;
import com.azure.ai.openai.models.CompletionsOptions;
import com.microsoft.semantickernel.Kernel;
import com.microsoft.semantickernel.aiservices.openai.OpenAiService;
import com.microsoft.semantickernel.aiservices.openai.implementation.OpenAIRequestSettings;
import com.microsoft.semantickernel.exceptions.AIException;
import com.microsoft.semantickernel.exceptions.AIException.ErrorCodes;
@@ -23,12 +24,7 @@ import reactor.core.publisher.Mono;
/**
* An OpenAI implementation of a {@link TextGenerationService}.
*/
public class OpenAITextGenerationService implements TextGenerationService {
private final OpenAIAsyncClient client;
@Nullable
private final String serviceId;
private final String modelId;
public class OpenAITextGenerationService extends OpenAiService implements TextGenerationService {
/**
* Creates a new {@link OpenAITextGenerationService}.
@@ -41,9 +37,7 @@ public class OpenAITextGenerationService implements TextGenerationService {
OpenAIAsyncClient client,
String modelId,
@Nullable String serviceId) {
this.serviceId = serviceId;
this.client = client;
this.modelId = modelId;
super(client, serviceId, modelId);
}
/**
@@ -55,12 +49,6 @@ public class OpenAITextGenerationService implements TextGenerationService {
return new Builder();
}
@Override
@Nullable
public String getServiceId() {
return serviceId;
}
@Override
public Mono<List<TextContent>> getTextContentsAsync(
String prompt,
@@ -86,7 +74,7 @@ public class OpenAITextGenerationService implements TextGenerationService {
CompletionsOptions completionsOptions = getCompletionsOptions(text, requestSettings);
return client
return getClient()
.getCompletionsWithResponse(getModelId(), completionsOptions,
OpenAIRequestSettings.getRequestOptions())
.flatMap(completionsResult -> {
@@ -146,12 +134,6 @@ public class OpenAITextGenerationService implements TextGenerationService {
return options;
}
@Nullable
@Override
public String getModelId() {
return modelId;
}
/**
* Builder for a TextGenerationService
*/
java/aiservices/openai/src/main/resources/META-INF/services/com.microsoft.semantickernel.services.audio.AudioToTextService$Builder
+1
View File
@@ -0,0 +1 @@
com.microsoft.semantickernel.aiservices.openai.audio.OpenAiAudioToTextService$Builder
java/aiservices/openai/src/main/resources/META-INF/services/com.microsoft.semantickernel.services.audio.TextToAudioService$Builder
+1
View File
@@ -0,0 +1 @@
com.microsoft.semantickernel.aiservices.openai.audio.OpenAiTextToAudioService$Builder
java/pom.xml
+1
View File
@@ -605,6 +605,7 @@
-XepOpt:NullAway:AnnotatedPackages=com.microsoft.semantickernel
-Xep:AlmostJavadoc:OFF -Xep:MissingSummary:OFF
-Xep:UnusedVariable:OFF -Xep:EmptyBlockTag:OFF
-XepExcludedPaths:.*/src/test/java/.*
</arg>
</compilerArgs>
<annotationProcessorPaths>
java/samples/booking-agent-m365/src/main/java/com/microsoft/semantickernel/BookingAgent.java
+26 -25
View File
@@ -1,3 +1,4 @@
// Copyright (c) Microsoft. All rights reserved.
package com.microsoft.semantickernel;
import com.azure.ai.openai.OpenAIAsyncClient;
@@ -25,7 +26,7 @@ public class BookingAgent {
private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY");
private static final String CLIENT_ENDPOINT = System.getenv("CLIENT_ENDPOINT");
private static final String MODEL_ID = System.getenv()
.getOrDefault("MODEL_ID", "gpt-35-turbo-2");
.getOrDefault("MODEL_ID", "gpt-35-turbo-2");
// Config for Graph
// https://learn.microsoft.com/en-us/graph/tutorials/java?tabs=aad&tutorial-step=1
@@ -41,13 +42,13 @@ public class BookingAgent {
public static void initializeGraph() {
InteractiveBrowserCredential interactiveBrowserCredential = new InteractiveBrowserCredentialBuilder()
.clientId(CLIENT_ID)
.tenantId(TENANT_ID)
.redirectUrl(REDIRECT_URL)
.build();
.clientId(CLIENT_ID)
.tenantId(TENANT_ID)
.redirectUrl(REDIRECT_URL)
.build();
userClient = new GraphServiceClient(interactiveBrowserCredential,
"User.Read", "BookingsAppointment.ReadWrite.All");
"User.Read", "BookingsAppointment.ReadWrite.All");
}
public static void main(String[] args) throws NoSuchMethodException {
@@ -56,34 +57,33 @@ public class BookingAgent {
OpenAIAsyncClient client;
if (AZURE_CLIENT_KEY != null) {
client = new OpenAIClientBuilder()
.credential(new AzureKeyCredential(AZURE_CLIENT_KEY))
.endpoint(CLIENT_ENDPOINT)
.buildAsyncClient();
.credential(new AzureKeyCredential(AZURE_CLIENT_KEY))
.endpoint(CLIENT_ENDPOINT)
.buildAsyncClient();
} else {
client = new OpenAIClientBuilder()
.credential(new KeyCredential(CLIENT_KEY))
.buildAsyncClient();
.credential(new KeyCredential(CLIENT_KEY))
.buildAsyncClient();
}
initializeGraph();
KernelPlugin plugin = KernelPluginFactory.createFromObject(new BookingPlugin(
userClient,
BOOKING_BUSINESS_ID,
SERVICE_ID,
"UTC"
), "BookingRestaurant");
userClient,
BOOKING_BUSINESS_ID,
SERVICE_ID,
"UTC"), "BookingRestaurant");
ChatCompletionService chat = OpenAIChatCompletion.builder()
.withModelId(MODEL_ID)
.withOpenAIAsyncClient(client)
.build();
.withModelId(MODEL_ID)
.withOpenAIAsyncClient(client)
.build();
Kernel kernel = Kernel.builder()
.withPlugin(plugin)
.withAIService(ChatCompletionService.class, chat)
.build();
.withPlugin(plugin)
.withAIService(ChatCompletionService.class, chat)
.build();
ChatHistory chatHistory = new ChatHistory();
@@ -94,10 +94,11 @@ public class BookingAgent {
while ((userMessage = scanner.nextLine()) != null) {
chatHistory.addUserMessage(userMessage);
InvocationContext invocationContext = InvocationContext.builder().
withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(true)).build();
InvocationContext invocationContext = InvocationContext.builder()
.withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(true)).build();
ChatMessageContent<?> result = chat.getChatMessageContentsAsync(chatHistory, kernel, invocationContext).block().get(0);
ChatMessageContent<?> result = chat
.getChatMessageContentsAsync(chatHistory, kernel, invocationContext).block().get(0);
chatHistory.addAssistantMessage(result.getContent());
System.out.println("Assistant > " + result);
java/samples/booking-agent-m365/src/main/java/com/microsoft/semantickernel/BookingPlugin.java
+46 -87
View File
@@ -1,3 +1,4 @@
// Copyright (c) Microsoft. All rights reserved.
package com.microsoft.semantickernel;
import com.microsoft.graph.models.BookingAppointment;
@@ -23,36 +24,20 @@ public class BookingPlugin {
private final String userTimeZone;
public BookingPlugin(GraphServiceClient graphServiceClient,
String bookingBusinessId,
String serviceId,
String userTimeZone) {
String bookingBusinessId,
String serviceId,
String userTimeZone) {
this.graphServiceClient = graphServiceClient;
this.bookingBusinessId = bookingBusinessId;
this.serviceId = serviceId;
this.userTimeZone = userTimeZone;
}
@DefineKernelFunction(
name = "bookTable",
description = "Book a table at a restaurant",
returnType = "string"
)
@DefineKernelFunction(name = "bookTable", description = "Book a table at a restaurant", returnType = "string")
public String bookTable(
@KernelFunctionParameter(
name = "restaurant",
description = "The name of the restaurant")
String restaurant,
@KernelFunctionParameter(
name = "date",
description = "The date of the reservation in UTC",
type = OffsetDateTime.class)
OffsetDateTime date,
@KernelFunctionParameter(
name = "partySize",
description = "The number of people in the party",
type = int.class)
int partySize
) {
@KernelFunctionParameter(name = "restaurant", description = "The name of the restaurant") String restaurant,
@KernelFunctionParameter(name = "date", description = "The date of the reservation in UTC", type = OffsetDateTime.class) OffsetDateTime date,
@KernelFunctionParameter(name = "partySize", description = "The number of people in the party", type = int.class) int partySize) {
BookingAppointment bookingAppointment = new BookingAppointment();
bookingAppointment.setOdataType("#microsoft.graph.bookingAppointment");
bookingAppointment.setCustomerTimeZone(userTimeZone);
@@ -82,33 +67,27 @@ public class BookingPlugin {
bookingAppointment.setMaximumAttendeesCount(partySize);
bookingAppointment.setFilledAttendeesCount(partySize);
graphServiceClient.solutions().bookingBusinesses().byBookingBusinessId(bookingBusinessId).appointments().post(bookingAppointment);
graphServiceClient.solutions().bookingBusinesses().byBookingBusinessId(bookingBusinessId)
.appointments().post(bookingAppointment);
return "Successful booking at " + restaurant;
}
@DefineKernelFunction(
name = "listReservations",
description = "List all reservations for a restaurant",
returnType = "string"
)
@DefineKernelFunction(name = "listReservations", description = "List all reservations for a restaurant", returnType = "string")
public String listReservations(
@KernelFunctionParameter(
name = "restaurant",
description = "The name of the restaurant")
String restaurant
) {
List<BookingAppointment> appointments = graphServiceClient.solutions().bookingBusinesses().byBookingBusinessId(bookingBusinessId).appointments().get().getValue();
@KernelFunctionParameter(name = "restaurant", description = "The name of the restaurant") String restaurant) {
List<BookingAppointment> appointments = graphServiceClient.solutions().bookingBusinesses()
.byBookingBusinessId(bookingBusinessId).appointments().get().getValue();
StringBuilder appointmentsList = new StringBuilder();
for (BookingAppointment appointment : appointments) {
if (appointment.getServiceLocation().getDisplayName().equals(restaurant)) {
appointmentsList.append(appointment.getServiceLocation().getDisplayName())
.append(" at ")
.append(appointment.getStartDateTime().getDateTime())
.append(" for ")
.append(appointment.getMaximumAttendeesCount())
.append(" people\n");
.append(" at ")
.append(appointment.getStartDateTime().getDateTime())
.append(" for ")
.append(appointment.getMaximumAttendeesCount())
.append(" people\n");
}
}
@@ -120,11 +99,13 @@ public class BookingPlugin {
}
private BookingAppointment getAppointment(String restaurant, OffsetDateTime date) {
List<BookingAppointment> appointments = graphServiceClient.solutions().bookingBusinesses().byBookingBusinessId(bookingBusinessId).appointments().get().getValue();
List<BookingAppointment> appointments = graphServiceClient.solutions().bookingBusinesses()
.byBookingBusinessId(bookingBusinessId).appointments().get().getValue();
for (BookingAppointment appointment : appointments) {
if (appointment.getServiceLocation().getDisplayName().equals(restaurant)
&& OffsetDateTime.parse(appointment.getStartDateTime().getDateTime()).equals(date)) {
&& OffsetDateTime.parse(appointment.getStartDateTime().getDateTime())
.equals(date)) {
return appointment;
}
}
@@ -132,68 +113,46 @@ public class BookingPlugin {
return null;
}
@DefineKernelFunction(
name = "modifyReservation",
description = "Modify an existing reservation at a restaurant",
returnType = "string"
)
@DefineKernelFunction(name = "modifyReservation", description = "Modify an existing reservation at a restaurant", returnType = "string")
public String modifyReservation(
@KernelFunctionParameter(
name = "restaurant",
description = "The name of the restaurant")
String restaurant,
@KernelFunctionParameter(
name = "date",
description = "The date of the reservation in UTC",
type = OffsetDateTime.class)
OffsetDateTime date,
@KernelFunctionParameter(
name = "updatedDate",
description = "The updated date of the reservation in UTC",
type = OffsetDateTime.class)
OffsetDateTime updatedDate
){
@KernelFunctionParameter(name = "restaurant", description = "The name of the restaurant") String restaurant,
@KernelFunctionParameter(name = "date", description = "The date of the reservation in UTC", type = OffsetDateTime.class) OffsetDateTime date,
@KernelFunctionParameter(name = "updatedDate", description = "The updated date of the reservation in UTC", type = OffsetDateTime.class) OffsetDateTime updatedDate) {
BookingAppointment appointment = getAppointment(restaurant, date);
if (appointment == null) {
return "No reservation found for " + restaurant + " on " + date;
}
appointment.setStartDateTime(new DateTimeTimeZone() {{
setDateTime(updatedDate.toString());
setTimeZone("UTC");
}});
appointment.setEndDateTime(new DateTimeTimeZone() {{
setDateTime(updatedDate.plusHours(BOOKING_HOURS).toString());
setTimeZone("UTC");
}});
appointment.setStartDateTime(new DateTimeTimeZone() {
{
setDateTime(updatedDate.toString());
setTimeZone("UTC");
}
});
appointment.setEndDateTime(new DateTimeTimeZone() {
{
setDateTime(updatedDate.plusHours(BOOKING_HOURS).toString());
setTimeZone("UTC");
}
});
graphServiceClient.solutions().bookingBusinesses().byBookingBusinessId(bookingBusinessId).appointments().byBookingAppointmentId(appointment.getId()).patch(appointment);
graphServiceClient.solutions().bookingBusinesses().byBookingBusinessId(bookingBusinessId)
.appointments().byBookingAppointmentId(appointment.getId()).patch(appointment);
return "Reservation updated at " + restaurant + " from " + date + " to " + updatedDate;
}
@DefineKernelFunction(
name = "cancelReservation",
description = "Cancel a reservation at a restaurant",
returnType = "string"
)
@DefineKernelFunction(name = "cancelReservation", description = "Cancel a reservation at a restaurant", returnType = "string")
public String cancelReservation(
@KernelFunctionParameter(
name = "restaurant",
description = "The name of the restaurant")
String restaurant,
@KernelFunctionParameter(
name = "date",
description = "The date of the reservation in UTC",
type = OffsetDateTime.class)
OffsetDateTime date
) {
@KernelFunctionParameter(name = "restaurant", description = "The name of the restaurant") String restaurant,
@KernelFunctionParameter(name = "date", description = "The date of the reservation in UTC", type = OffsetDateTime.class) OffsetDateTime date) {
BookingAppointment appointment = getAppointment(restaurant, date);
if (appointment == null) {
return "No reservation found for " + restaurant + " on " + date;
}
graphServiceClient.solutions().bookingBusinesses().byBookingBusinessId(bookingBusinessId).appointments().byBookingAppointmentId(appointment.getId()).delete();
graphServiceClient.solutions().bookingBusinesses().byBookingBusinessId(bookingBusinessId)
.appointments().byBookingAppointmentId(appointment.getId()).delete();
return "Reservation cancelled at " + restaurant + " on " + date;
}
java/samples/pom.xml
+1 -1
View File
@@ -21,7 +21,7 @@
</modules>
<properties>
<com.uber.nullaway.version>0.10.10</com.uber.nullaway.version>
<com.uber.nullaway.version>0.10.21</com.uber.nullaway.version>
<google.errorprone.core.version>2.19.1</google.errorprone.core.version>
<log4j2.version>2.20.0</log4j2.version>
<maven.compiler-plugin.version>3.11.0</maven.compiler-plugin.version>
java/samples/sample-code/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example59_OpenAIFunctionCalling.java
+43 -35
View File
@@ -40,7 +40,6 @@ public class Example59_OpenAIFunctionCalling {
private static final String MODEL_ID = System.getenv()
.getOrDefault("MODEL_ID", "gpt-3.5-turbo-1106");
// Define functions that can be called by the model
public static class HelperFunctions {
@@ -53,15 +52,24 @@ public class Example59_OpenAIFunctionCalling {
public String getWeatherForCity(
@KernelFunctionParameter(name = "cityName", description = "Name of the city") String cityName) {
switch (cityName) {
case "Thrapston": return "80 and sunny";
case "Boston": return "61 and rainy";
case "London": return "55 and cloudy";
case "Miami": return "80 and sunny";
case "Paris": return "60 and rainy";
case "Tokyo": return "50 and sunny";
case "Sydney": return "75 and sunny";
case "Tel Aviv": return "80 and sunny";
default: return "31 and snowing";
case "Thrapston":
return "80 and sunny";
case "Boston":
return "61 and rainy";
case "London":
return "55 and cloudy";
case "Miami":
return "80 and sunny";
case "Paris":
return "60 and rainy";
case "Tokyo":
return "50 and sunny";
case "Sydney":
return "75 and sunny";
case "Tel Aviv":
return "80 and sunny";
default:
return "31 and snowing";
}
}
}
@@ -95,7 +103,6 @@ public class Example59_OpenAIFunctionCalling {
.withPlugin(plugin)
.build();
System.out.println("======== Example 1: Use automated function calling ========");
var function = KernelFunctionFromPrompt.builder()
@@ -116,47 +123,49 @@ public class Example59_OpenAIFunctionCalling {
.block();
System.out.println(result.getResult());
System.out.println("======== Example 2: Use manual function calling ========");
var chatHistory = new ChatHistory();
chatHistory.addUserMessage("Given the current time of day and weather, what is the likely color of the sky in Boston?");
chatHistory.addUserMessage(
"Given the current time of day and weather, what is the likely color of the sky in Boston?");
while(true) {
while (true) {
var messages = chat.getChatMessageContentsAsync(
chatHistory,
kernel,
InvocationContext.builder().withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(false)).build()
)
chatHistory,
kernel,
InvocationContext.builder()
.withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(false)).build())
.block();
messages.stream()
.filter(it -> it.getContent() != null)
.forEach(it -> System.out.println(it.getContent()));
.forEach(it -> System.out.println(it.getContent()));
List<OpenAIFunctionToolCall> toolCalls =
messages.stream()
.filter(it -> it instanceof OpenAIChatMessageContent)
.map(it -> (OpenAIChatMessageContent<?>)it)
.map(OpenAIChatMessageContent::getToolCall)
.flatMap(List::stream)
.collect(Collectors.toList());
List<OpenAIFunctionToolCall> toolCalls = messages.stream()
.filter(it -> it instanceof OpenAIChatMessageContent)
.map(it -> (OpenAIChatMessageContent<?>) it)
.map(OpenAIChatMessageContent::getToolCall)
.flatMap(List::stream)
.collect(Collectors.toList());
if (toolCalls.isEmpty()) {
break;
}
messages.stream()
.forEach(it -> chatHistory.addMessage(it));
.forEach(it -> chatHistory.addMessage(it));
for(var toolCall : toolCalls) {
for (var toolCall : toolCalls) {
String content = null;
try {
// getFunction will throw an exception if the function is not found
var fn = kernel.getFunction(toolCall.getPluginName(), toolCall.getFunctionName());
FunctionResult<?> fnResult = fn.invokeAsync(kernel, toolCall.getArguments(), null, null).block();
content = (String)fnResult.getResult();
var fn = kernel.getFunction(toolCall.getPluginName(),
toolCall.getFunctionName());
FunctionResult<?> fnResult = fn
.invokeAsync(kernel, toolCall.getArguments(), null, null).block();
content = (String) fnResult.getResult();
} catch (IllegalArgumentException e) {
content = "Unable to find function. Please try again!";
}
@@ -167,11 +176,10 @@ public class Example59_OpenAIFunctionCalling {
StandardCharsets.UTF_8,
new FunctionResultMetadata(new CaseInsensitiveMap<>() {
{
put(FunctionResultMetadata.ID, ContextVariable.of(toolCall.getId()));
put(FunctionResultMetadata.ID, ContextVariable.of(toolCall.getId()));
}
})
);
}
}));
}
}
}
java/samples/sample-code/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example82_Audio.java
+125
View File
@@ -0,0 +1,125 @@
// Copyright (c) Microsoft. All rights reserved.
package com.microsoft.semantickernel.samples.syntaxexamples;
import com.azure.ai.openai.OpenAIAsyncClient;
import com.azure.ai.openai.OpenAIClientBuilder;
import com.azure.core.credential.AzureKeyCredential;
import com.azure.core.credential.KeyCredential;
import com.azure.core.util.BinaryData;
import com.microsoft.semantickernel.services.audio.AudioContent;
import com.microsoft.semantickernel.services.audio.AudioToTextExecutionSettings;
import com.microsoft.semantickernel.services.audio.AudioToTextService;
import com.microsoft.semantickernel.services.audio.TextToAudioExecutionSettings;
import com.microsoft.semantickernel.services.audio.TextToAudioService;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
public class Example82_Audio {
private static final String CLIENT_KEY = System.getenv("CLIENT_KEY");
private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY");
// Only required if AZURE_CLIENT_KEY is set
private static final String CLIENT_ENDPOINT = System.getenv("CLIENT_ENDPOINT");
private static final String MODEL_ID = System.getenv()
.getOrDefault("MODEL_ID", "gpt-35-turbo");
private static final String TextToAudioModel = "tts-1";
private static final String AudioToTextModel = "whisper-1";
public static void main(String[] args) throws IOException {
System.out.println("======== Example82_Audio ========");
File file = null;
try {
file = Files.createTempFile("audio", ".mp3").toFile();
file.deleteOnExit();
textToAudioAsync(file);
audioToTextAsync(file);
} finally {
if (file != null) {
file.delete();
}
}
}
public static void textToAudioAsync(File audioFile) throws IOException {
OpenAIAsyncClient client;
if (AZURE_CLIENT_KEY != null) {
client = new OpenAIClientBuilder()
.credential(new AzureKeyCredential(AZURE_CLIENT_KEY))
.endpoint(CLIENT_ENDPOINT)
.buildAsyncClient();
} else {
client = new OpenAIClientBuilder()
.credential(new KeyCredential(CLIENT_KEY))
.buildAsyncClient();
}
var textToAudioService = TextToAudioService.builder()
.withModelId(TextToAudioModel)
.withOpenAIAsyncClient(client)
.build();
String sampleText = "Hello, my name is John. I am a software engineer. I am working on a project to convert text to audio.";
// Set execution settings (optional)
TextToAudioExecutionSettings executionSettings = TextToAudioExecutionSettings.builder()
.withVoice("alloy")
.withResponseFormat("mp3")
.withSpeed(1.0)
.build();
// Convert text to audio
AudioContent audioContent = textToAudioService.getAudioContentAsync(
sampleText,
executionSettings)
.block();
// Save audio content to a file
Files.write(audioFile.toPath(), audioContent.getData());
}
public static void audioToTextAsync(File audioFile) {
OpenAIAsyncClient client;
if (AZURE_CLIENT_KEY != null) {
client = new OpenAIClientBuilder()
.credential(new AzureKeyCredential(AZURE_CLIENT_KEY))
.endpoint(CLIENT_ENDPOINT)
.buildAsyncClient();
} else {
client = new OpenAIClientBuilder()
.credential(new KeyCredential(CLIENT_KEY))
.buildAsyncClient();
}
var textToAudioService = AudioToTextService.builder()
.withModelId(AudioToTextModel)
.withOpenAIAsyncClient(client)
.build();
byte[] data = BinaryData.fromFile(audioFile.toPath()).toBytes();
// Set execution settings (optional)
AudioToTextExecutionSettings executionSettings = AudioToTextExecutionSettings.builder()
.withLanguage("en")
.withPrompt("sample prompt")
.withResponseFormat("json")
.withTemperature(0.3)
.withFilename(audioFile.getName())
.build();
String text = textToAudioService.getTextContentsAsync(
AudioContent.builder()
.withData(data)
.build(),
executionSettings).block();
System.out.println(text);
}
}
java/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariable.java
+15 -2
View File
@@ -295,11 +295,13 @@ public class ContextVariable<T> {
* {@code ContextVariable} to a prompt string. This method is useful when the convert of this
* {@code ContextVariableType} does not create the expected prompt string.
*
* @param types the types to use when converting the value
* @param types the types to use when converting the value, if {@code null} the global types
* are used
* @param converter the converter to use when converting the value
* @return the value of this {@code ContextVariable} as a prompt string
*/
public String toPromptString(ContextVariableTypes types,
public String toPromptString(
@Nullable ContextVariableTypes types,
ContextVariableTypeConverter<T> converter) {
return converter.toPromptString(types, value);
}
@@ -316,6 +318,17 @@ public class ContextVariable<T> {
return toPromptString(types, type.getConverter());
}
/**
* Use the given {@code ContextVariableTypeConverter} to convert the value of this
* {@code ContextVariable} to a prompt string. This method is useful when the convert of this
* {@code ContextVariableType} does not create the expected prompt string.
*
* @return the value of this {@code ContextVariable} as a prompt string
*/
public String toPromptString() {
return toPromptString(null, type.getConverter());
}
/**
* Returns true if the value of this {@code ContextVariable} is {@code null} or empty.
*
java/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java
+5 -1
View File
@@ -170,13 +170,17 @@ public class ContextVariableTypeConverter<T> {
* Convert the type to a prompt string using the {@code toPromptString} function provided to the
* constructor.
*
* @param types the context variable types, if {@code null} the global types are used
* @param t the type to convert
* @return the prompt string
*/
public String toPromptString(ContextVariableTypes types, @Nullable T t) {
public String toPromptString(@Nullable ContextVariableTypes types, @Nullable T t) {
if (t == null) {
return "";
}
if (types == null) {
types = ContextVariableTypes.getGlobalTypes();
}
return toPromptString.toPromptString(types, t);
}
java/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypes.java
+4
View File
@@ -94,6 +94,10 @@ public class ContextVariableTypes {
variableTypes = new HashMap<>();
}
public static ContextVariableTypes getGlobalTypes() {
return new ContextVariableTypes(DEFAULT_TYPES);
}
/**
* Create a new collection of context variable types.
*
java/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PreToolCallEvent.java
+4 -1
View File
@@ -5,6 +5,7 @@ import com.microsoft.semantickernel.contextvariables.ContextVariableTypes;
import com.microsoft.semantickernel.semanticfunctions.KernelFunction;
import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import javax.annotation.Nullable;
/**
* Represents a KernelHookEvent that is raised before a tool call is invoked.
@@ -13,6 +14,7 @@ public class PreToolCallEvent implements KernelHookEvent {
private final ContextVariableTypes contextVariableTypes;
private final String functionName;
@Nullable
private final KernelFunctionArguments arguments;
private final KernelFunction<?> function;
@@ -27,7 +29,7 @@ public class PreToolCallEvent implements KernelHookEvent {
@SuppressFBWarnings("EI_EXPOSE_REP2")
public PreToolCallEvent(
String functionName,
KernelFunctionArguments arguments,
@Nullable KernelFunctionArguments arguments,
KernelFunction<?> function,
ContextVariableTypes contextVariableTypes) {
this.functionName = functionName;
@@ -37,6 +39,7 @@ public class PreToolCallEvent implements KernelHookEvent {
}
@SuppressFBWarnings("EI_EXPOSE_REP")
@Nullable
public KernelFunctionArguments getArguments() {
return arguments;
}
java/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionArguments.java
+2 -1
View File
@@ -6,6 +6,7 @@ import com.microsoft.semantickernel.contextvariables.CaseInsensitiveMap;
import com.microsoft.semantickernel.contextvariables.ContextVariable;
import com.microsoft.semantickernel.contextvariables.ContextVariableType;
import com.microsoft.semantickernel.contextvariables.ContextVariableTypeConverter;
import com.microsoft.semantickernel.contextvariables.ContextVariableTypes;
import com.microsoft.semantickernel.exceptions.SKException;
import java.util.Collection;
import java.util.HashMap;
@@ -89,7 +90,7 @@ public class KernelFunctionArguments implements Map<String, ContextVariable<?>>
+ System.lineSeparator()
+ entry.getKey()
+ ": "
+ entry.getValue(),
+ entry.getValue().toPromptString(ContextVariableTypes.getGlobalTypes()),
(a, b) -> a + b);
}
java/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/AudioContent.java
+126
View File
@@ -0,0 +1,126 @@
// Copyright (c) Microsoft. All rights reserved.
package com.microsoft.semantickernel.services.audio;
import com.microsoft.semantickernel.exceptions.SKException;
import java.util.Arrays;
import java.util.Map;
import javax.annotation.Nullable;
/**
* Represents audio content.
*/
public class AudioContent {
private final byte[] data;
@Nullable
private final String modelId;
/**
* Creates an instance of audio content.
*
* @param data The audio data.
* @param modelId The model ID.
*/
public AudioContent(byte[] data, @Nullable String modelId) {
this.data = Arrays.copyOf(data, data.length);
this.modelId = modelId;
}
/**
* Gets the audio data.
*
* @return The audio data.
*/
public byte[] getData() {
return Arrays.copyOf(data, data.length);
}
/**
* Gets the model ID.
*
* @return The model ID.
*/
@Nullable
public String getModelId() {
return modelId;
}
/**
* Gets the inner content.
*
* @return The inner content.
*/
@Nullable
public String getInnerContent() {
return null;
}
/**
* Gets the metadata.
*
* @return The metadata.
*/
@Nullable
public Map<String, Object> getMetadata() {
return null;
}
/**
* Creates a new builder.
*
* @return The builder.
*/
public static Builder builder() {
return new Builder();
}
/**
* Represents a builder for audio content.
*/
public static class Builder {
@Nullable
private byte[] data = null;
@Nullable
private String modelId = null;
/**
* Sets the audio data.
*
* @param data The audio data.
* @return The builder.
*/
public Builder withData(byte[] data) {
this.data = Arrays.copyOf(data, data.length);
return this;
}
/**
* Sets the model ID.
*
* @param modelId The model ID.
* @return The builder.
*/
public Builder withModelId(String modelId) {
this.modelId = modelId;
return this;
}
/**
* Builds the audio content.
*
* @return The audio content.
*/
public AudioContent build() {
if (data == null) {
throw new SKException("Data is required");
}
if (modelId == null) {
throw new SKException("Model ID is required");
}
return new AudioContent(data, modelId);
}
}
}
java/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/AudioToTextExecutionSettings.java
+233
View File
@@ -0,0 +1,233 @@
// Copyright (c) Microsoft. All rights reserved.
package com.microsoft.semantickernel.services.audio;
import javax.annotation.Nullable;
/**
* Represents audio content.
*/
public class AudioToTextExecutionSettings {
@Nullable
private final String deploymentName;
@Nullable
private final String filename;
private final String responseFormat;
@Nullable
private final String language;
@Nullable
private final String prompt;
@Nullable
private final Double temperature;
/**
* Creates an instance of audio to text execution settings.
*
* @param deploymentName The deployment name.
* @param filename The filename.
* @param responseFormat The response format.
* @param language The language.
* @param prompt The prompt.
* @param temperature The temperature.
*/
public AudioToTextExecutionSettings(
@Nullable String deploymentName,
@Nullable String filename,
String responseFormat,
@Nullable String language,
@Nullable String prompt,
@Nullable Double temperature) {
this.deploymentName = deploymentName;
this.filename = filename;
this.responseFormat = responseFormat;
this.language = language;
this.prompt = prompt;
this.temperature = temperature;
}
/**
* The deployment name to use for audio transcription.
* <p>
* When making a request against Azure OpenAI, this should be the customizable name of the
* "model deployment" (example: my-gpt4-deployment) and not the name of the model itself
* (example: gpt-4).
* <p>
* When using non-Azure OpenAI, this corresponds to "model" in the request options and should
* use the appropriate name of the model (example: gpt-4).
*
* @return The deployment name.
*/
@Nullable
public String getDeploymentName() {
return deploymentName;
}
/**
* The optional filename or descriptive identifier to associate with the audio data.
*
* @return The filename or descriptive identifier.
*/
@Nullable
public String getFilename() {
return filename;
}
/**
* The requested format of the transcription response data, which will influence the content and
* detail of the result.
*
* @return The response format.
*/
public String getResponseFormat() {
return responseFormat;
}
/**
* The language of the audio data as two-letter ISO-639-1 language code (e.g. 'en' or 'es').
*
* @return The language of the audio data.
*/
@Nullable
public String getLanguage() {
return language;
}
/**
* An optional hint to guide the model's style or continue from a prior audio segment. The
* written language of the prompt should match the primary spoken language of the audio data.
*
* @return The prompt.
*/
@Nullable
public String getPrompt() {
return prompt;
}
/**
* The randomness of the generated text. Select a value from 0.0 to 1.0. 0 is the default.
*
* @return The temperature.
*/
@Nullable
public Double getTemperature() {
return temperature;
}
/**
* Creates a new builder.
*
* @return The builder.
*/
public static Builder builder() {
return new Builder();
}
/**
* Represents a builder for audio to text execution settings.
*/
public static class Builder {
@Nullable
private String deploymentName = null;
@Nullable
private String filename = null;
@Nullable
private String responseFormat = null;
@Nullable
private String language = null;
@Nullable
private String prompt = null;
@Nullable
private Double temperature = null;
/**
* Sets the deployment name to use for audio transcription.
*
* @param deploymentName The deployment name.
* @return The builder.
*/
public Builder withDeploymentName(String deploymentName) {
this.deploymentName = deploymentName;
return this;
}
/**
* Sets the filename or descriptive identifier to associate with the audio data.
*
* @param filename The filename or descriptive identifier.
* @return The builder.
*/
public Builder withFilename(String filename) {
this.filename = filename;
return this;
}
/**
* The requested format of the transcription response data, which will influence the content
* and detail of the result.
*
* @param responseFormat The response format. Supported formats are json, text, srt,
* verbose_json, or vtt. Default is 'json'.
* @return The builder.
*/
public Builder withResponseFormat(String responseFormat) {
this.responseFormat = responseFormat;
return this;
}
/**
* The language of the audio data as two-letter ISO-639-1 language code (e.g. 'en' or
* 'es').
*
* @param language The language of the audio data.
* @return The builder.
*/
public Builder withLanguage(String language) {
this.language = language;
return this;
}
/**
* An optional hint to guide the model's style or continue from a prior audio segment. The
* written language of the prompt should match the primary spoken language of the audio
* data.
*
* @param prompt The prompt.
* @return The builder.
*/
public Builder withPrompt(String prompt) {
this.prompt = prompt;
return this;
}
/**
* The randomness of the generated text. Select a value from 0.0 to 1.0. 0 is the default.
*
* @param temperature The temperature.
* @return The builder.
*/
public Builder withTemperature(Double temperature) {
this.temperature = temperature;
return this;
}
/**
* Builds the audio to text execution settings.
*
* @return The audio to text execution settings.
*/
public AudioToTextExecutionSettings build() {
if (responseFormat == null) {
responseFormat = "json";
}
return new AudioToTextExecutionSettings(deploymentName, filename,
responseFormat, language, prompt, temperature);
}
}
}
java/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/AudioToTextService.java
+38
View File
@@ -0,0 +1,38 @@
// Copyright (c) Microsoft. All rights reserved.
package com.microsoft.semantickernel.services.audio;
import com.microsoft.semantickernel.implementation.ServiceLoadUtil;
import com.microsoft.semantickernel.services.AIService;
import com.microsoft.semantickernel.services.openai.OpenAiServiceBuilder;
import javax.annotation.Nullable;
import reactor.core.publisher.Mono;
/**
* Provides audio to text service.
*/
public interface AudioToTextService extends AIService {
/**
* Get text contents from audio content.
*
* @param content Audio content.
* @param executionSettings The AI execution settings (optional).
* @return Text contents from audio content.
*/
Mono<String> getTextContentsAsync(
AudioContent content,
@Nullable AudioToTextExecutionSettings executionSettings);
static Builder builder() {
return ServiceLoadUtil.findServiceLoader(Builder.class,
"com.microsoft.semantickernel.aiservices.openai.audio.OpenAiAudioToTextService$Builder")
.get();
}
/**
* Builder for the AudioToTextService.
*/
abstract class Builder extends OpenAiServiceBuilder<AudioToTextService, Builder> {
}
}
java/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/TextToAudioExecutionSettings.java
+131
View File
@@ -0,0 +1,131 @@
// Copyright (c) Microsoft. All rights reserved.
package com.microsoft.semantickernel.services.audio;
import com.microsoft.semantickernel.exceptions.SKException;
import edu.umd.cs.findbugs.annotations.Nullable;
/**
* Represents the settings for text to audio execution.
*/
public class TextToAudioExecutionSettings {
private final String voice;
private final String responseFormat;
@Nullable
private final Double speed;
/**
* Creates a new instance of the settings.
*
* @param voice The voice.
* @param responseFormat The response format.
* @param speed The speed.
*/
public TextToAudioExecutionSettings(
String voice,
String responseFormat,
@Nullable Double speed) {
this.voice = voice;
this.responseFormat = responseFormat;
this.speed = speed;
}
/**
* Gets the voice.
*
* @return The voice.
*/
public String getVoice() {
return voice;
}
/**
* Gets the response format.
*
* @return The response format.
*/
public String getResponseFormat() {
return responseFormat;
}
/**
* Gets the speed.
*
* @return The speed.
*/
@Nullable
public Double getSpeed() {
return speed;
}
/**
* Creates a new builder.
*
* @return The builder.
*/
public static TextToAudioExecutionSettings.Builder builder() {
return new Builder();
}
/**
* Represents a builder for text to audio execution settings.
*/
public static class Builder {
@Nullable
private String voice = null;
@Nullable
private String responseFormat = null;
@Nullable
private Double speed = null;
/**
* Sets the voice to use for the audio generation.
*
* @param voice The voice.
* @return The builder.
*/
public Builder withVoice(String voice) {
this.voice = voice;
return this;
}
/**
* Sets the response format, e.g "mp3", "opus", "aac", "flak"
*
* @param responseFormat The response format.
* @return The builder.
*/
public Builder withResponseFormat(String responseFormat) {
this.responseFormat = responseFormat;
return this;
}
/**
* Sets the speed of the audio generation.
*
* @param speed The speed.
* @return The builder.
*/
public Builder withSpeed(Double speed) {
this.speed = speed;
return this;
}
/**
* Builds the settings.
*
* @return The settings.
*/
public TextToAudioExecutionSettings build() {
if (voice == null) {
throw new SKException("Voice must be set");
}
if (responseFormat == null) {
throw new SKException("Response format must be set");
}
return new TextToAudioExecutionSettings(voice, responseFormat, speed);
}
}
}
java/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/TextToAudioService.java
+42
View File
@@ -0,0 +1,42 @@
// Copyright (c) Microsoft. All rights reserved.
package com.microsoft.semantickernel.services.audio;
import com.microsoft.semantickernel.implementation.ServiceLoadUtil;
import com.microsoft.semantickernel.services.AIService;
import com.microsoft.semantickernel.services.openai.OpenAiServiceBuilder;
import reactor.core.publisher.Mono;
/**
* Provides audio to text service.
*/
public interface TextToAudioService extends AIService {
/**
* Get audio content from text.
*
* @param sampleText The sample text.
* @param executionSettings The AI execution settings.
* @return Audio content from text.
*/
Mono<AudioContent> getAudioContentAsync(String sampleText,
TextToAudioExecutionSettings executionSettings);
/**
* Gets the builder for the TextToAudioService.
*
* @return The builder.
*/
static Builder builder() {
return ServiceLoadUtil.findServiceLoader(Builder.class,
"com.microsoft.semantickernel.aiservices.openai.audio.OpenAiTextToAudioService$Builder")
.get();
}
/**
* Builder for the TextToAudioService.
*/
abstract class Builder extends
OpenAiServiceBuilder<TextToAudioService, Builder> {
}
}
java/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatCompletionService.java
+2 -45
View File
@@ -1,13 +1,12 @@
// Copyright (c) Microsoft. All rights reserved.
package com.microsoft.semantickernel.services.chatcompletion;
import com.azure.ai.openai.OpenAIAsyncClient;
import com.azure.ai.openai.OpenAIClient;
import com.microsoft.semantickernel.Kernel;
import com.microsoft.semantickernel.builders.SemanticKernelBuilder;
import com.microsoft.semantickernel.implementation.ServiceLoadUtil;
import com.microsoft.semantickernel.orchestration.InvocationContext;
import com.microsoft.semantickernel.services.TextAIService;
import com.microsoft.semantickernel.services.openai.OpenAiServiceBuilder;
import java.util.List;
import javax.annotation.Nullable;
import reactor.core.publisher.Mono;
@@ -62,50 +61,8 @@ public interface ChatCompletionService extends TextAIService {
* {@link ChatCompletionService} must implement the {@link SemanticKernelBuilder#build()}
* method.
*/
abstract class Builder implements SemanticKernelBuilder<ChatCompletionService> {
abstract class Builder extends OpenAiServiceBuilder<ChatCompletionService, Builder> {
@Nullable
protected OpenAIAsyncClient client;
@Nullable
protected String modelId;
@Nullable
protected String serviceId;
/**
* Sets the model ID to use with the ChatCompletion service.
*
* @param modelId the model ID to use with the ChatCompletion service
* @return this builder
*/
public Builder withModelId(String modelId) {
this.modelId = modelId;
return this;
}
/**
* Sets the {@link OpenAIClient} to use for communication with the ChatCompletion service.
*
* @param openAIClient the {@link OpenAIClient} to use for communication with the
* ChatCompletion service
* @return this builder
*/
public Builder withOpenAIAsyncClient(OpenAIAsyncClient openAIClient) {
this.client = openAIClient;
return this;
}
/**
* Sets the service ID to use with the ChatCompletion service.
*
* @param serviceId the service ID to use with the ChatCompletion service
* @return this builder
*/
public Builder withServiceId(String serviceId) {
this.serviceId = serviceId;
return this;
}
}
}
java/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/openai/OpenAiServiceBuilder.java
+57
View File
@@ -0,0 +1,57 @@
// Copyright (c) Microsoft. All rights reserved.
package com.microsoft.semantickernel.services.openai;
import com.azure.ai.openai.OpenAIAsyncClient;
import com.microsoft.semantickernel.builders.SemanticKernelBuilder;
import javax.annotation.Nullable;
/**
* Builder for an OpenAI service.
*/
public abstract class OpenAiServiceBuilder<T, U extends OpenAiServiceBuilder<T, U>> implements
SemanticKernelBuilder<T> {
@Nullable
protected String modelId;
@Nullable
protected OpenAIAsyncClient client;
@Nullable
protected String serviceId;
/**
* Sets the model ID for the service
*
* @param modelId The model ID
* @return The builder
*/
public U withModelId(String modelId) {
this.modelId = modelId;
return (U) this;
}
/**
* Sets the OpenAI client for the service
*
* @param client The OpenAI client
* @return The builder
*/
public U withOpenAIAsyncClient(OpenAIAsyncClient client) {
this.client = client;
return (U) this;
}
/**
* Sets the service ID for the service
*
* @param serviceId The service ID
* @return The builder
*/
public U withServiceId(String serviceId) {
this.serviceId = serviceId;
return (U) this;
}
@Override
public abstract T build();
}
java/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/TextGenerationService.java
+2 -47
View File
@@ -1,12 +1,11 @@
// Copyright (c) Microsoft. All rights reserved.
package com.microsoft.semantickernel.services.textcompletion;
import com.azure.ai.openai.OpenAIAsyncClient;
import com.microsoft.semantickernel.Kernel;
import com.microsoft.semantickernel.builders.SemanticKernelBuilder;
import com.microsoft.semantickernel.implementation.ServiceLoadUtil;
import com.microsoft.semantickernel.orchestration.PromptExecutionSettings;
import com.microsoft.semantickernel.services.TextAIService;
import com.microsoft.semantickernel.services.openai.OpenAiServiceBuilder;
import java.util.List;
import javax.annotation.Nullable;
import reactor.core.publisher.Flux;
@@ -61,50 +60,6 @@ public interface TextGenerationService extends TextAIService {
/**
* Builder for a TextGenerationService
*/
abstract class Builder implements SemanticKernelBuilder<TextGenerationService> {
@Nullable
protected String modelId;
@Nullable
protected OpenAIAsyncClient client;
@Nullable
protected String serviceId;
/**
* Sets the model ID for the service
*
* @param modelId The model ID
* @return The builder
*/
public Builder withModelId(String modelId) {
this.modelId = modelId;
return this;
}
/**
* Sets the OpenAI client for the service
*
* @param client The OpenAI client
* @return The builder
*/
public Builder withOpenAIAsyncClient(OpenAIAsyncClient client) {
this.client = client;
return this;
}
/**
* Sets the service ID for the service
*
* @param serviceId The service ID
* @return The builder
*/
public Builder withServiceId(String serviceId) {
this.serviceId = serviceId;
return this;
}
@Override
public abstract TextGenerationService build();
abstract class Builder extends OpenAiServiceBuilder<TextGenerationService, Builder> {
}
}
java/semantickernel-bom/pom.xml
+1 -1
View File
@@ -82,7 +82,7 @@
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-ai-openai</artifactId>
<version>1.0.0-beta.7</version>
<version>1.0.0-beta.8</version>
</dependency>
<dependency>
<groupId>com.azure</groupId>