mirror of
https://github.com/microsoft/semantic-kernel.git
synced 2026-10-12 02:19:06 +00:00
Dmytro Struk
1490eb6d02
### Motivation and Context <!-- Thank you for your contribution to the semantic-kernel repo! Please help reviewers and future users, providing the following information: 1. Why is this change required? 2. What problem does it solve? 3. What scenario does it contribute to? 4. If it fixes an open issue, please link to the issue here. --> This PR contains new interface for audio-to-text services and implementation of OpenAI connector to use Azure/OpenAI Whisper model. The implementation relies on Azure .NET SDK. ### Description <!-- Describe your changes, the overall approach, the underlying design. These notes will help understanding how your code works. Thanks! --> 1. Added new `IAudioToTextService` interface. 2. Added new `AzureOpenAIAudioToTextService` and `OpenAIAudioToTextService` implementations. 3. Added new `OpenAIAudioToTextExecutionSettings` to define audio-to-text configuration (e.g. filename, language etc.). 4. Added unit and integration tests. ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone 😄
96 lines
5.0 KiB
C#
96 lines
5.0 KiB
C#
// Copyright (c) Microsoft. All rights reserved.
|
|
|
|
using System.Collections.Generic;
|
|
using System.Diagnostics.CodeAnalysis;
|
|
using System.Net.Http;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using Azure.AI.OpenAI;
|
|
using Azure.Core;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.SemanticKernel.AudioToText;
|
|
using Microsoft.SemanticKernel.Contents;
|
|
using Microsoft.SemanticKernel.Services;
|
|
|
|
namespace Microsoft.SemanticKernel.Connectors.OpenAI;
|
|
|
|
/// <summary>
|
|
/// Azure OpenAI audio-to-text service.
|
|
/// </summary>
|
|
[Experimental("SKEXP0005")]
|
|
public sealed class AzureOpenAIAudioToTextService : IAudioToTextService
|
|
{
|
|
/// <summary>Core implementation shared by Azure OpenAI services.</summary>
|
|
private readonly AzureOpenAIClientCore _core;
|
|
|
|
/// <inheritdoc/>
|
|
public IReadOnlyDictionary<string, object?> Attributes => this._core.Attributes;
|
|
|
|
/// <summary>
|
|
/// Creates an instance of the <see cref="AzureOpenAIAudioToTextService"/> with API key auth.
|
|
/// </summary>
|
|
/// <param name="deploymentName">Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource</param>
|
|
/// <param name="endpoint">Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart</param>
|
|
/// <param name="apiKey">Azure OpenAI API key, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart</param>
|
|
/// <param name="modelId">Azure OpenAI model id, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource</param>
|
|
/// <param name="httpClient">Custom <see cref="HttpClient"/> for HTTP requests.</param>
|
|
/// <param name="loggerFactory">The <see cref="ILoggerFactory"/> to use for logging. If null, no logging will be performed.</param>
|
|
public AzureOpenAIAudioToTextService(
|
|
string deploymentName,
|
|
string endpoint,
|
|
string apiKey,
|
|
string? modelId = null,
|
|
HttpClient? httpClient = null,
|
|
ILoggerFactory? loggerFactory = null)
|
|
{
|
|
this._core = new(deploymentName, endpoint, apiKey, httpClient, loggerFactory?.CreateLogger(typeof(AzureOpenAIAudioToTextService)));
|
|
this._core.AddAttribute(AIServiceExtensions.ModelIdKey, modelId);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates an instance of the <see cref="AzureOpenAIAudioToTextService"/> with AAD auth.
|
|
/// </summary>
|
|
/// <param name="deploymentName">Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource</param>
|
|
/// <param name="endpoint">Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart</param>
|
|
/// <param name="credentials">Token credentials, e.g. DefaultAzureCredential, ManagedIdentityCredential, EnvironmentCredential, etc.</param>
|
|
/// <param name="modelId">Azure OpenAI model id, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource</param>
|
|
/// <param name="httpClient">Custom <see cref="HttpClient"/> for HTTP requests.</param>
|
|
/// <param name="loggerFactory">The <see cref="ILoggerFactory"/> to use for logging. If null, no logging will be performed.</param>
|
|
public AzureOpenAIAudioToTextService(
|
|
string deploymentName,
|
|
string endpoint,
|
|
TokenCredential credentials,
|
|
string? modelId = null,
|
|
HttpClient? httpClient = null,
|
|
ILoggerFactory? loggerFactory = null)
|
|
{
|
|
this._core = new(deploymentName, endpoint, credentials, httpClient, loggerFactory?.CreateLogger(typeof(AzureOpenAIAudioToTextService)));
|
|
this._core.AddAttribute(AIServiceExtensions.ModelIdKey, modelId);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates an instance of the <see cref="AzureOpenAIAudioToTextService"/> using the specified <see cref="OpenAIClient"/>.
|
|
/// </summary>
|
|
/// <param name="deploymentName">Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource</param>
|
|
/// <param name="openAIClient">Custom <see cref="OpenAIClient"/>.</param>
|
|
/// <param name="modelId">Azure OpenAI model id, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource</param>
|
|
/// <param name="loggerFactory">The <see cref="ILoggerFactory"/> to use for logging. If null, no logging will be performed.</param>
|
|
public AzureOpenAIAudioToTextService(
|
|
string deploymentName,
|
|
OpenAIClient openAIClient,
|
|
string? modelId = null,
|
|
ILoggerFactory? loggerFactory = null)
|
|
{
|
|
this._core = new(deploymentName, openAIClient, loggerFactory?.CreateLogger(typeof(AzureOpenAIAudioToTextService)));
|
|
this._core.AddAttribute(AIServiceExtensions.ModelIdKey, modelId);
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public Task<TextContent> GetTextContentAsync(
|
|
AudioContent content,
|
|
PromptExecutionSettings? executionSettings = null,
|
|
Kernel? kernel = null,
|
|
CancellationToken cancellationToken = default)
|
|
=> this._core.GetTextContentFromAudioAsync(content, executionSettings, cancellationToken);
|
|
}
|