mirror of
https://github.com/microsoft/semantic-kernel.git
synced 2026-10-12 02:19:06 +00:00
01afd3f128
### Description Updating KernelSyntaxExamples to use the same environment variable keys as the Integration tests. This is a step toward making these tests simpler to run, and even running them as automation in CI tests. This is a first pass -- more cleanup to be done. ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [X] The code builds clean without any errors or warnings - [X] The PR follows SK Contribution Guidelines (https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) - [X] The code follows the .NET coding conventions (https://learn.microsoft.com/dotnet/csharp/fundamentals/coding-style/coding-conventions) verified with `dotnet format` - [X] All unit tests pass, and I have added new tests where possible - [X] I didn't break anyone 😄 --------- Co-authored-by: name <email> Co-authored-by: Dmytro Struk <[email protected]>
93 lines
2.7 KiB
C#
93 lines
2.7 KiB
C#
// Copyright (c) Microsoft. All rights reserved.
|
|
|
|
using System;
|
|
using System.Threading.Tasks;
|
|
using Microsoft.SemanticKernel.Connectors.AI.OpenAI.Tokenizers;
|
|
|
|
// ReSharper disable once InconsistentNaming
|
|
/// <summary>
|
|
/// This sample shows how to count tokens using GPT tokenizer. The number of tokens affects
|
|
/// API calls cost and each models has a maximum amount of tokens it can process and generate.
|
|
/// This example is specific to OpenAI models, which use the tokenization described here:
|
|
/// https://platform.openai.com/tokenizer
|
|
/// If you use Semantic Kernel with other models, the tokenization logic is most probably different,
|
|
/// and you should not use the GPT tokenizer.
|
|
/// </summary>
|
|
public static class Example29_Tokenizer
|
|
{
|
|
public static Task RunAsync()
|
|
{
|
|
// Example 1
|
|
string sentence = "Some text on one line";
|
|
int tokenCount = GPT3Tokenizer.Encode(sentence).Count;
|
|
|
|
Console.WriteLine("---");
|
|
Console.WriteLine(sentence);
|
|
Console.WriteLine("Tokens: " + tokenCount);
|
|
Console.WriteLine("---\n\n");
|
|
|
|
// Example 2
|
|
sentence = "⭐⭐";
|
|
tokenCount = GPT3Tokenizer.Encode(sentence).Count;
|
|
|
|
Console.WriteLine("The following example contains emojis which require several tokens.");
|
|
Console.WriteLine("---");
|
|
Console.WriteLine(sentence);
|
|
Console.WriteLine("Tokens: " + tokenCount);
|
|
Console.WriteLine("---\n\n");
|
|
|
|
// Example 3
|
|
sentence = "Some text on\ntwo lines";
|
|
tokenCount = GPT3Tokenizer.Encode(sentence).Count;
|
|
|
|
Console.WriteLine("The following example uses Unix '\\n' line separator.");
|
|
Console.WriteLine("---");
|
|
Console.WriteLine(sentence);
|
|
Console.WriteLine("Tokens: " + tokenCount);
|
|
Console.WriteLine("---\n\n");
|
|
|
|
// Example 4
|
|
sentence = "Some text on\r\ntwo lines";
|
|
tokenCount = GPT3Tokenizer.Encode(sentence).Count;
|
|
|
|
Console.WriteLine("The following example uses Windows '\\r\\n' line separator.");
|
|
Console.WriteLine("---");
|
|
Console.WriteLine(sentence);
|
|
Console.WriteLine("Tokens: " + tokenCount);
|
|
Console.WriteLine("---\n\n");
|
|
|
|
/*
|
|
Output:
|
|
---
|
|
Some text on one line
|
|
Tokens: 5
|
|
---
|
|
|
|
|
|
The following example contains emojis which require several tokens.
|
|
---
|
|
⭐⭐
|
|
Tokens: 6
|
|
---
|
|
|
|
|
|
The following example uses Unix '\n' line separator.
|
|
---
|
|
Some text on
|
|
two lines
|
|
Tokens: 6
|
|
---
|
|
|
|
|
|
The following example uses Windows '\r\n' line separator.
|
|
---
|
|
Some text on
|
|
two lines
|
|
Tokens: 7
|
|
---
|
|
*/
|
|
|
|
return Task.CompletedTask;
|
|
}
|
|
}
|