mirror of
https://github.com/microsoft/semantic-kernel.git
synced 2026-10-12 02:19:06 +00:00
Redis connector for Semantic Memory (#1178)
### Motivation and Context Redis supports cosine similarity search by the [RediSearch](https://redis.io/docs/stack/search) module. ### Description Add a new Semantic Memory Connector leveraging Redis with RediSearch module. Update the syntax example to demonstrate how it is used. Co-authored-by: Dmytro Struk <[email protected]>
This commit is contained in:
@@ -68,6 +68,7 @@
|
||||
| Sqlite | ✅ | ❌ | ❌ | Vector optimization requires [sqlite-vss](https://github.com/asg017/sqlite-vss) |
|
||||
| Postgres | ✅ | ❌ | ❌ | Vector optimization requires [pgvector](https://github.com/pgvector/pgvector) |
|
||||
| CosmosDB | ✅ | ❌ | ❌ | CosmosDB is not optimized for vector storage |
|
||||
| Redis | ✅ | ❌ | ❌ | Vector optimization requires [RediSearch](https://redis.io/docs/stack/search) |
|
||||
|
||||
## Connectors and Skill Libraries
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
<PackageVersion Include="Azure.Search.Documents" Version="11.5.0-beta.2" />
|
||||
<PackageVersion Include="Microsoft.Bcl.HashCode" Version="[1.1.0, )" />
|
||||
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="[7.0.0, )" />
|
||||
<PackageVersion Include="NRedisStack" Version="0.6.1" />
|
||||
<PackageVersion Include="Pgvector" Version="0.1.3" />
|
||||
<PackageVersion Include="System.Linq.Async" Version="6.0.1" />
|
||||
<PackageVersion Include="System.Text.Json" Version="[7.0.2, )" />
|
||||
|
||||
@@ -72,6 +72,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.CosmosDB"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.Postgres", "src\Connectors\Connectors.Memory.Postgres\Connectors.Memory.Postgres.csproj", "{C9F957FA-A70F-4A6D-8F95-23FCD7F4FB87}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.Redis", "src\Connectors\Connectors.Memory.Redis\Connectors.Memory.Redis.csproj", "{3720F5ED-FB4D-485E-8A93-CDE60DEF0805}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.AI.OpenAI", "src\Connectors\Connectors.AI.OpenAI\Connectors.AI.OpenAI.csproj", "{AFA81EB7-F869-467D-8A90-744305D80AAC}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SemanticKernel.Abstractions", "src\SemanticKernel.Abstractions\SemanticKernel.Abstractions.csproj", "{627742DB-1E52-468A-99BD-6FF1A542D25B}"
|
||||
@@ -304,6 +306,12 @@ Global
|
||||
{4D91A3E0-C404-495B-AD4A-411C4E83CF54}.Publish|Any CPU.Build.0 = Release|Any CPU
|
||||
{4D91A3E0-C404-495B-AD4A-411C4E83CF54}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{4D91A3E0-C404-495B-AD4A-411C4E83CF54}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{3720F5ED-FB4D-485E-8A93-CDE60DEF0805}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{3720F5ED-FB4D-485E-8A93-CDE60DEF0805}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{3720F5ED-FB4D-485E-8A93-CDE60DEF0805}.Publish|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{3720F5ED-FB4D-485E-8A93-CDE60DEF0805}.Publish|Any CPU.Build.0 = Debug|Any CPU
|
||||
{3720F5ED-FB4D-485E-8A93-CDE60DEF0805}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{3720F5ED-FB4D-485E-8A93-CDE60DEF0805}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{50FAE231-6F24-4779-9D02-12ABBC9A49E2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{50FAE231-6F24-4779-9D02-12ABBC9A49E2}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{50FAE231-6F24-4779-9D02-12ABBC9A49E2}.Publish|Any CPU.ActiveCfg = Publish|Any CPU
|
||||
@@ -353,6 +361,7 @@ Global
|
||||
{4D3DAE63-41C6-4E1C-A35A-E77BDFC40675} = {831DDCA2-7D2C-4C31-80DB-6BDB3E1F7AE0}
|
||||
{6AAB0620-33A1-4A98-A63B-6560B9BA47A4} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C}
|
||||
{4D91A3E0-C404-495B-AD4A-411C4E83CF54} = {FA3720F1-C99A-49B2-9577-A940257098BF}
|
||||
{3720F5ED-FB4D-485E-8A93-CDE60DEF0805} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C}
|
||||
{50FAE231-6F24-4779-9D02-12ABBC9A49E2} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C}
|
||||
EndGlobalSection
|
||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<!-- THIS PROPERTY GROUP MUST COME FIRST -->
|
||||
<AssemblyName>Microsoft.SemanticKernel.Connectors.Memory.Redis</AssemblyName>
|
||||
<RootNamespace>$(AssemblyName)</RootNamespace>
|
||||
<TargetFramework>netstandard2.0</TargetFramework>
|
||||
</PropertyGroup>
|
||||
|
||||
<!-- IMPORT NUGET PACKAGE SHARED PROPERTIES -->
|
||||
<Import Project="$(RepoRoot)/dotnet/nuget/nuget-package.props" />
|
||||
<Import Project="$(RepoRoot)/dotnet/src/InternalUtilities/InternalUtilities.props" />
|
||||
|
||||
<PropertyGroup>
|
||||
<!-- NuGet Package Settings -->
|
||||
<Title>Semantic Kernel - Redis Connector</Title>
|
||||
<Description>Redis connector for Semantic Kernel skills and semantic memory</Description>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="NRedisStack" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\SemanticKernel\SemanticKernel.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,32 @@
|
||||
# Microsoft.SemanticKernel.Connectors.Memory.Redis
|
||||
|
||||
This connector uses Redis to implement Semantic Memory. It requires the [RediSearch](https://redis.io/docs/stack/search) module to be enabled on Redis to implement vector similarity search.
|
||||
|
||||
## What is RediSearch?
|
||||
|
||||
[RediSearch](https://redis.io/docs/stack/search) is a source-available Redis module that enables querying, secondary indexing, and full-text search for Redis. These features enable multi-field queries, aggregation, exact phrase matching, numeric filtering, geo filtering and vector similarity semantic search on top of text queries.
|
||||
|
||||
How to set up the RediSearch, please refer to its [documentation](https://redis.io/docs/stack/search/quick_start/).
|
||||
Or use the [Redis Enterprise](https://redis.io/docs/about/redis-enterprise/), see [Azure Marketplace](https://azuremarketplace.microsoft.com/en-us/marketplace/apps/garantiadata.redis_enterprise_1sp_public_preview?tab=Overview), [AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-e6y7ork67pjwg?sr=0-2&ref_=beagle&applicationId=AWSMPContessa), or [Google Marketplace](https://console.cloud.google.com/marketplace/details/redislabs-public/redis-enterprise?pli=1).
|
||||
|
||||
## Quick start
|
||||
|
||||
1. Run with Docker:
|
||||
|
||||
```bash
|
||||
docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest
|
||||
```
|
||||
|
||||
2. To use Redis as a semantic memory store:
|
||||
|
||||
```csharp
|
||||
using ConnectionMultiplexer connectionMultiplexer = await ConnectionMultiplexer.ConnectAsync("localhost:6379");
|
||||
IDatabase database = connectionMultiplexer.GetDatabase();
|
||||
RedisMemoryStore memoryStore = new RedisMemoryStore(database, vectorSize: 1536);
|
||||
|
||||
IKernel kernel = Kernel.Builder
|
||||
.WithLogger(ConsoleLogger.Log)
|
||||
.WithOpenAITextEmbeddingGenerationService("text-embedding-ada-002", Env.Var("OPENAI_API_KEY"))
|
||||
.WithMemoryStorage(memoryStore)
|
||||
.Build();
|
||||
```
|
||||
@@ -0,0 +1,299 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.SemanticKernel.AI.Embeddings;
|
||||
using Microsoft.SemanticKernel.Memory;
|
||||
using NRedisStack;
|
||||
using NRedisStack.RedisStackCommands;
|
||||
using NRedisStack.Search;
|
||||
using NRedisStack.Search.Literals.Enums;
|
||||
using StackExchange.Redis;
|
||||
|
||||
namespace Microsoft.SemanticKernel.Connectors.Memory.Redis;
|
||||
|
||||
/// <summary>
|
||||
/// An implementation of <see cref="IMemoryStore"/> for Redis.
|
||||
/// </summary>
|
||||
/// <remarks>The embedded data is saved to the Redis server database specified in the constructor.
|
||||
/// Similarity search capability is provided through the RediSearch module. Use RediSearch's "Index" to implement "Collection".
|
||||
/// </remarks>
|
||||
public sealed class RedisMemoryStore : IMemoryStore
|
||||
{
|
||||
/// <summary>
|
||||
/// Create a new instance of semantic memory using Redis.
|
||||
/// </summary>
|
||||
/// <param name="database">The database of the redis server.</param>
|
||||
/// <param name="vectorSize">Embedding vector size</param>
|
||||
public RedisMemoryStore(IDatabase database, int vectorSize)
|
||||
{
|
||||
this._database = database;
|
||||
this._vectorSize = vectorSize;
|
||||
this._ft = database.FT();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<string> GetCollectionsAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
|
||||
{
|
||||
foreach (var index in await this._ft._ListAsync().ConfigureAwait(false))
|
||||
{
|
||||
yield return ((string)index!);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task CreateCollectionAsync(string collectionName, CancellationToken cancellationToken = default)
|
||||
{
|
||||
FTCreateParams ftCreateParams = FTCreateParams.CreateParams().On(IndexDataType.HASH).Prefix($"{collectionName}:");
|
||||
Schema schema = new Schema()
|
||||
.AddTextField("key")
|
||||
.AddTextField("metadata")
|
||||
.AddNumericField("timestamp")
|
||||
.AddVectorField("embedding", VECTOR_INDEX_ALGORITHM, new Dictionary<string, object> {
|
||||
{"TYPE", VECTOR_TYPE},
|
||||
{"DIM", this._vectorSize},
|
||||
{"DISTANCE_METRIC", VECTOR_DISTANCE_METRIC},
|
||||
});
|
||||
|
||||
await this._ft.CreateAsync(collectionName, ftCreateParams, schema).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<bool> DoesCollectionExistAsync(string collectionName, CancellationToken cancellationToken = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
await this._ft.InfoAsync(collectionName).ConfigureAwait(false);
|
||||
return true;
|
||||
}
|
||||
catch (RedisServerException ex) when (ex.Message == MESSAGE_WHEN_INDEX_DOES_NOT_EXIST)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task DeleteCollectionAsync(string collectionName, CancellationToken cancellationToken = default)
|
||||
{
|
||||
// dd: If `true`, all documents will be deleted.
|
||||
await this._ft.DropIndexAsync(collectionName, dd: true).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<MemoryRecord?> GetAsync(string collectionName, string key, bool withEmbedding = false, CancellationToken cancellationToken = default)
|
||||
{
|
||||
return await this.InternalGetAsync(collectionName, key, withEmbedding, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async IAsyncEnumerable<MemoryRecord> GetBatchAsync(string collectionName, IEnumerable<string> keys, bool withEmbeddings = false,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken = default)
|
||||
{
|
||||
foreach (var key in keys)
|
||||
{
|
||||
var result = await this.InternalGetAsync(collectionName, key, withEmbeddings, cancellationToken).ConfigureAwait(false);
|
||||
if (result != null)
|
||||
{
|
||||
yield return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<string> UpsertAsync(string collectionName, MemoryRecord record, CancellationToken cancellationToken = default)
|
||||
{
|
||||
record.Key = record.Metadata.Id;
|
||||
|
||||
await this._database.HashSetAsync(GetRedisKey(collectionName, record.Key), new[] {
|
||||
new HashEntry("key", record.Key),
|
||||
new HashEntry("metadata", record.GetSerializedMetadata()),
|
||||
new HashEntry("embedding", MemoryMarshal.Cast<float, byte>(record.Embedding.AsReadOnlySpan()).ToArray()),
|
||||
new HashEntry("timestamp", ToTimestampLong(record.Timestamp))
|
||||
}, flags: CommandFlags.None).ConfigureAwait(false);
|
||||
|
||||
return record.Key;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async IAsyncEnumerable<string> UpsertBatchAsync(string collectionName, IEnumerable<MemoryRecord> records, [EnumeratorCancellation] CancellationToken cancellationToken = default)
|
||||
{
|
||||
foreach (var record in records)
|
||||
{
|
||||
yield return await this.UpsertAsync(collectionName, record, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task RemoveAsync(string collectionName, string key, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await this._database.KeyDeleteAsync(GetRedisKey(collectionName, key), flags: CommandFlags.None).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task RemoveBatchAsync(string collectionName, IEnumerable<string> keys, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await this._database.KeyDeleteAsync(keys.Select(key => GetRedisKey(collectionName, key)).ToArray(), flags: CommandFlags.None).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync(
|
||||
string collectionName,
|
||||
Embedding<float> embedding,
|
||||
int limit,
|
||||
double minRelevanceScore = 0,
|
||||
bool withEmbeddings = false,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (limit <= 0)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
var query = new Query($"*=>[KNN {limit} @embedding $embedding AS vector_score]")
|
||||
.AddParam("embedding", MemoryMarshal.Cast<float, byte>(embedding.AsReadOnlySpan()).ToArray())
|
||||
.SetSortBy("vector_score")
|
||||
.ReturnFields("key", "metadata", "embedding", "timestamp", "vector_score")
|
||||
.Limit(0, limit)
|
||||
.Dialect(QUERY_DIALECT);
|
||||
|
||||
var results = await this._ft.SearchAsync(collectionName, query).ConfigureAwait(false);
|
||||
|
||||
foreach (var document in results.Documents)
|
||||
{
|
||||
double similarity = this.GetSimilarity(document);
|
||||
if (similarity < minRelevanceScore)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
Embedding<float> convertedEmbedding = withEmbeddings && document["embedding"].HasValue
|
||||
?
|
||||
new Embedding<float>(MemoryMarshal.Cast<byte, float>((byte[])document["embedding"]!).ToArray())
|
||||
:
|
||||
Embedding<float>.Empty;
|
||||
|
||||
yield return (MemoryRecord.FromJsonMetadata(
|
||||
json: document["metadata"]!,
|
||||
embedding: convertedEmbedding,
|
||||
key: document["key"],
|
||||
timestamp: ParseTimestamp((long?)document["timestamp"])), similarity);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task<(MemoryRecord, double)?> GetNearestMatchAsync(string collectionName, Embedding<float> embedding, double minRelevanceScore = 0, bool withEmbedding = false,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
return await this.GetNearestMatchesAsync(
|
||||
collectionName: collectionName,
|
||||
embedding: embedding,
|
||||
limit: 1,
|
||||
minRelevanceScore: minRelevanceScore,
|
||||
withEmbeddings: withEmbedding,
|
||||
cancellationToken: cancellationToken).FirstOrDefaultAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
#region constants ================================================================================
|
||||
|
||||
/// <summary>
|
||||
/// Vector similarity index algorithm. The default value is "HNSW".
|
||||
/// <see href="https://redis.io/docs/stack/search/reference/vectors/#create-a-vector-field"/>
|
||||
/// </summary>
|
||||
internal const Schema.VectorField.VectorAlgo VECTOR_INDEX_ALGORITHM = Schema.VectorField.VectorAlgo.HNSW;
|
||||
|
||||
/// <summary>
|
||||
/// Vector type. Supported types are FLOAT32 and FLOAT64. The default value is "FLOAT32".
|
||||
/// </summary>
|
||||
internal const string VECTOR_TYPE = "FLOAT32";
|
||||
|
||||
/// <summary>
|
||||
/// Supported distance metric, one of {L2, IP, COSINE}. The default value is "COSINE".
|
||||
/// </summary>
|
||||
internal const string VECTOR_DISTANCE_METRIC = "COSINE";
|
||||
|
||||
/// <summary>
|
||||
/// Query dialect. To use a vector similarity query, specify DIALECT 2 or higher. The default value is "2".
|
||||
/// <see href="https://redis.io/docs/stack/search/reference/vectors/#querying-vector-fields"/>
|
||||
/// </summary>
|
||||
internal const int QUERY_DIALECT = 2;
|
||||
|
||||
/// <summary>
|
||||
/// Message when index does not exist.
|
||||
/// <see href="https://github.com/RediSearch/RediSearch/blob/master/src/info_command.c#L96"/>
|
||||
/// </summary>
|
||||
internal const string MESSAGE_WHEN_INDEX_DOES_NOT_EXIST = "Unknown Index name";
|
||||
|
||||
#endregion
|
||||
|
||||
#region private ================================================================================
|
||||
|
||||
private readonly IDatabase _database;
|
||||
private readonly int _vectorSize;
|
||||
private readonly SearchCommands _ft;
|
||||
|
||||
private static long ToTimestampLong(DateTimeOffset? timestamp)
|
||||
{
|
||||
if (timestamp.HasValue)
|
||||
{
|
||||
return timestamp.Value.ToUnixTimeMilliseconds();
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseTimestamp(long? timestamp)
|
||||
{
|
||||
if (timestamp.HasValue && timestamp > 0)
|
||||
{
|
||||
return DateTimeOffset.FromUnixTimeMilliseconds(timestamp.Value);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static RedisKey GetRedisKey(string collectionName, string key)
|
||||
{
|
||||
return new RedisKey($"{collectionName}:{key}");
|
||||
}
|
||||
|
||||
private async Task<MemoryRecord?> InternalGetAsync(string collectionName, string key, bool withEmbedding, CancellationToken cancellationToken)
|
||||
{
|
||||
HashEntry[] hashEntries = await this._database.HashGetAllAsync(GetRedisKey(collectionName, key), flags: CommandFlags.None).ConfigureAwait(false);
|
||||
|
||||
if (hashEntries.Length == 0) { return null; }
|
||||
|
||||
if (withEmbedding)
|
||||
{
|
||||
RedisValue embedding = hashEntries.FirstOrDefault(x => x.Name == "embedding").Value;
|
||||
return MemoryRecord.FromJsonMetadata(
|
||||
json: hashEntries.FirstOrDefault(x => x.Name == "metadata").Value!,
|
||||
embedding: embedding.HasValue ? new Embedding<float>(MemoryMarshal.Cast<byte, float>((byte[])embedding!).ToArray()) : Embedding<float>.Empty,
|
||||
key: hashEntries.FirstOrDefault(x => x.Name == "key").Value,
|
||||
timestamp: ParseTimestamp((long?)hashEntries.FirstOrDefault(x => x.Name == "timestamp").Value));
|
||||
}
|
||||
|
||||
return MemoryRecord.FromJsonMetadata(
|
||||
json: hashEntries.FirstOrDefault(x => x.Name == "metadata").Value!,
|
||||
embedding: Embedding<float>.Empty,
|
||||
key: hashEntries.FirstOrDefault(x => x.Name == "key").Value,
|
||||
timestamp: ParseTimestamp((long?)hashEntries.FirstOrDefault(x => x.Name == "timestamp").Value));
|
||||
}
|
||||
|
||||
private double GetSimilarity(Document document)
|
||||
{
|
||||
RedisValue vectorScoreValue = document["vector_score"];
|
||||
|
||||
if (vectorScoreValue.IsNullOrEmpty || !vectorScoreValue.TryParse(out double vectorScore))
|
||||
{
|
||||
throw new RedisMemoryStoreException("Invalid or missing vector score value.");
|
||||
}
|
||||
|
||||
return 1 - vectorScore;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
|
||||
namespace Microsoft.SemanticKernel.Connectors.Memory.Redis;
|
||||
|
||||
#pragma warning disable RCS1194 // Implement exception constructors
|
||||
|
||||
/// <summary>
|
||||
/// Exception thrown by the Redis connector
|
||||
/// </summary>
|
||||
public class RedisMemoryStoreException : Exception
|
||||
{
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="RedisMemoryStoreException"/> class.
|
||||
/// </summary>
|
||||
/// <param name="message">Exception message.</param>
|
||||
internal RedisMemoryStoreException(string? message) : base(message)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="RedisMemoryStoreException"/> class.
|
||||
/// </summary>
|
||||
/// <param name="message">Exception message.</param>
|
||||
/// <param name="innerException">Inner exception.</param>
|
||||
internal RedisMemoryStoreException(string? message, Exception? innerException) : base(message, innerException)
|
||||
{
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,7 @@
|
||||
<ProjectReference Include="..\Connectors.Memory.Pinecone\Connectors.Memory.Pinecone.csproj" />
|
||||
<ProjectReference Include="..\Connectors.Memory.DuckDB\Connectors.Memory.DuckDB.csproj" />
|
||||
<ProjectReference Include="..\Connectors.Memory.Qdrant\Connectors.Memory.Qdrant.csproj" />
|
||||
<ProjectReference Include="..\Connectors.Memory.Redis\Connectors.Memory.Redis.csproj" />
|
||||
<ProjectReference Include="..\Connectors.Memory.Sqlite\Connectors.Memory.Sqlite.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -0,0 +1,986 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.SemanticKernel.AI.Embeddings;
|
||||
using Microsoft.SemanticKernel.AI.Embeddings.VectorOperations;
|
||||
using Microsoft.SemanticKernel.Connectors.Memory.Redis;
|
||||
using Microsoft.SemanticKernel.Memory;
|
||||
using Microsoft.SemanticKernel.Memory.Collections;
|
||||
using Moq;
|
||||
using StackExchange.Redis;
|
||||
using Xunit;
|
||||
|
||||
namespace SemanticKernel.Connectors.UnitTests.Memory.Redis;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests of <see cref="RedisMemoryStore"/>.
|
||||
/// </summary>
|
||||
public class RedisMemoryStoreTests
|
||||
{
|
||||
private readonly Mock<IDatabase> _mockDatabase;
|
||||
private readonly Dictionary<string, List<MemoryRecord>> _collections;
|
||||
|
||||
public RedisMemoryStoreTests()
|
||||
{
|
||||
this._mockDatabase = new Mock<IDatabase>();
|
||||
this._collections = new();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConnectionCanBeInitialized()
|
||||
{
|
||||
// Arrange
|
||||
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ItCanCreateAndGetCollectionAsync()
|
||||
{
|
||||
// Arrange
|
||||
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
|
||||
string collection = "test_collection";
|
||||
this.MockCreateIndex(collection);
|
||||
|
||||
// Act
|
||||
await store.CreateCollectionAsync(collection);
|
||||
var collections = store.GetCollectionsAsync();
|
||||
|
||||
// Assert
|
||||
Assert.NotEmpty(collections.ToEnumerable());
|
||||
Assert.True(await collections.ContainsAsync(collection));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ItCanCheckIfCollectionExistsAsync()
|
||||
{
|
||||
// Arrange
|
||||
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
|
||||
string collection = "my_collection";
|
||||
this.MockCreateIndex(collection);
|
||||
|
||||
// Act
|
||||
await store.CreateCollectionAsync(collection);
|
||||
|
||||
// Assert
|
||||
Assert.True(await store.DoesCollectionExistAsync("my_collection"));
|
||||
Assert.False(await store.DoesCollectionExistAsync("my_collection2"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CollectionsCanBeDeletedAsync()
|
||||
{
|
||||
// Arrange
|
||||
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
|
||||
string collection = "test_collection";
|
||||
this.MockCreateIndex(collection, () =>
|
||||
{
|
||||
this.MockDropIndex(collection);
|
||||
});
|
||||
|
||||
await store.CreateCollectionAsync(collection);
|
||||
var collections = await store.GetCollectionsAsync().ToListAsync();
|
||||
Assert.True(collections.Count > 0);
|
||||
|
||||
// Act
|
||||
foreach (var c in collections)
|
||||
{
|
||||
await store.DeleteCollectionAsync(c);
|
||||
}
|
||||
|
||||
// Assert
|
||||
var collections2 = store.GetCollectionsAsync();
|
||||
Assert.True(await collections2.CountAsync() == 0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ItCanInsertIntoNonExistentCollectionAsync()
|
||||
{
|
||||
// Arrange
|
||||
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
|
||||
MemoryRecord testRecord = MemoryRecord.LocalRecord(
|
||||
id: "test",
|
||||
text: "text",
|
||||
description: "description",
|
||||
embedding: new Embedding<float>(new float[] { 1, 2, 3 }),
|
||||
key: null,
|
||||
timestamp: null);
|
||||
string collection = "random collection";
|
||||
string redisKey = $"{collection}:{testRecord.Metadata.Id}";
|
||||
byte[] embedding = MemoryMarshal.Cast<float, byte>(testRecord.Embedding.AsReadOnlySpan()).ToArray();
|
||||
this._mockDatabase
|
||||
.Setup<Task>(x => x.HashSetAsync(
|
||||
It.Is<RedisKey>(x => x == redisKey),
|
||||
It.Is<HashEntry[]>(x => x.Length == 4 &&
|
||||
x[0].Name == "key" && x[1].Name == "metadata" && x[2].Name == "embedding" && x[3].Name == "timestamp" &&
|
||||
x[0].Value == testRecord.Key && x[1].Value == testRecord.GetSerializedMetadata() && embedding.SequenceEqual((byte[])x[2].Value!) && x[3].Value == -1
|
||||
),
|
||||
It.IsAny<CommandFlags>())
|
||||
)
|
||||
.Callback(() =>
|
||||
{
|
||||
this._mockDatabase
|
||||
.Setup<Task<HashEntry[]>>(x => x.HashGetAllAsync(It.Is<RedisKey>(x => x == redisKey), It.IsAny<CommandFlags>()))
|
||||
.ReturnsAsync(new[] {
|
||||
new HashEntry("key", testRecord.Key),
|
||||
new HashEntry("metadata", testRecord.GetSerializedMetadata()),
|
||||
new HashEntry("embedding", embedding),
|
||||
new HashEntry("timestamp", -1)
|
||||
});
|
||||
});
|
||||
|
||||
// Arrange
|
||||
var key = await store.UpsertAsync(collection, testRecord);
|
||||
var actual = await store.GetAsync(collection, key, true);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(actual);
|
||||
Assert.Equal(testRecord.Metadata.Id, key);
|
||||
Assert.Equal(testRecord.Metadata.Id, actual.Key);
|
||||
Assert.Equal(testRecord.Embedding.Vector, actual.Embedding.Vector);
|
||||
Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text);
|
||||
Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description);
|
||||
Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName);
|
||||
Assert.Equal(testRecord.Metadata.Id, actual.Metadata.Id);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync()
|
||||
{
|
||||
// Arrange
|
||||
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
|
||||
MemoryRecord testRecord = MemoryRecord.LocalRecord(
|
||||
id: "test",
|
||||
text: "text",
|
||||
description: "description",
|
||||
embedding: new Embedding<float>(new float[] { 1, 2, 3 }),
|
||||
key: null,
|
||||
timestamp: null);
|
||||
string collection = "test_collection";
|
||||
string redisKey = $"{collection}:{testRecord.Metadata.Id}";
|
||||
|
||||
this.MockCreateIndex(collection, () =>
|
||||
{
|
||||
this.MockHashSet(collection, testRecord);
|
||||
});
|
||||
|
||||
// Act
|
||||
await store.CreateCollectionAsync(collection);
|
||||
var key = await store.UpsertAsync(collection, testRecord);
|
||||
var actualDefault = await store.GetAsync(collection, key);
|
||||
var actualWithEmbedding = await store.GetAsync(collection, key, true);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(actualDefault);
|
||||
Assert.NotNull(actualWithEmbedding);
|
||||
Assert.Empty(actualDefault.Embedding.Vector);
|
||||
Assert.NotEmpty(actualWithEmbedding.Embedding.Vector);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ItCanUpsertAndRetrieveARecordWithNoTimestampAsync()
|
||||
{
|
||||
// Arrange
|
||||
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
|
||||
MemoryRecord testRecord = MemoryRecord.LocalRecord(
|
||||
id: "test",
|
||||
text: "text",
|
||||
description: "description",
|
||||
embedding: new Embedding<float>(new float[] { 1, 2, 3 }),
|
||||
key: null,
|
||||
timestamp: null);
|
||||
string collection = "test_collection";
|
||||
this.MockCreateIndex(collection, () =>
|
||||
{
|
||||
this.MockHashSet(collection, testRecord);
|
||||
});
|
||||
|
||||
// Act
|
||||
await store.CreateCollectionAsync(collection);
|
||||
var key = await store.UpsertAsync(collection, testRecord);
|
||||
var actual = await store.GetAsync(collection, key, true);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(actual);
|
||||
Assert.Equal(testRecord.Metadata.Id, key);
|
||||
Assert.Equal(testRecord.Metadata.Id, actual.Key);
|
||||
Assert.Equal(testRecord.Embedding.Vector, actual.Embedding.Vector);
|
||||
Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text);
|
||||
Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description);
|
||||
Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName);
|
||||
Assert.Equal(testRecord.Metadata.Id, actual.Metadata.Id);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ItCanUpsertAndRetrieveARecordWithTimestampAsync()
|
||||
{
|
||||
// Arrange
|
||||
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
|
||||
MemoryRecord testRecord = MemoryRecord.LocalRecord(
|
||||
id: "test",
|
||||
text: "text",
|
||||
description: "description",
|
||||
embedding: new Embedding<float>(new float[] { 1, 2, 3 }),
|
||||
key: null,
|
||||
timestamp: DateTimeOffset.UtcNow);
|
||||
string collection = "test_collection";
|
||||
this.MockCreateIndex(collection, () =>
|
||||
{
|
||||
this.MockHashSet(collection, testRecord);
|
||||
});
|
||||
|
||||
// Act
|
||||
await store.CreateCollectionAsync(collection);
|
||||
var key = await store.UpsertAsync(collection, testRecord);
|
||||
var actual = await store.GetAsync(collection, key, true);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(actual);
|
||||
Assert.Equal(testRecord.Metadata.Id, key);
|
||||
Assert.Equal(testRecord.Metadata.Id, actual.Key);
|
||||
Assert.Equal(testRecord.Embedding.Vector, actual.Embedding.Vector);
|
||||
Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text);
|
||||
Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description);
|
||||
Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName);
|
||||
Assert.Equal(testRecord.Metadata.Id, actual.Metadata.Id);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpsertReplacesExistingRecordWithSameIdAsync()
|
||||
{
|
||||
// Arrange
|
||||
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
|
||||
string commonId = "test";
|
||||
MemoryRecord testRecord = MemoryRecord.LocalRecord(
|
||||
id: commonId,
|
||||
text: "text",
|
||||
description: "description",
|
||||
embedding: new Embedding<float>(new float[] { 1, 2, 3 }));
|
||||
MemoryRecord testRecord2 = MemoryRecord.LocalRecord(
|
||||
id: commonId,
|
||||
text: "text2",
|
||||
description: "description2",
|
||||
embedding: new Embedding<float>(new float[] { 1, 2, 4 }));
|
||||
string collection = "test_collection";
|
||||
this.MockCreateIndex(collection, () =>
|
||||
{
|
||||
this.MockHashSet(collection, testRecord);
|
||||
this.MockHashSet(collection, testRecord2);
|
||||
});
|
||||
|
||||
// Act
|
||||
await store.CreateCollectionAsync(collection);
|
||||
var key = await store.UpsertAsync(collection, testRecord);
|
||||
var key2 = await store.UpsertAsync(collection, testRecord2);
|
||||
var actual = await store.GetAsync(collection, key, true);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(actual);
|
||||
Assert.Equal(testRecord.Metadata.Id, key);
|
||||
Assert.Equal(testRecord2.Metadata.Id, actual.Key);
|
||||
Assert.NotEqual(testRecord.Embedding.Vector, actual.Embedding.Vector);
|
||||
Assert.Equal(testRecord2.Embedding.Vector, actual.Embedding.Vector);
|
||||
Assert.NotEqual(testRecord.Metadata.Text, actual.Metadata.Text);
|
||||
Assert.Equal(testRecord2.Metadata.Description, actual.Metadata.Description);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExistingRecordCanBeRemovedAsync()
|
||||
{
|
||||
// Arrange
|
||||
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
|
||||
MemoryRecord testRecord = MemoryRecord.LocalRecord(
|
||||
id: "test",
|
||||
text: "text",
|
||||
description: "description",
|
||||
embedding: new Embedding<float>(new float[] { 1, 2, 3 }));
|
||||
string collection = "test_collection";
|
||||
this.MockCreateIndex(collection, () =>
|
||||
{
|
||||
this.MockHashSet(collection, testRecord, () =>
|
||||
{
|
||||
this.MockKeyDelete(collection, testRecord.Metadata.Id);
|
||||
});
|
||||
});
|
||||
|
||||
// Act
|
||||
await store.CreateCollectionAsync(collection);
|
||||
var key = await store.UpsertAsync(collection, testRecord);
|
||||
await store.RemoveAsync(collection, key);
|
||||
var actual = await store.GetAsync(collection, key);
|
||||
|
||||
// Assert
|
||||
Assert.Null(actual);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RemovingNonExistingRecordDoesNothingAsync()
|
||||
{
|
||||
// Arrange
|
||||
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
|
||||
string collection = "test_collection";
|
||||
this.MockCreateIndex(collection, () =>
|
||||
{
|
||||
this.MockKeyDelete(collection, "key");
|
||||
});
|
||||
|
||||
// Act
|
||||
await store.CreateCollectionAsync(collection);
|
||||
await store.RemoveAsync(collection, "key");
|
||||
var actual = await store.GetAsync(collection, "key");
|
||||
|
||||
// Assert
|
||||
Assert.Null(actual);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ItCanListAllDatabaseCollectionsAsync()
|
||||
{
|
||||
// Arrange
|
||||
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
|
||||
string[] testCollections = { "random_collection1", "random_collection2", "random_collection3" };
|
||||
foreach (var collection in testCollections)
|
||||
{
|
||||
this.MockCreateIndex(collection, () =>
|
||||
{
|
||||
this.MockDropIndex(collection);
|
||||
});
|
||||
}
|
||||
await store.CreateCollectionAsync(testCollections[0]);
|
||||
await store.CreateCollectionAsync(testCollections[1]);
|
||||
await store.CreateCollectionAsync(testCollections[2]);
|
||||
|
||||
// Act
|
||||
var collections = await store.GetCollectionsAsync().ToListAsync();
|
||||
|
||||
// Assert
|
||||
foreach (var collection in testCollections)
|
||||
{
|
||||
Assert.True(await store.DoesCollectionExistAsync(collection));
|
||||
}
|
||||
|
||||
Assert.NotNull(collections);
|
||||
Assert.NotEmpty(collections);
|
||||
Assert.Equal(testCollections.Length, collections.Count);
|
||||
Assert.True(collections.Contains(testCollections[0]),
|
||||
$"Collections does not contain the newly-created collection {testCollections[0]}");
|
||||
Assert.True(collections.Contains(testCollections[1]),
|
||||
$"Collections does not contain the newly-created collection {testCollections[1]}");
|
||||
Assert.True(collections.Contains(testCollections[2]),
|
||||
$"Collections does not contain the newly-created collection {testCollections[2]}");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync()
|
||||
{
|
||||
// Arrange
|
||||
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
|
||||
var compareEmbedding = new Embedding<float>(new float[] { 1, 1, 1 });
|
||||
string collection = "test_collection";
|
||||
int topN = 4;
|
||||
double threshold = -1;
|
||||
var testEmbeddings = new[]
|
||||
{
|
||||
new Embedding<float>(new float[] { 1, 1, 1 }),
|
||||
new Embedding<float>(new float[] { -1, -1, -1 }),
|
||||
new Embedding<float>(new float[] { 1, 2, 3 }),
|
||||
new Embedding<float>(new float[] { -1, -2, -3 }),
|
||||
new Embedding<float>(new float[] { 1, -1, -2 })
|
||||
};
|
||||
var testRecords = new List<MemoryRecord>();
|
||||
for (int i = 0; i < testEmbeddings.Length; i++)
|
||||
{
|
||||
testRecords.Add(MemoryRecord.LocalRecord(
|
||||
id: "test" + i,
|
||||
text: "text" + i,
|
||||
description: "description" + i,
|
||||
embedding: testEmbeddings[i]));
|
||||
}
|
||||
this.MockCreateIndex(collection, () =>
|
||||
{
|
||||
for (int i = 0; i < testRecords.Count; i++)
|
||||
{
|
||||
if (i + 1 < testRecords.Count)
|
||||
{
|
||||
this.MockHashSet(collection, testRecords[i]);
|
||||
}
|
||||
else
|
||||
{
|
||||
this.MockHashSet(collection, testRecords[i], () => this.MockSearch(collection, compareEmbedding, topN, threshold));
|
||||
}
|
||||
}
|
||||
});
|
||||
await store.CreateCollectionAsync(collection);
|
||||
foreach (var testRecord in testRecords)
|
||||
{
|
||||
_ = await store.UpsertAsync(collection, testRecord);
|
||||
}
|
||||
|
||||
// Act
|
||||
var topNResults = store.GetNearestMatchesAsync(collection, compareEmbedding, limit: topN, minRelevanceScore: threshold).ToEnumerable().ToArray();
|
||||
|
||||
// Assert
|
||||
Assert.Equal(topN, topNResults.Length);
|
||||
for (int j = 0; j < topN - 1; j++)
|
||||
{
|
||||
int compare = topNResults[j].Item2.CompareTo(topNResults[j + 1].Item2);
|
||||
Assert.True(compare >= 0);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync()
|
||||
{
|
||||
// Arrange
|
||||
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
|
||||
var compareEmbedding = new Embedding<float>(new float[] { 1, 1, 1 });
|
||||
string collection = "test_collection";
|
||||
int topN = 1;
|
||||
double threshold = 0.75;
|
||||
var testEmbeddings = new[]
|
||||
{
|
||||
new Embedding<float>(new float[] { 1, 1, 1 }),
|
||||
new Embedding<float>(new float[] { -1, -1, -1 }),
|
||||
new Embedding<float>(new float[] { 1, 2, 3 }),
|
||||
new Embedding<float>(new float[] { -1, -2, -3 }),
|
||||
new Embedding<float>(new float[] { 1, -1, -2 })
|
||||
};
|
||||
var testRecords = new List<MemoryRecord>();
|
||||
for (int i = 0; i < testEmbeddings.Length; i++)
|
||||
{
|
||||
testRecords.Add(MemoryRecord.LocalRecord(
|
||||
id: "test" + i,
|
||||
text: "text" + i,
|
||||
description: "description" + i,
|
||||
embedding: testEmbeddings[i]));
|
||||
}
|
||||
this.MockCreateIndex(collection, () =>
|
||||
{
|
||||
for (int i = 0; i < testRecords.Count; i++)
|
||||
{
|
||||
if (i + 1 < testRecords.Count)
|
||||
{
|
||||
this.MockHashSet(collection, testRecords[i]);
|
||||
}
|
||||
else
|
||||
{
|
||||
this.MockHashSet(collection, testRecords[i], () =>
|
||||
{
|
||||
this.MockSearch(collection, compareEmbedding, topN, threshold);
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
await store.CreateCollectionAsync(collection);
|
||||
foreach (var testRecord in testRecords)
|
||||
{
|
||||
_ = await store.UpsertAsync(collection, testRecord);
|
||||
}
|
||||
|
||||
// Act
|
||||
var topNResultDefault = await store.GetNearestMatchAsync(collection, compareEmbedding, minRelevanceScore: threshold);
|
||||
var topNResultWithEmbedding = await store.GetNearestMatchAsync(collection, compareEmbedding, minRelevanceScore: threshold, withEmbedding: true);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(topNResultDefault);
|
||||
Assert.NotNull(topNResultWithEmbedding);
|
||||
Assert.Empty(topNResultDefault.Value.Item1.Embedding.Vector);
|
||||
Assert.NotEmpty(topNResultWithEmbedding.Value.Item1.Embedding.Vector);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetNearestMatchAsyncReturnsExpectedAsync()
|
||||
{
|
||||
// Arrange
|
||||
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
|
||||
var compareEmbedding = new Embedding<float>(new float[] { 1, 1, 1 });
|
||||
string collection = "test_collection";
|
||||
int topN = 1;
|
||||
double threshold = 0.75;
|
||||
var testEmbeddings = new[]
|
||||
{
|
||||
new Embedding<float>(new float[] { 1, 1, 1 }),
|
||||
new Embedding<float>(new float[] { -1, -1, -1 }),
|
||||
new Embedding<float>(new float[] { 1, 2, 3 }),
|
||||
new Embedding<float>(new float[] { -1, -2, -3 }),
|
||||
new Embedding<float>(new float[] { 1, -1, -2 })
|
||||
};
|
||||
var testRecords = new List<MemoryRecord>();
|
||||
for (int i = 0; i < testEmbeddings.Length; i++)
|
||||
{
|
||||
testRecords.Add(MemoryRecord.LocalRecord(
|
||||
id: "test" + i,
|
||||
text: "text" + i,
|
||||
description: "description" + i,
|
||||
embedding: testEmbeddings[i]));
|
||||
}
|
||||
this.MockCreateIndex(collection, () =>
|
||||
{
|
||||
for (int i = 0; i < testRecords.Count; i++)
|
||||
{
|
||||
if (i + 1 < testRecords.Count)
|
||||
{
|
||||
this.MockHashSet(collection, testRecords[i]);
|
||||
}
|
||||
else
|
||||
{
|
||||
this.MockHashSet(collection, testRecords[i], () =>
|
||||
{
|
||||
this.MockSearch(collection, compareEmbedding, topN, threshold);
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
await store.CreateCollectionAsync(collection);
|
||||
foreach (var testRecord in testRecords)
|
||||
{
|
||||
_ = await store.UpsertAsync(collection, testRecord);
|
||||
}
|
||||
|
||||
// Act
|
||||
var topNResult = await store.GetNearestMatchAsync(collection, compareEmbedding, minRelevanceScore: threshold);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(topNResult);
|
||||
Assert.Equal("test0", topNResult.Value.Item1.Metadata.Id);
|
||||
Assert.True(topNResult.Value.Item2 >= threshold);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetNearestMatchesDifferentiatesIdenticalVectorsByKeyAsync()
|
||||
{
|
||||
// Arrange
|
||||
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
|
||||
var compareEmbedding = new Embedding<float>(new float[] { 1, 1, 1 });
|
||||
int topN = 4;
|
||||
double threshold = 0.75;
|
||||
string collection = "test_collection";
|
||||
var testRecords = new List<MemoryRecord>();
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
testRecords.Add(MemoryRecord.LocalRecord(
|
||||
id: "test" + i,
|
||||
text: "text" + i,
|
||||
description: "description" + i,
|
||||
embedding: new Embedding<float>(new float[] { 1, 1, 1 })));
|
||||
}
|
||||
this.MockCreateIndex(collection, () =>
|
||||
{
|
||||
for (int i = 0; i < testRecords.Count; i++)
|
||||
{
|
||||
if (i + 1 < testRecords.Count)
|
||||
{
|
||||
this.MockHashSet(collection, testRecords[i]);
|
||||
}
|
||||
else
|
||||
{
|
||||
this.MockHashSet(collection, testRecords[i], () =>
|
||||
{
|
||||
this.MockSearch(collection, compareEmbedding, topN, threshold);
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
await store.CreateCollectionAsync(collection);
|
||||
foreach (var testRecord in testRecords)
|
||||
{
|
||||
_ = await store.UpsertAsync(collection, testRecord);
|
||||
}
|
||||
|
||||
// Act
|
||||
var topNResults = store.GetNearestMatchesAsync(collection, compareEmbedding, limit: topN, minRelevanceScore: threshold).ToEnumerable().ToArray();
|
||||
IEnumerable<string> topNKeys = topNResults.Select(x => x.Item1.Key).ToImmutableSortedSet();
|
||||
|
||||
// Assert
|
||||
Assert.Equal(topN, topNResults.Length);
|
||||
Assert.Equal(topN, topNKeys.Count());
|
||||
|
||||
for (int i = 0; i < topNResults.Length; i++)
|
||||
{
|
||||
int compare = topNResults[i].Item2.CompareTo(threshold);
|
||||
Assert.True(compare >= 0);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ItCanBatchUpsertRecordsAsync()
|
||||
{
|
||||
// Arrange
|
||||
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
|
||||
int numRecords = 10;
|
||||
string collection = "test_collection";
|
||||
IEnumerable<MemoryRecord> records = this.CreateBatchRecords(numRecords);
|
||||
this.MockCreateIndex(collection, () =>
|
||||
{
|
||||
foreach (var testRecord in records)
|
||||
{
|
||||
this.MockHashSet(collection, testRecord);
|
||||
}
|
||||
});
|
||||
|
||||
// Act
|
||||
await store.CreateCollectionAsync(collection);
|
||||
var keys = store.UpsertBatchAsync(collection, records);
|
||||
var resultRecords = store.GetBatchAsync(collection, keys.ToEnumerable());
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(keys);
|
||||
Assert.Equal(numRecords, keys.ToEnumerable().Count());
|
||||
Assert.Equal(numRecords, resultRecords.ToEnumerable().Count());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ItCanBatchGetRecordsAsync()
|
||||
{
|
||||
// Arrange
|
||||
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
|
||||
int numRecords = 10;
|
||||
string collection = "test_collection";
|
||||
IEnumerable<MemoryRecord> records = this.CreateBatchRecords(numRecords);
|
||||
this.MockCreateIndex(collection, () =>
|
||||
{
|
||||
foreach (var testRecord in records)
|
||||
{
|
||||
this.MockHashSet(collection, testRecord);
|
||||
}
|
||||
});
|
||||
var keys = store.UpsertBatchAsync(collection, records);
|
||||
|
||||
// Act
|
||||
await store.CreateCollectionAsync(collection);
|
||||
var results = store.GetBatchAsync(collection, keys.ToEnumerable());
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(keys);
|
||||
Assert.NotNull(results);
|
||||
Assert.Equal(numRecords, results.ToEnumerable().Count());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ItCanBatchRemoveRecordsAsync()
|
||||
{
|
||||
// Arrange
|
||||
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
|
||||
int numRecords = 10;
|
||||
string collection = "test_collection";
|
||||
IEnumerable<MemoryRecord> records = this.CreateBatchRecords(numRecords);
|
||||
this.MockCreateIndex(collection, () =>
|
||||
{
|
||||
foreach (var testRecord in records)
|
||||
{
|
||||
this.MockHashSet(collection, testRecord, () => this.MockKeyDelete(collection, testRecord.Metadata.Id));
|
||||
}
|
||||
this.MockKeyDelete(collection, records.Select(x => x.Metadata.Id));
|
||||
});
|
||||
await store.CreateCollectionAsync(collection);
|
||||
|
||||
List<string> keys = new();
|
||||
|
||||
// Act
|
||||
await foreach (var key in store.UpsertBatchAsync(collection, records))
|
||||
{
|
||||
keys.Add(key);
|
||||
}
|
||||
|
||||
await store.RemoveBatchAsync(collection, keys);
|
||||
|
||||
// Assert
|
||||
await foreach (var result in store.GetBatchAsync(collection, keys))
|
||||
{
|
||||
Assert.Null(result);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetNearestMatchAsyncThrowsExceptionOnInvalidVectorScoreAsync()
|
||||
{
|
||||
// Arrange
|
||||
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
|
||||
var compareEmbedding = new Embedding<float>(new float[] { 1, 1, 1 });
|
||||
string collection = "test_collection";
|
||||
int topN = 1;
|
||||
double threshold = 0.75;
|
||||
var testEmbeddings = new[]
|
||||
{
|
||||
new Embedding<float>(new float[] { 1, 1, 1 }),
|
||||
new Embedding<float>(new float[] { -1, -1, -1 }),
|
||||
new Embedding<float>(new float[] { 1, 2, 3 }),
|
||||
new Embedding<float>(new float[] { -1, -2, -3 }),
|
||||
new Embedding<float>(new float[] { 1, -1, -2 })
|
||||
};
|
||||
var testRecords = new List<MemoryRecord>();
|
||||
for (int i = 0; i < testEmbeddings.Length; i++)
|
||||
{
|
||||
testRecords.Add(MemoryRecord.LocalRecord(
|
||||
id: "test" + i,
|
||||
text: "text" + i,
|
||||
description: "description" + i,
|
||||
embedding: testEmbeddings[i]));
|
||||
}
|
||||
this.MockCreateIndex(collection, () =>
|
||||
{
|
||||
for (int i = 0; i < testRecords.Count; i++)
|
||||
{
|
||||
if (i + 1 < testRecords.Count)
|
||||
{
|
||||
this.MockHashSet(collection, testRecords[i]);
|
||||
}
|
||||
else
|
||||
{
|
||||
this.MockHashSet(collection, testRecords[i], () =>
|
||||
{
|
||||
this.MockSearch(collection, compareEmbedding, topN, threshold, returnStringVectorScore: true);
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
await store.CreateCollectionAsync(collection);
|
||||
foreach (var testRecord in testRecords)
|
||||
{
|
||||
_ = await store.UpsertAsync(collection, testRecord);
|
||||
}
|
||||
|
||||
// Assert
|
||||
RedisMemoryStoreException ex = await Assert.ThrowsAsync<RedisMemoryStoreException>(async () =>
|
||||
{
|
||||
// Act
|
||||
await store.GetNearestMatchAsync(collection, compareEmbedding, minRelevanceScore: threshold);
|
||||
});
|
||||
Assert.Equal(ex.Message, "Invalid or missing vector score value.");
|
||||
}
|
||||
|
||||
#region private
|
||||
|
||||
private void MockCreateIndex(string collection, Action? callback = null)
|
||||
{
|
||||
this._mockDatabase
|
||||
.Setup<Task<RedisResult>>(x => x.ExecuteAsync(
|
||||
It.Is<string>(x => x == "FT.CREATE"),
|
||||
It.Is<object[]>(x => x[0].ToString() == collection))
|
||||
)
|
||||
.ReturnsAsync(RedisResult.Create("OK", ResultType.SimpleString))
|
||||
.Callback(() =>
|
||||
{
|
||||
this._collections.TryAdd(collection, new());
|
||||
|
||||
this._mockDatabase
|
||||
.Setup<Task<RedisResult>>(x => x.ExecuteAsync(
|
||||
It.Is<string>(x => x == "FT.INFO"),
|
||||
It.Is<object[]>(x => x[0].ToString() == collection))
|
||||
)
|
||||
.ReturnsAsync(RedisResult.Create(new[] {
|
||||
RedisResult.Create("index_name", ResultType.BulkString),
|
||||
RedisResult.Create(collection, ResultType.BulkString)
|
||||
}));
|
||||
|
||||
this._mockDatabase
|
||||
.Setup<Task<RedisResult>>(x => x.ExecuteAsync(
|
||||
It.Is<string>(x => x == "FT._LIST"),
|
||||
It.IsAny<object[]>())
|
||||
)
|
||||
.ReturnsAsync(RedisResult.Create(this._collections.Select(x => RedisResult.Create(x.Key, ResultType.BulkString)).ToArray()));
|
||||
|
||||
callback?.Invoke();
|
||||
});
|
||||
|
||||
this._mockDatabase
|
||||
.Setup<Task<RedisResult>>(x => x.ExecuteAsync(It.Is<string>(x => x == "FT.INFO"), It.IsAny<object[]>()))
|
||||
.Throws(new RedisServerException("Unknown Index name"));
|
||||
}
|
||||
|
||||
private void MockDropIndex(string collection, Action? callback = null)
|
||||
{
|
||||
this._mockDatabase
|
||||
.Setup<Task<RedisResult>>(x => x.ExecuteAsync(
|
||||
It.Is<string>(x => x == "FT.DROPINDEX"),
|
||||
It.Is<object[]>(x => x[0].ToString() == collection && x[1].ToString() == "DD"))
|
||||
)
|
||||
.ReturnsAsync(RedisResult.Create("OK", ResultType.SimpleString))
|
||||
.Callback(() =>
|
||||
{
|
||||
this._collections.Remove(collection);
|
||||
|
||||
this._mockDatabase
|
||||
.Setup<Task<RedisResult>>(x => x.ExecuteAsync(
|
||||
It.Is<string>(x => x == "FT.INFO"),
|
||||
It.Is<object[]>(x => x[0].ToString() == collection))
|
||||
)
|
||||
.Throws(new RedisServerException("Unknown Index name"));
|
||||
|
||||
this._mockDatabase
|
||||
.Setup<Task<RedisResult>>(x => x.ExecuteAsync(
|
||||
It.Is<string>(x => x == "FT._LIST"),
|
||||
It.IsAny<object[]>())
|
||||
)
|
||||
.ReturnsAsync(RedisResult.Create(this._collections.Select(x => RedisResult.Create(x.Key, ResultType.BulkString)).ToArray()));
|
||||
});
|
||||
}
|
||||
|
||||
private void MockHashSet(string collection, MemoryRecord record, Action? callback = null)
|
||||
{
|
||||
string redisKey = $"{collection}:{record.Metadata.Id}";
|
||||
byte[] embedding = MemoryMarshal.Cast<float, byte>(record.Embedding.AsReadOnlySpan()).ToArray();
|
||||
long timestamp = record.Timestamp?.ToUnixTimeMilliseconds() ?? -1;
|
||||
|
||||
this._mockDatabase
|
||||
.Setup<Task>(x => x.HashSetAsync(
|
||||
It.Is<RedisKey>(x => x == redisKey),
|
||||
It.Is<HashEntry[]>(x => x.Length == 4 &&
|
||||
x[0].Name == "key" && x[1].Name == "metadata" && x[2].Name == "embedding" && x[3].Name == "timestamp" &&
|
||||
x[0].Value == record.Key && x[1].Value == record.GetSerializedMetadata() && embedding.SequenceEqual((byte[])x[2].Value!) && x[3].Value == timestamp
|
||||
),
|
||||
It.IsAny<CommandFlags>())
|
||||
)
|
||||
.Callback(() =>
|
||||
{
|
||||
(this._collections[collection] ??= new()).Add(record);
|
||||
|
||||
this._mockDatabase
|
||||
.Setup<Task<HashEntry[]>>(x => x.HashGetAllAsync(It.Is<RedisKey>(x => x == redisKey), It.IsAny<CommandFlags>()))
|
||||
.ReturnsAsync(new[] {
|
||||
new HashEntry("key", record.Key),
|
||||
new HashEntry("metadata", record.GetSerializedMetadata()),
|
||||
new HashEntry("embedding", embedding),
|
||||
new HashEntry("timestamp", timestamp)
|
||||
});
|
||||
|
||||
callback?.Invoke();
|
||||
});
|
||||
}
|
||||
|
||||
private void MockKeyDelete(string collection, string key, Action? callback = null)
|
||||
{
|
||||
string redisKey = $"{collection}:{key}";
|
||||
|
||||
this._mockDatabase
|
||||
.Setup<Task<bool>>(x => x.KeyDeleteAsync(
|
||||
It.Is<RedisKey>(x => x == redisKey),
|
||||
It.IsAny<CommandFlags>())
|
||||
)
|
||||
.ReturnsAsync(true)
|
||||
.Callback(() =>
|
||||
{
|
||||
(this._collections[collection] ??= new()).RemoveAll(x => x.Key == key);
|
||||
|
||||
this._mockDatabase
|
||||
.Setup<Task<HashEntry[]>>(x => x.HashGetAllAsync(It.Is<RedisKey>(x => x == redisKey), It.IsAny<CommandFlags>()))
|
||||
.ReturnsAsync(Array.Empty<HashEntry>());
|
||||
|
||||
callback?.Invoke();
|
||||
});
|
||||
}
|
||||
|
||||
private void MockKeyDelete(string collection, IEnumerable<string> keys, Action? callback = null)
|
||||
{
|
||||
RedisKey[] redisKeys = keys.Distinct().Select(key => new RedisKey($"{collection}:{key}")).ToArray();
|
||||
|
||||
this._mockDatabase
|
||||
.Setup<Task<long>>(x => x.KeyDeleteAsync(
|
||||
It.Is<RedisKey[]>(x => redisKeys.SequenceEqual(x)),
|
||||
It.IsAny<CommandFlags>())
|
||||
)
|
||||
.ReturnsAsync(redisKeys.Length)
|
||||
.Callback(() =>
|
||||
{
|
||||
(this._collections[collection] ??= new()).RemoveAll(x => keys.Contains(x.Key));
|
||||
|
||||
foreach (var redisKey in redisKeys)
|
||||
{
|
||||
this._mockDatabase
|
||||
.Setup<Task<HashEntry[]>>(x => x.HashGetAllAsync(It.Is<RedisKey>(x => x == redisKey), It.IsAny<CommandFlags>()))
|
||||
.ReturnsAsync(Array.Empty<HashEntry>());
|
||||
}
|
||||
|
||||
callback?.Invoke();
|
||||
});
|
||||
}
|
||||
|
||||
private void MockSearch(string collection, Embedding<float> compareEmbedding, int topN, double threshold, bool returnStringVectorScore = false)
|
||||
{
|
||||
TopNCollection<MemoryRecord> embeddings = new(topN);
|
||||
|
||||
List<MemoryRecord> records = this._collections.TryGetValue(collection, out var value) ? value : new();
|
||||
|
||||
foreach (var record in records)
|
||||
{
|
||||
double similarity = compareEmbedding
|
||||
.AsReadOnlySpan()
|
||||
.CosineSimilarity(record.Embedding.AsReadOnlySpan());
|
||||
if (similarity >= threshold)
|
||||
{
|
||||
embeddings.Add(new(record, similarity));
|
||||
}
|
||||
}
|
||||
|
||||
embeddings.SortByScore();
|
||||
|
||||
string redisKey = $"{collection}";
|
||||
|
||||
var redisResults = new List<RedisResult>();
|
||||
redisResults.Add(RedisResult.Create(embeddings.Count));
|
||||
|
||||
foreach (var item in embeddings)
|
||||
{
|
||||
long timestamp = item.Value.Timestamp?.ToUnixTimeMilliseconds() ?? -1;
|
||||
byte[] embedding = MemoryMarshal.Cast<float, byte>(item.Value.Embedding.AsReadOnlySpan()).ToArray();
|
||||
redisResults.Add(RedisResult.Create($"{collection}:{item.Value.Metadata.Id}", ResultType.BulkString));
|
||||
redisResults.Add(RedisResult.Create(
|
||||
new RedisResult[]
|
||||
{
|
||||
RedisResult.Create("key", ResultType.BulkString),
|
||||
RedisResult.Create(item.Value.Metadata.Id, ResultType.BulkString),
|
||||
RedisResult.Create("metadata", ResultType.BulkString),
|
||||
RedisResult.Create(item.Value.GetSerializedMetadata(), ResultType.BulkString),
|
||||
RedisResult.Create("embedding", ResultType.BulkString),
|
||||
RedisResult.Create(embedding, ResultType.BulkString),
|
||||
RedisResult.Create("timestamp", ResultType.BulkString),
|
||||
RedisResult.Create(timestamp, ResultType.BulkString),
|
||||
RedisResult.Create("vector_score", ResultType.BulkString),
|
||||
RedisResult.Create(returnStringVectorScore ? $"score:{1-item.Score.Value}" : 1-item.Score.Value, ResultType.BulkString),
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
this._mockDatabase
|
||||
.Setup<Task<RedisResult>>(x => x.ExecuteAsync(
|
||||
It.Is<string>(x => x == "FT.SEARCH"),
|
||||
It.Is<object[]>(x => x[0].ToString() == collection && x[1].ToString() == $"*=>[KNN {topN} @embedding $embedding AS vector_score]"))
|
||||
)
|
||||
.ReturnsAsync(RedisResult.Create(redisResults.ToArray()));
|
||||
}
|
||||
|
||||
private IEnumerable<MemoryRecord> CreateBatchRecords(int numRecords)
|
||||
{
|
||||
Assert.True(numRecords % 2 == 0, "Number of records must be even");
|
||||
Assert.True(numRecords > 0, "Number of records must be greater than 0");
|
||||
|
||||
IEnumerable<MemoryRecord> records = new List<MemoryRecord>(numRecords);
|
||||
for (int i = 0; i < numRecords / 2; i++)
|
||||
{
|
||||
var testRecord = MemoryRecord.LocalRecord(
|
||||
id: "test" + i,
|
||||
text: "text" + i,
|
||||
description: "description" + i,
|
||||
embedding: new Embedding<float>(new float[] { 1, 1, 1 }));
|
||||
records = records.Append(testRecord);
|
||||
}
|
||||
|
||||
for (int i = numRecords / 2; i < numRecords; i++)
|
||||
{
|
||||
var testRecord = MemoryRecord.ReferenceRecord(
|
||||
externalId: "test" + i,
|
||||
sourceName: "sourceName" + i,
|
||||
description: "description" + i,
|
||||
embedding: new Embedding<float>(new float[] { 1, 2, 3 }));
|
||||
records = records.Append(testRecord);
|
||||
}
|
||||
|
||||
return records;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.SemanticKernel;
|
||||
using Microsoft.SemanticKernel.Connectors.Memory.Redis;
|
||||
using Microsoft.SemanticKernel.Memory;
|
||||
using RepoUtils;
|
||||
using StackExchange.Redis;
|
||||
|
||||
// ReSharper disable once InconsistentNaming
|
||||
public static class Example47_Redis
|
||||
{
|
||||
private const string MemoryCollectionName = "redis-test";
|
||||
|
||||
public static async Task RunAsync()
|
||||
{
|
||||
string configuration = Env.Var("REDIS_CONFIGURATION");
|
||||
using ConnectionMultiplexer connectionMultiplexer = await ConnectionMultiplexer.ConnectAsync(configuration);
|
||||
IDatabase database = connectionMultiplexer.GetDatabase();
|
||||
RedisMemoryStore memoryStore = new(database, vectorSize: 1536);
|
||||
IKernel kernel = Kernel.Builder
|
||||
.WithLogger(ConsoleLogger.Log)
|
||||
.WithOpenAITextCompletionService("text-davinci-003", Env.Var("OPENAI_API_KEY"))
|
||||
.WithOpenAITextEmbeddingGenerationService("text-embedding-ada-002", Env.Var("OPENAI_API_KEY"))
|
||||
.WithMemoryStorage(memoryStore)
|
||||
.Build();
|
||||
|
||||
Console.WriteLine("== Printing Collections in DB ==");
|
||||
var collections = memoryStore.GetCollectionsAsync();
|
||||
await foreach (var collection in collections)
|
||||
{
|
||||
Console.WriteLine(collection);
|
||||
}
|
||||
|
||||
Console.WriteLine("== Adding Memories ==");
|
||||
|
||||
var key1 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat1", text: "british short hair");
|
||||
var key2 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat2", text: "orange tabby");
|
||||
var key3 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat3", text: "norwegian forest cat");
|
||||
|
||||
Console.WriteLine("== Printing Collections in DB ==");
|
||||
collections = memoryStore.GetCollectionsAsync();
|
||||
await foreach (var collection in collections)
|
||||
{
|
||||
Console.WriteLine(collection);
|
||||
}
|
||||
|
||||
Console.WriteLine("== Retrieving Memories Through the Kernel ==");
|
||||
MemoryQueryResult? lookup = await kernel.Memory.GetAsync(MemoryCollectionName, "cat1");
|
||||
Console.WriteLine(lookup != null ? lookup.Metadata.Text : "ERROR: memory not found");
|
||||
|
||||
Console.WriteLine("== Retrieving Memories Directly From the Store ==");
|
||||
var memory1 = await memoryStore.GetAsync(MemoryCollectionName, key1);
|
||||
var memory2 = await memoryStore.GetAsync(MemoryCollectionName, key2);
|
||||
var memory3 = await memoryStore.GetAsync(MemoryCollectionName, key3);
|
||||
Console.WriteLine(memory1 != null ? memory1.Metadata.Text : "ERROR: memory not found");
|
||||
Console.WriteLine(memory2 != null ? memory2.Metadata.Text : "ERROR: memory not found");
|
||||
Console.WriteLine(memory3 != null ? memory3.Metadata.Text : "ERROR: memory not found");
|
||||
|
||||
Console.WriteLine("== Similarity Searching Memories: My favorite color is orange ==");
|
||||
var searchResults = kernel.Memory.SearchAsync(MemoryCollectionName, "My favorite color is orange", limit: 3, minRelevanceScore: 0.8);
|
||||
|
||||
await foreach (var item in searchResults)
|
||||
{
|
||||
Console.WriteLine(item.Metadata.Text + " : " + item.Relevance);
|
||||
}
|
||||
|
||||
Console.WriteLine("== Removing Collection {0} ==", MemoryCollectionName);
|
||||
await memoryStore.DeleteCollectionAsync(MemoryCollectionName);
|
||||
|
||||
Console.WriteLine("== Printing Collections in DB ==");
|
||||
await foreach (var collection in collections)
|
||||
{
|
||||
Console.WriteLine(collection);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,7 @@
|
||||
<ProjectReference Include="..\..\..\dotnet\src\Connectors\Connectors.Memory.AzureCognitiveSearch\Connectors.Memory.AzureCognitiveSearch.csproj" />
|
||||
<ProjectReference Include="..\..\..\dotnet\src\Connectors\Connectors.Memory.Postgres\Connectors.Memory.Postgres.csproj" />
|
||||
<ProjectReference Include="..\..\..\dotnet\src\Connectors\Connectors.Memory.Weaviate\Connectors.Memory.Weaviate.csproj" />
|
||||
<ProjectReference Include="..\..\..\dotnet\src\Connectors\Connectors.Memory.Redis\Connectors.Memory.Redis.csproj" />
|
||||
<ProjectReference Include="..\..\..\dotnet\src\Extensions\Planning.ActionPlanner\Planning.ActionPlanner.csproj" />
|
||||
<ProjectReference Include="..\..\..\dotnet\src\Extensions\Planning.SequentialPlanner\Planning.SequentialPlanner.csproj" />
|
||||
<ProjectReference Include="..\..\..\dotnet\src\Connectors\Connectors.Memory.Pinecone\Connectors.Memory.Pinecone.csproj" />
|
||||
|
||||
@@ -123,6 +123,7 @@ public static class Program
|
||||
Console.WriteLine("== DONE ==");
|
||||
|
||||
await Example39_Postgres.RunAsync();
|
||||
Console.WriteLine("== DONE ==");
|
||||
|
||||
await Example40_DIContainer.RunAsync();
|
||||
Console.WriteLine("== DONE ==");
|
||||
@@ -144,5 +145,8 @@ public static class Program
|
||||
|
||||
await Example46_Weaviate.RunAsync();
|
||||
Console.WriteLine("== DONE ==");
|
||||
|
||||
await Example47_Redis.RunAsync();
|
||||
Console.WriteLine("== DONE ==");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ dotnet user-secrets set "WEAVIATE_PORT" "..."
|
||||
dotnet user-secrets set "WEAVIATE_APIKEY" "..."
|
||||
dotnet user-secrets set "GITHUB_PERSONAL_ACCESS_TOKEN" "github_pat_..."
|
||||
dotnet user-secrets set "POSTGRES_CONNECTIONSTRING" "..."
|
||||
dotnet user-secrets set "REDIS_CONFIGURATION" "..."
|
||||
```
|
||||
|
||||
To set your secrets with environment variables, use these names:
|
||||
@@ -56,6 +57,7 @@ To set your secrets with environment variables, use these names:
|
||||
* WEAVIATE_APIKEY
|
||||
* GITHUB_PERSONAL_ACCESS_TOKEN
|
||||
* POSTGRES_CONNECTIONSTRING
|
||||
* REDIS_CONFIGURATION
|
||||
* AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME
|
||||
* AZURE_OPENAI_EMBEDDINGS_ENDPOINT
|
||||
* AZURE_OPENAI_EMBEDDINGS_KEY
|
||||
|
||||
Reference in New Issue
Block a user