diff --git a/FEATURE_MATRIX.md b/FEATURE_MATRIX.md index 56f27cd1e8..d14056f86d 100644 --- a/FEATURE_MATRIX.md +++ b/FEATURE_MATRIX.md @@ -68,6 +68,7 @@ | Sqlite | ✅ | ❌ | ❌ | Vector optimization requires [sqlite-vss](https://github.com/asg017/sqlite-vss) | | Postgres | ✅ | ❌ | ❌ | Vector optimization requires [pgvector](https://github.com/pgvector/pgvector) | | CosmosDB | ✅ | ❌ | ❌ | CosmosDB is not optimized for vector storage | +| Redis | ✅ | ❌ | ❌ | Vector optimization requires [RediSearch](https://redis.io/docs/stack/search) | ## Connectors and Skill Libraries diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 8f2c8edc39..f55a98f69e 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -8,6 +8,7 @@ + diff --git a/dotnet/SK-dotnet.sln b/dotnet/SK-dotnet.sln index 1d4bcf0c14..1ebb115647 100644 --- a/dotnet/SK-dotnet.sln +++ b/dotnet/SK-dotnet.sln @@ -72,6 +72,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.CosmosDB" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.Postgres", "src\Connectors\Connectors.Memory.Postgres\Connectors.Memory.Postgres.csproj", "{C9F957FA-A70F-4A6D-8F95-23FCD7F4FB87}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.Redis", "src\Connectors\Connectors.Memory.Redis\Connectors.Memory.Redis.csproj", "{3720F5ED-FB4D-485E-8A93-CDE60DEF0805}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.AI.OpenAI", "src\Connectors\Connectors.AI.OpenAI\Connectors.AI.OpenAI.csproj", "{AFA81EB7-F869-467D-8A90-744305D80AAC}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SemanticKernel.Abstractions", "src\SemanticKernel.Abstractions\SemanticKernel.Abstractions.csproj", "{627742DB-1E52-468A-99BD-6FF1A542D25B}" @@ -304,6 +306,12 @@ Global {4D91A3E0-C404-495B-AD4A-411C4E83CF54}.Publish|Any CPU.Build.0 = Release|Any CPU {4D91A3E0-C404-495B-AD4A-411C4E83CF54}.Release|Any CPU.ActiveCfg = Release|Any CPU {4D91A3E0-C404-495B-AD4A-411C4E83CF54}.Release|Any CPU.Build.0 = Release|Any CPU + {3720F5ED-FB4D-485E-8A93-CDE60DEF0805}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3720F5ED-FB4D-485E-8A93-CDE60DEF0805}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3720F5ED-FB4D-485E-8A93-CDE60DEF0805}.Publish|Any CPU.ActiveCfg = Debug|Any CPU + {3720F5ED-FB4D-485E-8A93-CDE60DEF0805}.Publish|Any CPU.Build.0 = Debug|Any CPU + {3720F5ED-FB4D-485E-8A93-CDE60DEF0805}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3720F5ED-FB4D-485E-8A93-CDE60DEF0805}.Release|Any CPU.Build.0 = Release|Any CPU {50FAE231-6F24-4779-9D02-12ABBC9A49E2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {50FAE231-6F24-4779-9D02-12ABBC9A49E2}.Debug|Any CPU.Build.0 = Debug|Any CPU {50FAE231-6F24-4779-9D02-12ABBC9A49E2}.Publish|Any CPU.ActiveCfg = Publish|Any CPU @@ -353,6 +361,7 @@ Global {4D3DAE63-41C6-4E1C-A35A-E77BDFC40675} = {831DDCA2-7D2C-4C31-80DB-6BDB3E1F7AE0} {6AAB0620-33A1-4A98-A63B-6560B9BA47A4} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} {4D91A3E0-C404-495B-AD4A-411C4E83CF54} = {FA3720F1-C99A-49B2-9577-A940257098BF} + {3720F5ED-FB4D-485E-8A93-CDE60DEF0805} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} {50FAE231-6F24-4779-9D02-12ABBC9A49E2} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/Connectors.Memory.Redis.csproj b/dotnet/src/Connectors/Connectors.Memory.Redis/Connectors.Memory.Redis.csproj new file mode 100644 index 0000000000..bbf205a041 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Redis/Connectors.Memory.Redis.csproj @@ -0,0 +1,28 @@ + + + + + Microsoft.SemanticKernel.Connectors.Memory.Redis + $(AssemblyName) + netstandard2.0 + + + + + + + + + Semantic Kernel - Redis Connector + Redis connector for Semantic Kernel skills and semantic memory + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/README.md b/dotnet/src/Connectors/Connectors.Memory.Redis/README.md new file mode 100644 index 0000000000..296925394d --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Redis/README.md @@ -0,0 +1,32 @@ +# Microsoft.SemanticKernel.Connectors.Memory.Redis + +This connector uses Redis to implement Semantic Memory. It requires the [RediSearch](https://redis.io/docs/stack/search) module to be enabled on Redis to implement vector similarity search. + +## What is RediSearch? + +[RediSearch](https://redis.io/docs/stack/search) is a source-available Redis module that enables querying, secondary indexing, and full-text search for Redis. These features enable multi-field queries, aggregation, exact phrase matching, numeric filtering, geo filtering and vector similarity semantic search on top of text queries. + +How to set up the RediSearch, please refer to its [documentation](https://redis.io/docs/stack/search/quick_start/). +Or use the [Redis Enterprise](https://redis.io/docs/about/redis-enterprise/), see [Azure Marketplace](https://azuremarketplace.microsoft.com/en-us/marketplace/apps/garantiadata.redis_enterprise_1sp_public_preview?tab=Overview), [AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-e6y7ork67pjwg?sr=0-2&ref_=beagle&applicationId=AWSMPContessa), or [Google Marketplace](https://console.cloud.google.com/marketplace/details/redislabs-public/redis-enterprise?pli=1). + +## Quick start + +1. Run with Docker: + +```bash +docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest +``` + +2. To use Redis as a semantic memory store: + +```csharp +using ConnectionMultiplexer connectionMultiplexer = await ConnectionMultiplexer.ConnectAsync("localhost:6379"); +IDatabase database = connectionMultiplexer.GetDatabase(); +RedisMemoryStore memoryStore = new RedisMemoryStore(database, vectorSize: 1536); + +IKernel kernel = Kernel.Builder + .WithLogger(ConsoleLogger.Log) + .WithOpenAITextEmbeddingGenerationService("text-embedding-ada-002", Env.Var("OPENAI_API_KEY")) + .WithMemoryStorage(memoryStore) + .Build(); +``` diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisMemoryStore.cs b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisMemoryStore.cs new file mode 100644 index 0000000000..265a7e4b30 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisMemoryStore.cs @@ -0,0 +1,299 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.AI.Embeddings; +using Microsoft.SemanticKernel.Memory; +using NRedisStack; +using NRedisStack.RedisStackCommands; +using NRedisStack.Search; +using NRedisStack.Search.Literals.Enums; +using StackExchange.Redis; + +namespace Microsoft.SemanticKernel.Connectors.Memory.Redis; + +/// +/// An implementation of for Redis. +/// +/// The embedded data is saved to the Redis server database specified in the constructor. +/// Similarity search capability is provided through the RediSearch module. Use RediSearch's "Index" to implement "Collection". +/// +public sealed class RedisMemoryStore : IMemoryStore +{ + /// + /// Create a new instance of semantic memory using Redis. + /// + /// The database of the redis server. + /// Embedding vector size + public RedisMemoryStore(IDatabase database, int vectorSize) + { + this._database = database; + this._vectorSize = vectorSize; + this._ft = database.FT(); + } + + /// + public async IAsyncEnumerable GetCollectionsAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + foreach (var index in await this._ft._ListAsync().ConfigureAwait(false)) + { + yield return ((string)index!); + } + } + + /// + public async Task CreateCollectionAsync(string collectionName, CancellationToken cancellationToken = default) + { + FTCreateParams ftCreateParams = FTCreateParams.CreateParams().On(IndexDataType.HASH).Prefix($"{collectionName}:"); + Schema schema = new Schema() + .AddTextField("key") + .AddTextField("metadata") + .AddNumericField("timestamp") + .AddVectorField("embedding", VECTOR_INDEX_ALGORITHM, new Dictionary { + {"TYPE", VECTOR_TYPE}, + {"DIM", this._vectorSize}, + {"DISTANCE_METRIC", VECTOR_DISTANCE_METRIC}, + }); + + await this._ft.CreateAsync(collectionName, ftCreateParams, schema).ConfigureAwait(false); + } + + /// + public async Task DoesCollectionExistAsync(string collectionName, CancellationToken cancellationToken = default) + { + try + { + await this._ft.InfoAsync(collectionName).ConfigureAwait(false); + return true; + } + catch (RedisServerException ex) when (ex.Message == MESSAGE_WHEN_INDEX_DOES_NOT_EXIST) + { + return false; + } + } + + /// + public async Task DeleteCollectionAsync(string collectionName, CancellationToken cancellationToken = default) + { + // dd: If `true`, all documents will be deleted. + await this._ft.DropIndexAsync(collectionName, dd: true).ConfigureAwait(false); + } + + /// + public async Task GetAsync(string collectionName, string key, bool withEmbedding = false, CancellationToken cancellationToken = default) + { + return await this.InternalGetAsync(collectionName, key, withEmbedding, cancellationToken).ConfigureAwait(false); + } + + /// + public async IAsyncEnumerable GetBatchAsync(string collectionName, IEnumerable keys, bool withEmbeddings = false, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + foreach (var key in keys) + { + var result = await this.InternalGetAsync(collectionName, key, withEmbeddings, cancellationToken).ConfigureAwait(false); + if (result != null) + { + yield return result; + } + } + } + + /// + public async Task UpsertAsync(string collectionName, MemoryRecord record, CancellationToken cancellationToken = default) + { + record.Key = record.Metadata.Id; + + await this._database.HashSetAsync(GetRedisKey(collectionName, record.Key), new[] { + new HashEntry("key", record.Key), + new HashEntry("metadata", record.GetSerializedMetadata()), + new HashEntry("embedding", MemoryMarshal.Cast(record.Embedding.AsReadOnlySpan()).ToArray()), + new HashEntry("timestamp", ToTimestampLong(record.Timestamp)) + }, flags: CommandFlags.None).ConfigureAwait(false); + + return record.Key; + } + + /// + public async IAsyncEnumerable UpsertBatchAsync(string collectionName, IEnumerable records, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + foreach (var record in records) + { + yield return await this.UpsertAsync(collectionName, record, cancellationToken).ConfigureAwait(false); + } + } + + /// + public async Task RemoveAsync(string collectionName, string key, CancellationToken cancellationToken = default) + { + await this._database.KeyDeleteAsync(GetRedisKey(collectionName, key), flags: CommandFlags.None).ConfigureAwait(false); + } + + /// + public async Task RemoveBatchAsync(string collectionName, IEnumerable keys, CancellationToken cancellationToken = default) + { + await this._database.KeyDeleteAsync(keys.Select(key => GetRedisKey(collectionName, key)).ToArray(), flags: CommandFlags.None).ConfigureAwait(false); + } + + /// + public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync( + string collectionName, + Embedding embedding, + int limit, + double minRelevanceScore = 0, + bool withEmbeddings = false, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + if (limit <= 0) + { + yield break; + } + + var query = new Query($"*=>[KNN {limit} @embedding $embedding AS vector_score]") + .AddParam("embedding", MemoryMarshal.Cast(embedding.AsReadOnlySpan()).ToArray()) + .SetSortBy("vector_score") + .ReturnFields("key", "metadata", "embedding", "timestamp", "vector_score") + .Limit(0, limit) + .Dialect(QUERY_DIALECT); + + var results = await this._ft.SearchAsync(collectionName, query).ConfigureAwait(false); + + foreach (var document in results.Documents) + { + double similarity = this.GetSimilarity(document); + if (similarity < minRelevanceScore) + { + yield break; + } + + Embedding convertedEmbedding = withEmbeddings && document["embedding"].HasValue + ? + new Embedding(MemoryMarshal.Cast((byte[])document["embedding"]!).ToArray()) + : + Embedding.Empty; + + yield return (MemoryRecord.FromJsonMetadata( + json: document["metadata"]!, + embedding: convertedEmbedding, + key: document["key"], + timestamp: ParseTimestamp((long?)document["timestamp"])), similarity); + } + } + + /// + public async Task<(MemoryRecord, double)?> GetNearestMatchAsync(string collectionName, Embedding embedding, double minRelevanceScore = 0, bool withEmbedding = false, + CancellationToken cancellationToken = default) + { + return await this.GetNearestMatchesAsync( + collectionName: collectionName, + embedding: embedding, + limit: 1, + minRelevanceScore: minRelevanceScore, + withEmbeddings: withEmbedding, + cancellationToken: cancellationToken).FirstOrDefaultAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + } + + #region constants ================================================================================ + + /// + /// Vector similarity index algorithm. The default value is "HNSW". + /// + /// + internal const Schema.VectorField.VectorAlgo VECTOR_INDEX_ALGORITHM = Schema.VectorField.VectorAlgo.HNSW; + + /// + /// Vector type. Supported types are FLOAT32 and FLOAT64. The default value is "FLOAT32". + /// + internal const string VECTOR_TYPE = "FLOAT32"; + + /// + /// Supported distance metric, one of {L2, IP, COSINE}. The default value is "COSINE". + /// + internal const string VECTOR_DISTANCE_METRIC = "COSINE"; + + /// + /// Query dialect. To use a vector similarity query, specify DIALECT 2 or higher. The default value is "2". + /// + /// + internal const int QUERY_DIALECT = 2; + + /// + /// Message when index does not exist. + /// + /// + internal const string MESSAGE_WHEN_INDEX_DOES_NOT_EXIST = "Unknown Index name"; + + #endregion + + #region private ================================================================================ + + private readonly IDatabase _database; + private readonly int _vectorSize; + private readonly SearchCommands _ft; + + private static long ToTimestampLong(DateTimeOffset? timestamp) + { + if (timestamp.HasValue) + { + return timestamp.Value.ToUnixTimeMilliseconds(); + } + return -1; + } + + private static DateTimeOffset? ParseTimestamp(long? timestamp) + { + if (timestamp.HasValue && timestamp > 0) + { + return DateTimeOffset.FromUnixTimeMilliseconds(timestamp.Value); + } + + return null; + } + + private static RedisKey GetRedisKey(string collectionName, string key) + { + return new RedisKey($"{collectionName}:{key}"); + } + + private async Task InternalGetAsync(string collectionName, string key, bool withEmbedding, CancellationToken cancellationToken) + { + HashEntry[] hashEntries = await this._database.HashGetAllAsync(GetRedisKey(collectionName, key), flags: CommandFlags.None).ConfigureAwait(false); + + if (hashEntries.Length == 0) { return null; } + + if (withEmbedding) + { + RedisValue embedding = hashEntries.FirstOrDefault(x => x.Name == "embedding").Value; + return MemoryRecord.FromJsonMetadata( + json: hashEntries.FirstOrDefault(x => x.Name == "metadata").Value!, + embedding: embedding.HasValue ? new Embedding(MemoryMarshal.Cast((byte[])embedding!).ToArray()) : Embedding.Empty, + key: hashEntries.FirstOrDefault(x => x.Name == "key").Value, + timestamp: ParseTimestamp((long?)hashEntries.FirstOrDefault(x => x.Name == "timestamp").Value)); + } + + return MemoryRecord.FromJsonMetadata( + json: hashEntries.FirstOrDefault(x => x.Name == "metadata").Value!, + embedding: Embedding.Empty, + key: hashEntries.FirstOrDefault(x => x.Name == "key").Value, + timestamp: ParseTimestamp((long?)hashEntries.FirstOrDefault(x => x.Name == "timestamp").Value)); + } + + private double GetSimilarity(Document document) + { + RedisValue vectorScoreValue = document["vector_score"]; + + if (vectorScoreValue.IsNullOrEmpty || !vectorScoreValue.TryParse(out double vectorScore)) + { + throw new RedisMemoryStoreException("Invalid or missing vector score value."); + } + + return 1 - vectorScore; + } + + #endregion +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisMemoryStoreException.cs b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisMemoryStoreException.cs new file mode 100644 index 0000000000..80454a9bee --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisMemoryStoreException.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; + +namespace Microsoft.SemanticKernel.Connectors.Memory.Redis; + +#pragma warning disable RCS1194 // Implement exception constructors + +/// +/// Exception thrown by the Redis connector +/// +public class RedisMemoryStoreException : Exception +{ + /// + /// Initializes a new instance of the class. + /// + /// Exception message. + internal RedisMemoryStoreException(string? message) : base(message) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Exception message. + /// Inner exception. + internal RedisMemoryStoreException(string? message, Exception? innerException) : base(message, innerException) + { + } +} diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Connectors.UnitTests.csproj b/dotnet/src/Connectors/Connectors.UnitTests/Connectors.UnitTests.csproj index bd8f9069ba..39cdb35867 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/Connectors.UnitTests.csproj +++ b/dotnet/src/Connectors/Connectors.UnitTests/Connectors.UnitTests.csproj @@ -31,6 +31,7 @@ + diff --git a/dotnet/src/Connectors/Connectors.UnitTests/Memory/Redis/RedisMemoryStoreTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Redis/RedisMemoryStoreTests.cs new file mode 100644 index 0000000000..2d2c3dad7c --- /dev/null +++ b/dotnet/src/Connectors/Connectors.UnitTests/Memory/Redis/RedisMemoryStoreTests.cs @@ -0,0 +1,986 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Runtime.InteropServices; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.AI.Embeddings; +using Microsoft.SemanticKernel.AI.Embeddings.VectorOperations; +using Microsoft.SemanticKernel.Connectors.Memory.Redis; +using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Memory.Collections; +using Moq; +using StackExchange.Redis; +using Xunit; + +namespace SemanticKernel.Connectors.UnitTests.Memory.Redis; + +/// +/// Unit tests of . +/// +public class RedisMemoryStoreTests +{ + private readonly Mock _mockDatabase; + private readonly Dictionary> _collections; + + public RedisMemoryStoreTests() + { + this._mockDatabase = new Mock(); + this._collections = new(); + } + + [Fact] + public void ConnectionCanBeInitialized() + { + // Arrange + RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + } + + [Fact] + public async Task ItCanCreateAndGetCollectionAsync() + { + // Arrange + RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + string collection = "test_collection"; + this.MockCreateIndex(collection); + + // Act + await store.CreateCollectionAsync(collection); + var collections = store.GetCollectionsAsync(); + + // Assert + Assert.NotEmpty(collections.ToEnumerable()); + Assert.True(await collections.ContainsAsync(collection)); + } + + [Fact] + public async Task ItCanCheckIfCollectionExistsAsync() + { + // Arrange + RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + string collection = "my_collection"; + this.MockCreateIndex(collection); + + // Act + await store.CreateCollectionAsync(collection); + + // Assert + Assert.True(await store.DoesCollectionExistAsync("my_collection")); + Assert.False(await store.DoesCollectionExistAsync("my_collection2")); + } + + [Fact] + public async Task CollectionsCanBeDeletedAsync() + { + // Arrange + RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + string collection = "test_collection"; + this.MockCreateIndex(collection, () => + { + this.MockDropIndex(collection); + }); + + await store.CreateCollectionAsync(collection); + var collections = await store.GetCollectionsAsync().ToListAsync(); + Assert.True(collections.Count > 0); + + // Act + foreach (var c in collections) + { + await store.DeleteCollectionAsync(c); + } + + // Assert + var collections2 = store.GetCollectionsAsync(); + Assert.True(await collections2.CountAsync() == 0); + } + + [Fact] + public async Task ItCanInsertIntoNonExistentCollectionAsync() + { + // Arrange + RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + MemoryRecord testRecord = MemoryRecord.LocalRecord( + id: "test", + text: "text", + description: "description", + embedding: new Embedding(new float[] { 1, 2, 3 }), + key: null, + timestamp: null); + string collection = "random collection"; + string redisKey = $"{collection}:{testRecord.Metadata.Id}"; + byte[] embedding = MemoryMarshal.Cast(testRecord.Embedding.AsReadOnlySpan()).ToArray(); + this._mockDatabase + .Setup(x => x.HashSetAsync( + It.Is(x => x == redisKey), + It.Is(x => x.Length == 4 && + x[0].Name == "key" && x[1].Name == "metadata" && x[2].Name == "embedding" && x[3].Name == "timestamp" && + x[0].Value == testRecord.Key && x[1].Value == testRecord.GetSerializedMetadata() && embedding.SequenceEqual((byte[])x[2].Value!) && x[3].Value == -1 + ), + It.IsAny()) + ) + .Callback(() => + { + this._mockDatabase + .Setup>(x => x.HashGetAllAsync(It.Is(x => x == redisKey), It.IsAny())) + .ReturnsAsync(new[] { + new HashEntry("key", testRecord.Key), + new HashEntry("metadata", testRecord.GetSerializedMetadata()), + new HashEntry("embedding", embedding), + new HashEntry("timestamp", -1) + }); + }); + + // Arrange + var key = await store.UpsertAsync(collection, testRecord); + var actual = await store.GetAsync(collection, key, true); + + // Assert + Assert.NotNull(actual); + Assert.Equal(testRecord.Metadata.Id, key); + Assert.Equal(testRecord.Metadata.Id, actual.Key); + Assert.Equal(testRecord.Embedding.Vector, actual.Embedding.Vector); + Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text); + Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description); + Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName); + Assert.Equal(testRecord.Metadata.Id, actual.Metadata.Id); + } + + [Fact] + public async Task GetAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync() + { + // Arrange + RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + MemoryRecord testRecord = MemoryRecord.LocalRecord( + id: "test", + text: "text", + description: "description", + embedding: new Embedding(new float[] { 1, 2, 3 }), + key: null, + timestamp: null); + string collection = "test_collection"; + string redisKey = $"{collection}:{testRecord.Metadata.Id}"; + + this.MockCreateIndex(collection, () => + { + this.MockHashSet(collection, testRecord); + }); + + // Act + await store.CreateCollectionAsync(collection); + var key = await store.UpsertAsync(collection, testRecord); + var actualDefault = await store.GetAsync(collection, key); + var actualWithEmbedding = await store.GetAsync(collection, key, true); + + // Assert + Assert.NotNull(actualDefault); + Assert.NotNull(actualWithEmbedding); + Assert.Empty(actualDefault.Embedding.Vector); + Assert.NotEmpty(actualWithEmbedding.Embedding.Vector); + } + + [Fact] + public async Task ItCanUpsertAndRetrieveARecordWithNoTimestampAsync() + { + // Arrange + RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + MemoryRecord testRecord = MemoryRecord.LocalRecord( + id: "test", + text: "text", + description: "description", + embedding: new Embedding(new float[] { 1, 2, 3 }), + key: null, + timestamp: null); + string collection = "test_collection"; + this.MockCreateIndex(collection, () => + { + this.MockHashSet(collection, testRecord); + }); + + // Act + await store.CreateCollectionAsync(collection); + var key = await store.UpsertAsync(collection, testRecord); + var actual = await store.GetAsync(collection, key, true); + + // Assert + Assert.NotNull(actual); + Assert.Equal(testRecord.Metadata.Id, key); + Assert.Equal(testRecord.Metadata.Id, actual.Key); + Assert.Equal(testRecord.Embedding.Vector, actual.Embedding.Vector); + Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text); + Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description); + Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName); + Assert.Equal(testRecord.Metadata.Id, actual.Metadata.Id); + } + + [Fact] + public async Task ItCanUpsertAndRetrieveARecordWithTimestampAsync() + { + // Arrange + RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + MemoryRecord testRecord = MemoryRecord.LocalRecord( + id: "test", + text: "text", + description: "description", + embedding: new Embedding(new float[] { 1, 2, 3 }), + key: null, + timestamp: DateTimeOffset.UtcNow); + string collection = "test_collection"; + this.MockCreateIndex(collection, () => + { + this.MockHashSet(collection, testRecord); + }); + + // Act + await store.CreateCollectionAsync(collection); + var key = await store.UpsertAsync(collection, testRecord); + var actual = await store.GetAsync(collection, key, true); + + // Assert + Assert.NotNull(actual); + Assert.Equal(testRecord.Metadata.Id, key); + Assert.Equal(testRecord.Metadata.Id, actual.Key); + Assert.Equal(testRecord.Embedding.Vector, actual.Embedding.Vector); + Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text); + Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description); + Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName); + Assert.Equal(testRecord.Metadata.Id, actual.Metadata.Id); + } + + [Fact] + public async Task UpsertReplacesExistingRecordWithSameIdAsync() + { + // Arrange + RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + string commonId = "test"; + MemoryRecord testRecord = MemoryRecord.LocalRecord( + id: commonId, + text: "text", + description: "description", + embedding: new Embedding(new float[] { 1, 2, 3 })); + MemoryRecord testRecord2 = MemoryRecord.LocalRecord( + id: commonId, + text: "text2", + description: "description2", + embedding: new Embedding(new float[] { 1, 2, 4 })); + string collection = "test_collection"; + this.MockCreateIndex(collection, () => + { + this.MockHashSet(collection, testRecord); + this.MockHashSet(collection, testRecord2); + }); + + // Act + await store.CreateCollectionAsync(collection); + var key = await store.UpsertAsync(collection, testRecord); + var key2 = await store.UpsertAsync(collection, testRecord2); + var actual = await store.GetAsync(collection, key, true); + + // Assert + Assert.NotNull(actual); + Assert.Equal(testRecord.Metadata.Id, key); + Assert.Equal(testRecord2.Metadata.Id, actual.Key); + Assert.NotEqual(testRecord.Embedding.Vector, actual.Embedding.Vector); + Assert.Equal(testRecord2.Embedding.Vector, actual.Embedding.Vector); + Assert.NotEqual(testRecord.Metadata.Text, actual.Metadata.Text); + Assert.Equal(testRecord2.Metadata.Description, actual.Metadata.Description); + } + + [Fact] + public async Task ExistingRecordCanBeRemovedAsync() + { + // Arrange + RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + MemoryRecord testRecord = MemoryRecord.LocalRecord( + id: "test", + text: "text", + description: "description", + embedding: new Embedding(new float[] { 1, 2, 3 })); + string collection = "test_collection"; + this.MockCreateIndex(collection, () => + { + this.MockHashSet(collection, testRecord, () => + { + this.MockKeyDelete(collection, testRecord.Metadata.Id); + }); + }); + + // Act + await store.CreateCollectionAsync(collection); + var key = await store.UpsertAsync(collection, testRecord); + await store.RemoveAsync(collection, key); + var actual = await store.GetAsync(collection, key); + + // Assert + Assert.Null(actual); + } + + [Fact] + public async Task RemovingNonExistingRecordDoesNothingAsync() + { + // Arrange + RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + string collection = "test_collection"; + this.MockCreateIndex(collection, () => + { + this.MockKeyDelete(collection, "key"); + }); + + // Act + await store.CreateCollectionAsync(collection); + await store.RemoveAsync(collection, "key"); + var actual = await store.GetAsync(collection, "key"); + + // Assert + Assert.Null(actual); + } + + [Fact] + public async Task ItCanListAllDatabaseCollectionsAsync() + { + // Arrange + RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + string[] testCollections = { "random_collection1", "random_collection2", "random_collection3" }; + foreach (var collection in testCollections) + { + this.MockCreateIndex(collection, () => + { + this.MockDropIndex(collection); + }); + } + await store.CreateCollectionAsync(testCollections[0]); + await store.CreateCollectionAsync(testCollections[1]); + await store.CreateCollectionAsync(testCollections[2]); + + // Act + var collections = await store.GetCollectionsAsync().ToListAsync(); + + // Assert + foreach (var collection in testCollections) + { + Assert.True(await store.DoesCollectionExistAsync(collection)); + } + + Assert.NotNull(collections); + Assert.NotEmpty(collections); + Assert.Equal(testCollections.Length, collections.Count); + Assert.True(collections.Contains(testCollections[0]), + $"Collections does not contain the newly-created collection {testCollections[0]}"); + Assert.True(collections.Contains(testCollections[1]), + $"Collections does not contain the newly-created collection {testCollections[1]}"); + Assert.True(collections.Contains(testCollections[2]), + $"Collections does not contain the newly-created collection {testCollections[2]}"); + } + + [Fact] + public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() + { + // Arrange + RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + string collection = "test_collection"; + int topN = 4; + double threshold = -1; + var testEmbeddings = new[] + { + new Embedding(new float[] { 1, 1, 1 }), + new Embedding(new float[] { -1, -1, -1 }), + new Embedding(new float[] { 1, 2, 3 }), + new Embedding(new float[] { -1, -2, -3 }), + new Embedding(new float[] { 1, -1, -2 }) + }; + var testRecords = new List(); + for (int i = 0; i < testEmbeddings.Length; i++) + { + testRecords.Add(MemoryRecord.LocalRecord( + id: "test" + i, + text: "text" + i, + description: "description" + i, + embedding: testEmbeddings[i])); + } + this.MockCreateIndex(collection, () => + { + for (int i = 0; i < testRecords.Count; i++) + { + if (i + 1 < testRecords.Count) + { + this.MockHashSet(collection, testRecords[i]); + } + else + { + this.MockHashSet(collection, testRecords[i], () => this.MockSearch(collection, compareEmbedding, topN, threshold)); + } + } + }); + await store.CreateCollectionAsync(collection); + foreach (var testRecord in testRecords) + { + _ = await store.UpsertAsync(collection, testRecord); + } + + // Act + var topNResults = store.GetNearestMatchesAsync(collection, compareEmbedding, limit: topN, minRelevanceScore: threshold).ToEnumerable().ToArray(); + + // Assert + Assert.Equal(topN, topNResults.Length); + for (int j = 0; j < topN - 1; j++) + { + int compare = topNResults[j].Item2.CompareTo(topNResults[j + 1].Item2); + Assert.True(compare >= 0); + } + } + + [Fact] + public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync() + { + // Arrange + RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + string collection = "test_collection"; + int topN = 1; + double threshold = 0.75; + var testEmbeddings = new[] + { + new Embedding(new float[] { 1, 1, 1 }), + new Embedding(new float[] { -1, -1, -1 }), + new Embedding(new float[] { 1, 2, 3 }), + new Embedding(new float[] { -1, -2, -3 }), + new Embedding(new float[] { 1, -1, -2 }) + }; + var testRecords = new List(); + for (int i = 0; i < testEmbeddings.Length; i++) + { + testRecords.Add(MemoryRecord.LocalRecord( + id: "test" + i, + text: "text" + i, + description: "description" + i, + embedding: testEmbeddings[i])); + } + this.MockCreateIndex(collection, () => + { + for (int i = 0; i < testRecords.Count; i++) + { + if (i + 1 < testRecords.Count) + { + this.MockHashSet(collection, testRecords[i]); + } + else + { + this.MockHashSet(collection, testRecords[i], () => + { + this.MockSearch(collection, compareEmbedding, topN, threshold); + }); + } + } + }); + await store.CreateCollectionAsync(collection); + foreach (var testRecord in testRecords) + { + _ = await store.UpsertAsync(collection, testRecord); + } + + // Act + var topNResultDefault = await store.GetNearestMatchAsync(collection, compareEmbedding, minRelevanceScore: threshold); + var topNResultWithEmbedding = await store.GetNearestMatchAsync(collection, compareEmbedding, minRelevanceScore: threshold, withEmbedding: true); + + // Assert + Assert.NotNull(topNResultDefault); + Assert.NotNull(topNResultWithEmbedding); + Assert.Empty(topNResultDefault.Value.Item1.Embedding.Vector); + Assert.NotEmpty(topNResultWithEmbedding.Value.Item1.Embedding.Vector); + } + + [Fact] + public async Task GetNearestMatchAsyncReturnsExpectedAsync() + { + // Arrange + RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + string collection = "test_collection"; + int topN = 1; + double threshold = 0.75; + var testEmbeddings = new[] + { + new Embedding(new float[] { 1, 1, 1 }), + new Embedding(new float[] { -1, -1, -1 }), + new Embedding(new float[] { 1, 2, 3 }), + new Embedding(new float[] { -1, -2, -3 }), + new Embedding(new float[] { 1, -1, -2 }) + }; + var testRecords = new List(); + for (int i = 0; i < testEmbeddings.Length; i++) + { + testRecords.Add(MemoryRecord.LocalRecord( + id: "test" + i, + text: "text" + i, + description: "description" + i, + embedding: testEmbeddings[i])); + } + this.MockCreateIndex(collection, () => + { + for (int i = 0; i < testRecords.Count; i++) + { + if (i + 1 < testRecords.Count) + { + this.MockHashSet(collection, testRecords[i]); + } + else + { + this.MockHashSet(collection, testRecords[i], () => + { + this.MockSearch(collection, compareEmbedding, topN, threshold); + }); + } + } + }); + await store.CreateCollectionAsync(collection); + foreach (var testRecord in testRecords) + { + _ = await store.UpsertAsync(collection, testRecord); + } + + // Act + var topNResult = await store.GetNearestMatchAsync(collection, compareEmbedding, minRelevanceScore: threshold); + + // Assert + Assert.NotNull(topNResult); + Assert.Equal("test0", topNResult.Value.Item1.Metadata.Id); + Assert.True(topNResult.Value.Item2 >= threshold); + } + + [Fact] + public async Task GetNearestMatchesDifferentiatesIdenticalVectorsByKeyAsync() + { + // Arrange + RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + int topN = 4; + double threshold = 0.75; + string collection = "test_collection"; + var testRecords = new List(); + for (int i = 0; i < 10; i++) + { + testRecords.Add(MemoryRecord.LocalRecord( + id: "test" + i, + text: "text" + i, + description: "description" + i, + embedding: new Embedding(new float[] { 1, 1, 1 }))); + } + this.MockCreateIndex(collection, () => + { + for (int i = 0; i < testRecords.Count; i++) + { + if (i + 1 < testRecords.Count) + { + this.MockHashSet(collection, testRecords[i]); + } + else + { + this.MockHashSet(collection, testRecords[i], () => + { + this.MockSearch(collection, compareEmbedding, topN, threshold); + }); + } + } + }); + await store.CreateCollectionAsync(collection); + foreach (var testRecord in testRecords) + { + _ = await store.UpsertAsync(collection, testRecord); + } + + // Act + var topNResults = store.GetNearestMatchesAsync(collection, compareEmbedding, limit: topN, minRelevanceScore: threshold).ToEnumerable().ToArray(); + IEnumerable topNKeys = topNResults.Select(x => x.Item1.Key).ToImmutableSortedSet(); + + // Assert + Assert.Equal(topN, topNResults.Length); + Assert.Equal(topN, topNKeys.Count()); + + for (int i = 0; i < topNResults.Length; i++) + { + int compare = topNResults[i].Item2.CompareTo(threshold); + Assert.True(compare >= 0); + } + } + + [Fact] + public async Task ItCanBatchUpsertRecordsAsync() + { + // Arrange + RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + int numRecords = 10; + string collection = "test_collection"; + IEnumerable records = this.CreateBatchRecords(numRecords); + this.MockCreateIndex(collection, () => + { + foreach (var testRecord in records) + { + this.MockHashSet(collection, testRecord); + } + }); + + // Act + await store.CreateCollectionAsync(collection); + var keys = store.UpsertBatchAsync(collection, records); + var resultRecords = store.GetBatchAsync(collection, keys.ToEnumerable()); + + // Assert + Assert.NotNull(keys); + Assert.Equal(numRecords, keys.ToEnumerable().Count()); + Assert.Equal(numRecords, resultRecords.ToEnumerable().Count()); + } + + [Fact] + public async Task ItCanBatchGetRecordsAsync() + { + // Arrange + RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + int numRecords = 10; + string collection = "test_collection"; + IEnumerable records = this.CreateBatchRecords(numRecords); + this.MockCreateIndex(collection, () => + { + foreach (var testRecord in records) + { + this.MockHashSet(collection, testRecord); + } + }); + var keys = store.UpsertBatchAsync(collection, records); + + // Act + await store.CreateCollectionAsync(collection); + var results = store.GetBatchAsync(collection, keys.ToEnumerable()); + + // Assert + Assert.NotNull(keys); + Assert.NotNull(results); + Assert.Equal(numRecords, results.ToEnumerable().Count()); + } + + [Fact] + public async Task ItCanBatchRemoveRecordsAsync() + { + // Arrange + RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + int numRecords = 10; + string collection = "test_collection"; + IEnumerable records = this.CreateBatchRecords(numRecords); + this.MockCreateIndex(collection, () => + { + foreach (var testRecord in records) + { + this.MockHashSet(collection, testRecord, () => this.MockKeyDelete(collection, testRecord.Metadata.Id)); + } + this.MockKeyDelete(collection, records.Select(x => x.Metadata.Id)); + }); + await store.CreateCollectionAsync(collection); + + List keys = new(); + + // Act + await foreach (var key in store.UpsertBatchAsync(collection, records)) + { + keys.Add(key); + } + + await store.RemoveBatchAsync(collection, keys); + + // Assert + await foreach (var result in store.GetBatchAsync(collection, keys)) + { + Assert.Null(result); + } + } + + [Fact] + public async Task GetNearestMatchAsyncThrowsExceptionOnInvalidVectorScoreAsync() + { + // Arrange + RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3); + var compareEmbedding = new Embedding(new float[] { 1, 1, 1 }); + string collection = "test_collection"; + int topN = 1; + double threshold = 0.75; + var testEmbeddings = new[] + { + new Embedding(new float[] { 1, 1, 1 }), + new Embedding(new float[] { -1, -1, -1 }), + new Embedding(new float[] { 1, 2, 3 }), + new Embedding(new float[] { -1, -2, -3 }), + new Embedding(new float[] { 1, -1, -2 }) + }; + var testRecords = new List(); + for (int i = 0; i < testEmbeddings.Length; i++) + { + testRecords.Add(MemoryRecord.LocalRecord( + id: "test" + i, + text: "text" + i, + description: "description" + i, + embedding: testEmbeddings[i])); + } + this.MockCreateIndex(collection, () => + { + for (int i = 0; i < testRecords.Count; i++) + { + if (i + 1 < testRecords.Count) + { + this.MockHashSet(collection, testRecords[i]); + } + else + { + this.MockHashSet(collection, testRecords[i], () => + { + this.MockSearch(collection, compareEmbedding, topN, threshold, returnStringVectorScore: true); + }); + } + } + }); + await store.CreateCollectionAsync(collection); + foreach (var testRecord in testRecords) + { + _ = await store.UpsertAsync(collection, testRecord); + } + + // Assert + RedisMemoryStoreException ex = await Assert.ThrowsAsync(async () => + { + // Act + await store.GetNearestMatchAsync(collection, compareEmbedding, minRelevanceScore: threshold); + }); + Assert.Equal(ex.Message, "Invalid or missing vector score value."); + } + + #region private + + private void MockCreateIndex(string collection, Action? callback = null) + { + this._mockDatabase + .Setup>(x => x.ExecuteAsync( + It.Is(x => x == "FT.CREATE"), + It.Is(x => x[0].ToString() == collection)) + ) + .ReturnsAsync(RedisResult.Create("OK", ResultType.SimpleString)) + .Callback(() => + { + this._collections.TryAdd(collection, new()); + + this._mockDatabase + .Setup>(x => x.ExecuteAsync( + It.Is(x => x == "FT.INFO"), + It.Is(x => x[0].ToString() == collection)) + ) + .ReturnsAsync(RedisResult.Create(new[] { + RedisResult.Create("index_name", ResultType.BulkString), + RedisResult.Create(collection, ResultType.BulkString) + })); + + this._mockDatabase + .Setup>(x => x.ExecuteAsync( + It.Is(x => x == "FT._LIST"), + It.IsAny()) + ) + .ReturnsAsync(RedisResult.Create(this._collections.Select(x => RedisResult.Create(x.Key, ResultType.BulkString)).ToArray())); + + callback?.Invoke(); + }); + + this._mockDatabase + .Setup>(x => x.ExecuteAsync(It.Is(x => x == "FT.INFO"), It.IsAny())) + .Throws(new RedisServerException("Unknown Index name")); + } + + private void MockDropIndex(string collection, Action? callback = null) + { + this._mockDatabase + .Setup>(x => x.ExecuteAsync( + It.Is(x => x == "FT.DROPINDEX"), + It.Is(x => x[0].ToString() == collection && x[1].ToString() == "DD")) + ) + .ReturnsAsync(RedisResult.Create("OK", ResultType.SimpleString)) + .Callback(() => + { + this._collections.Remove(collection); + + this._mockDatabase + .Setup>(x => x.ExecuteAsync( + It.Is(x => x == "FT.INFO"), + It.Is(x => x[0].ToString() == collection)) + ) + .Throws(new RedisServerException("Unknown Index name")); + + this._mockDatabase + .Setup>(x => x.ExecuteAsync( + It.Is(x => x == "FT._LIST"), + It.IsAny()) + ) + .ReturnsAsync(RedisResult.Create(this._collections.Select(x => RedisResult.Create(x.Key, ResultType.BulkString)).ToArray())); + }); + } + + private void MockHashSet(string collection, MemoryRecord record, Action? callback = null) + { + string redisKey = $"{collection}:{record.Metadata.Id}"; + byte[] embedding = MemoryMarshal.Cast(record.Embedding.AsReadOnlySpan()).ToArray(); + long timestamp = record.Timestamp?.ToUnixTimeMilliseconds() ?? -1; + + this._mockDatabase + .Setup(x => x.HashSetAsync( + It.Is(x => x == redisKey), + It.Is(x => x.Length == 4 && + x[0].Name == "key" && x[1].Name == "metadata" && x[2].Name == "embedding" && x[3].Name == "timestamp" && + x[0].Value == record.Key && x[1].Value == record.GetSerializedMetadata() && embedding.SequenceEqual((byte[])x[2].Value!) && x[3].Value == timestamp + ), + It.IsAny()) + ) + .Callback(() => + { + (this._collections[collection] ??= new()).Add(record); + + this._mockDatabase + .Setup>(x => x.HashGetAllAsync(It.Is(x => x == redisKey), It.IsAny())) + .ReturnsAsync(new[] { + new HashEntry("key", record.Key), + new HashEntry("metadata", record.GetSerializedMetadata()), + new HashEntry("embedding", embedding), + new HashEntry("timestamp", timestamp) + }); + + callback?.Invoke(); + }); + } + + private void MockKeyDelete(string collection, string key, Action? callback = null) + { + string redisKey = $"{collection}:{key}"; + + this._mockDatabase + .Setup>(x => x.KeyDeleteAsync( + It.Is(x => x == redisKey), + It.IsAny()) + ) + .ReturnsAsync(true) + .Callback(() => + { + (this._collections[collection] ??= new()).RemoveAll(x => x.Key == key); + + this._mockDatabase + .Setup>(x => x.HashGetAllAsync(It.Is(x => x == redisKey), It.IsAny())) + .ReturnsAsync(Array.Empty()); + + callback?.Invoke(); + }); + } + + private void MockKeyDelete(string collection, IEnumerable keys, Action? callback = null) + { + RedisKey[] redisKeys = keys.Distinct().Select(key => new RedisKey($"{collection}:{key}")).ToArray(); + + this._mockDatabase + .Setup>(x => x.KeyDeleteAsync( + It.Is(x => redisKeys.SequenceEqual(x)), + It.IsAny()) + ) + .ReturnsAsync(redisKeys.Length) + .Callback(() => + { + (this._collections[collection] ??= new()).RemoveAll(x => keys.Contains(x.Key)); + + foreach (var redisKey in redisKeys) + { + this._mockDatabase + .Setup>(x => x.HashGetAllAsync(It.Is(x => x == redisKey), It.IsAny())) + .ReturnsAsync(Array.Empty()); + } + + callback?.Invoke(); + }); + } + + private void MockSearch(string collection, Embedding compareEmbedding, int topN, double threshold, bool returnStringVectorScore = false) + { + TopNCollection embeddings = new(topN); + + List records = this._collections.TryGetValue(collection, out var value) ? value : new(); + + foreach (var record in records) + { + double similarity = compareEmbedding + .AsReadOnlySpan() + .CosineSimilarity(record.Embedding.AsReadOnlySpan()); + if (similarity >= threshold) + { + embeddings.Add(new(record, similarity)); + } + } + + embeddings.SortByScore(); + + string redisKey = $"{collection}"; + + var redisResults = new List(); + redisResults.Add(RedisResult.Create(embeddings.Count)); + + foreach (var item in embeddings) + { + long timestamp = item.Value.Timestamp?.ToUnixTimeMilliseconds() ?? -1; + byte[] embedding = MemoryMarshal.Cast(item.Value.Embedding.AsReadOnlySpan()).ToArray(); + redisResults.Add(RedisResult.Create($"{collection}:{item.Value.Metadata.Id}", ResultType.BulkString)); + redisResults.Add(RedisResult.Create( + new RedisResult[] + { + RedisResult.Create("key", ResultType.BulkString), + RedisResult.Create(item.Value.Metadata.Id, ResultType.BulkString), + RedisResult.Create("metadata", ResultType.BulkString), + RedisResult.Create(item.Value.GetSerializedMetadata(), ResultType.BulkString), + RedisResult.Create("embedding", ResultType.BulkString), + RedisResult.Create(embedding, ResultType.BulkString), + RedisResult.Create("timestamp", ResultType.BulkString), + RedisResult.Create(timestamp, ResultType.BulkString), + RedisResult.Create("vector_score", ResultType.BulkString), + RedisResult.Create(returnStringVectorScore ? $"score:{1-item.Score.Value}" : 1-item.Score.Value, ResultType.BulkString), + }) + ); + } + + this._mockDatabase + .Setup>(x => x.ExecuteAsync( + It.Is(x => x == "FT.SEARCH"), + It.Is(x => x[0].ToString() == collection && x[1].ToString() == $"*=>[KNN {topN} @embedding $embedding AS vector_score]")) + ) + .ReturnsAsync(RedisResult.Create(redisResults.ToArray())); + } + + private IEnumerable CreateBatchRecords(int numRecords) + { + Assert.True(numRecords % 2 == 0, "Number of records must be even"); + Assert.True(numRecords > 0, "Number of records must be greater than 0"); + + IEnumerable records = new List(numRecords); + for (int i = 0; i < numRecords / 2; i++) + { + var testRecord = MemoryRecord.LocalRecord( + id: "test" + i, + text: "text" + i, + description: "description" + i, + embedding: new Embedding(new float[] { 1, 1, 1 })); + records = records.Append(testRecord); + } + + for (int i = numRecords / 2; i < numRecords; i++) + { + var testRecord = MemoryRecord.ReferenceRecord( + externalId: "test" + i, + sourceName: "sourceName" + i, + description: "description" + i, + embedding: new Embedding(new float[] { 1, 2, 3 })); + records = records.Append(testRecord); + } + + return records; + } + + #endregion +} diff --git a/samples/dotnet/kernel-syntax-examples/Example47_Redis.cs b/samples/dotnet/kernel-syntax-examples/Example47_Redis.cs new file mode 100644 index 0000000000..d5fb6c4622 --- /dev/null +++ b/samples/dotnet/kernel-syntax-examples/Example47_Redis.cs @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.Memory.Redis; +using Microsoft.SemanticKernel.Memory; +using RepoUtils; +using StackExchange.Redis; + +// ReSharper disable once InconsistentNaming +public static class Example47_Redis +{ + private const string MemoryCollectionName = "redis-test"; + + public static async Task RunAsync() + { + string configuration = Env.Var("REDIS_CONFIGURATION"); + using ConnectionMultiplexer connectionMultiplexer = await ConnectionMultiplexer.ConnectAsync(configuration); + IDatabase database = connectionMultiplexer.GetDatabase(); + RedisMemoryStore memoryStore = new(database, vectorSize: 1536); + IKernel kernel = Kernel.Builder + .WithLogger(ConsoleLogger.Log) + .WithOpenAITextCompletionService("text-davinci-003", Env.Var("OPENAI_API_KEY")) + .WithOpenAITextEmbeddingGenerationService("text-embedding-ada-002", Env.Var("OPENAI_API_KEY")) + .WithMemoryStorage(memoryStore) + .Build(); + + Console.WriteLine("== Printing Collections in DB =="); + var collections = memoryStore.GetCollectionsAsync(); + await foreach (var collection in collections) + { + Console.WriteLine(collection); + } + + Console.WriteLine("== Adding Memories =="); + + var key1 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat1", text: "british short hair"); + var key2 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat2", text: "orange tabby"); + var key3 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat3", text: "norwegian forest cat"); + + Console.WriteLine("== Printing Collections in DB =="); + collections = memoryStore.GetCollectionsAsync(); + await foreach (var collection in collections) + { + Console.WriteLine(collection); + } + + Console.WriteLine("== Retrieving Memories Through the Kernel =="); + MemoryQueryResult? lookup = await kernel.Memory.GetAsync(MemoryCollectionName, "cat1"); + Console.WriteLine(lookup != null ? lookup.Metadata.Text : "ERROR: memory not found"); + + Console.WriteLine("== Retrieving Memories Directly From the Store =="); + var memory1 = await memoryStore.GetAsync(MemoryCollectionName, key1); + var memory2 = await memoryStore.GetAsync(MemoryCollectionName, key2); + var memory3 = await memoryStore.GetAsync(MemoryCollectionName, key3); + Console.WriteLine(memory1 != null ? memory1.Metadata.Text : "ERROR: memory not found"); + Console.WriteLine(memory2 != null ? memory2.Metadata.Text : "ERROR: memory not found"); + Console.WriteLine(memory3 != null ? memory3.Metadata.Text : "ERROR: memory not found"); + + Console.WriteLine("== Similarity Searching Memories: My favorite color is orange =="); + var searchResults = kernel.Memory.SearchAsync(MemoryCollectionName, "My favorite color is orange", limit: 3, minRelevanceScore: 0.8); + + await foreach (var item in searchResults) + { + Console.WriteLine(item.Metadata.Text + " : " + item.Relevance); + } + + Console.WriteLine("== Removing Collection {0} ==", MemoryCollectionName); + await memoryStore.DeleteCollectionAsync(MemoryCollectionName); + + Console.WriteLine("== Printing Collections in DB =="); + await foreach (var collection in collections) + { + Console.WriteLine(collection); + } + } +} diff --git a/samples/dotnet/kernel-syntax-examples/KernelSyntaxExamples.csproj b/samples/dotnet/kernel-syntax-examples/KernelSyntaxExamples.csproj index 38b6202bd2..57cd720f74 100644 --- a/samples/dotnet/kernel-syntax-examples/KernelSyntaxExamples.csproj +++ b/samples/dotnet/kernel-syntax-examples/KernelSyntaxExamples.csproj @@ -34,6 +34,7 @@ + diff --git a/samples/dotnet/kernel-syntax-examples/Program.cs b/samples/dotnet/kernel-syntax-examples/Program.cs index 74ccca1226..a37e8fc556 100644 --- a/samples/dotnet/kernel-syntax-examples/Program.cs +++ b/samples/dotnet/kernel-syntax-examples/Program.cs @@ -123,6 +123,7 @@ public static class Program Console.WriteLine("== DONE =="); await Example39_Postgres.RunAsync(); + Console.WriteLine("== DONE =="); await Example40_DIContainer.RunAsync(); Console.WriteLine("== DONE =="); @@ -144,5 +145,8 @@ public static class Program await Example46_Weaviate.RunAsync(); Console.WriteLine("== DONE =="); + + await Example47_Redis.RunAsync(); + Console.WriteLine("== DONE =="); } } diff --git a/samples/dotnet/kernel-syntax-examples/README.md b/samples/dotnet/kernel-syntax-examples/README.md index fcaec1352b..34ed2accc3 100644 --- a/samples/dotnet/kernel-syntax-examples/README.md +++ b/samples/dotnet/kernel-syntax-examples/README.md @@ -36,6 +36,7 @@ dotnet user-secrets set "WEAVIATE_PORT" "..." dotnet user-secrets set "WEAVIATE_APIKEY" "..." dotnet user-secrets set "GITHUB_PERSONAL_ACCESS_TOKEN" "github_pat_..." dotnet user-secrets set "POSTGRES_CONNECTIONSTRING" "..." +dotnet user-secrets set "REDIS_CONFIGURATION" "..." ``` To set your secrets with environment variables, use these names: @@ -56,6 +57,7 @@ To set your secrets with environment variables, use these names: * WEAVIATE_APIKEY * GITHUB_PERSONAL_ACCESS_TOKEN * POSTGRES_CONNECTIONSTRING +* REDIS_CONFIGURATION * AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME * AZURE_OPENAI_EMBEDDINGS_ENDPOINT * AZURE_OPENAI_EMBEDDINGS_KEY