Redis connector for Semantic Memory (#1178)

### Motivation and Context
Redis supports cosine similarity search by the
[RediSearch](https://redis.io/docs/stack/search) module.

### Description
Add a new Semantic Memory Connector leveraging Redis with RediSearch
module. Update the syntax example to demonstrate how it is used.

Co-authored-by: Dmytro Struk <[email protected]>
This commit is contained in:
Jadyn
2023-06-09 16:15:05 +00:00
committed by GitHub
co-authored by Dmytro Struk
parent 471ec9c19a
commit 5078c2f719
13 changed files with 1472 additions and 0 deletions
+1
View File
@@ -68,6 +68,7 @@
| Sqlite | ✅ | ❌ | ❌ | Vector optimization requires [sqlite-vss](https://github.com/asg017/sqlite-vss) |
| Postgres | ✅ | ❌ | ❌ | Vector optimization requires [pgvector](https://github.com/pgvector/pgvector) |
| CosmosDB | ✅ | ❌ | ❌ | CosmosDB is not optimized for vector storage |
| Redis | ✅ | ❌ | ❌ | Vector optimization requires [RediSearch](https://redis.io/docs/stack/search) |
## Connectors and Skill Libraries
+1
View File
@@ -8,6 +8,7 @@
<PackageVersion Include="Azure.Search.Documents" Version="11.5.0-beta.2" />
<PackageVersion Include="Microsoft.Bcl.HashCode" Version="[1.1.0, )" />
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="[7.0.0, )" />
<PackageVersion Include="NRedisStack" Version="0.6.1" />
<PackageVersion Include="Pgvector" Version="0.1.3" />
<PackageVersion Include="System.Linq.Async" Version="6.0.1" />
<PackageVersion Include="System.Text.Json" Version="[7.0.2, )" />
+9
View File
@@ -72,6 +72,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.CosmosDB"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.Postgres", "src\Connectors\Connectors.Memory.Postgres\Connectors.Memory.Postgres.csproj", "{C9F957FA-A70F-4A6D-8F95-23FCD7F4FB87}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.Redis", "src\Connectors\Connectors.Memory.Redis\Connectors.Memory.Redis.csproj", "{3720F5ED-FB4D-485E-8A93-CDE60DEF0805}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.AI.OpenAI", "src\Connectors\Connectors.AI.OpenAI\Connectors.AI.OpenAI.csproj", "{AFA81EB7-F869-467D-8A90-744305D80AAC}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SemanticKernel.Abstractions", "src\SemanticKernel.Abstractions\SemanticKernel.Abstractions.csproj", "{627742DB-1E52-468A-99BD-6FF1A542D25B}"
@@ -304,6 +306,12 @@ Global
{4D91A3E0-C404-495B-AD4A-411C4E83CF54}.Publish|Any CPU.Build.0 = Release|Any CPU
{4D91A3E0-C404-495B-AD4A-411C4E83CF54}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4D91A3E0-C404-495B-AD4A-411C4E83CF54}.Release|Any CPU.Build.0 = Release|Any CPU
{3720F5ED-FB4D-485E-8A93-CDE60DEF0805}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3720F5ED-FB4D-485E-8A93-CDE60DEF0805}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3720F5ED-FB4D-485E-8A93-CDE60DEF0805}.Publish|Any CPU.ActiveCfg = Debug|Any CPU
{3720F5ED-FB4D-485E-8A93-CDE60DEF0805}.Publish|Any CPU.Build.0 = Debug|Any CPU
{3720F5ED-FB4D-485E-8A93-CDE60DEF0805}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3720F5ED-FB4D-485E-8A93-CDE60DEF0805}.Release|Any CPU.Build.0 = Release|Any CPU
{50FAE231-6F24-4779-9D02-12ABBC9A49E2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{50FAE231-6F24-4779-9D02-12ABBC9A49E2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{50FAE231-6F24-4779-9D02-12ABBC9A49E2}.Publish|Any CPU.ActiveCfg = Publish|Any CPU
@@ -353,6 +361,7 @@ Global
{4D3DAE63-41C6-4E1C-A35A-E77BDFC40675} = {831DDCA2-7D2C-4C31-80DB-6BDB3E1F7AE0}
{6AAB0620-33A1-4A98-A63B-6560B9BA47A4} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C}
{4D91A3E0-C404-495B-AD4A-411C4E83CF54} = {FA3720F1-C99A-49B2-9577-A940257098BF}
{3720F5ED-FB4D-485E-8A93-CDE60DEF0805} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C}
{50FAE231-6F24-4779-9D02-12ABBC9A49E2} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
@@ -0,0 +1,28 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<!-- THIS PROPERTY GROUP MUST COME FIRST -->
<AssemblyName>Microsoft.SemanticKernel.Connectors.Memory.Redis</AssemblyName>
<RootNamespace>$(AssemblyName)</RootNamespace>
<TargetFramework>netstandard2.0</TargetFramework>
</PropertyGroup>
<!-- IMPORT NUGET PACKAGE SHARED PROPERTIES -->
<Import Project="$(RepoRoot)/dotnet/nuget/nuget-package.props" />
<Import Project="$(RepoRoot)/dotnet/src/InternalUtilities/InternalUtilities.props" />
<PropertyGroup>
<!-- NuGet Package Settings -->
<Title>Semantic Kernel - Redis Connector</Title>
<Description>Redis connector for Semantic Kernel skills and semantic memory</Description>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="NRedisStack" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\SemanticKernel\SemanticKernel.csproj" />
</ItemGroup>
</Project>
@@ -0,0 +1,32 @@
# Microsoft.SemanticKernel.Connectors.Memory.Redis
This connector uses Redis to implement Semantic Memory. It requires the [RediSearch](https://redis.io/docs/stack/search) module to be enabled on Redis to implement vector similarity search.
## What is RediSearch?
[RediSearch](https://redis.io/docs/stack/search) is a source-available Redis module that enables querying, secondary indexing, and full-text search for Redis. These features enable multi-field queries, aggregation, exact phrase matching, numeric filtering, geo filtering and vector similarity semantic search on top of text queries.
How to set up the RediSearch, please refer to its [documentation](https://redis.io/docs/stack/search/quick_start/).
Or use the [Redis Enterprise](https://redis.io/docs/about/redis-enterprise/), see [Azure Marketplace](https://azuremarketplace.microsoft.com/en-us/marketplace/apps/garantiadata.redis_enterprise_1sp_public_preview?tab=Overview), [AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-e6y7ork67pjwg?sr=0-2&ref_=beagle&applicationId=AWSMPContessa), or [Google Marketplace](https://console.cloud.google.com/marketplace/details/redislabs-public/redis-enterprise?pli=1).
## Quick start
1. Run with Docker:
```bash
docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest
```
2. To use Redis as a semantic memory store:
```csharp
using ConnectionMultiplexer connectionMultiplexer = await ConnectionMultiplexer.ConnectAsync("localhost:6379");
IDatabase database = connectionMultiplexer.GetDatabase();
RedisMemoryStore memoryStore = new RedisMemoryStore(database, vectorSize: 1536);
IKernel kernel = Kernel.Builder
.WithLogger(ConsoleLogger.Log)
.WithOpenAITextEmbeddingGenerationService("text-embedding-ada-002", Env.Var("OPENAI_API_KEY"))
.WithMemoryStorage(memoryStore)
.Build();
```
@@ -0,0 +1,299 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.SemanticKernel.AI.Embeddings;
using Microsoft.SemanticKernel.Memory;
using NRedisStack;
using NRedisStack.RedisStackCommands;
using NRedisStack.Search;
using NRedisStack.Search.Literals.Enums;
using StackExchange.Redis;
namespace Microsoft.SemanticKernel.Connectors.Memory.Redis;
/// <summary>
/// An implementation of <see cref="IMemoryStore"/> for Redis.
/// </summary>
/// <remarks>The embedded data is saved to the Redis server database specified in the constructor.
/// Similarity search capability is provided through the RediSearch module. Use RediSearch's "Index" to implement "Collection".
/// </remarks>
public sealed class RedisMemoryStore : IMemoryStore
{
/// <summary>
/// Create a new instance of semantic memory using Redis.
/// </summary>
/// <param name="database">The database of the redis server.</param>
/// <param name="vectorSize">Embedding vector size</param>
public RedisMemoryStore(IDatabase database, int vectorSize)
{
this._database = database;
this._vectorSize = vectorSize;
this._ft = database.FT();
}
/// <inheritdoc />
public async IAsyncEnumerable<string> GetCollectionsAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
{
foreach (var index in await this._ft._ListAsync().ConfigureAwait(false))
{
yield return ((string)index!);
}
}
/// <inheritdoc />
public async Task CreateCollectionAsync(string collectionName, CancellationToken cancellationToken = default)
{
FTCreateParams ftCreateParams = FTCreateParams.CreateParams().On(IndexDataType.HASH).Prefix($"{collectionName}:");
Schema schema = new Schema()
.AddTextField("key")
.AddTextField("metadata")
.AddNumericField("timestamp")
.AddVectorField("embedding", VECTOR_INDEX_ALGORITHM, new Dictionary<string, object> {
{"TYPE", VECTOR_TYPE},
{"DIM", this._vectorSize},
{"DISTANCE_METRIC", VECTOR_DISTANCE_METRIC},
});
await this._ft.CreateAsync(collectionName, ftCreateParams, schema).ConfigureAwait(false);
}
/// <inheritdoc />
public async Task<bool> DoesCollectionExistAsync(string collectionName, CancellationToken cancellationToken = default)
{
try
{
await this._ft.InfoAsync(collectionName).ConfigureAwait(false);
return true;
}
catch (RedisServerException ex) when (ex.Message == MESSAGE_WHEN_INDEX_DOES_NOT_EXIST)
{
return false;
}
}
/// <inheritdoc />
public async Task DeleteCollectionAsync(string collectionName, CancellationToken cancellationToken = default)
{
// dd: If `true`, all documents will be deleted.
await this._ft.DropIndexAsync(collectionName, dd: true).ConfigureAwait(false);
}
/// <inheritdoc />
public async Task<MemoryRecord?> GetAsync(string collectionName, string key, bool withEmbedding = false, CancellationToken cancellationToken = default)
{
return await this.InternalGetAsync(collectionName, key, withEmbedding, cancellationToken).ConfigureAwait(false);
}
/// <inheritdoc/>
public async IAsyncEnumerable<MemoryRecord> GetBatchAsync(string collectionName, IEnumerable<string> keys, bool withEmbeddings = false,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
foreach (var key in keys)
{
var result = await this.InternalGetAsync(collectionName, key, withEmbeddings, cancellationToken).ConfigureAwait(false);
if (result != null)
{
yield return result;
}
}
}
/// <inheritdoc />
public async Task<string> UpsertAsync(string collectionName, MemoryRecord record, CancellationToken cancellationToken = default)
{
record.Key = record.Metadata.Id;
await this._database.HashSetAsync(GetRedisKey(collectionName, record.Key), new[] {
new HashEntry("key", record.Key),
new HashEntry("metadata", record.GetSerializedMetadata()),
new HashEntry("embedding", MemoryMarshal.Cast<float, byte>(record.Embedding.AsReadOnlySpan()).ToArray()),
new HashEntry("timestamp", ToTimestampLong(record.Timestamp))
}, flags: CommandFlags.None).ConfigureAwait(false);
return record.Key;
}
/// <inheritdoc/>
public async IAsyncEnumerable<string> UpsertBatchAsync(string collectionName, IEnumerable<MemoryRecord> records, [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
foreach (var record in records)
{
yield return await this.UpsertAsync(collectionName, record, cancellationToken).ConfigureAwait(false);
}
}
/// <inheritdoc />
public async Task RemoveAsync(string collectionName, string key, CancellationToken cancellationToken = default)
{
await this._database.KeyDeleteAsync(GetRedisKey(collectionName, key), flags: CommandFlags.None).ConfigureAwait(false);
}
/// <inheritdoc/>
public async Task RemoveBatchAsync(string collectionName, IEnumerable<string> keys, CancellationToken cancellationToken = default)
{
await this._database.KeyDeleteAsync(keys.Select(key => GetRedisKey(collectionName, key)).ToArray(), flags: CommandFlags.None).ConfigureAwait(false);
}
/// <inheritdoc />
public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync(
string collectionName,
Embedding<float> embedding,
int limit,
double minRelevanceScore = 0,
bool withEmbeddings = false,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
if (limit <= 0)
{
yield break;
}
var query = new Query($"*=>[KNN {limit} @embedding $embedding AS vector_score]")
.AddParam("embedding", MemoryMarshal.Cast<float, byte>(embedding.AsReadOnlySpan()).ToArray())
.SetSortBy("vector_score")
.ReturnFields("key", "metadata", "embedding", "timestamp", "vector_score")
.Limit(0, limit)
.Dialect(QUERY_DIALECT);
var results = await this._ft.SearchAsync(collectionName, query).ConfigureAwait(false);
foreach (var document in results.Documents)
{
double similarity = this.GetSimilarity(document);
if (similarity < minRelevanceScore)
{
yield break;
}
Embedding<float> convertedEmbedding = withEmbeddings && document["embedding"].HasValue
?
new Embedding<float>(MemoryMarshal.Cast<byte, float>((byte[])document["embedding"]!).ToArray())
:
Embedding<float>.Empty;
yield return (MemoryRecord.FromJsonMetadata(
json: document["metadata"]!,
embedding: convertedEmbedding,
key: document["key"],
timestamp: ParseTimestamp((long?)document["timestamp"])), similarity);
}
}
/// <inheritdoc/>
public async Task<(MemoryRecord, double)?> GetNearestMatchAsync(string collectionName, Embedding<float> embedding, double minRelevanceScore = 0, bool withEmbedding = false,
CancellationToken cancellationToken = default)
{
return await this.GetNearestMatchesAsync(
collectionName: collectionName,
embedding: embedding,
limit: 1,
minRelevanceScore: minRelevanceScore,
withEmbeddings: withEmbedding,
cancellationToken: cancellationToken).FirstOrDefaultAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
}
#region constants ================================================================================
/// <summary>
/// Vector similarity index algorithm. The default value is "HNSW".
/// <see href="https://redis.io/docs/stack/search/reference/vectors/#create-a-vector-field"/>
/// </summary>
internal const Schema.VectorField.VectorAlgo VECTOR_INDEX_ALGORITHM = Schema.VectorField.VectorAlgo.HNSW;
/// <summary>
/// Vector type. Supported types are FLOAT32 and FLOAT64. The default value is "FLOAT32".
/// </summary>
internal const string VECTOR_TYPE = "FLOAT32";
/// <summary>
/// Supported distance metric, one of {L2, IP, COSINE}. The default value is "COSINE".
/// </summary>
internal const string VECTOR_DISTANCE_METRIC = "COSINE";
/// <summary>
/// Query dialect. To use a vector similarity query, specify DIALECT 2 or higher. The default value is "2".
/// <see href="https://redis.io/docs/stack/search/reference/vectors/#querying-vector-fields"/>
/// </summary>
internal const int QUERY_DIALECT = 2;
/// <summary>
/// Message when index does not exist.
/// <see href="https://github.com/RediSearch/RediSearch/blob/master/src/info_command.c#L96"/>
/// </summary>
internal const string MESSAGE_WHEN_INDEX_DOES_NOT_EXIST = "Unknown Index name";
#endregion
#region private ================================================================================
private readonly IDatabase _database;
private readonly int _vectorSize;
private readonly SearchCommands _ft;
private static long ToTimestampLong(DateTimeOffset? timestamp)
{
if (timestamp.HasValue)
{
return timestamp.Value.ToUnixTimeMilliseconds();
}
return -1;
}
private static DateTimeOffset? ParseTimestamp(long? timestamp)
{
if (timestamp.HasValue && timestamp > 0)
{
return DateTimeOffset.FromUnixTimeMilliseconds(timestamp.Value);
}
return null;
}
private static RedisKey GetRedisKey(string collectionName, string key)
{
return new RedisKey($"{collectionName}:{key}");
}
private async Task<MemoryRecord?> InternalGetAsync(string collectionName, string key, bool withEmbedding, CancellationToken cancellationToken)
{
HashEntry[] hashEntries = await this._database.HashGetAllAsync(GetRedisKey(collectionName, key), flags: CommandFlags.None).ConfigureAwait(false);
if (hashEntries.Length == 0) { return null; }
if (withEmbedding)
{
RedisValue embedding = hashEntries.FirstOrDefault(x => x.Name == "embedding").Value;
return MemoryRecord.FromJsonMetadata(
json: hashEntries.FirstOrDefault(x => x.Name == "metadata").Value!,
embedding: embedding.HasValue ? new Embedding<float>(MemoryMarshal.Cast<byte, float>((byte[])embedding!).ToArray()) : Embedding<float>.Empty,
key: hashEntries.FirstOrDefault(x => x.Name == "key").Value,
timestamp: ParseTimestamp((long?)hashEntries.FirstOrDefault(x => x.Name == "timestamp").Value));
}
return MemoryRecord.FromJsonMetadata(
json: hashEntries.FirstOrDefault(x => x.Name == "metadata").Value!,
embedding: Embedding<float>.Empty,
key: hashEntries.FirstOrDefault(x => x.Name == "key").Value,
timestamp: ParseTimestamp((long?)hashEntries.FirstOrDefault(x => x.Name == "timestamp").Value));
}
private double GetSimilarity(Document document)
{
RedisValue vectorScoreValue = document["vector_score"];
if (vectorScoreValue.IsNullOrEmpty || !vectorScoreValue.TryParse(out double vectorScore))
{
throw new RedisMemoryStoreException("Invalid or missing vector score value.");
}
return 1 - vectorScore;
}
#endregion
}
@@ -0,0 +1,30 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
namespace Microsoft.SemanticKernel.Connectors.Memory.Redis;
#pragma warning disable RCS1194 // Implement exception constructors
/// <summary>
/// Exception thrown by the Redis connector
/// </summary>
public class RedisMemoryStoreException : Exception
{
/// <summary>
/// Initializes a new instance of the <see cref="RedisMemoryStoreException"/> class.
/// </summary>
/// <param name="message">Exception message.</param>
internal RedisMemoryStoreException(string? message) : base(message)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="RedisMemoryStoreException"/> class.
/// </summary>
/// <param name="message">Exception message.</param>
/// <param name="innerException">Inner exception.</param>
internal RedisMemoryStoreException(string? message, Exception? innerException) : base(message, innerException)
{
}
}
@@ -31,6 +31,7 @@
<ProjectReference Include="..\Connectors.Memory.Pinecone\Connectors.Memory.Pinecone.csproj" />
<ProjectReference Include="..\Connectors.Memory.DuckDB\Connectors.Memory.DuckDB.csproj" />
<ProjectReference Include="..\Connectors.Memory.Qdrant\Connectors.Memory.Qdrant.csproj" />
<ProjectReference Include="..\Connectors.Memory.Redis\Connectors.Memory.Redis.csproj" />
<ProjectReference Include="..\Connectors.Memory.Sqlite\Connectors.Memory.Sqlite.csproj" />
</ItemGroup>
@@ -0,0 +1,986 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading.Tasks;
using Microsoft.SemanticKernel.AI.Embeddings;
using Microsoft.SemanticKernel.AI.Embeddings.VectorOperations;
using Microsoft.SemanticKernel.Connectors.Memory.Redis;
using Microsoft.SemanticKernel.Memory;
using Microsoft.SemanticKernel.Memory.Collections;
using Moq;
using StackExchange.Redis;
using Xunit;
namespace SemanticKernel.Connectors.UnitTests.Memory.Redis;
/// <summary>
/// Unit tests of <see cref="RedisMemoryStore"/>.
/// </summary>
public class RedisMemoryStoreTests
{
private readonly Mock<IDatabase> _mockDatabase;
private readonly Dictionary<string, List<MemoryRecord>> _collections;
public RedisMemoryStoreTests()
{
this._mockDatabase = new Mock<IDatabase>();
this._collections = new();
}
[Fact]
public void ConnectionCanBeInitialized()
{
// Arrange
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
}
[Fact]
public async Task ItCanCreateAndGetCollectionAsync()
{
// Arrange
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
string collection = "test_collection";
this.MockCreateIndex(collection);
// Act
await store.CreateCollectionAsync(collection);
var collections = store.GetCollectionsAsync();
// Assert
Assert.NotEmpty(collections.ToEnumerable());
Assert.True(await collections.ContainsAsync(collection));
}
[Fact]
public async Task ItCanCheckIfCollectionExistsAsync()
{
// Arrange
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
string collection = "my_collection";
this.MockCreateIndex(collection);
// Act
await store.CreateCollectionAsync(collection);
// Assert
Assert.True(await store.DoesCollectionExistAsync("my_collection"));
Assert.False(await store.DoesCollectionExistAsync("my_collection2"));
}
[Fact]
public async Task CollectionsCanBeDeletedAsync()
{
// Arrange
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
string collection = "test_collection";
this.MockCreateIndex(collection, () =>
{
this.MockDropIndex(collection);
});
await store.CreateCollectionAsync(collection);
var collections = await store.GetCollectionsAsync().ToListAsync();
Assert.True(collections.Count > 0);
// Act
foreach (var c in collections)
{
await store.DeleteCollectionAsync(c);
}
// Assert
var collections2 = store.GetCollectionsAsync();
Assert.True(await collections2.CountAsync() == 0);
}
[Fact]
public async Task ItCanInsertIntoNonExistentCollectionAsync()
{
// Arrange
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
MemoryRecord testRecord = MemoryRecord.LocalRecord(
id: "test",
text: "text",
description: "description",
embedding: new Embedding<float>(new float[] { 1, 2, 3 }),
key: null,
timestamp: null);
string collection = "random collection";
string redisKey = $"{collection}:{testRecord.Metadata.Id}";
byte[] embedding = MemoryMarshal.Cast<float, byte>(testRecord.Embedding.AsReadOnlySpan()).ToArray();
this._mockDatabase
.Setup<Task>(x => x.HashSetAsync(
It.Is<RedisKey>(x => x == redisKey),
It.Is<HashEntry[]>(x => x.Length == 4 &&
x[0].Name == "key" && x[1].Name == "metadata" && x[2].Name == "embedding" && x[3].Name == "timestamp" &&
x[0].Value == testRecord.Key && x[1].Value == testRecord.GetSerializedMetadata() && embedding.SequenceEqual((byte[])x[2].Value!) && x[3].Value == -1
),
It.IsAny<CommandFlags>())
)
.Callback(() =>
{
this._mockDatabase
.Setup<Task<HashEntry[]>>(x => x.HashGetAllAsync(It.Is<RedisKey>(x => x == redisKey), It.IsAny<CommandFlags>()))
.ReturnsAsync(new[] {
new HashEntry("key", testRecord.Key),
new HashEntry("metadata", testRecord.GetSerializedMetadata()),
new HashEntry("embedding", embedding),
new HashEntry("timestamp", -1)
});
});
// Arrange
var key = await store.UpsertAsync(collection, testRecord);
var actual = await store.GetAsync(collection, key, true);
// Assert
Assert.NotNull(actual);
Assert.Equal(testRecord.Metadata.Id, key);
Assert.Equal(testRecord.Metadata.Id, actual.Key);
Assert.Equal(testRecord.Embedding.Vector, actual.Embedding.Vector);
Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text);
Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description);
Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName);
Assert.Equal(testRecord.Metadata.Id, actual.Metadata.Id);
}
[Fact]
public async Task GetAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync()
{
// Arrange
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
MemoryRecord testRecord = MemoryRecord.LocalRecord(
id: "test",
text: "text",
description: "description",
embedding: new Embedding<float>(new float[] { 1, 2, 3 }),
key: null,
timestamp: null);
string collection = "test_collection";
string redisKey = $"{collection}:{testRecord.Metadata.Id}";
this.MockCreateIndex(collection, () =>
{
this.MockHashSet(collection, testRecord);
});
// Act
await store.CreateCollectionAsync(collection);
var key = await store.UpsertAsync(collection, testRecord);
var actualDefault = await store.GetAsync(collection, key);
var actualWithEmbedding = await store.GetAsync(collection, key, true);
// Assert
Assert.NotNull(actualDefault);
Assert.NotNull(actualWithEmbedding);
Assert.Empty(actualDefault.Embedding.Vector);
Assert.NotEmpty(actualWithEmbedding.Embedding.Vector);
}
[Fact]
public async Task ItCanUpsertAndRetrieveARecordWithNoTimestampAsync()
{
// Arrange
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
MemoryRecord testRecord = MemoryRecord.LocalRecord(
id: "test",
text: "text",
description: "description",
embedding: new Embedding<float>(new float[] { 1, 2, 3 }),
key: null,
timestamp: null);
string collection = "test_collection";
this.MockCreateIndex(collection, () =>
{
this.MockHashSet(collection, testRecord);
});
// Act
await store.CreateCollectionAsync(collection);
var key = await store.UpsertAsync(collection, testRecord);
var actual = await store.GetAsync(collection, key, true);
// Assert
Assert.NotNull(actual);
Assert.Equal(testRecord.Metadata.Id, key);
Assert.Equal(testRecord.Metadata.Id, actual.Key);
Assert.Equal(testRecord.Embedding.Vector, actual.Embedding.Vector);
Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text);
Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description);
Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName);
Assert.Equal(testRecord.Metadata.Id, actual.Metadata.Id);
}
[Fact]
public async Task ItCanUpsertAndRetrieveARecordWithTimestampAsync()
{
// Arrange
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
MemoryRecord testRecord = MemoryRecord.LocalRecord(
id: "test",
text: "text",
description: "description",
embedding: new Embedding<float>(new float[] { 1, 2, 3 }),
key: null,
timestamp: DateTimeOffset.UtcNow);
string collection = "test_collection";
this.MockCreateIndex(collection, () =>
{
this.MockHashSet(collection, testRecord);
});
// Act
await store.CreateCollectionAsync(collection);
var key = await store.UpsertAsync(collection, testRecord);
var actual = await store.GetAsync(collection, key, true);
// Assert
Assert.NotNull(actual);
Assert.Equal(testRecord.Metadata.Id, key);
Assert.Equal(testRecord.Metadata.Id, actual.Key);
Assert.Equal(testRecord.Embedding.Vector, actual.Embedding.Vector);
Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text);
Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description);
Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName);
Assert.Equal(testRecord.Metadata.Id, actual.Metadata.Id);
}
[Fact]
public async Task UpsertReplacesExistingRecordWithSameIdAsync()
{
// Arrange
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
string commonId = "test";
MemoryRecord testRecord = MemoryRecord.LocalRecord(
id: commonId,
text: "text",
description: "description",
embedding: new Embedding<float>(new float[] { 1, 2, 3 }));
MemoryRecord testRecord2 = MemoryRecord.LocalRecord(
id: commonId,
text: "text2",
description: "description2",
embedding: new Embedding<float>(new float[] { 1, 2, 4 }));
string collection = "test_collection";
this.MockCreateIndex(collection, () =>
{
this.MockHashSet(collection, testRecord);
this.MockHashSet(collection, testRecord2);
});
// Act
await store.CreateCollectionAsync(collection);
var key = await store.UpsertAsync(collection, testRecord);
var key2 = await store.UpsertAsync(collection, testRecord2);
var actual = await store.GetAsync(collection, key, true);
// Assert
Assert.NotNull(actual);
Assert.Equal(testRecord.Metadata.Id, key);
Assert.Equal(testRecord2.Metadata.Id, actual.Key);
Assert.NotEqual(testRecord.Embedding.Vector, actual.Embedding.Vector);
Assert.Equal(testRecord2.Embedding.Vector, actual.Embedding.Vector);
Assert.NotEqual(testRecord.Metadata.Text, actual.Metadata.Text);
Assert.Equal(testRecord2.Metadata.Description, actual.Metadata.Description);
}
[Fact]
public async Task ExistingRecordCanBeRemovedAsync()
{
// Arrange
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
MemoryRecord testRecord = MemoryRecord.LocalRecord(
id: "test",
text: "text",
description: "description",
embedding: new Embedding<float>(new float[] { 1, 2, 3 }));
string collection = "test_collection";
this.MockCreateIndex(collection, () =>
{
this.MockHashSet(collection, testRecord, () =>
{
this.MockKeyDelete(collection, testRecord.Metadata.Id);
});
});
// Act
await store.CreateCollectionAsync(collection);
var key = await store.UpsertAsync(collection, testRecord);
await store.RemoveAsync(collection, key);
var actual = await store.GetAsync(collection, key);
// Assert
Assert.Null(actual);
}
[Fact]
public async Task RemovingNonExistingRecordDoesNothingAsync()
{
// Arrange
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
string collection = "test_collection";
this.MockCreateIndex(collection, () =>
{
this.MockKeyDelete(collection, "key");
});
// Act
await store.CreateCollectionAsync(collection);
await store.RemoveAsync(collection, "key");
var actual = await store.GetAsync(collection, "key");
// Assert
Assert.Null(actual);
}
[Fact]
public async Task ItCanListAllDatabaseCollectionsAsync()
{
// Arrange
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
string[] testCollections = { "random_collection1", "random_collection2", "random_collection3" };
foreach (var collection in testCollections)
{
this.MockCreateIndex(collection, () =>
{
this.MockDropIndex(collection);
});
}
await store.CreateCollectionAsync(testCollections[0]);
await store.CreateCollectionAsync(testCollections[1]);
await store.CreateCollectionAsync(testCollections[2]);
// Act
var collections = await store.GetCollectionsAsync().ToListAsync();
// Assert
foreach (var collection in testCollections)
{
Assert.True(await store.DoesCollectionExistAsync(collection));
}
Assert.NotNull(collections);
Assert.NotEmpty(collections);
Assert.Equal(testCollections.Length, collections.Count);
Assert.True(collections.Contains(testCollections[0]),
$"Collections does not contain the newly-created collection {testCollections[0]}");
Assert.True(collections.Contains(testCollections[1]),
$"Collections does not contain the newly-created collection {testCollections[1]}");
Assert.True(collections.Contains(testCollections[2]),
$"Collections does not contain the newly-created collection {testCollections[2]}");
}
[Fact]
public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync()
{
// Arrange
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
var compareEmbedding = new Embedding<float>(new float[] { 1, 1, 1 });
string collection = "test_collection";
int topN = 4;
double threshold = -1;
var testEmbeddings = new[]
{
new Embedding<float>(new float[] { 1, 1, 1 }),
new Embedding<float>(new float[] { -1, -1, -1 }),
new Embedding<float>(new float[] { 1, 2, 3 }),
new Embedding<float>(new float[] { -1, -2, -3 }),
new Embedding<float>(new float[] { 1, -1, -2 })
};
var testRecords = new List<MemoryRecord>();
for (int i = 0; i < testEmbeddings.Length; i++)
{
testRecords.Add(MemoryRecord.LocalRecord(
id: "test" + i,
text: "text" + i,
description: "description" + i,
embedding: testEmbeddings[i]));
}
this.MockCreateIndex(collection, () =>
{
for (int i = 0; i < testRecords.Count; i++)
{
if (i + 1 < testRecords.Count)
{
this.MockHashSet(collection, testRecords[i]);
}
else
{
this.MockHashSet(collection, testRecords[i], () => this.MockSearch(collection, compareEmbedding, topN, threshold));
}
}
});
await store.CreateCollectionAsync(collection);
foreach (var testRecord in testRecords)
{
_ = await store.UpsertAsync(collection, testRecord);
}
// Act
var topNResults = store.GetNearestMatchesAsync(collection, compareEmbedding, limit: topN, minRelevanceScore: threshold).ToEnumerable().ToArray();
// Assert
Assert.Equal(topN, topNResults.Length);
for (int j = 0; j < topN - 1; j++)
{
int compare = topNResults[j].Item2.CompareTo(topNResults[j + 1].Item2);
Assert.True(compare >= 0);
}
}
[Fact]
public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync()
{
// Arrange
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
var compareEmbedding = new Embedding<float>(new float[] { 1, 1, 1 });
string collection = "test_collection";
int topN = 1;
double threshold = 0.75;
var testEmbeddings = new[]
{
new Embedding<float>(new float[] { 1, 1, 1 }),
new Embedding<float>(new float[] { -1, -1, -1 }),
new Embedding<float>(new float[] { 1, 2, 3 }),
new Embedding<float>(new float[] { -1, -2, -3 }),
new Embedding<float>(new float[] { 1, -1, -2 })
};
var testRecords = new List<MemoryRecord>();
for (int i = 0; i < testEmbeddings.Length; i++)
{
testRecords.Add(MemoryRecord.LocalRecord(
id: "test" + i,
text: "text" + i,
description: "description" + i,
embedding: testEmbeddings[i]));
}
this.MockCreateIndex(collection, () =>
{
for (int i = 0; i < testRecords.Count; i++)
{
if (i + 1 < testRecords.Count)
{
this.MockHashSet(collection, testRecords[i]);
}
else
{
this.MockHashSet(collection, testRecords[i], () =>
{
this.MockSearch(collection, compareEmbedding, topN, threshold);
});
}
}
});
await store.CreateCollectionAsync(collection);
foreach (var testRecord in testRecords)
{
_ = await store.UpsertAsync(collection, testRecord);
}
// Act
var topNResultDefault = await store.GetNearestMatchAsync(collection, compareEmbedding, minRelevanceScore: threshold);
var topNResultWithEmbedding = await store.GetNearestMatchAsync(collection, compareEmbedding, minRelevanceScore: threshold, withEmbedding: true);
// Assert
Assert.NotNull(topNResultDefault);
Assert.NotNull(topNResultWithEmbedding);
Assert.Empty(topNResultDefault.Value.Item1.Embedding.Vector);
Assert.NotEmpty(topNResultWithEmbedding.Value.Item1.Embedding.Vector);
}
[Fact]
public async Task GetNearestMatchAsyncReturnsExpectedAsync()
{
// Arrange
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
var compareEmbedding = new Embedding<float>(new float[] { 1, 1, 1 });
string collection = "test_collection";
int topN = 1;
double threshold = 0.75;
var testEmbeddings = new[]
{
new Embedding<float>(new float[] { 1, 1, 1 }),
new Embedding<float>(new float[] { -1, -1, -1 }),
new Embedding<float>(new float[] { 1, 2, 3 }),
new Embedding<float>(new float[] { -1, -2, -3 }),
new Embedding<float>(new float[] { 1, -1, -2 })
};
var testRecords = new List<MemoryRecord>();
for (int i = 0; i < testEmbeddings.Length; i++)
{
testRecords.Add(MemoryRecord.LocalRecord(
id: "test" + i,
text: "text" + i,
description: "description" + i,
embedding: testEmbeddings[i]));
}
this.MockCreateIndex(collection, () =>
{
for (int i = 0; i < testRecords.Count; i++)
{
if (i + 1 < testRecords.Count)
{
this.MockHashSet(collection, testRecords[i]);
}
else
{
this.MockHashSet(collection, testRecords[i], () =>
{
this.MockSearch(collection, compareEmbedding, topN, threshold);
});
}
}
});
await store.CreateCollectionAsync(collection);
foreach (var testRecord in testRecords)
{
_ = await store.UpsertAsync(collection, testRecord);
}
// Act
var topNResult = await store.GetNearestMatchAsync(collection, compareEmbedding, minRelevanceScore: threshold);
// Assert
Assert.NotNull(topNResult);
Assert.Equal("test0", topNResult.Value.Item1.Metadata.Id);
Assert.True(topNResult.Value.Item2 >= threshold);
}
[Fact]
public async Task GetNearestMatchesDifferentiatesIdenticalVectorsByKeyAsync()
{
// Arrange
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
var compareEmbedding = new Embedding<float>(new float[] { 1, 1, 1 });
int topN = 4;
double threshold = 0.75;
string collection = "test_collection";
var testRecords = new List<MemoryRecord>();
for (int i = 0; i < 10; i++)
{
testRecords.Add(MemoryRecord.LocalRecord(
id: "test" + i,
text: "text" + i,
description: "description" + i,
embedding: new Embedding<float>(new float[] { 1, 1, 1 })));
}
this.MockCreateIndex(collection, () =>
{
for (int i = 0; i < testRecords.Count; i++)
{
if (i + 1 < testRecords.Count)
{
this.MockHashSet(collection, testRecords[i]);
}
else
{
this.MockHashSet(collection, testRecords[i], () =>
{
this.MockSearch(collection, compareEmbedding, topN, threshold);
});
}
}
});
await store.CreateCollectionAsync(collection);
foreach (var testRecord in testRecords)
{
_ = await store.UpsertAsync(collection, testRecord);
}
// Act
var topNResults = store.GetNearestMatchesAsync(collection, compareEmbedding, limit: topN, minRelevanceScore: threshold).ToEnumerable().ToArray();
IEnumerable<string> topNKeys = topNResults.Select(x => x.Item1.Key).ToImmutableSortedSet();
// Assert
Assert.Equal(topN, topNResults.Length);
Assert.Equal(topN, topNKeys.Count());
for (int i = 0; i < topNResults.Length; i++)
{
int compare = topNResults[i].Item2.CompareTo(threshold);
Assert.True(compare >= 0);
}
}
[Fact]
public async Task ItCanBatchUpsertRecordsAsync()
{
// Arrange
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
int numRecords = 10;
string collection = "test_collection";
IEnumerable<MemoryRecord> records = this.CreateBatchRecords(numRecords);
this.MockCreateIndex(collection, () =>
{
foreach (var testRecord in records)
{
this.MockHashSet(collection, testRecord);
}
});
// Act
await store.CreateCollectionAsync(collection);
var keys = store.UpsertBatchAsync(collection, records);
var resultRecords = store.GetBatchAsync(collection, keys.ToEnumerable());
// Assert
Assert.NotNull(keys);
Assert.Equal(numRecords, keys.ToEnumerable().Count());
Assert.Equal(numRecords, resultRecords.ToEnumerable().Count());
}
[Fact]
public async Task ItCanBatchGetRecordsAsync()
{
// Arrange
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
int numRecords = 10;
string collection = "test_collection";
IEnumerable<MemoryRecord> records = this.CreateBatchRecords(numRecords);
this.MockCreateIndex(collection, () =>
{
foreach (var testRecord in records)
{
this.MockHashSet(collection, testRecord);
}
});
var keys = store.UpsertBatchAsync(collection, records);
// Act
await store.CreateCollectionAsync(collection);
var results = store.GetBatchAsync(collection, keys.ToEnumerable());
// Assert
Assert.NotNull(keys);
Assert.NotNull(results);
Assert.Equal(numRecords, results.ToEnumerable().Count());
}
[Fact]
public async Task ItCanBatchRemoveRecordsAsync()
{
// Arrange
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
int numRecords = 10;
string collection = "test_collection";
IEnumerable<MemoryRecord> records = this.CreateBatchRecords(numRecords);
this.MockCreateIndex(collection, () =>
{
foreach (var testRecord in records)
{
this.MockHashSet(collection, testRecord, () => this.MockKeyDelete(collection, testRecord.Metadata.Id));
}
this.MockKeyDelete(collection, records.Select(x => x.Metadata.Id));
});
await store.CreateCollectionAsync(collection);
List<string> keys = new();
// Act
await foreach (var key in store.UpsertBatchAsync(collection, records))
{
keys.Add(key);
}
await store.RemoveBatchAsync(collection, keys);
// Assert
await foreach (var result in store.GetBatchAsync(collection, keys))
{
Assert.Null(result);
}
}
[Fact]
public async Task GetNearestMatchAsyncThrowsExceptionOnInvalidVectorScoreAsync()
{
// Arrange
RedisMemoryStore store = new(this._mockDatabase.Object, vectorSize: 3);
var compareEmbedding = new Embedding<float>(new float[] { 1, 1, 1 });
string collection = "test_collection";
int topN = 1;
double threshold = 0.75;
var testEmbeddings = new[]
{
new Embedding<float>(new float[] { 1, 1, 1 }),
new Embedding<float>(new float[] { -1, -1, -1 }),
new Embedding<float>(new float[] { 1, 2, 3 }),
new Embedding<float>(new float[] { -1, -2, -3 }),
new Embedding<float>(new float[] { 1, -1, -2 })
};
var testRecords = new List<MemoryRecord>();
for (int i = 0; i < testEmbeddings.Length; i++)
{
testRecords.Add(MemoryRecord.LocalRecord(
id: "test" + i,
text: "text" + i,
description: "description" + i,
embedding: testEmbeddings[i]));
}
this.MockCreateIndex(collection, () =>
{
for (int i = 0; i < testRecords.Count; i++)
{
if (i + 1 < testRecords.Count)
{
this.MockHashSet(collection, testRecords[i]);
}
else
{
this.MockHashSet(collection, testRecords[i], () =>
{
this.MockSearch(collection, compareEmbedding, topN, threshold, returnStringVectorScore: true);
});
}
}
});
await store.CreateCollectionAsync(collection);
foreach (var testRecord in testRecords)
{
_ = await store.UpsertAsync(collection, testRecord);
}
// Assert
RedisMemoryStoreException ex = await Assert.ThrowsAsync<RedisMemoryStoreException>(async () =>
{
// Act
await store.GetNearestMatchAsync(collection, compareEmbedding, minRelevanceScore: threshold);
});
Assert.Equal(ex.Message, "Invalid or missing vector score value.");
}
#region private
private void MockCreateIndex(string collection, Action? callback = null)
{
this._mockDatabase
.Setup<Task<RedisResult>>(x => x.ExecuteAsync(
It.Is<string>(x => x == "FT.CREATE"),
It.Is<object[]>(x => x[0].ToString() == collection))
)
.ReturnsAsync(RedisResult.Create("OK", ResultType.SimpleString))
.Callback(() =>
{
this._collections.TryAdd(collection, new());
this._mockDatabase
.Setup<Task<RedisResult>>(x => x.ExecuteAsync(
It.Is<string>(x => x == "FT.INFO"),
It.Is<object[]>(x => x[0].ToString() == collection))
)
.ReturnsAsync(RedisResult.Create(new[] {
RedisResult.Create("index_name", ResultType.BulkString),
RedisResult.Create(collection, ResultType.BulkString)
}));
this._mockDatabase
.Setup<Task<RedisResult>>(x => x.ExecuteAsync(
It.Is<string>(x => x == "FT._LIST"),
It.IsAny<object[]>())
)
.ReturnsAsync(RedisResult.Create(this._collections.Select(x => RedisResult.Create(x.Key, ResultType.BulkString)).ToArray()));
callback?.Invoke();
});
this._mockDatabase
.Setup<Task<RedisResult>>(x => x.ExecuteAsync(It.Is<string>(x => x == "FT.INFO"), It.IsAny<object[]>()))
.Throws(new RedisServerException("Unknown Index name"));
}
private void MockDropIndex(string collection, Action? callback = null)
{
this._mockDatabase
.Setup<Task<RedisResult>>(x => x.ExecuteAsync(
It.Is<string>(x => x == "FT.DROPINDEX"),
It.Is<object[]>(x => x[0].ToString() == collection && x[1].ToString() == "DD"))
)
.ReturnsAsync(RedisResult.Create("OK", ResultType.SimpleString))
.Callback(() =>
{
this._collections.Remove(collection);
this._mockDatabase
.Setup<Task<RedisResult>>(x => x.ExecuteAsync(
It.Is<string>(x => x == "FT.INFO"),
It.Is<object[]>(x => x[0].ToString() == collection))
)
.Throws(new RedisServerException("Unknown Index name"));
this._mockDatabase
.Setup<Task<RedisResult>>(x => x.ExecuteAsync(
It.Is<string>(x => x == "FT._LIST"),
It.IsAny<object[]>())
)
.ReturnsAsync(RedisResult.Create(this._collections.Select(x => RedisResult.Create(x.Key, ResultType.BulkString)).ToArray()));
});
}
private void MockHashSet(string collection, MemoryRecord record, Action? callback = null)
{
string redisKey = $"{collection}:{record.Metadata.Id}";
byte[] embedding = MemoryMarshal.Cast<float, byte>(record.Embedding.AsReadOnlySpan()).ToArray();
long timestamp = record.Timestamp?.ToUnixTimeMilliseconds() ?? -1;
this._mockDatabase
.Setup<Task>(x => x.HashSetAsync(
It.Is<RedisKey>(x => x == redisKey),
It.Is<HashEntry[]>(x => x.Length == 4 &&
x[0].Name == "key" && x[1].Name == "metadata" && x[2].Name == "embedding" && x[3].Name == "timestamp" &&
x[0].Value == record.Key && x[1].Value == record.GetSerializedMetadata() && embedding.SequenceEqual((byte[])x[2].Value!) && x[3].Value == timestamp
),
It.IsAny<CommandFlags>())
)
.Callback(() =>
{
(this._collections[collection] ??= new()).Add(record);
this._mockDatabase
.Setup<Task<HashEntry[]>>(x => x.HashGetAllAsync(It.Is<RedisKey>(x => x == redisKey), It.IsAny<CommandFlags>()))
.ReturnsAsync(new[] {
new HashEntry("key", record.Key),
new HashEntry("metadata", record.GetSerializedMetadata()),
new HashEntry("embedding", embedding),
new HashEntry("timestamp", timestamp)
});
callback?.Invoke();
});
}
private void MockKeyDelete(string collection, string key, Action? callback = null)
{
string redisKey = $"{collection}:{key}";
this._mockDatabase
.Setup<Task<bool>>(x => x.KeyDeleteAsync(
It.Is<RedisKey>(x => x == redisKey),
It.IsAny<CommandFlags>())
)
.ReturnsAsync(true)
.Callback(() =>
{
(this._collections[collection] ??= new()).RemoveAll(x => x.Key == key);
this._mockDatabase
.Setup<Task<HashEntry[]>>(x => x.HashGetAllAsync(It.Is<RedisKey>(x => x == redisKey), It.IsAny<CommandFlags>()))
.ReturnsAsync(Array.Empty<HashEntry>());
callback?.Invoke();
});
}
private void MockKeyDelete(string collection, IEnumerable<string> keys, Action? callback = null)
{
RedisKey[] redisKeys = keys.Distinct().Select(key => new RedisKey($"{collection}:{key}")).ToArray();
this._mockDatabase
.Setup<Task<long>>(x => x.KeyDeleteAsync(
It.Is<RedisKey[]>(x => redisKeys.SequenceEqual(x)),
It.IsAny<CommandFlags>())
)
.ReturnsAsync(redisKeys.Length)
.Callback(() =>
{
(this._collections[collection] ??= new()).RemoveAll(x => keys.Contains(x.Key));
foreach (var redisKey in redisKeys)
{
this._mockDatabase
.Setup<Task<HashEntry[]>>(x => x.HashGetAllAsync(It.Is<RedisKey>(x => x == redisKey), It.IsAny<CommandFlags>()))
.ReturnsAsync(Array.Empty<HashEntry>());
}
callback?.Invoke();
});
}
private void MockSearch(string collection, Embedding<float> compareEmbedding, int topN, double threshold, bool returnStringVectorScore = false)
{
TopNCollection<MemoryRecord> embeddings = new(topN);
List<MemoryRecord> records = this._collections.TryGetValue(collection, out var value) ? value : new();
foreach (var record in records)
{
double similarity = compareEmbedding
.AsReadOnlySpan()
.CosineSimilarity(record.Embedding.AsReadOnlySpan());
if (similarity >= threshold)
{
embeddings.Add(new(record, similarity));
}
}
embeddings.SortByScore();
string redisKey = $"{collection}";
var redisResults = new List<RedisResult>();
redisResults.Add(RedisResult.Create(embeddings.Count));
foreach (var item in embeddings)
{
long timestamp = item.Value.Timestamp?.ToUnixTimeMilliseconds() ?? -1;
byte[] embedding = MemoryMarshal.Cast<float, byte>(item.Value.Embedding.AsReadOnlySpan()).ToArray();
redisResults.Add(RedisResult.Create($"{collection}:{item.Value.Metadata.Id}", ResultType.BulkString));
redisResults.Add(RedisResult.Create(
new RedisResult[]
{
RedisResult.Create("key", ResultType.BulkString),
RedisResult.Create(item.Value.Metadata.Id, ResultType.BulkString),
RedisResult.Create("metadata", ResultType.BulkString),
RedisResult.Create(item.Value.GetSerializedMetadata(), ResultType.BulkString),
RedisResult.Create("embedding", ResultType.BulkString),
RedisResult.Create(embedding, ResultType.BulkString),
RedisResult.Create("timestamp", ResultType.BulkString),
RedisResult.Create(timestamp, ResultType.BulkString),
RedisResult.Create("vector_score", ResultType.BulkString),
RedisResult.Create(returnStringVectorScore ? $"score:{1-item.Score.Value}" : 1-item.Score.Value, ResultType.BulkString),
})
);
}
this._mockDatabase
.Setup<Task<RedisResult>>(x => x.ExecuteAsync(
It.Is<string>(x => x == "FT.SEARCH"),
It.Is<object[]>(x => x[0].ToString() == collection && x[1].ToString() == $"*=>[KNN {topN} @embedding $embedding AS vector_score]"))
)
.ReturnsAsync(RedisResult.Create(redisResults.ToArray()));
}
private IEnumerable<MemoryRecord> CreateBatchRecords(int numRecords)
{
Assert.True(numRecords % 2 == 0, "Number of records must be even");
Assert.True(numRecords > 0, "Number of records must be greater than 0");
IEnumerable<MemoryRecord> records = new List<MemoryRecord>(numRecords);
for (int i = 0; i < numRecords / 2; i++)
{
var testRecord = MemoryRecord.LocalRecord(
id: "test" + i,
text: "text" + i,
description: "description" + i,
embedding: new Embedding<float>(new float[] { 1, 1, 1 }));
records = records.Append(testRecord);
}
for (int i = numRecords / 2; i < numRecords; i++)
{
var testRecord = MemoryRecord.ReferenceRecord(
externalId: "test" + i,
sourceName: "sourceName" + i,
description: "description" + i,
embedding: new Embedding<float>(new float[] { 1, 2, 3 }));
records = records.Append(testRecord);
}
return records;
}
#endregion
}
@@ -0,0 +1,78 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Threading.Tasks;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.Memory.Redis;
using Microsoft.SemanticKernel.Memory;
using RepoUtils;
using StackExchange.Redis;
// ReSharper disable once InconsistentNaming
public static class Example47_Redis
{
private const string MemoryCollectionName = "redis-test";
public static async Task RunAsync()
{
string configuration = Env.Var("REDIS_CONFIGURATION");
using ConnectionMultiplexer connectionMultiplexer = await ConnectionMultiplexer.ConnectAsync(configuration);
IDatabase database = connectionMultiplexer.GetDatabase();
RedisMemoryStore memoryStore = new(database, vectorSize: 1536);
IKernel kernel = Kernel.Builder
.WithLogger(ConsoleLogger.Log)
.WithOpenAITextCompletionService("text-davinci-003", Env.Var("OPENAI_API_KEY"))
.WithOpenAITextEmbeddingGenerationService("text-embedding-ada-002", Env.Var("OPENAI_API_KEY"))
.WithMemoryStorage(memoryStore)
.Build();
Console.WriteLine("== Printing Collections in DB ==");
var collections = memoryStore.GetCollectionsAsync();
await foreach (var collection in collections)
{
Console.WriteLine(collection);
}
Console.WriteLine("== Adding Memories ==");
var key1 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat1", text: "british short hair");
var key2 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat2", text: "orange tabby");
var key3 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat3", text: "norwegian forest cat");
Console.WriteLine("== Printing Collections in DB ==");
collections = memoryStore.GetCollectionsAsync();
await foreach (var collection in collections)
{
Console.WriteLine(collection);
}
Console.WriteLine("== Retrieving Memories Through the Kernel ==");
MemoryQueryResult? lookup = await kernel.Memory.GetAsync(MemoryCollectionName, "cat1");
Console.WriteLine(lookup != null ? lookup.Metadata.Text : "ERROR: memory not found");
Console.WriteLine("== Retrieving Memories Directly From the Store ==");
var memory1 = await memoryStore.GetAsync(MemoryCollectionName, key1);
var memory2 = await memoryStore.GetAsync(MemoryCollectionName, key2);
var memory3 = await memoryStore.GetAsync(MemoryCollectionName, key3);
Console.WriteLine(memory1 != null ? memory1.Metadata.Text : "ERROR: memory not found");
Console.WriteLine(memory2 != null ? memory2.Metadata.Text : "ERROR: memory not found");
Console.WriteLine(memory3 != null ? memory3.Metadata.Text : "ERROR: memory not found");
Console.WriteLine("== Similarity Searching Memories: My favorite color is orange ==");
var searchResults = kernel.Memory.SearchAsync(MemoryCollectionName, "My favorite color is orange", limit: 3, minRelevanceScore: 0.8);
await foreach (var item in searchResults)
{
Console.WriteLine(item.Metadata.Text + " : " + item.Relevance);
}
Console.WriteLine("== Removing Collection {0} ==", MemoryCollectionName);
await memoryStore.DeleteCollectionAsync(MemoryCollectionName);
Console.WriteLine("== Printing Collections in DB ==");
await foreach (var collection in collections)
{
Console.WriteLine(collection);
}
}
}
@@ -34,6 +34,7 @@
<ProjectReference Include="..\..\..\dotnet\src\Connectors\Connectors.Memory.AzureCognitiveSearch\Connectors.Memory.AzureCognitiveSearch.csproj" />
<ProjectReference Include="..\..\..\dotnet\src\Connectors\Connectors.Memory.Postgres\Connectors.Memory.Postgres.csproj" />
<ProjectReference Include="..\..\..\dotnet\src\Connectors\Connectors.Memory.Weaviate\Connectors.Memory.Weaviate.csproj" />
<ProjectReference Include="..\..\..\dotnet\src\Connectors\Connectors.Memory.Redis\Connectors.Memory.Redis.csproj" />
<ProjectReference Include="..\..\..\dotnet\src\Extensions\Planning.ActionPlanner\Planning.ActionPlanner.csproj" />
<ProjectReference Include="..\..\..\dotnet\src\Extensions\Planning.SequentialPlanner\Planning.SequentialPlanner.csproj" />
<ProjectReference Include="..\..\..\dotnet\src\Connectors\Connectors.Memory.Pinecone\Connectors.Memory.Pinecone.csproj" />
@@ -123,6 +123,7 @@ public static class Program
Console.WriteLine("== DONE ==");
await Example39_Postgres.RunAsync();
Console.WriteLine("== DONE ==");
await Example40_DIContainer.RunAsync();
Console.WriteLine("== DONE ==");
@@ -144,5 +145,8 @@ public static class Program
await Example46_Weaviate.RunAsync();
Console.WriteLine("== DONE ==");
await Example47_Redis.RunAsync();
Console.WriteLine("== DONE ==");
}
}
@@ -36,6 +36,7 @@ dotnet user-secrets set "WEAVIATE_PORT" "..."
dotnet user-secrets set "WEAVIATE_APIKEY" "..."
dotnet user-secrets set "GITHUB_PERSONAL_ACCESS_TOKEN" "github_pat_..."
dotnet user-secrets set "POSTGRES_CONNECTIONSTRING" "..."
dotnet user-secrets set "REDIS_CONFIGURATION" "..."
```
To set your secrets with environment variables, use these names:
@@ -56,6 +57,7 @@ To set your secrets with environment variables, use these names:
* WEAVIATE_APIKEY
* GITHUB_PERSONAL_ACCESS_TOKEN
* POSTGRES_CONNECTIONSTRING
* REDIS_CONFIGURATION
* AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME
* AZURE_OPENAI_EMBEDDINGS_ENDPOINT
* AZURE_OPENAI_EMBEDDINGS_KEY