From e8b31a228c36f5592eea262eea511edbb2317d27 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Fri, 10 Jan 2025 14:17:08 +0000 Subject: [PATCH] .Net: Add shared integration tests for checking vector search scores (#10144) ### Motivation and Context Adds tests for verifying the scores as mentioned in #10103 Only adding tests for passing connectors right now. ### Description Adding a common base test class with integration tests for checking the returned vector scores. Adding subclasses for InMemory, Qdrant and Postgres, since these are passing end to end. ### Contribution Checklist - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone :smile: --- .../BaseVectorStoreRecordCollectionTests.cs | 138 ++++++++++++++++++ ...nMemoryVectorStoreRecordCollectionTests.cs | 31 ++++ ...ostgresVectorStoreRecordCollectionTests.cs | 34 +++++ .../Postgres/PostgresVectorStoreFixture.cs | 16 +- ...nQdrantVectorStoreRecordCollectionTests.cs | 35 +++++ 5 files changed, 246 insertions(+), 8 deletions(-) create mode 100644 dotnet/src/IntegrationTests/Connectors/Memory/BaseVectorStoreRecordCollectionTests.cs create mode 100644 dotnet/src/IntegrationTests/Connectors/Memory/InMemory/CommonInMemoryVectorStoreRecordCollectionTests.cs create mode 100644 dotnet/src/IntegrationTests/Connectors/Memory/Postgres/CommonPostgresVectorStoreRecordCollectionTests.cs create mode 100644 dotnet/src/IntegrationTests/Connectors/Memory/Qdrant/CommonQdrantVectorStoreRecordCollectionTests.cs diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/BaseVectorStoreRecordCollectionTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/BaseVectorStoreRecordCollectionTests.cs new file mode 100644 index 0000000000..9f325cab6e --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/Memory/BaseVectorStoreRecordCollectionTests.cs @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using Xunit; + +namespace SemanticKernel.IntegrationTests.Connectors.Memory; + +/// +/// Base class for common integration tests that should pass for any . +/// +/// The type of key to use with the record collection. +public abstract class BaseVectorStoreRecordCollectionTests + where TKey : notnull +{ + protected abstract TKey Key1 { get; } + protected abstract TKey Key2 { get; } + protected abstract TKey Key3 { get; } + protected abstract TKey Key4 { get; } + + protected abstract HashSet GetSupportedDistanceFunctions(); + + protected abstract IVectorStoreRecordCollection GetTargetRecordCollection(string recordCollectionName, VectorStoreRecordDefinition? vectorStoreRecordDefinition); + + protected virtual int DelayAfterIndexCreateInMilliseconds { get; } = 0; + + protected virtual int DelayAfterUploadInMilliseconds { get; } = 0; + + [Theory] + [InlineData(DistanceFunction.CosineDistance, 0, 2, 1, new int[] { 0, 2, 1 })] + [InlineData(DistanceFunction.CosineSimilarity, 1, -1, 0, new int[] { 0, 2, 1 })] + [InlineData(DistanceFunction.DotProductSimilarity, 1, -1, 0, new int[] { 0, 2, 1 })] + [InlineData(DistanceFunction.EuclideanDistance, 0, 2, 1.73, new int[] { 0, 2, 1 })] + [InlineData(DistanceFunction.EuclideanSquaredDistance, 0, 4, 3, new int[] { 0, 2, 1 })] + [InlineData(DistanceFunction.Hamming, 0, 1, 3, new int[] { 0, 1, 2 })] + [InlineData(DistanceFunction.ManhattanDistance, 0, 2, 3, new int[] { 0, 1, 2 })] + public async Task VectorSearchShouldReturnExpectedScoresAsync(string distanceFunction, double expectedExactMatchScore, double expectedOppositeScore, double expectedOrthogonalScore, int[] resultOrder) + { + var keyDictionary = new Dictionary + { + { 0, this.Key1 }, + { 1, this.Key2 }, + { 2, this.Key3 }, + }; + var scoreDictionary = new Dictionary + { + { 0, expectedExactMatchScore }, + { 1, expectedOppositeScore }, + { 2, expectedOrthogonalScore }, + }; + + // Don't test unsupported distance functions. + var supportedDistanceFunctions = this.GetSupportedDistanceFunctions(); + if (!supportedDistanceFunctions.Contains(distanceFunction)) + { + return; + } + + // Arrange + var definition = CreateKeyWithVectorRecordDefinition(4, distanceFunction); + var sut = this.GetTargetRecordCollection>( + $"scorebydistancefunction{distanceFunction}", + definition); + + await sut.CreateCollectionIfNotExistsAsync(); + await Task.Delay(this.DelayAfterIndexCreateInMilliseconds); + + // Create two vectors that are opposite to each other and records that use these + // plus a further vector that is orthogonal to the base vector. + var baseVector = new ReadOnlyMemory([1, 0, 0, 0]); + var oppositeVector = new ReadOnlyMemory([-1, 0, 0, 0]); + var orthogonalVector = new ReadOnlyMemory([0f, -1f, -1f, 0f]); + + var baseRecord = new KeyWithVectorRecord + { + Key = this.Key1, + Vector = baseVector, + }; + + var oppositeRecord = new KeyWithVectorRecord + { + Key = this.Key2, + Vector = oppositeVector, + }; + + var orthogonalRecord = new KeyWithVectorRecord + { + Key = this.Key3, + Vector = orthogonalVector, + }; + + await sut.UpsertBatchAsync([baseRecord, oppositeRecord, orthogonalRecord]).ToListAsync(); + await Task.Delay(this.DelayAfterUploadInMilliseconds); + + // Act + var searchResult = await sut.VectorizedSearchAsync(baseVector); + + // Assert + var results = await searchResult.Results.ToListAsync(); + Assert.Equal(3, results.Count); + + Assert.Equal(keyDictionary[resultOrder[0]], results[0].Record.Key); + Assert.Equal(Math.Round(scoreDictionary[resultOrder[0]], 2), Math.Round(results[0].Score!.Value, 2)); + + Assert.Equal(keyDictionary[resultOrder[1]], results[1].Record.Key); + Assert.Equal(Math.Round(scoreDictionary[resultOrder[1]], 2), Math.Round(results[1].Score!.Value, 2)); + + Assert.Equal(keyDictionary[resultOrder[2]], results[2].Record.Key); + Assert.Equal(Math.Round(scoreDictionary[resultOrder[2]], 2), Math.Round(results[2].Score!.Value, 2)); + + // Cleanup + await sut.DeleteCollectionAsync(); + } + + private static VectorStoreRecordDefinition CreateKeyWithVectorRecordDefinition(int vectorDimensions, string distanceFunction) + { + var definition = new VectorStoreRecordDefinition + { + Properties = + [ + new VectorStoreRecordKeyProperty("Key", typeof(TKey)), + new VectorStoreRecordVectorProperty("Vector", typeof(ReadOnlyMemory)) { Dimensions = vectorDimensions, DistanceFunction = distanceFunction }, + ], + }; + + return definition; + } + + private sealed class KeyWithVectorRecord + { + public required TRecordKey Key { get; set; } + + public ReadOnlyMemory Vector { get; set; } + } +} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/InMemory/CommonInMemoryVectorStoreRecordCollectionTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/InMemory/CommonInMemoryVectorStoreRecordCollectionTests.cs new file mode 100644 index 0000000000..74c4f4b449 --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/Memory/InMemory/CommonInMemoryVectorStoreRecordCollectionTests.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel.Connectors.InMemory; + +namespace SemanticKernel.IntegrationTests.Connectors.Memory.InMemory; + +/// +/// Inherits common integration tests that should pass for any . +/// +public class CommonInMemoryVectorStoreRecordCollectionTests() : BaseVectorStoreRecordCollectionTests +{ + protected override string Key1 => "1"; + protected override string Key2 => "2"; + protected override string Key3 => "3"; + protected override string Key4 => "4"; + + protected override IVectorStoreRecordCollection GetTargetRecordCollection(string recordCollectionName, VectorStoreRecordDefinition? vectorStoreRecordDefinition) + { + return new InMemoryVectorStoreRecordCollection(recordCollectionName, new() + { + VectorStoreRecordDefinition = vectorStoreRecordDefinition + }); + } + + protected override HashSet GetSupportedDistanceFunctions() + { + return [DistanceFunction.CosineDistance, DistanceFunction.CosineSimilarity, DistanceFunction.DotProductSimilarity, DistanceFunction.EuclideanDistance]; + } +} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Postgres/CommonPostgresVectorStoreRecordCollectionTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Postgres/CommonPostgresVectorStoreRecordCollectionTests.cs new file mode 100644 index 0000000000..e8783adf37 --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Postgres/CommonPostgresVectorStoreRecordCollectionTests.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel.Connectors.Postgres; +using Xunit; + +namespace SemanticKernel.IntegrationTests.Connectors.Memory.Postgres; + +/// +/// Inherits common integration tests that should pass for any . +/// +/// Postres setup and teardown. +[Collection("PostgresVectorStoreCollection")] +public class CommonPostgresVectorStoreRecordCollectionTests(PostgresVectorStoreFixture fixture) : BaseVectorStoreRecordCollectionTests +{ + protected override string Key1 => "1"; + protected override string Key2 => "2"; + protected override string Key3 => "3"; + protected override string Key4 => "4"; + + protected override IVectorStoreRecordCollection GetTargetRecordCollection(string recordCollectionName, VectorStoreRecordDefinition? vectorStoreRecordDefinition) + { + return new PostgresVectorStoreRecordCollection(fixture.DataSource!, recordCollectionName, new() + { + VectorStoreRecordDefinition = vectorStoreRecordDefinition + }); + } + + protected override HashSet GetSupportedDistanceFunctions() + { + return [DistanceFunction.CosineDistance, DistanceFunction.CosineSimilarity, DistanceFunction.DotProductSimilarity, DistanceFunction.EuclideanDistance, DistanceFunction.ManhattanDistance]; + } +} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Postgres/PostgresVectorStoreFixture.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Postgres/PostgresVectorStoreFixture.cs index 5888a513ac..f0d194fb6d 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Postgres/PostgresVectorStoreFixture.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Postgres/PostgresVectorStoreFixture.cs @@ -34,7 +34,7 @@ public class PostgresVectorStoreFixture : IAsyncLifetime /// /// Holds the Npgsql data source to use for tests. /// - private NpgsqlDataSource? _dataSource; + public NpgsqlDataSource? DataSource { get; private set; } private string _connectionString = null!; private string _databaseName = null!; @@ -42,14 +42,14 @@ public class PostgresVectorStoreFixture : IAsyncLifetime /// /// Gets a vector store to use for tests. /// - public IVectorStore VectorStore => new PostgresVectorStore(this._dataSource!); + public IVectorStore VectorStore => new PostgresVectorStore(this.DataSource!); /// /// Get a database connection /// public NpgsqlConnection GetConnection() { - return this._dataSource!.OpenConnection(); + return this.DataSource!.OpenConnection(); } public IVectorStoreRecordCollection GetCollection( @@ -81,7 +81,7 @@ public class PostgresVectorStoreFixture : IAsyncLifetime NpgsqlDataSourceBuilder dataSourceBuilder = new(connectionStringBuilder.ToString()); dataSourceBuilder.UseVector(); - this._dataSource = dataSourceBuilder.Build(); + this.DataSource = dataSourceBuilder.Build(); // Wait for the postgres container to be ready and create the test database using the initial data source. var initialDataSource = NpgsqlDataSource.Create(this._connectionString); @@ -124,7 +124,7 @@ public class PostgresVectorStoreFixture : IAsyncLifetime private async Task CreateTableAsync() { - NpgsqlConnection connection = await this._dataSource!.OpenConnectionAsync().ConfigureAwait(false); + NpgsqlConnection connection = await this.DataSource!.OpenConnectionAsync().ConfigureAwait(false); await using (connection) { @@ -150,9 +150,9 @@ public class PostgresVectorStoreFixture : IAsyncLifetime /// An async task. public async Task DisposeAsync() { - if (this._dataSource != null) + if (this.DataSource != null) { - this._dataSource.Dispose(); + this.DataSource.Dispose(); } await this.DropDatabaseAsync(); @@ -218,7 +218,7 @@ public class PostgresVectorStoreFixture : IAsyncLifetime await command.ExecuteNonQueryAsync(); } - await using (NpgsqlConnection conn = await this._dataSource!.OpenConnectionAsync()) + await using (NpgsqlConnection conn = await this.DataSource!.OpenConnectionAsync()) { await using (NpgsqlCommand command = new("CREATE EXTENSION vector", conn)) { diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Qdrant/CommonQdrantVectorStoreRecordCollectionTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Qdrant/CommonQdrantVectorStoreRecordCollectionTests.cs new file mode 100644 index 0000000000..3bafffc6a3 --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Qdrant/CommonQdrantVectorStoreRecordCollectionTests.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel.Connectors.Qdrant; +using Xunit; + +namespace SemanticKernel.IntegrationTests.Connectors.Memory.Qdrant; + +/// +/// Inherits common integration tests that should pass for any . +/// +/// Qdrant setup and teardown. +[Collection("QdrantVectorStoreCollection")] +public class CommonQdrantVectorStoreRecordCollectionTests(QdrantVectorStoreFixture fixture) : BaseVectorStoreRecordCollectionTests +{ + protected override ulong Key1 => 1; + protected override ulong Key2 => 2; + protected override ulong Key3 => 3; + protected override ulong Key4 => 4; + + protected override IVectorStoreRecordCollection GetTargetRecordCollection(string recordCollectionName, VectorStoreRecordDefinition? vectorStoreRecordDefinition) + { + return new QdrantVectorStoreRecordCollection(fixture.QdrantClient, recordCollectionName, new() + { + HasNamedVectors = true, + VectorStoreRecordDefinition = vectorStoreRecordDefinition + }); + } + + protected override HashSet GetSupportedDistanceFunctions() + { + return [DistanceFunction.CosineSimilarity, DistanceFunction.EuclideanDistance, DistanceFunction.ManhattanDistance]; + } +}