microsoft/semantic-kernel
1
0
Fork 0
mirror of https://github.com/microsoft/semantic-kernel.git synced 2026-10-12 02:19:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

.Net: Update SK to use MEVD packages and move MEVD projects out of main solution (#14117)

Browse Source 
### Motivation and Context

- The MEVD packages are not shipped from this repo anymore.

### Description

- Moving the packages out of the main SK solution
- Update all projects to use MEVD.Abstractions package directly
- Revert some previous code changes to make the MEVD implementations
compatible with the last MEVD.Abstractions package released from this
repo.

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [ ] The code builds clean without any errors or warnings
- [ ] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [ ] All unit tests pass, and I have added new tests where possible
- [ ] I didn't break anyone 😄
This commit is contained in:
westey
2026-06-24 16:51:21 +00:00
committed by GitHub
parent 88e1060b99
commit 82f244233b
86 changed files with 135 additions and 4868 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/dotnet-build-and-test.yml
+3 -3
View File
@@ -81,7 +81,7 @@ jobs:
- name: Build dotnet solutions
shell: bash
run: |
export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | tr '\n' ' ')
export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | grep -v "MEVD.slnx" | tr '\n' ' ')
for solution in $SOLUTIONS; do
dotnet build $solution -c ${{ matrix.configuration }} --warnaserror
done
@@ -89,7 +89,7 @@ jobs:
- name: Package install check
shell: bash
run: |
export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | tr '\n' ' ')
export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | grep -v "MEVD.slnx" | tr '\n' ' ')
for solution in $SOLUTIONS; do
dotnet pack $solution -c ${{ matrix.configuration }} --no-build --no-restore --output ./artifacts
done
@@ -115,7 +115,7 @@ jobs:
- name: Run Unit Tests
shell: bash
run: |
export UT_PROJECTS=$(find ./dotnet -type f -name "*.UnitTests.csproj" | grep -v -E "(Experimental.Orchestration.Flow.UnitTests.csproj|Experimental.Assistants.UnitTests.csproj)" | tr '\n' ' ')
export UT_PROJECTS=$(find ./dotnet -type f -name "*.UnitTests.csproj" | grep -v -E "(Experimental.Orchestration.Flow.UnitTests.csproj|Experimental.Assistants.UnitTests.csproj)" | grep -v "test/VectorData/" | tr '\n' ' ')
for project in $UT_PROJECTS; do
dotnet test -c ${{ matrix.configuration }} $project --no-build -v Normal --logger trx --collect:"XPlat Code Coverage" --results-directory:"TestResults/Coverage/" -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.ExcludeByAttribute=GeneratedCodeAttribute,CompilerGeneratedAttribute,ExcludeFromCodeCoverageAttribute
done
dotnet/Directory.Packages.props
+11
View File
@@ -152,6 +152,7 @@
<PackageVersion Include="Microsoft.Extensions.TimeProvider.Testing" Version="10.2.0" />
<PackageVersion Include="Microsoft.Extensions.FileProviders.Physical" Version="10.0.2" />
<PackageVersion Include="Microsoft.Extensions.FileProviders.Embedded" Version="10.0.2" />
<PackageVersion Include="Microsoft.Extensions.VectorData.Abstractions" Version="10.1.0" />
<!-- Test -->
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
<PackageVersion Include="Moq" Version="[4.18.4]" />
@@ -200,6 +201,16 @@
<PackageVersion Include="Testcontainers.Redis" Version="4.6.0" />
<PackageVersion Include="Microsoft.Data.SqlClient" Version="6.1.2" />
<PackageVersion Include="Qdrant.Client" Version="1.15.1" />
<!-- VectorData Dependencies -->
<PackageVersion Include="Microsoft.SemanticKernel.Connectors.AzureAISearch" Version="1.74.0-preview" />
<PackageVersion Include="Microsoft.SemanticKernel.Connectors.CosmosMongoDB" Version="1.74.0-preview" />
<PackageVersion Include="Microsoft.SemanticKernel.Connectors.CosmosNoSql" Version="1.74.0-preview" />
<PackageVersion Include="Microsoft.SemanticKernel.Connectors.InMemory" Version="1.74.0-preview" />
<PackageVersion Include="Microsoft.SemanticKernel.Connectors.PgVector" Version="1.74.0-preview" />
<PackageVersion Include="Microsoft.SemanticKernel.Connectors.Pinecone" Version="1.74.0-preview" />
<PackageVersion Include="Microsoft.SemanticKernel.Connectors.Qdrant" Version="1.74.0-preview" />
<PackageVersion Include="Microsoft.SemanticKernel.Connectors.Redis" Version="1.74.0-preview" />
<PackageVersion Include="Microsoft.SemanticKernel.Connectors.Weaviate" Version="1.74.0-preview" />
<!-- Symbols -->
<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
<!-- Toolset -->
dotnet/MEVD.slnf
-50
View File
@@ -1,50 +0,0 @@
{
"solution": {
"path": "SK-dotnet.slnx",
"projects":
[
"src/VectorData/AzureAISearch/AzureAISearch.csproj",
"src/VectorData/CosmosMongoDB/CosmosMongoDB.csproj",
"src/VectorData/CosmosNoSql/CosmosNoSql.csproj",
"src/VectorData/Chroma/Chroma.csproj",
"src/VectorData/InMemory/InMemory.csproj",
"src/VectorData/Milvus/Milvus.csproj",
"src/VectorData/MongoDB/MongoDB.csproj",
"src/VectorData/Pinecone/Pinecone.csproj",
"src/VectorData/PgVector/PgVector.csproj",
"src/VectorData/Qdrant/Qdrant.csproj",
"src/VectorData/Redis/Redis.csproj",
"src/VectorData/SqliteVec/SqliteVec.csproj",
"src/VectorData/SqlServer/SqlServer.csproj",
"src/VectorData/Weaviate/Weaviate.csproj",
"src/VectorData/VectorData.Abstractions/VectorData.Abstractions.csproj",
"test/VectorData/AzureAISearch.UnitTests/AzureAISearch.UnitTests.csproj",
"test/VectorData/AzureAISearch.ConformanceTests/AzureAISearch.ConformanceTests.csproj",
"test/VectorData/CosmosMongoDB.UnitTests/CosmosMongoDB.UnitTests.csproj",
"test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoDB.ConformanceTests.csproj",
"test/VectorData/CosmosNoSql.UnitTests/CosmosNoSql.UnitTests.csproj",
"test/VectorData/CosmosNoSql.ConformanceTests/CosmosNoSql.ConformanceTests.csproj",
"test/VectorData/InMemory.UnitTests/InMemory.UnitTests.csproj",
"test/VectorData/InMemory.ConformanceTests/InMemory.ConformanceTests.csproj",
"test/VectorData/MongoDB.UnitTests/MongoDB.UnitTests.csproj",
"test/VectorData/MongoDB.ConformanceTests/MongoDB.ConformanceTests.csproj",
"test/VectorData/Pinecone.UnitTests/Pinecone.UnitTests.csproj",
"test/VectorData/Pinecone.ConformanceTests/Pinecone.ConformanceTests.csproj",
"test/VectorData/PgVector.UnitTests/PgVector.UnitTests.csproj",
"test/VectorData/PgVector.ConformanceTests/PgVector.ConformanceTests.csproj",
"test/VectorData/Qdrant.UnitTests/Qdrant.UnitTests.csproj",
"test/VectorData/Qdrant.ConformanceTests/Qdrant.ConformanceTests.csproj",
"test/VectorData/Redis.UnitTests/Redis.UnitTests.csproj",
"test/VectorData/Redis.ConformanceTests/Redis.ConformanceTests.csproj",
"test/VectorData/SqliteVec.UnitTests/SqliteVec.UnitTests.csproj",
"test/VectorData/SqliteVec.ConformanceTests/SqliteVec.ConformanceTests.csproj",
"test/VectorData/SqlServer.ConformanceTests/SqlServer.ConformanceTests.csproj",
"test/VectorData/Weaviate.UnitTests/Weaviate.UnitTests.csproj",
"test/VectorData/Weaviate.ConformanceTests/Weaviate.ConformanceTests.csproj",
"test/VectorData/VectorData.ConformanceTests/VectorData.ConformanceTests.csproj"
]
}
}
dotnet/MEVD.slnx
+66
View File
@@ -0,0 +1,66 @@
<Solution>
<Folder Name="/Solution Items/">
<File Path="../.editorconfig" />
<File Path="../.github/workflows/dotnet-format.yml" />
<File Path="../.gitignore" />
<File Path="../nuget.config" />
<File Path="../README.md" />
<File Path="Directory.Build.props" />
<File Path="Directory.Build.targets" />
<File Path="Directory.Packages.props" />
<File Path="docs/EXPERIMENTS.md" />
<File Path="global.json" />
</Folder>
<Folder Name="/Solution Items/nuget/">
<File Path="nuget/icon.png" />
<File Path="nuget/nuget-package.props" />
<File Path="nuget/NUGET.md" />
<File Path="nuget/VECTORDATA-CONNECTORS-NUGET.md" />
</Folder>
<Folder Name="/src/" />
<Folder Name="/src/VectorData/">
<File Path="src/VectorData/Directory.Build.props" />
<Project Path="src/VectorData/AzureAISearch/AzureAISearch.csproj" />
<Project Path="src/VectorData/Chroma/Chroma.csproj" />
<Project Path="src/VectorData/CosmosMongoDB/CosmosMongoDB.csproj" />
<Project Path="src/VectorData/CosmosNoSql/CosmosNoSql.csproj" />
<Project Path="src/VectorData/InMemory/InMemory.csproj" />
<Project Path="src/VectorData/Milvus/Milvus.csproj" />
<Project Path="src/VectorData/MongoDB/MongoDB.csproj" />
<Project Path="src/VectorData/PgVector/PgVector.csproj" />
<Project Path="src/VectorData/Pinecone/Pinecone.csproj" />
<Project Path="src/VectorData/Qdrant/Qdrant.csproj" />
<Project Path="src/VectorData/Redis/Redis.csproj" />
<Project Path="src/VectorData/SqliteVec/SqliteVec.csproj" />
<Project Path="src/VectorData/SqlServer/SqlServer.csproj" />
<Project Path="src/VectorData/Weaviate/Weaviate.csproj" />
</Folder>
<Folder Name="/test/" />
<Folder Name="/test/VectorData/">
<Project Path="test/VectorData/AzureAISearch.ConformanceTests/AzureAISearch.ConformanceTests.csproj" />
<Project Path="test/VectorData/AzureAISearch.UnitTests/AzureAISearch.UnitTests.csproj" />
<Project Path="test/VectorData/Chroma.UnitTests/Chroma.UnitTests.csproj" />
<Project Path="test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoDB.ConformanceTests.csproj" />
<Project Path="test/VectorData/CosmosMongoDB.UnitTests/CosmosMongoDB.UnitTests.csproj" />
<Project Path="test/VectorData/CosmosNoSql.ConformanceTests/CosmosNoSql.ConformanceTests.csproj" />
<Project Path="test/VectorData/CosmosNoSql.UnitTests/CosmosNoSql.UnitTests.csproj" />
<Project Path="test/VectorData/InMemory.ConformanceTests/InMemory.ConformanceTests.csproj" />
<Project Path="test/VectorData/InMemory.UnitTests/InMemory.UnitTests.csproj" />
<Project Path="test/VectorData/MongoDB.ConformanceTests/MongoDB.ConformanceTests.csproj" />
<Project Path="test/VectorData/MongoDB.UnitTests/MongoDB.UnitTests.csproj" />
<Project Path="test/VectorData/PgVector.ConformanceTests/PgVector.ConformanceTests.csproj" />
<Project Path="test/VectorData/PgVector.UnitTests/PgVector.UnitTests.csproj" />
<Project Path="test/VectorData/Pinecone.ConformanceTests/Pinecone.ConformanceTests.csproj" />
<Project Path="test/VectorData/Pinecone.UnitTests/Pinecone.UnitTests.csproj" />
<Project Path="test/VectorData/Qdrant.ConformanceTests/Qdrant.ConformanceTests.csproj" />
<Project Path="test/VectorData/Qdrant.UnitTests/Qdrant.UnitTests.csproj" />
<Project Path="test/VectorData/Redis.ConformanceTests/Redis.ConformanceTests.csproj" />
<Project Path="test/VectorData/Redis.UnitTests/Redis.UnitTests.csproj" />
<Project Path="test/VectorData/SqliteVec.ConformanceTests/SqliteVec.ConformanceTests.csproj" />
<Project Path="test/VectorData/SqliteVec.UnitTests/SqliteVec.UnitTests.csproj" />
<Project Path="test/VectorData/SqlServer.ConformanceTests/SqlServer.ConformanceTests.csproj" />
<Project Path="test/VectorData/VectorData.ConformanceTests/VectorData.ConformanceTests.csproj" />
<Project Path="test/VectorData/Weaviate.ConformanceTests/Weaviate.ConformanceTests.csproj" />
<Project Path="test/VectorData/Weaviate.UnitTests/Weaviate.UnitTests.csproj" />
</Folder>
</Solution>
dotnet/SK-dotnet.slnx
-47
View File
@@ -121,24 +121,6 @@
<Project Path="src/Connectors/Connectors.OpenAI.UnitTests/Connectors.OpenAI.UnitTests.csproj" />
<Project Path="src/Connectors/Connectors.OpenAI/Connectors.OpenAI.csproj" />
</Folder>
<Folder Name="/src/VectorData/">
<File Path="src/VectorData/Directory.Build.props" />
<Project Path="src/VectorData/AzureAISearch/AzureAISearch.csproj" />
<Project Path="src/VectorData/Chroma/Chroma.csproj" />
<Project Path="src/VectorData/CosmosMongoDB/CosmosMongoDB.csproj" />
<Project Path="src/VectorData/CosmosNoSql/CosmosNoSql.csproj" />
<Project Path="src/VectorData/InMemory/InMemory.csproj" />
<Project Path="src/VectorData/Milvus/Milvus.csproj" />
<Project Path="src/VectorData/MongoDB/MongoDB.csproj" />
<Project Path="src/VectorData/PgVector/PgVector.csproj" />
<Project Path="src/VectorData/Pinecone/Pinecone.csproj" />
<Project Path="src/VectorData/Qdrant/Qdrant.csproj" />
<Project Path="src/VectorData/Redis/Redis.csproj" />
<Project Path="src/VectorData/SqliteVec/SqliteVec.csproj" />
<Project Path="src/VectorData/SqlServer/SqlServer.csproj" />
<Project Path="src/VectorData/VectorData.Abstractions/VectorData.Abstractions.csproj" />
<Project Path="src/VectorData/Weaviate/Weaviate.csproj" />
</Folder>
<Folder Name="/src/experimental/">
<Project Path="src/Experimental/Orchestration.Flow.IntegrationTests/Experimental.Orchestration.Flow.IntegrationTests.csproj" />
<Project Path="src/Experimental/Orchestration.Flow.UnitTests/Experimental.Orchestration.Flow.UnitTests.csproj" />
@@ -292,33 +274,4 @@
<Project Path="src/Plugins/Plugins.UnitTests/Plugins.UnitTests.csproj" />
<Project Path="src/Plugins/Plugins.Web/Plugins.Web.csproj" />
</Folder>
<Folder Name="/test/" />
<Folder Name="/test/VectorData/">
<Project Path="test/VectorData/AzureAISearch.ConformanceTests/AzureAISearch.ConformanceTests.csproj" />
<Project Path="test/VectorData/AzureAISearch.UnitTests/AzureAISearch.UnitTests.csproj" />
<Project Path="test/VectorData/Chroma.UnitTests/Chroma.UnitTests.csproj" />
<Project Path="test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoDB.ConformanceTests.csproj" />
<Project Path="test/VectorData/CosmosMongoDB.UnitTests/CosmosMongoDB.UnitTests.csproj" />
<Project Path="test/VectorData/CosmosNoSql.ConformanceTests/CosmosNoSql.ConformanceTests.csproj" />
<Project Path="test/VectorData/CosmosNoSql.UnitTests/CosmosNoSql.UnitTests.csproj" />
<Project Path="test/VectorData/InMemory.ConformanceTests/InMemory.ConformanceTests.csproj" />
<Project Path="test/VectorData/InMemory.UnitTests/InMemory.UnitTests.csproj" />
<Project Path="test/VectorData/MongoDB.ConformanceTests/MongoDB.ConformanceTests.csproj" />
<Project Path="test/VectorData/MongoDB.UnitTests/MongoDB.UnitTests.csproj" />
<Project Path="test/VectorData/PgVector.ConformanceTests/PgVector.ConformanceTests.csproj" />
<Project Path="test/VectorData/PgVector.UnitTests/PgVector.UnitTests.csproj" />
<Project Path="test/VectorData/Pinecone.ConformanceTests/Pinecone.ConformanceTests.csproj" />
<Project Path="test/VectorData/Pinecone.UnitTests/Pinecone.UnitTests.csproj" />
<Project Path="test/VectorData/Qdrant.ConformanceTests/Qdrant.ConformanceTests.csproj" />
<Project Path="test/VectorData/Qdrant.UnitTests/Qdrant.UnitTests.csproj" />
<Project Path="test/VectorData/Redis.ConformanceTests/Redis.ConformanceTests.csproj" />
<Project Path="test/VectorData/Redis.UnitTests/Redis.UnitTests.csproj" />
<Project Path="test/VectorData/SqliteVec.ConformanceTests/SqliteVec.ConformanceTests.csproj" />
<Project Path="test/VectorData/SqliteVec.UnitTests/SqliteVec.UnitTests.csproj" />
<Project Path="test/VectorData/SqlServer.ConformanceTests/SqlServer.ConformanceTests.csproj" />
<Project Path="test/VectorData/VectorData.ConformanceTests/VectorData.ConformanceTests.csproj" />
<Project Path="test/VectorData/VectorData.UnitTests/VectorData.UnitTests.csproj" />
<Project Path="test/VectorData/Weaviate.ConformanceTests/Weaviate.ConformanceTests.csproj" />
<Project Path="test/VectorData/Weaviate.UnitTests/Weaviate.UnitTests.csproj" />
</Folder>
</Solution>
dotnet/samples/Concepts/Concepts.csproj
+9 -11
View File
@@ -49,6 +49,15 @@
<PackageReference Include="AWSSDK.Core" />
<PackageReference Include="AWSSDK.SecurityToken" />
<PackageReference Include="Grpc.Net.ClientFactory" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.AzureAISearch" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.CosmosMongoDB" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.InMemory" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.Qdrant" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.Pinecone" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.PgVector" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.Redis" />
<PackageReference Include="SharpCompress" /> <!-- Pin to patched version; overrides transitive 0.30.1 from MongoDB.Driver (GHSA-6c8g-7p36-r338) -->
<PackageReference Include="Snappier" /> <!-- Pin to patched version; overrides transitive 1.0.0 from MongoDB.Driver (GHSA-pggp-6c3x-2xmx) -->
</ItemGroup>
<PropertyGroup>
@@ -71,17 +80,6 @@
<ProjectReference Include="..\..\src\Connectors\Connectors.Onnx\Connectors.Onnx.csproj" />
<ProjectReference Include="..\..\src\Connectors\Connectors.Ollama\Connectors.Ollama.csproj" />
<ProjectReference Include="..\..\src\Connectors\Connectors.OpenAI\Connectors.OpenAI.csproj" />
<ProjectReference Include="..\..\src\VectorData\AzureAISearch\AzureAISearch.csproj" />
<ProjectReference Include="..\..\src\VectorData\CosmosMongoDB\CosmosMongoDB.csproj" />
<ProjectReference Include="..\..\src\VectorData\Chroma\Chroma.csproj" />
<ProjectReference Include="..\..\src\VectorData\InMemory\InMemory.csproj" />
<ProjectReference Include="..\..\src\VectorData\MongoDB\MongoDB.csproj" />
<ProjectReference Include="..\..\src\VectorData\Pinecone\Pinecone.csproj" />
<ProjectReference Include="..\..\src\VectorData\PgVector\PgVector.csproj" />
<ProjectReference Include="..\..\src\VectorData\Qdrant\Qdrant.csproj" />
<ProjectReference Include="..\..\src\VectorData\Redis\Redis.csproj" />
<ProjectReference Include="..\..\src\VectorData\SqliteVec\SqliteVec.csproj" />
<ProjectReference Include="..\..\src\VectorData\Weaviate\Weaviate.csproj" />
<ProjectReference Include="..\..\src\Experimental\Orchestration.Flow\Experimental.Orchestration.Flow.csproj" />
<ProjectReference Include="..\..\src\Extensions\PromptTemplates.Handlebars\PromptTemplates.Handlebars.csproj" />
<ProjectReference Include="..\..\src\Extensions\PromptTemplates.Liquid\PromptTemplates.Liquid.csproj" />
dotnet/samples/Concepts/Memory/HuggingFace_TextEmbeddingCustomHttpHandler.cs
+3 -4
View File
@@ -3,7 +3,7 @@
using System.Text.Json;
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel.Connectors.HuggingFace;
using Microsoft.SemanticKernel.Connectors.SqliteVec;
using Microsoft.SemanticKernel.Connectors.InMemory;
using Microsoft.SemanticKernel.Embeddings;
#pragma warning disable CS8602 // Dereference of a possibly null reference.
@@ -32,12 +32,11 @@ public class HuggingFace_TextEmbeddingCustomHttpHandler(ITestOutputHelper output
})
);
var sqliteCollection = new SqliteCollection<string, Record>(
"Data Source=./../../../Sqlite.sqlite",
var inMemoryCollection = new InMemoryCollection<string, Record>(
name: "Test",
new() { EmbeddingGenerator = hf.AsEmbeddingGenerator() });
await sqliteCollection.UpsertAsync(new Record
await inMemoryCollection.UpsertAsync(new Record
{
Id = "1",
Text = "THIS IS A SAMPLE",
dotnet/samples/Demos/AgentFrameworkWithAspire/ChatWithAgent.ApiService/ChatWithAgent.ApiService.csproj
+1 -1
View File
@@ -14,13 +14,13 @@
<ItemGroup>
<PackageReference Include="Aspire.Azure.AI.OpenAI" />
<PackageReference Include="Aspire.Azure.Search.Documents" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.AzureAISearch" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\..\src\Agents\Core\Agents.Core.csproj" />
<ProjectReference Include="..\..\..\..\src\Connectors\Connectors.AzureOpenAI\Connectors.AzureOpenAI.csproj" />
<ProjectReference Include="..\..\..\..\src\Connectors\Connectors.OpenAI\Connectors.OpenAI.csproj" />
<ProjectReference Include="..\..\..\..\src\VectorData\AzureAISearch\AzureAISearch.csproj" />
<ProjectReference Include="..\..\..\..\src\Extensions\PromptTemplates.Handlebars\PromptTemplates.Handlebars.csproj" />
<ProjectReference Include="..\..\..\..\src\Functions\Functions.Yaml\Functions.Yaml.csproj" />
<ProjectReference Include="..\..\..\..\src\SemanticKernel.Abstractions\SemanticKernel.Abstractions.csproj" />
dotnet/samples/Demos/ModelContextProtocolClientServer/MCPServer/MCPServer.csproj
+1 -1
View File
@@ -32,11 +32,11 @@
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Hosting" />
<PackageReference Include="ModelContextProtocol" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.InMemory" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\..\src\Agents\Core\Agents.Core.csproj" />
<ProjectReference Include="..\..\..\..\src\VectorData\InMemory\InMemory.csproj" />
<ProjectReference Include="..\..\..\..\src\Connectors\Connectors.OpenAI\Connectors.OpenAI.csproj" />
<ProjectReference Include="..\..\..\..\src\Extensions\PromptTemplates.Handlebars\PromptTemplates.Handlebars.csproj" />
<ProjectReference Include="..\..\..\..\src\SemanticKernel.Abstractions\SemanticKernel.Abstractions.csproj" />
dotnet/samples/Demos/OnnxSimpleRAG/OnnxSimpleRAG.csproj
+1 -1
View File
@@ -8,7 +8,6 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\src\VectorData\InMemory\InMemory.csproj" />
<ProjectReference Include="..\..\..\src\Connectors\Connectors.Onnx\Connectors.Onnx.csproj" />
<ProjectReference Include="..\..\..\src\Extensions\PromptTemplates.Handlebars\PromptTemplates.Handlebars.csproj" />
<ProjectReference Include="..\..\..\src\SemanticKernel.Abstractions\SemanticKernel.Abstractions.csproj" />
@@ -17,6 +16,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.InMemory" />
</ItemGroup>
<ItemGroup>
<None Update="Facts\*.txt">
dotnet/samples/Demos/VectorStoreRAG/VectorStoreRAG.csproj
+9 -7
View File
@@ -15,17 +15,19 @@
<PackageReference Include="Microsoft.Extensions.Hosting" />
<PackageReference Include="Microsoft.Extensions.Options.DataAnnotations" />
<PackageReference Include="PdfPig" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.AzureAISearch" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.CosmosMongoDB" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.CosmosNoSql" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.InMemory" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.Qdrant" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.Redis" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.Weaviate" />
<PackageReference Include="SharpCompress" /> <!-- Pin to patched version; overrides transitive 0.30.1 from MongoDB.Driver (GHSA-6c8g-7p36-r338) -->
<PackageReference Include="Snappier" /> <!-- Pin to patched version; overrides transitive 1.0.0 from MongoDB.Driver (GHSA-pggp-6c3x-2xmx) -->
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\src\Connectors\Connectors.AzureOpenAI\Connectors.AzureOpenAI.csproj" />
<ProjectReference Include="..\..\..\src\VectorData\AzureAISearch\AzureAISearch.csproj" />
<ProjectReference Include="..\..\..\src\VectorData\CosmosMongoDB\CosmosMongoDB.csproj" />
<ProjectReference Include="..\..\..\src\VectorData\CosmosNoSql\CosmosNoSql.csproj" />
<ProjectReference Include="..\..\..\src\VectorData\InMemory\InMemory.csproj" />
<ProjectReference Include="..\..\..\src\VectorData\Qdrant\Qdrant.csproj" />
<ProjectReference Include="..\..\..\src\VectorData\Redis\Redis.csproj" />
<ProjectReference Include="..\..\..\src\VectorData\Weaviate\Weaviate.csproj" />
<ProjectReference Include="..\..\..\src\Extensions\PromptTemplates.Handlebars\PromptTemplates.Handlebars.csproj" />
<ProjectReference Include="..\..\..\src\SemanticKernel.Core\SemanticKernel.Core.csproj" />
</ItemGroup>
dotnet/samples/GettingStartedWithTextSearch/GettingStartedWithTextSearch.csproj
+1 -1
View File
@@ -35,6 +35,7 @@
<PackageReference Include="Microsoft.Extensions.Logging.Console" />
<PackageReference Include="Microsoft.ML.Tokenizers" />
<PackageReference Include="System.Numerics.Tensors" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.InMemory" />
</ItemGroup>
<Import Project="$(RepoRoot)/dotnet/src/InternalUtilities/samples/SamplesInternalUtilities.props" />
@@ -42,7 +43,6 @@
<ItemGroup>
<ProjectReference Include="..\..\src\Connectors\Connectors.AzureOpenAI\Connectors.AzureOpenAI.csproj" />
<ProjectReference Include="..\..\src\Connectors\Connectors.OpenAI\Connectors.OpenAI.csproj" />
<ProjectReference Include="..\..\src\VectorData\InMemory\InMemory.csproj" />
<ProjectReference Include="..\..\src\Extensions\PromptTemplates.Handlebars\PromptTemplates.Handlebars.csproj" />
<ProjectReference Include="..\..\src\Plugins\Plugins.Web\Plugins.Web.csproj" />
<ProjectReference Include="..\..\src\SemanticKernel.Abstractions\SemanticKernel.Abstractions.csproj" />
dotnet/samples/GettingStartedWithVectorStores/GettingStartedWithVectorStores.csproj
+3 -4
View File
@@ -32,16 +32,15 @@
<PackageReference Include="Microsoft.Extensions.AI.OpenAI" />
<PackageReference Include="Microsoft.Extensions.Logging" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.AzureAISearch" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.InMemory" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.Redis" />
</ItemGroup>
<Import Project="$(RepoRoot)/dotnet/src/InternalUtilities/samples/SamplesInternalUtilities.props" />
<ItemGroup>
<ProjectReference Include="..\..\src\Connectors\Connectors.AzureOpenAI\Connectors.AzureOpenAI.csproj" />
<ProjectReference Include="..\..\src\VectorData\AzureAISearch\AzureAISearch.csproj" />
<ProjectReference Include="..\..\src\VectorData\InMemory\InMemory.csproj" />
<ProjectReference Include="..\..\src\VectorData\Qdrant\Qdrant.csproj" />
<ProjectReference Include="..\..\src\VectorData\Redis\Redis.csproj" />
<ProjectReference Include="..\..\src\SemanticKernel.Abstractions\SemanticKernel.Abstractions.csproj" />
<ProjectReference Include="..\..\src\SemanticKernel.Core\SemanticKernel.Core.csproj" />
</ItemGroup>
dotnet/src/IntegrationTests/IntegrationTests.csproj
-12
View File
@@ -72,18 +72,6 @@
<ProjectReference Include="..\Connectors\Connectors.Onnx\Connectors.Onnx.csproj" />
<ProjectReference Include="..\Connectors\Connectors.MistralAI\Connectors.MistralAI.csproj" />
<ProjectReference Include="..\Functions\Functions.OpenApi.Extensions\Functions.OpenApi.Extensions.csproj" />
<ProjectReference Include="..\VectorData\AzureAISearch\AzureAISearch.csproj" />
<ProjectReference Include="..\VectorData\CosmosMongoDB\CosmosMongoDB.csproj" />
<ProjectReference Include="..\VectorData\CosmosNoSql\CosmosNoSql.csproj" />
<ProjectReference Include="..\VectorData\Chroma\Chroma.csproj" />
<ProjectReference Include="..\VectorData\InMemory\InMemory.csproj" />
<ProjectReference Include="..\VectorData\Milvus\Milvus.csproj" />
<ProjectReference Include="..\VectorData\MongoDB\MongoDB.csproj" />
<ProjectReference Include="..\VectorData\PgVector\PgVector.csproj" />
<ProjectReference Include="..\VectorData\Qdrant\Qdrant.csproj" />
<ProjectReference Include="..\VectorData\Redis\Redis.csproj" />
<ProjectReference Include="..\VectorData\SqliteVec\SqliteVec.csproj" />
<ProjectReference Include="..\VectorData\Weaviate\Weaviate.csproj" />
<ProjectReference Include="..\Experimental\Process.Abstractions\Process.Abstractions.csproj" />
<ProjectReference Include="..\Experimental\Process.Core\Process.Core.csproj" />
<ProjectReference Include="..\Experimental\Process.LocalRuntime\Process.LocalRuntime.csproj" />
dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoModelBuilder.cs
+8 -5
View File
@@ -27,14 +27,17 @@ internal class MongoModelBuilder() : CollectionModelBuilder(s_validationOptions)
UsesExternalSerializer = true,
};
protected override void ProcessProperty(PropertyInfo? clrProperty, VectorStoreProperty? definitionProperty, Type? type)
[RequiresUnreferencedCode("Traverses the CLR type's properties with reflection, so not compatible with trimming")]
protected override void ProcessTypeProperties(Type type, VectorStoreCollectionDefinition? definition)
{
base.ProcessProperty(clrProperty, definitionProperty, type);
base.ProcessTypeProperties(type, definition);
if (clrProperty?.GetCustomAttribute<BsonElementAttribute>() is { } bsonElementAttribute
&& this.PropertyMap.TryGetValue(clrProperty.Name, out var property))
foreach (var property in this.Properties)
{
property.StorageName = bsonElementAttribute.ElementName;
if (property.PropertyInfo?.GetCustomAttribute<BsonElementAttribute>() is { } bsonElementAttribute)
{
property.StorageName = bsonElementAttribute.ElementName;
}
}
}
dotnet/src/SemanticKernel.Abstractions/SemanticKernel.Abstractions.csproj
+1 -4
View File
@@ -23,13 +23,10 @@
<Description>Semantic Kernel interfaces and abstractions. This package is automatically installed by Semantic Kernel packages if needed.</Description>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\VectorData\VectorData.Abstractions\VectorData.Abstractions.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Bcl.HashCode" />
<PackageReference Include="Microsoft.Extensions.AI" />
<PackageReference Include="Microsoft.Extensions.VectorData.Abstractions" />
<PackageReference Include="System.Linq.AsyncEnumerable" />
<PackageReference Include="System.Text.Json" />
</ItemGroup>
dotnet/src/SemanticKernel.UnitTests/SemanticKernel.UnitTests.csproj
+1 -1
View File
@@ -25,11 +25,11 @@
</PackageReference>
<PackageReference Include="Microsoft.ML.Tokenizers" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.InMemory" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Connectors\Connectors.AzureOpenAI\Connectors.AzureOpenAI.csproj" />
<ProjectReference Include="..\VectorData\InMemory\InMemory.csproj" />
<ProjectReference Include="..\SemanticKernel.Abstractions\SemanticKernel.Abstractions.csproj" />
<ProjectReference Include="..\SemanticKernel.Core\SemanticKernel.Core.csproj" />
</ItemGroup>
dotnet/src/VectorData/AzureAISearch/AzureAISearch.csproj
+1 -4
View File
@@ -26,6 +26,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
<PackageReference Include="Microsoft.Extensions.VectorData.Abstractions" />
<PackageReference Include="Azure.Search.Documents" />
<PackageReference Include="System.Linq.AsyncEnumerable" />
</ItemGroup>
@@ -37,8 +38,4 @@
<ItemGroup>
<InternalsVisibleTo Include="SemanticKernel.Connectors.AzureAISearch.UnitTests" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\VectorData.Abstractions\VectorData.Abstractions.csproj" />
</ItemGroup>
</Project>
dotnet/src/VectorData/CosmosMongoDB/CosmosMongoDB.csproj
+1 -4
View File
@@ -42,10 +42,7 @@
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
<PackageReference Include="System.Text.Json" />
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\VectorData.Abstractions\VectorData.Abstractions.csproj" />
<PackageReference Include="Microsoft.Extensions.VectorData.Abstractions" />
</ItemGroup>
</Project>
dotnet/src/VectorData/CosmosNoSql/CosmosNoSql.csproj
+1 -4
View File
@@ -32,13 +32,10 @@
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
<PackageReference Include="Microsoft.Extensions.VectorData.Abstractions" />
<PackageReference Include="System.Linq.AsyncEnumerable" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\VectorData.Abstractions\VectorData.Abstractions.csproj" />
</ItemGroup>
<ItemGroup>
<InternalsVisibleTo Include="SemanticKernel.Connectors.CosmosNoSql.UnitTests" />
</ItemGroup>
dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlMapper.cs
+3 -3
View File
@@ -41,8 +41,8 @@ internal sealed class CosmosNoSqlMapper<TRecord> : ICosmosNoSqlMapper<TRecord>
// The key property in Azure CosmosDB NoSQL is always named 'id'.
// But the external JSON serializer used just above isn't aware of that, and will produce a JSON object with another name, taking into
// account e.g. naming policies. SerializedKeyName gets populated in the model builder - containing that name - once VectorStoreModelBuildingOptions.ReservedKeyPropertyName is set
RenameJsonProperty(jsonObject, this._keyProperty.SerializedKeyName!, CosmosNoSqlConstants.ReservedKeyPropertyName);
// account e.g. naming policies. TemporaryStorageName gets populated in the model builder - containing that name - once VectorStoreModelBuildingOptions.ReservedKeyPropertyName is set
RenameJsonProperty(jsonObject, this._keyProperty.TemporaryStorageName!, CosmosNoSqlConstants.ReservedKeyPropertyName);
// Go over the vector properties; inject any generated embeddings to overwrite the JSON serialized above.
// Also, for Embedding<T> properties we also need to overwrite with a simple array (since Embedding<T> gets serialized as a complex object).
@@ -116,7 +116,7 @@ internal sealed class CosmosNoSqlMapper<TRecord> : ICosmosNoSqlMapper<TRecord>
public TRecord MapFromStorageToDataModel(JsonObject storageModel, bool includeVectors)
{
// See above comment.
RenameJsonProperty(storageModel, CosmosNoSqlConstants.ReservedKeyPropertyName, this._keyProperty.SerializedKeyName!);
RenameJsonProperty(storageModel, CosmosNoSqlConstants.ReservedKeyPropertyName, this._keyProperty.TemporaryStorageName!);
foreach (var vectorProperty in this._model.VectorProperties)
{
dotnet/src/VectorData/InMemory/InMemory.csproj
+1 -4
View File
@@ -34,11 +34,8 @@
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
<PackageReference Include="Microsoft.Extensions.VectorData.Abstractions" />
<PackageReference Include="System.Linq.AsyncEnumerable" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\VectorData.Abstractions\VectorData.Abstractions.csproj" />
</ItemGroup>
</Project>
dotnet/src/VectorData/MongoDB/MongoDB.csproj
+1 -4
View File
@@ -37,10 +37,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\VectorData.Abstractions\VectorData.Abstractions.csproj" />
<PackageReference Include="Microsoft.Extensions.VectorData.Abstractions" />
</ItemGroup>
<ItemGroup>
dotnet/src/VectorData/PgVector/PgVector.csproj
+1 -4
View File
@@ -32,16 +32,13 @@
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
<PackageReference Include="Microsoft.Extensions.VectorData.Abstractions" />
<PackageReference Include="Npgsql">
<VersionOverride Condition="!$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))">8.0.7</VersionOverride>
</PackageReference>
<PackageReference Include="Pgvector" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\VectorData.Abstractions\VectorData.Abstractions.csproj" />
</ItemGroup>
<ItemGroup>
<InternalsVisibleTo Include="SemanticKernel.Connectors.PgVector.UnitTests" />
<InternalsVisibleTo Include="DynamicProxyGenAssembly2" />
dotnet/src/VectorData/Pinecone/Pinecone.csproj
+1 -4
View File
@@ -28,6 +28,7 @@
<PackageReference Include="Pinecone.Client" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
<PackageReference Include="Microsoft.Extensions.VectorData.Abstractions" />
</ItemGroup>
<ItemGroup Condition=" '$(TargetFramework)' == 'netstandard2.0' Or '$(TargetFramework)' == 'net462' ">
@@ -39,8 +40,4 @@
<InternalsVisibleTo Include="DynamicProxyGenAssembly2" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\VectorData.Abstractions\VectorData.Abstractions.csproj" />
</ItemGroup>
</Project>
dotnet/src/VectorData/Qdrant/Qdrant.csproj
+1 -4
View File
@@ -27,6 +27,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
<PackageReference Include="Microsoft.Extensions.VectorData.Abstractions" />
<PackageReference Include="Qdrant.Client" />
<PackageReference Include="System.Linq.AsyncEnumerable" />
</ItemGroup>
@@ -40,8 +41,4 @@
<InternalsVisibleTo Include="DynamicProxyGenAssembly2" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\VectorData.Abstractions\VectorData.Abstractions.csproj" />
</ItemGroup>
</Project>
dotnet/src/VectorData/Redis/Redis.csproj
+1 -4
View File
@@ -26,6 +26,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
<PackageReference Include="Microsoft.Extensions.VectorData.Abstractions" />
<PackageReference Include="NRedisStack" />
</ItemGroup>
@@ -33,8 +34,4 @@
<InternalsVisibleTo Include="Microsoft.SemanticKernel.Connectors.Redis.UnitTests" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\VectorData.Abstractions\VectorData.Abstractions.csproj" />
</ItemGroup>
</Project>
dotnet/src/VectorData/SqlServer/SqlServer.csproj
+1 -4
View File
@@ -30,13 +30,10 @@
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
<PackageReference Include="Microsoft.Extensions.VectorData.Abstractions" />
<PackageReference Include="Microsoft.Data.SqlClient" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\VectorData.Abstractions\VectorData.Abstractions.csproj" />
</ItemGroup>
<ItemGroup>
<InternalsVisibleTo Include="SqlServer.ConformanceTests" />
</ItemGroup>
dotnet/src/VectorData/SqliteVec/SqliteVec.csproj
+1 -4
View File
@@ -31,16 +31,13 @@
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
<PackageReference Include="Microsoft.Extensions.VectorData.Abstractions" />
<PackageReference Include="Microsoft.Data.Sqlite" />
<PackageReference Include="System.Linq.AsyncEnumerable" />
<PackageReference Include="System.Numerics.Tensors" />
<PackageReference Include="sqlite-vec" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\VectorData.Abstractions\VectorData.Abstractions.csproj" />
</ItemGroup>
<ItemGroup>
<InternalsVisibleTo Include="SemanticKernel.Connectors.SqliteVec.UnitTests" />
</ItemGroup>
dotnet/src/VectorData/VectorData.Abstractions/.editorconfig
-3
View File
@@ -1,3 +0,0 @@
# Suppress missing documentation warnings for generated code (strings)
[*.Designer.cs]
dotnet_diagnostic.CS1591.severity = none
dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/AnyTagEqualToFilterClause.cs
-33
View File
@@ -1,33 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Represents a filter clause that filters by checking if a field consisting of a list of values contains a specific value.
/// </summary>
[Obsolete("Use LINQ expressions via VectorSearchOptions<TRecord>.Filter instead. This type will be removed in a future version.")]
public sealed class AnyTagEqualToFilterClause : FilterClause
{
/// <summary>
/// Initializes a new instance of the <see cref="AnyTagEqualToFilterClause"/> class.
/// </summary>
/// <param name="fieldName">The name of the field with the list of values.</param>
/// <param name="value">The value that the list should contain.</param>
public AnyTagEqualToFilterClause(string fieldName, string value)
{
this.FieldName = fieldName;
this.Value = value;
}
/// <summary>
/// Gets the name of the field with the list of values.
/// </summary>
public string FieldName { get; private set; }
/// <summary>
/// Gets the value that the list should contain.
/// </summary>
public string Value { get; private set; }
}
dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/EqualToFilterClause.cs
-33
View File
@@ -1,33 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Represents a filter clause that filters using equality of a field value.
/// </summary>
[Obsolete("Use LINQ expressions via VectorSearchOptions<TRecord>.Filter instead. This type will be removed in a future version.")]
public sealed class EqualToFilterClause : FilterClause
{
/// <summary>
/// Initializes a new instance of the <see cref="EqualToFilterClause"/> class.
/// </summary>
/// <param name="fieldName">Field name.</param>
/// <param name="value">Field value.</param>
public EqualToFilterClause(string fieldName, object value)
{
this.FieldName = fieldName;
this.Value = value;
}
/// <summary>
/// Gets the field name to match.
/// </summary>
public string FieldName { get; private set; }
/// <summary>
/// Gets the field value to match.
/// </summary>
public object Value { get; private set; }
}
dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/FilterClause.cs
-23
View File
@@ -1,23 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Defines a base class for filter clauses.
/// </summary>
/// <remarks>
/// A <see cref="FilterClause"/> is used to request that the underlying search service should
/// filter search results based on the specified criteria.
/// </remarks>
[Obsolete("Use LINQ expressions via VectorSearchOptions<TRecord>.Filter instead. This type will be removed in a future version.")]
public abstract class FilterClause
{
/// <summary>
/// Initializes a new instance of the <see cref="FilterClause"/> class.
/// </summary>
protected FilterClause()
{
}
}
dotnet/src/VectorData/VectorData.Abstractions/PACKAGE.md
-51
View File
@@ -1,51 +0,0 @@
## About
Contains abstractions for accessing Vector Databases and Vector Indexes.
## Key Features
- Base abstract classes and interfaces for Vector Database implementation. Vector Database implementations are provided separately in other packages, for example `Microsoft.SemanticKernel.Connectors.AzureAISearch`.
- Abstractions include:
- Creating, listing and deleting collections with custom schema support.
- Creating, retrieving, updating and deleting records.
- Similarty search using vector embeddings.
- Search using filters.
- Hybrid search combining vector similarity and keyword search.
- Built-in embedding generation using `Microsoft.Extensions.AI`.
## How to Use
This package is typically used with an implementation of the vector database abstractions such as `Microsoft.SemanticKernel.Connectors.AzureAISearch`.
## Main Types
The main types provided by this library are:
- [Microsoft.Extensions.VectorData.VectorStore](https://learn.microsoft.com/dotnet/api/microsoft.extensions.vectordata.vectorstore)
- [Microsoft.Extensions.VectorData.VectorStoreCollection](https://learn.microsoft.com/dotnet/api/microsoft.extensions.vectordata.vectorstorecollection-2)
## Additional Documentation
- [Conceptual documentation](https://learn.microsoft.com/en-us/semantic-kernel/concepts/vector-store-connectors)
## Related Packages
Vector Database implementations:
- [Microsoft.SemanticKernel.Connectors.AzureAISearch](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.AzureAISearch)
- [Microsoft.SemanticKernel.Connectors.CosmosMongoDB](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.CosmosMongoDB)
- [Microsoft.SemanticKernel.Connectors.CosmosNoSQL](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.CosmosNoSQL)
- [Elastic.SemanticKernel.Connectors.Elasticsearch](https://www.nuget.org/packages/Elastic.SemanticKernel.Connectors.Elasticsearch)
- [Microsoft.SemanticKernel.Connectors.InMemory](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.InMemory)
- [Microsoft.SemanticKernel.Connectors.MongoDB](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.MongoDB)
- [Microsoft.SemanticKernel.Connectors.PgVector](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.PgVector)
- [Microsoft.SemanticKernel.Connectors.Pinecone](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.Pinecone)
- [Microsoft.SemanticKernel.Connectors.Qdrant](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.Qdrant)
- [Microsoft.SemanticKernel.Connectors.Redis](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.Redis)
- [Microsoft.SemanticKernel.Connectors.SqliteVec](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.SqliteVec)
- [Microsoft.SemanticKernel.Connectors.SqlServer](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.SqlServer)
- [Microsoft.SemanticKernel.Connectors.Weaviate](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.Weaviate)
## Feedback & Contributing
Microsoft.Extensions.VectorData.Abstractions is released as open source under the [MIT license](https://licenses.nuget.org/MIT). Bug reports and contributions are welcome at [the GitHub repository](https://github.com/microsoft/semantic-kernel).
dotnet/src/VectorData/VectorData.Abstractions/Properties/AssemblyInfo.cs
-3
View File
@@ -1,3 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
[assembly: System.Resources.NeutralResourcesLanguage("en-US")]
dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionJsonModelBuilder.cs
-104
View File
@@ -1,104 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Diagnostics.CodeAnalysis;
using System.Reflection;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.AI;
namespace Microsoft.Extensions.VectorData.ProviderServices;
/// <summary>
/// Represents a model builder that performs logic specific to connectors that use System.Text.Json for serialization.
/// This is an internal support type meant for use by connectors only and not by applications.
/// </summary>
[Experimental("MEVD9001")]
public abstract class CollectionJsonModelBuilder : CollectionModelBuilder
{
private JsonSerializerOptions? _jsonSerializerOptions;
/// <summary>
/// Constructs a new <see cref="CollectionJsonModelBuilder"/>.
/// </summary>
protected CollectionJsonModelBuilder(CollectionModelBuildingOptions options)
: base(options)
{
}
/// <summary>
/// Builds and returns a <see cref="CollectionModel"/> from the given <paramref name="recordType"/> and <paramref name="definition"/>.
/// </summary>
[RequiresDynamicCode("This model building variant is not compatible with NativeAOT. See BuildDynamic() for dynamic mapping, and a third variant accepting source-generated delegates will be introduced in the future.")]
[RequiresUnreferencedCode("This model building variant is not compatible with trimming. See BuildDynamic() for dynamic mapping, and a third variant accepting source-generated delegates will be introduced in the future.")]
public virtual CollectionModel Build(
Type recordType,
Type keyType,
VectorStoreCollectionDefinition? definition,
IEmbeddingGenerator? defaultEmbeddingGenerator,
JsonSerializerOptions jsonSerializerOptions)
{
this._jsonSerializerOptions = jsonSerializerOptions;
return this.Build(recordType, keyType, definition, defaultEmbeddingGenerator);
}
/// <summary>
/// Builds and returns a <see cref="CollectionModel"/> for dynamic mapping scenarios from the given <paramref name="definition"/>.
/// </summary>
public virtual CollectionModel BuildDynamic(
VectorStoreCollectionDefinition definition,
IEmbeddingGenerator? defaultEmbeddingGenerator,
JsonSerializerOptions jsonSerializerOptions)
{
this._jsonSerializerOptions = jsonSerializerOptions;
return this.BuildDynamic(definition, defaultEmbeddingGenerator);
}
/// <inheritdoc/>
protected override void Customize()
{
// This mimics the naming behavior of the System.Text.Json serializer, which we use for serialization/deserialization.
// The property storage names in the model must in sync with the serializer configuration, since the model is used e.g. for filtering
// even if serialization/deserialization doesn't use the model.
var namingPolicy = this._jsonSerializerOptions?.PropertyNamingPolicy;
foreach (var property in this.Properties)
{
var keyPropertyWithReservedName = this.Options.ReservedKeyStorageName is not null && property is KeyPropertyModel;
string storageName;
if (property.PropertyInfo?.GetCustomAttribute<JsonPropertyNameAttribute>() is { } jsonPropertyNameAttribute)
{
if (keyPropertyWithReservedName && jsonPropertyNameAttribute.Name != this.Options.ReservedKeyStorageName)
{
throw new InvalidOperationException($"The key property for your connector must always have the reserved name '{this.Options.ReservedKeyStorageName}' and cannot be changed.");
}
storageName = jsonPropertyNameAttribute.Name;
}
else if (namingPolicy is not null)
{
storageName = namingPolicy.ConvertName(property.ModelName);
}
else
{
storageName = property.ModelName;
}
if (keyPropertyWithReservedName)
{
// Some providers (Weaviate, Cosmos NoSQL) have a fixed, reserved storage name for keys (id), and at the same time use an external
// JSON serializer to serialize the entire user POCO. Since the serializer is unaware of the reserved storage name, it will produce
// a storage name as usual, based on the .NET property's name, possibly with a naming policy applied to it. The connector then needs
// to look that up and replace with the reserved name.
((KeyPropertyModel)property).SerializedKeyName = storageName;
}
else
{
property.StorageName = storageName;
}
}
}
}
dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModel.cs
-243
View File
@@ -1,243 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Linq.Expressions;
using System.Reflection;
using System.Runtime.CompilerServices;
namespace Microsoft.Extensions.VectorData.ProviderServices;
/// <summary>
/// Represents a record in a vector store collection.
/// This is an internal support type meant for use by connectors only and not by applications.
/// </summary>
[Experimental("MEVD9001")]
public sealed class CollectionModel
{
private readonly Type _recordType;
private readonly Func<object> _recordFactory;
private KeyPropertyModel? _singleKeyProperty;
private VectorPropertyModel? _singleVectorProperty;
private DataPropertyModel? _singleFullTextSearchProperty;
/// <summary>
/// Gets the key properties of the record.
/// </summary>
public IReadOnlyList<KeyPropertyModel> KeyProperties { get; }
/// <summary>
/// Gets the data properties of the record.
/// </summary>
public IReadOnlyList<DataPropertyModel> DataProperties { get; }
/// <summary>
/// Gets the vector properties of the record.
/// </summary>
public IReadOnlyList<VectorPropertyModel> VectorProperties { get; }
/// <summary>
/// Gets all properties of the record, of all types.
/// </summary>
public IReadOnlyList<PropertyModel> Properties { get; }
/// <summary>
/// Gets all properties of the record, of all types, indexed by their model name.
/// </summary>
public IReadOnlyDictionary<string, PropertyModel> PropertyMap { get; }
/// <summary>
/// Gets a value that indicates whether any of the vector properties in the model require embedding generation.
/// </summary>
public bool EmbeddingGenerationRequired { get; }
internal CollectionModel(
Type recordType,
Func<object> recordFactory,
IReadOnlyList<KeyPropertyModel> keyProperties,
IReadOnlyList<DataPropertyModel> dataProperties,
IReadOnlyList<VectorPropertyModel> vectorProperties,
IReadOnlyDictionary<string, PropertyModel> propertyMap)
{
this._recordType = recordType;
this._recordFactory = recordFactory;
this.KeyProperties = keyProperties;
this.DataProperties = dataProperties;
this.VectorProperties = vectorProperties;
this.PropertyMap = propertyMap;
this.Properties = propertyMap.Values.ToList();
this.EmbeddingGenerationRequired = vectorProperties.Any(p => p.EmbeddingType != p.Type);
}
/// <summary>
/// Returns the single key property in the model, and throws if there are multiple key properties.
/// </summary>
public KeyPropertyModel KeyProperty => this._singleKeyProperty ??= this.KeyProperties.Single();
/// <summary>
/// Returns the single vector property in the model, and throws if there are multiple vector properties.
/// Suitable for connectors where validation is in place for single vectors only (<see cref="CollectionModelBuildingOptions.SupportsMultipleVectors"/>).
/// </summary>
public VectorPropertyModel VectorProperty => this._singleVectorProperty ??= this.VectorProperties.Single();
/// <summary>
/// Instantiates a new record of the specified type.
/// </summary>
// TODO: the pattern of first instantiating via parameterless constructor and then populating the properties isn't compatible
// with read-only types, where properties have no setters. Supporting those would be problematic given the that different
// connectors have completely different representations of the data coming back from the database, and which needs to be
// populated.
public TRecord CreateRecord<TRecord>()
{
Debug.Assert(typeof(TRecord) == this._recordType, "Type mismatch between record type and model type.");
return (TRecord)this._recordFactory();
}
/// <summary>
/// Gets the vector property with the provided name if a name is provided, and falls back
/// to a vector property in the schema if not.
/// </summary>
/// <param name="searchOptions">The search options, which defines the vector property name.</param>
/// <exception cref="InvalidOperationException"><para>The provided property name is not a valid text data property name.</para><para>OR</para><para>No name was provided and there's more than one vector property.</para></exception>
public VectorPropertyModel GetVectorPropertyOrSingle<TRecord>(VectorSearchOptions<TRecord> searchOptions)
{
if (searchOptions.VectorProperty is not null)
{
return this.GetMatchingProperty<TRecord, VectorPropertyModel>(searchOptions.VectorProperty, data: false);
}
// If vector property name is not provided, check if there is a single vector property, or throw if there are no vectors or more than one.
// TODO: Make a single switch expression + coalesce from the following - dotnet format fails on it for now
if (this._singleVectorProperty is null)
{
switch (this.VectorProperties)
{
case [var singleProperty]:
this._singleVectorProperty = singleProperty;
break;
case { Count: 0 }:
throw new InvalidOperationException($"The '{this._recordType.Name}' type does not have any vector properties.");
default:
throw new InvalidOperationException($"The '{this._recordType.Name}' type has multiple vector properties, please specify your chosen property via options.");
}
}
return this._singleVectorProperty;
}
/// <summary>
/// Gets the text data property with the provided name that has full text search indexing enabled, or falls back
/// to a text data property in the schema if no name is provided.
/// </summary>
/// <param name="expression">The full text search property selector.</param>
/// <exception cref="InvalidOperationException"><para>The provided property name is not a valid text data property name.</para><para>OR</para><para>No name was provided and there's more than one text data property with full text search indexing enabled.</para></exception>
public DataPropertyModel GetFullTextDataPropertyOrSingle<TRecord>(Expression<Func<TRecord, object?>>? expression)
{
if (expression is not null)
{
var property = this.GetMatchingProperty<TRecord, DataPropertyModel>(expression, data: true);
return property.IsFullTextIndexed
? property
: throw new InvalidOperationException($"The property '{property.ModelName}' on '{this._recordType.Name}' must have full text search indexing enabled.");
}
if (this._singleFullTextSearchProperty is null)
{
// If text data property name is not provided, check if a single full text indexed text property exists or throw otherwise.
var fullTextStringProperties = this.DataProperties
.Where(l => l.Type == typeof(string) && l.IsFullTextIndexed)
.ToList();
// If text data property name is not provided, check if a single full text indexed text property exists or throw otherwise.
switch (fullTextStringProperties)
{
// If there is a single property, use it.
// If there are no properties, throw.
// If there are multiple properties, throw.
case [var singleProperty]:
this._singleFullTextSearchProperty = singleProperty;
break;
case { Count: 0 }:
throw new InvalidOperationException($"The '{this._recordType.Name}' type does not have any text data properties that have full text indexing enabled.");
default:
throw new InvalidOperationException($"The '{this._recordType.Name}' type has multiple text data properties that have full text indexing enabled, please specify your chosen property via options.");
}
}
return this._singleFullTextSearchProperty;
}
/// <summary>
/// Gets the data or key property selected by the provided expression.
/// </summary>
/// <param name="expression">The property selector.</param>
/// <exception cref="InvalidOperationException">The provided property name is not a valid data or key property name.</exception>
public PropertyModel GetDataOrKeyProperty<TRecord>(Expression<Func<TRecord, object?>> expression)
=> this.GetMatchingProperty<TRecord, PropertyModel>(expression, data: true);
private TProperty GetMatchingProperty<TRecord, TProperty>(Expression<Func<TRecord, object?>> expression, bool data)
where TProperty : PropertyModel
{
var node = expression.Body;
// First, unwrap any object convert node: r => (object)r.PropertyName becomes r => r.PropertyName
if (expression.Body is UnaryExpression { NodeType: ExpressionType.Convert } convert
&& convert.Type == typeof(object))
{
node = convert.Operand;
}
var propertyName = node switch
{
// Simple member expression over the lambda parameter (r => r.PropertyName)
MemberExpression { Member: PropertyInfo clrProperty } member when member.Expression == expression.Parameters[0]
=> clrProperty.Name,
// Dictionary access over the lambda parameter, in dynamic mapping (r => r["PropertyName"])
MethodCallExpression { Method.Name: "get_Item", Arguments: [var keyExpression] } methodCall
=> keyExpression switch
{
ConstantExpression { Value: string text } => text,
MemberExpression field when TryGetCapturedValue(field, out object? capturedValue) && capturedValue is string text => text,
_ => throw new InvalidOperationException("Invalid dictionary key expression")
},
_ => throw new InvalidOperationException("Property selector lambda is invalid")
};
if (!this.PropertyMap.TryGetValue(propertyName, out var property))
{
throw new InvalidOperationException($"Property '{propertyName}' could not be found.");
}
return property is TProperty typedProperty
? typedProperty
: throw new InvalidOperationException($"Property '{propertyName}' isn't of type '{typeof(TProperty).Name}'.");
static bool TryGetCapturedValue(Expression expression, out object? capturedValue)
{
if (expression is MemberExpression { Expression: ConstantExpression constant, Member: FieldInfo fieldInfo }
&& constant.Type.Attributes.HasFlag(TypeAttributes.NestedPrivate)
&& Attribute.IsDefined(constant.Type, typeof(CompilerGeneratedAttribute), inherit: true))
{
capturedValue = fieldInfo.GetValue(constant.Value);
return true;
}
capturedValue = null;
return false;
}
}
}
dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModelBuilder.cs
-609
View File
@@ -1,609 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Reflection;
using Microsoft.Extensions.AI;
namespace Microsoft.Extensions.VectorData.ProviderServices;
/// <summary>
/// Represents a builder for a <see cref="CollectionModel"/>.
/// This is an internal support type meant for use by connectors only and not by applications.
/// </summary>
/// <remarks>This class is single-use only, and not thread-safe.</remarks>
[Experimental("MEVD9001")]
public abstract class CollectionModelBuilder
{
/// <summary>
/// Gets the options for building the model.
/// </summary>
protected CollectionModelBuildingOptions Options { get; }
/// <summary>
/// Gets the key properties of the record.
/// </summary>
protected List<KeyPropertyModel> KeyProperties { get; } = [];
/// <summary>
/// Gets the data properties of the record.
/// </summary>
protected List<DataPropertyModel> DataProperties { get; } = [];
/// <summary>
/// Gets the vector properties of the record.
/// </summary>
protected List<VectorPropertyModel> VectorProperties { get; } = [];
/// <summary>
/// Gets all properties of the record, of all types.
/// </summary>
protected IEnumerable<PropertyModel> Properties => this.PropertyMap.Values;
/// <summary>
/// Gets all properties of the record, of all types, indexed by their model name.
/// </summary>
protected Dictionary<string, PropertyModel> PropertyMap { get; } = [];
/// <summary>
/// Gets the default embedding generator to use for vector properties, when none is specified at the property or collection level.
/// </summary>
protected IEmbeddingGenerator? DefaultEmbeddingGenerator { get; private set; }
/// <summary>
/// Gets the collection's generic key type parameter (<c>TKey</c>), if provided.
/// Used by <see cref="ValidateKeyProperty"/> to validate that <c>TKey</c> corresponds to the key property type on the model.
/// </summary>
protected Type? KeyType { get; private set; }
/// <summary>
/// Constructs a new <see cref="CollectionModelBuilder"/>.
/// </summary>
protected CollectionModelBuilder(CollectionModelBuildingOptions options)
=> this.Options = options;
/// <summary>
/// Builds and returns an <see cref="CollectionModel"/> from the given <paramref name="recordType"/> and <paramref name="definition"/>.
/// </summary>
/// <param name="recordType">The CLR type of the record.</param>
/// <param name="keyType">The collection's generic key type parameter (<c>TKey</c>), used to validate correspondence with the key property type.</param>
/// <param name="definition">An optional record definition that overrides attribute-based configuration.</param>
/// <param name="defaultEmbeddingGenerator">An optional default embedding generator for vector properties.</param>
[RequiresDynamicCode("This model building variant is not compatible with NativeAOT. See BuildDynamic() for dynamic mapping, and a third variant accepting source-generated delegates will be introduced in the future.")]
[RequiresUnreferencedCode("This model building variant is not compatible with trimming. See BuildDynamic() for dynamic mapping, and a third variant accepting source-generated delegates will be introduced in the future.")]
public virtual CollectionModel Build(Type recordType, Type keyType, VectorStoreCollectionDefinition? definition, IEmbeddingGenerator? defaultEmbeddingGenerator)
{
this.KeyType = keyType;
if (recordType == typeof(Dictionary<string, object?>))
{
throw new ArgumentException("Dynamic mapping with Dictionary<string, object?> requires calling BuildDynamic().");
}
this.DefaultEmbeddingGenerator = definition?.EmbeddingGenerator ?? defaultEmbeddingGenerator;
// Build a lookup of definition properties by name for matching with CLR properties.
Dictionary<string, VectorStoreProperty>? definitionByName = null;
if (definition is not null)
{
definitionByName = [];
foreach (var p in definition.Properties)
{
definitionByName[p.Name] = p;
}
}
// Process CLR properties, matching to definition properties where available.
// TODO: This traverses the CLR type's properties, making it incompatible with trimming (and NativeAOT).
// TODO: We could put [DynamicallyAccessedMembers] to preserve all properties, but that approach wouldn't
// TODO: work with hierarchical data models (#10957).
foreach (var clrProperty in recordType.GetProperties())
{
VectorStoreProperty? definitionProperty = null;
_ = definitionByName?.TryGetValue(clrProperty.Name, out definitionProperty);
this.ProcessProperty(clrProperty, definitionProperty, recordType);
}
// Go over the properties, configure POCO accessors and validate type compatibility.
foreach (var property in this.Properties)
{
var clrProperty = recordType.GetProperty(property.ModelName)
?? throw new InvalidOperationException($"Property '{property.ModelName}' not found on CLR type '{recordType.FullName}'.");
var clrPropertyType = clrProperty.PropertyType;
if ((Nullable.GetUnderlyingType(clrPropertyType) ?? clrPropertyType) != (Nullable.GetUnderlyingType(property.Type) ?? property.Type))
{
throw new InvalidOperationException(
$"Property '{property.ModelName}' has a different CLR type in the record definition ('{property.Type.Name}') and on the .NET property ('{clrProperty.PropertyType}').");
}
property.ConfigurePocoAccessors(clrProperty);
}
this.Customize();
this.Validate(recordType, definition);
// Extra validation for non-dynamic mapping scenarios: ensure the type has a parameterless constructor.
if (!this.Options.UsesExternalSerializer && recordType.GetConstructor(Type.EmptyTypes) is null)
{
throw new NotSupportedException($"Type '{recordType.Name}' must have a parameterless constructor.");
}
return new(recordType, () => Activator.CreateInstance(recordType)!, this.KeyProperties, this.DataProperties, this.VectorProperties, this.PropertyMap);
}
/// <summary>
/// Builds and returns an <see cref="CollectionModel"/> for dynamic mapping scenarios from the given <paramref name="definition"/>.
/// </summary>
/// <param name="definition">The record definition describing the collection's schema.</param>
/// <param name="defaultEmbeddingGenerator">An optional default embedding generator for vector properties.</param>
public virtual CollectionModel BuildDynamic(VectorStoreCollectionDefinition definition, IEmbeddingGenerator? defaultEmbeddingGenerator)
{
if (definition is null)
{
throw new ArgumentException("Vector store record definition must be provided for dynamic mapping.");
}
this.DefaultEmbeddingGenerator = defaultEmbeddingGenerator;
foreach (var defProp in definition.Properties)
{
this.ProcessProperty(clrProperty: null, defProp, type: null);
}
this.Customize();
this.Validate(type: null, definition);
foreach (var property in this.Properties)
{
property.ConfigureDynamicAccessors();
}
return new(typeof(Dictionary<string, object?>), static () => new Dictionary<string, object?>(), this.KeyProperties, this.DataProperties, this.VectorProperties, this.PropertyMap);
}
/// <summary>
/// As part of building the model, this method processes a single property, accepting both a CLR <see cref="PropertyInfo"/>
/// (from which attributes are read) and a <see cref="VectorStoreProperty"/> from the user-provided record definition.
/// Either may be <see langword="null"/>, but not both.
/// When both are provided, the record definition values override attribute-configured values.
/// </summary>
protected virtual void ProcessProperty(PropertyInfo? clrProperty, VectorStoreProperty? definitionProperty, Type? type)
{
Debug.Assert(clrProperty is not null || definitionProperty is not null);
VectorStoreKeyAttribute? keyAttribute = null;
VectorStoreDataAttribute? dataAttribute = null;
VectorStoreVectorAttribute? vectorAttribute = null;
if (clrProperty is not null)
{
// Read attributes from CLR property.
keyAttribute = clrProperty.GetCustomAttribute<VectorStoreKeyAttribute>();
dataAttribute = clrProperty.GetCustomAttribute<VectorStoreDataAttribute>();
vectorAttribute = clrProperty.GetCustomAttribute<VectorStoreVectorAttribute>();
// Validate that at most one mapping attribute is present.
if ((keyAttribute is not null ? 1 : 0) + (dataAttribute is not null ? 1 : 0) + (vectorAttribute is not null ? 1 : 0) > 1)
{
throw new InvalidOperationException(
$"Property '{type!.Name}.{clrProperty.Name}' has multiple of {nameof(VectorStoreKeyAttribute)}, {nameof(VectorStoreDataAttribute)} or {nameof(VectorStoreVectorAttribute)}. Only one of these attributes can be specified on a property.");
}
// If no mapping attribute and no definition, skip this property.
if (keyAttribute is null && dataAttribute is null && vectorAttribute is null && definitionProperty is null)
{
return;
}
// Validate kind compatibility between attribute and definition.
if (definitionProperty is not null
&& ((keyAttribute is not null && definitionProperty is not VectorStoreKeyProperty)
|| (dataAttribute is not null && definitionProperty is not VectorStoreDataProperty)
|| (vectorAttribute is not null && definitionProperty is not VectorStoreVectorProperty)))
{
string definitionKind = definitionProperty switch
{
VectorStoreKeyProperty => "key",
VectorStoreDataProperty => "data",
VectorStoreVectorProperty => "vector",
_ => throw new ArgumentException($"Unknown type '{definitionProperty.GetType().FullName}' in vector store record definition.")
};
throw new InvalidOperationException(
$"Property '{clrProperty.Name}' is present in the {nameof(VectorStoreCollectionDefinition)} as a {definitionKind} property, but the .NET property on type '{type?.Name}' has an incompatible attribute.");
}
}
string propertyName = clrProperty?.Name ?? definitionProperty!.Name;
Type propertyType = clrProperty?.PropertyType
?? definitionProperty!.Type
?? throw new InvalidOperationException(VectorDataStrings.MissingTypeOnPropertyDefinition(definitionProperty!));
PropertyModel property;
string? attributeStorageName = null;
if (keyAttribute is not null || definitionProperty is VectorStoreKeyProperty)
{
var keyProperty = new KeyPropertyModel(propertyName, propertyType);
if (keyAttribute is not null)
{
keyProperty.IsAutoGenerated = keyAttribute.IsAutoGeneratedNullable ?? this.SupportsKeyAutoGeneration(keyProperty.Type);
attributeStorageName = keyAttribute.StorageName;
}
// Definition values override attribute values.
if (definitionProperty is VectorStoreKeyProperty defKey)
{
keyProperty.IsAutoGenerated = defKey.IsAutoGenerated ?? this.SupportsKeyAutoGeneration(keyProperty.Type);
}
this.KeyProperties.Add(keyProperty);
property = keyProperty;
}
else if (dataAttribute is not null || definitionProperty is VectorStoreDataProperty)
{
var dataProperty = new DataPropertyModel(propertyName, propertyType);
if (dataAttribute is not null)
{
dataProperty.IsIndexed = dataAttribute.IsIndexed;
dataProperty.IsFullTextIndexed = dataAttribute.IsFullTextIndexed;
attributeStorageName = dataAttribute.StorageName;
}
// Definition values override attribute values.
if (definitionProperty is VectorStoreDataProperty defData)
{
dataProperty.IsIndexed = defData.IsIndexed;
dataProperty.IsFullTextIndexed = defData.IsFullTextIndexed;
}
this.DataProperties.Add(dataProperty);
property = dataProperty;
}
else if (vectorAttribute is not null || definitionProperty is VectorStoreVectorProperty)
{
// If a definition exists, create via the definition to preserve generic type info (VectorStoreVectorProperty<TInput>).
var vectorProperty = definitionProperty is VectorStoreVectorProperty defVec
? defVec.CreatePropertyModel()
: new VectorPropertyModel(propertyName, propertyType);
if (vectorAttribute is not null)
{
vectorProperty.Dimensions = vectorAttribute.Dimensions;
vectorProperty.IndexKind = vectorAttribute.IndexKind;
vectorProperty.DistanceFunction = vectorAttribute.DistanceFunction;
attributeStorageName = vectorAttribute.StorageName;
}
// Definition values override attribute values.
if (definitionProperty is VectorStoreVectorProperty defVectorProp)
{
vectorProperty.Dimensions = defVectorProp.Dimensions;
if (defVectorProp.IndexKind is not null)
{
vectorProperty.IndexKind = defVectorProp.IndexKind;
}
if (defVectorProp.DistanceFunction is not null)
{
vectorProperty.DistanceFunction = defVectorProp.DistanceFunction;
}
}
this.ConfigureVectorPropertyEmbedding(
vectorProperty,
(definitionProperty as VectorStoreVectorProperty)?.EmbeddingGenerator ?? this.DefaultEmbeddingGenerator,
(definitionProperty as VectorStoreVectorProperty)?.EmbeddingType);
this.VectorProperties.Add(vectorProperty);
property = vectorProperty;
}
else
{
throw new UnreachableException();
}
// Apply storage name: attribute first, then definition (which takes precedence).
this.SetPropertyStorageName(property, attributeStorageName, type);
if (definitionProperty is not null)
{
this.SetPropertyStorageName(property, definitionProperty.StorageName, type);
}
if (definitionProperty?.ProviderAnnotations is not null)
{
property.ProviderAnnotations = new Dictionary<string, object?>(definitionProperty.ProviderAnnotations);
}
if (clrProperty is not null)
{
property.PropertyInfo = clrProperty;
}
this.PropertyMap.Add(propertyName, property);
}
private void SetPropertyStorageName(PropertyModel property, string? storageName, Type? type)
{
if (property is KeyPropertyModel && this.Options.ReservedKeyStorageName is not null)
{
// If we have ReservedKeyStorageName, there can only be a single key property (validated in the constructor)
property.StorageName = this.Options.ReservedKeyStorageName;
return;
}
if (storageName is null)
{
return;
}
// If a custom serializer is used (e.g. JsonSerializer), it would ignore our own attributes/config, and
// our model needs to be in sync with the serializer's behavior (for e.g. storage names in filters).
// So we ignore the config here as well.
// TODO: Consider throwing here instead of ignoring
if (this.Options.UsesExternalSerializer && type != null)
{
return;
}
property.StorageName = storageName;
}
/// <summary>
/// Gets the embedding types supported by this provider, in priority order.
/// The first type whose embedding generator is compatible with the input type will be used.
/// </summary>
/// <remarks>
/// Override this property in connectors that support additional embedding types beyond <see cref="Embedding{T}"/> of <see langword="float"/>.
/// </remarks>
protected virtual IReadOnlyList<EmbeddingGenerationDispatcher> EmbeddingGenerationDispatchers { get; }
= [EmbeddingGenerationDispatcher.Create<Embedding<float>>()];
/// <summary>
/// Attempts to resolve the embedding type for the given vector property, iterating over <see cref="EmbeddingGenerationDispatchers"/> in priority order.
/// </summary>
private (Type? EmbeddingType, EmbeddingGenerationDispatcher? Handler) ResolveEmbeddingType(
VectorPropertyModel vectorProperty,
IEmbeddingGenerator embeddingGenerator,
Type? userRequestedEmbeddingType)
{
foreach (var supported in this.EmbeddingGenerationDispatchers)
{
if (supported.ResolveEmbeddingType(vectorProperty, embeddingGenerator, userRequestedEmbeddingType) is { } resolved)
{
return (resolved, supported);
}
}
return (null, null);
}
/// <summary>
/// Resolves the embedding handler for a native vector property type, where embedding generation is only needed for search.
/// Since the property type is already a valid native type, we only check if the generator can produce the
/// embedding output type (regardless of input type, which is only known at search time).
/// </summary>
private EmbeddingGenerationDispatcher? ResolveSearchOnlyEmbeddingHandler(VectorPropertyModel vectorProperty, IEmbeddingGenerator embeddingGenerator)
{
foreach (var supported in this.EmbeddingGenerationDispatchers)
{
if (supported.CanGenerateEmbedding(vectorProperty, embeddingGenerator))
{
return supported;
}
}
return null;
}
/// <summary>
/// Configures embedding generation for a vector property. Sets the embedding generator, resolves the embedding type,
/// and assigns the appropriate <see cref="EmbeddingGenerationDispatcher"/>.
/// </summary>
/// <remarks>
/// If the property's type is natively supported (e.g. <see cref="ReadOnlyMemory{T}"/> of <see langword="float"/>), the embedding type
/// is set to the property's type; if a generator is also configured, a search-only dispatcher is resolved so that search can convert
/// arbitrary inputs (e.g. string) to embeddings.
/// Otherwise, if a generator is configured, the embedding type is resolved from it. If resolution fails, the embedding type remains
/// <see langword="null"/> and an error is deferred to the validation phase.
/// </remarks>
private void ConfigureVectorPropertyEmbedding(
VectorPropertyModel vectorProperty,
IEmbeddingGenerator? embeddingGenerator,
Type? userRequestedEmbeddingType)
{
vectorProperty.EmbeddingGenerator = embeddingGenerator;
if (this.IsVectorPropertyTypeValid(vectorProperty.Type, out _))
{
if (userRequestedEmbeddingType is not null && userRequestedEmbeddingType != vectorProperty.Type)
{
throw new InvalidOperationException(VectorDataStrings.DifferentEmbeddingTypeSpecifiedForNativelySupportedType(vectorProperty, userRequestedEmbeddingType));
}
vectorProperty.EmbeddingType = vectorProperty.Type;
// Even for native types, if an embedding generator is configured, resolve the dispatcher
// so that search can convert arbitrary inputs (e.g. string) to embeddings.
if (embeddingGenerator is not null)
{
vectorProperty.EmbeddingGenerationDispatcher = this.ResolveSearchOnlyEmbeddingHandler(vectorProperty, embeddingGenerator);
}
}
else if (embeddingGenerator is not null)
{
// The property type isn't a valid embedding type, but an embedding generator is configured.
// Try to resolve the embedding type from it: if the configured generator supports translating the input type (e.g. string) to
// an output type supported by the provider, we set that as the embedding type.
// If this fails, EmbeddingType remains null and we defer the error to the validation phase.
var (embeddingType, handler) = this.ResolveEmbeddingType(vectorProperty, embeddingGenerator, userRequestedEmbeddingType);
vectorProperty.EmbeddingType = embeddingType;
vectorProperty.EmbeddingGenerationDispatcher = handler;
}
// If the property type isn't valid and there's no embedding generator, that's an error.
// But we throw later, in validation, to allow for provider customization to correct this invalid state after this step.
}
/// <summary>
/// Extension hook for connectors to be able to customize the model.
/// </summary>
protected virtual void Customize()
{
}
/// <summary>
/// Validates the model after all properties have been processed.
/// </summary>
protected virtual void Validate(Type? type, VectorStoreCollectionDefinition? definition)
{
if (this.KeyProperties.Count > 1)
{
throw new NotSupportedException($"Multiple key properties found on {TypeMessage()}the provided {nameof(VectorStoreCollectionDefinition)} while only one is supported.");
}
if (this.KeyProperties.Count == 0)
{
throw new NotSupportedException($"No key property found on {TypeMessage()}the provided {nameof(VectorStoreCollectionDefinition)} while at least one is required.");
}
if (this.Options.RequiresAtLeastOneVector && this.VectorProperties.Count == 0)
{
throw new NotSupportedException($"No vector property found on {TypeMessage()}the provided {nameof(VectorStoreCollectionDefinition)} while at least one is required.");
}
if (!this.Options.SupportsMultipleVectors && this.VectorProperties.Count > 1)
{
throw new NotSupportedException($"Multiple vector properties found on {TypeMessage()}the provided {nameof(VectorStoreCollectionDefinition)} while only one is supported.");
}
var storageNameMap = new Dictionary<string, PropertyModel>();
foreach (var property in this.PropertyMap.Values)
{
this.ValidateProperty(property, definition);
if (storageNameMap.TryGetValue(property.StorageName, out var otherproperty))
{
throw new InvalidOperationException($"Property '{property.ModelName}' is being mapped to storage name '{property.StorageName}', but property '{otherproperty.ModelName}' is already mapped to the same storage name.");
}
storageNameMap[property.StorageName] = property;
}
string TypeMessage() => type is null ? "" : $"type '{type.Name}' or ";
}
/// <summary>
/// Validates a single property, performing validation on it.
/// </summary>
protected virtual void ValidateProperty(PropertyModel propertyModel, VectorStoreCollectionDefinition? definition)
{
var type = propertyModel.Type;
Debug.Assert(propertyModel.Type is not null);
switch (propertyModel)
{
case KeyPropertyModel keyProperty:
if (keyProperty.IsAutoGenerated && !this.SupportsKeyAutoGeneration(keyProperty.Type))
{
throw new NotSupportedException(
$"Property '{keyProperty.ModelName}' is configured for auto-generation, but key properties of type '{keyProperty.Type.Name}' do not support auto-generation.");
}
this.ValidateKeyProperty(keyProperty);
break;
case DataPropertyModel dataProperty:
if (!this.IsDataPropertyTypeValid(dataProperty.Type, out var supportedTypes))
{
throw new NotSupportedException(
$"Property '{dataProperty.ModelName}' has unsupported type '{type.Name}'. Data properties must be one of the supported types: {supportedTypes}.");
}
break;
case VectorPropertyModel vectorProperty:
if (vectorProperty.EmbeddingType is null)
{
if (this.IsVectorPropertyTypeValid(vectorProperty.Type, out string? supportedVectorTypes))
{
throw new UnreachableException("EmbeddingType cannot be null when the property type is supported.");
}
if (vectorProperty.EmbeddingGenerator is null)
{
throw new InvalidOperationException(VectorDataStrings.UnsupportedVectorPropertyWithoutEmbeddingGenerator(vectorProperty));
}
// If the user has configured a desired embedding type (done to use en embedding type other than the provider's default one), throw errors tailored to that.
// Throw errors related to that.
var userRequestedEmbeddingType = definition?.Properties.OfType<VectorStoreVectorProperty>().SingleOrDefault(p => p.Name == vectorProperty.ModelName)?.EmbeddingType;
if (userRequestedEmbeddingType is not null)
{
throw new InvalidOperationException(this.IsVectorPropertyTypeValid(userRequestedEmbeddingType, out _)
? VectorDataStrings.ConfiguredEmbeddingTypeIsUnsupportedByTheGenerator(vectorProperty, userRequestedEmbeddingType, supportedVectorTypes)
: VectorDataStrings.ConfiguredEmbeddingTypeIsUnsupportedByTheProvider(vectorProperty, userRequestedEmbeddingType, supportedVectorTypes));
}
throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGenerator(vectorProperty, vectorProperty.EmbeddingGenerator, supportedVectorTypes));
}
if (!this.IsVectorPropertyTypeValid(vectorProperty.EmbeddingType, out string? supportedVectorTypes2))
{
// Should in principle never happen, only with incorrect provider customization.
throw new InvalidOperationException($"Property '{vectorProperty.ModelName}' has unsupported embedding type '{vectorProperty.EmbeddingType.Name}'. Vector properties must be one of the supported types: {supportedVectorTypes2}.");
}
if (vectorProperty.Dimensions <= 0)
{
throw new InvalidOperationException($"Vector property '{propertyModel.ModelName}' must have a positive number of dimensions.");
}
break;
default:
throw new UnreachableException();
}
}
/// <summary>
/// Configures auto-generation for the given key property.
/// Defaults to configuring <see cref="Guid" /> key properties as auto-generated, and throwing if auto-generation is requested for
/// any other type.
/// </summary>
protected virtual bool SupportsKeyAutoGeneration(Type keyPropertyType)
=> keyPropertyType == typeof(Guid);
/// <summary>
/// Validates the key property. The default implementation validates that the collection's generic key type (<see cref="KeyType"/>)
/// corresponds to the key property type on the model, if <see cref="KeyType"/> was provided.
/// Provider overrides should call the base implementation.
/// </summary>
protected virtual void ValidateKeyProperty(KeyPropertyModel keyProperty)
{
if (this.KeyType is not null && this.KeyType != typeof(object) && this.KeyType != keyProperty.Type)
{
throw new InvalidOperationException(
$"The collection's generic key type is '{this.KeyType.Name}', but the key property '{keyProperty.ModelName}' has type '{keyProperty.Type.Name}'. The generic key type must match the key property type.");
}
}
/// <summary>
/// Validates that the .NET type for a data property is supported by the provider.
/// </summary>
protected abstract bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes);
/// <summary>
/// Validates that the .NET type for a vector property is supported by the provider.
/// </summary>
protected abstract bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes);
}
dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModelBuildingOptions.cs
-34
View File
@@ -1,34 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Diagnostics.CodeAnalysis;
namespace Microsoft.Extensions.VectorData.ProviderServices;
/// <summary>
/// Contains options affecting model building; passed to <see cref="CollectionModelBuilder"/>.
/// This is an internal support type meant for use by connectors only and not by applications.
/// </summary>
[Experimental("MEVD9001")]
public sealed class CollectionModelBuildingOptions
{
/// <summary>
/// Gets a value that indicates whether multiple vector properties are supported.
/// </summary>
public required bool SupportsMultipleVectors { get; init; }
/// <summary>
/// Gets a value that indicates whether at least one vector property is required.
/// </summary>
public required bool RequiresAtLeastOneVector { get; init; }
/// <summary>
/// Gets a value that indicates whether an external serializer will be used (for example, System.Text.Json).
/// </summary>
public bool UsesExternalSerializer { get; init; }
/// <summary>
/// Gets the special, reserved name for the key property of the database.
/// When set, the model builder manages the key storage name, and users cannot customize it.
/// </summary>
public string? ReservedKeyStorageName { get; init; }
}
dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/DataPropertyModel.cs
-34
View File
@@ -1,34 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Diagnostics.CodeAnalysis;
namespace Microsoft.Extensions.VectorData.ProviderServices;
/// <summary>
/// Represents a data property on a vector store record.
/// This is an internal support type meant for use by connectors only and not by applications.
/// </summary>
[Experimental("MEVD9001")]
public class DataPropertyModel(string modelName, Type type) : PropertyModel(modelName, type)
{
/// <summary>
/// Gets or sets a value indicating whether this data property is indexed.
/// </summary>
/// <value>
/// The default is <see langword="false" />.
/// </value>
public bool IsIndexed { get; set; }
/// <summary>
/// Gets or sets a value indicating whether this data property is indexed for full-text search.
/// </summary>
/// <value>
/// The default is <see langword="false" />.
/// </value>
public bool IsFullTextIndexed { get; set; }
/// <inheritdoc/>
public override string ToString()
=> $"{this.ModelName} (Data, {this.Type.Name})";
}
dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/EmbeddingGenerationDispatcher.cs
-86
View File
@@ -1,86 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
namespace Microsoft.Extensions.VectorData.ProviderServices;
/// <summary>
/// Represents a supported embedding type for a vector store provider.
/// This is an internal support type meant for use by connectors only and not by applications.
/// </summary>
/// <remarks>
/// Each instance encapsulates both build-time embedding type resolution and runtime embedding generation
/// for a specific <see cref="Embedding"/> subtype.
/// </remarks>
[Experimental("MEVD9001")]
public abstract class EmbeddingGenerationDispatcher
{
/// <summary>
/// Gets the <see cref="Embedding"/> type that this instance supports.
/// </summary>
public abstract Type EmbeddingType { get; }
/// <summary>
/// Attempts to resolve the embedding type for the given <paramref name="vectorProperty"/>, using the given <paramref name="embeddingGenerator"/>.
/// </summary>
/// <returns>The resolved embedding type, or <see langword="null"/> if the generator does not support this embedding type.</returns>
public abstract Type? ResolveEmbeddingType(VectorPropertyModel vectorProperty, IEmbeddingGenerator embeddingGenerator, Type? userRequestedEmbeddingType);
/// <summary>
/// Generates embeddings of this type from the given <paramref name="values"/>, using the embedding generator configured on the <paramref name="vectorProperty"/>.
/// </summary>
public abstract Task<IReadOnlyList<Embedding>> GenerateEmbeddingsAsync(VectorPropertyModel vectorProperty, IEnumerable<object?> values, CancellationToken cancellationToken);
/// <summary>
/// Generates a single embedding of this type from the given <paramref name="value"/>, using the embedding generator configured on the <paramref name="vectorProperty"/>.
/// </summary>
public abstract Task<Embedding> GenerateEmbeddingAsync(VectorPropertyModel vectorProperty, object? value, CancellationToken cancellationToken);
/// <summary>
/// Checks whether the given <paramref name="embeddingGenerator"/> can produce embeddings of this type for any of the input types
/// supported by the given <paramref name="vectorProperty"/>.
/// This is used for native vector property types (e.g., <see cref="ReadOnlyMemory{T}"/> of <see langword="float"/>), where embedding generation
/// is only needed for search and the input type is not known at model-build time.
/// </summary>
public abstract bool CanGenerateEmbedding(VectorPropertyModel vectorProperty, IEmbeddingGenerator embeddingGenerator);
/// <summary>
/// Creates a new <see cref="EmbeddingGenerationDispatcher"/> for the given <typeparamref name="TEmbedding"/> type.
/// </summary>
public static EmbeddingGenerationDispatcher Create<TEmbedding>()
where TEmbedding : Embedding
=> new EmbeddingGenerationDispatcher<TEmbedding>();
}
/// <summary>
/// A <see cref="EmbeddingGenerationDispatcher"/> implementation for a specific <typeparamref name="TEmbedding"/> type.
/// This is an internal support type meant for use by connectors only and not by applications.
/// </summary>
[Experimental("MEVD9001")]
public sealed class EmbeddingGenerationDispatcher<TEmbedding> : EmbeddingGenerationDispatcher
where TEmbedding : Embedding
{
/// <inheritdoc />
public override Type EmbeddingType => typeof(TEmbedding);
/// <inheritdoc />
public override Type? ResolveEmbeddingType(VectorPropertyModel vectorProperty, IEmbeddingGenerator embeddingGenerator, Type? userRequestedEmbeddingType)
=> vectorProperty.ResolveEmbeddingType<TEmbedding>(embeddingGenerator, userRequestedEmbeddingType);
/// <inheritdoc />
public override bool CanGenerateEmbedding(VectorPropertyModel vectorProperty, IEmbeddingGenerator embeddingGenerator)
=> vectorProperty.CanGenerateEmbedding<TEmbedding>(embeddingGenerator);
/// <inheritdoc />
public override Task<IReadOnlyList<Embedding>> GenerateEmbeddingsAsync(VectorPropertyModel vectorProperty, IEnumerable<object?> values, CancellationToken cancellationToken)
=> vectorProperty.GenerateEmbeddingsCoreAsync<TEmbedding>(values, cancellationToken);
/// <inheritdoc />
public override Task<Embedding> GenerateEmbeddingAsync(VectorPropertyModel vectorProperty, object? value, CancellationToken cancellationToken)
=> vectorProperty.GenerateEmbeddingCoreAsync<TEmbedding>(value, cancellationToken);
}
dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterPreprocessingOptions.cs
-22
View File
@@ -1,22 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Diagnostics.CodeAnalysis;
namespace Microsoft.Extensions.VectorData.ProviderServices.Filter;
/// <summary>
/// Options for filter expression preprocessing.
/// This is an internal support type meant for use by connectors only and not by applications.
/// </summary>
[Experimental("MEVD9001")]
public class FilterPreprocessingOptions
{
/// <summary>
/// Whether the connector supports parameterization.
/// </summary>
/// <remarks>
/// If <see langword="false"/>, the visitor will inline captured variables and constant member accesses as simple constant nodes.
/// If <see langword="true"/>, these will instead be replaced with <see cref="QueryParameterExpression"/> nodes.
/// </remarks>
public bool SupportsParameterization { get; init; }
}
dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterTranslatorBase.cs
-403
View File
@@ -1,403 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Linq.Expressions;
using System.Reflection;
namespace Microsoft.Extensions.VectorData.ProviderServices.Filter;
/// <summary>
/// Base class for filter translators used by vector data connectors.
/// Provides common functionality for preprocessing filter expressions and matching common patterns.
/// This is an internal support type meant for use by connectors only and not by applications.
/// </summary>
[Experimental("MEVD9001")]
public abstract class FilterTranslatorBase
{
/// <summary>
/// The collection model for the current translation operation.
/// </summary>
protected CollectionModel Model { get; private set; } = null!;
/// <summary>
/// The parameter expression representing the record in the filter lambda.
/// </summary>
protected ParameterExpression RecordParameter { get; private set; } = null!;
/// <summary>
/// Preprocesses the filter expression before translation.
/// Sets <see cref="Model"/> and <see cref="RecordParameter"/>, runs the preprocessing visitor,
/// and returns the preprocessed expression.
/// </summary>
/// <param name="lambdaExpression">The filter lambda expression to preprocess.</param>
/// <param name="model">The collection model containing property information.</param>
/// <param name="options">Options controlling the preprocessing behavior.</param>
/// <returns>The preprocessed expression ready for translation.</returns>
protected Expression PreprocessFilter(LambdaExpression lambdaExpression, CollectionModel model, FilterPreprocessingOptions options)
{
this.Model = model;
this.RecordParameter = lambdaExpression.Parameters[0];
var preprocessor = new FilterTranslationPreprocessor(options.SupportsParameterization);
return preprocessor.Preprocess(lambdaExpression.Body);
}
/// <summary>
/// Tries to match a Contains method call expression and extract the source collection and item expressions.
/// </summary>
/// <param name="methodCall">The method call expression to match.</param>
/// <param name="source">When successful, the source collection expression.</param>
/// <param name="item">When successful, the item expression being searched for.</param>
/// <returns><see langword="true"/> if the expression is a recognized Contains pattern; otherwise, <see langword="false"/>.</returns>
protected static bool TryMatchContains(
MethodCallExpression methodCall,
[NotNullWhen(true)] out Expression? source,
[NotNullWhen(true)] out Expression? item)
{
switch (methodCall)
{
// Enumerable.Contains()
case { Method.Name: nameof(Enumerable.Contains), Arguments: [var src, var itm] }
when methodCall.Method.DeclaringType == typeof(Enumerable):
source = src;
item = itm;
return true;
// List<T>.Contains()
case
{
Method:
{
Name: nameof(Enumerable.Contains),
DeclaringType: { IsGenericType: true } declaringType
},
Object: Expression src,
Arguments: [var itm]
} when declaringType.GetGenericTypeDefinition() == typeof(List<>):
source = src;
item = itm;
return true;
// C# 14 made changes to overload resolution to prefer Span-based overloads when those exist ("first-class spans");
// this makes MemoryExtensions.Contains() be resolved rather than Enumerable.Contains() (see above).
// MemoryExtensions.Contains() also accepts a Span argument for the source, adding an implicit cast we need to remove.
// See https://github.com/dotnet/runtime/issues/109757 for more context.
// Note that MemoryExtensions.Contains has an optional 3rd ComparisonType parameter; we only match when
// it's null.
case { Method.Name: nameof(MemoryExtensions.Contains), Arguments: [var spanArg, var itm, ..] }
when methodCall.Method.DeclaringType == typeof(MemoryExtensions)
&& (methodCall.Arguments.Count is 2
|| (methodCall.Arguments.Count is 3 && methodCall.Arguments[2] is ConstantExpression { Value: null }))
&& TryUnwrapSpanImplicitCast(spanArg, out var src):
source = src;
item = itm;
return true;
default:
source = null;
item = null;
return false;
}
}
/// <summary>
/// Tries to bind an expression to a property in the collection model.
/// </summary>
/// <param name="expression">The expression to bind.</param>
/// <param name="propertyModel">When successful, the property model that was bound.</param>
/// <returns><see langword="true"/> if the expression was successfully bound to a property; otherwise, <see langword="false"/>.</returns>
protected virtual bool TryBindProperty(Expression expression, [NotNullWhen(true)] out PropertyModel? propertyModel)
{
var unwrappedExpression = expression;
while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert)
{
unwrappedExpression = convert.Operand;
}
var modelName = unwrappedExpression switch
{
// Regular member access for strongly-typed POCO binding (e.g. r => r.SomeInt == 8)
MemberExpression memberExpression when memberExpression.Expression == this.RecordParameter
=> memberExpression.Member.Name,
// Dictionary lookup for weakly-typed dynamic binding (e.g. r => r["SomeInt"] == 8)
MethodCallExpression
{
Method: { Name: "get_Item", DeclaringType: var declaringType },
Arguments: [ConstantExpression { Value: string keyName }]
} methodCall when methodCall.Object == this.RecordParameter && declaringType == typeof(Dictionary<string, object?>)
=> keyName,
_ => null
};
if (modelName is null)
{
propertyModel = null;
return false;
}
if (!this.Model.PropertyMap.TryGetValue(modelName, out propertyModel))
{
throw new InvalidOperationException($"Property name '{modelName}' provided as part of the filter clause is not a valid property name.");
}
// Now that we have the property, go over all wrapping Convert nodes again to ensure that they're compatible with the property type
var unwrappedPropertyType = Nullable.GetUnderlyingType(propertyModel.Type) ?? propertyModel.Type;
unwrappedExpression = expression;
while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert)
{
var convertType = Nullable.GetUnderlyingType(convert.Type) ?? convert.Type;
if (convertType != unwrappedPropertyType && convertType != typeof(object))
{
throw new InvalidCastException($"Property '{propertyModel.ModelName}' is being cast to type '{convert.Type.Name}', but its configured type is '{propertyModel.Type.Name}'.");
}
unwrappedExpression = convert.Operand;
}
return true;
}
/// <summary>
/// Tries to unwrap an implicit cast to Span or ReadOnlySpan that may be present in expressions
/// when C# 14's first-class span support causes MemoryExtensions methods to be resolved.
/// </summary>
/// <param name="expression">The expression to unwrap.</param>
/// <param name="result">When successful, the unwrapped expression.</param>
/// <returns><see langword="true"/> if a span implicit cast was unwrapped; otherwise, <see langword="false"/>.</returns>
protected static bool TryUnwrapSpanImplicitCast(Expression expression, [NotNullWhen(true)] out Expression? result)
{
// Different versions of the compiler seem to generate slightly different expression tree representations for this
// implicit cast:
var (unwrapped, castDeclaringType) = expression switch
{
UnaryExpression
{
NodeType: ExpressionType.Convert,
Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType },
Operand: var operand
} => (operand, implicitCastDeclaringType),
MethodCallExpression
{
Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType },
Arguments: [var firstArgument]
} => (firstArgument, implicitCastDeclaringType),
// After the preprocessor runs, the Convert node may have Method: null because the visitor
// recreates the UnaryExpression with a different operand type (QueryParameterExpression).
// Handle this case by checking if the target type is Span<T> or ReadOnlySpan<T>.
UnaryExpression
{
NodeType: ExpressionType.Convert,
Method: null,
Type: { IsGenericType: true } targetType,
Operand: var operand
} when targetType.GetGenericTypeDefinition() is var gtd
&& (gtd == typeof(Span<>) || gtd == typeof(ReadOnlySpan<>))
=> (operand, targetType),
_ => (null, null)
};
// For the dynamic case, there's a Convert node representing an up-cast to object[]; unwrap that too.
// Also handle cases where the preprocessor adds a Convert node back to the array type.
while (unwrapped is UnaryExpression
{
NodeType: ExpressionType.Convert,
Method: null,
Operand: var innerOperand
})
{
unwrapped = innerOperand;
}
if (unwrapped is not null
&& castDeclaringType?.GetGenericTypeDefinition() is var genericTypeDefinition
&& (genericTypeDefinition == typeof(Span<>) || genericTypeDefinition == typeof(ReadOnlySpan<>)))
{
result = unwrapped;
return true;
}
result = null;
return false;
}
#region FilterTranslationPreprocessor
/// <summary>
/// A processor for user-provided filter expressions which performs various common transformations before actual translation takes place.
/// </summary>
private sealed class FilterTranslationPreprocessor : ExpressionVisitor
{
private readonly bool _supportsParameterization;
private List<string>? _parameterNames;
internal FilterTranslationPreprocessor(bool supportsParameterization)
{
this._supportsParameterization = supportsParameterization;
}
internal Expression Preprocess(Expression node)
{
if (this._supportsParameterization)
{
this._parameterNames = [];
}
return this.Visit(node);
}
/// <inheritdoc />
protected override Expression VisitMember(MemberExpression node)
{
var visited = (MemberExpression)base.VisitMember(node);
// This identifies field and property access over constants, which can be evaluated immediately.
// This covers captured variables, since those are actually member accesses over compiled-generated closure types:
// var x = 8;
// _ = await collection.SearchAsync(vector, top: 3, new() { Filter = r => r.Int == x });
//
// This also covers member variables:
// _ = await collection.SearchAsync(vector, top: 3, new() { Filter = r => r.Int == this._x });
// ... as "this" here is represented by a ConstantExpression node in the tree.
//
// Some databases - mostly relational ones - support out-of-band parameters which can be referenced via placeholders
// from the query itself. For those databases, we transform the member access to QueryParameterExpression (this simplifies things for those
// connectors, and centralizes the pattern matching in a single centralized place).
// For databases which don't support parameters, we simply inline the evaluated member access as a constant in the tree, so that translators don't
// even need to be aware of it.
// Evaluate the MemberExpression to get the actual value, either for instance members (expression is a ConstantExpression) or for
// static members (expression is null).
object? baseValue;
switch (visited.Expression)
{
// Member access over constant (i.e. instance members)
case ConstantExpression { Value: var v }:
baseValue = v;
break;
// Member constant over null (i.e. static members)
case null:
baseValue = null;
break;
// Member constant over something that has already been parameterized (i.e. nested member access, e.g. r=> r.Int == this.SomeWrapper.Something)
case QueryParameterExpression p:
baseValue = p.Value;
// The previous parameter is getting replaced by the new one we're creating here, so remove its name from the list of parameter names.
this._parameterNames!.Remove(p.Name);
break;
default:
return visited;
}
object? evaluatedValue;
var memberInfo = visited.Member;
switch (memberInfo)
{
case FieldInfo fieldInfo:
evaluatedValue = fieldInfo.GetValue(baseValue);
break;
case PropertyInfo { GetMethod.IsStatic: false } propertyInfo when baseValue is null:
throw new InvalidOperationException($"Cannot access member '{propertyInfo.Name}' on null object.");
case PropertyInfo propertyInfo:
evaluatedValue = propertyInfo.GetValue(baseValue);
break;
default:
return visited;
}
// Inline the evaluated value (if the connector doesn't support parameterization, or if the field is readonly),
if (!this._supportsParameterization)
{
return Expression.Constant(evaluatedValue, visited.Type);
}
// Otherwise, transform the node to a QueryParameterExpression which the connector will then translate to a parameter (e.g. SqlParameter).
// TODO: Share the same parameter when it references the same captured value
// Make sure parameter names are unique.
var origName = memberInfo.Name;
var name = origName;
for (var i = 0; this._parameterNames!.Contains(name); i++)
{
name = $"{origName}_{i}";
}
this._parameterNames.Add(name);
return new QueryParameterExpression(name, evaluatedValue, visited.Type);
}
/// <inheritdoc />
protected override Expression VisitNew(NewExpression node)
{
var visited = (NewExpression)base.VisitNew(node);
// Recognize certain well-known constructors where we can evaluate immediately, converting the NewExpression to a ConstantExpression.
// This is particularly useful for converting inline instantiation of DateTime, DateTimeOffset, DateOnly, and TimeOnly to constants, which can then be easily translated.
switch (visited.Constructor)
{
case ConstructorInfo constructor when constructor.DeclaringType == typeof(DateTimeOffset) || constructor.DeclaringType == typeof(DateTime)
#if NET
|| constructor.DeclaringType == typeof(DateOnly) || constructor.DeclaringType == typeof(TimeOnly)
#endif
:
var constantArguments = new object?[visited.Arguments.Count];
// We first do a fast path to check if all arguments are constants; this catches the common case of e.g. new DateTime(2023, 10, 1).
// If an argument isn't a constant (e.g. new DateTimeOffset(..., TimeSpan.FromHours(2))), we fall back to trying the LINQ interpreter
// as a general-purpose expression evaluator - but note that this is considerably slower.
for (var i = 0; i < visited.Arguments.Count; i++)
{
if (visited.Arguments[i] is ConstantExpression constantArgument)
{
constantArguments[i] = constantArgument.Value;
}
else
{
// There's a non-constant argument - try the LINQ interpreter.
#pragma warning disable CA1031 // Do not catch general exception types
try
{
var evaluated = Expression.Lambda<Func<object>>(Expression.Convert(visited, typeof(object)))
#if NET
.Compile(preferInterpretation: true)
#else
.Compile()
#endif
.Invoke();
return Expression.Constant(evaluated, constructor.DeclaringType);
}
catch
{
return visited;
}
#pragma warning restore CA1031
}
}
var constantValue = constructor.Invoke(constantArguments);
return Expression.Constant(constantValue, constructor.DeclaringType);
}
return visited;
}
}
#endregion FilterTranslationPreprocessor
}
dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/QueryParameterExpression.cs
-33
View File
@@ -1,33 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Diagnostics.CodeAnalysis;
using System.Linq.Expressions;
namespace Microsoft.Extensions.VectorData.ProviderServices.Filter;
/// <summary>
/// An expression representation a query parameter (captured variable) in the filter expression.
/// </summary>
[Experimental("MEVD9001")]
public class QueryParameterExpression(string name, object? value, Type type) : Expression
{
/// <summary>
/// The name of the parameter.
/// </summary>
public string Name { get; } = name;
/// <summary>
/// The value of the parameter.
/// </summary>
public object? Value { get; } = value;
/// <inheritdoc />
public override ExpressionType NodeType => ExpressionType.Extension;
/// <inheritdoc />
public override Type Type => type;
/// <inheritdoc />
protected override Expression VisitChildren(ExpressionVisitor visitor) => this;
}
dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/IRecordCreator.cs
-8
View File
@@ -1,8 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
namespace Microsoft.Extensions.VectorData.ProviderServices;
internal interface IRecordCreator
{
TRecord Create<TRecord>();
}
dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/KeyPropertyModel.cs
-31
View File
@@ -1,31 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Diagnostics.CodeAnalysis;
namespace Microsoft.Extensions.VectorData.ProviderServices;
/// <summary>
/// Represents a key property on a vector store record.
/// This is an internal support type meant for use by connectors only and not by applications.
/// </summary>
[Experimental("MEVD9001")]
public class KeyPropertyModel(string modelName, Type type) : PropertyModel(modelName, type)
{
/// <summary>
/// Gets or sets whether this key property's value is auto-generated or not.
/// </summary>
public bool IsAutoGenerated { get; set; }
/// <summary>
/// Gets or sets the name that the JSON serializer will produce for this key property.
/// This is needed for connectors that use an external JSON serializer combined with a reserved key storage name
/// (e.g. CosmosDB NoSQL uses "id"): the serializer produces a JSON object with the policy-transformed name, and
/// the connector needs to find and replace it with the reserved storage name.
/// </summary>
public string? SerializedKeyName { get; set; }
/// <inheritdoc/>
public override string ToString()
=> $"{this.ModelName} (Key, {this.Type.Name}{(this.IsAutoGenerated ? ", auto-generated" : "")})";
}
dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/PropertyModel.cs
-162
View File
@@ -1,162 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Reflection;
namespace Microsoft.Extensions.VectorData.ProviderServices;
/// <summary>
/// Represents a property on a vector store record.
/// This is an internal support type meant for use by connectors only and not by applications.
/// </summary>
[Experimental("MEVD9001")]
public abstract class PropertyModel(string modelName, Type type)
{
private string? _storageName;
private Func<object, object?>? _getter;
private Action<object, object?>? _setter;
/// <summary>
/// Gets or sets the model name of the property. If the property corresponds to a .NET property, this name is the name of that property.
/// </summary>
public string ModelName { get; set; } = modelName;
/// <summary>
/// Gets or sets the storage name of the property. This is the name to which the property is mapped in the vector store.
/// </summary>
public string StorageName
{
get => this._storageName ?? this.ModelName;
set => this._storageName = value;
}
/// <summary>
/// Gets or sets the CLR type of the property.
/// </summary>
public Type Type { get; set; } = type;
/// <summary>
/// Gets or sets the reflection <see cref="PropertyInfo"/> for the .NET property.
/// </summary>
/// <value>
/// The reflection <see cref="PropertyInfo"/> for the .NET property.
/// <see langword="null"/> when using dynamic mapping.
/// </value>
public PropertyInfo? PropertyInfo { get; set; }
/// <summary>
/// Gets or sets a dictionary of provider-specific annotations for this property.
/// </summary>
/// <remarks>
/// This allows setting database-specific configuration options that aren't universal across all vector stores.
/// </remarks>
public Dictionary<string, object?>? ProviderAnnotations { get; set; }
/// <summary>
/// Gets whether the property type is nullable. For value types, this is <see langword="true"/> when the type is
/// <see cref="Nullable{T}"/>. For reference types on .NET 6+, this uses NRT annotations via
/// <c>NullabilityInfoContext</c> when a <see cref="PropertyInfo"/> is available
/// (i.e., POCO mapping); otherwise, reference types are assumed nullable.
/// </summary>
public bool IsNullable
{
get
{
// Value types: nullable only if Nullable<T>
if (this.Type.IsValueType)
{
return Nullable.GetUnderlyingType(this.Type) is not null;
}
// Reference types: check NRT annotation via NullabilityInfoContext when available
#if NET
if (this.PropertyInfo is { } propertyInfo)
{
var nullabilityInfo = new NullabilityInfoContext().Create(propertyInfo);
return nullabilityInfo.ReadState != NullabilityState.NotNull;
}
#endif
// Dynamic mapping or old framework: assume nullable for reference types
return true;
}
}
/// <summary>
/// Configures the property accessors using a CLR <see cref="System.Reflection.PropertyInfo"/> for POCO mapping.
/// </summary>
// TODO: Implement compiled delegates for better performance, #11122
// TODO: Implement source-generated accessors for NativeAOT, #10256
internal void ConfigurePocoAccessors(PropertyInfo propertyInfo)
{
this.PropertyInfo = propertyInfo;
this._getter = propertyInfo.GetValue;
this._setter = (record, value) =>
{
// If the value is null, no need to set the property (it's the CLR default)
if (value is not null)
{
propertyInfo.SetValue(record, value);
}
};
}
/// <summary>
/// Configures the property accessors for dynamic mapping using <see cref="Dictionary{TKey, TValue}"/>.
/// </summary>
internal void ConfigureDynamicAccessors()
{
var modelName = this.ModelName;
var propertyType = this.Type;
this._getter = record =>
{
var dictionary = (Dictionary<string, object?>)record;
var value = dictionary.TryGetValue(modelName, out var tempValue) ? tempValue : null;
if (value is not null && value.GetType() != (Nullable.GetUnderlyingType(propertyType) ?? propertyType))
{
throw new InvalidCastException($"Property '{modelName}' has a value of type '{value.GetType().Name}', but its configured type is '{propertyType.Name}'.");
}
return value;
};
this._setter = (record, value) => ((Dictionary<string, object?>)record)[modelName] = value;
}
/// <summary>
/// Reads the property from the given <paramref name="record"/>, returning the value as an <see cref="object"/>.
/// </summary>
public object? GetValueAsObject(object record)
{
Debug.Assert(this._getter is not null, "Property accessors have not been configured.");
return this._getter!(record);
}
/// <summary>
/// Writes the property from the given <paramref name="record"/>, accepting the value to write as an <see cref="object"/>.
/// </summary>
public void SetValueAsObject(object record, object? value)
{
Debug.Assert(this._setter is not null, "Property accessors have not been configured.");
this._setter!(record, value);
}
/// <summary>
/// Reads the property from the given <paramref name="record"/>.
/// </summary>
// TODO: actually implement the generic accessors to avoid boxing, and make use of them in connectors
public T GetValue<T>(object record)
=> (T)(object)this.GetValueAsObject(record)!;
/// <summary>
/// Writes the property from the given <paramref name="record"/>.
/// </summary>
// TODO: actually implement the generic accessors to avoid boxing, and make use of them in connectors
public void SetValue<T>(object record, T value)
=> this.SetValueAsObject(record, value);
}
dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorDataStrings.cs
-79
View File
@@ -1,79 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using Microsoft.Extensions.AI;
namespace Microsoft.Extensions.VectorData.ProviderServices;
#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member
/// <summary>
/// Exposes methods for constructing strings that should be used by providers when throwing exceptions.
/// </summary>
[Experimental("MEVD9001")]
public static class VectorDataStrings
{
public static string ConfiguredEmbeddingTypeIsUnsupportedByTheGenerator(VectorPropertyModel vectorProperty, Type userRequestedEmbeddingType, string supportedVectorTypes)
=> $"Vector property '{vectorProperty.ModelName}' has embedding type '{TypeName(userRequestedEmbeddingType)}' configured, but that type isn't supported by your embedding generator.";
public static string ConfiguredEmbeddingTypeIsUnsupportedByTheProvider(VectorPropertyModel vectorProperty, Type userRequestedEmbeddingType, string supportedVectorTypes)
=> $"Vector property '{vectorProperty.ModelName}' has embedding type '{TypeName(userRequestedEmbeddingType)}' configured, but that type isn't supported by your provider. Supported types are {supportedVectorTypes}.";
public static string EmbeddingGeneratorWithInvalidEmbeddingType(VectorPropertyModel vectorProperty)
=> $"An embedding generator was configured on property '{vectorProperty.ModelName}', but output embedding type '{vectorProperty.EmbeddingType.Name}' isn't supported by the connector.";
public static string EmbeddingPropertyTypeIncompatibleWithEmbeddingGenerator(VectorPropertyModel vectorProperty)
=> $"Property '{vectorProperty.ModelName}' has embedding type '{TypeName(vectorProperty.Type)}', but an embedding generator is configured on the property. Remove the embedding generator or change the property's .NET type to a non-embedding input type to the generator (e.g. string).";
public static string DifferentEmbeddingTypeSpecifiedForNativelySupportedType(VectorPropertyModel vectorProperty, Type embeddingType)
=> $"Property '{vectorProperty.ModelName}' has {nameof(VectorStoreVectorProperty.EmbeddingType)} configured to '{TypeName(embeddingType)}', but the property already has natively supported '{TypeName(vectorProperty.Type)}'. {nameof(VectorStoreVectorProperty.EmbeddingType)} only needs to be specified for properties that require embedding generation.";
public static string GetCollectionWithDictionaryNotSupported
=> "Dynamic mapping via Dictionary<string, object?> is not supported via this method, call GetDynamicCollection() instead.";
public static string IncludeVectorsNotSupportedWithEmbeddingGeneration
=> "When an embedding generator is configured, `Include Vectors` cannot be enabled.";
public static string IncompatibleEmbeddingGenerator(VectorPropertyModel vectorProperty, IEmbeddingGenerator embeddingGenerator, string supportedOutputTypes)
=> $"Embedding generator '{TypeName(embeddingGenerator.GetType())}' on vector property '{vectorProperty.ModelName}' cannot convert the input type '{TypeName(vectorProperty.Type)}' to a supported vector type (one of: {supportedOutputTypes}).";
public static string IncompatibleEmbeddingGeneratorWasConfiguredForInputType(Type inputType, Type embeddingGeneratorType)
=> $"An input of type '{TypeName(inputType)}' was provided, but an incompatible embedding generator of type '{TypeName(embeddingGeneratorType)}' was configured.";
public static string InvalidSearchInputAndNoEmbeddingGeneratorWasConfigured(Type inputType, string supportedVectorTypes)
=> $"A value of type '{TypeName(inputType)}' was passed to 'SearchAsync', but that isn't a supported vector type by your provider and no embedding generator was configured. The supported vector types are: {supportedVectorTypes}.";
public static string MissingTypeOnPropertyDefinition(VectorStoreProperty property)
=> $"Property '{property.Name}' has no type specified in its definition, and does not have a corresponding .NET property. Specify the type on the definition.";
public static string UnsupportedVectorPropertyWithoutEmbeddingGenerator(VectorPropertyModel vectorProperty)
=> $"Vector property '{vectorProperty.ModelName}' has type '{TypeName(vectorProperty.Type)}' which isn't supported by your provider, and no embedding generator is configured. Configure a generator that supports converting '{TypeName(vectorProperty.Type)}' to vector type supported by your provider.";
public static string NonDynamicCollectionWithDictionaryNotSupported(Type dynamicCollectionType)
=> $"Dynamic mapping via Dictionary<string, object?> is not supported via this class, use '{TypeName(dynamicCollectionType)}' instead.";
private static string TypeName(this Type type)
{
var i = type.Name.IndexOf('`');
if (i == -1)
{
return type.Name switch
{
"Int32" => "int",
"Int64" => "long",
"Boolean" => "bool",
"Double" => "double",
"Single" => "float",
"String" => "string",
_ => type.Name
};
}
var genericTypeName = type.Name.Substring(0, i);
var genericArgs = string.Join(", ", type.GetGenericArguments().Select(t => t.TypeName()));
return $"{genericTypeName}<{genericArgs}>";
}
}
dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorPropertyModel.cs
-198
View File
@@ -1,198 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
namespace Microsoft.Extensions.VectorData.ProviderServices;
/// <summary>
/// Represents a vector property on a vector store record.
/// This is an internal support type meant for use by connectors only and not by applications.
/// </summary>
[Experimental("MEVD9001")]
public class VectorPropertyModel(string modelName, Type type) : PropertyModel(modelName, type)
{
private int _dimensions;
/// <summary>
/// Gets or sets the number of dimensions that the vector has.
/// </summary>
/// <remarks>
/// This property is required when creating collections, but can be omitted if not using that functionality.
/// If not provided when trying to create a collection, create will fail.
/// </remarks>
public int Dimensions
{
get => this._dimensions;
set
{
if (value <= 0)
{
throw new ArgumentOutOfRangeException(nameof(value), "Dimensions must be greater than zero.");
}
this._dimensions = value;
}
}
/// <summary>
/// Gets or sets the kind of index to use.
/// </summary>
/// <value>
/// The default varies by database type. For more information, see the documentation of your chosen database connector.
/// </value>
/// <seealso cref="Microsoft.Extensions.VectorData.IndexKind"/>
public string? IndexKind { get; set; }
/// <summary>
/// Gets or sets the distance function to use when comparing vectors.
/// </summary>
/// <value>
/// The default varies by database type. For more information, see the documentation of your chosen database connector.
/// </value>
/// <seealso cref="Microsoft.Extensions.VectorData.DistanceFunction"/>
public string? DistanceFunction { get; set; }
/// <summary>
/// Gets or sets the type representing the embedding stored in the database if <see cref="EmbeddingGenerator"/> is set.
/// Otherwise, this property is identical to <see cref="PropertyModel.Type"/>.
/// </summary>
/// <remarks>
/// This property may be <see langword="null"/> during model building while the embedding type is being resolved,
/// but is guaranteed to be non-null after building completes (validation ensures this).
/// </remarks>
[AllowNull]
public Type EmbeddingType { get; set; } = null!;
/// <summary>
/// Gets or sets the embedding generator to use for this property.
/// </summary>
public IEmbeddingGenerator? EmbeddingGenerator { get; set; }
/// <summary>
/// Gets or sets the <see cref="EmbeddingGenerationDispatcher"/> that was resolved for this property during model building.
/// This handler is used for runtime embedding generation dispatch.
/// </summary>
/// <remarks>
/// This is <see langword="null"/> for vector properties whose type is natively supported by the provider
/// (e.g., <see cref="ReadOnlyMemory{T}"/> of <see langword="float"/>, <see langword="float"/>[], <see cref="Embedding{T}"/>),
/// since no embedding generation is needed.
/// </remarks>
public EmbeddingGenerationDispatcher? EmbeddingGenerationDispatcher { get; set; }
/// <summary>
/// Checks whether the given <paramref name="embeddingGenerator" /> can produce embeddings of type <typeparamref name="TEmbedding" />
/// for any input type known to this property model. The base implementation checks for <see cref="string"/> and <see cref="DataContent"/>;
/// <see cref="VectorPropertyModel{TInput}"/> also checks for <c>TInput</c>.
/// </summary>
/// <remarks>This is used for native vector property types, where the input type isn't known at model-build time.</remarks>
public virtual bool CanGenerateEmbedding<TEmbedding>(IEmbeddingGenerator embeddingGenerator)
where TEmbedding : Embedding
=> embeddingGenerator is IEmbeddingGenerator<string, TEmbedding>
|| embeddingGenerator is IEmbeddingGenerator<DataContent, TEmbedding>;
/// <summary>
/// Checks whether the <see cref="EmbeddingGenerator"/> configured on this property supports the given embedding type.
/// The implementation on this non-generic <see cref="VectorPropertyModel"/> checks for <see cref="string"/>
/// and <see cref="DataContent"/> as input types for <see cref="EmbeddingGenerator"/>.
/// </summary>
public virtual Type? ResolveEmbeddingType<TEmbedding>(IEmbeddingGenerator embeddingGenerator, Type? userRequestedEmbeddingType)
where TEmbedding : Embedding
=> embeddingGenerator switch
{
// On the TInput side, this out-of-the-box/simple implementation supports string and DataContent only
// (users who want arbitrary TInput types need to use the generic subclass of this type).
// The TEmbedding side is provided by the connector via the generic type parameter to this method, as the connector controls/knows which embedding types are supported.
// Note that if the user has manually specified an embedding type (e.g. to choose Embedding<Half> rather than the default Embedding<float>),
// that's provided via the userRequestedEmbeddingType argument; we use that as a filter.
IEmbeddingGenerator<string, TEmbedding> when this.Type == typeof(string) && (userRequestedEmbeddingType is null || userRequestedEmbeddingType == typeof(TEmbedding))
=> typeof(TEmbedding),
IEmbeddingGenerator<DataContent, TEmbedding> when this.Type == typeof(DataContent) && (userRequestedEmbeddingType is null || userRequestedEmbeddingType == typeof(TEmbedding))
=> typeof(TEmbedding),
null => throw new ArgumentNullException(nameof(embeddingGenerator), "This method should only be called when an embedding generator is configured."),
_ => null
};
/// <summary>
/// Generates embeddings for the given <paramref name="values"/>, using the configured <see cref="EmbeddingGenerationDispatcher"/>.
/// </summary>
/// <exception cref="InvalidOperationException">Thrown if no <see cref="EmbeddingGenerationDispatcher"/> is configured on this property.</exception>
public Task<IReadOnlyList<Embedding>> GenerateEmbeddingsAsync(IEnumerable<object?> values, CancellationToken cancellationToken)
=> this.EmbeddingGenerationDispatcher is not { } dispatcher
? throw new InvalidOperationException($"No embedding generation is configured for property '{this.ModelName}'.")
: dispatcher.GenerateEmbeddingsAsync(this, values, cancellationToken);
/// <summary>
/// Generates a single embedding for the given <paramref name="value"/>, using the configured <see cref="EmbeddingGenerationDispatcher"/>.
/// </summary>
/// <exception cref="InvalidOperationException">Thrown if no <see cref="EmbeddingGenerationDispatcher"/> is configured on this property.</exception>
public Task<Embedding> GenerateEmbeddingAsync(object? value, CancellationToken cancellationToken)
=> this.EmbeddingGenerationDispatcher is not { } dispatcher
? throw new InvalidOperationException($"No embedding generation is configured for property '{this.ModelName}'.")
: dispatcher.GenerateEmbeddingAsync(this, value, cancellationToken);
/// <summary>
/// Core method to generate a batch of embeddings. Called by <see cref="EmbeddingGenerationDispatcher{TEmbedding}"/> with the correct type parameter.
/// </summary>
internal virtual async Task<IReadOnlyList<Embedding>> GenerateEmbeddingsCoreAsync<TEmbedding>(IEnumerable<object?> values, CancellationToken cancellationToken)
where TEmbedding : Embedding
=> this.EmbeddingGenerator switch
{
IEmbeddingGenerator<string, TEmbedding> generator when this.EmbeddingType == typeof(TEmbedding)
=> await generator.GenerateAsync(
values.Select(v => v is string s
? s
: throw new InvalidOperationException($"Property '{this.ModelName}' was configured with an embedding generator accepting a string, but {v?.GetType().Name ?? "null"} was provided.")),
options: null,
cancellationToken).ConfigureAwait(false),
IEmbeddingGenerator<DataContent, TEmbedding> generator when this.EmbeddingType == typeof(TEmbedding)
=> await generator.GenerateAsync(
values.Select(v => v is DataContent c
? c
: throw new InvalidOperationException($"Property '{this.ModelName}' was configured with an embedding generator accepting a {nameof(DataContent)}, but {v?.GetType().Name ?? "null"} was provided.")),
options: null,
cancellationToken).ConfigureAwait(false),
null => throw new UnreachableException("This method should only be called when an embedding generator is configured."),
_ => throw new InvalidOperationException(
$"The embedding generator configured on property '{this.ModelName}' cannot produce an embedding of type '{typeof(TEmbedding).Name}' for the given input type."),
};
/// <summary>
/// Core method to generate a single embedding. Called by <see cref="EmbeddingGenerationDispatcher{TEmbedding}"/> with the correct type parameter.
/// </summary>
internal virtual async Task<Embedding> GenerateEmbeddingCoreAsync<TEmbedding>(object? value, CancellationToken cancellationToken)
where TEmbedding : Embedding
=> this.EmbeddingGenerator switch
{
IEmbeddingGenerator<string, TEmbedding> generator when value is string s
=> await generator.GenerateAsync(s, options: null, cancellationToken).ConfigureAwait(false),
IEmbeddingGenerator<DataContent, TEmbedding> generator when value is DataContent c
=> await generator.GenerateAsync(c, options: null, cancellationToken).ConfigureAwait(false),
null => throw new UnreachableException("This method should only be called when an embedding generator is configured."),
_ => throw new InvalidOperationException(
VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(value?.GetType() ?? typeof(object), this.EmbeddingGenerator!.GetType())),
};
/// <summary>
/// Returns the types of input that this property model supports.
/// </summary>
public virtual Type[] GetSupportedInputTypes() => [typeof(string), typeof(DataContent)];
/// <inheritdoc/>
public override string ToString()
=> $"{this.ModelName} (Vector, {this.Type.Name})";
}
dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorPropertyModel{TInput}.cs
-70
View File
@@ -1,70 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
namespace Microsoft.Extensions.VectorData.ProviderServices;
/// <inheritdoc />
[Experimental("MEVD9001")]
public sealed class VectorPropertyModel<TInput>(string modelName) : VectorPropertyModel(modelName, typeof(TInput))
{
/// <inheritdoc />
public override bool CanGenerateEmbedding<TEmbedding>(IEmbeddingGenerator embeddingGenerator)
=> embeddingGenerator is IEmbeddingGenerator<TInput, TEmbedding>
|| base.CanGenerateEmbedding<TEmbedding>(embeddingGenerator);
/// <inheritdoc />
public override Type? ResolveEmbeddingType<TEmbedding>(IEmbeddingGenerator embeddingGenerator, Type? userRequestedEmbeddingType)
=> embeddingGenerator switch
{
IEmbeddingGenerator<TInput, TEmbedding> when this.Type == typeof(TInput) && (userRequestedEmbeddingType is null || userRequestedEmbeddingType == typeof(TEmbedding))
=> typeof(TEmbedding),
null => throw new ArgumentNullException(nameof(embeddingGenerator), "This method should only be called when an embedding generator is configured."),
_ => null
};
/// <inheritdoc />
internal override async Task<IReadOnlyList<Embedding>> GenerateEmbeddingsCoreAsync<TEmbedding>(IEnumerable<object?> values, CancellationToken cancellationToken)
{
switch (this.EmbeddingGenerator)
{
case IEmbeddingGenerator<TInput, TEmbedding> generator when this.EmbeddingType == typeof(TEmbedding):
return await generator.GenerateAsync(
values.Select(v => v is TInput s
? s
: throw new InvalidOperationException($"Property '{this.ModelName}' was configured with an embedding generator accepting a {typeof(TInput).Name}, but {v?.GetType().Name ?? "null"} was provided.")),
options: null,
cancellationToken).ConfigureAwait(false);
case null:
throw new UnreachableException("This method should only be called when an embedding generator is configured.");
default:
throw new InvalidOperationException(
$"The embedding generator configured on property '{this.ModelName}' cannot produce an embedding of type '{typeof(TEmbedding).Name}' for the given input type.");
}
}
/// <inheritdoc />
internal override async Task<Embedding> GenerateEmbeddingCoreAsync<TEmbedding>(object? value, CancellationToken cancellationToken)
{
if (this.EmbeddingGenerator is IEmbeddingGenerator<TInput, TEmbedding> generator && value is TInput t)
{
return await generator.GenerateAsync(t, options: null, cancellationToken).ConfigureAwait(false);
}
// Fall through to base class which checks for string and DataContent input types.
return await base.GenerateEmbeddingCoreAsync<TEmbedding>(value, cancellationToken).ConfigureAwait(false);
}
/// <inheritdoc />
public override Type[] GetSupportedInputTypes() => [typeof(TInput)];
}
dotnet/src/VectorData/VectorData.Abstractions/README.md
+1
View File
@@ -0,0 +1 @@
The code for Microsoft.Extensions.VectorData.Abstractions can now be found in the [dotnet/extensions repository](https://github.com/dotnet/extensions/tree/main/src/Libraries/Microsoft.Extensions.VectorData.Abstractions)
dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreDataAttribute.cs
-41
View File
@@ -1,41 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Defines an attribute to mark a property on a record class as 'data'.
/// </summary>
/// <remarks>
/// Marking a property as 'data' means that the property is not a key and not a vector. But optionally,
/// this property can have an associated vector field containing an embedding for this data.
/// The characteristics defined here influence how the property is treated by the vector store.
/// </remarks>
[AttributeUsage(AttributeTargets.Property, AllowMultiple = false)]
public sealed class VectorStoreDataAttribute : Attribute
{
/// <summary>
/// Gets or sets a value indicating whether this data property is indexed.
/// </summary>
/// <value>
/// The default is <see langword="false" />.
/// </value>
public bool IsIndexed { get; init; }
/// <summary>
/// Gets or sets a value indicating whether this data property is indexed for full-text search.
/// </summary>
/// <value>
/// The default is <see langword="false" />.
/// </value>
public bool IsFullTextIndexed { get; init; }
/// <summary>
/// Gets or sets an optional name to use for the property in storage, if different from the property name.
/// </summary>
/// <remarks>
/// For example, the property name might be "MyProperty" and the storage name might be "my_property".
/// </remarks>
public string? StorageName { get; init; }
}
dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreKeyAttribute.cs
-46
View File
@@ -1,46 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Defines an attribute to mark a property on a record class as the key under which the record is stored in a vector store.
/// </summary>
/// <remarks>
/// The characteristics defined here influence how the property is treated by the vector store.
/// </remarks>
[AttributeUsage(AttributeTargets.Property, AllowMultiple = false)]
public sealed class VectorStoreKeyAttribute : Attribute
{
/// <summary>
/// Gets or sets an optional name to use for the property in storage, if different from the property name.
/// </summary>
/// <remarks>
/// For example, the property name might be "MyProperty" and the storage name might be "my_property".
/// </remarks>
public string? StorageName { get; init; }
/// <summary>
/// Gets or sets whether this key property's value is auto-generated or not.
/// </summary>
/// <remarks>
/// The availability of auto-generated properties - as well as the .NET types supported for them - varies across provider implementations.
/// The getter returns <see langword="false" /> when the value has not been explicitly set; use <see cref="IsAutoGeneratedNullable" /> to distinguish
/// between "explicitly set to false" and "not set".
/// The getter does not throw, even though it cannot distinguish "explicitly set to false" and "not set"; this is a workaround for a C# compiler
/// limitation that does not allow <see cref="Nullable{T}"/> to be used as an attribute argument.
/// </remarks>
public bool IsAutoGenerated
{
// The getter returns GetValueOrDefault() rather than throwing, as a workaround for a C# compiler limitation:
// Nullable<bool> cannot be used as a compile-time attribute argument, so the public property must be bool.
get => this.IsAutoGeneratedNullable.GetValueOrDefault();
set => this.IsAutoGeneratedNullable = value;
}
/// <summary>
/// Gets whether this key property's value is auto-generated or not, or <see langword="null" /> if not set.
/// </summary>
internal bool? IsAutoGeneratedNullable { get; private set; }
}
dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreVectorAttribute.cs
-68
View File
@@ -1,68 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Defines an attribute to mark a property on a record class as a vector.
/// </summary>
/// <remarks>
/// The characteristics defined here influence how the property is treated by the vector store.
/// </remarks>
[AttributeUsage(AttributeTargets.Property, AllowMultiple = false)]
public sealed class VectorStoreVectorAttribute : Attribute
{
/// <summary>
/// Initializes a new instance of the <see cref="VectorStoreVectorAttribute"/> class.
/// </summary>
/// <param name="Dimensions">The number of dimensions that the vector has.</param>
public VectorStoreVectorAttribute(int Dimensions)
{
if (Dimensions <= 0)
{
throw new ArgumentOutOfRangeException(nameof(Dimensions), "Dimensions must be greater than zero.");
}
this.Dimensions = Dimensions;
}
/// <summary>
/// Gets the number of dimensions that the vector has.
/// </summary>
/// <remarks>
/// This property is required when creating collections, but can be omitted if not using that functionality.
/// If not provided when trying to create a collection, create will fail.
/// </remarks>
public int Dimensions { get; private set; }
/// <summary>
/// Gets or sets the kind of index to use.
/// </summary>
/// <value>
/// The default value varies by database type. See the documentation of your chosen database connector for more information.
/// </value>
/// <seealso cref="IndexKind"/>
#pragma warning disable CA1019 // Define accessors for attribute arguments: The constructor overload that contains this property is obsolete.
public string? IndexKind { get; init; }
#pragma warning restore CA1019
/// <summary>
/// Gets or sets the distance function to use when comparing vectors.
/// </summary>
/// <value>
/// The default value varies by database type. See the documentation of your chosen database connector for more information.
/// </value>
/// <seealso cref="DistanceFunction"/>
#pragma warning disable CA1019 // Define accessors for attribute arguments: The constructor overload that contains this property is obsolete.
public string? DistanceFunction { get; init; }
#pragma warning restore CA1019
/// <summary>
/// Gets or sets an optional name to use for the property in storage, if different from the property name.
/// </summary>
/// <remarks>
/// For example, the property name might be "MyProperty" and the storage name might be "my_property".
/// </remarks>
public string? StorageName { get; init; }
}
dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/DistanceFunction.cs
-80
View File
@@ -1,80 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Defines a list of well-known distance functions that can be used to compare vectors.
/// </summary>
/// <remarks>
/// Not all Vector Store connectors support all distance functions, and some connectors might
/// support additional distance functions that aren't defined here.
/// For more information on what's supported, see the documentation for each connector.
/// </remarks>
public static class DistanceFunction
{
/// <summary>
/// Specifies the function that measures the cosine (angular) similarity between two vectors.
/// </summary>
/// <remarks>
/// Cosine similarity measures only the angle between the two vectors, without taking into account the length of the vectors.
/// ConsineSimilarity = 1 - CosineDistance.
/// -1 means vectors are opposite.
/// 0 means vectors are orthogonal.
/// 1 means vectors are identical.
/// </remarks>
public const string CosineSimilarity = nameof(CosineSimilarity);
/// <summary>
/// Specifies the function that measures the cosine (angular) distance between two vectors.
/// </summary>
/// <remarks>
/// CosineDistance = 1 - CosineSimilarity.
/// 2 means vectors are opposite.
/// 1 means vectors are orthogonal.
/// 0 means vectors are identical.
/// </remarks>
public const string CosineDistance = nameof(CosineDistance);
/// <summary>
/// Specifies the dot product similarity function, which measures both the length and angle between two vectors.
/// </summary>
/// <remarks>
/// The higher the value, the more similar the vectors.
/// </remarks>
public const string DotProductSimilarity = nameof(DotProductSimilarity);
/// <summary>
/// Specifies the negative dot product similarity function, which measures both the length and angle between two vectors.
/// </summary>
/// <remarks>
/// The value of NegativeDotProduct = -1 * DotProductSimilarity.
/// The higher the value, the greater the distance between the vectors and the less similar the vectors.
/// </remarks>
public const string NegativeDotProductSimilarity = nameof(NegativeDotProductSimilarity);
/// <summary>
/// Specifies the function that measures the Euclidean distance between two vectors.
/// </summary>
/// <remarks>
/// Also known as l2-norm.
/// </remarks>
public const string EuclideanDistance = nameof(EuclideanDistance);
/// <summary>
/// Specifies the function that measures the Euclidean squared distance between two vectors.
/// </summary>
/// <remarks>
/// Also known as l2-squared.
/// </remarks>
public const string EuclideanSquaredDistance = nameof(EuclideanSquaredDistance);
/// <summary>
/// Specifies the function that measures the number of differences between vectors at each dimension.
/// </summary>
public const string HammingDistance = nameof(HammingDistance);
/// <summary>
/// Specifies the function that measures the Manhattan distance between two vectors.
/// </summary>
public const string ManhattanDistance = nameof(ManhattanDistance);
}
dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/IndexKind.cs
-60
View File
@@ -1,60 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Defines a list of well-known index types that can be used to index vectors.
/// </summary>
/// <remarks>
/// Not all Vector Store connectors support all index types, and some connectors might
/// support additional index types that aren't defined here. For more information on what's
/// supported, see the documentation for each connector.
/// </remarks>
public static class IndexKind
{
/// <summary>
/// Specifies the Hierarchical Navigable Small World, which performs an approximate nearest neighbor (ANN) search.
/// </summary>
/// <remarks>
/// This search has lower accuracy than exhaustive k nearest neighbor, but is faster and more efficient.
/// </remarks>
public const string Hnsw = nameof(Hnsw);
/// <summary>
/// Specifies the brute force search to find the nearest neighbors.
/// </summary>
/// <remarks>
/// This search calculates the distances between all pairs of data points, so it has a linear time complexity that grows directly proportional to the number of points.
/// It's also referred to as "exhaustive k nearest neighbor" in some databases.
/// This search has high recall accuracy, but is slower and more expensive than HNSW.
/// It works better with smaller datasets.
/// </remarks>
public const string Flat = nameof(Flat);
/// <summary>
/// Specifies an Inverted File with Flat Compression.
/// </summary>
/// <remarks>
/// This search is designed to enhance search efficiency by narrowing the search area through the use of neighbor partitions or clusters.
/// Also referred to as approximate nearest neighbor (ANN) search.
/// </remarks>
public const string IvfFlat = nameof(IvfFlat);
/// <summary>
/// Specifies the Disk-based Approximate Nearest Neighbor algorithm, which is designed for efficiently searching for approximate nearest neighbors (ANN) in high-dimensional spaces.
/// </summary>
/// <remarks>
/// The primary focus of DiskANN is to handle large-scale datasets that can't fit entirely into memory, leveraging disk storage to store the data while maintaining fast search times.
/// </remarks>
public const string DiskAnn = nameof(DiskAnn);
/// <summary>
/// Specifies an index that compresses vectors using DiskANN-based quantization methods for better efficiency in the kNN search.
/// </summary>
public const string QuantizedFlat = nameof(QuantizedFlat);
/// <summary>
/// Specifies a dynamic index that switches automatically from <see cref="Flat"/> to <see cref="Hnsw"/> indexes.
/// </summary>
public const string Dynamic = nameof(Dynamic);
}
dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreCollectionDefinition.cs
-33
View File
@@ -1,33 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using Microsoft.Extensions.AI;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Describes the properties of a record in a vector store collection.
/// </summary>
/// <remarks>
/// Each property contains additional information about how the property will be treated by the vector store.
/// </remarks>
public sealed class VectorStoreCollectionDefinition
{
private IList<VectorStoreProperty>? _properties;
/// <summary>
/// Gets or sets the list of properties that are stored in the record.
/// </summary>
[AllowNull]
public IList<VectorStoreProperty> Properties
{
get => this._properties ??= [];
set => this._properties = value;
}
/// <summary>
/// Gets or sets the default embedding generator for vector properties in this collection.
/// </summary>
public IEmbeddingGenerator? EmbeddingGenerator { get; set; }
}
dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreDataProperty.cs
-40
View File
@@ -1,40 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Defines a data property on a vector store record.
/// </summary>
/// <remarks>
/// The characteristics defined here influence how the property is treated by the vector store.
/// </remarks>
public sealed class VectorStoreDataProperty : VectorStoreProperty
{
/// <summary>
/// Initializes a new instance of the <see cref="VectorStoreDataProperty"/> class.
/// </summary>
/// <param name="name">The name of the property on the data model. If the record is mapped to a .NET type, this corresponds to the .NET property name on that type.</param>
/// <param name="type">The type of the property. Required when using a record type of <c>Dictionary&lt;string, object?&gt;</c> (dynamic mapping), but can be omitted when mapping any other .NET type.</param>
public VectorStoreDataProperty(string name, Type? type = null)
: base(name, type)
{
}
/// <summary>
/// Gets or sets a value indicating whether this data property is indexed.
/// </summary>
/// <value>
/// The default is <see langword="false" />.
/// </value>
public bool IsIndexed { get; set; }
/// <summary>
/// Gets or sets a value indicating whether this data property is indexed for full-text search.
/// </summary>
/// <value>
/// The default is <see langword="false" />.
/// </value>
public bool IsFullTextIndexed { get; set; }
}
dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreKeyProperty.cs
-32
View File
@@ -1,32 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Defines a key property on a vector store record.
/// </summary>
/// <remarks>
/// The characteristics defined here influence how the property is treated by the vector store.
/// </remarks>
public sealed class VectorStoreKeyProperty : VectorStoreProperty
{
/// <summary>
/// Initializes a new instance of the <see cref="VectorStoreKeyProperty"/> class.
/// </summary>
/// <param name="name">The name of the property on the data model. If the record is mapped to a .NET type, this corresponds to the .NET property name on that type.</param>
/// <param name="type">The type of the property. Required when using a record type of <c>Dictionary&lt;string, object?&gt;</c> (dynamic mapping), but can be omitted when mapping any other .NET type.</param>
public VectorStoreKeyProperty(string name, Type? type = null)
: base(name, type)
{
}
/// <summary>
/// Gets or sets whether this key property's value is auto-generated or not.
/// </summary>
/// <remarks>
/// The availability of auto-generated properties - as well as the .NET types supported for them - varies across provider implementations.
/// </remarks>
public bool? IsAutoGenerated { get; set; }
}
dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreProperty.cs
-71
View File
@@ -1,71 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Defines a base property class for properties on a vector store record.
/// </summary>
/// <remarks>
/// The characteristics defined here influence how the property is treated by the vector store.
/// </remarks>
public abstract class VectorStoreProperty
{
/// <summary>
/// Initializes a new instance of the <see cref="VectorStoreProperty"/> class.
/// </summary>
/// <param name="name">The name of the property on the data model. If the record is mapped to a .NET type, this corresponds to the .NET property name on that type.</param>
/// <param name="type">The type of the property.</param>
private protected VectorStoreProperty(string name, Type? type)
{
if (string.IsNullOrWhiteSpace(name))
{
throw new ArgumentException("Value cannot be null or whitespace.", nameof(name));
}
this.Name = name;
this.Type = type;
}
private protected VectorStoreProperty(VectorStoreProperty source)
{
this.Name = source.Name;
this.StorageName = source.StorageName;
this.Type = source.Type;
this.ProviderAnnotations = source.ProviderAnnotations is not null
? new Dictionary<string, object?>(source.ProviderAnnotations)
: null;
}
/// <summary>
/// Gets or sets the name of the property on the data model.
/// </summary>
public string Name { get; set; }
/// <summary>
/// Gets or sets an optional name to use for the property in storage, if different from the property name.
/// </summary>
/// <remarks>
/// For example, the property name might be "MyProperty" and the storage name might be "my_property".
/// This property is only respected by implementations that don't support a well-known
/// serialization mechanism like JSON, in which case the attributes used by that serialization system will
/// be used.
/// </remarks>
public string? StorageName { get; set; }
/// <summary>
/// Gets or sets the type of the property.
/// </summary>
public Type? Type { get; set; }
/// <summary>
/// Gets or sets a dictionary of provider-specific annotations for this property.
/// </summary>
/// <remarks>
/// This allows setting database-specific configuration options that aren't universal across all vector stores.
/// Use provider-specific extension methods to set and get values in a strongly-typed manner.
/// </remarks>
public Dictionary<string, object?>? ProviderAnnotations { get; set; }
}
dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreVectorProperty.cs
-105
View File
@@ -1,105 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.VectorData.ProviderServices;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Defines a vector property on a vector store record.
/// </summary>
/// <remarks>
/// The characteristics defined here influence how the property is treated by the vector store.
/// </remarks>
public class VectorStoreVectorProperty : VectorStoreProperty
{
private int _dimensions;
/// <summary>
/// Initializes a new instance of the <see cref="VectorStoreVectorProperty"/> class.
/// </summary>
/// <param name="name">The name of the property on the data model. If the record is mapped to a .NET type, this corresponds to the .NET property name on that type.</param>
/// <param name="dimensions">The number of dimensions that the vector has.</param>
public VectorStoreVectorProperty(string name, int dimensions)
: base(name, type: null)
{
this.Dimensions = dimensions;
}
/// <summary>
/// Initializes a new instance of the <see cref="VectorStoreVectorProperty"/> class.
/// </summary>
/// <param name="name">The name of the property on the data model. If the record is mapped to a .NET type, this corresponds to the .NET property name on that type.</param>
/// <param name="type">The type of the property.</param>
/// <param name="dimensions">The number of dimensions that the vector has.</param>
public VectorStoreVectorProperty(string name, Type type, int dimensions)
: base(name, type)
{
this.Dimensions = dimensions;
}
/// <summary>
/// Gets or sets the default embedding generator to use for this property.
/// </summary>
/// <remarks>
/// If not set, embedding generation will be performed in the database, if supported by your connector.
/// Otherwise, if your database does not support embedding generation, only pregenerated embeddings can be used (for example, <c>ReadOnlyMemory&lt;float&gt;</c>).
/// </remarks>
public IEmbeddingGenerator? EmbeddingGenerator { get; set; }
/// <summary>
/// Gets or sets the number of dimensions that the vector has.
/// </summary>
/// <remarks>
/// This property is required when creating collections, but can be omitted if not using that functionality.
/// If not provided when trying to create a collection, create will fail.
/// </remarks>
public int Dimensions
{
get => this._dimensions;
set
{
if (value <= 0)
{
throw new ArgumentOutOfRangeException(nameof(value), "Dimensions must be greater than zero.");
}
this._dimensions = value;
}
}
/// <summary>
/// Gets or sets the kind of index to use.
/// </summary>
/// <value>
/// The default varies by database type. See the documentation of your chosen database connector for more information.
/// </value>
/// <seealso cref="Microsoft.Extensions.VectorData.IndexKind"/>
public string? IndexKind { get; set; }
/// <summary>
/// Gets or sets the distance function to use when comparing vectors.
/// </summary>
/// <value>
/// The default varies by database type. See the documentation of your chosen database connector for more information.
/// </value>
/// <seealso cref="Microsoft.Extensions.VectorData.DistanceFunction"/>
public string? DistanceFunction { get; set; }
/// <summary>
/// Gets or sets the desired embedding type (for example, <c>Embedding&lt;Half&gt;</c>) for cases where the default (typically <c>Embedding&lt;float&gt;</c>) isn't suitable.
/// </summary>
public Type? EmbeddingType { get; set; }
internal virtual VectorPropertyModel CreatePropertyModel()
=> new(this.Name, this.Type ?? throw new InvalidOperationException(VectorDataStrings.MissingTypeOnPropertyDefinition(this)))
{
Dimensions = this.Dimensions,
IndexKind = this.IndexKind,
DistanceFunction = this.DistanceFunction,
EmbeddingGenerator = this.EmbeddingGenerator,
EmbeddingType = this.EmbeddingType!
};
}
dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreVectorProperty{TInput}.cs
-36
View File
@@ -1,36 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.AI;
using Microsoft.Extensions.VectorData.ProviderServices;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Defines a vector property on a vector store record.
/// </summary>
/// <remarks>
/// <para>
/// The characteristics defined here influence how the property is treated by the vector store.
/// </para>
/// <para>
/// This generic version of <see cref="VectorStoreVectorProperty"/> only needs to be used when an <see cref="IEmbeddingGenerator"/> is
/// configured on the property, and a custom .NET type is used as input (any type other than <see cref="string"/> or <see cref="DataContent"/>).
/// </para>
/// </remarks>
public class VectorStoreVectorProperty<TInput> : VectorStoreVectorProperty
{
/// <inheritdoc />
public VectorStoreVectorProperty(string propertyName, int dimensions)
: base(propertyName, typeof(TInput), dimensions)
{
}
internal override VectorPropertyModel CreatePropertyModel()
=> new VectorPropertyModel<TInput>(this.Name)
{
Dimensions = this.Dimensions,
IndexKind = this.IndexKind,
DistanceFunction = this.DistanceFunction,
EmbeddingGenerator = this.EmbeddingGenerator
};
}
dotnet/src/VectorData/VectorData.Abstractions/RecordOptions/FilteredRecordRetrievalOptions.cs
-117
View File
@@ -1,117 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Linq.Expressions;
using System.Threading;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Defines options for calling <see cref="VectorStoreCollection{TKey, TRecord}.GetAsync(Expression{Func{TRecord, bool}}, int, FilteredRecordRetrievalOptions{TRecord}, CancellationToken)"/>.
/// </summary>
/// <typeparam name="TRecord">The type of the record.</typeparam>
public sealed class FilteredRecordRetrievalOptions<TRecord>
{
private int _skip = 0;
/// <summary>
/// Gets or sets the number of results to skip before returning results, that is, the index of the first result to return.
/// </summary>
/// <exception cref="ArgumentOutOfRangeException">The value is less than 0.</exception>
public int Skip
{
get => this._skip;
set
{
if (value < 0)
{
throw new ArgumentOutOfRangeException(nameof(value), "Skip must be greater than or equal to 0.");
}
this._skip = value;
}
}
/// <summary>
/// Gets or sets the data property to order by.
/// </summary>
/// <value>
/// If not provided, the order of returned results is non-deterministic.
/// </value>
public Func<OrderByDefinition, OrderByDefinition>? OrderBy { get; set; }
/// <summary>
/// Gets or sets a value indicating whether to include vectors in the retrieval result.
/// </summary>
public bool IncludeVectors { get; set; }
/// <summary>
/// Represents a builder for sorting.
/// </summary>
// This type does not derive any collection in order to avoid Intellisense suggesting LINQ methods.
public sealed class OrderByDefinition
{
private readonly List<SortInfo> _values = [];
/// <summary>
/// Gets the expressions to sort by.
/// </summary>
/// <remarks>This property is intended to be consumed by the connectors to retrieve the configuration.</remarks>
public IReadOnlyList<SortInfo> Values => this._values;
/// <summary>
/// Creates an ascending sort.
/// </summary>
public OrderByDefinition Ascending(Expression<Func<TRecord, object?>> propertySelector)
{
if (propertySelector is null)
{
throw new ArgumentNullException(nameof(propertySelector));
}
this._values.Add(new(propertySelector, true));
return this;
}
/// <summary>
/// Creates a descending sort.
/// </summary>
public OrderByDefinition Descending(Expression<Func<TRecord, object?>> propertySelector)
{
if (propertySelector is null)
{
throw new ArgumentNullException(nameof(propertySelector));
}
this._values.Add(new(propertySelector, false));
return this;
}
/// <summary>
/// Provides a way to define property ordering.
/// </summary>
/// <remarks>This class is intended to be consumed by the connectors to retrieve the configuration.</remarks>
public sealed class SortInfo
{
internal SortInfo(Expression<Func<TRecord, object?>> propertySelector, bool isAscending)
{
this.PropertySelector = propertySelector;
this.Ascending = isAscending;
}
/// <summary>
/// Gets the expression to select the property to sort by.
/// </summary>
public Expression<Func<TRecord, object?>> PropertySelector { get; }
/// <summary>
/// Gets a value that indicates whether the sort is ascending; otherwise, false.
/// </summary>
/// <value>
/// <see langword="true"/> if the sort is ascending; otherwise, <see langword="false"/>.
/// </value>
public bool Ascending { get; }
}
}
}
dotnet/src/VectorData/VectorData.Abstractions/RecordOptions/RecordRetrievalOptions.cs
-18
View File
@@ -1,18 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using System.Threading;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Defines options for calling <see cref="VectorStoreCollection{TKey, TRecord}.GetAsync(TKey, RecordRetrievalOptions?, CancellationToken)"/>
/// or <see cref="VectorStoreCollection{TKey, TRecord}.GetAsync(IEnumerable{TKey}, RecordRetrievalOptions?, CancellationToken)"/>.
/// </summary>
public class RecordRetrievalOptions
{
/// <summary>
/// Gets or sets a value indicating whether to include vectors in the retrieval result.
/// </summary>
public bool IncludeVectors { get; set; }
}
dotnet/src/VectorData/VectorData.Abstractions/Throw.cs
-14
View File
@@ -1,14 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
namespace Microsoft.Extensions.VectorData;
internal static class Throw
{
/// <summary>Throws an exception indicating that a required service is not available.</summary>
public static InvalidOperationException CreateMissingServiceException(Type serviceType, object? serviceKey) =>
new(serviceKey is null ?
$"No service of type '{serviceType}' is available." :
$"No service of type '{serviceType}' for the key '{serviceKey}' is available.");
}
dotnet/src/VectorData/VectorData.Abstractions/VectorData.Abstractions.csproj
-71
View File
@@ -1,71 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<!-- THIS PROPERTY GROUP MUST COME FIRST -->
<AssemblyName>Microsoft.Extensions.VectorData.Abstractions</AssemblyName>
<RootNamespace>Microsoft.Extensions.VectorData</RootNamespace>
<TargetFrameworks>net10.0;net8.0;netstandard2.0;net462</TargetFrameworks>
<IsAotCompatible Condition="$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))">true</IsAotCompatible>
<!-- temporarily disabled due to mysterious build issue -->
<EnablePackageValidation>false</EnablePackageValidation>
</PropertyGroup>
<Import Project="$(RepoRoot)/dotnet/nuget/nuget-package.props" />
<PropertyGroup>
<PackageVersion>10.1.0</PackageVersion>
<AssemblyVersion>10.0.0.0</AssemblyVersion>
<!-- Package validation. Baseline Version should be the latest version available on NuGet. -->
<PackageValidationBaselineVersion>10.0.1</PackageValidationBaselineVersion>
<Title>Microsoft.Extensions.VectorData.Abstractions</Title>
<PackageId>$(AssemblyName)</PackageId>
<PackageDescription>Abstractions for vector database access.
Commonly Used Types:
Microsoft.Extensions.VectorData.IVectorStore
Microsoft.Extensions.VectorData.IVectorStoreRecordCollection&lt;TKey, TRecord&gt;</PackageDescription>
<PackageIcon>neticon.png</PackageIcon>
<PackageIconUrl>neticon.png</PackageIconUrl>
<PackageReadmeFile>PACKAGE.md</PackageReadmeFile>
<Product></Product>
<PackageTags>Vector, Database, SDK</PackageTags>
<Description>$(PackageDescription)</Description>
<PackageProjectUrl>https://dot.net/</PackageProjectUrl>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
</ItemGroup>
<ItemGroup Condition=" '$(TargetFramework)' == 'netstandard2.0' Or '$(TargetFramework)' == 'net462' ">
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" />
<PackageReference Include="Microsoft.Bcl.Memory" />
<PackageReference Include="System.Text.Json" />
</ItemGroup>
<ItemGroup Condition=" '$(TargetFramework)' == 'net462' ">
<PackageReference Include="System.ValueTuple" />
</ItemGroup>
<ItemGroup>
<!-- Include neticon.png and PACKAGE.md in the project. -->
<None Include="neticon.png" Pack="true" PackagePath="." />
<None Include="PACKAGE.md" Pack="true" PackagePath="." />
</ItemGroup>
<ItemGroup Condition=" '$(TargetFramework)' == 'netstandard2.0' Or '$(TargetFramework)' == 'net462' ">
<Compile Include="$(RepoRoot)/dotnet/src/InternalUtilities/src/Diagnostics/CompilerServicesAttributes.cs" Link="Shared/Diagnostics/%(Filename)%(Extension)" />
<Compile Include="$(RepoRoot)/dotnet/src/InternalUtilities/src/Diagnostics/DynamicallyAccessedMembersAttribute.cs" Link="Shared/Diagnostics/%(Filename)%(Extension)" />
<Compile Include="$(RepoRoot)/dotnet/src/InternalUtilities/src/Diagnostics/ExperimentalAttribute.cs" Link="Shared/Diagnostics/%(Filename)%(Extension)" />
<Compile Include="$(RepoRoot)/dotnet/src/InternalUtilities/src/Diagnostics/IsExternalInit.cs" Link="Shared/Diagnostics/%(Filename)%(Extension)" />
<Compile Include="$(RepoRoot)/dotnet/src/InternalUtilities/src/Diagnostics/NullableAttributes.cs" Link="Shared/Diagnostics/%(Filename)%(Extension)" />
<Compile Include="$(RepoRoot)/dotnet/src/InternalUtilities/src/Diagnostics/RequiresDynamicCodeAttribute.cs" Link="Shared/Diagnostics/%(Filename)%(Extension)" />
<Compile Include="$(RepoRoot)/dotnet/src/InternalUtilities/src/Diagnostics/RequiresUnreferencedCodeAttribute.cs" Link="Shared/Diagnostics/%(Filename)%(Extension)" />
<Compile Include="$(RepoRoot)/dotnet/src/InternalUtilities/src/Diagnostics/UnconditionalSuppressMessageAttribute.cs" Link="Shared/Diagnostics/%(Filename)%(Extension)" />
<Compile Include="$(RepoRoot)/dotnet/src/InternalUtilities/src/Diagnostics/UnreachableException.cs" Link="Shared/Diagnostics/%(Filename)%(Extension)" />
</ItemGroup>
<ItemGroup>
<Compile Include="$(RepoRoot)/dotnet/src/InternalUtilities/src/Diagnostics/Verify.cs" Link="Shared/Diagnostics/%(Filename)%(Extension)" />
</ItemGroup>
</Project>
dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/HybridSearchOptions.cs
-77
View File
@@ -1,77 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Linq.Expressions;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Defines options for hybrid search when using a dense vector and string keywords to do the search.
/// </summary>
public class HybridSearchOptions<TRecord>
{
private int _skip = 0;
/// <summary>
/// Gets or sets a search filter to use before doing the hybrid search.
/// </summary>
public Expression<Func<TRecord, bool>>? Filter { get; set; }
/// <summary>
/// Gets or sets the target dense vector property to search on.
/// Only needs to be set when the collection has multiple vector properties.
/// </summary>
/// <remarks>
/// If this property isn't set, <see cref="IKeywordHybridSearchable{TRecord}.HybridSearchAsync{TInput}(TInput, System.Collections.Generic.ICollection{string}, int, Microsoft.Extensions.VectorData.HybridSearchOptions{TRecord}?, System.Threading.CancellationToken)"/> checks if there is a vector property to use by default, and
/// throws if either none or multiple exist.
/// </remarks>
public Expression<Func<TRecord, object?>>? VectorProperty { get; set; }
/// <summary>
/// Gets or sets the additional target property to do the text or keyword search on.
/// The property must have full text indexing enabled.
/// </summary>
/// <remarks>
/// If this property isn't set, <see cref="IKeywordHybridSearchable{TRecord}.HybridSearchAsync{TInput}(TInput, System.Collections.Generic.ICollection{string}, int, Microsoft.Extensions.VectorData.HybridSearchOptions{TRecord}?, System.Threading.CancellationToken)"/> checks if there is a text property with full text indexing enabled, and
/// throws an exception if either none or multiple exist.
/// </remarks>
public Expression<Func<TRecord, object?>>? AdditionalProperty { get; set; }
/// <summary>
/// Gets or sets the number of results to skip before returning results, that is, the index of the first result to return.
/// </summary>
/// <exception cref="ArgumentOutOfRangeException">The value is less than 0.</exception>
public int Skip
{
get => this._skip;
set
{
if (value < 0)
{
throw new ArgumentOutOfRangeException(nameof(value), "Skip must be greater than or equal to 0.");
}
this._skip = value;
}
}
/// <summary>
/// Gets or sets a value indicating whether to include vectors in the retrieval result.
/// </summary>
public bool IncludeVectors { get; set; }
/// <summary>
/// Gets or sets the score threshold to filter results.
/// </summary>
/// <remarks>
/// <para>
/// The meaning of the score is a combination of the distance function configured for <see cref="VectorProperty"/> and the text
/// relevance score for the full-text search on <see cref="AdditionalProperty"/>.
/// </para>
/// <para>
/// The range of scores also depends on the distance function; for example, cosine similarity/distance scores
/// fall within 0 to 1, while Euclidean distance is unbounded. Scores can also differ between vector databases.
/// </para>
/// </remarks>
public double? ScoreThreshold { get; set; }
}
dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/IKeywordHybridSearchable.cs
-74
View File
@@ -1,74 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections;
using System.Collections.Generic;
using System.Threading;
using Microsoft.Extensions.AI;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Contains a method for performing a hybrid search using a vector and keywords.
/// </summary>
/// <typeparam name="TRecord">The record data model to use for retrieving data from the store.</typeparam>
public interface IKeywordHybridSearchable<TRecord>
{
/// <summary>
/// Performs a hybrid search for records that match the given embedding and keywords, after applying the provided filters.
/// </summary>
/// <typeparam name="TInput">The type of the input value on which to perform the vector similarity search.</typeparam>
/// <param name="searchValue">The value on which to perform the similarity search. See the remarks section for more details.</param>
/// <param name="keywords">A collection of keywords to search the store with.</param>
/// <param name="top">The maximum number of results to return.</param>
/// <param name="options">The options that control the behavior of the search.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>The records found by the hybrid search, including their result scores.</returns>
/// <remarks>
/// The types supported for the <paramref name="searchValue"/> vary based on the provider being used and the embedding generation configured:
///
/// <list type="bullet">
/// <item>
/// A <see cref="string"/> or <see cref="DataContent"/> (for images, sound...) if an appropriate <see cref="IEmbeddingGenerator"/> has been configured that accepts that type as input.
/// For example, register an <see cref="IEmbeddingGenerator"/> that accepts <see cref="string"/> as input in your dependency injection container, and then pass in a
/// <see cref="string"/> argument to this method; the argument will be automatically passed to the <see cref="IEmbeddingGenerator"/> to generate the embedding and perform the search.
/// Some databases support generating embeddings at the database side. In this case, you can pass in a <see cref="string"/> or <see cref="DataContent"/> without configuring an
/// <see cref="IEmbeddingGenerator"/> with Microsoft.Extensions.VectorData. The provider will simply send your argument to the database as-is for embedding generation.
/// </item>
/// <item>
/// Arbitrary .NET types can also be passed in as long as an appropriate <see cref="IEmbeddingGenerator"/> has been configured; for example, you can create your own <see cref="IEmbeddingGenerator"/>
/// that accepts your own custom types as input, and uses another <see cref="IEmbeddingGenerator"/> to generate embedding from multiple properties. For .NET types beyond <see cref="string"/> and
/// <see cref="DataContent"/>, you must use the generic <see cref="VectorStoreVectorProperty{TInput}"/> in your record definition.
/// </item>
/// <item>
/// To work with embeddings directly, pass in a <see cref="ReadOnlyMemory{T}"/> or a .NET array of the appropriate type. Most providers support at least <c>ReadOnlyMemory&lt;float&gt;</c> and <c>float[]</c>,
/// but some support other types (for example, <c>ReadOnlyMemory&lt;Half&gt;</c>, <see cref="BitArray"/>). Some providers might also support their own custom types as well, for example, to represent sparse embeddings.
/// Consult your provider's documentation for supported types.
/// </item>
/// <item>
/// If you're using <see cref="IEmbeddingGenerator"/> directly in your code, that type returns an <see cref="Embedding"/> (for example, <c>Embedding{float}</c>),
/// which can also be passed in directly, as long as the provider supports the specific embedding type. However, consider registering your <see cref="IEmbeddingGenerator"/> with the provider
/// instead and pass in the input type (for example, <see cref="string"/>).
/// </item>
/// </list>
/// </remarks>
IAsyncEnumerable<VectorSearchResult<TRecord>> HybridSearchAsync<TInput>(
TInput searchValue,
ICollection<string> keywords,
int top,
HybridSearchOptions<TRecord>? options = default,
CancellationToken cancellationToken = default)
where TInput : notnull;
/// <summary>Asks the <see cref="IKeywordHybridSearchable{TRecord}"/> for an object of the specified type <paramref name="serviceType"/>.</summary>
/// <param name="serviceType">The type of object being requested.</param>
/// <param name="serviceKey">An optional key that can be used to help identify the target service.</param>
/// <returns>The found object, otherwise <see langword="null"/>.</returns>
/// <exception cref="ArgumentNullException"><paramref name="serviceType"/> is <see langword="null"/>.</exception>
/// <remarks>
/// The purpose of this method is to allow for the retrieval of strongly typed services that might be provided by the <see cref="IKeywordHybridSearchable{TRecord}"/>,
/// including itself or any services it might be wrapping. For example, to access the <see cref="VectorStoreCollectionMetadata"/> for the instance,
/// <see cref="GetService"/> can be used to request it.
/// </remarks>
object? GetService(Type serviceType, object? serviceKey = null);
}
dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/IVectorSearchable.cs
-72
View File
@@ -1,72 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections;
using System.Collections.Generic;
using System.Threading;
using Microsoft.Extensions.AI;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Defines an interface for performing vector searches on a vector store.
/// </summary>
/// <typeparam name="TRecord">The record data model to use for retrieving data from the store.</typeparam>
public interface IVectorSearchable<TRecord>
{
/// <summary>
/// Searches the vector store for records that are similar to the given value.
/// </summary>
/// <typeparam name="TInput">The type of the input value on which to perform the similarity search.</typeparam>
/// <param name="searchValue">The value on which to perform the similarity search. See the remarks section for more details.</param>
/// <param name="top">The maximum number of results to return.</param>
/// <param name="options">The options that control the behavior of the search.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>The records found by the vector search, including their result scores.</returns>
/// <remarks>
/// The types supported for the <paramref name="searchValue"/> vary based on the provider being used and the embedding generation configured:
///
/// <list type="bullet">
/// <item>
/// A <see cref="string"/> or <see cref="DataContent"/> (for images, sound...) if an appropriate <see cref="IEmbeddingGenerator"/> has been configured that accepts that type as input.
/// For example, register an <see cref="IEmbeddingGenerator"/> that accepts <see cref="string"/> as input in your dependency injection container, and then pass in a
/// <see cref="string"/> argument to this method; the argument will be automatically passed to the <see cref="IEmbeddingGenerator"/> to generate the embedding and perform the search.
/// Some databases support generating embeddings at the database side. In this case, you can pass in a <see cref="string"/> or <see cref="DataContent"/> without configuring an
/// <see cref="IEmbeddingGenerator"/> with Microsoft.Extensions.VectorData. The provider will simply send your argument to the database as-is for embedding generation.
/// </item>
/// <item>
/// Arbitrary .NET types can also be passed in as long as an appropriate <see cref="IEmbeddingGenerator"/> has been configured; for example, you can create your own <see cref="IEmbeddingGenerator"/>
/// that accepts your own custom types as input, and uses another <see cref="IEmbeddingGenerator"/> to generate embedding from multiple properties. For .NET types beyond <see cref="string"/> and
/// <see cref="DataContent"/>, you must use the generic <see cref="VectorStoreVectorProperty{TInput}"/> in your record definition.
/// </item>
/// <item>
/// To work with embeddings directly, pass in a <see cref="ReadOnlyMemory{T}"/> or a .NET array of the appropriate type. Most providers support at least <c>ReadOnlyMemory&lt;float&gt;</c> and <c>float[]</c>,
/// but some support other types (for example, <c>ReadOnlyMemory&lt;Half&gt;</c>, <see cref="BitArray"/>). Some providers might also support their own custom types as well, for example, to represent sparse embeddings.
/// Consult your provider's documentation for supported types.
/// </item>
/// <item>
/// If you're using <see cref="IEmbeddingGenerator"/> directly in your code, that type returns an <see cref="Embedding"/> (for example, <c>Embedding{float}</c>),
/// which can also be passed in directly, as long as the provider supports the specific embedding type. However, consider registering your <see cref="IEmbeddingGenerator"/> with the provider
/// instead and pass in the input type (for example, <see cref="string"/>).
/// </item>
/// </list>
/// </remarks>
IAsyncEnumerable<VectorSearchResult<TRecord>> SearchAsync<TInput>(
TInput searchValue,
int top,
VectorSearchOptions<TRecord>? options = default,
CancellationToken cancellationToken = default)
where TInput : notnull;
/// <summary>Asks the <see cref="IVectorSearchable{TRecord}"/> for an object of the specified type <paramref name="serviceType"/>.</summary>
/// <param name="serviceType">The type of object being requested.</param>
/// <param name="serviceKey">An optional key that can be used to help identify the target service.</param>
/// <returns>The found object, otherwise <see langword="null"/>.</returns>
/// <exception cref="ArgumentNullException"><paramref name="serviceType"/> is <see langword="null"/>.</exception>
/// <remarks>
/// The purpose of this method is to allow for the retrieval of strongly typed services that might be provided by the <see cref="IVectorSearchable{TRecord}"/>,
/// including itself or any services it might be wrapping. For example, to access the <see cref="VectorStoreCollectionMetadata"/> for the instance,
/// <see cref="GetService"/> can be used to request it.
/// </remarks>
object? GetService(Type serviceType, object? serviceKey = null);
}
dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/KeywordHybridSearchExtensions.cs
-31
View File
@@ -1,31 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
namespace Microsoft.Extensions.VectorData;
/// <summary>Provides a collection of static methods for extending <see cref="IKeywordHybridSearchable{TRecord}"/> instances.</summary>
public static class KeywordHybridSearchExtensions
{
/// <summary>
/// Asks the <see cref="IKeywordHybridSearchable{TRecord}"/> for an object of the specified type <paramref name="serviceType"/>
/// and throw an exception if one isn't available.
/// </summary>
/// <typeparam name="TRecord">The record data model to use for retrieving data from the store.</typeparam>
/// <param name="keywordHybridSearch">The keyword hybrid search.</param>
/// <param name="serviceType">The type of object being requested.</param>
/// <param name="serviceKey">An optional key that can be used to help identify the target service.</param>
/// <returns>The found object.</returns>
/// <exception cref="ArgumentNullException"><paramref name="keywordHybridSearch"/> is <see langword="null"/>.</exception>
/// <exception cref="ArgumentNullException"><paramref name="serviceType"/> is <see langword="null"/>.</exception>
/// <exception cref="InvalidOperationException">No service of the requested type for the specified key is available.</exception>
public static object GetRequiredService<TRecord>(this IKeywordHybridSearchable<TRecord> keywordHybridSearch, Type serviceType, object? serviceKey = null)
{
if (keywordHybridSearch is null) { throw new ArgumentNullException(nameof(keywordHybridSearch)); }
if (serviceType is null) { throw new ArgumentNullException(nameof(serviceType)); }
return
keywordHybridSearch.GetService(serviceType, serviceKey) ??
throw Throw.CreateMissingServiceException(serviceType, serviceKey);
}
}
dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/RecordSearchOptions.cs
-70
View File
@@ -1,70 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Linq.Expressions;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Defines options for vector search via <see cref="VectorStoreCollection{TKey, TRecord}.SearchAsync{TInput}"/>.
/// </summary>
public class VectorSearchOptions<TRecord>
{
private int _skip = 0;
/// <summary>
/// Gets or sets a search filter to use before doing the vector search.
/// </summary>
public Expression<Func<TRecord, bool>>? Filter { get; set; }
/// <summary>
/// Gets or sets the vector property to search on.
/// Only needs to be set when the collection has multiple vector properties.
/// </summary>
/// <remarks>
/// If this property isn't set provided, <see cref="VectorStoreCollection{TKey, TRecord}.SearchAsync{TInput}"/> checks if there is a vector property to use by default, and
/// throws an exception if either none or multiple exist.
/// </remarks>
public Expression<Func<TRecord, object?>>? VectorProperty { get; set; }
/// <summary>
/// Gets or sets the number of results to skip before returning results, that is, the index of the first result to return.
/// </summary>
/// <exception cref="ArgumentOutOfRangeException">The value is less than 0.</exception>
public int Skip
{
get => this._skip;
set
{
if (value < 0)
{
throw new ArgumentOutOfRangeException(nameof(value), "Skip must be greater than or equal to 0.");
}
this._skip = value;
}
}
/// <summary>
/// Gets or sets a value indicating whether to include vectors in the retrieval result.
/// </summary>
public bool IncludeVectors { get; set; }
/// <summary>
/// Gets or sets the score threshold to filter results.
/// </summary>
/// <remarks>
/// <para>
/// The meaning of the score depends on the distance function configured for the vector property.
/// For similarity functions (e.g. <see cref="DistanceFunction.CosineSimilarity" />, <see cref="DistanceFunction.DotProductSimilarity" />),
/// higher scores indicate more similar results, and results with scores lower than the threshold will be filtered out.
/// For distance functions (e.g. <see cref="DistanceFunction.CosineDistance" />, <see cref="DistanceFunction.EuclideanDistance" />),
/// lower scores indicate more similar results, and results with scores higher than the threshold will be filtered out.
/// </para>
/// <para>
/// The range of scores also depends on the distance function; for example, cosine similarity/distance scores
/// fall within 0 to 1, while Euclidean distance is unbounded. Scores can also differ between vector databases.
/// </para>
/// </remarks>
public double? ScoreThreshold { get; set; }
}
dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/VectorSearchExtensions.cs
-31
View File
@@ -1,31 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
namespace Microsoft.Extensions.VectorData;
/// <summary>Provides a collection of static methods for extending <see cref="IVectorSearchable{TRecord}"/> instances.</summary>
public static class VectorSearchExtensions
{
/// <summary>
/// Asks the <see cref="IVectorSearchable{TRecord}"/> for an object of the specified type <paramref name="serviceType"/>
/// and throws an exception if one isn't available.
/// </summary>
/// <typeparam name="TRecord">The record data model to use for retrieving data from the store.</typeparam>
/// <param name="vectorSearch">The vector search.</param>
/// <param name="serviceType">The type of object being requested.</param>
/// <param name="serviceKey">An optional key that can be used to help identify the target service.</param>
/// <returns>The found object.</returns>
/// <exception cref="ArgumentNullException"><paramref name="vectorSearch"/> is <see langword="null"/>.</exception>
/// <exception cref="ArgumentNullException"><paramref name="serviceType"/> is <see langword="null"/>.</exception>
/// <exception cref="InvalidOperationException">No service of the requested type for the specified key is available.</exception>
public static object GetRequiredService<TRecord>(this IVectorSearchable<TRecord> vectorSearch, Type serviceType, object? serviceKey = null)
{
if (vectorSearch is null) { throw new ArgumentNullException(nameof(vectorSearch)); }
if (serviceType is null) { throw new ArgumentNullException(nameof(serviceType)); }
return
vectorSearch.GetService(serviceType, serviceKey) ??
throw Throw.CreateMissingServiceException(serviceType, serviceKey);
}
}
dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/VectorSearchResult.cs
-31
View File
@@ -1,31 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Represents a single search result from a vector search.
/// </summary>
/// <typeparam name="TRecord">The record data model to use for retrieving data from the store.</typeparam>
public sealed class VectorSearchResult<TRecord>
{
/// <summary>
/// Initializes a new instance of the <see cref="VectorSearchResult{TRecord}"/> class.
/// </summary>
/// <param name="record">The record that was found by the search.</param>
/// <param name="score">The score of this result in relation to the search query.</param>
public VectorSearchResult(TRecord record, double? score)
{
this.Record = record;
this.Score = score;
}
/// <summary>
/// Gets the record that was found by the search.
/// </summary>
public TRecord Record { get; }
/// <summary>
/// Gets the score of this result in relation to the search query.
/// </summary>
public double? Score { get; }
}
dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStore.cs
-98
View File
@@ -1,98 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Threading;
using System.Threading.Tasks;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Represents a vector store that contains collections of records.
/// </summary>
/// <remarks>
/// <para>This type can be used with collections of any schema type, but requires you to provide schema information when getting a collection.</para>
/// <para>Unless otherwise documented, implementations of this abstract base class can be expected to be thread-safe, and can be used concurrently from multiple threads.</para>
/// </remarks>
public abstract class VectorStore : IDisposable
{
/// <summary>
/// Gets a collection from the vector store.
/// </summary>
/// <typeparam name="TKey">The data type of the record key.</typeparam>
/// <typeparam name="TRecord">The record data model to use for adding, updating, and retrieving data from the collection.</typeparam>
/// <param name="name">The name of the collection.</param>
/// <param name="definition">The schema of the record type.</param>
/// <returns>A new <see cref="VectorStoreCollection{TKey, TRecord}"/> instance for managing the records in the collection.</returns>
/// <remarks>
/// To successfully request a collection, either <typeparamref name="TRecord"/> must be annotated with attributes that define the schema of
/// the record type, or <paramref name="definition"/> must be provided.
/// </remarks>
/// <seealso cref="VectorStoreKeyAttribute"/>
/// <seealso cref="VectorStoreDataAttribute"/>
/// <seealso cref="VectorStoreVectorAttribute"/>
[RequiresDynamicCode("This API is not compatible with NativeAOT. For dynamic mapping via Dictionary<string, object?>, use GetCollectionDynamic() instead.")]
[RequiresUnreferencedCode("This API is not compatible with trimming. For dynamic mapping via Dictionary<string, object?>, use GetCollectionDynamic() instead.")]
public abstract VectorStoreCollection<TKey, TRecord> GetCollection<TKey, TRecord>(string name, VectorStoreCollectionDefinition? definition = null)
where TKey : notnull
where TRecord : class;
/// <summary>
/// Gets a collection from the vector store, using dynamic mapping; the record type is represented as a <see cref="Dictionary{TKey, TValue}"/>.
/// </summary>
/// <param name="name">The name of the collection.</param>
/// <param name="definition">The schema of the record type.</param>
/// <returns>A new <see cref="VectorStoreCollection{TKey, TRecord}"/> instance for managing the records in the collection.</returns>
public abstract VectorStoreCollection<object, Dictionary<string, object?>> GetDynamicCollection(string name, VectorStoreCollectionDefinition definition);
/// <summary>
/// Retrieves the names of all the collections in the vector store.
/// </summary>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>The list of names of all the collections in the vector store.</returns>
public abstract IAsyncEnumerable<string> ListCollectionNamesAsync(CancellationToken cancellationToken = default);
/// <summary>
/// Checks if the collection exists in the vector store.
/// </summary>
/// <param name="name">The name of the collection.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns><see langword="true"/> if the collection exists, <see langword="false"/> otherwise.</returns>
public abstract Task<bool> CollectionExistsAsync(string name, CancellationToken cancellationToken = default);
/// <summary>
/// Deletes the collection from the vector store.
/// </summary>
/// <param name="name">The name of the collection to delete.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>A <see cref="Task"/> that completes when the collection has been deleted.</returns>
public abstract Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default);
/// <summary>Asks the <see cref="VectorStore"/> for an object of the specified type <paramref name="serviceType"/>.</summary>
/// <param name="serviceType">The type of object being requested.</param>
/// <param name="serviceKey">An optional key that can be used to help identify the target service.</param>
/// <returns>The found object, otherwise <see langword="null"/>.</returns>
/// <exception cref="ArgumentNullException"><paramref name="serviceType"/> is <see langword="null"/>.</exception>
/// <remarks>
/// The purpose of this method is to allow for the retrieval of strongly typed services that might be provided by the <see cref="VectorStore"/>,
/// including itself or any services it might be wrapping. For example, to access the <see cref="VectorStoreMetadata"/> for the instance,
/// <see cref="GetService"/> can be used to request it.
/// </remarks>
public abstract object? GetService(Type serviceType, object? serviceKey = null);
/// <summary>
/// Disposes the <see cref="VectorStore"/> and releases any resources it holds.
/// </summary>
/// <param name="disposing"><see langword="true"/> if called from <see cref="Dispose()"/>; <see langword="false"/> if called from a finalizer.</param>
protected virtual void Dispose(bool disposing)
{
}
/// <inheritdoc/>
public void Dispose()
{
this.Dispose(disposing: true);
GC.SuppressFinalize(this);
}
}
dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollection.cs
-202
View File
@@ -1,202 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Linq.Expressions;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.SemanticKernel;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Represents a named collection of records in a vector store, and can be used to search and manage records, and to create or delete the collection itself.
/// </summary>
/// <typeparam name="TKey">The data type of the record key.</typeparam>
/// <typeparam name="TRecord">The record data model to use for adding, updating, and retrieving data from the store.</typeparam>
/// <remarks>
/// <para>Unless otherwise documented, implementations of this abstract base class can be expected to be thread-safe, and can be used concurrently from multiple threads.</para>
/// </remarks>
#pragma warning disable CA1711 // Identifiers should not have incorrect suffix (Collection)
public abstract class VectorStoreCollection<TKey, TRecord> : IVectorSearchable<TRecord>, IDisposable
#pragma warning restore CA1711
where TKey : notnull
where TRecord : class
{
/// <summary>
/// Gets the name of the collection.
/// </summary>
public abstract string Name { get; }
/// <summary>
/// Checks if the collection exists in the vector store.
/// </summary>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns><see langword="true"/> if the collection exists, <see langword="false"/> otherwise.</returns>
public abstract Task<bool> CollectionExistsAsync(CancellationToken cancellationToken = default);
/// <summary>
/// Creates this collection in the vector store if it doesn't already exist.
/// </summary>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>A <see cref="Task"/> that completes when the collection has been created.</returns>
public abstract Task EnsureCollectionExistsAsync(CancellationToken cancellationToken = default);
/// <summary>
/// Deletes the collection from the vector store if it exists.
/// </summary>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>A <see cref="Task"/> that completes when the collection has been deleted.</returns>
public abstract Task EnsureCollectionDeletedAsync(CancellationToken cancellationToken = default);
/// <summary>
/// Gets a record from the vector store. Does not guarantee that the collection exists.
/// Returns null if the record is not found.
/// </summary>
/// <param name="key">The unique ID associated with the record to get.</param>
/// <param name="options">Optional options for retrieving the record.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>The record if found, otherwise null.</returns>
/// <exception cref="VectorStoreException">The command fails to execute for any reason.</exception>
public abstract Task<TRecord?> GetAsync(TKey key, RecordRetrievalOptions? options = default, CancellationToken cancellationToken = default);
/// <summary>
/// Gets a batch of records from the vector store. Does not guarantee that the collection exists.
/// </summary>
/// <param name="keys">The unique IDs associated with the record to get.</param>
/// <param name="options">Optional options for retrieving the records.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>The records associated with the specified unique keys.</returns>
/// <remarks>
/// <para>
/// The exact method of retrieval is implementation-specific and can vary based on database support.
/// The default implementation of this method retrieves the records one after the other, but implementations which supporting batching can override to provide a more efficient implementation.
/// </para>
/// <para>
/// Only found records are returned, so the result set might be smaller than the requested keys.
/// </para>
/// <para>
/// This method throws for any issues other than records not being found.
/// </para>
/// </remarks>
/// <exception cref="VectorStoreException">The command fails to execute for any reason.</exception>
public virtual async IAsyncEnumerable<TRecord> GetAsync(IEnumerable<TKey> keys, RecordRetrievalOptions? options = default, [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
Verify.NotNull(keys);
foreach (var key in keys)
{
var record = await this.GetAsync(key, options, cancellationToken).ConfigureAwait(false);
if (record is not null)
{
yield return record;
}
}
}
/// <summary>
/// Deletes a record from the vector store. Does not guarantee that the collection exists.
/// </summary>
/// <param name="key">The unique ID associated with the record to remove.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>The unique identifier for the record.</returns>
/// <exception cref="VectorStoreException">The command fails to execute for any reason other than that the record does not exist.</exception>
public abstract Task DeleteAsync(TKey key, CancellationToken cancellationToken = default);
/// <summary>
/// Deletes a batch of records from the vector store. Does not guarantee that the collection exists.
/// </summary>
/// <param name="keys">The unique IDs associated with the records to remove.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>A <see cref="Task"/> that completes when the records have been deleted.</returns>
/// <remarks>
/// <para>
/// The exact method of deleting is implementation-specific and can vary based on database support.
/// The default implementation of this method deletes the records one after the other, but implementations which supporting batching can override to provide a more efficient implementation.
/// </para>
/// <para>
/// If a record isn't found, it is ignored and the batch succeeds.
/// If any record can't be deleted for any other reason, the operation throws. Some records might have already been deleted while others might not have, so the entire operation should be retried.
/// </para>
/// </remarks>
/// <exception cref="VectorStoreException">The command fails to execute for any reason other than that a record does not exist.</exception>
public virtual async Task DeleteAsync(IEnumerable<TKey> keys, CancellationToken cancellationToken = default)
{
Verify.NotNull(keys);
foreach (var key in keys)
{
await this.DeleteAsync(key, cancellationToken).ConfigureAwait(false);
}
}
/// <summary>
/// Upserts a record into the vector store. Does not guarantee that the collection exists.
/// If the record already exists, it is updated.
/// If the record does not exist, it is created.
/// </summary>
/// <param name="record">The record to upsert.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <exception cref="VectorStoreException">The command fails to execute for any reason.</exception>
public abstract Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default);
/// <summary>
/// Upserts a batch of records into the vector store. Does not guarantee that the collection exists.
/// If the record already exists, it is updated.
/// If the record does not exist, it is created.
/// </summary>
/// <param name="records">The records to upsert.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <remarks>
/// <para>
/// The exact method of upserting the batch is implementation-specific and can vary based on database support.
/// </para>
/// <para>
/// Similarly, the error behavior can vary across databases: where possible, the batch should be upserted atomically, so that any errors cause the entire batch to be rolled
/// back. Where not supported, some records might be upserted while others are not. If key properties are set by the user, then the entire upsert operation is idempotent,
/// and can simply be retried again if an error occurs. However, if store-generated keys are in use, the upsert operation is no longer idempotent; in that case, if the
/// database doesn't guarantee atomicity, retrying could cause duplicate records to be created.
/// </para>
/// <para>
/// Implementations of <see cref="VectorStoreCollection{TKey,TRecord}"/> should implement this method in a way which performs embedding generation once for the batch, rather than
/// generating an embedding for each record separately. This is why a default implementation that calls <see cref="UpsertAsync(TRecord, CancellationToken)"/> is not provided.
/// </para>
/// </remarks>
/// <exception cref="VectorStoreException">The command fails to execute for any reason.</exception>
public abstract Task UpsertAsync(IEnumerable<TRecord> records, CancellationToken cancellationToken = default);
/// <summary>
/// Gets matching records from the vector store. Does not guarantee that the collection exists.
/// </summary>
/// <param name="filter">The predicate to filter the records.</param>
/// <param name="top">The maximum number of results to return.</param>
/// <param name="options">Options for retrieving the records.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>The records that match the given predicate.</returns>
/// <exception cref="VectorStoreException">The command fails to execute for any reason.</exception>
public abstract IAsyncEnumerable<TRecord> GetAsync(Expression<Func<TRecord, bool>> filter, int top, FilteredRecordRetrievalOptions<TRecord>? options = null, CancellationToken cancellationToken = default);
/// <inheritdoc />
public abstract IAsyncEnumerable<VectorSearchResult<TRecord>> SearchAsync<TInput>(TInput searchValue, int top, VectorSearchOptions<TRecord>? options = null, CancellationToken cancellationToken = default)
where TInput : notnull;
/// <inheritdoc />
public abstract object? GetService(Type serviceType, object? serviceKey = null);
/// <summary>
/// Disposes the <see cref="VectorStoreCollection{TKey, TRecord}"/> and releases any resources it holds.
/// </summary>
/// <param name="disposing"><see langword="true"/> if called from <see cref="Dispose()"/>; <see langword="false"/> if called from a finalizer.</param>
protected virtual void Dispose(bool disposing)
{
}
/// <inheritdoc/>
public void Dispose()
{
this.Dispose(disposing: true);
GC.SuppressFinalize(this);
}
}
dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollectionMetadata.cs
-25
View File
@@ -1,25 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
namespace Microsoft.Extensions.VectorData;
/// <summary>Provides metadata about an <see cref="VectorStoreCollection{TKey, TRecord}"/>.</summary>
public class VectorStoreCollectionMetadata
{
/// <summary>Gets or sets the name of the vector store system.</summary>
/// <remarks>
/// Where possible, this value maps to the "db.system.name" attribute defined in the
/// OpenTelemetry Semantic Conventions for database calls and systems; see <see href="https://opentelemetry.io/docs/specs/semconv/database/"/>.
/// Example: redis, sqlite, mysql.
/// </remarks>
public string? VectorStoreSystemName { get; init; }
/// <summary>
/// Gets or sets the name of the vector store (database).
/// </summary>
public string? VectorStoreName { get; init; }
/// <summary>
/// Gets or sets the name of a collection (table, container) within the vector store (database).
/// </summary>
public string? CollectionName { get; init; }
}
dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollectionOptions.cs
-40
View File
@@ -1,40 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.AI;
namespace Microsoft.Extensions.VectorData;
/// <summary>Defines an abstract base class for options passed to a collection.</summary>
public abstract class VectorStoreCollectionOptions
{
/// <summary>
/// Initializes a new instance of the <see cref="VectorStoreCollectionOptions"/> class.
/// </summary>
protected VectorStoreCollectionOptions()
{
}
/// <summary>
/// Initializes a new instance of the <see cref="VectorStoreCollectionOptions"/> class.
/// </summary>
protected VectorStoreCollectionOptions(VectorStoreCollectionOptions? source)
{
this.Definition = source?.Definition;
this.EmbeddingGenerator = source?.EmbeddingGenerator;
}
/// <summary>
/// Gets or sets an optional record definition that defines the schema of the record type.
/// </summary>
/// <remarks>
/// If not provided, the schema will be inferred from the record model class using reflection.
/// In this case, the record model properties must be annotated with the appropriate attributes to indicate their usage.
/// See <see cref="VectorStoreKeyAttribute"/>, <see cref="VectorStoreDataAttribute"/>, and <see cref="VectorStoreVectorAttribute"/>.
/// </remarks>
public VectorStoreCollectionDefinition? Definition { get; set; }
/// <summary>
/// Gets or sets the default embedding generator to use when generating vectors embeddings with this collection.
/// </summary>
public IEmbeddingGenerator? EmbeddingGenerator { get; set; }
}
dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreException.cs
-58
View File
@@ -1,58 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
namespace Microsoft.Extensions.VectorData;
/// <summary>
/// Defines a base exception type for any type of failure when using vector stores.
/// </summary>
public class VectorStoreException : Exception
{
/// <summary>
/// Initializes a new instance of the <see cref="VectorStoreException"/> class.
/// </summary>
public VectorStoreException()
{
}
/// <summary>
/// Initializes a new instance of the <see cref="VectorStoreException"/> class with a specified error message.
/// </summary>
/// <param name="message">The error message that explains the reason for the exception.</param>
public VectorStoreException(string? message) : base(message)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="VectorStoreException"/> class with a specified error message and a reference to the inner exception that's the cause of this exception.
/// </summary>
/// <param name="message">The error message that explains the reason for the exception.</param>
/// <param name="innerException">The exception that's the cause of the current exception, or a null reference if no inner exception is specified.</param>
public VectorStoreException(string? message, Exception? innerException) : base(message, innerException)
{
}
/// <summary>Gets or sets the name of the vector store system.</summary>
/// <remarks>
/// Where possible, this value maps to the "db.system.name" attribute defined in the
/// OpenTelemetry Semantic Conventions for database calls and systems; see <see href="https://opentelemetry.io/docs/specs/semconv/database/"/>.
/// Example: redis, sqlite, mysql.
/// </remarks>
public string? VectorStoreSystemName { get; init; }
/// <summary>
/// Gets or sets the name of the vector store (database).
/// </summary>
public string? VectorStoreName { get; init; }
/// <summary>
/// Gets or sets the name of the vector store collection that the failing operation was performed on.
/// </summary>
public string? CollectionName { get; init; }
/// <summary>
/// Gets or sets the name of the vector store operation that failed.
/// </summary>
public string? OperationName { get; init; }
}
dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreExtensions.cs
-31
View File
@@ -1,31 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
namespace Microsoft.Extensions.VectorData;
/// <summary>Provides a collection of static methods for extending <see cref="VectorStore"/> instances.</summary>
public static class VectorStoreExtensions
{
/// <summary>
/// Asks the <see cref="VectorStore"/> for an object of the specified type <paramref name="serviceType"/>
/// and throws an exception if one isn't available.
/// </summary>
/// <typeparam name="TRecord">The record data model to use for retrieving data from the store.</typeparam>
/// <param name="vectorStore">The vector store.</param>
/// <param name="serviceType">The type of object being requested.</param>
/// <param name="serviceKey">An optional key that can be used to help identify the target service.</param>
/// <returns>The found object.</returns>
/// <exception cref="ArgumentNullException"><paramref name="vectorStore"/> is <see langword="null"/>.</exception>
/// <exception cref="ArgumentNullException"><paramref name="serviceType"/> is <see langword="null"/>.</exception>
/// <exception cref="InvalidOperationException">No service of the requested type for the specified key is available.</exception>
public static object GetRequiredService<TRecord>(this VectorStore vectorStore, Type serviceType, object? serviceKey = null)
{
if (vectorStore is null) { throw new ArgumentNullException(nameof(vectorStore)); }
if (serviceType is null) { throw new ArgumentNullException(nameof(serviceType)); }
return
vectorStore.GetService(serviceType, serviceKey) ??
throw Throw.CreateMissingServiceException(serviceType, serviceKey);
}
}
dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreMetadata.cs
-20
View File
@@ -1,20 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
namespace Microsoft.Extensions.VectorData;
/// <summary>Provides metadata about an <see cref="VectorStore"/>.</summary>
public class VectorStoreMetadata
{
/// <summary>Gets or sets the name of the vector store system.</summary>
/// <remarks>
/// Where possible, this value maps to the "db.system.name" attribute defined in the
/// OpenTelemetry Semantic Conventions for database calls and systems; see <see href="https://opentelemetry.io/docs/specs/semconv/database/"/>.
/// Example: redis, sqlite, mysql.
/// </remarks>
public string? VectorStoreSystemName { get; init; }
/// <summary>
/// Gets or sets the name of the vector store (database).
/// </summary>
public string? VectorStoreName { get; init; }
}
dotnet/src/VectorData/VectorData.Abstractions/neticon.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 6.8 KiB

dotnet/src/VectorData/Weaviate/Weaviate.csproj
+1 -4
View File
@@ -27,6 +27,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
<PackageReference Include="Microsoft.Extensions.VectorData.Abstractions" />
<PackageReference Include="System.Linq.AsyncEnumerable" />
</ItemGroup>
@@ -34,10 +35,6 @@
<PackageReference Include="System.Text.Json" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\VectorData.Abstractions\VectorData.Abstractions.csproj" />
</ItemGroup>
<ItemGroup>
<InternalsVisibleTo Include="SemanticKernel.Connectors.Weaviate.UnitTests" />
</ItemGroup>
dotnet/test/VectorData/VectorData.ConformanceTests/VectorData.ConformanceTests.csproj
+1 -4
View File
@@ -15,13 +15,10 @@
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
<PackageReference Include="Microsoft.Extensions.Configuration" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" />
<PackageReference Include="Microsoft.Extensions.VectorData.Abstractions" />
<PackageReference Include="System.Linq.AsyncEnumerable" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\src\VectorData\VectorData.Abstractions\VectorData.Abstractions.csproj" />
</ItemGroup>
<ItemGroup Condition=" '$(TargetFramework)' == 'net472' ">
<PackageReference Include="System.Memory.Data" />
</ItemGroup>
dotnet/test/VectorData/VectorData.UnitTests/CollectionModelBuilderTests.cs
-525
View File
@@ -1,525 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.VectorData;
using Microsoft.Extensions.VectorData.ProviderServices;
using Xunit;
namespace VectorData.UnitTests;
#pragma warning disable CA2000 // Dispose objects before losing scope
public class CollectionModelBuilderTests
{
[Fact]
public void Default_embedding_generator_without_record_definition()
{
using var embeddingGenerator = new FakeEmbeddingGenerator<string, Embedding<float>>();
var model = new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), definition: null, embeddingGenerator);
// The embedding's .NET type (Embedding<float>) is inferred from the embedding generator.
Assert.Same(embeddingGenerator, model.VectorProperty.EmbeddingGenerator);
Assert.Same(typeof(string), model.VectorProperty.Type);
Assert.Same(typeof(Embedding<float>), model.VectorProperty.EmbeddingType);
}
[Fact]
public void Default_embedding_generator_with_clr_type_and_record_definition()
{
using var embeddingGenerator = new FakeEmbeddingGenerator<string, Embedding<Half>>();
var recordDefinition = new VectorStoreCollectionDefinition
{
Properties =
[
new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)),
new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)),
new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(string), dimensions: 3)
{
// The following configures the property to be Embedding<Half> (non-default embedding type for this connector)
EmbeddingType = typeof(Embedding<Half>)
}
]
};
var model = new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), recordDefinition, embeddingGenerator);
// The embedding's .NET type (Embedding<float>) is inferred from the embedding generator.
Assert.Same(embeddingGenerator, model.VectorProperty.EmbeddingGenerator);
Assert.Same(typeof(string), model.VectorProperty.Type);
Assert.Same(typeof(Embedding<Half>), model.VectorProperty.EmbeddingType);
}
[Fact]
public void Default_embedding_generator_with_dynamic()
{
using var embeddingGenerator = new FakeEmbeddingGenerator<string, Embedding<float>>();
var recordDefinition = new VectorStoreCollectionDefinition
{
Properties =
[
new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)),
new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)),
new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(string), dimensions: 3)
]
};
var model = new CustomModelBuilder().BuildDynamic(recordDefinition, embeddingGenerator);
// The embedding's .NET type (Embedding<float>) is inferred from the embedding generator.
Assert.Same(embeddingGenerator, model.VectorProperty.EmbeddingGenerator);
Assert.Same(typeof(string), model.VectorProperty.Type);
Assert.Same(typeof(Embedding<float>), model.VectorProperty.EmbeddingType);
}
[Fact]
public void Default_embedding_generator_with_dynamic_and_non_default_EmbeddingType()
{
using var embeddingGenerator = new FakeEmbeddingGenerator<string, Embedding<Half>>();
var recordDefinition = new VectorStoreCollectionDefinition
{
Properties =
[
new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)),
new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)),
new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(string), dimensions: 3)
{
EmbeddingType = typeof(Embedding<Half>)
}
]
};
var model = new CustomModelBuilder().BuildDynamic(recordDefinition, embeddingGenerator);
Assert.Same(embeddingGenerator, model.VectorProperty.EmbeddingGenerator);
Assert.Same(typeof(string), model.VectorProperty.Type);
Assert.Same(typeof(Embedding<Half>), model.VectorProperty.EmbeddingType);
}
[Fact]
public void Property_embedding_generator_takes_precedence_over_default_generator()
{
using var propertyEmbeddingGenerator = new FakeEmbeddingGenerator<string, Embedding<float>>();
using var defaultEmbeddingGenerator = new FakeEmbeddingGenerator<string, Embedding<float>>();
var recordDefinition = new VectorStoreCollectionDefinition
{
Properties =
[
new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)),
new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)),
new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(string), dimensions: 3)
{
EmbeddingGenerator = propertyEmbeddingGenerator
}
]
};
var model = new CustomModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator);
Assert.Same(propertyEmbeddingGenerator, model.VectorProperty.EmbeddingGenerator);
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public void Embedding_property_type_with_default_embedding_generator(bool dynamic)
{
using var embeddingGenerator = new FakeEmbeddingGenerator<string, Embedding<float>>();
var model = dynamic
? new CustomModelBuilder().BuildDynamic(
new VectorStoreCollectionDefinition
{
Properties =
[
new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)),
new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)),
new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(ReadOnlyMemory<float>), dimensions: 3)
]
},
embeddingGenerator)
: new CustomModelBuilder().Build(typeof(RecordWithEmbeddingVectorProperty), typeof(int), definition: null, embeddingGenerator);
var vectorProperty = model.VectorProperty;
Assert.Same(embeddingGenerator, vectorProperty.EmbeddingGenerator);
Assert.Same(typeof(ReadOnlyMemory<float>), vectorProperty.Type);
}
[Fact]
public void Embedding_property_type_with_property_embedding_generator()
{
using var embeddingGenerator = new FakeEmbeddingGenerator<int, Embedding<float>>();
var model = new CustomModelBuilder().Build(
typeof(RecordWithEmbeddingVectorProperty),
typeof(int),
new VectorStoreCollectionDefinition
{
Properties =
[
new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)),
new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)),
new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(ReadOnlyMemory<float>), dimensions: 3)
{
EmbeddingGenerator = embeddingGenerator
}
]
},
embeddingGenerator);
var vectorProperty = model.VectorProperty;
Assert.Same(embeddingGenerator, vectorProperty.EmbeddingGenerator);
Assert.Same(typeof(ReadOnlyMemory<float>), vectorProperty.EmbeddingType);
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public void Custom_input_type(bool dynamic)
{
using var embeddingGenerator = new FakeEmbeddingGenerator<Customer, Embedding<float>>();
// TODO: Allow custom input type without a record definition (i.e. generic attribute)
var recordDefinition = new VectorStoreCollectionDefinition
{
Properties =
[
new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)),
new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)),
new VectorStoreVectorProperty<Customer>(nameof(RecordWithEmbeddingVectorProperty.Embedding), dimensions: 3)
]
};
var model = dynamic
? new CustomModelBuilder().BuildDynamic(recordDefinition, embeddingGenerator)
: new CustomModelBuilder().Build(typeof(RecordWithCustomerVectorProperty), typeof(int), recordDefinition, embeddingGenerator);
var vectorProperty = model.VectorProperty;
Assert.Same(embeddingGenerator, vectorProperty.EmbeddingGenerator);
Assert.Same(typeof(Customer), vectorProperty.Type);
Assert.Same(typeof(Embedding<float>), vectorProperty.EmbeddingType);
}
[Fact]
public void Incompatible_embedding_on_embedding_generator_throws()
{
// Embedding<long> is not a supported embedding type by the connector
using var embeddingGenerator = new FakeEmbeddingGenerator<string, Embedding<long>>();
var exception = Assert.Throws<InvalidOperationException>(() =>
new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), definition: null, embeddingGenerator));
Assert.Equal($"Embedding generator 'FakeEmbeddingGenerator<string, Embedding<long>>' on vector property '{nameof(RecordWithStringVectorProperty.Embedding)}' cannot convert the input type 'string' to a supported vector type (one of: ReadOnlyMemory<float>, Embedding<float>, float[], ReadOnlyMemory<Half>, Embedding<Half>, Half[]).", exception.Message);
}
[Fact]
public void Incompatible_input_on_embedding_generator_throws()
{
// int is not a supported input type for the embedding generator
using var embeddingGenerator = new FakeEmbeddingGenerator<int, Embedding<float>>();
var exception = Assert.Throws<InvalidOperationException>(() =>
new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), definition: null, embeddingGenerator));
Assert.Equal($"Embedding generator 'FakeEmbeddingGenerator<int, Embedding<float>>' on vector property '{nameof(RecordWithStringVectorProperty.Embedding)}' cannot convert the input type 'string' to a supported vector type (one of: ReadOnlyMemory<float>, Embedding<float>, float[], ReadOnlyMemory<Half>, Embedding<Half>, Half[]).", exception.Message);
}
[Fact]
public void Non_embedding_vector_property_without_embedding_generator_throws()
{
var exception = Assert.Throws<InvalidOperationException>(() =>
new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), definition: null, defaultEmbeddingGenerator: null));
Assert.Equal($"Vector property '{nameof(RecordWithStringVectorProperty.Embedding)}' has type 'string' which isn't supported by your provider, and no embedding generator is configured. Configure a generator that supports converting 'string' to vector type supported by your provider.", exception.Message);
}
[Fact]
public void EmbeddingType_not_supported_by_provider()
{
using var embeddingGenerator = new FakeEmbeddingGenerator<string, Embedding<Half>>();
var recordDefinition = new VectorStoreCollectionDefinition
{
Properties =
[
new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)),
new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)),
new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(string), dimensions: 3)
{
EmbeddingType = typeof(Embedding<byte>) // The provider supports float/Half only, not byte
}
]
};
var exception = Assert.Throws<InvalidOperationException>(() =>
new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), recordDefinition, embeddingGenerator));
Assert.Equal("Vector property 'Embedding' has embedding type 'Embedding<Byte>' configured, but that type isn't supported by your provider. Supported types are ReadOnlyMemory<float>, Embedding<float>, float[], ReadOnlyMemory<Half>, Embedding<Half>, Half[].", exception.Message);
}
[Fact]
public void EmbeddingType_not_supported_by_generator()
{
using var embeddingGenerator = new FakeEmbeddingGenerator<string, Embedding<float>>();
var recordDefinition = new VectorStoreCollectionDefinition
{
Properties =
[
new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)),
new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)),
new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(string), dimensions: 3)
{
EmbeddingType = typeof(Embedding<Half>) // The generator (instantiated above) supports only Embedding<float>
}
]
};
var exception = Assert.Throws<InvalidOperationException>(() =>
new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), recordDefinition, embeddingGenerator));
Assert.Equal("Vector property 'Embedding' has embedding type 'Embedding<Half>' configured, but that type isn't supported by your embedding generator.", exception.Message);
}
[Fact]
public void Missing_Type_on_property_definition()
{
var recordDefinition = new VectorStoreCollectionDefinition
{
Properties =
[
new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)),
new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)),
new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(ReadOnlyMemory<float>), dimensions: 3)
]
};
// Key
recordDefinition.Properties[0].Type = null;
var exception = Assert.Throws<InvalidOperationException>(() => new CustomModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null));
Assert.Equal($"Property '{nameof(RecordWithEmbeddingVectorProperty.Id)}' has no type specified in its definition, and does not have a corresponding .NET property. Specify the type on the definition.", exception.Message);
// Data
recordDefinition.Properties[0].Type = typeof(int);
recordDefinition.Properties[1].Type = null;
exception = Assert.Throws<InvalidOperationException>(() => new CustomModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null));
Assert.Equal($"Property '{nameof(RecordWithEmbeddingVectorProperty.Name)}' has no type specified in its definition, and does not have a corresponding .NET property. Specify the type on the definition.", exception.Message);
// Vector
recordDefinition.Properties[1].Type = typeof(string);
recordDefinition.Properties[2].Type = null;
exception = Assert.Throws<InvalidOperationException>(() => new CustomModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null));
Assert.Equal($"Property '{nameof(RecordWithEmbeddingVectorProperty.Embedding)}' has no type specified in its definition, and does not have a corresponding .NET property. Specify the type on the definition.", exception.Message);
}
public class RecordWithStringVectorProperty
{
[VectorStoreKey]
public int Id { get; set; }
[VectorStoreData]
public string Name { get; set; }
[VectorStoreVector(Dimensions: 3)]
public string Embedding { get; set; }
}
public class RecordWithEmbeddingVectorProperty
{
[VectorStoreKey]
public int Id { get; set; }
[VectorStoreData]
public string Name { get; set; }
[VectorStoreVector(Dimensions: 3)]
public ReadOnlyMemory<float> Embedding { get; set; }
}
public class RecordWithCustomerVectorProperty
{
[VectorStoreKey]
public int Id { get; set; }
[VectorStoreData]
public string Name { get; set; }
[VectorStoreVector(Dimensions: 3)]
public Customer Embedding { get; set; }
}
public class Customer
{
public string FirstName { get; set; }
public string LastName { get; set; }
}
private sealed class CustomModelBuilder(CollectionModelBuildingOptions? options = null)
: CollectionModelBuilder(options ?? s_defaultOptions)
{
private static readonly CollectionModelBuildingOptions s_defaultOptions = new()
{
SupportsMultipleVectors = true,
RequiresAtLeastOneVector = false
};
protected override void ValidateKeyProperty(KeyPropertyModel keyProperty)
{
var type = keyProperty.Type;
if (type != typeof(string) && type != typeof(int))
{
throw new NotSupportedException(
$"Property '{keyProperty.ModelName}' has unsupported type '{type.Name}'. Key properties must be one of the supported types: string, int.");
}
}
protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes)
{
supportedTypes = "string, int";
if (Nullable.GetUnderlyingType(type) is Type underlyingType)
{
type = underlyingType;
}
return type == typeof(string) || type == typeof(int);
}
protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes)
=> IsVectorPropertyTypeValidCore(type, out supportedTypes);
internal static bool IsVectorPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes)
{
supportedTypes = "ReadOnlyMemory<float>, Embedding<float>, float[], ReadOnlyMemory<Half>, Embedding<Half>, Half[]";
if (Nullable.GetUnderlyingType(type) is Type underlyingType)
{
type = underlyingType;
}
return type == typeof(ReadOnlyMemory<float>)
|| type == typeof(Embedding<float>)
|| type == typeof(float[])
|| type == typeof(ReadOnlyMemory<Half>)
|| type == typeof(Embedding<Half>)
|| type == typeof(Half[]);
}
protected override IReadOnlyList<EmbeddingGenerationDispatcher> EmbeddingGenerationDispatchers { get; } =
[
EmbeddingGenerationDispatcher.Create<Embedding<float>>(),
EmbeddingGenerationDispatcher.Create<Embedding<Half>>()
];
}
[Fact]
public void IsAutoGenerated_attribute_true_overrides_default()
{
// Guid key with explicit IsAutoGenerated = true; the attribute should override SupportsKeyAutoGeneration.
var model = new GuidKeyModelBuilder().Build(
typeof(RecordWithGuidKeyAutoGeneratedTrue), typeof(Guid), definition: null, defaultEmbeddingGenerator: null);
Assert.True(model.KeyProperty.IsAutoGenerated);
}
[Fact]
public void IsAutoGenerated_attribute_false_overrides_default()
{
// Guid key with explicit IsAutoGenerated = false; the attribute should override SupportsKeyAutoGeneration,
// which would otherwise return true for Guid.
var model = new GuidKeyModelBuilder().Build(
typeof(RecordWithGuidKeyAutoGeneratedFalse), typeof(Guid), definition: null, defaultEmbeddingGenerator: null);
Assert.False(model.KeyProperty.IsAutoGenerated);
}
[Fact]
public void IsAutoGenerated_omitted_falls_back_to_SupportsKeyAutoGeneration_true()
{
// Guid key with no IsAutoGenerated attribute set; should fall back to SupportsKeyAutoGeneration, which returns true for Guid.
var model = new GuidKeyModelBuilder().Build(
typeof(RecordWithGuidKeyNoIsAutoGenerated), typeof(Guid), definition: null, defaultEmbeddingGenerator: null);
Assert.True(model.KeyProperty.IsAutoGenerated);
}
[Fact]
public void IsAutoGenerated_omitted_falls_back_to_SupportsKeyAutoGeneration_false()
{
// int key with no IsAutoGenerated attribute set; should fall back to SupportsKeyAutoGeneration, which returns false for non-Guid.
var model = new GuidKeyModelBuilder().Build(
typeof(RecordWithIntKeyNoIsAutoGenerated), typeof(int), definition: null, defaultEmbeddingGenerator: null);
Assert.False(model.KeyProperty.IsAutoGenerated);
}
public class RecordWithGuidKeyAutoGeneratedTrue
{
[VectorStoreKey(IsAutoGenerated = true)]
public Guid Id { get; set; }
}
public class RecordWithGuidKeyAutoGeneratedFalse
{
[VectorStoreKey(IsAutoGenerated = false)]
public Guid Id { get; set; }
}
public class RecordWithGuidKeyNoIsAutoGenerated
{
[VectorStoreKey]
public Guid Id { get; set; }
}
public class RecordWithIntKeyNoIsAutoGenerated
{
[VectorStoreKey]
public int Id { get; set; }
}
private sealed class GuidKeyModelBuilder()
: CollectionModelBuilder(new CollectionModelBuildingOptions
{
SupportsMultipleVectors = true,
RequiresAtLeastOneVector = false
})
{
protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes)
{
supportedTypes = null;
return true;
}
protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes)
{
supportedTypes = null;
return true;
}
}
private sealed class FakeEmbeddingGenerator<TInput, TEmbedding> : IEmbeddingGenerator<TInput, TEmbedding>
where TEmbedding : Embedding
{
public Task<GeneratedEmbeddings<TEmbedding>> GenerateAsync(
IEnumerable<TInput> values,
EmbeddingGenerationOptions? options = null,
CancellationToken cancellationToken = default)
=> throw new UnreachableException();
public object? GetService(Type serviceType, object? serviceKey = null)
=> throw new UnreachableException();
public void Dispose() { }
}
}
dotnet/test/VectorData/VectorData.UnitTests/PropertyModelTests.cs
-139
View File
@@ -1,139 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using Microsoft.Extensions.VectorData.ProviderServices;
using Xunit;
namespace VectorData.UnitTests;
public class PropertyModelTests
{
#region Value type nullability
[Fact]
public void IsNullable_NonNullableValueType_ReturnsFalse()
{
var property = new DataPropertyModel("test", typeof(int));
Assert.False(property.IsNullable);
}
[Fact]
public void IsNullable_NullableValueType_ReturnsTrue()
{
var property = new DataPropertyModel("test", typeof(int?));
Assert.True(property.IsNullable);
}
[Fact]
public void IsNullable_Guid_ReturnsFalse()
{
var property = new DataPropertyModel("test", typeof(Guid));
Assert.False(property.IsNullable);
}
[Fact]
public void IsNullable_NullableGuid_ReturnsTrue()
{
var property = new DataPropertyModel("test", typeof(Guid?));
Assert.True(property.IsNullable);
}
[Fact]
public void IsNullable_ReadOnlyMemoryFloat_ReturnsFalse()
{
var property = new DataPropertyModel("test", typeof(ReadOnlyMemory<float>));
Assert.False(property.IsNullable);
}
[Fact]
public void IsNullable_NullableReadOnlyMemoryFloat_ReturnsTrue()
{
var property = new DataPropertyModel("test", typeof(ReadOnlyMemory<float>?));
Assert.True(property.IsNullable);
}
#endregion
#region Reference type nullability (dynamic mapping, no PropertyInfo)
[Fact]
public void IsNullable_String_WithoutPropertyInfo_ReturnsTrue()
{
// Without PropertyInfo (dynamic mapping), reference types are assumed nullable
var property = new DataPropertyModel("test", typeof(string));
Assert.True(property.IsNullable);
}
[Fact]
public void IsNullable_ByteArray_WithoutPropertyInfo_ReturnsTrue()
{
var property = new DataPropertyModel("test", typeof(byte[]));
Assert.True(property.IsNullable);
}
#endregion
#if NET
#region NRT detection via NullabilityInfoContext (POCO mapping with PropertyInfo)
[Fact]
public void IsNullable_NonNullableString_WithPropertyInfo_ReturnsFalse()
{
var propertyInfo = typeof(NrtTestRecord).GetProperty(nameof(NrtTestRecord.NonNullableString))!;
var property = new DataPropertyModel("test", typeof(string)) { PropertyInfo = propertyInfo };
Assert.False(property.IsNullable);
}
[Fact]
public void IsNullable_NullableString_WithPropertyInfo_ReturnsTrue()
{
var propertyInfo = typeof(NrtTestRecord).GetProperty(nameof(NrtTestRecord.NullableString))!;
var property = new DataPropertyModel("test", typeof(string)) { PropertyInfo = propertyInfo };
Assert.True(property.IsNullable);
}
[Fact]
public void IsNullable_NonNullableByteArray_WithPropertyInfo_ReturnsFalse()
{
var propertyInfo = typeof(NrtTestRecord).GetProperty(nameof(NrtTestRecord.NonNullableByteArray))!;
var property = new DataPropertyModel("test", typeof(byte[])) { PropertyInfo = propertyInfo };
Assert.False(property.IsNullable);
}
[Fact]
public void IsNullable_NullableByteArray_WithPropertyInfo_ReturnsTrue()
{
var propertyInfo = typeof(NrtTestRecord).GetProperty(nameof(NrtTestRecord.NullableByteArray))!;
var property = new DataPropertyModel("test", typeof(byte[])) { PropertyInfo = propertyInfo };
Assert.True(property.IsNullable);
}
[Fact]
public void IsNullable_ValueType_WithPropertyInfo_StillUsesTypeCheck()
{
var propertyInfo = typeof(NrtTestRecord).GetProperty(nameof(NrtTestRecord.NonNullableInt))!;
var property = new DataPropertyModel("test", typeof(int)) { PropertyInfo = propertyInfo };
Assert.False(property.IsNullable);
var nullablePropertyInfo = typeof(NrtTestRecord).GetProperty(nameof(NrtTestRecord.NullableInt))!;
var nullableProperty = new DataPropertyModel("test", typeof(int?)) { PropertyInfo = nullablePropertyInfo };
Assert.True(nullableProperty.IsNullable);
}
#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor
#pragma warning disable CA1812 // Class is used via reflection
private sealed class NrtTestRecord
{
public string NonNullableString { get; set; }
public string? NullableString { get; set; }
public byte[] NonNullableByteArray { get; set; }
public byte[]? NullableByteArray { get; set; }
public int NonNullableInt { get; set; }
public int? NullableInt { get; set; }
}
#pragma warning restore CA1812
#pragma warning restore CS8618
#endregion
#endif
}
dotnet/test/VectorData/VectorData.UnitTests/VectorData.UnitTests.csproj
-43
View File
@@ -1,43 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<AssemblyName>VectorData.UnitTests</AssemblyName>
<RootNamespace>VectorData.UnitTests</RootNamespace>
<TargetFramework>net10.0</TargetFramework>
<IsTestProject>true</IsTestProject>
<Nullable>enable</Nullable>
<ImplicitUsings>disable</ImplicitUsings>
<IsPackable>false</IsPackable>
<NoWarn>$(NoWarn);VSTHRD111,CA2007,CS8618</NoWarn>
<NoWarn>$(NoWarn);MEVD9001</NoWarn> <!-- Experimental MEVD connector-facing APIs -->
<NoWarn>$(NoWarn);CA1515</NoWarn> <!-- Because an application's API isn't typically referenced from outside the assembly, types can be made internal -->
<NoWarn>$(NoWarn);CA1707</NoWarn> <!-- Remove the underscores from member name -->
<NoWarn>$(NoWarn);CA1716</NoWarn> <!-- Rename virtual/interface member so that it no longer conflicts with the reserved language keyword -->
<NoWarn>$(NoWarn);CA1720</NoWarn> <!-- Identifier contains type name -->
<NoWarn>$(NoWarn);CA1721</NoWarn> <!-- The property name X is confusing given the existence of method Y. Rename or remove one of these members. -->
<NoWarn>$(NoWarn);CA1861</NoWarn> <!-- Prefer 'static readonly' fields over constant array arguments if the called method is called repeatedly and is not mutating the passed array -->
<NoWarn>$(NoWarn);CA1863</NoWarn> <!-- Cache a 'CompositeFormat' for repeated use in this formatting operation -->
<NoWarn>$(NoWarn);CA2007;VSTHRD111</NoWarn> <!-- Consider calling ConfigureAwait on the awaited task -->
<NoWarn>$(NoWarn);CS1591</NoWarn> <!-- Missing XML comment for publicly visible type or member -->
<NoWarn>$(NoWarn);IDE1006</NoWarn> <!-- Naming rule violation: Missing suffix: 'Async' -->
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" />
<PackageReference Include="Moq" />
<PackageReference Include="xunit" />
<PackageReference Include="xunit.runner.visualstudio">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="coverlet.collector">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\src\VectorData\VectorData.Abstractions\VectorData.Abstractions.csproj" />
</ItemGroup>
</Project>