mirror of
https://github.com/microsoft/semantic-kernel.git
synced 2026-10-12 02:19:06 +00:00
.Net: [MEVD] Map DateTime to timestamptz on PostgreSQL (#13514)
DateTime now maps to the PG timestamptz type by default, which represents a UTC timestamp; this aligns with Npgsql and EF Core. However, now that we have property annotations, I've also added a new `WithStoreType()` that allows the user to specify a database type explicitly. For now, this is only supported for setting `timestamp with time zone` (short: `timestamp`), which is the other PG date/time type, for unspecified timestamps (timezone is unknown or implicitly assumed). This allows users who do want to store non-UTC timestamp to do so. Closes #10641
This commit is contained in:
@@ -375,7 +375,7 @@ public class PostgresCollection<TKey, TRecord> : VectorStoreCollection<TKey, TRe
|
||||
|
||||
using var connection = await this._dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
using var command = connection.CreateCommand();
|
||||
PostgresSqlBuilder.BuildDeleteBatchCommand(command, this._schema, this.Name, this._model.KeyProperty.StorageName, listOfKeys);
|
||||
PostgresSqlBuilder.BuildDeleteBatchCommand(command, this._schema, this.Name, this._model.KeyProperty, listOfKeys);
|
||||
|
||||
await this.RunOperationAsync("DeleteBatch", () => command.ExecuteNonQueryAsync(cancellationToken)).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Linq.Expressions;
|
||||
using System.Text;
|
||||
@@ -29,18 +30,29 @@ internal sealed class PostgresFilterTranslator : SqlFilterTranslator
|
||||
{
|
||||
switch (value)
|
||||
{
|
||||
// TODO: This aligns with our mapping of DateTime to PG's timestamp (as opposed to timestamptz) - we probably want to
|
||||
// change that to timestamptz (aligning with Npgsql and EF). See #10641.
|
||||
case DateTime dateTime:
|
||||
this._sql.Append('\'').Append(dateTime.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFF", CultureInfo.InvariantCulture)).Append('\'');
|
||||
return;
|
||||
switch (dateTime.Kind)
|
||||
{
|
||||
case DateTimeKind.Utc:
|
||||
this._sql.Append('\'').Append(dateTime.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFFZ", CultureInfo.InvariantCulture)).Append('\'');
|
||||
return;
|
||||
|
||||
case DateTimeKind.Unspecified:
|
||||
case DateTimeKind.Local:
|
||||
this._sql.Append('\'').Append(dateTime.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFF", CultureInfo.InvariantCulture)).Append('\'');
|
||||
return;
|
||||
|
||||
default:
|
||||
throw new UnreachableException();
|
||||
}
|
||||
|
||||
case DateTimeOffset dateTimeOffset:
|
||||
if (dateTimeOffset.Offset != TimeSpan.Zero)
|
||||
{
|
||||
throw new NotSupportedException("DateTimeOffset with non-zero offset is not supported with PostgreSQL");
|
||||
throw new NotSupportedException("DateTimeOffset with non-zero offset is not supported with PostgreSQL. Use DateTimeOffset.UtcNow or convert to UTC.");
|
||||
}
|
||||
|
||||
this._sql.Append('\'').Append(dateTimeOffset.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFF", CultureInfo.InvariantCulture)).Append("Z'");
|
||||
this._sql.Append('\'').Append(dateTimeOffset.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFFZ", CultureInfo.InvariantCulture)).Append('\'');
|
||||
return;
|
||||
|
||||
// Array constants (ARRAY[1, 2, 3])
|
||||
|
||||
@@ -5,6 +5,7 @@ using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.Extensions.VectorData.ProviderServices;
|
||||
using Pgvector;
|
||||
|
||||
@@ -92,6 +93,22 @@ internal class PostgresModelBuilder() : CollectionModelBuilder(PostgresModelBuil
|
||||
type == typeof(SparseVector);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void ValidateProperty(PropertyModel propertyModel, VectorStoreCollectionDefinition? definition)
|
||||
{
|
||||
base.ValidateProperty(propertyModel, definition);
|
||||
|
||||
if (propertyModel.IsTimestampWithoutTimezone())
|
||||
{
|
||||
var type = Nullable.GetUnderlyingType(propertyModel.Type) ?? propertyModel.Type;
|
||||
if (type != typeof(DateTime))
|
||||
{
|
||||
throw new NotSupportedException(
|
||||
$"Property '{propertyModel.ModelName}' has store type 'timestamp' configured, but this is only supported for DateTime properties. The property type is '{propertyModel.Type.Name}'.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override Type? ResolveEmbeddingType(
|
||||
VectorPropertyModel vectorProperty,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.Extensions.VectorData.ProviderServices;
|
||||
|
||||
@@ -11,6 +12,9 @@ namespace Microsoft.SemanticKernel.Connectors.PgVector;
|
||||
public static class PostgresPropertyExtensions
|
||||
{
|
||||
private const string FullTextSearchLanguageKey = "Postgres:FullTextSearchLanguage";
|
||||
private const string StoreTypeKey = "Postgres:StoreType";
|
||||
|
||||
#region Full-text search language
|
||||
|
||||
/// <summary>
|
||||
/// Sets the PostgreSQL full-text search language for a data property.
|
||||
@@ -50,4 +54,69 @@ public static class PostgresPropertyExtensions
|
||||
=> property.ProviderAnnotations?.TryGetValue(FullTextSearchLanguageKey, out var value) == true && value is string language
|
||||
? language
|
||||
: PostgresConstants.DefaultFullTextSearchLanguage;
|
||||
|
||||
#endregion Full-text search language
|
||||
|
||||
#region Store type
|
||||
|
||||
/// <summary>
|
||||
/// Sets the PostgreSQL store type for a property, overriding the default type mapping.
|
||||
/// </summary>
|
||||
/// <param name="property">The property to configure.</param>
|
||||
/// <param name="storeType">
|
||||
/// The PostgreSQL type name. Currently, only <c>"timestamp"</c> and <c>"timestamp without time zone"</c>
|
||||
/// are supported, and only on <see cref="DateTime"/> properties. This causes the property to be stored as
|
||||
/// PostgreSQL <c>timestamp</c> (without time zone) instead of the default <c>timestamptz</c>.
|
||||
/// </param>
|
||||
/// <returns>The same property instance for method chaining.</returns>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// By default, .NET <see cref="DateTime"/> properties are mapped to PostgreSQL <c>timestamptz</c> (timestamp with time zone),
|
||||
/// which requires UTC values. Use this method to map to <c>timestamp</c> (without time zone) instead, which stores
|
||||
/// local/unspecified date-time values without time zone information.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// When using <c>timestamp</c>, <see cref="DateTime"/> values with <see cref="DateTimeKind.Unspecified"/> or
|
||||
/// <see cref="DateTimeKind.Local"/> kind should be used. Values read back from the database will have
|
||||
/// <see cref="DateTimeKind.Unspecified"/>.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <exception cref="NotSupportedException">Thrown if the <paramref name="storeType"/> is not a supported value.</exception>
|
||||
public static TProperty WithStoreType<TProperty>(this TProperty property, string storeType)
|
||||
where TProperty : VectorStoreProperty
|
||||
{
|
||||
if (!IsTimestampStoreType(storeType))
|
||||
{
|
||||
throw new NotSupportedException(
|
||||
$"Store type '{storeType}' is not supported. Only 'timestamp' and 'timestamp without time zone' are supported.");
|
||||
}
|
||||
|
||||
property.ProviderAnnotations ??= [];
|
||||
property.ProviderAnnotations[StoreTypeKey] = storeType;
|
||||
return property;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the PostgreSQL store type configured for a property.
|
||||
/// </summary>
|
||||
/// <param name="property">The property to read from.</param>
|
||||
/// <returns>The configured store type, or <see langword="null"/> if not set.</returns>
|
||||
public static string? GetStoreType(this VectorStoreProperty property)
|
||||
=> property.ProviderAnnotations?.TryGetValue(StoreTypeKey, out var value) == true
|
||||
? value as string
|
||||
: null;
|
||||
|
||||
/// <summary>
|
||||
/// Gets whether the property model has been configured with a <c>timestamp</c> (without time zone) store type.
|
||||
/// </summary>
|
||||
internal static bool IsTimestampWithoutTimezone(this PropertyModel property)
|
||||
=> property.ProviderAnnotations?.TryGetValue(StoreTypeKey, out var value) == true
|
||||
&& value is string storeType
|
||||
&& IsTimestampStoreType(storeType);
|
||||
|
||||
private static bool IsTimestampStoreType(string storeType)
|
||||
=> string.Equals(storeType, "timestamp", StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(storeType, "timestamp without time zone", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
#endregion Store type
|
||||
}
|
||||
|
||||
@@ -38,8 +38,11 @@ internal static class PostgresPropertyMapping
|
||||
var value => throw new NotSupportedException($"Mapping for type '{value.GetType().Name}' to a vector is not supported.")
|
||||
};
|
||||
|
||||
public static NpgsqlDbType? GetNpgsqlDbType(Type propertyType) =>
|
||||
(Nullable.GetUnderlyingType(propertyType) ?? propertyType) switch
|
||||
/// <summary>
|
||||
/// Gets the NpgsqlDbType for a property, taking into account any store type annotation.
|
||||
/// </summary>
|
||||
internal static NpgsqlDbType? GetNpgsqlDbType(PropertyModel property)
|
||||
=> (Nullable.GetUnderlyingType(property.Type) ?? property.Type) switch
|
||||
{
|
||||
Type t when t == typeof(bool) => NpgsqlDbType.Boolean,
|
||||
Type t when t == typeof(short) => NpgsqlDbType.Smallint,
|
||||
@@ -50,19 +53,21 @@ internal static class PostgresPropertyMapping
|
||||
Type t when t == typeof(decimal) => NpgsqlDbType.Numeric,
|
||||
Type t when t == typeof(string) => NpgsqlDbType.Text,
|
||||
Type t when t == typeof(byte[]) => NpgsqlDbType.Bytea,
|
||||
Type t when t == typeof(DateTime) => NpgsqlDbType.Timestamp,
|
||||
Type t when t == typeof(DateTimeOffset) => NpgsqlDbType.TimestampTz,
|
||||
Type t when t == typeof(Guid) => NpgsqlDbType.Uuid,
|
||||
Type t when t == typeof(DateTimeOffset) => NpgsqlDbType.TimestampTz,
|
||||
|
||||
// DateTime properties map to PG's "timestamp with time zone" (UTC timestamps) by default, aligning with Npgsql/EF/etc.
|
||||
// Users can explicitly opt into "timestamp without time zone".
|
||||
Type t when t == typeof(DateTime) && property.IsTimestampWithoutTimezone() => NpgsqlDbType.Timestamp,
|
||||
Type t when t == typeof(DateTime) => NpgsqlDbType.TimestampTz,
|
||||
|
||||
_ => null
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Maps a .NET type to a PostgreSQL type name.
|
||||
/// Maps a .NET type to a PostgreSQL type name, taking into account any store type annotation on the property.
|
||||
/// </summary>
|
||||
/// <param name="propertyType">The .NET type.</param>
|
||||
/// <returns>Tuple of the the PostgreSQL type name and whether it can be NULL</returns>
|
||||
public static (string PgType, bool IsNullable) GetPostgresTypeName(Type propertyType)
|
||||
internal static (string PgType, bool IsNullable) GetPostgresTypeName(PropertyModel property)
|
||||
{
|
||||
static bool TryGetBaseType(Type type, [NotNullWhen(true)] out string? typeName)
|
||||
{
|
||||
@@ -77,7 +82,7 @@ internal static class PostgresPropertyMapping
|
||||
Type t when t == typeof(decimal) => "NUMERIC",
|
||||
Type t when t == typeof(string) => "TEXT",
|
||||
Type t when t == typeof(byte[]) => "BYTEA",
|
||||
Type t when t == typeof(DateTime) => "TIMESTAMP",
|
||||
Type t when t == typeof(DateTime) => "TIMESTAMPTZ",
|
||||
Type t when t == typeof(DateTimeOffset) => "TIMESTAMPTZ",
|
||||
Type t when t == typeof(Guid) => "UUID",
|
||||
_ => null
|
||||
@@ -86,30 +91,43 @@ internal static class PostgresPropertyMapping
|
||||
return typeName is not null;
|
||||
}
|
||||
|
||||
var propertyType = property.Type;
|
||||
|
||||
// TODO: Handle NRTs properly via NullabilityInfoContext
|
||||
|
||||
(string PgType, bool IsNullable) result;
|
||||
|
||||
if (TryGetBaseType(propertyType, out string? pgType))
|
||||
{
|
||||
return (pgType, !propertyType.IsValueType);
|
||||
result = (pgType, !propertyType.IsValueType);
|
||||
}
|
||||
|
||||
// Handle nullable types (e.g. Nullable<int>)
|
||||
if (Nullable.GetUnderlyingType(propertyType) is Type unwrappedType
|
||||
else if (Nullable.GetUnderlyingType(propertyType) is Type unwrappedType
|
||||
&& TryGetBaseType(unwrappedType, out string? underlyingPgType))
|
||||
{
|
||||
return (underlyingPgType, true);
|
||||
result = (underlyingPgType, true);
|
||||
}
|
||||
|
||||
// Handle collections
|
||||
if ((propertyType.IsArray && TryGetBaseType(propertyType.GetElementType()!, out string? elementPgType))
|
||||
else if ((propertyType.IsArray && TryGetBaseType(propertyType.GetElementType()!, out string? elementPgType))
|
||||
|| (propertyType.IsGenericType
|
||||
&& propertyType.GetGenericTypeDefinition() == typeof(List<>)
|
||||
&& TryGetBaseType(propertyType.GetGenericArguments()[0], out elementPgType)))
|
||||
{
|
||||
return (elementPgType + "[]", true);
|
||||
result = (elementPgType + "[]", true);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new NotSupportedException($"Type {propertyType.Name} is not supported by this store.");
|
||||
}
|
||||
|
||||
throw new NotSupportedException($"Type {propertyType.Name} is not supported by this store.");
|
||||
if (property.IsTimestampWithoutTimezone())
|
||||
{
|
||||
// Replace TIMESTAMPTZ with TIMESTAMP in the PG type name.
|
||||
// This handles both "TIMESTAMPTZ" and "TIMESTAMPTZ[]" cases.
|
||||
result = ("TIMESTAMP", result.IsNullable);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -64,7 +64,7 @@ WHERE table_schema = $1 AND table_type = 'BASE TABLE'
|
||||
createTableCommand.AppendIdentifier(schema).Append('.').AppendIdentifier(tableName).AppendLine(" (");
|
||||
|
||||
// Add the key column
|
||||
var keyStoreType = PostgresPropertyMapping.GetPostgresTypeName(model.KeyProperty.Type).PgType;
|
||||
var keyStoreType = PostgresPropertyMapping.GetPostgresTypeName(model.KeyProperty).PgType;
|
||||
createTableCommand.Append(" ").AppendIdentifier(keyName).Append(' ').Append(keyStoreType);
|
||||
if (model.KeyProperty.IsAutoGenerated)
|
||||
{
|
||||
@@ -90,7 +90,7 @@ WHERE table_schema = $1 AND table_type = 'BASE TABLE'
|
||||
// Add the data columns
|
||||
foreach (var dataProperty in model.DataProperties)
|
||||
{
|
||||
var dataPgTypeInfo = PostgresPropertyMapping.GetPostgresTypeName(dataProperty.Type);
|
||||
var dataPgTypeInfo = PostgresPropertyMapping.GetPostgresTypeName(dataProperty);
|
||||
createTableCommand.Append(" ").AppendIdentifier(dataProperty.StorageName).Append(' ').Append(dataPgTypeInfo.PgType);
|
||||
if (!dataPgTypeInfo.IsNullable)
|
||||
{
|
||||
@@ -224,7 +224,22 @@ WHERE table_schema = $1 AND table_type = 'BASE TABLE'
|
||||
value = PostgresPropertyMapping.MapVectorForStorageModel(value);
|
||||
}
|
||||
|
||||
batchCommand.Parameters.Add(new() { Value = value ?? DBNull.Value });
|
||||
NpgsqlDbType? npgsqlDbType = null;
|
||||
|
||||
if (property.Type == typeof(DateTime))
|
||||
{
|
||||
npgsqlDbType = property.IsTimestampWithoutTimezone()
|
||||
? NpgsqlDbType.Timestamp
|
||||
// DateTime properties map to PG's "timestamp with time zone" (timestamptz) by default.
|
||||
// Npgsql would silently send non-UTC DateTimes as 'timestamp' (without timezone), and PG would
|
||||
// implicitly cast to timestamptz using the session timezone, producing incorrect results.
|
||||
// Explicitly set NpgsqlDbType.TimestampTz to ensure Npgsql rejects non-UTC DateTimes.
|
||||
: NpgsqlDbType.TimestampTz;
|
||||
}
|
||||
|
||||
batchCommand.Parameters.Add(npgsqlDbType.HasValue
|
||||
? new() { Value = value ?? DBNull.Value, NpgsqlDbType = npgsqlDbType.Value }
|
||||
: new() { Value = value ?? DBNull.Value });
|
||||
}
|
||||
|
||||
batch.BatchCommands.Add(batchCommand);
|
||||
@@ -357,7 +372,7 @@ WHERE table_schema = $1 AND table_type = 'BASE TABLE'
|
||||
internal static void BuildGetBatchCommand<TKey>(NpgsqlCommand command, string schema, string tableName, CollectionModel model, List<TKey> keys, bool includeVectors = false)
|
||||
where TKey : notnull
|
||||
{
|
||||
NpgsqlDbType? keyType = PostgresPropertyMapping.GetNpgsqlDbType(model.KeyProperty.Type) ?? throw new UnreachableException($"Unsupported key type {model.KeyProperty.Type.Name}");
|
||||
NpgsqlDbType? keyType = PostgresPropertyMapping.GetNpgsqlDbType(model.KeyProperty) ?? throw new UnreachableException($"Unsupported key type {model.KeyProperty.Type.Name}");
|
||||
|
||||
StringBuilder sql = new();
|
||||
sql.Append("SELECT ");
|
||||
@@ -403,9 +418,9 @@ WHERE table_schema = $1 AND table_type = 'BASE TABLE'
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
internal static void BuildDeleteBatchCommand<TKey>(NpgsqlCommand command, string schema, string tableName, string keyColumn, List<TKey> keys)
|
||||
internal static void BuildDeleteBatchCommand<TKey>(NpgsqlCommand command, string schema, string tableName, KeyPropertyModel keyProperty, List<TKey> keys)
|
||||
{
|
||||
NpgsqlDbType? keyType = PostgresPropertyMapping.GetNpgsqlDbType(typeof(TKey)) ?? throw new ArgumentException($"Unsupported key type {typeof(TKey).Name}");
|
||||
NpgsqlDbType? keyType = PostgresPropertyMapping.GetNpgsqlDbType(keyProperty) ?? throw new ArgumentException($"Unsupported key type {typeof(TKey).Name}");
|
||||
|
||||
for (int i = 0; i < keys.Count; i++)
|
||||
{
|
||||
@@ -417,7 +432,7 @@ WHERE table_schema = $1 AND table_type = 'BASE TABLE'
|
||||
|
||||
StringBuilder sql = new();
|
||||
sql.Append("DELETE FROM ").AppendIdentifier(schema).Append('.').AppendIdentifier(tableName).AppendLine()
|
||||
.Append("WHERE ").AppendIdentifier(keyColumn).Append(" = ANY($1);");
|
||||
.Append("WHERE ").AppendIdentifier(keyProperty.StorageName).Append(" = ANY($1);");
|
||||
|
||||
command.CommandText = sql.ToString();
|
||||
Debug.Assert(command.Parameters.Count == 0);
|
||||
|
||||
@@ -10,8 +10,23 @@ namespace PgVector.ConformanceTests.TypeTests;
|
||||
public class PostgresDataTypeTests(PostgresDataTypeTests.Fixture fixture)
|
||||
: DataTypeTests<Guid, DataTypeTests<Guid>.DefaultRecord>(fixture), IClassFixture<PostgresDataTypeTests.Fixture>
|
||||
{
|
||||
// Npgsql maps DateTime to timestamptz by default, and so requires UTC DateTimes (the base test uses Unspecified).
|
||||
public override Task DateTime()
|
||||
=> Assert.ThrowsAsync<ArgumentException>(base.DateTime);
|
||||
|
||||
public virtual Task DateTime_utc()
|
||||
=> this.Test<DateTime>(
|
||||
"DateTime",
|
||||
System.DateTime.SpecifyKind(new DateTime(2020, 1, 1, 12, 30, 45), DateTimeKind.Utc),
|
||||
System.DateTime.SpecifyKind(new DateTime(2021, 2, 3, 13, 40, 55), DateTimeKind.Utc),
|
||||
instantiationExpression: () => System.DateTime.SpecifyKind(new DateTime(2020, 1, 1, 12, 30, 45), DateTimeKind.Utc));
|
||||
|
||||
// PostgreSQL does not support representing an offset, so only DateTimeOffsets with offset=0 are supported.
|
||||
public override Task DateTimeOffset()
|
||||
=> Assert.ThrowsAsync<ArgumentException>(base.DateTimeOffset);
|
||||
|
||||
// PostgreSQL does not support representing an offset, so only DateTimeOffsets with offset=0 are supported.
|
||||
public virtual Task DateTimeOffset_with_offset_zero()
|
||||
=> this.Test<DateTimeOffset>(
|
||||
"DateTimeOffset",
|
||||
new DateTimeOffset(2020, 1, 1, 12, 30, 45, TimeSpan.FromHours(0)),
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.Extensions.VectorData.ProviderServices;
|
||||
using Microsoft.SemanticKernel.Connectors.PgVector;
|
||||
using Xunit;
|
||||
|
||||
namespace SemanticKernel.Connectors.PgVector.UnitTests;
|
||||
|
||||
public sealed class PostgresFilterTranslatorTests
|
||||
{
|
||||
[Fact]
|
||||
public void DateTime_Utc_FormattedWithZ()
|
||||
{
|
||||
// Inline new DateTime(..., DateTimeKind.Utc) in expression tree to exercise TranslateConstant
|
||||
var sql = TranslateFilter<TestRecord>(r => r.Created == new DateTime(2024, 6, 15, 10, 30, 45, DateTimeKind.Utc));
|
||||
Assert.Contains("'2024-06-15T10:30:45Z'", sql);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DateTime_Unspecified_FormattedWithoutZ()
|
||||
{
|
||||
// Inline new DateTime(...) has Kind=Unspecified
|
||||
var sql = TranslateFilter<TestRecord>(r => r.Created == new DateTime(2024, 6, 15, 10, 30, 45));
|
||||
Assert.Contains("'2024-06-15T10:30:45'", sql);
|
||||
Assert.DoesNotContain("Z'", sql);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DateTimeOffset_Utc_FormattedWithZ()
|
||||
{
|
||||
// Use a ConstantExpression directly to bypass VisitNew issues with TimeSpan.Zero
|
||||
var sql = TranslateFilterWithConstant<TestRecord, DateTimeOffset>(
|
||||
nameof(TestRecord.CreatedOffset),
|
||||
new DateTimeOffset(2024, 6, 15, 10, 30, 45, TimeSpan.Zero));
|
||||
Assert.Contains("'2024-06-15T10:30:45Z'", sql);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DateTimeOffset_NonZeroOffset_Throws()
|
||||
{
|
||||
Assert.ThrowsAny<NotSupportedException>(() =>
|
||||
TranslateFilterWithConstant<TestRecord, DateTimeOffset>(
|
||||
nameof(TestRecord.CreatedOffset),
|
||||
new DateTimeOffset(2024, 6, 15, 10, 30, 45, TimeSpan.FromHours(2))));
|
||||
}
|
||||
|
||||
private static string TranslateFilter<TRecord>(Expression<Func<TRecord, bool>> filter)
|
||||
{
|
||||
var model = BuildModel();
|
||||
var sb = new StringBuilder();
|
||||
var translator = new PostgresFilterTranslator(model, filter, startParamIndex: 1, sb);
|
||||
translator.Translate(appendWhere: false);
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
private static string TranslateFilterWithConstant<TRecord, TValue>(string propertyName, TValue value)
|
||||
{
|
||||
var model = BuildModel();
|
||||
var param = Expression.Parameter(typeof(TRecord), "r");
|
||||
var prop = Expression.Property(param, propertyName);
|
||||
var constant = Expression.Constant(value, typeof(TValue));
|
||||
var body = Expression.Equal(prop, constant);
|
||||
var filter = Expression.Lambda<Func<TRecord, bool>>(body, param);
|
||||
|
||||
var sb = new StringBuilder();
|
||||
var translator = new PostgresFilterTranslator(model, filter, startParamIndex: 1, sb);
|
||||
translator.Translate(appendWhere: false);
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
private static CollectionModel BuildModel()
|
||||
{
|
||||
var definition = new VectorStoreCollectionDefinition
|
||||
{
|
||||
Properties =
|
||||
[
|
||||
new VectorStoreKeyProperty("Id", typeof(Guid)),
|
||||
new VectorStoreDataProperty("Created", typeof(DateTime)),
|
||||
new VectorStoreDataProperty("CreatedOffset", typeof(DateTimeOffset)),
|
||||
new VectorStoreVectorProperty("Embedding", typeof(ReadOnlyMemory<float>), 10)
|
||||
]
|
||||
};
|
||||
|
||||
return new PostgresModelBuilder().BuildDynamic(definition, defaultEmbeddingGenerator: null);
|
||||
}
|
||||
|
||||
#pragma warning disable CA1812 // internal class that is apparently never instantiated.
|
||||
private sealed record TestRecord
|
||||
{
|
||||
public Guid Id { get; set; }
|
||||
public DateTime Created { get; set; }
|
||||
public DateTimeOffset CreatedOffset { get; set; }
|
||||
public ReadOnlyMemory<float> Embedding { get; set; }
|
||||
}
|
||||
#pragma warning restore CA1812
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.SemanticKernel.Connectors.PgVector;
|
||||
using Xunit;
|
||||
|
||||
namespace SemanticKernel.Connectors.PgVector.UnitTests;
|
||||
|
||||
public sealed class PostgresPropertyExtensionsTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("timestamp")]
|
||||
[InlineData("TIMESTAMP")]
|
||||
[InlineData("timestamp without time zone")]
|
||||
[InlineData("TIMESTAMP WITHOUT TIME ZONE")]
|
||||
public void WithStoreType_Timestamp_SetsAnnotation(string storeType)
|
||||
{
|
||||
var property = new VectorStoreDataProperty("test", typeof(DateTime));
|
||||
|
||||
var result = property.WithStoreType(storeType);
|
||||
|
||||
Assert.Same(property, result);
|
||||
Assert.Equal(storeType, property.GetStoreType());
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("timestamptz")]
|
||||
[InlineData("timestamp with time zone")]
|
||||
[InlineData("integer")]
|
||||
[InlineData("text")]
|
||||
[InlineData("")]
|
||||
public void WithStoreType_UnsupportedType_Throws(string storeType)
|
||||
{
|
||||
var property = new VectorStoreDataProperty("test", typeof(DateTime));
|
||||
|
||||
Assert.Throws<NotSupportedException>(() => property.WithStoreType(storeType));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetStoreType_NotSet_ReturnsNull()
|
||||
{
|
||||
var property = new VectorStoreDataProperty("test", typeof(DateTime));
|
||||
|
||||
Assert.Null(property.GetStoreType());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WithStoreType_OnlyAllowedOnDateTimeProperties()
|
||||
{
|
||||
// Setting the annotation on a non-DateTime property should succeed at the property level
|
||||
// (validation happens in the model builder), but building the model should throw.
|
||||
var definition = new VectorStoreCollectionDefinition
|
||||
{
|
||||
Properties =
|
||||
[
|
||||
new VectorStoreKeyProperty("id", typeof(Guid)),
|
||||
new VectorStoreDataProperty("name", typeof(string)).WithStoreType("timestamp"),
|
||||
new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory<float>), 10)
|
||||
]
|
||||
};
|
||||
|
||||
Assert.Throws<NotSupportedException>(() =>
|
||||
new PostgresModelBuilder().BuildDynamic(definition, defaultEmbeddingGenerator: null));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WithStoreType_AllowedOnDateTimeProperty()
|
||||
{
|
||||
var definition = new VectorStoreCollectionDefinition
|
||||
{
|
||||
Properties =
|
||||
[
|
||||
new VectorStoreKeyProperty("id", typeof(Guid)),
|
||||
new VectorStoreDataProperty("created", typeof(DateTime)).WithStoreType("timestamp"),
|
||||
new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory<float>), 10)
|
||||
]
|
||||
};
|
||||
|
||||
// Should not throw
|
||||
var model = new PostgresModelBuilder().BuildDynamic(definition, defaultEmbeddingGenerator: null);
|
||||
Assert.NotNull(model);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WithStoreType_AllowedOnKeyProperty()
|
||||
{
|
||||
var property = new VectorStoreKeyProperty("id", typeof(DateTime)).WithStoreType("timestamp");
|
||||
Assert.Equal("timestamp", property.GetStoreType());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WithStoreType_OnKeyProperty_NonDateTime_Throws()
|
||||
{
|
||||
var definition = new VectorStoreCollectionDefinition
|
||||
{
|
||||
Properties =
|
||||
[
|
||||
new VectorStoreKeyProperty("id", typeof(Guid)).WithStoreType("timestamp"),
|
||||
new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory<float>), 10)
|
||||
]
|
||||
};
|
||||
|
||||
Assert.Throws<NotSupportedException>(() =>
|
||||
new PostgresModelBuilder().BuildDynamic(definition, defaultEmbeddingGenerator: null));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WithStoreType_AllowedOnNullableDateTimeProperty()
|
||||
{
|
||||
var definition = new VectorStoreCollectionDefinition
|
||||
{
|
||||
Properties =
|
||||
[
|
||||
new VectorStoreKeyProperty("id", typeof(Guid)),
|
||||
new VectorStoreDataProperty("created", typeof(DateTime?)).WithStoreType("timestamp"),
|
||||
new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory<float>), 10)
|
||||
]
|
||||
};
|
||||
|
||||
// Should not throw
|
||||
var model = new PostgresModelBuilder().BuildDynamic(definition, defaultEmbeddingGenerator: null);
|
||||
Assert.NotNull(model);
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ using System.Collections.Generic;
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.Extensions.VectorData.ProviderServices;
|
||||
using Microsoft.SemanticKernel.Connectors.PgVector;
|
||||
using NpgsqlTypes;
|
||||
using Pgvector;
|
||||
using Xunit;
|
||||
|
||||
@@ -50,14 +51,15 @@ public sealed class PostgresPropertyMappingTests
|
||||
(typeof(List<double>), "DOUBLE PRECISION[]"),
|
||||
(typeof(List<string>), "TEXT[]"),
|
||||
(typeof(List<bool>), "BOOLEAN[]"),
|
||||
(typeof(List<DateTime>), "TIMESTAMP[]"),
|
||||
(typeof(List<DateTime>), "TIMESTAMPTZ[]"),
|
||||
(typeof(List<Guid>), "UUID[]"),
|
||||
};
|
||||
|
||||
// Act & Assert
|
||||
foreach (var (type, expectedValue) in typesAndExpectedValues)
|
||||
{
|
||||
var (pgType, _) = PostgresPropertyMapping.GetPostgresTypeName(type);
|
||||
var property = new DataPropertyModel("test", type);
|
||||
var (pgType, _) = PostgresPropertyMapping.GetPostgresTypeName(property);
|
||||
Assert.Equal(expectedValue, pgType);
|
||||
}
|
||||
}
|
||||
@@ -81,7 +83,8 @@ public sealed class PostgresPropertyMappingTests
|
||||
// Act & Assert
|
||||
foreach (var (type, expectedValue) in typesAndExpectedValues)
|
||||
{
|
||||
var (_, isNullable) = PostgresPropertyMapping.GetPostgresTypeName(type);
|
||||
var property = new DataPropertyModel("test", type);
|
||||
var (_, isNullable) = PostgresPropertyMapping.GetPostgresTypeName(property);
|
||||
Assert.Equal(expectedValue, isNullable);
|
||||
}
|
||||
}
|
||||
@@ -142,4 +145,53 @@ public sealed class PostgresPropertyMappingTests
|
||||
// Act & Assert
|
||||
Assert.Throws<NotSupportedException>(() => PostgresPropertyMapping.GetIndexInfo([vectorProperty]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetPostgresTypeName_DateTime_WithTimestampAnnotation_ReturnsTimestamp()
|
||||
{
|
||||
var dataProperty = new DataPropertyModel("created", typeof(DateTime));
|
||||
dataProperty.ProviderAnnotations = new() { ["Postgres:StoreType"] = "timestamp" };
|
||||
|
||||
var (pgType, _) = PostgresPropertyMapping.GetPostgresTypeName(dataProperty);
|
||||
Assert.Equal("TIMESTAMP", pgType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetPostgresTypeName_DateTime_WithoutAnnotation_ReturnsTimestampTz()
|
||||
{
|
||||
var dataProperty = new DataPropertyModel("created", typeof(DateTime));
|
||||
|
||||
var (pgType, _) = PostgresPropertyMapping.GetPostgresTypeName(dataProperty);
|
||||
Assert.Equal("TIMESTAMPTZ", pgType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetPostgresTypeName_NullableDateTime_WithTimestampAnnotation_ReturnsTimestamp()
|
||||
{
|
||||
var dataProperty = new DataPropertyModel("created", typeof(DateTime?));
|
||||
dataProperty.ProviderAnnotations = new() { ["Postgres:StoreType"] = "timestamp" };
|
||||
|
||||
var (pgType, isNullable) = PostgresPropertyMapping.GetPostgresTypeName(dataProperty);
|
||||
Assert.Equal("TIMESTAMP", pgType);
|
||||
Assert.True(isNullable);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetNpgsqlDbType_DateTime_WithTimestampAnnotation_ReturnsTimestamp()
|
||||
{
|
||||
var dataProperty = new DataPropertyModel("created", typeof(DateTime));
|
||||
dataProperty.ProviderAnnotations = new() { ["Postgres:StoreType"] = "timestamp" };
|
||||
|
||||
var dbType = PostgresPropertyMapping.GetNpgsqlDbType(dataProperty);
|
||||
Assert.Equal(NpgsqlDbType.Timestamp, dbType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetNpgsqlDbType_DateTime_WithoutAnnotation_ReturnsTimestampTz()
|
||||
{
|
||||
var dataProperty = new DataPropertyModel("created", typeof(DateTime));
|
||||
|
||||
var dbType = PostgresPropertyMapping.GetNpgsqlDbType(dataProperty);
|
||||
Assert.Equal(NpgsqlDbType.TimestampTz, dbType);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.Extensions.VectorData.ProviderServices;
|
||||
using Microsoft.SemanticKernel.Connectors.PgVector;
|
||||
using Npgsql;
|
||||
using Xunit;
|
||||
@@ -75,6 +76,35 @@ public class PostgresSqlBuilderTests
|
||||
this._output.WriteLine(sql);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TestBuildCreateTableCommand_WithTimestampStoreType()
|
||||
{
|
||||
var recordDefinition = new VectorStoreCollectionDefinition()
|
||||
{
|
||||
Properties = [
|
||||
new VectorStoreKeyProperty("id", typeof(long)),
|
||||
new VectorStoreDataProperty("created_utc", typeof(DateTime)),
|
||||
new VectorStoreDataProperty("created_local", typeof(DateTime)).WithStoreType("timestamp"),
|
||||
new VectorStoreDataProperty("created_nullable", typeof(DateTime?)).WithStoreType("timestamp without time zone"),
|
||||
new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory<float>), 10)
|
||||
]
|
||||
};
|
||||
|
||||
var model = new PostgresModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null);
|
||||
|
||||
var sql = PostgresSqlBuilder.BuildCreateTableSql("public", "testcollection", model, pgVersion: new Version(18, 0));
|
||||
|
||||
Assert.Contains("\"created_utc\" TIMESTAMPTZ NOT NULL", sql);
|
||||
Assert.Contains("\"created_local\" TIMESTAMP NOT NULL", sql);
|
||||
Assert.Contains("\"created_nullable\" TIMESTAMP", sql);
|
||||
// Make sure it's TIMESTAMP (not TIMESTAMPTZ) for the nullable one
|
||||
var idx = sql.IndexOf("\"created_nullable\"", StringComparison.Ordinal);
|
||||
var fragment = sql.Substring(idx, sql.IndexOf('\n', idx) - idx);
|
||||
Assert.DoesNotContain("TIMESTAMPTZ", fragment);
|
||||
|
||||
this._output.WriteLine(sql);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(IndexKind.Hnsw, DistanceFunction.EuclideanDistance, true)]
|
||||
[InlineData(IndexKind.Hnsw, DistanceFunction.EuclideanDistance, false)]
|
||||
@@ -373,7 +403,8 @@ public class PostgresSqlBuilderTests
|
||||
|
||||
// Act
|
||||
using var command = new NpgsqlCommand();
|
||||
PostgresSqlBuilder.BuildDeleteBatchCommand(command, "public", "testcollection", "id", keys);
|
||||
var keyProperty = new KeyPropertyModel("id", typeof(long));
|
||||
PostgresSqlBuilder.BuildDeleteBatchCommand(command, "public", "testcollection", keyProperty, keys);
|
||||
|
||||
// Assert
|
||||
Assert.Contains("DELETE", command.CommandText);
|
||||
|
||||
Reference in New Issue
Block a user