.Net: [MEVD] Map DateTime to timestamptz on PostgreSQL (#13514)

DateTime now maps to the PG timestamptz type by default, which
represents a UTC timestamp; this aligns with Npgsql and EF Core.

However, now that we have property annotations, I've also added a new
`WithStoreType()` that allows the user to specify a database type
explicitly. For now, this is only supported for setting `timestamp with
time zone` (short: `timestamp`), which is the other PG date/time type,
for unspecified timestamps (timezone is unknown or implicitly assumed).
This allows users who do want to store non-UTC timestamp to do so.

Closes #10641
This commit is contained in:
Shay Rojansky
2026-02-11 10:54:37 +00:00
committed by GitHub
parent 898b9fe691
commit e8b4862d03
11 changed files with 489 additions and 35 deletions
@@ -375,7 +375,7 @@ public class PostgresCollection<TKey, TRecord> : VectorStoreCollection<TKey, TRe
using var connection = await this._dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
using var command = connection.CreateCommand();
PostgresSqlBuilder.BuildDeleteBatchCommand(command, this._schema, this.Name, this._model.KeyProperty.StorageName, listOfKeys);
PostgresSqlBuilder.BuildDeleteBatchCommand(command, this._schema, this.Name, this._model.KeyProperty, listOfKeys);
await this.RunOperationAsync("DeleteBatch", () => command.ExecuteNonQueryAsync(cancellationToken)).ConfigureAwait(false);
}
@@ -3,6 +3,7 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.Linq.Expressions;
using System.Text;
@@ -29,18 +30,29 @@ internal sealed class PostgresFilterTranslator : SqlFilterTranslator
{
switch (value)
{
// TODO: This aligns with our mapping of DateTime to PG's timestamp (as opposed to timestamptz) - we probably want to
// change that to timestamptz (aligning with Npgsql and EF). See #10641.
case DateTime dateTime:
this._sql.Append('\'').Append(dateTime.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFF", CultureInfo.InvariantCulture)).Append('\'');
return;
switch (dateTime.Kind)
{
case DateTimeKind.Utc:
this._sql.Append('\'').Append(dateTime.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFFZ", CultureInfo.InvariantCulture)).Append('\'');
return;
case DateTimeKind.Unspecified:
case DateTimeKind.Local:
this._sql.Append('\'').Append(dateTime.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFF", CultureInfo.InvariantCulture)).Append('\'');
return;
default:
throw new UnreachableException();
}
case DateTimeOffset dateTimeOffset:
if (dateTimeOffset.Offset != TimeSpan.Zero)
{
throw new NotSupportedException("DateTimeOffset with non-zero offset is not supported with PostgreSQL");
throw new NotSupportedException("DateTimeOffset with non-zero offset is not supported with PostgreSQL. Use DateTimeOffset.UtcNow or convert to UTC.");
}
this._sql.Append('\'').Append(dateTimeOffset.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFF", CultureInfo.InvariantCulture)).Append("Z'");
this._sql.Append('\'').Append(dateTimeOffset.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFFZ", CultureInfo.InvariantCulture)).Append('\'');
return;
// Array constants (ARRAY[1, 2, 3])
@@ -5,6 +5,7 @@ using System.Collections;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.VectorData;
using Microsoft.Extensions.VectorData.ProviderServices;
using Pgvector;
@@ -92,6 +93,22 @@ internal class PostgresModelBuilder() : CollectionModelBuilder(PostgresModelBuil
type == typeof(SparseVector);
}
/// <inheritdoc />
protected override void ValidateProperty(PropertyModel propertyModel, VectorStoreCollectionDefinition? definition)
{
base.ValidateProperty(propertyModel, definition);
if (propertyModel.IsTimestampWithoutTimezone())
{
var type = Nullable.GetUnderlyingType(propertyModel.Type) ?? propertyModel.Type;
if (type != typeof(DateTime))
{
throw new NotSupportedException(
$"Property '{propertyModel.ModelName}' has store type 'timestamp' configured, but this is only supported for DateTime properties. The property type is '{propertyModel.Type.Name}'.");
}
}
}
/// <inheritdoc />
protected override Type? ResolveEmbeddingType(
VectorPropertyModel vectorProperty,
@@ -1,5 +1,6 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using Microsoft.Extensions.VectorData;
using Microsoft.Extensions.VectorData.ProviderServices;
@@ -11,6 +12,9 @@ namespace Microsoft.SemanticKernel.Connectors.PgVector;
public static class PostgresPropertyExtensions
{
private const string FullTextSearchLanguageKey = "Postgres:FullTextSearchLanguage";
private const string StoreTypeKey = "Postgres:StoreType";
#region Full-text search language
/// <summary>
/// Sets the PostgreSQL full-text search language for a data property.
@@ -50,4 +54,69 @@ public static class PostgresPropertyExtensions
=> property.ProviderAnnotations?.TryGetValue(FullTextSearchLanguageKey, out var value) == true && value is string language
? language
: PostgresConstants.DefaultFullTextSearchLanguage;
#endregion Full-text search language
#region Store type
/// <summary>
/// Sets the PostgreSQL store type for a property, overriding the default type mapping.
/// </summary>
/// <param name="property">The property to configure.</param>
/// <param name="storeType">
/// The PostgreSQL type name. Currently, only <c>"timestamp"</c> and <c>"timestamp without time zone"</c>
/// are supported, and only on <see cref="DateTime"/> properties. This causes the property to be stored as
/// PostgreSQL <c>timestamp</c> (without time zone) instead of the default <c>timestamptz</c>.
/// </param>
/// <returns>The same property instance for method chaining.</returns>
/// <remarks>
/// <para>
/// By default, .NET <see cref="DateTime"/> properties are mapped to PostgreSQL <c>timestamptz</c> (timestamp with time zone),
/// which requires UTC values. Use this method to map to <c>timestamp</c> (without time zone) instead, which stores
/// local/unspecified date-time values without time zone information.
/// </para>
/// <para>
/// When using <c>timestamp</c>, <see cref="DateTime"/> values with <see cref="DateTimeKind.Unspecified"/> or
/// <see cref="DateTimeKind.Local"/> kind should be used. Values read back from the database will have
/// <see cref="DateTimeKind.Unspecified"/>.
/// </para>
/// </remarks>
/// <exception cref="NotSupportedException">Thrown if the <paramref name="storeType"/> is not a supported value.</exception>
public static TProperty WithStoreType<TProperty>(this TProperty property, string storeType)
where TProperty : VectorStoreProperty
{
if (!IsTimestampStoreType(storeType))
{
throw new NotSupportedException(
$"Store type '{storeType}' is not supported. Only 'timestamp' and 'timestamp without time zone' are supported.");
}
property.ProviderAnnotations ??= [];
property.ProviderAnnotations[StoreTypeKey] = storeType;
return property;
}
/// <summary>
/// Gets the PostgreSQL store type configured for a property.
/// </summary>
/// <param name="property">The property to read from.</param>
/// <returns>The configured store type, or <see langword="null"/> if not set.</returns>
public static string? GetStoreType(this VectorStoreProperty property)
=> property.ProviderAnnotations?.TryGetValue(StoreTypeKey, out var value) == true
? value as string
: null;
/// <summary>
/// Gets whether the property model has been configured with a <c>timestamp</c> (without time zone) store type.
/// </summary>
internal static bool IsTimestampWithoutTimezone(this PropertyModel property)
=> property.ProviderAnnotations?.TryGetValue(StoreTypeKey, out var value) == true
&& value is string storeType
&& IsTimestampStoreType(storeType);
private static bool IsTimestampStoreType(string storeType)
=> string.Equals(storeType, "timestamp", StringComparison.OrdinalIgnoreCase)
|| string.Equals(storeType, "timestamp without time zone", StringComparison.OrdinalIgnoreCase);
#endregion Store type
}
@@ -38,8 +38,11 @@ internal static class PostgresPropertyMapping
var value => throw new NotSupportedException($"Mapping for type '{value.GetType().Name}' to a vector is not supported.")
};
public static NpgsqlDbType? GetNpgsqlDbType(Type propertyType) =>
(Nullable.GetUnderlyingType(propertyType) ?? propertyType) switch
/// <summary>
/// Gets the NpgsqlDbType for a property, taking into account any store type annotation.
/// </summary>
internal static NpgsqlDbType? GetNpgsqlDbType(PropertyModel property)
=> (Nullable.GetUnderlyingType(property.Type) ?? property.Type) switch
{
Type t when t == typeof(bool) => NpgsqlDbType.Boolean,
Type t when t == typeof(short) => NpgsqlDbType.Smallint,
@@ -50,19 +53,21 @@ internal static class PostgresPropertyMapping
Type t when t == typeof(decimal) => NpgsqlDbType.Numeric,
Type t when t == typeof(string) => NpgsqlDbType.Text,
Type t when t == typeof(byte[]) => NpgsqlDbType.Bytea,
Type t when t == typeof(DateTime) => NpgsqlDbType.Timestamp,
Type t when t == typeof(DateTimeOffset) => NpgsqlDbType.TimestampTz,
Type t when t == typeof(Guid) => NpgsqlDbType.Uuid,
Type t when t == typeof(DateTimeOffset) => NpgsqlDbType.TimestampTz,
// DateTime properties map to PG's "timestamp with time zone" (UTC timestamps) by default, aligning with Npgsql/EF/etc.
// Users can explicitly opt into "timestamp without time zone".
Type t when t == typeof(DateTime) && property.IsTimestampWithoutTimezone() => NpgsqlDbType.Timestamp,
Type t when t == typeof(DateTime) => NpgsqlDbType.TimestampTz,
_ => null
};
/// <summary>
/// Maps a .NET type to a PostgreSQL type name.
/// Maps a .NET type to a PostgreSQL type name, taking into account any store type annotation on the property.
/// </summary>
/// <param name="propertyType">The .NET type.</param>
/// <returns>Tuple of the the PostgreSQL type name and whether it can be NULL</returns>
public static (string PgType, bool IsNullable) GetPostgresTypeName(Type propertyType)
internal static (string PgType, bool IsNullable) GetPostgresTypeName(PropertyModel property)
{
static bool TryGetBaseType(Type type, [NotNullWhen(true)] out string? typeName)
{
@@ -77,7 +82,7 @@ internal static class PostgresPropertyMapping
Type t when t == typeof(decimal) => "NUMERIC",
Type t when t == typeof(string) => "TEXT",
Type t when t == typeof(byte[]) => "BYTEA",
Type t when t == typeof(DateTime) => "TIMESTAMP",
Type t when t == typeof(DateTime) => "TIMESTAMPTZ",
Type t when t == typeof(DateTimeOffset) => "TIMESTAMPTZ",
Type t when t == typeof(Guid) => "UUID",
_ => null
@@ -86,30 +91,43 @@ internal static class PostgresPropertyMapping
return typeName is not null;
}
var propertyType = property.Type;
// TODO: Handle NRTs properly via NullabilityInfoContext
(string PgType, bool IsNullable) result;
if (TryGetBaseType(propertyType, out string? pgType))
{
return (pgType, !propertyType.IsValueType);
result = (pgType, !propertyType.IsValueType);
}
// Handle nullable types (e.g. Nullable<int>)
if (Nullable.GetUnderlyingType(propertyType) is Type unwrappedType
else if (Nullable.GetUnderlyingType(propertyType) is Type unwrappedType
&& TryGetBaseType(unwrappedType, out string? underlyingPgType))
{
return (underlyingPgType, true);
result = (underlyingPgType, true);
}
// Handle collections
if ((propertyType.IsArray && TryGetBaseType(propertyType.GetElementType()!, out string? elementPgType))
else if ((propertyType.IsArray && TryGetBaseType(propertyType.GetElementType()!, out string? elementPgType))
|| (propertyType.IsGenericType
&& propertyType.GetGenericTypeDefinition() == typeof(List<>)
&& TryGetBaseType(propertyType.GetGenericArguments()[0], out elementPgType)))
{
return (elementPgType + "[]", true);
result = (elementPgType + "[]", true);
}
else
{
throw new NotSupportedException($"Type {propertyType.Name} is not supported by this store.");
}
throw new NotSupportedException($"Type {propertyType.Name} is not supported by this store.");
if (property.IsTimestampWithoutTimezone())
{
// Replace TIMESTAMPTZ with TIMESTAMP in the PG type name.
// This handles both "TIMESTAMPTZ" and "TIMESTAMPTZ[]" cases.
result = ("TIMESTAMP", result.IsNullable);
}
return result;
}
/// <summary>
@@ -64,7 +64,7 @@ WHERE table_schema = $1 AND table_type = 'BASE TABLE'
createTableCommand.AppendIdentifier(schema).Append('.').AppendIdentifier(tableName).AppendLine(" (");
// Add the key column
var keyStoreType = PostgresPropertyMapping.GetPostgresTypeName(model.KeyProperty.Type).PgType;
var keyStoreType = PostgresPropertyMapping.GetPostgresTypeName(model.KeyProperty).PgType;
createTableCommand.Append(" ").AppendIdentifier(keyName).Append(' ').Append(keyStoreType);
if (model.KeyProperty.IsAutoGenerated)
{
@@ -90,7 +90,7 @@ WHERE table_schema = $1 AND table_type = 'BASE TABLE'
// Add the data columns
foreach (var dataProperty in model.DataProperties)
{
var dataPgTypeInfo = PostgresPropertyMapping.GetPostgresTypeName(dataProperty.Type);
var dataPgTypeInfo = PostgresPropertyMapping.GetPostgresTypeName(dataProperty);
createTableCommand.Append(" ").AppendIdentifier(dataProperty.StorageName).Append(' ').Append(dataPgTypeInfo.PgType);
if (!dataPgTypeInfo.IsNullable)
{
@@ -224,7 +224,22 @@ WHERE table_schema = $1 AND table_type = 'BASE TABLE'
value = PostgresPropertyMapping.MapVectorForStorageModel(value);
}
batchCommand.Parameters.Add(new() { Value = value ?? DBNull.Value });
NpgsqlDbType? npgsqlDbType = null;
if (property.Type == typeof(DateTime))
{
npgsqlDbType = property.IsTimestampWithoutTimezone()
? NpgsqlDbType.Timestamp
// DateTime properties map to PG's "timestamp with time zone" (timestamptz) by default.
// Npgsql would silently send non-UTC DateTimes as 'timestamp' (without timezone), and PG would
// implicitly cast to timestamptz using the session timezone, producing incorrect results.
// Explicitly set NpgsqlDbType.TimestampTz to ensure Npgsql rejects non-UTC DateTimes.
: NpgsqlDbType.TimestampTz;
}
batchCommand.Parameters.Add(npgsqlDbType.HasValue
? new() { Value = value ?? DBNull.Value, NpgsqlDbType = npgsqlDbType.Value }
: new() { Value = value ?? DBNull.Value });
}
batch.BatchCommands.Add(batchCommand);
@@ -357,7 +372,7 @@ WHERE table_schema = $1 AND table_type = 'BASE TABLE'
internal static void BuildGetBatchCommand<TKey>(NpgsqlCommand command, string schema, string tableName, CollectionModel model, List<TKey> keys, bool includeVectors = false)
where TKey : notnull
{
NpgsqlDbType? keyType = PostgresPropertyMapping.GetNpgsqlDbType(model.KeyProperty.Type) ?? throw new UnreachableException($"Unsupported key type {model.KeyProperty.Type.Name}");
NpgsqlDbType? keyType = PostgresPropertyMapping.GetNpgsqlDbType(model.KeyProperty) ?? throw new UnreachableException($"Unsupported key type {model.KeyProperty.Type.Name}");
StringBuilder sql = new();
sql.Append("SELECT ");
@@ -403,9 +418,9 @@ WHERE table_schema = $1 AND table_type = 'BASE TABLE'
}
/// <inheritdoc />
internal static void BuildDeleteBatchCommand<TKey>(NpgsqlCommand command, string schema, string tableName, string keyColumn, List<TKey> keys)
internal static void BuildDeleteBatchCommand<TKey>(NpgsqlCommand command, string schema, string tableName, KeyPropertyModel keyProperty, List<TKey> keys)
{
NpgsqlDbType? keyType = PostgresPropertyMapping.GetNpgsqlDbType(typeof(TKey)) ?? throw new ArgumentException($"Unsupported key type {typeof(TKey).Name}");
NpgsqlDbType? keyType = PostgresPropertyMapping.GetNpgsqlDbType(keyProperty) ?? throw new ArgumentException($"Unsupported key type {typeof(TKey).Name}");
for (int i = 0; i < keys.Count; i++)
{
@@ -417,7 +432,7 @@ WHERE table_schema = $1 AND table_type = 'BASE TABLE'
StringBuilder sql = new();
sql.Append("DELETE FROM ").AppendIdentifier(schema).Append('.').AppendIdentifier(tableName).AppendLine()
.Append("WHERE ").AppendIdentifier(keyColumn).Append(" = ANY($1);");
.Append("WHERE ").AppendIdentifier(keyProperty.StorageName).Append(" = ANY($1);");
command.CommandText = sql.ToString();
Debug.Assert(command.Parameters.Count == 0);
@@ -10,8 +10,23 @@ namespace PgVector.ConformanceTests.TypeTests;
public class PostgresDataTypeTests(PostgresDataTypeTests.Fixture fixture)
: DataTypeTests<Guid, DataTypeTests<Guid>.DefaultRecord>(fixture), IClassFixture<PostgresDataTypeTests.Fixture>
{
// Npgsql maps DateTime to timestamptz by default, and so requires UTC DateTimes (the base test uses Unspecified).
public override Task DateTime()
=> Assert.ThrowsAsync<ArgumentException>(base.DateTime);
public virtual Task DateTime_utc()
=> this.Test<DateTime>(
"DateTime",
System.DateTime.SpecifyKind(new DateTime(2020, 1, 1, 12, 30, 45), DateTimeKind.Utc),
System.DateTime.SpecifyKind(new DateTime(2021, 2, 3, 13, 40, 55), DateTimeKind.Utc),
instantiationExpression: () => System.DateTime.SpecifyKind(new DateTime(2020, 1, 1, 12, 30, 45), DateTimeKind.Utc));
// PostgreSQL does not support representing an offset, so only DateTimeOffsets with offset=0 are supported.
public override Task DateTimeOffset()
=> Assert.ThrowsAsync<ArgumentException>(base.DateTimeOffset);
// PostgreSQL does not support representing an offset, so only DateTimeOffsets with offset=0 are supported.
public virtual Task DateTimeOffset_with_offset_zero()
=> this.Test<DateTimeOffset>(
"DateTimeOffset",
new DateTimeOffset(2020, 1, 1, 12, 30, 45, TimeSpan.FromHours(0)),
@@ -0,0 +1,100 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Linq.Expressions;
using System.Text;
using Microsoft.Extensions.VectorData;
using Microsoft.Extensions.VectorData.ProviderServices;
using Microsoft.SemanticKernel.Connectors.PgVector;
using Xunit;
namespace SemanticKernel.Connectors.PgVector.UnitTests;
public sealed class PostgresFilterTranslatorTests
{
[Fact]
public void DateTime_Utc_FormattedWithZ()
{
// Inline new DateTime(..., DateTimeKind.Utc) in expression tree to exercise TranslateConstant
var sql = TranslateFilter<TestRecord>(r => r.Created == new DateTime(2024, 6, 15, 10, 30, 45, DateTimeKind.Utc));
Assert.Contains("'2024-06-15T10:30:45Z'", sql);
}
[Fact]
public void DateTime_Unspecified_FormattedWithoutZ()
{
// Inline new DateTime(...) has Kind=Unspecified
var sql = TranslateFilter<TestRecord>(r => r.Created == new DateTime(2024, 6, 15, 10, 30, 45));
Assert.Contains("'2024-06-15T10:30:45'", sql);
Assert.DoesNotContain("Z'", sql);
}
[Fact]
public void DateTimeOffset_Utc_FormattedWithZ()
{
// Use a ConstantExpression directly to bypass VisitNew issues with TimeSpan.Zero
var sql = TranslateFilterWithConstant<TestRecord, DateTimeOffset>(
nameof(TestRecord.CreatedOffset),
new DateTimeOffset(2024, 6, 15, 10, 30, 45, TimeSpan.Zero));
Assert.Contains("'2024-06-15T10:30:45Z'", sql);
}
[Fact]
public void DateTimeOffset_NonZeroOffset_Throws()
{
Assert.ThrowsAny<NotSupportedException>(() =>
TranslateFilterWithConstant<TestRecord, DateTimeOffset>(
nameof(TestRecord.CreatedOffset),
new DateTimeOffset(2024, 6, 15, 10, 30, 45, TimeSpan.FromHours(2))));
}
private static string TranslateFilter<TRecord>(Expression<Func<TRecord, bool>> filter)
{
var model = BuildModel();
var sb = new StringBuilder();
var translator = new PostgresFilterTranslator(model, filter, startParamIndex: 1, sb);
translator.Translate(appendWhere: false);
return sb.ToString();
}
private static string TranslateFilterWithConstant<TRecord, TValue>(string propertyName, TValue value)
{
var model = BuildModel();
var param = Expression.Parameter(typeof(TRecord), "r");
var prop = Expression.Property(param, propertyName);
var constant = Expression.Constant(value, typeof(TValue));
var body = Expression.Equal(prop, constant);
var filter = Expression.Lambda<Func<TRecord, bool>>(body, param);
var sb = new StringBuilder();
var translator = new PostgresFilterTranslator(model, filter, startParamIndex: 1, sb);
translator.Translate(appendWhere: false);
return sb.ToString();
}
private static CollectionModel BuildModel()
{
var definition = new VectorStoreCollectionDefinition
{
Properties =
[
new VectorStoreKeyProperty("Id", typeof(Guid)),
new VectorStoreDataProperty("Created", typeof(DateTime)),
new VectorStoreDataProperty("CreatedOffset", typeof(DateTimeOffset)),
new VectorStoreVectorProperty("Embedding", typeof(ReadOnlyMemory<float>), 10)
]
};
return new PostgresModelBuilder().BuildDynamic(definition, defaultEmbeddingGenerator: null);
}
#pragma warning disable CA1812 // internal class that is apparently never instantiated.
private sealed record TestRecord
{
public Guid Id { get; set; }
public DateTime Created { get; set; }
public DateTimeOffset CreatedOffset { get; set; }
public ReadOnlyMemory<float> Embedding { get; set; }
}
#pragma warning restore CA1812
}
@@ -0,0 +1,125 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel.Connectors.PgVector;
using Xunit;
namespace SemanticKernel.Connectors.PgVector.UnitTests;
public sealed class PostgresPropertyExtensionsTests
{
[Theory]
[InlineData("timestamp")]
[InlineData("TIMESTAMP")]
[InlineData("timestamp without time zone")]
[InlineData("TIMESTAMP WITHOUT TIME ZONE")]
public void WithStoreType_Timestamp_SetsAnnotation(string storeType)
{
var property = new VectorStoreDataProperty("test", typeof(DateTime));
var result = property.WithStoreType(storeType);
Assert.Same(property, result);
Assert.Equal(storeType, property.GetStoreType());
}
[Theory]
[InlineData("timestamptz")]
[InlineData("timestamp with time zone")]
[InlineData("integer")]
[InlineData("text")]
[InlineData("")]
public void WithStoreType_UnsupportedType_Throws(string storeType)
{
var property = new VectorStoreDataProperty("test", typeof(DateTime));
Assert.Throws<NotSupportedException>(() => property.WithStoreType(storeType));
}
[Fact]
public void GetStoreType_NotSet_ReturnsNull()
{
var property = new VectorStoreDataProperty("test", typeof(DateTime));
Assert.Null(property.GetStoreType());
}
[Fact]
public void WithStoreType_OnlyAllowedOnDateTimeProperties()
{
// Setting the annotation on a non-DateTime property should succeed at the property level
// (validation happens in the model builder), but building the model should throw.
var definition = new VectorStoreCollectionDefinition
{
Properties =
[
new VectorStoreKeyProperty("id", typeof(Guid)),
new VectorStoreDataProperty("name", typeof(string)).WithStoreType("timestamp"),
new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory<float>), 10)
]
};
Assert.Throws<NotSupportedException>(() =>
new PostgresModelBuilder().BuildDynamic(definition, defaultEmbeddingGenerator: null));
}
[Fact]
public void WithStoreType_AllowedOnDateTimeProperty()
{
var definition = new VectorStoreCollectionDefinition
{
Properties =
[
new VectorStoreKeyProperty("id", typeof(Guid)),
new VectorStoreDataProperty("created", typeof(DateTime)).WithStoreType("timestamp"),
new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory<float>), 10)
]
};
// Should not throw
var model = new PostgresModelBuilder().BuildDynamic(definition, defaultEmbeddingGenerator: null);
Assert.NotNull(model);
}
[Fact]
public void WithStoreType_AllowedOnKeyProperty()
{
var property = new VectorStoreKeyProperty("id", typeof(DateTime)).WithStoreType("timestamp");
Assert.Equal("timestamp", property.GetStoreType());
}
[Fact]
public void WithStoreType_OnKeyProperty_NonDateTime_Throws()
{
var definition = new VectorStoreCollectionDefinition
{
Properties =
[
new VectorStoreKeyProperty("id", typeof(Guid)).WithStoreType("timestamp"),
new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory<float>), 10)
]
};
Assert.Throws<NotSupportedException>(() =>
new PostgresModelBuilder().BuildDynamic(definition, defaultEmbeddingGenerator: null));
}
[Fact]
public void WithStoreType_AllowedOnNullableDateTimeProperty()
{
var definition = new VectorStoreCollectionDefinition
{
Properties =
[
new VectorStoreKeyProperty("id", typeof(Guid)),
new VectorStoreDataProperty("created", typeof(DateTime?)).WithStoreType("timestamp"),
new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory<float>), 10)
]
};
// Should not throw
var model = new PostgresModelBuilder().BuildDynamic(definition, defaultEmbeddingGenerator: null);
Assert.NotNull(model);
}
}
@@ -5,6 +5,7 @@ using System.Collections.Generic;
using Microsoft.Extensions.VectorData;
using Microsoft.Extensions.VectorData.ProviderServices;
using Microsoft.SemanticKernel.Connectors.PgVector;
using NpgsqlTypes;
using Pgvector;
using Xunit;
@@ -50,14 +51,15 @@ public sealed class PostgresPropertyMappingTests
(typeof(List<double>), "DOUBLE PRECISION[]"),
(typeof(List<string>), "TEXT[]"),
(typeof(List<bool>), "BOOLEAN[]"),
(typeof(List<DateTime>), "TIMESTAMP[]"),
(typeof(List<DateTime>), "TIMESTAMPTZ[]"),
(typeof(List<Guid>), "UUID[]"),
};
// Act & Assert
foreach (var (type, expectedValue) in typesAndExpectedValues)
{
var (pgType, _) = PostgresPropertyMapping.GetPostgresTypeName(type);
var property = new DataPropertyModel("test", type);
var (pgType, _) = PostgresPropertyMapping.GetPostgresTypeName(property);
Assert.Equal(expectedValue, pgType);
}
}
@@ -81,7 +83,8 @@ public sealed class PostgresPropertyMappingTests
// Act & Assert
foreach (var (type, expectedValue) in typesAndExpectedValues)
{
var (_, isNullable) = PostgresPropertyMapping.GetPostgresTypeName(type);
var property = new DataPropertyModel("test", type);
var (_, isNullable) = PostgresPropertyMapping.GetPostgresTypeName(property);
Assert.Equal(expectedValue, isNullable);
}
}
@@ -142,4 +145,53 @@ public sealed class PostgresPropertyMappingTests
// Act & Assert
Assert.Throws<NotSupportedException>(() => PostgresPropertyMapping.GetIndexInfo([vectorProperty]));
}
[Fact]
public void GetPostgresTypeName_DateTime_WithTimestampAnnotation_ReturnsTimestamp()
{
var dataProperty = new DataPropertyModel("created", typeof(DateTime));
dataProperty.ProviderAnnotations = new() { ["Postgres:StoreType"] = "timestamp" };
var (pgType, _) = PostgresPropertyMapping.GetPostgresTypeName(dataProperty);
Assert.Equal("TIMESTAMP", pgType);
}
[Fact]
public void GetPostgresTypeName_DateTime_WithoutAnnotation_ReturnsTimestampTz()
{
var dataProperty = new DataPropertyModel("created", typeof(DateTime));
var (pgType, _) = PostgresPropertyMapping.GetPostgresTypeName(dataProperty);
Assert.Equal("TIMESTAMPTZ", pgType);
}
[Fact]
public void GetPostgresTypeName_NullableDateTime_WithTimestampAnnotation_ReturnsTimestamp()
{
var dataProperty = new DataPropertyModel("created", typeof(DateTime?));
dataProperty.ProviderAnnotations = new() { ["Postgres:StoreType"] = "timestamp" };
var (pgType, isNullable) = PostgresPropertyMapping.GetPostgresTypeName(dataProperty);
Assert.Equal("TIMESTAMP", pgType);
Assert.True(isNullable);
}
[Fact]
public void GetNpgsqlDbType_DateTime_WithTimestampAnnotation_ReturnsTimestamp()
{
var dataProperty = new DataPropertyModel("created", typeof(DateTime));
dataProperty.ProviderAnnotations = new() { ["Postgres:StoreType"] = "timestamp" };
var dbType = PostgresPropertyMapping.GetNpgsqlDbType(dataProperty);
Assert.Equal(NpgsqlDbType.Timestamp, dbType);
}
[Fact]
public void GetNpgsqlDbType_DateTime_WithoutAnnotation_ReturnsTimestampTz()
{
var dataProperty = new DataPropertyModel("created", typeof(DateTime));
var dbType = PostgresPropertyMapping.GetNpgsqlDbType(dataProperty);
Assert.Equal(NpgsqlDbType.TimestampTz, dbType);
}
}
@@ -4,6 +4,7 @@ using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.VectorData;
using Microsoft.Extensions.VectorData.ProviderServices;
using Microsoft.SemanticKernel.Connectors.PgVector;
using Npgsql;
using Xunit;
@@ -75,6 +76,35 @@ public class PostgresSqlBuilderTests
this._output.WriteLine(sql);
}
[Fact]
public void TestBuildCreateTableCommand_WithTimestampStoreType()
{
var recordDefinition = new VectorStoreCollectionDefinition()
{
Properties = [
new VectorStoreKeyProperty("id", typeof(long)),
new VectorStoreDataProperty("created_utc", typeof(DateTime)),
new VectorStoreDataProperty("created_local", typeof(DateTime)).WithStoreType("timestamp"),
new VectorStoreDataProperty("created_nullable", typeof(DateTime?)).WithStoreType("timestamp without time zone"),
new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory<float>), 10)
]
};
var model = new PostgresModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null);
var sql = PostgresSqlBuilder.BuildCreateTableSql("public", "testcollection", model, pgVersion: new Version(18, 0));
Assert.Contains("\"created_utc\" TIMESTAMPTZ NOT NULL", sql);
Assert.Contains("\"created_local\" TIMESTAMP NOT NULL", sql);
Assert.Contains("\"created_nullable\" TIMESTAMP", sql);
// Make sure it's TIMESTAMP (not TIMESTAMPTZ) for the nullable one
var idx = sql.IndexOf("\"created_nullable\"", StringComparison.Ordinal);
var fragment = sql.Substring(idx, sql.IndexOf('\n', idx) - idx);
Assert.DoesNotContain("TIMESTAMPTZ", fragment);
this._output.WriteLine(sql);
}
[Theory]
[InlineData(IndexKind.Hnsw, DistanceFunction.EuclideanDistance, true)]
[InlineData(IndexKind.Hnsw, DistanceFunction.EuclideanDistance, false)]
@@ -373,7 +403,8 @@ public class PostgresSqlBuilderTests
// Act
using var command = new NpgsqlCommand();
PostgresSqlBuilder.BuildDeleteBatchCommand(command, "public", "testcollection", "id", keys);
var keyProperty = new KeyPropertyModel("id", typeof(long));
PostgresSqlBuilder.BuildDeleteBatchCommand(command, "public", "testcollection", keyProperty, keys);
// Assert
Assert.Contains("DELETE", command.CommandText);