.Net: Implement PostgreSQL hybrid search (#13502)

**Note: this PR is based on top of #13501, skip first commit**

And add provider-specific annotation support

Closes #11084
Closes #10359

Author: roji

dotnet/src/VectorData/PgVector/PostgresCollection.cs

@@ -24,7 +24,7 @@ namespace Microsoft.SemanticKernel.Connectors.PgVector;
 /// <typeparam name="TKey">The type of the key.</typeparam>
 /// <typeparam name="TRecord">The type of the record.</typeparam>
 #pragma warning disable CA1711 // Identifiers should not have incorrect suffix
-public class PostgresCollection<TKey, TRecord> : VectorStoreCollection<TKey, TRecord>
+public class PostgresCollection<TKey, TRecord> : VectorStoreCollection<TKey, TRecord>, IKeywordHybridSearchable<TRecord>
 #pragma warning restore CA1711 // Identifiers should not have incorrect suffix
     where TKey : notnull
     where TRecord : class
@@ -52,6 +52,9 @@ public class PostgresCollection<TKey, TRecord> : VectorStoreCollection<TKey, TRe
     /// <summary>The default options for vector search.</summary>
     private static readonly VectorSearchOptions<TRecord> s_defaultVectorSearchOptions = new();
 
+    /// <summary>The default options for hybrid search.</summary>
+    private static readonly HybridSearchOptions<TRecord> s_defaultHybridSearchOptions = new();
+
     /// <summary>
     /// Initializes a new instance of the <see cref="PostgresCollection{TKey, TRecord}"/> class.
     /// </summary>
@@ -396,43 +399,7 @@ public override async IAsyncEnumerable<VectorSearchResult<TRecord>> SearchAsync<
         }
 
         var vectorProperty = this._model.GetVectorPropertyOrSingle(options);
-
-        object vector = searchValue switch
-        {
-            // Dense float32
-            ReadOnlyMemory<float> r => r,
-            float[] f => new ReadOnlyMemory<float>(f),
-            Embedding<float> e => e.Vector,
-            _ when vectorProperty.EmbeddingGenerator is IEmbeddingGenerator<TInput, Embedding<float>> generator
-                => await generator.GenerateVectorAsync(searchValue, cancellationToken: cancellationToken).ConfigureAwait(false),
-
-#if NET
-            // Dense float16
-            ReadOnlyMemory<Half> r => r,
-            Half[] f => new ReadOnlyMemory<Half>(f),
-            Embedding<Half> e => e.Vector,
-            _ when vectorProperty.EmbeddingGenerator is IEmbeddingGenerator<TInput, Embedding<Half>> generator
-                => await generator.GenerateVectorAsync(searchValue, cancellationToken: cancellationToken).ConfigureAwait(false),
-#endif
-
-            // Dense Binary
-            BitArray b => b,
-            BinaryEmbedding e => e.Vector,
-            _ when vectorProperty.EmbeddingGenerator is IEmbeddingGenerator<TInput, BinaryEmbedding> generator
-                => await generator.GenerateAsync(searchValue, cancellationToken: cancellationToken).ConfigureAwait(false),
-
-            // Sparse
-            SparseVector sv => sv,
-            // TODO: Add a PG-specific SparseVectorEmbedding type
-
-            _ => vectorProperty.EmbeddingGenerator is null
-                ? throw new NotSupportedException(VectorDataStrings.InvalidSearchInputAndNoEmbeddingGeneratorWasConfigured(searchValue.GetType(), PostgresModelBuilder.SupportedVectorTypes))
-                : throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType()))
-        };
-
-        var pgVector = PostgresPropertyMapping.MapVectorForStorageModel(vector);
-
-        Verify.NotNull(pgVector);
+        var pgVector = await this.ConvertSearchInputToVectorAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false);
 
         // Simulating skip/offset logic locally, since OFFSET can work only with LIMIT in combination
         // and LIMIT is not supported in vector search extension, instead of LIMIT - "k" parameter is used.
@@ -460,6 +427,51 @@ _ when vectorProperty.EmbeddingGenerator is IEmbeddingGenerator<TInput, BinaryEm
         }
     }
 
+    /// <inheritdoc />
+    public async IAsyncEnumerable<VectorSearchResult<TRecord>> HybridSearchAsync<TInput>(
+        TInput searchValue,
+        ICollection<string> keywords,
+        int top,
+        HybridSearchOptions<TRecord>? options = null,
+        [EnumeratorCancellation] CancellationToken cancellationToken = default)
+        where TInput : notnull
+    {
+        Verify.NotNull(searchValue);
+        Verify.NotNull(keywords);
+        Verify.NotLessThan(top, 1);
+
+        options ??= s_defaultHybridSearchOptions;
+        if (options.IncludeVectors && this._model.EmbeddingGenerationRequired)
+        {
+            throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration);
+        }
+
+        var vectorProperty = this._model.GetVectorPropertyOrSingle<TRecord>(new() { VectorProperty = options.VectorProperty });
+        var textProperty = this._model.GetFullTextDataPropertyOrSingle(options.AdditionalProperty);
+        var pgVector = await this.ConvertSearchInputToVectorAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false);
+
+        using var connection = await this._dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
+        using var command = connection.CreateCommand();
+        PostgresSqlBuilder.BuildHybridSearchCommand(command, this._schema, this.Name, this._model, vectorProperty, textProperty, pgVector, keywords,
+#pragma warning disable CS0618 // VectorSearchFilter is obsolete
+            options.OldFilter,
+#pragma warning restore CS0618 // VectorSearchFilter is obsolete
+            options.Filter, options.Skip, options.IncludeVectors, top, options.ScoreThreshold);
+
+        using var reader = await connection.ExecuteWithErrorHandlingAsync(
+            this._collectionMetadata,
+            "HybridSearch",
+            () => command.ExecuteReaderAsync(cancellationToken),
+            cancellationToken).ConfigureAwait(false);
+
+        while (await reader.ReadWithErrorHandlingAsync(this._collectionMetadata, "HybridSearch", cancellationToken).ConfigureAwait(false))
+        {
+            yield return new VectorSearchResult<TRecord>(
+                this._mapper.MapFromStorageToDataModel(reader, options.IncludeVectors),
+                reader.GetDouble(reader.GetOrdinal(PostgresConstants.DistanceColumnName)));
+        }
+    }
+
     #endregion Search
 
     /// <inheritdoc />
@@ -513,11 +525,11 @@ private async Task InternalCreateCollectionAsync(bool ifNotExists, CancellationT
             batch.BatchCommands.Add(
                 new NpgsqlBatchCommand(PostgresSqlBuilder.BuildCreateTableSql(this._schema, this.Name, this._model, pgVersion, ifNotExists)));
 
-            foreach (var (column, kind, function, isVector) in PostgresPropertyMapping.GetIndexInfo(this._model.Properties))
+            foreach (var (column, kind, function, isVector, isFullText, fullTextLanguage) in PostgresPropertyMapping.GetIndexInfo(this._model.Properties))
             {
                 batch.BatchCommands.Add(
                     new NpgsqlBatchCommand(
-                        PostgresSqlBuilder.BuildCreateIndexSql(this._schema, this.Name, column, kind, function, isVector, ifNotExists)));
+                        PostgresSqlBuilder.BuildCreateIndexSql(this._schema, this.Name, column, kind, function, isVector, isFullText, fullTextLanguage, ifNotExists)));
             }
 
             await batch.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
@@ -535,4 +547,48 @@ private Task<T> RunOperationAsync<T>(string operationName, Func<Task<T>> operati
             this._collectionMetadata,
             operationName,
             operation);
+
+    /// <summary>
+    /// Converts a search input value to a PostgreSQL vector representation, generating embeddings if necessary.
+    /// </summary>
+    private async Task<object> ConvertSearchInputToVectorAsync<TInput>(TInput searchValue, VectorPropertyModel vectorProperty, CancellationToken cancellationToken)
+        where TInput : notnull
+    {
+        object vector = searchValue switch
+        {
+            // Dense float32
+            ReadOnlyMemory<float> r => r,
+            float[] f => new ReadOnlyMemory<float>(f),
+            Embedding<float> e => e.Vector,
+            _ when vectorProperty.EmbeddingGenerator is IEmbeddingGenerator<TInput, Embedding<float>> generator
+                => await generator.GenerateVectorAsync(searchValue, cancellationToken: cancellationToken).ConfigureAwait(false),
+
+#if NET
+            // Dense float16
+            ReadOnlyMemory<Half> r => r,
+            Half[] f => new ReadOnlyMemory<Half>(f),
+            Embedding<Half> e => e.Vector,
+            _ when vectorProperty.EmbeddingGenerator is IEmbeddingGenerator<TInput, Embedding<Half>> generator
+                => await generator.GenerateVectorAsync(searchValue, cancellationToken: cancellationToken).ConfigureAwait(false),
+#endif
+
+            // Dense Binary
+            BitArray b => b,
+            BinaryEmbedding e => e.Vector,
+            _ when vectorProperty.EmbeddingGenerator is IEmbeddingGenerator<TInput, BinaryEmbedding> generator
+                => await generator.GenerateAsync(searchValue, cancellationToken: cancellationToken).ConfigureAwait(false),
+
+            // Sparse
+            SparseVector sv => sv,
+            // TODO: Add a PG-specific SparseVectorEmbedding type
+
+            _ => vectorProperty.EmbeddingGenerator is null
+                ? throw new NotSupportedException(VectorDataStrings.InvalidSearchInputAndNoEmbeddingGeneratorWasConfigured(searchValue.GetType(), PostgresModelBuilder.SupportedVectorTypes))
+                : throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType()))
+        };
+
+        var pgVector = PostgresPropertyMapping.MapVectorForStorageModel(vector);
+        Verify.NotNull(pgVector);
+        return pgVector;
+    }
 }

dotnet/src/VectorData/PgVector/PostgresConstants.cs

@@ -24,6 +24,9 @@ internal static class PostgresConstants
     /// <summary>The default distance function.</summary>
     public const string DefaultDistanceFunction = DistanceFunction.CosineDistance;
 
+    /// <summary>The default full-text search language for PostgreSQL.</summary>
+    public const string DefaultFullTextSearchLanguage = "english";
+
     public static readonly Dictionary<string, int> IndexMaxDimensions = new()
     {
         { IndexKind.Hnsw, 2000 },

dotnet/src/VectorData/PgVector/PostgresPropertyExtensions.cs

@@ -0,0 +1,53 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.VectorData;
+using Microsoft.Extensions.VectorData.ProviderServices;
+
+namespace Microsoft.SemanticKernel.Connectors.PgVector;
+
+/// <summary>
+/// Extension methods for configuring PostgreSQL-specific properties on vector store property definitions.
+/// </summary>
+public static class PostgresPropertyExtensions
+{
+    private const string FullTextSearchLanguageKey = "Postgres:FullTextSearchLanguage";
+
+    /// <summary>
+    /// Sets the PostgreSQL full-text search language for a data property.
+    /// </summary>
+    /// <param name="property">The data property to configure.</param>
+    /// <param name="language">The PostgreSQL text search language name (e.g., "english", "spanish", "german").</param>
+    /// <returns>The same property instance for method chaining.</returns>
+    /// <remarks>
+    /// This language is used with PostgreSQL's <c>to_tsvector</c> and <c>plainto_tsquery</c> functions
+    /// when creating GIN indexes and performing full-text search operations.
+    /// Common language options include: "simple", "english", "spanish", "german", "french", etc.
+    /// See PostgreSQL documentation for the full list of available text search configurations.
+    /// </remarks>
+    public static VectorStoreDataProperty WithFullTextSearchLanguage(this VectorStoreDataProperty property, string? language)
+    {
+        property.ProviderAnnotations ??= [];
+        property.ProviderAnnotations[FullTextSearchLanguageKey] = language;
+        return property;
+    }
+
+    /// <summary>
+    /// Gets the PostgreSQL full-text search language configured for a data property.
+    /// </summary>
+    /// <param name="property">The data property to read from.</param>
+    /// <returns>The configured language, or <see langword="null"/> if not set.</returns>
+    public static string? GetFullTextSearchLanguage(this VectorStoreDataProperty property)
+        => property.ProviderAnnotations?.TryGetValue(FullTextSearchLanguageKey, out var value) == true
+            ? value as string
+            : null;
+
+    /// <summary>
+    /// Gets the PostgreSQL full-text search language configured for a data property model.
+    /// </summary>
+    /// <param name="property">The data property model to read from.</param>
+    /// <returns>The configured language, or the default language ("english") if not set.</returns>
+    internal static string GetFullTextSearchLanguageOrDefault(this DataPropertyModel property)
+        => property.ProviderAnnotations?.TryGetValue(FullTextSearchLanguageKey, out var value) == true && value is string language
+            ? language
+            : PostgresConstants.DefaultFullTextSearchLanguage;
+}

dotnet/src/VectorData/PgVector/PostgresPropertyMapping.cs

@@ -152,13 +152,13 @@ public static NpgsqlParameter GetNpgsqlParameter(object? value)
     /// Returns information about indexes to create, validating that the dimensions of the vector are supported.
     /// </summary>
     /// <param name="properties">The properties of the vector store record.</param>
-    /// <returns>A list of tuples containing the column name, index kind, and distance function for each property.</returns>
+    /// <returns>A list of tuples containing the column name, index kind, distance function, and full-text language for each property.</returns>
     /// <remarks>
     /// The default index kind is "Flat", which prevents the creation of an index.
     /// </remarks>
-    public static List<(string column, string kind, string function, bool isVector)> GetIndexInfo(IReadOnlyList<PropertyModel> properties)
+    public static List<(string column, string kind, string function, bool isVector, bool isFullText, string? fullTextLanguage)> GetIndexInfo(IReadOnlyList<PropertyModel> properties)
     {
-        var vectorIndexesToCreate = new List<(string column, string kind, string function, bool isVector)>();
+        var vectorIndexesToCreate = new List<(string column, string kind, string function, bool isVector, bool isFullText, string? fullTextLanguage)>();
         foreach (var property in properties)
         {
             switch (property)
@@ -185,15 +185,21 @@ public static NpgsqlParameter GetNpgsqlParameter(object? value)
                             );
                         }
 
-                        vectorIndexesToCreate.Add((vectorProperty.StorageName, indexKind, distanceFunction, isVector: true));
+                        vectorIndexesToCreate.Add((vectorProperty.StorageName, indexKind, distanceFunction, isVector: true, isFullText: false, fullTextLanguage: null));
                     }
 
                     break;
 
                 case DataPropertyModel dataProperty:
                     if (dataProperty.IsIndexed)
                     {
-                        vectorIndexesToCreate.Add((dataProperty.StorageName, "", "", isVector: false));
+                        vectorIndexesToCreate.Add((dataProperty.StorageName, kind: "", function: "", isVector: false, isFullText: false, fullTextLanguage: null));
+                    }
+
+                    if (dataProperty.IsFullTextIndexed)
+                    {
+                        var language = dataProperty.GetFullTextSearchLanguageOrDefault();
+                        vectorIndexesToCreate.Add((dataProperty.StorageName, kind: "", function: "", isVector: false, isFullText: true, fullTextLanguage: language));
                     }
                     break;
 

dotnet/src/VectorData/PgVector/PostgresServiceCollectionExtensions.cs

@@ -286,8 +286,8 @@ private static void AddAbstractions<TKey, TRecord>(IServiceCollection services,
         services.Add(new ServiceDescriptor(typeof(IVectorSearchable<TRecord>), serviceKey,
             static (sp, key) => sp.GetRequiredKeyedService<PostgresCollection<TKey, TRecord>>(key), lifetime));
 
-        // Once HybridSearch supports get implemented by PostgresCollection,
-        // we need to add IKeywordHybridSearchable abstraction here as well.
+        services.Add(new ServiceDescriptor(typeof(IKeywordHybridSearchable<TRecord>), serviceKey,
+            static (sp, key) => sp.GetRequiredKeyedService<PostgresCollection<TKey, TRecord>>(key), lifetime));
     }
 
     private static PostgresVectorStoreOptions? GetStoreOptions(IServiceProvider sp, Func<IServiceProvider, PostgresVectorStoreOptions?>? optionsProvider)