Add Support for Vector Data Type in SQL for REST (#3677)

## Why make this change?

- Solves issue #3654 

## What is this change?

These changes add the base to support the Vector data type for REST,
GraphQL, and MCP. This PR focuses mainly on the REST endpoint, ensuring
that it is able to both read and write to the vector columns through it.
Lastly, it also adds support to ensure that OpenAPI is able to show
columns with the Vector data type as `array: [numeric]`.

The following files were changed to show that DAB is able to support the
Vector data type and also change the type internally to system types it
can work with. Such as the `float` type for `DbType`, the `Numeric` type
for `JsonDataType`, and `SINGLE_TYPE` for GraphQL.
- SqlTypeConstants
- TypeHelper.cs
- SqlSchemaConverter.cs

The following file was changed to allow DAB to save the vector columns
as an array type, which is necessary for GraphQL to be able to know if
the column of an entity is a simple type or an array.
- MsSqlMetadataProvider.cs

The following files were changed to parse the information from the user
to a format that can be used by DAB to create the queries for SQL.
- BaseSqlQueryStructure.cs: Parses the array information by creating a
new function that checks for the simple type of the array (e.g If it is
an array of floats it will use the type float) and parse each value
inside the array.
- MsSqlQueryExecutor.cs: Parses the values given by the user and adds
them to an SqlVector object which allows for an easier way to query the
values.

The following file was changed to allow DAB's OpenAPI to support array
types by checking if the main type is `array` and then taking the
subtype and creating the appropriate item with it.
- OpenApiDocumentor.cs

Note: All of these changes are only applicable to Vectors of Float32,
support for Vectors of Float16 will have to wait as support for it in
.NET is still missing.

## How was this tested?

- [x] Integration Tests
- [ ] Unit Tests

Added tests that ensure the following scenarios on REST:
- Read regular vector 
- Read vector with max number of dimensions possible
- Read vector with null value
- Read vector with `$first` argument
- Read vector with `$after` argument
- Read vector with `$filter` argument - fails
- Read vector with `$orderby` argument - fails
- Insert regular vector 
- Insert vector with null value
- Insert vector with number values in scientific format
- Insert vector with number values in string format
- Insert vector with more dimensions than allowed - fails
- Insert vector with non-valid values - fails
- Update vector with new values

## Sample Request(s)
Note: Currently the values will be outputted in scientific notation,
this will be changed in the future
#3680

GET https://localhost:5001/api/dbo_normalvector/
<img width="429" height="609" alt="image"
src="https://github.com/user-attachments/assets/f0c16c4a-4e6a-4199-8d26-33469af2feab"
/>

POST https://localhost:5001/api/dbo_normalvector/
```
{
	"ProductID": 2100,
	"Embedding": [
		1.15202468461012559910005,
		0.00005,
		5.111111111
	]
}
```
<img width="303" height="319" alt="image"
src="https://github.com/user-attachments/assets/3b660ddd-dbe8-4f24-9025-fc526bbd64e4"
/>

PATCH https://localhost:5001/api/dbo_normalvector/ (Adding new row)
```
{
	"ProductID": 3100,
	"Embedding": [
		1.15202468461012559910005,
		0.00005,
		5.111111111
	]
}
```
<img width="472" height="318" alt="image"
src="https://github.com/user-attachments/assets/c22cf689-bf60-4a1f-934b-91bbad2de3ae"
/>

PATCH https://localhost:5001/api/dbo_normalvector/ (Update existing row)
```
{
	"ProductID": 2100,
	"Embedding": [
		25,
		1.2,
		510.35
	]
}
```
<img width="363" height="315" alt="image"
src="https://github.com/user-attachments/assets/7af1259b-834a-4881-a80f-ab011ee1123d"
/>

PUT https://localhost:5001/api/dbo_normalvector/ (Adding new row)
```
{
	"ProductID": 4100,
	"Embedding": [
		0,
		1,
		37
	]
}
```
<img width="310" height="289" alt="image"
src="https://github.com/user-attachments/assets/1aba2a11-5efe-443d-9f89-62b510fe4a42"
/>

PUT https://localhost:5001/api/dbo_normalvector/ (Edit existing row)
```
{
	"ProductID": 2100,
	"Embedding": [
		0.220224962,
		0.66666667,
		37
	]
}
```
<img width="386" height="303" alt="image"
src="https://github.com/user-attachments/assets/abf79182-716c-407f-af2a-0396e26a7c26"
/>

https://localhost:5001/swagger
<img width="348" height="142" alt="image"
src="https://github.com/user-attachments/assets/5d65470b-7df2-4725-a70d-ec9eeb69e7d0"
/>
<img width="302" height="277" alt="image"
src="https://github.com/user-attachments/assets/3b5c08b2-39ed-4bd8-a287-7b6fba3a7684"
/>

---------

Co-authored-by: souvikghosh04 <souvikofficial04@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
Co-authored-by: aaronburtle <93220300+aaronburtle@users.noreply.github.com>

Author: 4 people

config-generators/mssql-commands.txt

@@ -16,6 +16,8 @@ add Broker --config "dab-config.MsSql.json" --source brokers --permissions "anon
 add WebsiteUser --config "dab-config.MsSql.json" --source website_users --permissions "anonymous:create,read,delete,update"
 add WebsiteUser_MM --config "dab-config.MsSql.json" --source website_users_mm --graphql "websiteuser_mm:websiteusers_mm" --permissions "anonymous:*"
 add SupportedType --config "dab-config.MsSql.json" --source type_table --permissions "anonymous:create,read,delete,update"
+add VectorType --config "dab-config.MsSql.json" --source vector_type_table --rest true --graphql false --permissions "anonymous:create,read,delete,update"
+update VectorType --config "dab-config.MsSql.json" --permissions "authenticated:create,read,delete,update"
 add stocks_price --config "dab-config.MsSql.json" --source stocks_price --permissions "authenticated:create,read,update,delete"
 update stocks_price --config "dab-config.MsSql.json" --permissions "anonymous:read"
 update stocks_price --config "dab-config.MsSql.json" --permissions "TestNestedFilterFieldIsNull_ColumnForbidden:read" --fields.exclude "price"

src/Core/Models/SqlTypeConstants.cs

@@ -48,6 +48,7 @@ public static class SqlTypeConstants
         { "datetime2", true },            // SqlDbType.DateTime2
         { "datetimeoffset", true },       // SqlDbType.DateTimeOffset
         { "", false },                    // SqlDbType.Udt and SqlDbType.Structured provided by SQL as empty strings (unsupported)
-        { "numeric", true}                // Not present in SqlDbType, however can be returned by sql functions like LAG and should map to decimal.
+        { "numeric", true},                // Not present in SqlDbType, however can be returned by sql functions like LAG and should map to decimal.
+        { "vector", true }                // SqlDbType.Vector
     };
 }

src/Core/Resolvers/MsSqlQueryBuilder.cs

@@ -666,8 +666,7 @@ AND ty.name IN
                                     N'hierarchyid',
                                     N'sql_variant',
                                     N'xml',
-                                    N'rowversion',
-                                    N'vector'
+                                    N'rowversion'
                                 )
                         ) THEN 1
                         ELSE 0
@@ -712,8 +711,7 @@ AND ty.name IN
                                     N'hierarchyid',
                                     N'sql_variant',
                                     N'xml',
-                                    N'rowversion',
-                                    N'vector'
+                                    N'rowversion'
                                 )
                         )
                     )

src/Core/Resolvers/MsSqlQueryExecutor.cs

@@ -18,6 +18,7 @@
 using Azure.Identity;
 using Microsoft.AspNetCore.Http;
 using Microsoft.Data.SqlClient;
+using Microsoft.Data.SqlTypes;
 using Microsoft.Extensions.Logging;
 
 namespace Azure.DataApiBuilder.Core.Resolvers
@@ -695,6 +696,19 @@ public override SqlCommand PrepareDbCommand(
                         parameter.Size = parameterEntry.Value.Length.Value;
                     }
 
+                    // if sqldbtype is vector then set the value as an SqlVector object
+                    if (parameter.SqlDbType is SqlDbType.Vector)
+                    {
+                        List<float> values = new();
+                        foreach (float val in (Array)parameter.Value)
+                        {
+                            values.Add(val);
+                        }
+
+                        SqlVector<float> value = new(values.ToArray());
+                        parameter.Value = value;
+                    }
+
                     cmd.Parameters.Add(parameter);
                 }
             }

src/Core/Resolvers/QueryExecutor.cs

@@ -13,6 +13,7 @@
 using Azure.DataApiBuilder.Core.Models;
 using Azure.DataApiBuilder.Service.Exceptions;
 using Microsoft.AspNetCore.Http;
+using Microsoft.Data.SqlTypes;
 using Microsoft.Extensions.Logging;
 using Polly;
 using Polly.Retry;
@@ -502,7 +503,7 @@ public async Task<DbResultSet>
                             {
                                 if (!ConfigProvider.GetConfig().MaxResponseSizeLogicEnabled())
                                 {
-                                    dbResultSetRow.Columns.Add(columnName, dbDataReader[columnName]);
+                                    dbResultSetRow.Columns.Add(columnName, GetColumnInformation(dbDataReader, columnName));
                                 }
                                 else
                                 {
@@ -554,7 +555,7 @@ public DbResultSet
                             {
                                 if (!ConfigProvider.GetConfig().MaxResponseSizeLogicEnabled())
                                 {
-                                    dbResultSetRow.Columns.Add(columnName, dbDataReader[columnName]);
+                                    dbResultSetRow.Columns.Add(columnName, GetColumnInformation(dbDataReader, columnName));
                                 }
                                 else
                                 {
@@ -822,7 +823,7 @@ internal int StreamDataIntoDbResultSetRow(DbDataReader dbDataReader, DbResultSet
             {
                 dataRead = columnSize;
                 ValidateSize(availableBytes, dataRead);
-                dbResultSetRow.Columns.Add(columnName, dbDataReader[columnName]);
+                dbResultSetRow.Columns.Add(columnName, GetColumnInformation(dbDataReader, columnName));
             }
 
             return dataRead;
@@ -885,6 +886,18 @@ private void ValidateSize(long availableSizeBytes, long sizeToBeReadBytes)
             }
         }
 
+        /// <summary>
+        /// Helper function to get column information from the DbDataReader and handle special cases like SqlVector<float>.
+        /// </summary>
+        /// <param name="dbDataReader"></param>
+        /// <param name="columnName"></param>
+        /// <returns></returns>
+        private static object GetColumnInformation(DbDataReader dbDataReader, string columnName)
+        {
+            object value = dbDataReader[columnName];
+            return value is SqlVector<float> vector ? vector.Memory : value;
+        }
+
         internal virtual void AddDbExecutionTimeToMiddlewareContext(long time)
         {
             HttpContext? httpContext = HttpContextAccessor?.HttpContext;

src/Core/Resolvers/Sql Query Structures/BaseSqlQueryStructure.cs

@@ -4,6 +4,7 @@
 using System.Data;
 using System.Globalization;
 using System.Net;
+using System.Text.Json;
 using Azure.DataApiBuilder.Auth;
 using Azure.DataApiBuilder.Config.DatabasePrimitives;
 using Azure.DataApiBuilder.Config.ObjectModel;
@@ -452,10 +453,49 @@ protected static object ParseParamAsSystemType(string param, Type systemType)
                 "Guid" => Guid.Parse(param),
                 "TimeOnly" => TimeOnly.Parse(param),
                 "TimeSpan" => TimeOnly.Parse(param),
+                "Single[]" => ParseArrayIntoSystemType(param, systemType),
                 _ => throw new NotSupportedException($"{systemType.Name} is not supported")
             };
         }
 
+        /// <summary>
+        /// Takes the array of the parameter we are going to parse and converts each element to the specified system type.
+        /// </summary>
+        /// <param name="param"></param>
+        /// <param name="systemType"></param>
+        /// <returns></returns>
+        /// <exception cref="NotSupportedException"></exception>
+        /// <exception cref="FormatException"></exception>
+        private static object ParseArrayIntoSystemType(string param, Type systemType)
+        {
+            Type typeOfArray;
+            switch (systemType.Name)
+            {
+                case "Single[]":
+                    typeOfArray = typeof(Single);
+                    break;
+
+                default:
+                    throw new NotSupportedException($"{systemType.Name} is not supported");
+            }
+
+            try
+            {
+                object[] values = JsonSerializer.Deserialize<object[]>(param) ?? Array.Empty<object>();
+                for (int i = 0; i < values.Length; i++)
+                {
+                    string stringValue = values[i]?.ToString() ?? string.Empty;
+                    values[i] = ParseParamAsSystemType(stringValue, typeOfArray);
+                }
+
+                return values;
+            }
+            catch
+            {
+                throw new FormatException($"Expected an array for {systemType.Name} but got an unexpected value");
+            }
+        }
+
         /// <summary>
         /// Very similar to GQLArgumentToDictParams but only extracts the argument names from
         /// the specified field which means that the method does not need a middleware context

src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs

@@ -15,6 +15,7 @@
 using Azure.DataApiBuilder.Service.Exceptions;
 using Azure.DataApiBuilder.Service.GraphQLBuilder;
 using Microsoft.Data.SqlClient;
+using Microsoft.Data.SqlTypes;
 using Microsoft.Extensions.Logging;
 using static Azure.DataApiBuilder.Service.GraphQLBuilder.GraphQLNaming;
 
@@ -120,6 +121,15 @@ protected override void PopulateColumnDefinitionWithHasDefaultAndDbType(
                     columnDefinition.DbType = TypeHelper.GetDbTypeFromSystemType(columnDefinition.SystemType);
 
                     string sqlDbTypeName = (string)columnInfo["DATA_TYPE"];
+
+                    if (columnDefinition.SystemType == typeof(SqlVector<Single>))
+                    {
+                        sqlDbTypeName = "vector";   // Currently the "DATA_TYPE" column returns "varbinary" for vector type columns. This is a known issue https://learn.microsoft.com/en-us/sql/t-sql/data-types/vector-data-type?view=sql-server-ver17&tabs=csharp#known-issues
+                        columnDefinition.IsArrayType = true;
+                        columnDefinition.ElementSystemType = typeof(Single);
+                        columnDefinition.SystemType = columnDefinition.ElementSystemType.MakeArrayType();
+                    }
+
                     if (Enum.TryParse(sqlDbTypeName, ignoreCase: true, out SqlDbType sqlDbType))
                     {
                         // The DbType enum in .NET does not distinguish between VarChar and NVarChar. Both are mapped to DbType.String.

src/Core/Services/OpenAPI/OpenApiDocumentor.cs

@@ -1484,12 +1484,18 @@ private static OpenApiSchema CreateComponentSchema(
                 if (metadataProvider.TryGetBackingColumn(entityName, field, out string? backingColumnValue) && !string.IsNullOrEmpty(backingColumnValue))
                 {
                     string typeMetadata = string.Empty;
+                    string subTypeMetadata = string.Empty;
                     string formatMetadata = string.Empty;
                     string? fieldDescription = null;
 
                     if (dbObject.SourceDefinition.Columns.TryGetValue(backingColumnValue, out ColumnDefinition? columnDef))
                     {
                         typeMetadata = TypeHelper.GetJsonDataTypeFromSystemType(columnDef.SystemType).ToString().ToLower();
+
+                        if (string.Equals(typeMetadata, JsonDataType.Array.ToString().ToLower(), StringComparison.OrdinalIgnoreCase))
+                        {
+                            subTypeMetadata = TypeHelper.GetJsonDataTypeFromSystemType(columnDef.ElementSystemType!).ToString().ToLower();
+                        }
                     }
 
                     if (entityConfig?.Fields != null)
@@ -1502,7 +1508,8 @@ private static OpenApiSchema CreateComponentSchema(
                     {
                         Type = typeMetadata,
                         Format = formatMetadata,
-                        Description = fieldDescription
+                        Description = fieldDescription,
+                        Items = !string.IsNullOrWhiteSpace(subTypeMetadata) ? new OpenApiSchema() { Type = subTypeMetadata } : null
                     });
                 }
             }

src/Core/Services/TypeHelper.cs

@@ -8,6 +8,7 @@
 using Azure.DataApiBuilder.Core.Services.OpenAPI;
 using Azure.DataApiBuilder.Service.Exceptions;
 using HotChocolate.Language;
+using Microsoft.Data.SqlTypes;
 using Microsoft.OData.Edm;
 
 namespace Azure.DataApiBuilder.Core.Services
@@ -46,7 +47,8 @@ public static class TypeHelper
             [typeof(byte[])] = DbType.Binary,
             [typeof(TimeOnly)] = DbType.Time,
             [typeof(TimeSpan)] = DbType.Time,
-            [typeof(object)] = DbType.Object
+            [typeof(object)] = DbType.Object,
+            [typeof(SqlVector<Single>)] = DbType.Single
         };
 
         /// <summary>
@@ -77,7 +79,8 @@ public static class TypeHelper
             [typeof(TimeOnly)] = JsonDataType.String,
             [typeof(object)] = JsonDataType.Object,
             [typeof(DateTime)] = JsonDataType.String,
-            [typeof(DateTimeOffset)] = JsonDataType.String
+            [typeof(DateTimeOffset)] = JsonDataType.String,
+            [typeof(Single[])] = JsonDataType.Array
         };
 
         /// <summary>
@@ -111,7 +114,8 @@ public static class TypeHelper
             [SqlDbType.TinyInt] = typeof(byte),
             [SqlDbType.UniqueIdentifier] = typeof(Guid),
             [SqlDbType.VarBinary] = typeof(byte[]),
-            [SqlDbType.VarChar] = typeof(string)
+            [SqlDbType.VarChar] = typeof(string),
+            [SqlDbType.Vector] = typeof(float)
         };
 
         private static Dictionary<SqlDbType, DbType> _sqlDbDateTimeTypeToDbType = new()

src/Service.Tests/DatabaseSchema-MsSql.sql

@@ -40,6 +40,7 @@ DROP TABLE IF EXISTS stocks;
 DROP TABLE IF EXISTS comics;
 DROP TABLE IF EXISTS brokers;
 DROP TABLE IF EXISTS type_table;
+DROP TABLE IF EXISTS vector_type_table;
 DROP TABLE IF EXISTS trees;
 DROP TABLE IF EXISTS fungi;
 DROP TABLE IF EXISTS empty_table;
@@ -234,6 +235,12 @@ CREATE TABLE type_table(
     uuid_types uniqueidentifier DEFAULT newid()
 );
 
+CREATE TABLE vector_type_table(
+    id int IDENTITY(5001, 1) PRIMARY KEY,
+    vector_data vector(3),
+    vector_data_max vector(1998)
+);
+
 CREATE TABLE trees (
     treeId int PRIMARY KEY,
     species varchar(max),
@@ -616,6 +623,23 @@ VALUES
 INSERT INTO type_table(id, uuid_types) values(10, 'D1D021A8-47B4-4AE4-B718-98E89C41A161');
 SET IDENTITY_INSERT type_table OFF
 
+SET IDENTITY_INSERT vector_type_table ON
+INSERT INTO vector_type_table(id, vector_data)
+VALUES
+    (1, '[0.5, 0.25, 0.75]'),
+    (2, '[1.5, -2.5, 3.5]'),
+    (3, NULL),
+    (4, '[1.0, 2.0, 3.0]'),
+    (5, '[4.0, 5.0, 6.0]'),
+    (6, '[7.0, 8.0, 9.0]');
+
+INSERT INTO vector_type_table(id, vector_data_max)
+VALUES (7, CAST('[' + (
+    SELECT STRING_AGG(CAST(value AS NVARCHAR(MAX)), ',') WITHIN GROUP (ORDER BY value)
+    FROM GENERATE_SERIES(1, 1998)
+) + ']' AS vector(1998)));
+SET IDENTITY_INSERT vector_type_table OFF
+
 SET IDENTITY_INSERT sales ON
 INSERT INTO sales(id, item_name, subtotal, tax) VALUES (1, 'Watch', 249.00, 20.59), (2, 'Montior', 120.50, 11.12);
 SET IDENTITY_INSERT sales OFF