Implement TDS_BLOB support (#150)

senseibaka · c-j-hughes · commit ac719c1310ac · 2019-10-27T09:14:22.000+10:30
SAP ASE has supported the TDS_BLOB type within parameters since ASE 15.7. This fix adds corresponding support to the driver.
diff --git a/src/AdoNetCore.AseClient/Enum/BlobType.cs b/src/AdoNetCore.AseClient/Enum/BlobType.cs
@@ -0,0 +1,35 @@
+// ReSharper disable InconsistentNaming
+namespace AdoNetCore.AseClient.Enum
+{
+    public enum BlobType : byte
+    {
+        /// <summary>
+        /// Not set
+        /// </summary>
+        BLOB_UNSET = 0x00,
+        /// <summary>
+        /// The fully qualified name of the class (“com.foo.Bar”).
+        /// This is a Character String in the negotiated TDS character set currently in use on this connection.
+        /// </summary>
+        BLOB_FULLY_QUALIFIED_CLASS_NAME = 0x01,
+        /// <summary>
+        /// 4-byte integer (database ID) 4-byte integer(sysextypes number of this class definition in this database).
+        /// Both integers are in the byte-ordering negotiated for this connection.
+        /// </summary>
+        BLOB_INT32_CLASS_ID = 0x02,
+        /// <summary>
+        /// This is long character data and has no ClassID associated with it
+        /// </summary>
+        BLOB_LONGCHAR = 0x03,
+        /// <summary>
+        ///  This is long binary data and has no ClassID associated with it.
+        /// Appears in ribo as BLOB_VARBINARY
+        /// </summary>
+        BLOB_LONGBINARY = 0x04,
+        /// <summary>
+        /// This is unichar data with no ClassID associated with it.
+        /// Appears in ribo as BLOB_UTF16
+        /// </summary>
+        BLOB_UNICHAR = 0x05
+    }
+}
diff --git a/src/AdoNetCore.AseClient/Enum/SerializationType.cs b/src/AdoNetCore.AseClient/Enum/SerializationType.cs
@@ -0,0 +1,27 @@
+namespace AdoNetCore.AseClient.Enum
+{
+    public enum SerializationType : byte
+    {
+        //todo: figure out what these names might actually be
+        /// <summary>
+        /// Use the default serialization associated with the specified <see cref="BlobType"/>.
+        /// Allowed cases:
+        /// <see cref="BlobType.BLOB_LONGCHAR"/> -  Characters are in their native format, the character set of the data is the same as that of all other character data as negotiated on the connection during login.
+        /// <see cref="BlobType.BLOB_LONGBINARY"/> - Binary data in its normal form
+        /// <see cref="BlobType.BLOB_UNICHAR"/> - This is unichar data with normal UTF-16 encoding with byte-order identical to that of the client
+        /// </summary>
+        SER_DEFAULT = 0x00,
+        /// <summary>
+        /// Allowed cases:
+        /// <see cref="BlobType.BLOB_FULLY_QUALIFIED_CLASS_NAME"/> - Native Java Serialization
+        /// <see cref="BlobType.BLOB_INT32_CLASS_ID"/> - Native Java Serialization
+        /// <see cref="BlobType.BLOB_UNICHAR"/> - This is unichar data in its UTF-8 encoding.
+        /// </summary>
+        SER_SPECIAL1 = 0x01,
+        /// <summary>
+        /// Allowed cases:
+        /// <see cref="BlobType.BLOB_UNICHAR"/> - This is unichar data in SCSU (compressed) encoding
+        /// </summary>
+        SER_SPECIAL2 = 0x02
+    }
+}
diff --git a/src/AdoNetCore.AseClient/Internal/FormatItem.cs b/src/AdoNetCore.AseClient/Internal/FormatItem.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Data;
 using System.Diagnostics;
 using System.IO;
 using System.Text;
@@ -18,7 +19,13 @@ internal class FormatItem
         public int UserType { get; set; }
         public TdsDataType DataType { get; set; }
         public int? Length { get; set; }
+        /// <summary>
+        /// Relates to TDS_NUMN and TDS_DECN
+        /// </summary>
         public byte? Precision { get; set; }
+        /// <summary>
+        /// Relates to TDS_NUMN and TDS_DECN
+        /// </summary>
         public byte? Scale { get; set; }
         public string LocaleInfo { get; set; }
 
@@ -46,7 +53,15 @@ public string ParameterName
         /// <summary>
         /// Relates to TDS_BLOB
         /// </summary>
-        public string ClassId { get; set; }
+        public BlobType BlobType { get; set; }
+        /// <summary>
+        /// Relates to TDS_BLOB
+        /// </summary>
+        public byte[] ClassId { get; set; }
+        /// <summary>
+        /// Relates to TDS_BLOB
+        /// </summary>
+        public SerializationType SerializationType { get; set; }
 
         public static FormatItem CreateForParameter(AseParameter parameter, DbEnvironment env)
         {
@@ -61,9 +76,23 @@ public static FormatItem CreateForParameter(AseParameter parameter, DbEnvironmen
                 IsNullable = parameter.IsNullable,
                 Length = length,
                 DataType = TypeMap.GetTdsDataType(dbType, parameter.SendableValue, length, parameter.ParameterName),
-                UserType = TypeMap.GetTdsUserType(dbType),
+                UserType = TypeMap.GetTdsUserType(dbType)
             };
 
+            //fixup the FormatItem's BlobType for strings and byte arrays
+            if (format.DataType == TdsDataType.TDS_BLOB)
+            {
+                switch (parameter.DbType)
+                {
+                    case DbType.String:
+                        format.BlobType = BlobType.BLOB_UNICHAR;
+                        break;
+                    case DbType.Binary:
+                        format.BlobType = BlobType.BLOB_LONGBINARY;
+                        break;
+                }
+            }
+            
             //fixup the FormatItem's length,scale,precision for decimals
             if (format.IsDecimalType)
             {
@@ -175,6 +204,10 @@ private static void ReadTypeInfo(FormatItem format, Stream stream, Encoding enc)
                 case TdsDataType.TDS_LONGBINARY:
                     format.Length = stream.ReadInt();
                     break;
+                case TdsDataType.TDS_BLOB:
+                    format.BlobType = (BlobType)stream.ReadByte();
+                    format.ClassId = stream.ReadNullableUShortLengthPrefixedByteArray();
+                    break;
                 case TdsDataType.TDS_DECN:
                 case TdsDataType.TDS_NUMN:
                     format.Length = stream.ReadByte();
@@ -268,6 +301,11 @@ public void WriteForParameter(Stream stream, Encoding enc, TokenType srcTokenTyp
                 case TdsDataType.TDS_LONGBINARY:
                     stream.WriteUInt((uint)(Length ?? 0));
                     break;
+                case TdsDataType.TDS_BLOB:
+                    //according to spec, length isn't specified as part of the format token, but as part of the params token
+                    stream.WriteByte((byte)BlobType);
+                    stream.WriteNullableUShortPrefixedByteArray(ClassId);
+                    break;
                 case TdsDataType.TDS_DECN:
                 case TdsDataType.TDS_NUMN:
                     stream.WriteByte((byte)(Length ?? 1));
@@ -417,6 +455,14 @@ public string GetDataTypeName()
                         default:
                             return "binary";
                     }
+                case TdsDataType.TDS_BLOB:
+                    switch (BlobType)
+                    {
+                        case BlobType.BLOB_UNICHAR:
+                            return "unichar";
+                        default:
+                            return "blob";
+                    }
                 default:
                     return string.Empty;
             }
diff --git a/src/AdoNetCore.AseClient/Internal/StreamReadExtensions.cs b/src/AdoNetCore.AseClient/Internal/StreamReadExtensions.cs
@@ -137,6 +137,18 @@ public static byte[] ReadNullableByteLengthPrefixedByteArray(this Stream stream)
             return stream.ReadByteArray(length);
         }
 
+        public static byte[] ReadNullableUShortLengthPrefixedByteArray(this Stream stream)
+        {
+            var length = stream.ReadUShort();
+
+            if (length == 0)
+            {
+                return null;
+            }
+
+            return stream.ReadByteArray(length);
+        }
+
         public static byte[] ReadNullableIntLengthPrefixedByteArray(this Stream stream)
         {
             var length = stream.ReadInt();
diff --git a/src/AdoNetCore.AseClient/Internal/StreamWriteExtensions.cs b/src/AdoNetCore.AseClient/Internal/StreamWriteExtensions.cs
@@ -109,6 +109,26 @@ public static void WriteBytePrefixedByteArray(this Stream stream, byte[] value)
             stream.WriteByte(len);
             stream.Write(value, 0, len);
         }
+        public static void WriteNullableUShortPrefixedByteArray(this Stream stream, byte[] value)
+        {
+            var len = (ushort)(value?.Length ?? 0);
+            stream.WriteUShort(len);
+
+            if (len == 0)
+            {
+                return;
+            }
+
+            stream.Write(value, 0, len);
+        }
+
+        public static void WriteBlobSpecificIntPrefixedByteArray(this Stream stream, byte[] value)
+        {
+            var len = value.Length;
+            var endOfBlobLen = (uint) len | 0x80000000; //highest-order bit set means end of blob data
+            stream.WriteUInt(endOfBlobLen);
+            stream.Write(value, 0, len);
+        }
 
         public static void WriteIntPrefixedByteArray(this Stream stream, byte[] value)
         {
diff --git a/src/AdoNetCore.AseClient/Internal/TypeMap.cs b/src/AdoNetCore.AseClient/Internal/TypeMap.cs
@@ -9,6 +9,9 @@ namespace AdoNetCore.AseClient.Internal
     internal static class TypeMap
     {
         private const int VarLongBoundary = 255;
+        //Above this length, send strings as TDS_BLOBs
+        private const int StringAsBlobBoundary = 8192;
+        private const int BinaryAsBlobBoundary = 16384;
 
         private static readonly Dictionary<DbType, Func<object, int, TdsDataType>> DbToTdsMap = new Dictionary<DbType, Func<object, int, TdsDataType>>
         {
@@ -21,11 +24,16 @@ internal static class TypeMap
             {DbType.UInt32, (value, length) => value == DBNull.Value ? TdsDataType.TDS_UINTN : TdsDataType.TDS_UINT4},
             {DbType.Int64, (value, length) => value == DBNull.Value ? TdsDataType.TDS_INTN : TdsDataType.TDS_INT8},
             {DbType.UInt64, (value, length) => value == DBNull.Value ? TdsDataType.TDS_UINTN : TdsDataType.TDS_UINT8},
-            {DbType.String, (value, length) => TdsDataType.TDS_LONGBINARY},
+            {DbType.String, (value, length) => length <= StringAsBlobBoundary ? TdsDataType.TDS_LONGBINARY : TdsDataType.TDS_BLOB},
             {DbType.StringFixedLength, (value, length) => TdsDataType.TDS_LONGBINARY},
             {DbType.AnsiString, (value, length) => length <= VarLongBoundary ? TdsDataType.TDS_VARCHAR : TdsDataType.TDS_LONGCHAR},
             {DbType.AnsiStringFixedLength, (value, length) => length <= VarLongBoundary ? TdsDataType.TDS_VARCHAR : TdsDataType.TDS_LONGCHAR},
-            {DbType.Binary, (value, length) => length <= VarLongBoundary ? TdsDataType.TDS_BINARY : TdsDataType.TDS_LONGBINARY},
+            {DbType.Binary, (value, length) =>
+                length <= BinaryAsBlobBoundary
+                    ? length <= VarLongBoundary
+                        ? TdsDataType.TDS_BINARY
+                        : TdsDataType.TDS_LONGBINARY
+                    : TdsDataType.TDS_BLOB},
             {DbType.Guid, (value, length) => TdsDataType.TDS_BINARY},
             {DbType.Decimal, (value, length) => TdsDataType.TDS_NUMN},
             {DbType.Currency, (value, length) => TdsDataType.TDS_MONEYN},
diff --git a/src/AdoNetCore.AseClient/Internal/ValueWriter.cs b/src/AdoNetCore.AseClient/Internal/ValueWriter.cs
@@ -86,6 +86,7 @@ private static T Cast<T>(object value, FormatItem format, Encoding enc)
             { TdsDataType.TDS_VARBINARY, WriteTDS_VARBINARY },
             { TdsDataType.TDS_BINARY, WriteTDS_BINARY },
             { TdsDataType.TDS_LONGBINARY, WriteTDS_LONGBINARY },
+            { TdsDataType.TDS_BLOB, WriteTDS_BLOB },
             { TdsDataType.TDS_DECN, WriteTDS_DECN },
             { TdsDataType.TDS_NUMN, WriteTDS_NUMN },
             { TdsDataType.TDS_DATETIME, WriteTDS_DATETIME },
@@ -300,6 +301,29 @@ private static void WriteTDS_LONGBINARY(object value, Stream stream, FormatItem
                 }
             }
         }
+        private static void WriteTDS_BLOB(object value, Stream stream, FormatItem format, Encoding enc)
+        {
+            //byte serialization type
+            //short-prefixed class id
+            //n chunks of data
+            //    4-byte datalen (highest-order bit indicates if there are more chunks)
+            //    data
+            switch (value)
+            {
+                case string s:
+                    stream.WriteByte((byte)SerializationType.SER_DEFAULT);
+                    stream.WriteNullableUShortPrefixedByteArray(format.ClassId);
+                    stream.WriteBlobSpecificIntPrefixedByteArray(Encoding.Unicode.GetBytes(s));
+                    break;
+                case byte[] ba:
+                    stream.WriteByte((byte)SerializationType.SER_DEFAULT);
+                    stream.WriteNullableUShortPrefixedByteArray(format.ClassId);
+                    stream.WriteBlobSpecificIntPrefixedByteArray(ba);
+                    break;
+                default:
+                    throw new AseException($"TDS_BLOB support for {value.GetType().Name} not yet implemented");
+            }
+        }
 
         private static void WriteTDS_DECN(object value, Stream stream, FormatItem format, Encoding enc)
         {
diff --git a/test/AdoNetCore.AseClient.Tests/AdoNetCore.AseClient.Tests.csproj b/test/AdoNetCore.AseClient.Tests/AdoNetCore.AseClient.Tests.csproj
@@ -1,6 +1,6 @@
-﻿<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFrameworks>netcoreapp1.0;netcoreapp1.1;netcoreapp2.0;netcoreapp2.1;net46</TargetFrameworks>
+    <TargetFrameworks>netcoreapp1.0;netcoreapp1.1;netcoreapp2.0;netcoreapp2.1;netcoreapp2.2;net46</TargetFrameworks>
     <IsPackable>false</IsPackable>
   </PropertyGroup>
   <ItemGroup>
@@ -23,13 +23,13 @@
   <PropertyGroup Condition="'$(TargetFramework)' == 'net46'">
     <DefineConstants>$(DefineConstants);NET_FRAMEWORK</DefineConstants>
   </PropertyGroup>
-  <PropertyGroup Condition="'$(TargetFramework)' == 'netcoreapp2.1' Or '$(TargetFramework)' == 'netcoreapp2.0' Or '$(TargetFramework)' == 'netcoreapp1.1' Or '$(TargetFramework)' == 'netcoreapp1.0'">
+  <PropertyGroup Condition="'$(TargetFramework)' == 'netcoreapp2.2' Or '$(TargetFramework)' == 'netcoreapp2.1' Or '$(TargetFramework)' == 'netcoreapp2.0' Or '$(TargetFramework)' == 'netcoreapp1.1' Or '$(TargetFramework)' == 'netcoreapp1.0'">
     <DefineConstants>$(DefineConstants);NET_CORE</DefineConstants>
   </PropertyGroup>
-  <PropertyGroup Condition="'$(TargetFramework)' == 'netcoreapp2.1' Or '$(TargetFramework)' == 'netcoreapp2.0' Or '$(TargetFramework)' == 'net46' Or '$(TargetFramework)' == 'netstandard2.0'">
+  <PropertyGroup Condition="'$(TargetFramework)' == 'netcoreapp2.2' Or '$(TargetFramework)' == 'netcoreapp2.1' Or '$(TargetFramework)' == 'netcoreapp2.0' Or '$(TargetFramework)' == 'net46' Or '$(TargetFramework)' == 'netstandard2.0'">
     <DefineConstants>$(DefineConstants);ENABLE_SYSTEM_DATA_COMMON_EXTENSIONS</DefineConstants>
   </PropertyGroup>
-  <PropertyGroup Condition="'$(TargetFramework)' == 'netcoreapp2.1' Or '$(TargetFramework)' == 'net46'">
+  <PropertyGroup Condition="'$(TargetFramework)' == 'netcoreapp2.2' Or '$(TargetFramework)' == 'netcoreapp2.1' Or '$(TargetFramework)' == 'net46'">
     <DefineConstants>$(DefineConstants);ENABLE_DB_PROVIDERFACTORY</DefineConstants>
   </PropertyGroup>
   <ItemGroup Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">
diff --git a/test/AdoNetCore.AseClient.Tests/Integration/Insert/BinaryTests.cs b/test/AdoNetCore.AseClient.Tests/Integration/Insert/BinaryTests.cs
@@ -0,0 +1,92 @@
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Data.Common;
+using AdoNetCore.AseClient.Internal;
+using AdoNetCore.AseClient.Tests.ConnectionProvider;
+using Dapper;
+using NUnit.Framework;
+
+namespace AdoNetCore.AseClient.Tests.Integration.Insert
+{
+    [Category("basic")]
+#if NET_FRAMEWORK
+    [TestFixture(typeof(SapConnectionProvider), Explicit = true, Reason = "SAP AseClient tests are run for compatibility purposes.")]
+#endif
+    [TestFixture(typeof(CoreFxConnectionProvider))]
+    public class BinaryTests<T> where T : IConnectionProvider
+    {
+        private DbConnection GetConnection()
+        {
+            return Activator.CreateInstance<T>().GetConnection(ConnectionStrings.Pooled);
+        }
+
+        private const string SetUpSql = @"create table [dbo].[insert_image_tests] (image_field image null)";
+        private const string CleanUpSql = @"IF EXISTS(SELECT 1 FROM sysobjects WHERE name = 'insert_image_tests')
+BEGIN
+    drop table [dbo].[insert_image_tests]
+END";
+
+        [SetUp]
+        public void Setup()
+        {
+            Logger.Enable();
+
+            using (var connection = GetConnection())
+            {
+                connection.Execute(CleanUpSql);
+                connection.Execute(SetUpSql);
+            }
+        }
+
+        [TearDown]
+        public void TearDown()
+        {
+            using (var connection = GetConnection())
+            {
+                connection.Execute(CleanUpSql);
+            }
+        }
+
+        public static IEnumerable<TestCaseData> Insert_Parameter_Cases()
+        {
+            yield return new TestCaseData(1);
+            yield return new TestCaseData(10);
+            yield return new TestCaseData(100);
+            yield return new TestCaseData(127);
+            yield return new TestCaseData(1000);
+            yield return new TestCaseData(8192);
+            yield return new TestCaseData(8193);
+            yield return new TestCaseData(10000);
+            yield return new TestCaseData(16384);
+            yield return new TestCaseData(16385);
+            yield return new TestCaseData(100000);
+            yield return new TestCaseData(1000000);
+        }
+
+        [TestCaseSource(nameof(Insert_Parameter_Cases))]
+        public void Insert_Parameter_Dapper(int count)
+        {
+            var value = new byte[count];
+            using (var connection = GetConnection())
+            {
+                connection.Execute("set textsize 1000000");
+                var p = new DynamicParameters();
+                p.Add("@image_field", value, DbType.Binary);
+                connection.Execute("insert into [dbo].[insert_image_tests] (image_field) values (@image_field)", p);
+                var insertedLength = connection.QuerySingle<int>("select top 1 datalength(image_field) from [dbo].[insert_image_tests]");
+                Assert.AreEqual(value.Length, insertedLength);
+            }
+
+            Insert_Parameter_VerifyResult(GetConnection, "insert_image_tests", "image_field", value);
+        }
+
+        private void Insert_Parameter_VerifyResult(Func<DbConnection> getConnection, string table, string field, byte[] expected)
+        {
+            using (var connection = getConnection())
+            {
+                Assert.AreEqual(expected, connection.QuerySingle<byte[]>($"select top 1 {field} from [dbo].[{table}]"));
+            }
+        }
+    }
+}
diff --git a/test/AdoNetCore.AseClient.Tests/Integration/Insert/TextTests.cs b/test/AdoNetCore.AseClient.Tests/Integration/Insert/TextTests.cs
