Supplementary-character-aware collations now have correct behavior with various string functions.

Author: RyanLamansky

SqlServerSimulator.Tests/CollationDeclaredColumnTests.cs

@@ -979,5 +979,67 @@ public void CastAsCharN_WithPostfixCollateUtf8_MatchesByteBudget(string input, i
         AreEqual(expectedHex, actualHex);
     }
 
+    /// <summary>
+    /// Scalar functions switch indexing semantics on the input value's
+    /// collation: code-unit semantics under non-<c>_SC_</c>, codepoint
+    /// semantics under <c>_SC_</c>. All eight affected functions
+    /// probe-confirmed against SQL Server 2025 (2026-05-21) using a single
+    /// supplementary character (😀 = U+1F600) as the test fixture.
+    /// </summary>
+    [TestMethod]
+    [DataRow("len(N'😀' collate Latin1_General_100_CI_AS)", "2", DisplayName = "LEN non-SC = 2 code units")]
+    [DataRow("len(N'😀' collate Latin1_General_100_CI_AS_SC_UTF8)", "1", DisplayName = "LEN SC = 1 codepoint")]
+    [DataRow("len(N'a😀b' collate Latin1_General_100_CI_AS)", "4", DisplayName = "LEN non-SC a😀b = 4")]
+    [DataRow("len(N'a😀b' collate Latin1_General_100_CI_AS_SC_UTF8)", "3", DisplayName = "LEN SC a😀b = 3")]
+    [DataRow("charindex(N'X', N'😀X' collate Latin1_General_100_CI_AS)", "3", DisplayName = "CHARINDEX non-SC = position 3")]
+    [DataRow("charindex(N'X', N'😀X' collate Latin1_General_100_CI_AS_SC_UTF8)", "2", DisplayName = "CHARINDEX SC = position 2")]
+    [DataRow("patindex(N'%X%', N'😀X' collate Latin1_General_100_CI_AS)", "3", DisplayName = "PATINDEX non-SC = 3")]
+    [DataRow("patindex(N'%X%', N'😀X' collate Latin1_General_100_CI_AS_SC_UTF8)", "2", DisplayName = "PATINDEX SC = 2")]
+    [DataRow("unicode(N'😀' collate Latin1_General_100_CI_AS)", "55357", DisplayName = "UNICODE non-SC = 55357 high surrogate")]
+    [DataRow("unicode(N'😀' collate Latin1_General_100_CI_AS_SC_UTF8)", "128512", DisplayName = "UNICODE SC = 128512 codepoint")]
+    public void SupplementaryCharFunctions_PositionAndLengthDispatch(string expression, string expectedScalar)
+        => AreEqual(int.Parse(expectedScalar), new Simulation().ExecuteScalar($"select {expression}"));
+
+    /// <summary>
+    /// SUBSTRING / LEFT / RIGHT / REVERSE / STUFF return string results
+    /// whose UTF-16 byte content differs between SC and non-SC. Compares
+    /// the .NET string char-by-char to the expected UTF-16 LE byte hex —
+    /// lone surrogates that real SQL Server preserves under non-SC are
+    /// also preserved by the simulator (the nvarchar Encode/Decode path
+    /// byte-copies directly, bypassing <c>Encoding.Unicode</c>'s lone-
+    /// surrogate replacement).
+    /// </summary>
+    [TestMethod]
+    [DataRow("substring(N'😀X' collate Latin1_General_100_CI_AS, 1, 1)", "3DD8", DisplayName = "SUBSTRING non-SC pos 1 len 1 = lone high surrogate")]
+    [DataRow("substring(N'😀X' collate Latin1_General_100_CI_AS_SC_UTF8, 1, 1)", "3DD800DE", DisplayName = "SUBSTRING SC pos 1 len 1 = full emoji")]
+    [DataRow("substring(N'😀X' collate Latin1_General_100_CI_AS, 2, 1)", "00DE", DisplayName = "SUBSTRING non-SC pos 2 len 1 = lone low surrogate")]
+    [DataRow("substring(N'😀X' collate Latin1_General_100_CI_AS_SC_UTF8, 2, 1)", "5800", DisplayName = "SUBSTRING SC pos 2 len 1 = X")]
+    [DataRow("left(N'😀X' collate Latin1_General_100_CI_AS, 1)", "3DD8", DisplayName = "LEFT non-SC")]
+    [DataRow("left(N'😀X' collate Latin1_General_100_CI_AS_SC_UTF8, 1)", "3DD800DE", DisplayName = "LEFT SC")]
+    [DataRow("right(N'X😀' collate Latin1_General_100_CI_AS, 1)", "00DE", DisplayName = "RIGHT non-SC")]
+    [DataRow("right(N'X😀' collate Latin1_General_100_CI_AS_SC_UTF8, 1)", "3DD800DE", DisplayName = "RIGHT SC")]
+    [DataRow("reverse(N'😀X' collate Latin1_General_100_CI_AS)", "580000DE3DD8", DisplayName = "REVERSE non-SC tears surrogate pair")]
+    [DataRow("reverse(N'😀X' collate Latin1_General_100_CI_AS_SC_UTF8)", "58003DD800DE", DisplayName = "REVERSE SC keeps emoji intact")]
+    [DataRow("stuff(N'😀X' collate Latin1_General_100_CI_AS, 1, 1, N'Y')", "590000DE5800", DisplayName = "STUFF non-SC replaces 1 code unit (high surrogate)")]
+    [DataRow("stuff(N'😀X' collate Latin1_General_100_CI_AS_SC_UTF8, 1, 1, N'Y')", "59005800", DisplayName = "STUFF SC replaces 1 codepoint (whole emoji)")]
+    public void SupplementaryCharFunctions_ResultByteContent(string expression, string expectedHex)
+    {
+        var result = (string)new Simulation().ExecuteScalar($"select {expression}")!;
+        var actualHex = AsUtf16LeHex(result);
+        AreEqual(expectedHex, actualHex);
+    }
+
+    /// <summary>Renders a .NET string as UTF-16 LE byte hex, preserving lone surrogates.</summary>
+    private static string AsUtf16LeHex(string s)
+    {
+        var bytes = new byte[s.Length * 2];
+        for (var i = 0; i < s.Length; i++)
+        {
+            bytes[i * 2] = (byte)(s[i] & 0xFF);
+            bytes[(i * 2) + 1] = (byte)((s[i] >> 8) & 0xFF);
+        }
+        return Convert.ToHexString(bytes);
+    }
+
     private static void IsNull(object? value) => Microsoft.VisualStudio.TestTools.UnitTesting.Assert.IsNull(value is DBNull ? null : value);
 }

SqlServerSimulator/Collation.cs

@@ -283,14 +283,14 @@ private protected Collation()
     /// <c>collations.md</c>.</summary>
     internal static readonly CultureCollation Latin1General100CiAsScUtf8 = new(
         "Latin1_General_100_CI_AS_SC_UTF8", CultureInfo.InvariantCulture.Name, caseSensitive: false,
-        storageEncoding: Encoding.UTF8);
+        storageEncoding: Encoding.UTF8, isSupplementaryCharacterAware: true);
 
     /// <summary>"Latin1_General_100_CS_AS_SC_UTF8" — case-sensitive UTF-8
     /// variant. Sort / compare matches <see cref="Latin1GeneralCsAs"/>;
     /// <c>StorageEncoding</c> is UTF-8.</summary>
     internal static readonly CultureCollation Latin1General100CsAsScUtf8 = new(
         "Latin1_General_100_CS_AS_SC_UTF8", CultureInfo.InvariantCulture.Name, caseSensitive: true,
-        storageEncoding: Encoding.UTF8);
+        storageEncoding: Encoding.UTF8, isSupplementaryCharacterAware: true);
 
     /// <summary>"Latin1_General_100_BIN2_UTF8" — binary UTF-8 variant.
     /// On nvarchar / nchar storage the comparer body is
@@ -473,6 +473,25 @@ internal static (Collation Collation, Coercibility Coercibility)? Resolve(SqlTyp
     /// </summary>
     internal virtual Collation ForVarcharStorage() => this;
 
+    /// <summary>
+    /// True for collations whose name carries the <c>_SC_</c>
+    /// (supplementary-character-aware) flag. Real SQL Server's text functions
+    /// — <c>LEN</c>, <c>SUBSTRING</c>, <c>LEFT</c>, <c>RIGHT</c>,
+    /// <c>CHARINDEX</c>, <c>PATINDEX</c>, <c>STUFF</c>, <c>REVERSE</c>,
+    /// <c>UNICODE</c> — switch from UTF-16 code-unit semantics (non-SC) to
+    /// Unicode-codepoint semantics (SC) based on the input value's collation:
+    /// <c>LEN(N'😀')</c> returns 2 (code units) under non-SC and 1 (codepoint)
+    /// under SC; <c>SUBSTRING(N'😀X', 1, 1)</c> returns a lone high surrogate
+    /// under non-SC and the full emoji under SC. The simulator's default is
+    /// <see langword="false"/> (code-unit semantics, matching non-SC); the
+    /// two recognized <c>_SC_UTF8</c> collations override to
+    /// <see langword="true"/>. Sort behavior is governed separately by the
+    /// individual collations' <see cref="Compare"/> bodies — see the "Known
+    /// gaps" entry in <c>collations.md</c> for the residual non-<c>_SC_</c>
+    /// sort divergence on pre-v100 collations.
+    /// </summary>
+    internal virtual bool IsSupplementaryCharacterAware => false;
+
     /// <summary>
     /// The byte encoding the storage layer uses when this collation is
     /// pinned on a <c>varchar</c> / <c>char</c> column. Default is CP1252,
@@ -863,12 +882,15 @@ internal sealed class CultureCollation : Collation
 
         private readonly Encoding storageEncoding;
 
-        internal CultureCollation(string name, string cultureName, bool caseSensitive, bool kanaTypeSensitive = false, bool widthSensitive = false, Encoding? storageEncoding = null)
+        private readonly bool isSupplementaryCharacterAware;
+
+        internal CultureCollation(string name, string cultureName, bool caseSensitive, bool kanaTypeSensitive = false, bool widthSensitive = false, Encoding? storageEncoding = null, bool isSupplementaryCharacterAware = false)
         {
             this.name = name;
             this.caseSensitive = caseSensitive;
             this.compareInfo = CultureInfo.GetCultureInfo(cultureName).CompareInfo;
             this.storageEncoding = storageEncoding ?? CharSqlType.Cp1252Encoder;
+            this.isSupplementaryCharacterAware = isSupplementaryCharacterAware;
             var baseOpts = caseSensitive
                 ? CompareOptions.None
                 : CompareOptions.IgnoreCase;
@@ -891,6 +913,8 @@ internal CultureCollation(string name, string cultureName, bool caseSensitive, b
 
         internal override Encoding StorageEncoding => this.storageEncoding;
 
+        internal override bool IsSupplementaryCharacterAware => this.isSupplementaryCharacterAware;
+
         public override int Compare(string? x, string? y) =>
             x is null
                 ? (y is null ? 0 : -1)

SqlServerSimulator/Parser/Expressions/CharIndex.cs

@@ -34,22 +34,34 @@ public override SqlValue Run(RuntimeContext runtime)
         if (!SqlType.IsStringCategory(n.Type) || !SqlType.IsStringCategory(h.Type))
             throw new NotSupportedException("CHARINDEX expects string operands.");
 
-        var startIndex = 0;
+        var needleStr = n.AsString;
+        var haystackStr = h.AsString;
+        // CHARINDEX indexes in code units under non-SC collations and in
+        // codepoints under _SC_. Probe-confirmed against SQL Server 2025:
+        // CHARINDEX(N'X', N'😀X') = 3 under non-SC (surrogate pair occupies
+        // positions 1-2) and = 2 under _SC_UTF8 (emoji = position 1). The
+        // start argument is in the same unit as the result.
+        var isSc = h.Type.Collation?.IsSupplementaryCharacterAware == true;
+        var startUnits = 0;
         if (start is not null)
         {
             var startValue = start.Run(runtime);
             if (startValue.IsNull)
                 return SqlValue.Null(SqlType.Int32);
-            startIndex = Math.Max(0, startValue.CoerceTo(SqlType.Int32).AsInt32 - 1);
+            startUnits = Math.Max(0, startValue.CoerceTo(SqlType.Int32).AsInt32 - 1);
         }
-
-        var needleStr = n.AsString;
-        var haystackStr = h.AsString;
-        if (startIndex >= haystackStr.Length)
+        var startCu = isSc
+            ? SupplementaryCharacters.CodepointToCodeUnit(haystackStr, startUnits)
+            : startUnits;
+        if (startCu >= haystackStr.Length)
             return SqlValue.FromInt32(0);
 
-        var found = haystackStr.IndexOf(needleStr, startIndex, StringComparison.InvariantCultureIgnoreCase);
-        return SqlValue.FromInt32(found < 0 ? 0 : found + 1);
+        var foundCu = haystackStr.IndexOf(needleStr, startCu, StringComparison.InvariantCultureIgnoreCase);
+        return SqlValue.FromInt32(foundCu < 0
+            ? 0
+            : isSc
+                ? SupplementaryCharacters.CodeUnitToCodepoint(haystackStr, foundCu) + 1
+                : foundCu + 1);
     }
 
     public override SqlType GetSqlType(Func<MultiPartName, SqlType> resolveColumnType) => SqlType.Int32;

SqlServerSimulator/Parser/Expressions/Left.cs

@@ -34,7 +34,10 @@ public override SqlValue Run(RuntimeContext runtime)
             throw SimulatedSqlException.NegativeLengthNotAllowed("LEFT");
 
         var input = s.AsString;
-        return SqlValue.FromString(s.Type, len >= input.Length ? input : input[..len]);
+        var result = s.Type.Collation?.IsSupplementaryCharacterAware == true
+            ? SupplementaryCharacters.LeftByCodepoints(input, len)
+            : len >= input.Length ? input : input[..len];
+        return SqlValue.FromString(s.Type, result);
     }
 
     public override SqlType GetSqlType(Func<MultiPartName, SqlType> resolveColumnType) => source.GetSqlType(resolveColumnType);

SqlServerSimulator/Parser/Expressions/Length.cs

@@ -8,9 +8,13 @@ namespace SqlServerSimulator.Parser.Expressions;
 /// </summary>
 /// <remarks>
 /// SQL Server's quirk: <c>LEN</c> ignores trailing spaces but not leading
-/// spaces. The simulator measures characters in <see cref="string.Length"/>
-/// terms (UCS-2 code units for nvarchar; bytes-equal-chars for varchar in
-/// CP1252), matching SQL Server.
+/// spaces. The simulator measures by code unit (<see cref="string.Length"/>)
+/// under non-SC collations and by Unicode codepoint (rune count) under
+/// <c>_SC_</c>-flagged collations — probe-confirmed against SQL Server
+/// 2025: <c>LEN(N'😀')</c> = 2 under default / <c>Latin1_General_100_CI_AS</c>
+/// (code units, surrogate pair counts as 2) and = 1 under
+/// <c>Latin1_General_100_CI_AS_SC_UTF8</c> (the supplementary codepoint
+/// counts as 1).
 /// Reference: https://learn.microsoft.com/en-us/sql/t-sql/functions/len-transact-sql
 /// </remarks>
 internal sealed class Length(ParserContext context) : Expression
@@ -28,7 +32,10 @@ public override SqlValue Run(RuntimeContext runtime)
         if (!SqlType.IsStringCategory(value.Type))
             throw new NotSupportedException($"LEN expects a string operand; got {value.Type}.");
         var trimmed = value.AsString.TrimEnd(' ');
-        return SqlValue.FromInt32(trimmed.Length);
+        var length = value.Type.Collation?.IsSupplementaryCharacterAware == true
+            ? SupplementaryCharacters.CodepointCount(trimmed)
+            : trimmed.Length;
+        return SqlValue.FromInt32(length);
     }
 
     public override SqlType GetSqlType(Func<MultiPartName, SqlType> resolveColumnType) => SqlType.Int32;

SqlServerSimulator/Parser/Expressions/PatIndex.cs

@@ -57,8 +57,14 @@ public override SqlValue Run(RuntimeContext runtime)
             : p.CoerceTo(s.Type).AsString;
 
         var regex = LikePatternBuilder.BuildForPatIndex(patternString);
-        var match = regex.Match(s.AsString);
-        var position = match.Success ? (long)match.Index + 1 : 0L;
+        var subjectStr = s.AsString;
+        var match = regex.Match(subjectStr);
+        // Result position is code-unit-based under non-SC, codepoint-based
+        // under _SC_ — matches CHARINDEX dispatch on the subject's collation.
+        var position = !match.Success ? 0L
+            : s.Type.Collation?.IsSupplementaryCharacterAware == true
+                ? (long)SupplementaryCharacters.CodeUnitToCodepoint(subjectStr, match.Index) + 1
+                : (long)match.Index + 1;
         return isBig ? SqlValue.FromInt64(position) : SqlValue.FromInt32((int)position);
     }
 

SqlServerSimulator/Parser/Expressions/Reverse.cs

@@ -4,8 +4,14 @@ namespace SqlServerSimulator.Parser.Expressions;
 
 /// <summary>
 /// SQL <c>REVERSE(x)</c>: returns the source with its characters in reverse
-/// order. Surrogate pairs are reversed as a unit (their high/low order is
-/// preserved) so emoji and supplementary-plane characters don't get torn.
+/// order. Reverses by code unit under non-<c>_SC_</c> collations (surrogate
+/// pairs are split — the high/low halves swap positions) and by codepoint
+/// under <c>_SC_</c> collations (surrogate pairs stay intact). Probe-
+/// confirmed against SQL Server 2025: <c>REVERSE(N'😀X')</c> on a non-SC
+/// collation returns <c>X</c> followed by the swapped surrogate bytes,
+/// matching <c>0x580000DE3DD8</c>; the same call on
+/// <c>Latin1_General_100_CI_AS_SC_UTF8</c> returns <c>X</c> followed by an
+/// intact emoji (<c>0x58003DD800DE</c>).
 /// </summary>
 /// <remarks>Reference: https://learn.microsoft.com/en-us/sql/t-sql/functions/reverse-transact-sql</remarks>
 internal sealed class Reverse(ParserContext context) : Expression
@@ -21,27 +27,10 @@ public override SqlValue Run(RuntimeContext runtime)
             throw new NotSupportedException($"REVERSE expects a string operand; got {value.Type}.");
 
         var input = value.AsString;
-        var reversed = new char[input.Length];
-        var sourceIndex = 0;
-        var destIndex = input.Length;
-        while (sourceIndex < input.Length)
-        {
-            var c = input[sourceIndex];
-            if (char.IsHighSurrogate(c) && sourceIndex + 1 < input.Length && char.IsLowSurrogate(input[sourceIndex + 1]))
-            {
-                destIndex -= 2;
-                reversed[destIndex] = c;
-                reversed[destIndex + 1] = input[sourceIndex + 1];
-                sourceIndex += 2;
-            }
-            else
-            {
-                destIndex--;
-                reversed[destIndex] = c;
-                sourceIndex++;
-            }
-        }
-        return SqlValue.FromString(value.Type, new string(reversed));
+        var reversed = value.Type.Collation?.IsSupplementaryCharacterAware == true
+            ? SupplementaryCharacters.ReverseByCodepoints(input)
+            : SupplementaryCharacters.ReverseByCodeUnits(input);
+        return SqlValue.FromString(value.Type, reversed);
     }
 
     public override SqlType GetSqlType(Func<MultiPartName, SqlType> resolveColumnType) => source.GetSqlType(resolveColumnType);

SqlServerSimulator/Parser/Expressions/Right.cs

@@ -34,7 +34,10 @@ public override SqlValue Run(RuntimeContext runtime)
             throw SimulatedSqlException.NegativeLengthNotAllowed("RIGHT");
 
         var input = s.AsString;
-        return SqlValue.FromString(s.Type, len >= input.Length ? input : input[(input.Length - len)..]);
+        var result = s.Type.Collation?.IsSupplementaryCharacterAware == true
+            ? SupplementaryCharacters.RightByCodepoints(input, len)
+            : len >= input.Length ? input : input[(input.Length - len)..];
+        return SqlValue.FromString(s.Type, result);
     }
 
     public override SqlType GetSqlType(Func<MultiPartName, SqlType> resolveColumnType) => source.GetSqlType(resolveColumnType);

SqlServerSimulator/Parser/Expressions/Stuff.cs

@@ -54,15 +54,23 @@ public override SqlValue Run(RuntimeContext runtime)
         var startIndex = startValue.CoerceTo(SqlType.Int32).AsInt32;
         var len = lengthValue.CoerceTo(SqlType.Int32).AsInt32;
         var s = inputValue.AsString;
+        // STUFF indexes/lengths are code-unit-based under non-SC and
+        // codepoint-based under _SC_. Under _SC_ a delete count of 1
+        // removes one full codepoint (whole emoji) rather than splitting
+        // a surrogate pair. Probe-confirmed against SQL Server 2025.
+        var isSc = inputValue.Type.Collation?.IsSupplementaryCharacterAware == true;
+        var inputUnits = isSc ? SupplementaryCharacters.CodepointCount(s) : s.Length;
 
         // Invalid argument cases all map to NULL silently — matches SQL
         // Server's documented and probed behavior.
-        if (startIndex < 1 || startIndex > s.Length || len < 0)
+        if (startIndex < 1 || startIndex > inputUnits || len < 0)
             return SqlValue.Null(resultType);
 
-        var deleteCount = Math.Min(len, s.Length - (startIndex - 1));
+        var deleteCount = Math.Min(len, inputUnits - (startIndex - 1));
         var insertText = replacementValue.IsNull ? string.Empty : replacementValue.AsString;
-        var result = string.Concat(s.AsSpan(0, startIndex - 1), insertText, s.AsSpan(startIndex - 1 + deleteCount));
+        var sliceStartCu = isSc ? SupplementaryCharacters.CodepointToCodeUnit(s, startIndex - 1) : startIndex - 1;
+        var sliceEndCu = isSc ? SupplementaryCharacters.CodepointToCodeUnit(s, startIndex - 1 + deleteCount) : startIndex - 1 + deleteCount;
+        var result = string.Concat(s.AsSpan(0, sliceStartCu), insertText, s.AsSpan(sliceEndCu));
         return SqlValue.FromString(resultType, result);
     }