Optimizations and refinement to collations.

Author: RyanLamansky

SqlServerSimulator.Tests.Internal/CollationTests.cs

@@ -22,13 +22,13 @@ namespace SqlServerSimulator;
 [TestClass]
 public sealed class CollationTests
 {
-    private static readonly Collation Latin1General100CiAs = Collation.TryGet("Latin1_General_100_CI_AS")!;
+    private static readonly Collation Latin1General100CiAs = Collation.Get("Latin1_General_100_CI_AS");
 
-    private static readonly Collation Latin1GeneralCiAs = Collation.TryGet("Latin1_General_CI_AS")!;
+    private static readonly Collation Latin1GeneralCiAs = Collation.Get("Latin1_General_CI_AS");
 
-    private static readonly Collation Latin1GeneralCsAs = Collation.TryGet("Latin1_General_CS_AS")!;
+    private static readonly Collation Latin1GeneralCsAs = Collation.Get("Latin1_General_CS_AS");
 
-    private static readonly Collation Latin1GeneralBin = Collation.TryGet("Latin1_General_BIN")!;
+    private static readonly Collation Latin1GeneralBin = Collation.Get("Latin1_General_BIN");
 
     // ---- SQL_Latin1_General_CP1_CI_AS — internal-only contracts ----
 

SqlServerSimulator/Collation.Parser.cs

@@ -1,4 +1,5 @@
 using System.Collections.Concurrent;
+using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 using System.Globalization;
 using System.Text;
@@ -65,6 +66,21 @@ private enum CollationFlags : ushort
                 ? hit
                 : TryParse(name, out var parsed) ? interned.GetOrAdd(parsed.Name, parsed) : null;
 
+    /// <summary>
+    /// Resolves a collation name guaranteed valid at the call site — a
+    /// compile-time constant naming a collation real SQL Server ships. The
+    /// <c>Debug.Assert</c> is a refactoring tripwire: a regression in the
+    /// parser tables fails loud and early under the (Debug-built) test
+    /// suite, where collation resolution is among the hottest paths. Use
+    /// <see cref="TryGet"/> for any name that originates from SQL text.
+    /// </summary>
+    internal static Collation Get(string name)
+    {
+        var collation = TryGet(name);
+        Debug.Assert(collation is not null, $"Collation '{name}' is expected to always resolve but did not.");
+        return collation;
+    }
+
     /// <summary>
     /// True if <paramref name="name"/> is a recognized SQL Server collation
     /// name — grammatically parseable with a known prefix. Replaces the legacy

SqlServerSimulator/Collation.SqlLatin1Sort.cs

@@ -247,24 +247,28 @@ public override int GetHashCode(string obj)
         {
             if (!AllCp1252(obj))
                 return this.inner.GetHashCode(obj);
-            var (primary, secondary, _) = this.BuildKey(obj);
+
+            // Hash the primary weight run, a separator, then the secondary run —
+            // mirrors the order Compare consults them, so Compare-equal strings
+            // (equal at both levels) always hash equal. Streamed through the
+            // cursor so no weight lists are materialized.
             var hash = new HashCode();
-            foreach (var weight in primary)
-                hash.Add(weight);
+            var primary = this.NewCursor(obj);
+            while (primary.MoveNext())
+                hash.Add(primary.Primary);
             hash.Add(-1);
-            foreach (var weight in secondary)
-                hash.Add(weight);
+            var secondary = this.NewCursor(obj);
+            while (secondary.MoveNext())
+                hash.Add(secondary.Secondary);
             return hash.ToHashCode();
         }
 
         private int CompareInRepertoire(string x, string y)
         {
-            var (px, sx, tx) = this.BuildKey(x);
-            var (py, sy, ty) = this.BuildKey(y);
-            var primary = LexCompare(px, py);
+            var primary = this.CompareLevel(x, y, Level.Primary);
             if (primary != 0)
                 return primary;
-            var secondary = LexCompare(sx, sy);
+            var secondary = this.CompareLevel(x, y, Level.Secondary);
             if (secondary != 0)
                 return secondary;
 
@@ -273,58 +277,117 @@ private int CompareInRepertoire(string x, string y)
             // it has no minimal-weight characters. nvarchar treats a ligature as
             // equal to its expansion and instead breaks the tie on its
             // minimal-weight characters.
-            return this.varcharStorage ? LexCompare(tx, ty) : NvarcharIgnorableTiebreak(x, y);
+            return this.varcharStorage ? this.CompareLevel(x, y, Level.Tertiary) : NvarcharIgnorableTiebreak(x, y);
         }
 
-        // Builds the primary, secondary, and tertiary weight sequences (all
-        // parallel). Ignorable characters (nvarchar only) contribute to none;
-        // ligatures expand to their base letters' weights, and each expansion
-        // element carries tertiary weight 1 (plain characters carry 0) so a
-        // ligature outranks the identical run of plain letters.
-        private (List<int> Primary, List<int> Secondary, List<int> Tertiary) BuildKey(string s)
+        private enum Level
         {
-            var weights = this.varcharStorage ? varcharWeights : nvarcharWeights;
-            var expansions = this.varcharStorage ? varcharExpansion : nvarcharExpansion;
-            var primary = new List<int>(s.Length);
-            var secondary = new List<int>(s.Length);
-            var tertiary = new List<int>(s.Length);
-            foreach (var ch in s)
+            Primary,
+            Secondary,
+            Tertiary,
+        }
+
+        // Walks both operands through parallel weight cursors, comparing one
+        // level at a time. The shorter weight run sorts first once a shared
+        // prefix ties (matching the old list-length tie-break). The cursors are
+        // ref structs over the source strings — no weight lists are allocated,
+        // so the common single-pass primary comparison is allocation-free.
+        private int CompareLevel(string x, string y, Level level)
+        {
+            var cx = this.NewCursor(x);
+            var cy = this.NewCursor(y);
+            while (true)
+            {
+                var hasX = cx.MoveNext();
+                var hasY = cy.MoveNext();
+                if (!hasX || !hasY)
+                    return (hasX ? 1 : 0) - (hasY ? 1 : 0);
+                var wx = level switch { Level.Primary => cx.Primary, Level.Secondary => cx.Secondary, _ => cx.Tertiary };
+                var wy = level switch { Level.Primary => cy.Primary, Level.Secondary => cy.Secondary, _ => cy.Tertiary };
+                if (wx != wy)
+                    return wx < wy ? -1 : 1;
+            }
+        }
+
+        private WeightCursor NewCursor(string s) =>
+            this.varcharStorage
+                ? new WeightCursor(s, varcharWeights, varcharExpansion, skipIgnorable: false)
+                : new WeightCursor(s, nvarcharWeights, nvarcharExpansion, skipIgnorable: true);
+
+        // Yields the (primary, secondary, tertiary) weight of each collation
+        // element of a string in order: ignorables (nvarchar) are skipped,
+        // ligatures expand to their base letters with tertiary 1, plain
+        // characters carry tertiary 0. One element per MoveNext; the expansion
+        // of a ligature surfaces as consecutive elements.
+        private ref struct WeightCursor
+        {
+            private readonly string s;
+
+            private readonly FrozenDictionary<char, (int Primary, int Secondary)> weights;
+
+            private readonly FrozenDictionary<char, string> expansions;
+
+            private readonly bool skipIgnorable;
+
+            private int index;
+
+            private string? expansion;
+
+            private int expansionPos;
+
+            internal int Primary;
+
+            internal int Secondary;
+
+            internal int Tertiary;
+
+            internal WeightCursor(string s, FrozenDictionary<char, (int Primary, int Secondary)> weights, FrozenDictionary<char, string> expansions, bool skipIgnorable)
+            {
+                this.s = s;
+                this.weights = weights;
+                this.expansions = expansions;
+                this.skipIgnorable = skipIgnorable;
+            }
+
+            internal bool MoveNext()
             {
-                if (!this.varcharStorage && nvarcharIgnorable.Contains(ch))
-                    continue;
-                if (expansions.TryGetValue(ch, out var expansion))
+                if (this.expansion is not null)
                 {
-                    foreach (var baseChar in expansion)
+                    this.Emit(this.expansion[this.expansionPos++], tertiary: 1);
+                    if (this.expansionPos >= this.expansion.Length)
+                        this.expansion = null;
+                    return true;
+                }
+
+                while (this.index < this.s.Length)
+                {
+                    var ch = this.s[this.index++];
+                    if (this.skipIgnorable && nvarcharIgnorable.Contains(ch))
+                        continue;
+                    if (this.expansions.TryGetValue(ch, out var exp))
                     {
-                        var (basePrimary, baseSecondary) = weights[baseChar];
-                        primary.Add(basePrimary);
-                        secondary.Add(baseSecondary);
-                        tertiary.Add(1);
+                        this.expansion = exp;
+                        this.expansionPos = 0;
+                        this.Emit(exp[this.expansionPos++], tertiary: 1);
+                        if (this.expansionPos >= exp.Length)
+                            this.expansion = null;
+                        return true;
                     }
 
-                    continue;
+                    this.Emit(ch, tertiary: 0);
+                    return true;
                 }
 
-                var (charPrimary, charSecondary) = weights[ch];
-                primary.Add(charPrimary);
-                secondary.Add(charSecondary);
-                tertiary.Add(0);
+                return false;
             }
 
-            return (primary, secondary, tertiary);
-        }
-
-        private static int LexCompare(List<int> a, List<int> b)
-        {
-            var shared = Math.Min(a.Count, b.Count);
-            for (var i = 0; i < shared; i++)
+            private void Emit(char ch, int tertiary)
             {
-                var delta = a[i] - b[i];
-                if (delta != 0)
-                    return delta < 0 ? -1 : 1;
+                var (primary, secondary) = this.weights[ch];
+                this.Primary = primary;
+                this.Secondary = secondary;
+                this.Tertiary = tertiary;
             }
-
-            return a.Count.CompareTo(b.Count);
         }
 
         // Reached only when the primary and secondary keys are equal: the two

SqlServerSimulator/Collation.cs

@@ -33,10 +33,11 @@ internal enum Coercibility : byte
 /// The SQL Server equivalent to .NET's <see cref="IComparer{T}"/> for strings.
 /// Concrete subclasses are constructed by the parser
 /// (<see cref="TryParse(string, out Collation?)"/>) from the grammatical
-/// shape of the collation name; hand-tuned bespoke bodies can be registered
-/// via the override slot for names whose behavior diverges from the
-/// grammar-derived default. See <c>docs/claude/collations.md</c> for the
-/// catalog, parser, and override-slot details.
+/// shape of the collation name. One name whose sort diverges from the
+/// grammar-derived default gets a hand-tuned body: the parser special-cases
+/// <c>SQL_Latin1_General_CP1_CI_AS</c> and wraps its comparer in the
+/// byte-exact <see cref="SqlLatin1Cp1CiAsCollation"/>. See
+/// <c>docs/claude/collations.md</c> for the catalog and parser details.
 /// </summary>
 internal abstract partial class Collation : IComparer<string>, IEqualityComparer<string>
 {
@@ -49,8 +50,9 @@ private protected Collation()
     /// <summary>
     /// Human-readable description of this collation — matches the format
     /// emitted by <c>sys.fn_helpcollations()</c>. Generated by
-    /// <see cref="BuildDescription"/> for parser-derived instances;
-    /// override-registry entries supply their own text.
+    /// <see cref="BuildDescription"/> for parser-derived instances; the
+    /// byte-exact <see cref="SqlLatin1Cp1CiAsCollation"/> delegates to the
+    /// inner parser-built body for its text.
     /// </summary>
     public abstract string Description { get; }
 
@@ -68,7 +70,7 @@ private protected Collation()
     /// <summary>
     /// The simulator's hardcoded baseline collation —
     /// <c>SQL_Latin1_General_CP1_CI_AS</c>. Resolved once via
-    /// <see cref="TryGet"/>. Two consumers:
+    /// <see cref="Get"/>. Two consumers:
     /// <list type="bullet">
     /// <item>The initial value of
     /// <see cref="Simulation.ServerCollation"/>, which in turn seeds every
@@ -100,7 +102,7 @@ private protected Collation()
     /// the simulator's catalog views (<c>sys.objects.type_desc</c>,
     /// <c>sys.database_permissions.permission_name</c>, the various
     /// <c>type</c> / <c>state</c> char(1) / char(2) enum codes, etc.).
-    /// Resolved once via <see cref="TryGet"/> to
+    /// Resolved once via <see cref="Get"/> to
     /// <c>Latin1_General_100_CI_AS_KS_WS_SC</c> — the contained-database
     /// catalog collation real SQL Server reports through
     /// <c>sys.fn_helpcollations()</c>.
@@ -143,10 +145,10 @@ private protected Collation()
     // and the textual-order guarantee for static-field initializers
     // doesn't cross those.
     private static readonly Lazy<Collation> baselineLazy =
-        new(() => TryGet("SQL_Latin1_General_CP1_CI_AS") ?? throw new InvalidOperationException("Default collation failed to resolve."));
+        new(() => Get("SQL_Latin1_General_CP1_CI_AS"));
 
     private static readonly Lazy<Collation> catalogLazy =
-        new(() => TryGet("Latin1_General_100_CI_AS_KS_WS_SC") ?? throw new InvalidOperationException("Catalog collation failed to resolve."));
+        new(() => Get("Latin1_General_100_CI_AS_KS_WS_SC"));
 
     /// <summary>
     /// SQL Server's collation-coercibility resolution for two operands.
@@ -222,6 +224,26 @@ internal static (Collation Collation, Coercibility Coercibility)? Resolve(SqlTyp
     /// </summary>
     internal virtual Encoding StorageEncoding => CharSqlType.Cp1252Encoder;
 
+    /// <summary>
+    /// Reads the Unicode scalar at <paramref name="i"/>: a surrogate pair
+    /// combines into one 32-bit code point (<paramref name="advance"/> = 2),
+    /// otherwise the bare code unit (<paramref name="advance"/> = 1). Shared by
+    /// the code-point-ordered binary bodies (pre-Bin2 positions 1+ and the
+    /// UTF-8 body).
+    /// </summary>
+    private static int ScalarAt(string s, int i, out int advance)
+    {
+        var c = s[i];
+        if (char.IsHighSurrogate(c) && i + 1 < s.Length && char.IsLowSurrogate(s[i + 1]))
+        {
+            advance = 2;
+            return char.ConvertToUtf32(c, s[i + 1]);
+        }
+
+        advance = 1;
+        return c;
+    }
+
     /// <summary>
     /// Generic culture-driven collation. Pins a
     /// <see cref="CompareInfo"/> and a flag-derived
@@ -441,18 +463,6 @@ private static int ComparePreBin2(string x, string y)
             }
             return x.Length.CompareTo(y.Length);
         }
-
-        private static int ScalarAt(string s, int i, out int advance)
-        {
-            var c = s[i];
-            if (char.IsHighSurrogate(c) && i + 1 < s.Length && char.IsLowSurrogate(s[i + 1]))
-            {
-                advance = 2;
-                return char.ConvertToUtf32(c, s[i + 1]);
-            }
-            advance = 1;
-            return c;
-        }
     }
 
     /// <summary>
@@ -494,14 +504,23 @@ x is null
 
         public override int GetHashCode(string obj)
         {
-            var bytes = CharSqlType.Cp1252Encoder.GetBytes(obj);
+            // CP1252 is single-byte, so the byte count equals the char count.
+            var buffer = obj.Length <= 256 ? stackalloc byte[obj.Length] : new byte[obj.Length];
+            var length = CharSqlType.Cp1252Encoder.GetBytes(obj, buffer);
             var hash = default(HashCode);
-            hash.AddBytes(bytes);
+            hash.AddBytes(buffer[..length]);
             return hash.ToHashCode();
         }
 
-        private static int CompareBytes(string x, string y) =>
-            CharSqlType.Cp1252Encoder.GetBytes(x).AsSpan().SequenceCompareTo(CharSqlType.Cp1252Encoder.GetBytes(y));
+        private static int CompareBytes(string x, string y)
+        {
+            var encoding = CharSqlType.Cp1252Encoder;
+            var xBuffer = x.Length <= 256 ? stackalloc byte[x.Length] : new byte[x.Length];
+            var yBuffer = y.Length <= 256 ? stackalloc byte[y.Length] : new byte[y.Length];
+            var xLength = encoding.GetBytes(x, xBuffer);
+            var yLength = encoding.GetBytes(y, yBuffer);
+            return xBuffer[..xLength].SequenceCompareTo(yBuffer[..yLength]);
+        }
     }
 
     /// <summary>
@@ -561,17 +580,5 @@ public override int GetHashCode(string obj)
             hash.AddBytes(bytes);
             return hash.ToHashCode();
         }
-
-        private static int ScalarAt(string s, int i, out int advance)
-        {
-            var c = s[i];
-            if (char.IsHighSurrogate(c) && i + 1 < s.Length && char.IsLowSurrogate(s[i + 1]))
-            {
-                advance = 2;
-                return char.ConvertToUtf32(c, s[i + 1]);
-            }
-            advance = 1;
-            return c;
-        }
     }
 }

docs/claude/collations.md

@@ -139,6 +139,8 @@ Why a bespoke body instead of `CompareInfo`: real SQL Server sorts this collatio
 
 One hand-adjustment in the data: the legacy varchar CI_AI form classifies cedilla (`Ç`/`ç`) as a distinct primary letter, but its CI_AS *sort* folds it onto `c` (probe-confirmed `'Çm' < 'cn'`), so those two primary entries are pinned to `c`'s rank. Strings containing a **non-CP1252** character fall back to the inner `CultureCollation`'s `CompareInfo` two-pass (below) — close for arbitrary Unicode, exact for the CP1252 universe.
 
+Implementation: this is the engine's hottest string path (it backs `Baseline`), so `Compare` / `GetHashCode` stream each operand's weights through a `WeightCursor` `ref struct` — one element per `MoveNext`, ignorables skipped and ligatures expanded inline — rather than materializing weight lists. A comparison walks the cursors at the primary level, dropping to a second walk only on a primary tie and a third only on a secondary tie; the common case resolves in one pass with **zero allocation**. Keep the `AllCp1252` gate ahead of the streaming path: the "any non-CP1252 char ⇒ `CompareInfo` fallback for the whole pair" contract requires scanning both operands before choosing the repertoire path, which a bail-mid-walk detector would break.
+
 ## Symbol sort weighting (other SQL_\* / Windows / locale families)
 
 `CultureCollation.Compare` (the `CompareInfo`-routed comparer behind every collation **other than the default**) gives hyphen (`-`) and apostrophe (`'`) the **minimal-weight** treatment SQL Server applies, while every other symbol keeps a real primary weight: