Improved fidelity of SQL_Latin1_General_CP1_CI_AS implementation.

Author: RyanLamansky

SqlServerSimulator.Tests.Internal/CollationTests.cs

@@ -70,22 +70,27 @@ public void Win100_AccentSensitive()
     }
 
     [TestMethod]
-    public void Win100_ApostropheIsPrimaryWeightZero_InSort()
+    public void Win100_ApostropheIsMinimalWeight_InSort()
     {
         // Probe-confirmed against WideWorldImporters.Application.Cities:
-        // MIN of ('Aaronsburg', "'Aiea") is 'Aaronsburg' under
-        // Latin1_General_100_CI_AS because the apostrophe drops out of
-        // the primary sort key, leaving "Aiea" > "Aaronsburg".
+        // MIN of ('Aaronsburg', "'Aiea") is 'Aaronsburg' because the
+        // apostrophe drops out of the *primary* sort key, leaving "Aiea" >
+        // "Aaronsburg". The apostrophe is not weightless, though — it carries
+        // a secondary weight, so between two strings that share a primary key
+        // the copy bearing it sorts after: "A" < "'A", "OBrien" < "O'Brien"
+        // (probe-confirmed on SQL Server 2025, all three families identical).
         IsLessThan(0, Latin1General100CiAs.Compare("Aaronsburg", "'Aiea"));
-        AreEqual(0, Latin1General100CiAs.Compare("'A", "A"));
-        AreEqual(0, Latin1General100CiAs.Compare("O'Brien", "OBrien"));
+        IsGreaterThan(0, Latin1General100CiAs.Compare("'A", "A"));
+        IsGreaterThan(0, Latin1General100CiAs.Compare("O'Brien", "OBrien"));
     }
 
     [TestMethod]
-    public void Win100_HyphenIsPrimaryWeightZero_InSort()
+    public void Win100_HyphenIsMinimalWeight_InSort()
     {
-        AreEqual(0, Latin1General100CiAs.Compare("co-op", "coop"));
-        AreEqual(0, Latin1General100CiAs.Compare("re-do", "redo"));
+        // Same secondary-weight rule as the apostrophe: "coop" < "co-op",
+        // "redo" < "re-do".
+        IsGreaterThan(0, Latin1General100CiAs.Compare("co-op", "coop"));
+        IsGreaterThan(0, Latin1General100CiAs.Compare("re-do", "redo"));
     }
 
     [TestMethod]
@@ -127,7 +132,7 @@ public void Win80_BehavesLikeWin100_ForAsciiAndLatin1()
         // Latin scripts and a handful of newly-added supplementary code
         // points; none reach the regression bar).
         IsTrue(Latin1GeneralCiAs.Equals("abc", "ABC"));
-        AreEqual(0, Latin1GeneralCiAs.Compare("'A", "A"));
+        IsGreaterThan(0, Latin1GeneralCiAs.Compare("'A", "A"));
         IsFalse(Latin1GeneralCiAs.Equals("'A", "A"));
         IsLessThan(0, Latin1GeneralCiAs.Compare("Aaronsburg", "'Aiea"));
     }

SqlServerSimulator.Tests/CollationBehaviorTests.cs

@@ -5,7 +5,7 @@ namespace SqlServerSimulator;
 /// <summary>
 /// Public-surface coverage for the default collation's
 /// (<c>SQL_Latin1_General_CP1_CI_AS</c>) case-folding, accent-sensitivity,
-/// and primary-weight-zero sort/equality rules — exercised through
+/// and minimal-weight (hyphen / apostrophe) sort/equality rules — exercised through
 /// <c>=</c>, <c>ORDER BY</c>, <c>DISTINCT</c>, and <c>COLLATE</c>-name
 /// case-insensitive resolution. Counterpart to the internal-only
 /// <c>CollationTests</c>, which retains the algorithm-contract tests for
@@ -62,14 +62,15 @@ public void DefaultCollation_OrderBy_AsciiLowerVsUpper_IsCaseInsensitive()
     }
 
     /// <summary>
-    /// Probe-confirmed against SQL Server 2025: the default collation
-    /// applies <c>IgnoreSymbols</c> in sort — apostrophe drops out of the
-    /// primary sort key, so "'Aiea" sorts as "Aiea" which is greater than
-    /// "Aaronsburg". Equality keeps symbols significant — the direct
-    /// asymmetry probe lives in <c>CollationTests</c>.
+    /// Probe-confirmed against SQL Server 2025: apostrophe and hyphen carry
+    /// only a secondary sort weight, so they drop out of the *primary* key —
+    /// "'Aiea" sorts as "Aiea", which is greater than "Aaronsburg". (Other
+    /// symbols keep a real primary weight and sort ahead of letters; the
+    /// minimal-weight asymmetry and the secondary tie-break live in
+    /// <c>CollationTests</c>.) Equality keeps every symbol significant.
     /// </summary>
     [TestMethod]
-    public void DefaultCollation_OrderBy_ApostropheIsIgnored()
+    public void DefaultCollation_OrderBy_ApostropheHasMinimalWeight()
     {
         var sim = new Simulation();
         _ = sim.ExecuteNonQuery(
@@ -81,6 +82,46 @@ public void DefaultCollation_OrderBy_ApostropheIsIgnored()
         CollectionAssert.AreEqual(new[] { "Aaronsburg", "'Aiea" }, rows);
     }
 
+    /// <summary>
+    /// Probe-confirmed against SQL Server 2025: symbols other than hyphen /
+    /// apostrophe keep a real primary weight that sorts them ahead of digits
+    /// and letters, so MIN of ('#500-75', '00,', 'abc') is '#500-75'. (An
+    /// earlier ignore-all-symbols sort stripped the '#' and mis-ranked
+    /// "#500-75" among the digits as "50075" — the divergence this guards.)
+    /// </summary>
+    [TestMethod]
+    public void DefaultCollation_OrderBy_NonMinimalSymbolsSortFirst()
+    {
+        var sim = new Simulation();
+        _ = sim.ExecuteNonQuery(
+            "create table t (v nvarchar(20)); insert t values ('00,'), ('abc'), ('#500-75')");
+        AreEqual("#500-75", sim.ExecuteScalar("select min(v) from t"));
+        using var reader = sim.CreateCommand("select v from t order by v").ExecuteReader();
+        var rows = new List<string>();
+        while (reader.Read())
+            rows.Add(reader.GetString(0));
+        CollectionAssert.AreEqual(new[] { "#500-75", "00,", "abc" }, rows);
+    }
+
+    /// <summary>
+    /// The minimal-weight marks break ties only against an otherwise-identical
+    /// neighbor: "coop" sorts before "co-op" (probe-confirmed). MIN therefore
+    /// picks the mark-free spelling.
+    /// </summary>
+    [TestMethod]
+    public void DefaultCollation_OrderBy_MinimalWeightBreaksTieAfterPlainSpelling()
+    {
+        var sim = new Simulation();
+        _ = sim.ExecuteNonQuery(
+            "create table t (v nvarchar(20)); insert t values ('co-op'), ('coop')");
+        AreEqual("coop", sim.ExecuteScalar("select min(v) from t"));
+        using var reader = sim.CreateCommand("select v from t order by v").ExecuteReader();
+        var rows = new List<string>();
+        while (reader.Read())
+            rows.Add(reader.GetString(0));
+        CollectionAssert.AreEqual(new[] { "coop", "co-op" }, rows);
+    }
+
     /// <summary>
     /// DISTINCT relies on the comparer's hash/equality contract: case-
     /// folded equivalents collapse to a single bucket, accent-distinct

SqlServerSimulator/Collation.cs

@@ -227,10 +227,12 @@ internal static (Collation Collation, Coercibility Coercibility)? Resolve(SqlTyp
     /// <see cref="CompareInfo"/> and a flag-derived
     /// <see cref="CompareOptions"/> set; routes <see cref="Compare"/> /
     /// <see cref="Equals"/> / <see cref="GetHashCode"/> through them.
-    /// Sort options layer <see cref="CompareOptions.IgnoreSymbols"/> on
-    /// top of the equality options, matching real SQL Server's sort-only
-    /// ignore-symbols treatment (probe-confirmed across the SQL_*,
-    /// Windows, and locale families).
+    /// Hyphen and apostrophe get the two-pass minimal-weight treatment in
+    /// <see cref="Compare"/> (sort as if absent at the primary level, break
+    /// ties only against an otherwise-identical neighbor); every other symbol
+    /// keeps its real primary weight and sorts ahead of digits and letters.
+    /// Probe-confirmed identical across the SQL_*, Windows, and locale
+    /// families on SQL Server 2025 — see <c>docs/claude/collations.md</c>.
     /// </summary>
     internal sealed class CultureCollation : Collation
     {
@@ -244,8 +246,6 @@ internal sealed class CultureCollation : Collation
 
         private readonly CompareOptions equalityOptions;
 
-        private readonly CompareOptions sortOptions;
-
         private readonly Encoding storageEncoding;
 
         private readonly bool isSupplementaryCharacterAware;
@@ -268,11 +268,6 @@ internal CultureCollation(string name, string description, string cultureName, b
             if (!kanaTypeSensitive) baseOpts |= CompareOptions.IgnoreKanaType;
             if (!widthSensitive) baseOpts |= CompareOptions.IgnoreWidth;
             this.equalityOptions = baseOpts;
-            // Sort layers IgnoreSymbols on top — probe-confirmed against
-            // SQL Server 2025: every modeled family (SQL_*, Windows-style,
-            // locale) treats apostrophe / hyphen as primary-weight-zero in
-            // sort, while equality keeps them significant.
-            this.sortOptions = baseOpts | CompareOptions.IgnoreSymbols;
         }
 
         public override string Name => this.name;
@@ -285,10 +280,75 @@ internal CultureCollation(string name, string description, string cultureName, b
 
         internal override bool IsSupplementaryCharacterAware => this.isSupplementaryCharacterAware;
 
-        public override int Compare(string? x, string? y) =>
-            x is null
-                ? (y is null ? 0 : -1)
-                : y is null ? 1 : this.compareInfo.Compare(x, y, this.sortOptions);
+        public override int Compare(string? x, string? y)
+        {
+            if (x is null)
+                return y is null ? 0 : -1;
+            if (y is null)
+                return 1;
+
+            // Hyphen and apostrophe are the two marks SQL Server's collations
+            // weight only at a secondary level: at the primary level they sort
+            // as if absent (so "co-op" ranks beside "coop", "'Aiea" beside
+            // "Aiea"), while every other symbol keeps a real primary weight
+            // that sorts ahead of digits and letters. .NET exposes neither
+            // knob granularly — IgnoreSymbols drops *all* punctuation, the
+            // default keeps it all — so the two minimal marks are stripped for
+            // the primary pass and consulted only to break an otherwise-exact
+            // tie (the copy carrying one sorts after: "coop" < "co-op").
+            if (!ContainsMinimalPunctuation(x) && !ContainsMinimalPunctuation(y))
+                return this.compareInfo.Compare(x, y, this.equalityOptions);
+            var primary = this.compareInfo.Compare(
+                StripMinimalPunctuation(x), StripMinimalPunctuation(y), this.equalityOptions);
+            return primary != 0 ? primary : MinimalPunctuationTiebreak(x, y);
+        }
+
+        private static bool IsMinimalPunctuation(char c) => c is '-' or '\'';
+
+        private static bool ContainsMinimalPunctuation(string s)
+        {
+            foreach (var c in s)
+            {
+                if (IsMinimalPunctuation(c))
+                    return true;
+            }
+
+            return false;
+        }
+
+        private static string StripMinimalPunctuation(string s)
+        {
+            var buffer = s.Length <= 256 ? stackalloc char[s.Length] : new char[s.Length];
+            var count = 0;
+            foreach (var c in s)
+            {
+                if (!IsMinimalPunctuation(c))
+                    buffer[count++] = c;
+            }
+
+            return new string(buffer[..count]);
+        }
+
+        // Primary-equal strings differ only in their hyphen / apostrophe
+        // content; the copy carrying such a mark where the other has a real
+        // character (or its end) sorts later, matching "coop" &lt; "co-op",
+        // "cant" &lt; "can't", "A" &lt; "'A".
+        private static int MinimalPunctuationTiebreak(string x, string y)
+        {
+            var i = 0;
+            var j = 0;
+            while (i < x.Length && j < y.Length)
+            {
+                var xMinimal = IsMinimalPunctuation(x[i]);
+                var yMinimal = IsMinimalPunctuation(y[j]);
+                if (xMinimal != yMinimal)
+                    return xMinimal ? 1 : -1;
+                i++;
+                j++;
+            }
+
+            return (x.Length - i).CompareTo(y.Length - j);
+        }
 
         public override bool Equals(string? x, string? y) =>
             x is null

docs/claude/collations.md

@@ -119,13 +119,26 @@ The override registry (`overrides` ConcurrentDictionary in `Collation.Parser.cs`
 
 ### Behavioral notes by family
 
-- **SQL_\* family**: routed through invariant `CompareInfo` (unless the human-prefix description maps to a locale-specific culture, e.g., `SQL_Croatian_CP1250_CI_AS` → `hr-HR`); same IgnoreSymbols-in-sort treatment as the Windows family. Description carries the per-name SQL Server Sort Order number + Code Page (extracted from the `CP*` token).
-- **Windows-style Latin1_General**: invariant `CompareInfo`; IgnoreSymbols layered on sort options. `_BIN` engages the pre-2005 position-0-codeunit / position-1+-codepoint quirk; `_BIN2` is pure UTF-16 code-unit ordinal.
+- **SQL_\* family**: routed through invariant `CompareInfo` (unless the human-prefix description maps to a locale-specific culture, e.g., `SQL_Croatian_CP1250_CI_AS` → `hr-HR`); same two-pass minimal-punctuation sort treatment as the Windows family (see [Symbol sort weighting](#symbol-sort-weighting)). Description carries the per-name SQL Server Sort Order number + Code Page (extracted from the `CP*` token).
+- **Windows-style Latin1_General**: invariant `CompareInfo`; two-pass minimal-punctuation sort. `_BIN` engages the pre-2005 position-0-codeunit / position-1+-codepoint quirk; `_BIN2` is pure UTF-16 code-unit ordinal.
 - **`_UTF8` collations**: storage encoding flips from CP1252 to UTF-8 for varchar/char columns. `_BIN2_UTF8` substitutes `Utf8CodepointBinaryCollation` (codepoint-order = UTF-8 byte order) on varchar storage.
 - **`_SC_` collations** (and v140+ implicitly): set `IsSupplementaryCharacterAware` on the constructed instance, driving codepoint-aware LEN/SUBSTRING/etc. dispatch (see [`_SC_` function-semantics dispatch](#_sc_-function-semantics-dispatch)).
 - **`_KS_` / `_WS_` flags**: flip `CompareOptions.IgnoreKanaType` / `IgnoreWidth` off (default = both on).
 - **Locale prefixes** (Japanese, Chinese, Turkish, Korean, etc.): map to the closest .NET culture via `KnownPrefixes`; fall back to invariant when no clean .NET equivalent exists (Tamazight, Traditional_Spanish, Indic_General). Sort-parity caveat in [Locale-comparer sort-parity gap](#locale-comparer-sort-parity-gap) applies — equality / CI/CS / KS / WS folding align, secondary sort tiebreakers within equivalence classes may diverge.
 
+## Symbol sort weighting
+
+`CultureCollation.Compare` (the `CompareInfo`-routed comparer behind every SQL_\*, Windows, and locale family) gives hyphen (`-`) and apostrophe (`'`) the **minimal-weight** treatment SQL Server applies, while every other symbol keeps a real primary weight. Probe-confirmed identical across `SQL_Latin1_General_CP1_CI_AS`, `Latin1_General_100_CI_AS`, and `Latin1_General_CI_AS` on SQL Server 2025:
+
+- **Non-minimal symbols (`#`, `+`, `,`, `!`, `~`, `_`, …) sort first** — ahead of digits and letters. So `MIN('#500-75', '00,', 'abc')` is `'#500-75'`. .NET's `CompareOptions.IgnoreSymbols` would *strip* these (mis-ranking `'#500-75'` among the digits as `50075`); plain `CompareInfo` without it keeps them, which is what the comparer uses.
+- **Hyphen and apostrophe drop out of the primary key** — `'co-op'` ranks beside `'coop'`, `"'Aiea"` beside `'Aiea'` — but carry a secondary weight, so between two strings sharing a primary key the copy bearing the mark sorts *after*: `'coop' < 'co-op'`, `'cant' < "can't"`, `'A' < "'A"`.
+
+Implementation: a fast path (`compareInfo.Compare(x, y, equalityOptions)`) when neither operand contains a minimal mark; otherwise a primary pass over hyphen/apostrophe-stripped copies, then `MinimalPunctuationTiebreak` (a two-pointer scan where a minimal mark sorts after a real character). Equality and `GetHashCode` keep *every* symbol significant (only trailing spaces fold, handled at the `SqlValue` layer) — so `'co-op' = 'coop'` is false, matching real SQL Server.
+
+### Known gap: varchar symbol order under SQL_\* collations
+
+`SQL_*` collations sort **non-Unicode (`varchar`/`char`) data through a CP1252 code-page sort table** that differs from their Unicode (`nvarchar`/`nchar`) rules — the same collation name orders the same characters differently by storage type. Probe-confirmed: under `SQL_Latin1_General_CP1_CI_AS`, `varchar` sorts `'+' < '/'` but `nvarchar` sorts `'/' < '+'`. The simulator routes both through one Unicode `CompareInfo`, so it matches the nvarchar order for both. Bites base64-bearing `varchar` columns (`+`/`/` are base64's two special chars) — e.g. AdventureWorks `Person.Password.PasswordHash` (`varchar`), where live `MIN(PasswordHash)` starts `++…` (code-page order) but the simulator picks `//…` (Unicode order). Closing it requires the CP1252 SQL sort-order weight table for varchar storage, which the simulator doesn't ship.
+
 ## Locale-comparer sort-parity gap
 
 Probed against SQL Server 2025 with a curated word set per locale (mixed-case ASCII, accented Latin, hiragana / katakana / half-width katakana, common CJK characters and 2-character compounds). For each `(collation, storage)` pair, ORDER BY result vs `CompareInfo.Compare` ordering compared position-by-position: