Top-level ORDER BY after set operations computes sort keys by decoding only the ORDER BY columns from the byte[] rows instead of full-row decode.

RyanLamansky · RyanLamansky · commit ce255d450a9d · 2026-07-18T12:37:58.000-05:00
diff --git a/SqlServerSimulator.Tests/SetOperationTests.cs b/SqlServerSimulator.Tests/SetOperationTests.cs
@@ -139,6 +139,56 @@ public void TopLevelOrderBy_AppliesToCombinedResult()
     public void PerBranchOrderBy_RaisesMsg156()
         => _ = new Simulation().AssertSqlError("select 1 order by 1 union select 2", 156);
 
+    // The top-level (post-set-op) ORDER BY sorts the combined rows in their
+    // encoded byte[] form and decodes only the sort-key column per row; the
+    // non-key columns (here a string and a NULL-bearing int) must still drain
+    // correctly for every row after the sort, and NULL ordering / collation
+    // must match the single-SELECT path.
+    [TestMethod]
+    public void TopLevelOrderBy_MultiColumn_DrainsNonKeyColumnsAfterSort()
+    {
+        var simulation = new Simulation();
+        _ = simulation.ExecuteNonQuery("""
+            create table a (k int, s varchar(10), n int);
+            create table b (k int, s varchar(10), n int);
+            insert a values (3, 'gamma', 30), (1, 'alpha', null);
+            insert b values (2, 'Beta', 20), (4, 'delta', 40)
+            """);
+
+        using var reader = simulation.CreateCommand(
+            "select k, s, n from a union all select k, s, n from b order by s").ExecuteReader();
+        var rows = new List<(int K, string S, int? N)>();
+        while (reader.Read())
+            rows.Add((reader.GetInt32(0), reader.GetString(1), reader.IsDBNull(2) ? null : reader.GetInt32(2)));
+
+        // Case-insensitive default collation: alpha < Beta < delta < gamma.
+        CollectionAssert.AreEqual(
+            new[] { (1, "alpha", (int?)null), (2, "Beta", 20), (4, "delta", 40), (3, "gamma", 30) },
+            rows);
+    }
+
+    // Ordinal ORDER BY over a set-op resolves against the projected column and
+    // sorts NULLs first under ASC, exercising the ordinal branch of the
+    // top-level key decode.
+    [TestMethod]
+    public void TopLevelOrderBy_Ordinal_NullsFirst()
+    {
+        var simulation = new Simulation();
+        _ = simulation.ExecuteNonQuery("""
+            create table a (v int);
+            create table b (v int);
+            insert a values (3), (null);
+            insert b values (1), (2)
+            """);
+
+        using var reader = simulation.CreateCommand(
+            "select v from a union all select v from b order by 1").ExecuteReader();
+        var values = new List<int?>();
+        while (reader.Read())
+            values.Add(reader.IsDBNull(0) ? null : reader.GetInt32(0));
+        CollectionAssert.AreEqual(new int?[] { null, 1, 2, 3 }, values);
+    }
+
     // Non-set-op SELECT can ORDER BY a non-projected source column.
     [TestMethod]
     public void SingleSelect_OrderByNonProjectedSource_StillWorks()
diff --git a/SqlServerSimulator/Parser/Selection.Execution.OrderBy.cs b/SqlServerSimulator/Parser/Selection.Execution.OrderBy.cs
@@ -50,6 +50,44 @@ private static SqlValue[] ComputeOrderKeys(
         return keys;
     }
 
+    /// <summary>
+    /// Computes ORDER BY keys for the top-level (post-set-op) sort directly off
+    /// an encoded <c>byte[]</c> row, decoding only the columns an ORDER BY item
+    /// references rather than the whole tuple. References resolve against the
+    /// inner plan's projected column names / ordinals only — there are no source
+    /// columns to fall back to (an unresolved name is Msg 207), matching the
+    /// <see cref="ComputeOrderKeys"/> path this mirrors. <paramref name="columns"/>
+    /// is the schema's cached <see cref="HeapColumn"/>[] so each per-column decode
+    /// hits the RowLayout geometry cache.
+    /// </summary>
+    private static SqlValue[] ComputeTopLevelOrderKeys(
+        List<OrderBySpec> orderBy,
+        string[] columnNames,
+        HeapColumn[] columns,
+        byte[] rowBytes,
+        BatchContext batch)
+    {
+        SqlValue ResolveByOutputName(MultiPartName name)
+        {
+            for (var j = 0; j < columnNames.Length; j++)
+            {
+                if (BuiltInToken.Equals(columnNames[j], name.Leaf))
+                    return RowDecoder.DecodeColumn(columns, rowBytes, j);
+            }
+            throw SimulatedSqlException.InvalidColumnName(name);
+        }
+
+        var keys = new SqlValue[orderBy.Count];
+        for (var i = 0; i < orderBy.Count; i++)
+        {
+            var spec = orderBy[i];
+            keys[i] = spec.IsOrdinal
+                ? RowDecoder.DecodeColumn(columns, rowBytes, spec.Ordinal - 1)
+                : spec.Expr!.Run(new RuntimeContext(ResolveByOutputName, batch));
+        }
+        return keys;
+    }
+
     /// <summary>
     /// Lexicographic compare of two key tuples per the per-key descending
     /// flags. NULL is treated as the smallest value (NULL first under ASC,
diff --git a/SqlServerSimulator/Parser/Selection.Execution.SetOps.cs b/SqlServerSimulator/Parser/Selection.Execution.SetOps.cs
@@ -195,6 +195,10 @@ private static Selection ApplyTopLevelOrderBy(Selection inner, List<OrderBySpec>
         var schema = inner.Schema;
         var columnNames = inner.ColumnNames;
 
+        // Cached HeapColumn[] for the combined schema, so per-column key decodes
+        // hit RowLayout's identity-keyed geometry cache (see RowDecoder.ColumnsFor).
+        var keyColumns = RowDecoder.ColumnsFor(schema);
+
         return new Selection(schema, columnNames,
             hasOrderBy: true,
             hasTopOrOffsetOrFetch: inner.HasTopOrOffsetOrFetch || offsetExpression is not null || fetchExpression is not null,
@@ -214,23 +218,16 @@ private static Selection ApplyTopLevelOrderBy(Selection inner, List<OrderBySpec>
             }
             else
             {
+                // The inner set-op chain yields byte[] rows natively (branch
+                // dedup / coercion re-encode). Keep that form through the sort
+                // and let the reader / TDS cursor decode once at drain —
+                // eagerly decoding every column into SqlValue[] here only to
+                // re-materialize strings for the whole buffer measured slower
+                // and heavier than the lazy-from-bytes path. Sort keys decode
+                // only the ORDER BY columns off each row, not the full tuple.
                 var keyed = new List<(byte[] Row, SqlValue[] Keys)>(allRows.Count);
                 foreach (var rowBytes in allRows)
-                {
-                    var values = DecodeRowToValues(rowBytes, schema);
-                    SqlValue ResolveByOutputName(MultiPartName name)
-                    {
-                        for (var j = 0; j < columnNames.Length; j++)
-                        {
-                            if (BuiltInToken.Equals(columnNames[j], name.Leaf))
-                                return values[j];
-                        }
-                        throw SimulatedSqlException.InvalidColumnName(name);
-                    }
-
-                    var keys = ComputeOrderKeys(orderBy, values, columnNames, distinct: false, batch, ResolveByOutputName);
-                    keyed.Add((rowBytes, keys));
-                }
+                    keyed.Add((rowBytes, ComputeTopLevelOrderKeys(orderBy, columnNames, keyColumns, rowBytes, batch)));
 
                 keyed.Sort((a, b) => CompareOrderKeys(a.Keys, b.Keys, orderBy));
                 ordered = keyed.Select(r => r.Row);
diff --git a/docs/claude/backlog.md b/docs/claude/backlog.md
@@ -24,9 +24,9 @@ The endpoint ships with SQLBatch + RPC + Transaction Manager support and credent
 
 - **Tool shakedown** — point real client tools at the endpoint and harvest their exotic catalog queries / SET shapes into this backlog. Tool scope (user decision, 2026-07-16): tools a SQL Server + .NET developer already has — SSMS, sqlcmd, Visual Studio (SQL Server Object Explorer / DacFx), LINQPad; DBA-flavored tools like DBeaver are out of scope. **SSMS is the final boss** — an ongoing campaign, not a single leg: each surface is its own multi-round harvest, and clearing one unlocks the next. Cleared legs are recorded in the per-feature deep-dives (catalog surface in [`catalog-views.md`](catalog-views.md), wire behavior in [`tds-endpoint.md`](tds-endpoint.md)); the discovery harnesses are the gitignored `.vs/ssms-host` TDS host and the headless SMO property-bag drain. **Remaining frontier**: Table Designer, Activity Monitor, standard reports, and IntelliSense's background metadata harvest. (The sp_cursor* API-server-cursor RPC family — SSMS query-editor grid navigation / multi-row edit — shipped; see [`tds-endpoint.md`](tds-endpoint.md).) Candidate follow-on legs within tool scope: Visual Studio's SQL Server Object Explorer (DacFx-driven, a different query dialect from SMO) and LINQPad.
 - **SMO API sweep campaign** — `.vs/smo-sweep` (gitignored local harness) walks SMO's full reachable read surface against the self-hosted simulator and, identically, against the live reference, draining every `Property.Value` and `Script()`-ing every `IScriptable`; modes `sweep` / `sweep --live` / `diff` → sorted JSON reports + `reports/triage.md`; workflow = sweep both sides → triage → fix bundles → graduated `Tests.Smo` tests → re-sweep. Open items from the latest triage: (a) `DBCC SHOW_STATISTICS … WITH STATS_STREAM` (SMO `Statistic.Stream`) stays `NotSupportedException` — it wants the raw serialized statistics-histogram blob, which the simulator has no faithful source for; (b) the unmodeled runtime/OS surfaces SMO reaches as absent objects (backup history `msdb.dbo.backupset`, `sys.dm_tran_persistent_version_store_stats`, file-space/IO DMVs, `sys.dm_os_process_memory`, `master.dbo.sysprocesses`, `FILEPROPERTY`, registry/OS xps) — surfaced as `PropertyCannotBeRetrievedException` / defaults, the legitimate-gap category.
-- **ORDER BY result sets ride encoded `byte[]` rows** through the reader-cursor decode path instead of the projected `SqlValue[]` fast path the unordered FROM-bearing SELECT takes — an encode-then-re-decode round-trip per row. The next large-result-drain throughput lever (DacFx export chunks all carry ORDER BY, so it taxes every export row).
 - **Wire forms deferred by the codec**: `text`/`ntext`/`image` (textptr ROW form + table-name-bearing COLMETADATA); `sql_variant` RPC *parameters* (the result-column form ships — see [`tds-endpoint.md`](tds-endpoint.md); candidates for true-variant migration off inner-type substitution: SERVERPROPERTY / SESSIONPROPERTY / CONNECTIONPROPERTY / COLLATIONPROPERTY / LOGINPROPERTY / SESSION_CONTEXT / `sys.configurations`); UDT *parameters* for `geography`/`geometry`/`hierarchyid` (their result-column forms ship). **TVP parameters (0xF3, `SqlDbType.Structured`) ship** — `DataTable` / `IEnumerable<SqlDataRecord>` / `DbDataReader` sources over proc-RPC + sp_executesql, error parity for arity / type / NOT NULL / CHECK / PK / UNIQUE / unknown-type; see [`tds-endpoint.md`](tds-endpoint.md#tvp-parameters-sqldbtypestructured). Residual: `sql_variant` / UDT *columns inside* a TVP still reject.
 - **Smaller fidelity items**: mid-stream attention (cancel during a large result requires a concurrent reader); MARS; TDS 8.0 / `Encrypt=Strict` (ALPN via `SslApplicationProtocol`); user-supplied `X509Certificate2`; off-loopback binding — the last two are add-on-demand public-surface expansions.
+- **Chunked `OFFSET/FETCH` re-sorts the whole set per page**: an `ORDER BY … OFFSET @o FETCH @f` paginated drain (the DacFx chunked-export shape) buffers and sorts the entire result on every chunk, then skip/takes the page — so N chunks pay N full sorts (measured ~10× the single-sort µs/row at 15 chunks over 150k rows). The projection/drain representation is already the decoded-once `SqlValue[]` fast path (single-SELECT ORDER BY sorts projected rows, no byte[] re-decode — the prior gap here is closed); this residual is redundant re-sorting across pages, not a decode round-trip. A "sort once, serve many pages" lever would need cross-execution result caching the plan cache doesn't currently model.
 
 ### Built-in functions
 
diff --git a/docs/claude/query.md b/docs/claude/query.md
@@ -21,6 +21,11 @@
 - TOP + OFFSET → **Msg 10741**.
 - Counts resolve at parse time (constants, parameters, arithmetic).
 
+## Result drain / ORDER BY representation
+The FROM-bearing SELECT projection paths — streaming, buffered (ORDER BY / DISTINCT), windowed, and aggregate — all yield already-projected `SqlValue[]` rows, so `SimulatedSqlResultSet` serves the reader and TDS cursors directly with no encode-then-re-decode round-trip (see the `SimulatedSqlResultSet` doc + [`data-reader.md`](data-reader.md)). **ORDER BY on a single SELECT sorts those projected `SqlValue[]` rows** (`ProjectBuffered.materialized.Sort`), so ordered drains ride the same decoded-once fast path as unordered ones; the only ordered-vs-unordered cost is the inherent buffer + per-row key computation + `List.Sort`, not a decode round-trip, and peak retained memory is effectively unchanged (measured within 0.2% on a 150k-row wide drain).
+
+The **top-level ORDER BY after a set-op chain** (`ApplyTopLevelOrderBy`) is the exception: the inner UNION / INTERSECT / EXCEPT chain yields byte[] rows natively (branch dedup / coercion re-encode), so that path keeps the byte[] form through the sort and lets the drain cursor decode once — eagerly decoding every column into `SqlValue[]` for the whole buffer measured slower and heavier (strings re-materialized and retained across the sort). Sort keys decode only the ORDER BY columns off each row (`ComputeTopLevelOrderKeys`), not the full tuple.
+
 ## Aggregates
 `COUNT(*)` / `COUNT(expr)` / `COUNT(DISTINCT)` / `COUNT_BIG`, `SUM` / `AVG`, `MAX` / `MIN`, statistical (`STDEV` / `STDEVP` / `VAR` / `VARP`), `STRING_AGG`, `CHECKSUM_AGG`, `APPROX_COUNT_DISTINCT`. `AVG(int)` truncates; `AVG(decimal(p,s))` widens to `decimal(38, max(s,6))`.
 
