Seek-accelerate MERGE's match phase now supports WHEN NOT MATCHED BY SOURCE.

Author: RyanLamansky

SqlServerSimulator.Tests.Internal/IndexSeekTests.cs

@@ -1752,22 +1752,43 @@ merge tgt as t
     }
 
     [TestMethod]
-    public void MergeNotMatchedBySource_FullScans()
+    public void MergeNotMatchedBySource_SeeksThenComplementScans()
     {
         var c = FreshTarget();
-        // A NOT MATCHED BY SOURCE clause has to visit every target row, so the
-        // inversion declines and the target × source scan stands — no CacheBuild.
+        // A NOT MATCHED BY SOURCE clause must visit every target to find the
+        // unmatched ones, but the match phase still seeks (CacheBuild): the seek
+        // builds matchedByTarget, then one heap pass applies MATCHED to the hits
+        // and BY-SOURCE DELETE to the rest — no per-target source loop.
         var trace = ExecTraced(c, """
             merge tgt as t
             using (values (2, 200)) as s(id, v) on t.id = s.id
             when matched then update set v = s.v
             when not matched by source then delete;
             """);
-        DoesNotContain("CacheBuild", trace);
+        Contains("CacheBuild", trace);
         AreEqual("2", Seq(ReadRows(c, "select id from tgt order by id"))); // 1 and 3 deleted
         AreEqual(200, Convert.ToInt32(ReadVal(c, "select v from tgt where id = 2")));
     }
 
+    [TestMethod]
+    public void MergeNotMatchedBySource_ThreeWay_AllBranchesCorrect()
+    {
+        var c = FreshTarget(); // tgt = (1,10), (2,20), (3,30)
+        // id=2 matched → update; id=4 not matched by target → insert; id=1,3 not
+        // matched by source → delete. Exercises all three branches through the
+        // seek + complement-scan path.
+        Exec(c, """
+            merge tgt as t
+            using (values (2, 222), (4, 444)) as s(id, v) on t.id = s.id
+            when matched then update set v = s.v
+            when not matched then insert (id, v) values (s.id, s.v)
+            when not matched by source then delete;
+            """);
+        AreEqual("2,4", Seq(ReadRows(c, "select id from tgt order by id")));
+        AreEqual(222, Convert.ToInt32(ReadVal(c, "select v from tgt where id = 2")));
+        AreEqual(444, Convert.ToInt32(ReadVal(c, "select v from tgt where id = 4")));
+    }
+
     [TestMethod]
     public void MergeSeek_ResidualOnTerm_Filters()
     {

SqlServerSimulator/Simulation/Simulation.Merge.cs

@@ -1015,56 +1015,77 @@ SqlValue ResolveCombined(SqlValue[]? targetValues, SqlValue[]? sourceValues, Mul
         // Phase A finds, per matched target row, the source rows it matches, then
         // applies the WHEN MATCHED / WHEN NOT MATCHED BY SOURCE action. The
         // target × source scan is O(target × source). When the ON carries a
-        // seekable target equality, no NOT MATCHED BY SOURCE clause forces every
-        // target to be visited, and the target isn't a view (whose column names
+        // seekable target equality and the target isn't a view (whose column names
         // don't map to the base heap), the inverted path seeks matching targets
-        // per source row instead and touches only matched targets.
-        var hasNotMatchedBySource = whenClauses.Any(c => c.Kind == WhenClauseKind.NotMatchedBySource);
-        var targetSeek = sourceView is null && !hasNotMatchedBySource
+        // per source row instead — turning the match phase into O(source × log
+        // target). With no NOT MATCHED BY SOURCE clause it then touches only the
+        // matched targets; with one (which must visit every target to find the
+        // unmatched ones) it walks the heap once applying the precomputed matches,
+        // dropping the inner source loop either way.
+        var targetSeek = sourceView is null
             ? Selection.TryPrepareMergeTargetSeek(destinationTable, targetAlias, onPredicate, context.Batch)
             : null;
 
         if (targetSeek is not null)
         {
-            // Inverted Phase A: per source row, seek the matching target rows and
-            // group them by target address — first-source-wins via source-index
-            // order (sources iterate ascending), heap order restored by the final
-            // (page, slot) sort so the apply order matches the scan path's.
-            var matchedByTarget = new Dictionary<(int Page, int Slot), (List<int> Sources, byte[] Bytes)>();
+            var hasNotMatchedBySource = whenClauses.Any(c => c.Kind == WhenClauseKind.NotMatchedBySource);
+
+            // Match phase: per source row, seek the matching target rows and group
+            // them by target address — first-source-wins via source-index order
+            // (sources iterate ascending). The seek matched the equality prefix
+            // only, so the full ON is re-run per candidate (residual filter — a
+            // term like … AND t.active = 1, or a stale cache entry, is dropped).
+            var matchedByTarget = new Dictionary<(int Page, int Slot), List<int>>();
             for (var si = 0; si < sourceRows.Count; si++)
             {
                 var sourceValues = sourceRows[si];
                 foreach (var (page, slot, rowBytes) in targetSeek(name => ResolveCombined(null, sourceValues, name)))
                 {
-                    var targetValues = DecodeFullRow(destinationTable, rowBytes);
-                    EvaluateComputedColumns(destinationTable, targetValues, context.Batch);
-
-                    // The seek matched the equality prefix only; re-run the full ON
-                    // predicate so a residual term (… AND t.active = 1) or a stale
-                    // cache entry can't produce a false match.
-                    if (onPredicate.Run(new RuntimeContext(name => ResolveCombined(targetValues, sourceValues, name), context.Batch)) != true)
+                    var candidateValues = DecodeFullRow(destinationTable, rowBytes);
+                    EvaluateComputedColumns(destinationTable, candidateValues, context.Batch);
+                    if (onPredicate.Run(new RuntimeContext(name => ResolveCombined(candidateValues, sourceValues, name), context.Batch)) != true)
                         continue;
 
                     sourceMatched[si] = true;
-                    if (!matchedByTarget.TryGetValue((page, slot), out var entry))
-                        matchedByTarget[(page, slot)] = entry = ([], rowBytes);
-                    entry.Sources.Add(si);
+                    if (!matchedByTarget.TryGetValue((page, slot), out var sources))
+                        matchedByTarget[(page, slot)] = sources = [];
+                    sources.Add(si);
                 }
             }
 
-            foreach (var address in matchedByTarget.Keys.OrderBy(a => a.Page).ThenBy(a => a.Slot))
+            if (hasNotMatchedBySource)
             {
-                var (matchedSources, rowBytes) = matchedByTarget[address];
-                var targetValues = DecodeFullRow(destinationTable, rowBytes);
-                EvaluateComputedColumns(destinationTable, targetValues, context.Batch);
-                var sourceValues = sourceRows[matchedSources[0]];
-                var chosen = PickClause(whenClauses, WhenClauseKind.Matched, targetValues, sourceValues, context.Batch, ResolveCombined);
-                if (chosen is null)
-                    continue;
-                if (chosen.Action == MergeActionKind.Update && matchedSources.Count > 1)
-                    throw SimulatedSqlException.MergeMultiMatch();
-
-                ApplyChosenMatchedAction(context, destinationTable, sourceView, chosen, address.Page, address.Slot, targetValues, sourceValues, ResolveCombined, pendingUpdates, pendingDeletes);
+                // Complement pass: every target row in heap order — matched rows take
+                // their precomputed source list, unmatched rows fall to WHEN NOT
+                // MATCHED BY SOURCE. Heap-order interleaving matches the scan path's
+                // discovery order, but with no per-target source loop.
+                foreach (var (pageIndex, slotIndex, rowBytes) in destinationTable.Heap.EnumerateRowsWithAddress())
+                {
+                    var targetValues = DecodeFullRow(destinationTable, rowBytes);
+                    EvaluateComputedColumns(destinationTable, targetValues, context.Batch);
+                    if (matchedByTarget.TryGetValue((pageIndex, slotIndex), out var matchedSources))
+                    {
+                        ApplyMergeMatched(context, destinationTable, sourceView, whenClauses, pageIndex, slotIndex, targetValues, sourceRows, matchedSources, ResolveCombined, pendingUpdates, pendingDeletes);
+                    }
+                    else
+                    {
+                        var chosen = PickClause(whenClauses, WhenClauseKind.NotMatchedBySource, targetValues, sourceValues: null, context.Batch, ResolveCombined);
+                        if (chosen is not null)
+                            ApplyChosenMatchedAction(context, destinationTable, sourceView, chosen, pageIndex, slotIndex, targetValues, sourceValues: null, ResolveCombined, pendingUpdates, pendingDeletes);
+                    }
+                }
+            }
+            else
+            {
+                // No NOT MATCHED BY SOURCE: visit only matched targets, restoring
+                // heap order via the (page, slot) sort so the apply order matches
+                // the scan path's.
+                foreach (var address in matchedByTarget.Keys.OrderBy(a => a.Page).ThenBy(a => a.Slot))
+                {
+                    var targetValues = DecodeFullRow(destinationTable, destinationTable.Heap.ReadSlotBytes(address.Page, address.Slot)!);
+                    EvaluateComputedColumns(destinationTable, targetValues, context.Batch);
+                    ApplyMergeMatched(context, destinationTable, sourceView, whenClauses, address.Page, address.Slot, targetValues, sourceRows, matchedByTarget[address], ResolveCombined, pendingUpdates, pendingDeletes);
+                }
             }
         }
         else
@@ -1170,6 +1191,35 @@ SqlValue ResolveCombined(SqlValue[]? targetValues, SqlValue[]? sourceValues, Mul
         return null;
     }
 
+    // Applies the WHEN MATCHED action to one matched target row, given the source
+    // rows it matched (in source-index order, first wins). Picks the first
+    // applicable MATCHED clause, enforces the multiple-source-match guard
+    // (Msg 8672 for UPDATE), and queues the action. Shared by both inverted-seek
+    // apply passes (matched-only and the NOT MATCHED BY SOURCE complement scan).
+    private static void ApplyMergeMatched(
+        ParserContext context,
+        HeapTable destinationTable,
+        View? sourceView,
+        List<WhenClause> whenClauses,
+        int pageIndex,
+        int slotIndex,
+        SqlValue[] targetValues,
+        List<SqlValue[]> sourceRows,
+        List<int> matchedSources,
+        Func<SqlValue[]?, SqlValue[]?, MultiPartName, SqlValue> resolveCombined,
+        List<(int Page, int Slot, SqlValue[] OldValues, SqlValue[] NewValues, SqlValue[]? SourceValues)> pendingUpdates,
+        List<(int Page, int Slot, SqlValue[] OldValues, SqlValue[]? SourceValues)> pendingDeletes)
+    {
+        var sourceValues = sourceRows[matchedSources[0]];
+        var chosen = PickClause(whenClauses, WhenClauseKind.Matched, targetValues, sourceValues, context.Batch, resolveCombined);
+        if (chosen is null)
+            return;
+        if (chosen.Action == MergeActionKind.Update && matchedSources.Count > 1)
+            throw SimulatedSqlException.MergeMultiMatch();
+
+        ApplyChosenMatchedAction(context, destinationTable, sourceView, chosen, pageIndex, slotIndex, targetValues, sourceValues, resolveCombined, pendingUpdates, pendingDeletes);
+    }
+
     private static void ApplyChosenMatchedAction(
         ParserContext context,
         HeapTable destinationTable,

docs/claude/dml.md

@@ -74,7 +74,7 @@ Creates a destination table from the projection's inferred schema, then copies r
 `Simulation.Merge.cs:ExecuteMerge` is a single-pass walk:
 
 1. **Materialize source** once into `List<SqlValue[]>` via the parse-time `Func<BatchContext, List<SqlValue[]>>` materializer. `VALUES`-form evaluates the tuple expressions; `SELECT`-form runs `Selection.Execute` and decodes via `RowDecoder`; the bare-table / view form iterates the underlying heap or view selection respectively, then runs `EvaluateComputedColumns` per row so source-side computed columns are observable from the ON predicate / SET / INSERT projections.
-2. **Phase A — target × source**: for each target heap row, enumerate source rows; ON evaluates with a combined resolver wired to both target alias and source alias. Multiple-match collection feeds the Msg 8672 guard. For each target with ≥ 1 match, walk WHEN MATCHED clauses; first clause whose `AND` is satisfied (or absent) wins. For each target with 0 matches, walk WHEN NOT MATCHED BY SOURCE clauses the same way. Action gets queued (`pendingInserts` / `pendingUpdates` / `pendingDeletes`) along with the `(page, slot)` address + pre-update and post-update row snapshots. **Seek-accelerated when applicable**: when the ON carries a seekable target equality, there's no WHEN NOT MATCHED BY SOURCE clause, and the target isn't a view, Phase A inverts — it seeks matching targets per source row (`Selection.TryPrepareMergeTargetSeek`) and visits only matched targets, re-running the full ON per candidate (residual filter), first-source-wins and heap order preserved. ~9× faster on a large target; see [`indexes.md`](indexes.md#merge-target-seeking-loop-inversion). Declines to the full scan otherwise.
+2. **Phase A — target × source**: for each target heap row, enumerate source rows; ON evaluates with a combined resolver wired to both target alias and source alias. Multiple-match collection feeds the Msg 8672 guard. For each target with ≥ 1 match, walk WHEN MATCHED clauses; first clause whose `AND` is satisfied (or absent) wins. For each target with 0 matches, walk WHEN NOT MATCHED BY SOURCE clauses the same way. Action gets queued (`pendingInserts` / `pendingUpdates` / `pendingDeletes`) along with the `(page, slot)` address + pre-update and post-update row snapshots. **Seek-accelerated when applicable**: when the ON carries a seekable target equality and the target isn't a view, the match phase inverts — it seeks matching targets per source row (`Selection.TryPrepareMergeTargetSeek`), re-running the full ON per candidate (residual filter), first-source-wins and heap order preserved. With no WHEN NOT MATCHED BY SOURCE clause it then visits only matched targets; with one it walks the heap once applying the precomputed matches (BY-SOURCE for the rest) — dropping the inner source loop either way. ~9× faster on a large target; see [`indexes.md`](indexes.md#merge-target-seeking-loop-inversion). Declines to the full scan for a view target or a non-seekable ON.
 3. **Phase B — unmatched sources**: for each source row that didn't match any target, the single WHEN NOT MATCHED BY TARGET clause's AND condition is evaluated; if true, queue an INSERT.
 4. **Phase C — commit**: PK / UNIQUE validation runs on the union of pending inserts + updates via `EnforceKeyConstraintsForUpdate` (inserts use sentinel `(-1, i)` addresses). If a violation surfaces, every queued mutation is abandoned and the statement-atomic undo log already captures the no-heap-writes state. Then deletes tombstone, updates rewrite, inserts append, in that order.
 5. **Phase D — OUTPUT**: walk queued INSERT rows → UPDATE rows → DELETE rows; the `MergeOutputProjection` resolves `INSERTED.col` / `DELETED.col` / source-alias / `$action`. For each row, the unmatched side projects all-NULL.