Closed gaps in heap page reclamation by covering versioned autocommit and forwarded-row DELETE.

Author: RyanLamansky

CLAUDE.md

@@ -213,7 +213,7 @@ Per-feature deep-dives live under `docs/claude/`. Each entry below is a trigger:
 - `decimal` / `numeric`: backed by .NET `decimal`. Values requiring more than 28 significant digits aren't modeled (declarations up through `decimal(38, *)` accepted so storage byte-width matches).
 - `float` text formatting: .NET `G15`/`G7` rather than SQL Server's `1e+015`-style scientific.
 - Auto-generated constraint names: PK / UNIQUE shape `PK__<table8>__<16hex>` / `UQ__<table8>__<16hex>` (16-hex 64-bit FNV-1a); CK / FK / DF shape `CK__<table8>__[<col8>__]<8hex>` (8-hex 32-bit FNV-1a). Both are deterministic across runs, distinct from SQL Server's object-id-derived hex (so won't byte-match).
-- **Reclaimed heap space is reused, but the page lists shrink only from the tail**: superseded row bytes and off-row LOB chains are freed and reused like SQL Server (see `HeapPage.Compact` / `Heap.FreeLobChain`), so memory tracks the *peak concurrent* working set. The lists don't shrink on their own — a fully-dead interior page is reused in place but never removed from `Heap.Pages`, and a reclaimed slot keeps a 2-byte zero-extent directory entry, because mid-list removal / slot renumbering would break the stable `(page, slot)` addresses cursors, version Rids, and forward pointers depend on. `DBCC SHRINKDATABASE` / `DBCC SHRINKFILE` lower the high-water mark on demand by trimming the *trailing* run of fully-dead data pages and freed LOB pages (`Heap.TrimTrailingDeadPages` / `TrimTrailingFreeLobPages`, after a version-store GC pass); interior dead pages and any version- or lock-pinned tail page stay put. SHRINKDATABASE emits no result set (matches the real server's nothing-to-report case); SHRINKFILE returns the documented per-file row (`DbId, FileId, CurrentSize, MinimumSize, UsedPages, EstimatedPages`) with sizes synthesized from heap page totals, since no physical file model exists. Two residual leaks remain: deleting a *forwarded* row reclaims only the pointer slot, not the relocated target's bytes; and a LOB chain superseded by a versioning-on **autocommit** statement isn't reclaimed (version-store GC fires only at explicit-tx commit).
+- **Reclaimed heap space is reused, but the page lists shrink only from the tail**: superseded row bytes and off-row LOB chains are freed and reused like SQL Server (see `HeapPage.Compact` / `Heap.FreeLobChain`), so memory tracks the *peak concurrent* working set. The lists don't shrink on their own — a fully-dead interior page is reused in place but never removed from `Heap.Pages`, and a reclaimed slot keeps a 2-byte zero-extent directory entry, because mid-list removal / slot renumbering would break the stable `(page, slot)` addresses cursors, version Rids, and forward pointers depend on. `DBCC SHRINKDATABASE` / `DBCC SHRINKFILE` lower the high-water mark on demand by trimming the *trailing* run of fully-dead data pages and freed LOB pages (`Heap.TrimTrailingDeadPages` / `TrimTrailingFreeLobPages`, after a version-store GC pass); interior dead pages and any version- or lock-pinned tail page stay put. SHRINKDATABASE emits no result set (matches the real server's nothing-to-report case); SHRINKFILE returns the documented per-file row (`DbId, FileId, CurrentSize, MinimumSize, UsedPages, EstimatedPages`) with sizes synthesized from heap page totals, since no physical file model exists. Deleting a *forwarded* row reclaims both the pointer slot and the relocated target (`Heap.DeleteAt`'s forwarding branch); a versioning-on **autocommit** UPDATE/DELETE reclaims its superseded chains via a version-store GC pass at statement end when no snapshot is open (an open snapshot legitimately holds them until it closes).
 - **`GetBytes`/`GetChars` materialize, don't stream**: each call decodes the full column value via `RowDecoder` and slices into the caller's buffer. Behavior matches per-call observation; the streaming-memory guarantee doesn't.
 - **`SELECT INTO` string `+` reads as nullable**: real SQL Server projects `cs + 'x'` (both NOT NULL) as NOT NULL; the simulator can't statically distinguish string-concat from integer-add at projection-schema time (the dispatch happens runtime on operand types), so all `Add` results read as nullable. Conservative; no test reliance on string-`+`-non-null.
 - **`SELECT INTO` from a CTE drops identity + nullability**: CTE bindings synthesize their wrapper `HeapColumn` entries with `nullable: true` and no identity, so the analyzer treats CTE sources as derived plans. Real SQL Server preserves both through simple single-source CTEs. Fix requires propagating column metadata through CTE bindings.

SqlServerSimulator.Tests.Internal/Storage/LobReclamationTests.cs

@@ -136,6 +136,26 @@ public void VersionedUpdate_ReclaimsViaGarbageCollection()
         IsLessThanOrEqualTo(8, HeapFor(conn, "t").LobPages.Count, "Version-store GC should trim each committed tx's history entry and free its chain when no snapshot needs it.");
     }
 
+    // Under RCSI with no concurrent snapshot reader, even plain autocommit
+    // UPDATEs (no explicit transaction) must reclaim their superseded chains:
+    // version-store GC runs at the end of a versioned autocommit statement when
+    // no snapshot is open, so churn stays bounded without an explicit COMMIT to
+    // trigger collection.
+    [TestMethod]
+    public void VersionedAutocommitUpdate_ReclaimsWithoutExplicitTransaction()
+    {
+        var conn = new Simulation().CreateDbConnection();
+        conn.Open();
+        Exec(conn, "alter database simulated set read_committed_snapshot on");
+        Exec(conn, "create table t (id int not null primary key, v nvarchar(max) not null)");
+        Exec(conn, "insert t values (1, replicate(N'a', 100))");
+
+        for (var i = 0; i < 100; i++)
+            Exec(conn, "update t set v = replicate(N'b', 100) where id = 1");
+
+        IsLessThanOrEqualTo(8, HeapFor(conn, "t").LobPages.Count, "A versioned autocommit UPDATE with no open snapshot should GC its superseded chain immediately, not wait for an explicit-tx commit.");
+    }
+
     // Reclamation must not corrupt a surviving value: the live row reads back
     // correctly after extensive churn around it.
     [TestMethod]

SqlServerSimulator.Tests.Internal/Storage/PageReclamationTests.cs

@@ -115,6 +115,48 @@ public void Compaction_PreservesManyInterleavedSurvivors()
         }
     }
 
+    // Deleting a *forwarded* row must reclaim the relocated target's bytes, not
+    // just the pointer slot. Each cycle grows a row (forwarding it to a fresh
+    // target) then deletes it; without target reclamation the target pages
+    // accrete one per cycle.
+    [TestMethod]
+    public void ForwardedRowDeleteChurn_KeepsPagesBounded()
+    {
+        var conn = new Simulation().CreateDbConnection();
+        conn.Open();
+        Exec(conn, "create table t (id int not null primary key, v varchar(7000) not null)");
+
+        for (var i = 0; i < 100; i++)
+        {
+            Exec(conn, "insert t values (1, 'x')");
+            Exec(conn, "update t set v = replicate('a', 7000) where id = 1");
+            Exec(conn, "delete from t where id = 1");
+        }
+
+        IsLessThanOrEqualTo(6, HeapFor(conn, "t").Pages.Count, "Deleting a forwarded row should reclaim the relocated target's page space, not orphan a page per cycle.");
+    }
+
+    // Rolling back a forwarded-row DELETE must restore the row exactly once — the
+    // pointer and target both un-tombstone and the target re-registers as a
+    // forward target, so the row surfaces via the forwarder rather than also
+    // appearing as a standalone live row.
+    [TestMethod]
+    public void RolledBackForwardedRowDelete_RestoresRowExactlyOnce()
+    {
+        var conn = new Simulation().CreateDbConnection();
+        conn.Open();
+        Exec(conn, "create table t (id int not null primary key, v varchar(7000) not null)");
+        Exec(conn, "insert t values (1, 'x')");
+        Exec(conn, "update t set v = replicate('a', 7000) where id = 1"); // grows → forwards
+        Exec(conn, "begin tran; delete from t where id = 1; rollback");
+
+        using var cmd = conn.CreateCommand();
+        cmd.CommandText = "select count(*) from t";
+        AreEqual(1, cmd.ExecuteScalar());
+        cmd.CommandText = "select len(v) from t where id = 1";
+        AreEqual(7000, cmd.ExecuteScalar());
+    }
+
     // A rolled-back DELETE must not be reclaimed: compaction has to preserve an
     // uncommitted-tombstone's bytes so the row comes back intact.
     [TestMethod]

SqlServerSimulator/Simulation/Simulation.cs

@@ -1155,13 +1155,26 @@ private static SimulatedStatementOutcome RunMutation(ParserContext context, Func
             var outcome = body(context);
             if (statementVersionEntries is { } autoCommitEntries)
             {
+                // FinalizePendingEntries clears the list, so capture whether
+                // this statement versioned anything before the call.
+                var versionedThisStatement = autoCommitEntries.Count > 0;
                 Storage.VersionStore.FinalizePendingEntries(autoCommitEntries, context.CurrentDatabase);
                 // Auto-commit statement: its writes are now permanent, so
                 // commit the throwaway log — reclaiming chains superseded by
                 // this statement's UPDATE/DELETEs (unversioned path). Under an
                 // explicit tx (statementVersionEntries is null) the entries
                 // stay on the tx's log until COMMIT instead.
                 log.Commit();
+                // When this statement versioned its superseded rows, those
+                // images are pinned only by the HistoricalVersions just
+                // created. With no snapshot open nothing needs them, so collect
+                // now rather than leaving them until the next explicit-tx
+                // commit (the version-store analog of the unversioned
+                // log.Commit() above). An active snapshot legitimately needs the
+                // versions, so defer — and skip the scan — until it closes.
+                var autoCommitDatabase = context.CurrentDatabase;
+                if (versionedThisStatement && autoCommitDatabase.ActiveSnapshotTxs.IsEmpty)
+                    Storage.VersionStore.RunGarbageCollection(autoCommitDatabase);
             }
             // Table-variable writes are non-transactional and final on
             // statement success regardless of any enclosing tx, so their

SqlServerSimulator/Storage/Heap.cs

@@ -302,20 +302,41 @@ public IEnumerable<byte[]> EnumerateRows()
 
     /// <summary>
     /// Marks the row at <paramref name="pageIndex"/> / <paramref name="slotIndex"/>
-    /// as deleted. The slot is tombstoned at the page level; row payload bytes
-    /// are not reclaimed (the heap-page-byte leak quirk in CLAUDE.md). The
-    /// row's off-row LOB chains, however, are reclaimed: when
+    /// as deleted. The slot is tombstoned at the page level; on commit its undo
+    /// entry marks the slot reclaimable so <see cref="HeapPage.Compact"/> can
+    /// pack the bytes away. The row's off-row LOB chains are reclaimed too: when
     /// <paramref name="reclaimSuperseded"/> is set (no <c>HistoricalVersion</c>
     /// will pin them — see <see cref="VersionStore.WillCaptureVersions"/>) the
     /// recorded undo entry frees them on commit via
     /// <see cref="FreeLobChain"/>; otherwise version-store GC frees them once
     /// no snapshot needs the deleted row.
     /// </summary>
+    /// <remarks>
+    /// When the visible slot is a forwarding pointer, the row's payload (and any
+    /// off-row chains) lives at the relocated target, not the pointer slot — so
+    /// both are deleted, and the target is unregistered from
+    /// <see cref="ForwardTargets"/>. The target's Delete entry carries the
+    /// <paramref name="reclaimSuperseded"/> gate (it owns the row + chains); the
+    /// pointer's entry only reclaims its directory slot — its bytes are a
+    /// forward pointer, not a row, so they must never be decoded for LOB heads.
+    /// Rollback resurrects both slots and re-registers the target.
+    /// </remarks>
     public void DeleteAt(int pageIndex, int slotIndex, UndoLog? undoLog = null, bool reclaimSuperseded = false)
     {
         this.MutationGeneration++;
+        var page = this.Pages[pageIndex];
+        if (page.IsSlotForwarded(slotIndex))
+        {
+            var target = page.ReadForwardTarget(slotIndex);
+            undoLog?.RecordDelete(this, target.PageIndex, target.SlotIndex, reclaimSuperseded);
+            this.Pages[target.PageIndex].DeleteSlot(target.SlotIndex);
+            undoLog?.RecordForwardedPointerDelete(this, pageIndex, slotIndex, target);
+            page.DeleteSlot(slotIndex);
+            _ = this.ForwardTargets.Remove(target);
+            return;
+        }
         undoLog?.RecordDelete(this, pageIndex, slotIndex, reclaimSuperseded);
-        this.Pages[pageIndex].DeleteSlot(slotIndex);
+        page.DeleteSlot(slotIndex);
     }
 
     /// <summary>
@@ -421,6 +442,15 @@ internal void SwapForwardTargetForUndo((int Page, int Slot) oldTarget, (int Page
         _ = this.ForwardTargets.Add(oldTarget);
     }
 
+    /// <summary>
+    /// Undo callback for <see cref="UndoLog.RecordForwardedPointerDelete"/> —
+    /// re-registers a target whose forwarding row was deleted, so the
+    /// resurrected pointer surfaces the row once (via the forwarder) rather than
+    /// the target also appearing as a standalone live row.
+    /// </summary>
+    internal void ReinstateForwardTargetForUndo((int Page, int Slot) target) =>
+        _ = this.ForwardTargets.Add(target);
+
     /// <summary>
     /// Returns a fresh copy of the row bytes at the given Rid, dereferencing
     /// one level of forwarding so callers see the live row's payload even

SqlServerSimulator/Storage/UndoLog.cs

@@ -75,6 +75,9 @@ public void RecordForwardInstall(Heap heap, int pageIndex, int slotIndex, byte[]
     public void RecordForwardRetarget(Heap heap, int pageIndex, int slotIndex, (int Page, int Slot) oldTarget, (int Page, int Slot) newTarget) =>
         this.entries.Add(new SlotRewrite(heap, SlotRewriteKind.ForwardRetarget, pageIndex, slotIndex, [], oldTarget, newTarget, freeOnCommit: false));
 
+    public void RecordForwardedPointerDelete(Heap heap, int pageIndex, int slotIndex, (int Page, int Slot) target) =>
+        this.entries.Add(new ForwardedPointerDelete(heap, pageIndex, slotIndex, target));
+
     public void RecordTempTableCreation(ConcurrentDictionary<string, HeapTable> owner, string name) =>
         this.entries.Add(new TempTableCreation(owner, name));
 
@@ -275,6 +278,33 @@ public override void Commit()
         }
     }
 
+    /// <summary>
+    /// Undo entry for the forwarding-pointer half of a forwarded-row DELETE
+    /// (the relocated target is handled by a paired <see cref="UndoKind.Delete"/>
+    /// <see cref="SlotChange"/>). The pointer slot holds a forward pointer, not a
+    /// row, so commit reclaims only its directory slot — never decoding it for
+    /// LOB chains — and undo resurrects the pointer and re-registers the target.
+    /// </summary>
+    private sealed class ForwardedPointerDelete(Heap heap, int pageIndex, int slotIndex, (int Page, int Slot) target) : UndoEntry
+    {
+        public readonly Heap Heap = heap;
+        public readonly int PageIndex = pageIndex;
+        public readonly int SlotIndex = slotIndex;
+        public readonly (int Page, int Slot) Target = target;
+
+        public override void Undo()
+        {
+            this.Heap.Pages[this.PageIndex].UndeleteSlot(this.SlotIndex);
+            this.Heap.ReinstateForwardTargetForUndo(this.Target);
+        }
+
+        public override void Commit()
+        {
+            this.Heap.Pages[this.PageIndex].MarkSlotReclaimable(this.SlotIndex);
+            this.Heap.MarkPageReclaimable(this.PageIndex);
+        }
+    }
+
     private sealed class TempTableCreation(ConcurrentDictionary<string, HeapTable> owner, string name) : UndoEntry
     {
         public readonly ConcurrentDictionary<string, HeapTable> Owner = owner;