Implement auto-heal for legacy corrupt free list in WAL and add corresponding tests

Author: JKamsker

LiteDB.Tests/Issues/Issue1940_Tests.cs

@@ -0,0 +1,97 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using FluentAssertions;
+using Xunit;
+
+namespace LiteDB.Tests.Issues
+{
+    public class Issue1940_Tests
+    {
+        [Fact]
+        public void OpeningDatabaseWithLegacyCorruptFreeListInWalShouldAutoHeal()
+        {
+            var tempDirectory = Path.Combine(Path.GetTempPath(), $"litedb-issue1940-{Guid.NewGuid():N}");
+            Directory.CreateDirectory(tempDirectory);
+
+            try
+            {
+                var databasePath = Path.Combine(tempDirectory, "issue1940.db");
+                var logPath = Path.Combine(tempDirectory, "issue1940-log.db");
+
+                ZipFile.ExtractToDirectory(
+                    Path.Combine(AppContext.BaseDirectory, "Resources", "Issue1940_CorruptFreeEmptyList.zip"),
+                    tempDirectory,
+                    overwriteFiles: true);
+
+                Action firstOpen = () =>
+                {
+                    using var db = new LiteDatabase(databasePath);
+                    var col = db.GetCollection<LegacyCorruptWalDoc>("verify");
+
+                    col.EnsureIndex(x => x.Tags);
+                    col.Insert(this.CreateDocs());
+
+                    db.Checkpoint();
+                };
+
+                firstOpen.Should().NotThrow();
+
+                if (File.Exists(logPath))
+                {
+                    new FileInfo(logPath).Length.Should().Be(0);
+                }
+
+                Action secondOpen = () =>
+                {
+                    using var db = new LiteDatabase(databasePath);
+                    var col = db.GetCollection<LegacyCorruptWalDoc>("verify");
+
+                    col.Insert(this.CreateDocs());
+                };
+
+                secondOpen.Should().NotThrow();
+            }
+            finally
+            {
+                if (Directory.Exists(tempDirectory))
+                {
+                    Directory.Delete(tempDirectory, true);
+                }
+            }
+        }
+
+        private IEnumerable<LegacyCorruptWalDoc> CreateDocs()
+        {
+            yield return new LegacyCorruptWalDoc
+            {
+                Number = 1,
+                Payload = new string('a', 2048),
+                Tags = new List<string> { "alpha", "beta" },
+                Values = new List<string> { "one", "two" }
+            };
+
+            yield return new LegacyCorruptWalDoc
+            {
+                Number = 2,
+                Payload = new string('b', 1536),
+                Tags = new List<string> { "gamma", "delta" },
+                Values = new List<string> { "three", "four" }
+            };
+        }
+
+        private class LegacyCorruptWalDoc
+        {
+            public ObjectId Id { get; set; } = ObjectId.NewObjectId();
+
+            public int Number { get; set; }
+
+            public string Payload { get; set; } = string.Empty;
+
+            public List<string> Tags { get; set; } = new List<string>();
+
+            public List<string> Values { get; set; } = new List<string>();
+        }
+    }
+}

LiteDB.Tests/LiteDB.Tests.csproj

@@ -25,6 +25,9 @@
     <None Update="Resources\ingest-20250922-234735.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <None Update="Resources\Issue1940_CorruptFreeEmptyList.zip">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
 
   <ItemGroup Condition="'$(Configuration)' == 'Release'">

LiteDB.Tests/Resources/Issue1940_CorruptFreeEmptyList.zip

@@ -10,6 +10,109 @@ namespace LiteDB.Engine
 {
     public partial class LiteEngine
     {
+        private void HealCorruptedFreeEmptyPageList()
+        {
+            if (_header.FreeEmptyPageList == uint.MaxValue)
+            {
+                return;
+            }
+
+            using var reader = _disk.GetReader();
+
+            var current = _header.FreeEmptyPageList;
+            var visited = new HashSet<uint>();
+
+            while (current != uint.MaxValue)
+            {
+                if (current > _header.LastPageID || visited.Add(current) == false)
+                {
+                    this.RepairFreeEmptyPageList(current, null);
+                    return;
+                }
+
+                var page = this.ReadLatestPage(current, reader);
+
+                try
+                {
+                    if (page.PageType != PageType.Empty)
+                    {
+                        this.RepairFreeEmptyPageList(current, page.PageType);
+                        return;
+                    }
+
+                    current = page.NextPageID;
+                }
+                finally
+                {
+                    page.Buffer.Release();
+                }
+            }
+        }
+
+        private BasePage ReadLatestPage(uint pageID, DiskReader reader)
+        {
+            var position = _walIndex.GetPageIndex(pageID, int.MaxValue, out _);
+
+            if (position != long.MaxValue)
+            {
+                return new BasePage(reader.ReadPage(position, false, FileOrigin.Log));
+            }
+
+            return new BasePage(reader.ReadPage(BasePage.GetPagePosition(pageID), false, FileOrigin.Data));
+        }
+
+        private void RepairFreeEmptyPageList(uint pageID, PageType? pageType)
+        {
+            LOG(
+                pageType.HasValue
+                    ? $"detected legacy corruption in free empty page list at page {pageID} ({pageType.Value}); resetting header free list"
+                    : $"detected legacy corruption in free empty page list near page {pageID}; resetting header free list",
+                "RECOVERY");
+
+            lock (_header)
+            {
+                var savepoint = _header.Savepoint();
+
+                try
+                {
+                    _header.FreeEmptyPageList = uint.MaxValue;
+                    _header.TransactionID = uint.MaxValue;
+                    _header.IsConfirmed = false;
+                    _header.UpdateBuffer();
+
+                    if (_settings.ReadOnly == false)
+                    {
+                        this.PersistRecoveredHeader();
+                    }
+                }
+                catch
+                {
+                    _header.Restore(savepoint);
+                    throw;
+                }
+            }
+        }
+
+        private void PersistRecoveredHeader()
+        {
+            var transactionID = _walIndex.NextTransactionID();
+
+            _header.TransactionID = transactionID;
+            _header.IsConfirmed = true;
+
+            var buffer = _header.UpdateBuffer();
+            var clone = _disk.NewPage();
+
+            Buffer.BlockCopy(buffer.Array, buffer.Offset, clone.Array, clone.Offset, clone.Count);
+
+            _disk.WriteLogDisk(new[] { clone });
+            _walIndex.ConfirmTransaction(transactionID, new[] { new PagePosition(0, clone.Position) });
+
+            _header.TransactionID = uint.MaxValue;
+            _header.IsConfirmed = false;
+            _header.UpdateBuffer();
+        }
+
         /// <summary>
         /// Recovery datafile using a rebuild process. Run only on "Open" database
         /// </summary>
@@ -28,4 +131,4 @@ private void Recovery(Collation collation)
             rebuilder.Rebuild(options);
         }
     }
-}
+}

LiteDB/Engine/Engine/Recovery.cs

@@ -143,6 +143,8 @@ internal bool Open()
                 if (_disk.GetFileLength(FileOrigin.Log) > 0)
                 {
                     _walIndex.RestoreIndex(ref _header);
+
+                    this.HealCorruptedFreeEmptyPageList();
                 }
 
                 // initialize sort temp disk
@@ -266,4 +268,4 @@ protected virtual void Dispose(bool disposing)
             this.Close();
         }
     }
-}
+}