Merge pull request #17 from kduma-OSS/claude/pcf-partition-table-header-5QdNX

PCF: add end-of-file Trailer + chain direction; adopt in PFS-MS (Rust)

Author: kduma

.github/workflows/ci-pfs.yml

@@ -69,7 +69,7 @@ jobs:
       - name: Inspect generated test vector
         run: |
           ls -l pfs_ms_testvector.bin
-          test "$(wc -c < pfs_ms_testvector.bin)" = "2986"
+          test "$(wc -c < pfs_ms_testvector.bin)" = "3066"
       - uses: actions/upload-artifact@v4
         with:
           name: pfs-ms-testvector

implementations/dotnet/src/Pcf/Constants.cs

@@ -49,4 +49,30 @@ public static class Constants
 
     /// <summary>The NIL UID (all zero). MUST NOT label a live partition.</summary>
     public static readonly byte[] NilUid = new byte[UidSize];
+
+    /// <summary>
+    /// Sentinel value of <c>partition_table_offset</c> (header offset 12) meaning
+    /// the partition-table head is recorded in the file <see cref="Trailer"/> at
+    /// the end of the file rather than in the header (spec section 4, "File
+    /// Trailer"). The all-ones u64 can never be a real offset, so it is
+    /// unambiguous.
+    /// </summary>
+    public const ulong PtOffsetTrailer = 0xFFFF_FFFF_FFFF_FFFF;
+
+    /// <summary>Fixed size of the optional file trailer, in bytes.</summary>
+    public const long TrailerSize = 20;
+
+    /// <summary>
+    /// Trailer signature, 8 bytes: the file <see cref="Magic"/> reversed
+    /// (<c>0x1A 0x0A 0x0D 'T' 'R' 'P' 'K' 0x89</c>). Placed as the final 8 bytes
+    /// of the file so a reader can detect and validate the trailer at the end.
+    /// </summary>
+    public static readonly byte[] TrailerMagic =
+        { 0x1A, 0x0A, 0x0D, (byte)'T', (byte)'R', (byte)'P', (byte)'K', 0x89 };
+
+    /// <summary>Chain-direction flag: forward chain, head = first block.</summary>
+    public const byte ChainForward = 0;
+
+    /// <summary>Chain-direction flag: backward chain, head = last/newest block.</summary>
+    public const byte ChainBackward = 1;
 }

implementations/dotnet/src/Pcf/Container.cs

@@ -42,6 +42,17 @@ private sealed class BlockInfo
     private uint _defaultCapacity;
     private HashAlgo _tableHashAlgo;
 
+    /// <summary>
+    /// Resolved absolute offset of the partition-table head: the header pointer
+    /// for a classic file, or the offset from the file <see cref="Trailer"/>
+    /// when the header holds <see cref="Constants.PtOffsetTrailer"/>. 0 denotes
+    /// an empty table.
+    /// </summary>
+    private ulong _tableHead;
+
+    /// <summary>Chain-direction flags resolved at open time (see <see cref="Trailer"/>).</summary>
+    private byte _chainFlags;
+
     private Container(Stream storage)
     {
         _storage = storage;
@@ -100,10 +111,22 @@ public static Container CreateWith(Stream storage, uint firstBlockCapacity, Hash
         };
         c._defaultCapacity = cap;
         c._tableHashAlgo = tableHashAlgo;
+        c._tableHead = (ulong)Constants.HeaderSize;
+        c._chainFlags = Constants.ChainForward;
         return c;
     }
 
-    /// <summary>Open an existing container, validating the header (spec C1, C2).</summary>
+    /// <summary>
+    /// Open an existing container, validating the header (spec C1, C2).
+    ///
+    /// <para>When the header's <c>partition_table_offset</c> is the
+    /// <see cref="Constants.PtOffsetTrailer"/> sentinel, the partition-table head
+    /// and chain direction are read from the file <see cref="Trailer"/> (located
+    /// by scanning backward from the end of the file). Chain traversal is
+    /// identical in both directions (follow <c>next_table_offset</c> until 0);
+    /// the direction only conveys which end is newest, exposed via
+    /// <see cref="ChainIsBackward"/>.</para>
+    /// </summary>
     public static Container Open(Stream storage)
     {
         var c = new Container(storage)
@@ -117,8 +140,18 @@ public static Container Open(Stream storage)
         c.ReadAt(0, hb, 20);
         c._header = FileHeader.FromBytes(hb);
 
+        if (c._header.PartitionTableOffset == Constants.PtOffsetTrailer)
+        {
+            (c._tableHead, c._chainFlags) = c.LocateTrailer();
+        }
+        else
+        {
+            c._tableHead = c._header.PartitionTableOffset;
+            c._chainFlags = Constants.ChainForward;
+        }
+
         var blocks = new List<BlockInfo>();
-        ulong off = c._header.PartitionTableOffset;
+        ulong off = c._tableHead;
         while (off != 0)
         {
             (TableBlockHeader h, _) = c.ReadBlock(off);
@@ -144,9 +177,83 @@ public static Container Open(Stream storage)
     /// <summary>The backing store.</summary>
     public Stream Storage => _storage;
 
-    /// <summary>The parsed file header.</summary>
+    /// <summary>
+    /// The parsed file header. In trailer mode its <c>PartitionTableOffset</c>
+    /// holds the <see cref="Constants.PtOffsetTrailer"/> sentinel; use
+    /// <see cref="TableHead"/> for the resolved head.
+    /// </summary>
     public FileHeader Header => _header;
 
+    /// <summary>
+    /// The resolved absolute offset of the partition-table head (0 if empty).
+    /// This is the value to follow regardless of header-pointer vs trailer mode.
+    /// </summary>
+    public ulong TableHead => _tableHead;
+
+    /// <summary>
+    /// Whether the chain is backward-linked (head = newest block,
+    /// <c>next_table_offset</c> points at the previous/older block). Classic
+    /// header-pointer files are always forward.
+    /// </summary>
+    public bool ChainIsBackward => (_chainFlags & 1) != 0;
+
+    /// <summary>
+    /// Locate the most recent valid file trailer by scanning backward from the
+    /// end of the file for the last 20-byte window ending in
+    /// <see cref="Constants.TrailerMagic"/> whose recorded head is empty (0) or
+    /// references a parseable table block. Bytes after that trailer — an
+    /// incomplete or aborted append — are ignored, which gives append-only
+    /// writers crash recovery for free. In the clean case the trailer is the
+    /// final <see cref="Constants.TrailerSize"/> bytes.
+    /// </summary>
+    private (ulong, byte) LocateTrailer()
+    {
+        long fileLen = _storage.Seek(0, SeekOrigin.End);
+        var tb = new byte[20];
+        long end = fileLen;
+        while (end >= Constants.TrailerSize)
+        {
+            long start = end - Constants.TrailerSize;
+            ReadAt((ulong)start, tb, 20);
+            bool magicOk = true;
+            for (int i = 0; i < 8; i++)
+            {
+                if (tb[12 + i] != Constants.TrailerMagic[i])
+                {
+                    magicOk = false;
+                    break;
+                }
+            }
+            if (magicOk)
+            {
+                Trailer t = Trailer.FromBytes(tb);
+                if (t.PartitionTableOffset == 0)
+                {
+                    return (0, t.ChainFlags);
+                }
+                // Guard against the magic appearing inside an aborted tail: the
+                // recorded head must precede this trailer and parse as a block.
+                if (start >= Constants.TableHeaderSize
+                    && t.PartitionTableOffset <= (ulong)(start - Constants.TableHeaderSize))
+                {
+                    try
+                    {
+                        var bhb = new byte[74];
+                        ReadAt(t.PartitionTableOffset, bhb, 74);
+                        TableBlockHeader.FromBytes(bhb);
+                        return (t.PartitionTableOffset, t.ChainFlags);
+                    }
+                    catch (PcfException)
+                    {
+                        // Spurious magic in an aborted tail; keep scanning.
+                    }
+                }
+            }
+            end -= 1;
+        }
+        throw PcfException.BadTrailer();
+    }
+
     // ---- low-level I/O ----------------------------------------------------
 
     private void ReadAt(ulong off, byte[] buf, int len)
@@ -217,7 +324,7 @@ private void WriteBlock(ulong off, ulong next, HashAlgo algo, IReadOnlyList<Part
     public List<PartitionEntry> Entries()
     {
         var outp = new List<PartitionEntry>();
-        ulong off = _header.PartitionTableOffset;
+        ulong off = _tableHead;
         while (off != 0)
         {
             (TableBlockHeader h, List<PartitionEntry> entries) = ReadBlock(off);
@@ -254,7 +361,7 @@ public byte[] ReadPartitionData(PartitionEntry entry)
 
     private (ulong, int, PartitionEntry) Locate(byte[] uid)
     {
-        ulong off = _header.PartitionTableOffset;
+        ulong off = _tableHead;
         while (off != 0)
         {
             (TableBlockHeader h, List<PartitionEntry> entries) = ReadBlock(off);
@@ -423,7 +530,7 @@ public void RemovePartition(byte[] uid)
     /// </summary>
     public void Verify()
     {
-        ulong off = _header.PartitionTableOffset;
+        ulong off = _tableHead;
         while (off != 0)
         {
             (TableBlockHeader h, List<PartitionEntry> entries) = ReadBlock(off);
@@ -465,7 +572,7 @@ public byte[] CompactedImage()
     {
         // Gather live entries and their data, in chain order.
         var live = new List<KeyValuePair<PartitionEntry, byte[]>>();
-        ulong off = _header.PartitionTableOffset;
+        ulong off = _tableHead;
         while (off != 0)
         {
             (TableBlockHeader h, List<PartitionEntry> entries) = ReadBlock(off);
@@ -563,6 +670,33 @@ public void CompactInto(Stream output)
         output.Write(img, 0, img.Length);
     }
 
+    // ---- trailer mode -----------------------------------------------------
+
+    /// <summary>
+    /// Convert the file to trailer mode: append a fixed <see cref="Trailer"/> at
+    /// the end of the file recording the current partition-table head, then
+    /// overwrite the header's <c>partition_table_offset</c> with the
+    /// <see cref="Constants.PtOffsetTrailer"/> sentinel so the head is located
+    /// via that trailer. The chain built by this writer is forward-linked, so the
+    /// trailer records <see cref="Constants.ChainForward"/>.
+    /// </summary>
+    public void FinalizeWithTrailer()
+    {
+        var trailer = new Trailer
+        {
+            PartitionTableOffset = _tableHead,
+            ChainFlags = Constants.ChainForward,
+        };
+        long pos = _storage.Seek(0, SeekOrigin.End);
+        byte[] tb = trailer.ToBytes();
+        WriteAt((ulong)pos, tb, tb.Length);
+        _header.PartitionTableOffset = Constants.PtOffsetTrailer;
+        byte[] hb = _header.ToBytes();
+        WriteAt(0, hb, hb.Length);
+        _chainFlags = Constants.ChainForward;
+        _dataEof = (ulong)pos + (ulong)Constants.TrailerSize;
+    }
+
     // ---- helpers ----------------------------------------------------------
 
     private static uint Clamp(uint v, uint lo, uint hi) => v < lo ? lo : (v > hi ? hi : v);

implementations/dotnet/src/Pcf/PcfError.cs

@@ -44,6 +44,9 @@ public enum PcfError
 
     /// <summary>An attempt was made to add a partition whose UID already exists.</summary>
     DuplicateUid,
+
+    /// <summary>The header requested trailer-based location but no valid trailer exists.</summary>
+    BadTrailer,
 }
 
 /// <summary>An error raised by a PCF reader or writer operation.</summary>
@@ -93,4 +96,7 @@ internal static PcfException NotFound() =>
 
     internal static PcfException DuplicateUid() =>
         new(PcfError.DuplicateUid, "duplicate UID");
+
+    internal static PcfException BadTrailer() =>
+        new(PcfError.BadTrailer, "missing or invalid file trailer");
 }

implementations/dotnet/src/Pcf/Trailer.cs

@@ -0,0 +1,57 @@
+using System;
+
+namespace Pcf;
+
+/// <summary>
+/// The optional fixed 20-byte file trailer (spec section 4, "File Trailer").
+///
+/// <para>A trailer is present only when the file header's
+/// <c>partition_table_offset</c> holds the <see cref="Constants.PtOffsetTrailer"/>
+/// sentinel. It occupies the final <see cref="Constants.TrailerSize"/> bytes of
+/// the file and records the real offset of the partition-table head together
+/// with the chain direction. Because every append places a fresh trailer at the
+/// new end of file, the file's last bytes always point at the newest table —
+/// enabling append-only writers with no in-place header rewrite.</para>
+/// </summary>
+public sealed class Trailer
+{
+    /// <summary>Absolute offset of the partition-table head (0 = empty container).</summary>
+    public ulong PartitionTableOffset { get; set; }
+
+    /// <summary>Chain-direction flags; bit 0 selects forward (0) or backward (1).</summary>
+    public byte ChainFlags { get; set; }
+
+    /// <summary>Serialise to the on-disk 20-byte layout (reserved bytes 9..12 are zero).</summary>
+    public byte[] ToBytes()
+    {
+        var b = new byte[20];
+        LittleEndian.WriteU64(b, 0, PartitionTableOffset);
+        b[8] = ChainFlags;
+        Buffer.BlockCopy(Constants.TrailerMagic, 0, b, 12, 8);
+        return b;
+    }
+
+    /// <summary>
+    /// Parse from the on-disk 20-byte layout, validating the trailer magic.
+    /// Throws <see cref="PcfException"/> (BadTrailer) if the magic does not match.
+    /// </summary>
+    public static Trailer FromBytes(byte[] b)
+    {
+        if (b.Length < Constants.TrailerSize)
+        {
+            throw PcfException.BadTrailer();
+        }
+        for (int i = 0; i < 8; i++)
+        {
+            if (b[12 + i] != Constants.TrailerMagic[i])
+            {
+                throw PcfException.BadTrailer();
+            }
+        }
+        return new Trailer
+        {
+            PartitionTableOffset = LittleEndian.ReadU64(b, 0),
+            ChainFlags = b[8],
+        };
+    }
+}