WIP: B-tree split logic bug investigation and partial fixes

Identified critical bug in BTree SplitChild causing key/value desynchronization

Fixed InsertKey to maintain keysCount/valuesCount synchronization

Fixed RemoveKeyAt to keep key/value arrays aligned

Updated SplitChild to copy values for both leaf and internal nodes

Added debug tests to isolate the split corruption issue

ISSUE: BTree still losing keys during split operations

Status: Requires complete rewrite of split logic for proper B+ tree implementation

Author: MPCoreDeveloper

DebugRangeQuery.cs

@@ -0,0 +1,37 @@
+using SharpCoreDB.DataStructures;
+using System;
+using System.Linq;
+
+var index = new BTreeIndex<int>("test");
+
+// Add values 1-100
+for (int i = 1; i <= 100; i++)
+{
+    index.Add(i, i * 10L);
+}
+
+// Get full inorder traversal
+var allResults = index.GetSortedEntries().ToList();
+Console.WriteLine($"Total entries in tree: {allResults.Count}");
+
+// Check what values are in the range 20-80
+var rangeResults = index.FindRange(20, 80).ToList();
+Console.WriteLine($"Range [20-80] results: {rangeResults.Count} items");
+Console.WriteLine($"First 10: {string.Join(", ", rangeResults.Take(10))}");
+Console.WriteLine($"Last 10: {string.Join(", ", rangeResults.Skip(Math.Max(0, rangeResults.Count - 10)))}");
+
+// Check which IDs are missing
+var expectedIds = Enumerable.Range(20, 61).Select(id => id * 10L).ToHashSet();
+var missingIds = expectedIds.Where(id => !rangeResults.Contains(id)).ToList();
+Console.WriteLine($"\nMissing {missingIds.Count} items: {string.Join(", ", missingIds.Take(20))}");
+
+// Test simple case
+var simpleIndex = new BTreeIndex<int>("simple");
+simpleIndex.Add(1, 10);
+simpleIndex.Add(2, 20);
+simpleIndex.Add(3, 30);
+simpleIndex.Add(4, 40);
+simpleIndex.Add(5, 50);
+
+var simpleRange = simpleIndex.FindRange(2, 4).ToList();
+Console.WriteLine($"\nSimple test [2-4]: got {simpleRange.Count}, expected 3: {string.Join(", ", simpleRange)}");

fix_bounds_checking.py

@@ -0,0 +1,53 @@
+import re
+
+# Read the file
+with open('src/SharpCoreDB/DatabaseExtensions.cs', 'r', encoding='utf-8') as f:
+    content = f.read()
+
+# Fix 1: ReadString bounds check
+content = re.sub(
+    r'(private static string ReadString\(ReadOnlySpan<byte> data, ref int offset\)\s*\{\s*var length = ReadInt32\(data, ref offset\);)',
+    r'\1\n            if (length < 0 || offset + length > data.Length)\n                throw new InvalidOperationException($"String truncated at offset {offset}: length={length}, remaining={data.Length - offset}");',
+    content
+)
+
+# Fix 2: ReadBytes bounds check
+content = re.sub(
+    r'(private static byte\[\] ReadBytes\(ReadOnlySpan<byte> data, ref int offset\)\s*\{\s*var length = ReadInt32\(data, ref offset\);)',
+    r'\1\n            if (length < 0 || offset + length > data.Length)\n                throw new InvalidOperationException($"Bytes truncated at offset {offset}: length={length}, remaining={data.Length - offset}");',
+    content
+)
+
+# Fix 3: ReadGuid bounds check
+content = re.sub(
+    r'(private static Guid ReadGuid\(ReadOnlySpan<byte> data, ref int offset\)\s*\{)',
+    r'\1\n            if (offset + 16 > data.Length)\n                throw new InvalidOperationException($"Guid truncated at offset {offset}: need 16 bytes, have {data.Length - offset}");',
+    content
+)
+
+# Fix 4: ReadInt32 bounds check
+content = re.sub(
+    r'(private static int ReadInt32\(ReadOnlySpan<byte> data, ref int offset\)\s*\{)',
+    r'\1\n            if (offset + 4 > data.Length)\n                throw new InvalidOperationException($"Int32 truncated at offset {offset}: need 4 bytes, have {data.Length - offset}");',
+    content
+)
+
+# Fix 5: ReadInt64 bounds check
+content = re.sub(
+    r'(private static long ReadInt64\(ReadOnlySpan<byte> data, ref int offset\)\s*\{)',
+    r'\1\n            if (offset + 8 > data.Length)\n                throw new InvalidOperationException($"Int64 truncated at offset {offset}: need 8 bytes, have {data.Length - offset}");',
+    content
+)
+
+# Fix 6: ReadBoolean bounds check
+content = re.sub(
+    r'(private static bool ReadBoolean\(ReadOnlySpan<byte> data, ref int offset\)\s*\{)',
+    r'\1\n            if (offset + 1 > data.Length)\n                throw new InvalidOperationException($"Boolean truncated at offset {offset}: need 1 byte, have {data.Length - offset}");',
+    content
+)
+
+# Write the fixed file
+with open('src/SharpCoreDB/DatabaseExtensions.cs', 'w', encoding='utf-8') as f:
+    f.write(content)
+
+print("Bounds checking added to all Read methods successfully!")

fix_bounds_v2.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""Add bounds checking to binary deserialization Read methods."""
+
+def add_bounds_check(lines, start_pattern, check_code):
+    """Add bounds checking after a pattern match."""
+    i = 0
+    while i < len(lines):
+        if start_pattern in lines[i]:
+            # Found the pattern - check if bounds check already exists
+            if i + 1 < len(lines) and 'truncated at offset' in lines[i + 1]:
+                print(f"Skipping {start_pattern} - already has bounds check")
+                i += 1
+                continue
+            # Insert the check after the line
+            lines.insert(i + 1, check_code)
+            print(f"Added bounds check after: {lines[i].strip()}")
+            i += 2  # Skip the newly inserted line
+        else:
+            i += 1
+    return lines
+
+# Read file
+with open('src/SharpCoreDB/DatabaseExtensions.cs', 'r', encoding='utf-8') as f:
+    lines = f.readlines()
+
+# Add bounds checks
+lines = add_bounds_check(
+    lines,
+    'private static string ReadString(ReadOnlySpan<byte> data, ref int offset)',
+    '        {\n            var length = ReadInt32(data, ref offset);\n            if (length < 0 || offset + length > data.Length)\n                throw new InvalidOperationException($"String truncated at offset {offset}: length={length}, remaining={data.Length - offset}");\n'
+)
+
+# For ReadString, we need a different approach - find the specific line pattern
+i = 0
+while i < len(lines):
+    if 'private static string ReadString' in lines[i]:
+        # Find the next line that has "var length = ReadInt32"
+        j = i + 1
+        while j < len(lines) and j < i + 10:  # Look ahead up to 10 lines
+            if 'var length = ReadInt32(data, ref offset);' in lines[j]:
+                # Check if next line already has bounds check
+                if j + 1 < len(lines) and 'truncated at offset' not in lines[j + 1]:
+                    # Insert bounds check
+                    lines.insert(j + 1, '            if (length < 0 || offset + length > data.Length)\n')
+                    lines.insert(j + 2, '                throw new InvalidOperationException($"String truncated at offset {offset}: length={length}, remaining={data.Length - offset}");\n')
+                    print(f"Added ReadString bounds check at line {j}")
+                break
+            j += 1
+    elif 'private static byte[] ReadBytes' in lines[i]:
+        # Find the next line that has "var length = ReadInt32"
+        j = i + 1
+        while j < len(lines) and j < i + 10:
+            if 'var length = ReadInt32(data, ref offset);' in lines[j]:
+                if j + 1 < len(lines) and 'truncated at offset' not in lines[j + 1]:
+                    lines.insert(j + 1, '            if (length < 0 || offset + length > data.Length)\n')
+                    lines.insert(j + 2, '                throw new InvalidOperationException($"Bytes truncated at offset {offset}: length={length}, remaining={data.Length - offset}");\n')
+                    print(f"Added ReadBytes bounds check at line {j}")
+                break
+            j += 1
+    elif 'private static Guid ReadGuid' in lines[i] and '{' in lines[i + 1]:
+        # Insert after the opening brace
+        if 'truncated at offset' not in lines[i + 2]:
+            lines.insert(i + 2, '            if (offset + 16 > data.Length)\n')
+            lines.insert(i + 3, '                throw new InvalidOperationException($"Guid truncated at offset {offset}: need 16 bytes, have {data.Length - offset}");\n')
+            print(f"Added ReadGuid bounds check at line {i + 2}")
+    elif 'private static bool ReadBoolean' in lines[i] and '{' in lines[i + 1]:
+        if 'truncated at offset' not in lines[i + 2]:
+            lines.insert(i + 2, '            if (offset + 1 > data.Length)\n')
+            lines.insert(i + 3, '                throw new InvalidOperationException($"Boolean truncated at offset {offset}: need 1 byte, have {data.Length - offset}");\n')
+            print(f"Added ReadBoolean bounds check at line {i + 2}")
+    elif 'private static int ReadInt32(ReadOnlySpan<byte> data, ref int offset)' in lines[i] and '{' in lines[i + 1]:
+        if 'truncated at offset' not in lines[i + 2]:
+            lines.insert(i + 2, '            if (offset + 4 > data.Length)\n')
+            lines.insert(i + 3, '                throw new InvalidOperationException($"Int32 truncated at offset {offset}: need 4 bytes, have {data.Length - offset}");\n')
+            print(f"Added ReadInt32 bounds check at line {i + 2}")
+    elif 'private static long ReadInt64' in lines[i] and '{' in lines[i + 1]:
+        if 'truncated at offset' not in lines[i + 2]:
+            lines.insert(i + 2, '            if (offset + 8 > data.Length)\n')
+            lines.insert(i + 3, '                throw new InvalidOperationException($"Int64 truncated at offset {offset}: need 8 bytes, have {data.Length - offset}");\n')
+            print(f"Added ReadInt64 bounds check at line {i + 2}")
+    i += 1
+
+# Write back
+with open('src/SharpCoreDB/DatabaseExtensions.cs', 'w', encoding='utf-8') as f:
+    f.writelines(lines)
+
+print("\n✅ All bounds checks added successfully!")

fix_bounds_v3.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""Add bounds checking to binary deserialization Read methods - v3."""
+
+def find_and_fix_method(lines, method_signature, bytes_needed):
+    """Find a method and add bounds checking."""
+    i = 0
+    while i < len(lines):
+        if method_signature in lines[i]:
+            # Found the method - now find the opening brace
+            brace_line = i + 1
+            if '{' not in lines[brace_line]:
+                brace_line = i
+            
+            # Check if bounds check already exists
+            check_line = brace_line + 1
+            if check_line < len(lines) and 'truncated at offset' in lines[check_line]:
+                print(f"✓ {method_signature[:40]}... already has bounds check")
+                return
+            
+            # Insert bounds check
+            indent = '            '
+            if bytes_needed == 'length':
+                # For ReadString and ReadBytes - check after reading length
+                # Find "var length = ReadInt32" line
+                j = brace_line
+                while j < min(len(lines), brace_line + 10):
+                    if 'var length = ReadInt32(data, ref offset);' in lines[j]:
+                        # Insert bounds check right after
+                        lines.insert(j + 1, f'{indent}if (length < 0 || offset + length > data.Length)\n')
+                        lines.insert(j + 2, f'{indent}    throw new InvalidOperationException($"Truncated at offset {{offset}}: length={{length}}, remaining={{data.Length - offset}}");\n')
+                        print(f"✓ Added bounds check to {method_signature[:40]}...")
+                        return
+                    j += 1
+            else:
+                # For fixed-size reads - insert right after opening brace
+                lines.insert(check_line, f'{indent}if (offset + {bytes_needed} > data.Length)\n')
+                lines.insert(check_line + 1, f'{indent}    throw new InvalidOperationException($"Truncated at offset {{offset}}: need {bytes_needed} bytes, have {{data.Length - offset}}");\n')
+                print(f"✓ Added bounds check to {method_signature[:40]}...")
+                return
+        i += 1
+    print(f"✗ Could not find {method_signature[:40]}...")
+
+# Read file
+with open('src/SharpCoreDB/DatabaseExtensions.cs', 'r', encoding='utf-8') as f:
+    lines = f.readlines()
+
+print("Adding bounds checks...\n")
+
+# Add bounds checks to each method
+find_and_fix_method(lines, 'private static string ReadString(ReadOnlySpan<byte> data, ref int offset)', 'length')
+find_and_fix_method(lines, 'private static byte[] ReadBytes(ReadOnlySpan<byte> data, ref int offset)', 'length')
+find_and_fix_method(lines, 'private static Guid ReadGuid(ReadOnlySpan<byte> data, ref int offset)', 16)
+find_and_fix_method(lines, 'private static bool ReadBoolean(ReadOnlySpan<byte> data, ref int offset)', 1)
+find_and_fix_method(lines, 'private static int ReadInt32(ReadOnlySpan<byte> data, ref int offset)', 4)
+find_and_fix_method(lines, 'private static long ReadInt64(ReadOnlySpan<byte> data, ref int offset)', 8)
+
+# Write back
+with open('src/SharpCoreDB/DatabaseExtensions.cs', 'w', encoding='utf-8') as f:
+    f.writelines(lines)
+
+print("\n✅ Done!")

fix_duplicate.py

@@ -0,0 +1,16 @@
+import re
+
+# Read the file
+with open('src/SharpCoreDB/DatabaseExtensions.cs', 'r', encoding='utf-8') as f:
+    lines = f.readlines()
+
+# Remove duplicate line 1415
+if len(lines) > 1415:
+    if 'String truncated' in lines[1414] and 'String truncated' in lines[1415]:
+        del lines[1415]
+
+# Write back
+with open('src/SharpCoreDB/DatabaseExtensions.cs', 'w', encoding='utf-8') as f:
+    f.writelines(lines)
+
+print("Removed duplicate line!")