Improved calculations

Author: alexis-

src/EfCore.StorageEstimator/EfCore.StorageEstimator.csproj

@@ -5,7 +5,7 @@
 
     <!-- NuGet Package Properties (common properties inherited from Directory.Build.props) -->
     <PackageId>EfCore.StorageEstimator</PackageId>
-    <Version>1.0.0</Version>
+    <Version>1.1.0</Version>
     <Description>Estimate PostgreSQL storage footprint and database pressure from EF Core models using annotation-driven workload assumptions.</Description>
     <PackageTags>efcore;postgresql;storage;estimation;analysis</PackageTags>
   </PropertyGroup>

src/EfCore.StorageEstimator/Estimation/PostgreSqlStorageMath.cs

@@ -16,6 +16,7 @@ internal static class PostgreSqlStorageMath
   private const double IndexPageHeaderBytes       = 24;
   private const double BTreeSpecialSpaceBytes     = 16;
   private const double IndexTupleHeaderBytes      = 8;
+  private const double IndexNullBitmapHeaderBytes = 16;
   private const double IndexRowPointerBytes       = 4;
   private const int    MaxAlignment               = 8;
   private const int    DefaultVariableLengthBytes = 256;
@@ -107,11 +108,15 @@ private static EntityStorageEstimate EstimateEntity(
                             .Select(property => EstimateProperty(property, $"{entityPath}.{property.Name}", warnings))
                             .ToArray();
     var propertyLookup = propertyEstimates.ToDictionary(property => property.Name, StringComparer.Ordinal);
-    var tupleHeaderBytes = Align(
+    var tupleHeaderBytesWithoutNulls = Align(HeapTupleHeaderBytes, MaxAlignment);
+    var tupleHeaderBytesWithNulls = Align(
       HeapTupleHeaderBytes + GetNullBitmapBytes(properties),
       MaxAlignment);
-    var tupleDataBytes      = propertyEstimates.Sum(property => property.AverageStoredBytes);
-    var averageHeapRowBytes = Align(tupleHeaderBytes + tupleDataBytes, MaxAlignment) + HeapItemPointerBytes;
+    var averageHeapTupleBytes = EstimateAverageTupleBytes(
+      tupleHeaderBytesWithoutNulls,
+      tupleHeaderBytesWithNulls,
+      propertyEstimates);
+    var averageHeapRowBytes = averageHeapTupleBytes + HeapItemPointerBytes;
     var heapBytes           = EstimatePagedBytes(rowCount, averageHeapRowBytes, HeapPageHeaderBytes, 0);
     var indexEstimates = indexes
                          .Select(index => EstimateIndex(index, propertyLookup, rowCount))
@@ -138,7 +143,11 @@ private static PropertySizeEstimate EstimateProperty(
       warnings.Add(
         $"Using fallback fill rate 1.0 for nullable fixed-width property '{propertyPath}'. Add [{nameof(StorageFieldAttribute)}] to override it.");
 
-    return new PropertySizeEstimate(property.Name, size * fillRate);
+    return new PropertySizeEstimate(
+      property.Name,
+      size,
+      GetFixedAlignment(property, propertyPath, warnings),
+      fillRate);
   }
 
 
@@ -184,13 +193,23 @@ private static PropertySizeEstimate EstimateVariableLengthProperty(
     }
 
     if (IsNumericType(property.ClrType, property.ProviderClrType, property.StoreType))
-      return new PropertySizeEstimate(property.Name, averageLength.Value * fillRate);
+      return new PropertySizeEstimate(
+        property.Name,
+        averageLength.Value,
+        GetVariableLengthAlignment(averageLength.Value),
+        fillRate);
 
     var overheadBytes = averageLength <= 126
       ? 1
       : 4;
 
-    return new PropertySizeEstimate(property.Name, (averageLength.Value + overheadBytes) * fillRate);
+    var storedBytes = averageLength.Value + overheadBytes;
+
+    return new PropertySizeEstimate(
+      property.Name,
+      storedBytes,
+      GetVariableLengthAlignment(storedBytes),
+      fillRate);
   }
 
 
@@ -213,8 +232,14 @@ private static StorageIndexEstimate EstimateIndex(
     IReadOnlyDictionary<string, PropertySizeEstimate> propertyLookup,
     double                                            rowCount)
   {
-    var keyBytes          = index.Properties.Sum(property => propertyLookup[property.Name].AverageStoredBytes);
-    var averageEntryBytes = Align(IndexTupleHeaderBytes + keyBytes, MaxAlignment) + IndexRowPointerBytes;
+    var keyProperties = index.Properties
+                             .Select(property => propertyLookup[property.Name])
+                             .ToArray();
+    var averageTupleBytes = EstimateAverageTupleBytes(
+      IndexTupleHeaderBytes,
+      IndexNullBitmapHeaderBytes,
+      keyProperties);
+    var averageEntryBytes = averageTupleBytes + IndexRowPointerBytes;
     var estimatedBytes    = EstimatePagedBytes(rowCount, averageEntryBytes, IndexPageHeaderBytes, BTreeSpecialSpaceBytes) + HeapPageBytes;
 
     return new StorageIndexEstimate(
@@ -226,6 +251,143 @@ private static StorageIndexEstimate EstimateIndex(
   }
 
 
+  private static double EstimateAverageTupleBytes(
+    double                           headerBytesWithoutNulls,
+    double                           headerBytesWithNulls,
+    IReadOnlyList<PropertySizeEstimate> properties)
+  {
+    if (properties.Count == 0)
+      return Align(headerBytesWithoutNulls, MaxAlignment);
+
+    var allPresentProbability = properties.Aggregate(1.0d, (current, property) => current * property.FillRate);
+    var allPresentBytesWithoutNulls = EstimateTupleBytesForAlwaysPresentProperties(headerBytesWithoutNulls, properties);
+
+    if (allPresentProbability >= 1.0d)
+      return allPresentBytesWithoutNulls;
+
+    var allPresentBytesWithNulls = EstimateTupleBytesForAlwaysPresentProperties(headerBytesWithNulls, properties);
+    var expectedBytesWithNullableHeader = EstimateExpectedTupleBytes(
+      headerBytesWithNulls,
+      properties);
+
+    return (allPresentProbability * allPresentBytesWithoutNulls) +
+      expectedBytesWithNullableHeader -
+      (allPresentProbability * allPresentBytesWithNulls);
+  }
+
+
+  private static double EstimateTupleBytesForAlwaysPresentProperties(
+    double                           headerBytes,
+    IReadOnlyList<PropertySizeEstimate> properties)
+  {
+    var offset = headerBytes;
+
+    foreach (var property in properties)
+    {
+      offset = Align(offset, property.AlignmentBytes);
+      offset += property.BytesWhenPresent;
+    }
+
+    return Align(offset, MaxAlignment);
+  }
+
+
+  private static double EstimateExpectedTupleBytes(
+    double                           headerBytes,
+    IReadOnlyList<PropertySizeEstimate> properties)
+  {
+    var states = new Dictionary<int, TupleLayoutState>
+    {
+      [GetResidue(headerBytes)] = new TupleLayoutState(1.0d, headerBytes)
+    };
+
+    foreach (var property in properties)
+    {
+      var nextStates = new Dictionary<int, TupleLayoutState>();
+
+      foreach (var state in states)
+      {
+        var residue     = state.Key;
+        var layoutState = state.Value;
+
+        if (property.FillRate < 1.0d)
+          AddState(
+            nextStates,
+            residue,
+            new TupleLayoutState(
+              layoutState.Probability * (1.0d - property.FillRate),
+              layoutState.WeightedOffsetSum * (1.0d - property.FillRate)));
+
+        if (property.FillRate <= 0.0d)
+          continue;
+
+        var paddingBytes = GetPaddingBytes(residue, property.AlignmentBytes);
+        var addedBytes   = paddingBytes + property.BytesWhenPresent;
+
+        AddProbability(
+          nextStates,
+          GetResidue(residue + addedBytes),
+          new TupleLayoutState(
+            layoutState.Probability * property.FillRate,
+            (layoutState.WeightedOffsetSum * property.FillRate) + (layoutState.Probability * property.FillRate * addedBytes)));
+      }
+
+      states = nextStates;
+    }
+
+    return states.Sum(state => state.Value.WeightedOffsetSum + (state.Value.Probability * GetPaddingBytes(state.Key, MaxAlignment)));
+  }
+
+
+  private static int GetPaddingBytes(
+    int residue,
+    int alignmentBytes)
+  {
+    if (alignmentBytes <= 1)
+      return 0;
+
+    var misalignment = residue % alignmentBytes;
+
+    return misalignment == 0
+      ? 0
+      : alignmentBytes - misalignment;
+  }
+
+
+  private static int GetResidue(double offset)
+  {
+    return ((int)Math.Round(offset, MidpointRounding.AwayFromZero)) % MaxAlignment;
+  }
+
+
+  private static void AddProbability(
+    IDictionary<int, TupleLayoutState> states,
+    int                                residue,
+    TupleLayoutState                   state)
+  {
+    if (state.Probability <= 0.0d)
+      return;
+
+    AddState(states, residue, state);
+  }
+
+
+  private static void AddState(
+    IDictionary<int, TupleLayoutState> states,
+    int                                residue,
+    TupleLayoutState                   state)
+  {
+    if (state.Probability <= 0.0d)
+      return;
+
+    states[residue] = states.TryGetValue(residue, out var current)
+      ? new TupleLayoutState(
+        current.Probability + state.Probability,
+        current.WeightedOffsetSum + state.WeightedOffsetSum)
+      : state;
+  }
+
+
   private static double EstimatePagedBytes(
     double rowCount,
     double averageEntryBytes,
@@ -251,7 +413,7 @@ private static double GetNullBitmapBytes(IReadOnlyList<PropertySizingInput> prop
   }
 
 
-  private static double GetFixedSize(
+  private static int GetFixedSize(
     PropertySizingInput property,
     string              propertyPath,
     ICollection<string> warnings)
@@ -323,6 +485,64 @@ private static double GetFixedSize(
   }
 
 
+  private static int GetFixedAlignment(
+    PropertySizingInput property,
+    string              propertyPath,
+    ICollection<string> warnings)
+  {
+    var nonNullableClrType      = Nullable.GetUnderlyingType(property.ClrType) ?? property.ClrType;
+    var nonNullableProviderType = Nullable.GetUnderlyingType(property.ProviderClrType) ?? property.ProviderClrType;
+    var storeType               = property.StoreType ?? string.Empty;
+
+    if (nonNullableProviderType.IsEnum)
+      nonNullableProviderType = Enum.GetUnderlyingType(nonNullableProviderType);
+
+    if (nonNullableClrType.IsEnum)
+      nonNullableClrType = Enum.GetUnderlyingType(nonNullableClrType);
+
+    if (nonNullableProviderType == typeof(bool) || nonNullableClrType == typeof(bool) ||
+        nonNullableProviderType == typeof(byte) || nonNullableProviderType == typeof(sbyte) ||
+        nonNullableClrType == typeof(byte) || nonNullableClrType == typeof(sbyte) ||
+        nonNullableProviderType == typeof(Guid) || nonNullableClrType == typeof(Guid) ||
+        storeType.Contains("uuid", StringComparison.OrdinalIgnoreCase))
+      return 1;
+
+    if (nonNullableProviderType == typeof(short) || nonNullableProviderType == typeof(ushort) ||
+        nonNullableClrType == typeof(short) || nonNullableClrType == typeof(ushort))
+      return 2;
+
+    if (nonNullableProviderType == typeof(int) || nonNullableProviderType == typeof(uint) ||
+        nonNullableClrType == typeof(int) || nonNullableClrType == typeof(uint) ||
+        nonNullableProviderType == typeof(float) || nonNullableClrType == typeof(float) ||
+        storeType.Contains("date", StringComparison.OrdinalIgnoreCase))
+      return 4;
+
+    if (nonNullableProviderType == typeof(long) || nonNullableProviderType == typeof(ulong) ||
+        nonNullableClrType == typeof(long) || nonNullableClrType == typeof(ulong) ||
+        nonNullableProviderType == typeof(double) || nonNullableClrType == typeof(double) ||
+        nonNullableProviderType == typeof(DateTime) || nonNullableClrType == typeof(DateTime) ||
+        nonNullableProviderType == typeof(DateTimeOffset) || nonNullableClrType == typeof(DateTimeOffset) ||
+        nonNullableProviderType == typeof(TimeSpan) || nonNullableClrType == typeof(TimeSpan) ||
+        storeType.Contains("timestamp", StringComparison.OrdinalIgnoreCase) ||
+        storeType.Contains("interval", StringComparison.OrdinalIgnoreCase) ||
+        storeType.Contains("time with time zone", StringComparison.OrdinalIgnoreCase) ||
+        storeType.Contains("time", StringComparison.OrdinalIgnoreCase))
+      return 8;
+
+    warnings.Add($"Using fallback fixed-size alignment 8 for '{propertyPath}' because its PostgreSQL alignment could not be inferred.");
+
+    return 8;
+  }
+
+
+  private static int GetVariableLengthAlignment(double storedBytes)
+  {
+    return storedBytes <= 126
+      ? 1
+      : 4;
+  }
+
+
   private static bool IsNumericType(
     Type    clrType,
     Type    providerClrType,
@@ -512,12 +732,25 @@ public StoragePropertySchema ToSchema()
 
   private sealed class PropertySizeEstimate(
     string name,
-    double averageStoredBytes)
+    double bytesWhenPresent,
+    int    alignmentBytes,
+    double fillRate)
   {
     public string Name { get; } = name;
 
-    public double AverageStoredBytes { get; } = averageStoredBytes;
+    public double BytesWhenPresent { get; } = bytesWhenPresent;
+
+    public int AlignmentBytes { get; } = alignmentBytes;
+
+    public double FillRate { get; } = fillRate;
+
+    public double AverageStoredBytes => BytesWhenPresent * FillRate;
   }
 
+
+  private readonly record struct TupleLayoutState(
+    double Probability,
+    double WeightedOffsetSum);
+
   #endregion
 }

src/EfCore.StorageEstimator/Schema/EfCoreSchemaReader.cs

@@ -65,7 +65,6 @@ public EfCoreEntitySchema GetEntitySchema(IEntityType entityType)
     var propertyLookup = properties.ToDictionary(property => property.Name, StringComparer.Ordinal);
     var indexes = entityType
                   .GetIndexes()
-                  .Where(index => !IsPureForeignKeyIndex(entityType, index))
                   .Select(index => CreateIndexSchema(index, propertyLookup))
                   .ToArray();
     var entitySchema = new EfCoreEntitySchema(
@@ -279,17 +278,6 @@ private static bool IsVariableLength(
       storeType.Contains("decimal", StringComparison.OrdinalIgnoreCase);
   }
 
-
-  private static bool IsPureForeignKeyIndex(IEntityType entityType, IIndex index)
-  {
-    var indexPropertyNames = index.Properties.Select(property => property.Name).ToArray();
-
-    return entityType
-           .GetForeignKeys()
-           .Any(foreignKey => foreignKey.Properties.Select(property => property.Name).SequenceEqual(indexPropertyNames));
-  }
-
-
   private static bool IsScalarCollectionType(Type type)
   {
     if (type == typeof(string) || type == typeof(byte[]))