graph: Improve estimate of how much of a table pruning will remove

Our current estimate is way too naive and causes a lot of unnecessary table
rebuilds. Using statistics from pg_stats should improve that

Author: lutter

graph/src/components/store/mod.rs

@@ -1099,11 +1099,22 @@ impl PruneRequest {
             return None;
         }
 
-        // Estimate how much data we will throw away; we assume that
-        // entity versions are distributed evenly across all blocks so
-        // that `history_pct` will tell us how much of that data pruning
-        // will remove.
-        let removal_ratio = self.history_pct(stats) * (1.0 - stats.ratio);
+        let removal_ratio = if stats.block_range_upper.is_empty()
+            || ENV_VARS.store.prune_disable_range_bound_estimation
+        {
+            // Estimate how much data we will throw away; we assume that
+            // entity versions are distributed evenly across all blocks so
+            // that `history_pct` will tell us how much of that data pruning
+            // will remove.
+            self.history_pct(stats) * (1.0 - stats.ratio)
+        } else {
+            // This estimate is more accurate than the one above since it
+            // does not assume anything about the distribution of entities
+            // and versions but uses the estimates from Postgres statistics.
+            // Of course, we can only use it if we have statistics
+            self.remove_pct_from_bounds(stats)
+        };
+
         if removal_ratio >= self.rebuild_threshold {
             Some(PruningStrategy::Rebuild)
         } else if removal_ratio >= self.delete_threshold {
@@ -1128,6 +1139,18 @@ impl PruneRequest {
             1.0 - self.history_blocks as f64 / total_blocks as f64
         }
     }
+
+    /// Return the fraction of entities that we will remove according to the
+    /// histogram bounds in `stats`. That fraction can be estimated as the
+    /// fraction of histogram buckets that end before `self.earliest_block`
+    fn remove_pct_from_bounds(&self, stats: &VersionStats) -> f64 {
+        stats
+            .block_range_upper
+            .iter()
+            .filter(|b| **b <= self.earliest_block)
+            .count() as f64
+            / stats.block_range_upper.len() as f64
+    }
 }
 
 /// Represents an item retrieved from an

graph/src/env/store.rs

@@ -114,6 +114,10 @@ pub struct EnvVarsStore {
     /// For how many prune runs per deployment to keep status information.
     /// Set by `GRAPH_STORE_HISTORY_KEEP_STATUS`. The default is 5
     pub prune_keep_history: usize,
+    /// Temporary switch to disable range bound estimation for pruning.
+    /// Set by `GRAPH_STORE_PRUNE_DISABLE_RANGE_BOUND_ESTIMATION`.
+    /// Defaults to false. Remove after 2025-07-15
+    pub prune_disable_range_bound_estimation: bool,
     /// How long to accumulate changes into a batch before a write has to
     /// happen. Set by the environment variable
     /// `GRAPH_STORE_WRITE_BATCH_DURATION` in seconds. The default is 300s.
@@ -188,6 +192,7 @@ impl TryFrom<InnerStore> for EnvVarsStore {
             delete_threshold: x.delete_threshold.0,
             history_slack_factor: x.history_slack_factor.0,
             prune_keep_history: x.prune_keep_status,
+            prune_disable_range_bound_estimation: x.prune_disable_range_bound_estimation,
             write_batch_duration: Duration::from_secs(x.write_batch_duration_in_secs),
             write_batch_size: x.write_batch_size * 1_000,
             create_gin_indexes: x.create_gin_indexes,
@@ -263,6 +268,11 @@ pub struct InnerStore {
     history_slack_factor: HistorySlackF64,
     #[envconfig(from = "GRAPH_STORE_HISTORY_KEEP_STATUS", default = "5")]
     prune_keep_status: usize,
+    #[envconfig(
+        from = "GRAPH_STORE_PRUNE_DISABLE_RANGE_BOUND_ESTIMATION",
+        default = "false"
+    )]
+    prune_disable_range_bound_estimation: bool,
     #[envconfig(from = "GRAPH_STORE_WRITE_BATCH_DURATION", default = "300")]
     write_batch_duration_in_secs: u64,
     #[envconfig(from = "GRAPH_STORE_WRITE_BATCH_SIZE", default = "10000")]