checkpoint with some instrumentation

Author: lukesandberg

turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs

@@ -1099,7 +1099,7 @@ impl_operation!(LeafDistanceUpdate leaf_distance_update::LeafDistanceUpdateQueue
 pub use self::invalidate::TaskDirtyCause;
 pub use self::{
     aggregation_update::{
-        AggregatedDataUpdate, AggregationUpdateJob, ComputeDirtyAndCleanUpdate,
+        AggregatedDataUpdate, AggregationUpdateJob, ComputeDirtyAndCleanUpdate, LEAF_NUMBER,
         get_aggregation_number, get_uppers, is_aggregating_node, is_root_node,
     },
     cleanup_old_edges::OutdatedEdge,

turbopack/crates/turbo-tasks-backend/src/backend/storage.rs

@@ -5,13 +5,14 @@ use std::{
     sync::{Arc, atomic::AtomicBool},
 };
 
+use rustc_hash::FxHashMap;
 use smallvec::SmallVec;
 use thread_local::ThreadLocal;
 use turbo_bincode::TurboBincodeBuffer;
 use turbo_tasks::{FxDashMap, TaskId, parallel};
 
 use crate::{
-    backend::storage_schema::{Evictability, TaskStorage},
+    backend::storage_schema::{Evictability, TaskStorage, UnevictableReason},
     backing_storage::SnapshotItem,
     database::key_value_database::KeySpace,
     utils::{
@@ -307,35 +308,50 @@ impl Storage {
             "evict_after_snapshot must not be called during snapshot mode"
         );
 
-        let counts: Vec<(usize, usize)> = parallel::map_collect(self.map.shards(), |shard| {
-            let mut shard = shard.write();
-            let mut full = 0usize;
-            let mut data_only = 0usize;
-            // SAFETY: We hold the write lock for the duration of iteration.
-            for bucket in unsafe { shard.iter() } {
-                // SAFETY: The write lock guard outlives the bucket reference.
-                let (task_id, task) = unsafe { bucket.as_mut() };
-                if task_id.is_transient() {
-                    continue;
-                }
-                match task.get().evictability() {
-                    Evictability::Full => {
-                        // SAFETY: Erasing while iterating a RawTable is safe.
-                        unsafe { shard.erase(bucket) };
-                        full += 1;
+        let counts: Vec<(usize, usize, FxHashMap<UnevictableReason, usize>)> =
+            parallel::map_collect(self.map.shards(), |shard| {
+                let mut shard = shard.write();
+                let mut full = 0usize;
+                let mut data_only = 0usize;
+                let mut reason_counts: FxHashMap<UnevictableReason, usize> = FxHashMap::default();
+                // SAFETY: We hold the write lock for the duration of iteration.
+                for bucket in unsafe { shard.iter() } {
+                    // SAFETY: The write lock guard outlives the bucket reference.
+                    let (task_id, task) = unsafe { bucket.as_mut() };
+                    if task_id.is_transient() {
+                        continue;
                     }
-                    Evictability::DataOnly => {
-                        task.get_mut().drop_data();
-                        data_only += 1;
+                    match task.get().evictability() {
+                        Evictability::Full => {
+                            unsafe {
+                                shard.erase(bucket);
+                            }
+                            full += 1;
+                        }
+                        Evictability::DataOnly => {
+                            task.get_mut().drop_data();
+                            data_only += 1;
+                        }
+                        Evictability::No(reason) => {
+                            *reason_counts.entry(reason).or_default() += 1;
+                        }
                     }
-                    Evictability::No => {}
                 }
+                (full, data_only, reason_counts)
+            });
+        let mut full = 0usize;
+        let mut data_only = 0usize;
+        let mut reasons: FxHashMap<UnevictableReason, usize> = FxHashMap::default();
+        for (f, d, r) in counts {
+            full += f;
+            data_only += d;
+            for (reason, count) in r {
+                *reasons.entry(reason).or_default() += count;
             }
-            (full, data_only)
-        });
-        counts
-            .into_iter()
-            .fold((0, 0), |(a, b), (c, d)| (a + c, b + d))
+        }
+        let skipped: usize = reasons.values().sum();
+        eprintln!("eviction: {full} full, {data_only} data-only, {skipped} skipped ({reasons:?})",);
+        (full, data_only)
     }
 }
 

turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs

@@ -29,7 +29,7 @@ use turbo_tasks::{
 };
 
 use crate::{
-    backend::counter_map::CounterMap,
+    backend::{counter_map::CounterMap, operation::LEAF_NUMBER},
     data::{
         ActivenessState, AggregationNumber, CellRef, CollectibleRef, CollectiblesRef, Dirtyness,
         InProgressCellState, InProgressState, LeafDistance, OutputValue, RootType, TransientTask,
@@ -384,11 +384,21 @@ impl TaskFlags {
 // Eviction
 // =============================================================================
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum UnevictableReason {
+    InProgress,
+    TransientDependents,
+    TransientData,
+    TransientUppers,
+    SessionState,
+    Modified,
+}
+
 /// Eviction level for a task after a snapshot.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Evictability {
     /// Task cannot be evicted.
-    No,
+    No(UnevictableReason),
     /// Only the data category can be evicted (meta is still in use).
     DataOnly,
     /// The entire task can be evicted (removed from the storage map).
@@ -426,7 +436,7 @@ impl TaskStorage {
             || self.get_activeness().is_some()
             || self.get_transient_task_type().is_some()
         {
-            return Evictability::No;
+            return Evictability::No(UnevictableReason::InProgress);
         }
 
         // Check if full eviction is possible
@@ -436,28 +446,71 @@ impl TaskStorage {
         let data_evictable = flags.data_restored()
             && !flags.data_modified()
             && !flags.data_modified_during_snapshot();
+        if !data_evictable {
+            return Evictability::No(UnevictableReason::Modified);
+        }
 
-        if meta_evictable && data_evictable {
-            // Non-serializable cell data (e.g. process pool handles) cannot be restored from
-            // disk. Full eviction would permanently lose it. Downgrade to data-only eviction
-            // which preserves transient fields.
-            if self.transient_cell_data().is_some_and(|m| !m.is_empty()) {
-                return Evictability::DataOnly;
-            }
-            // Session-dependent tasks have transient `current_session_clean` state that cannot
-            // be restored from disk. Losing it would make the task appear dirty in the current
-            // session, causing redundant re-execution. Downgrade to data-only eviction.
-            if matches!(self.get_dirty(), Some(Dirtyness::SessionDependent)) {
+        // Data-category fields with `filter_transient` lose entries referencing transient
+        // tasks when serialized to disk. If the in-memory copy has such entries, evicting
+        // (and later restoring from disk) would silently drop those reverse-dependency
+        // edges, causing transient tasks (e.g., HMR update streams) to never be notified
+        // when cells/outputs change. Prevent data eviction in this case.
+        let has_transient_dependents = self
+            .output_dependent()
+            .iter()
+            .any(|task_id| task_id.is_transient())
+            || self
+                .cell_dependents()
+                .is_some_and(|deps| deps.iter().any(|(_, _, task_id)| task_id.is_transient()))
+            || self
+                .collectibles_dependents()
+                .is_some_and(|deps| deps.iter().any(|(_, id)| id.is_transient()));
+        // If any transient tasks are reading this one we need to not evict so the notifications
+        // still work
+        if has_transient_dependents {
+            return Evictability::No(UnevictableReason::TransientDependents);
+        }
+        // Check for non-serializable cell data (transient category, while this would be preserved
+        // by only
+        if self.transient_cell_data().is_some_and(|m| !m.is_empty())
+            || self.get_output().is_some_and(|o| o.is_transient())
+        {
+            return Evictability::No(UnevictableReason::TransientData);
+        }
+        // Meta fields with filter_transient (children, upper, followers, output,
+        // collectibles_dependents, etc.) lose transient entries when serialized.
+        // If the task participates in the aggregation graph with transient nodes,
+        // full eviction would break those relationships. Check key meta fields.
+        debug_assert!(
+            !self
+                .children()
+                .is_some_and(|c| c.iter().any(|id| id.is_transient())),
+            "persistent tasks cannot have transient children"
+        );
+        if self.upper().iter().any(|(id, _)| id.is_transient()) {
+            return Evictability::No(UnevictableReason::TransientUppers);
+        }
+
+        if self
+            .get_dirty()
+            .is_some_and(|d| matches!(d, Dirtyness::SessionDependent))
+        {
+            return Evictability::No(UnevictableReason::SessionState);
+        }
+        if meta_evictable {
+            // Session-dependent tasks have transient state (current_session_clean flag,
+            // Dirtyness::SessionDependent) that would be lost on full eviction.
+
+            // Aggregating nodes carry transient session-clean container counts that would
+            // be lost on full eviction, breaking has_dirty_containers() checks.
+            if self.aggregation_number.effective >= LEAF_NUMBER {
                 return Evictability::DataOnly;
             }
-            return Evictability::Full;
-        }
 
-        if data_evictable {
-            return Evictability::DataOnly;
+            return Evictability::Full;
         }
 
-        Evictability::No
+        return Evictability::DataOnly;
     }
 }