improve eviction semantics and add tests

Author: lukesandberg

test/e2e/filesystem-cache/evict-after-snapshot.test.ts

@@ -0,0 +1,89 @@
+import { nextTestSetup } from 'e2e-utils'
+import { retry, waitFor } from 'next-test-utils'
+
+describe('evict-after-snapshot', () => {
+  const envVars = [
+    'ENABLE_CACHING=1',
+    'TURBO_ENGINE_IGNORE_DIRTY=1',
+    'TURBO_ENGINE_SNAPSHOT_IDLE_TIMEOUT_MILLIS=1000',
+    'TURBO_ENGINE_EVICT_AFTER_SNAPSHOT=1',
+  ].join(' ')
+
+  const { skipped, next } = nextTestSetup({
+    files: __dirname,
+    skipDeployment: true,
+    patchFileDelay: 500,
+    packageJson: {
+      scripts: {
+        dev: `${envVars} next dev`,
+      },
+    },
+    installCommand: 'npm i',
+    startCommand: 'npm run dev',
+  })
+
+  if (skipped) {
+    return
+  }
+
+  async function waitForSnapshotAndEviction() {
+    // The idle timeout is 1s, give extra time for snapshot + eviction to complete
+    await waitFor(5000)
+  }
+
+  // Turbopack-only: eviction requires persistent caching
+  ;(process.env.IS_TURBOPACK_TEST ? it : it.skip)(
+    'should serve correct content after eviction and HMR',
+    async () => {
+      const browser = await next.browser('/')
+      await retry(async () => {
+        expect(await browser.elementByCss('p').text()).toBe('hello world')
+      })
+
+      for (let cycle = 1; cycle <= 3; cycle++) {
+        await waitForSnapshotAndEviction()
+
+        await next.patchFile(
+          'app/page.tsx',
+          (content) => content.replace('hello world', `cycle ${cycle}`),
+          async () => {
+            await retry(async () => {
+              expect(await browser.elementByCss('p').text()).toBe(
+                `cycle ${cycle}`
+              )
+            }, 10000)
+          }
+        )
+      }
+
+      await browser.close()
+    },
+    90000
+  )
+  ;(process.env.IS_TURBOPACK_TEST ? it : it.skip)(
+    'should handle client component HMR after eviction',
+    async () => {
+      const browser = await next.browser('/client')
+      await retry(async () => {
+        expect(await browser.elementByCss('p').text()).toBe('hello world')
+      })
+
+      await waitForSnapshotAndEviction()
+
+      await next.patchFile(
+        'app/client/page.tsx',
+        (content) => content.replace('hello world', 'hello eviction'),
+        async () => {
+          await retry(async () => {
+            expect(await browser.elementByCss('p').text()).toBe(
+              'hello eviction'
+            )
+          }, 10000)
+        }
+      )
+
+      await browser.close()
+    },
+    90000
+  )
+})

turbopack/crates/turbo-tasks-backend/src/backend/mod.rs

@@ -236,6 +236,19 @@ impl<B: BackingStorage> TurboTasksBackend<B> {
     pub fn backing_storage(&self) -> &B {
         &self.0.backing_storage
     }
+
+    /// Perform a snapshot and then evict all evictable tasks from memory.
+    ///
+    /// This is exposed for integration tests that need to verify the
+    /// snapshot → evict → restore cycle works correctly.
+    ///
+    /// Returns `(snapshot_had_new_data, full_evicted, data_only_evicted)`.
+    pub fn snapshot_and_evict(
+        &self,
+        turbo_tasks: &dyn TurboTasksBackendApi<TurboTasksBackend<B>>,
+    ) -> (bool, usize, usize) {
+        self.0.snapshot_and_evict(turbo_tasks)
+    }
 }
 
 impl<B: BackingStorage> TurboTasksBackendInner<B> {
@@ -371,6 +384,26 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
         self.options.evict_after_snapshot && self.should_persist()
     }
 
+    /// Perform a snapshot and then evict all evictable tasks from memory.
+    ///
+    /// This is exposed for integration tests that need to verify the
+    /// snapshot → evict → restore cycle works correctly.
+    ///
+    /// Returns `(snapshot_had_new_data, full_evicted, data_only_evicted)`.
+    pub fn snapshot_and_evict(
+        &self,
+        turbo_tasks: &dyn TurboTasksBackendApi<TurboTasksBackend<B>>,
+    ) -> (bool, usize, usize) {
+        assert!(
+            self.should_persist(),
+            "snapshot_and_evict requires persistence"
+        );
+        let snapshot_result = self.snapshot_and_persist(None, "test", turbo_tasks);
+        let had_new_data = snapshot_result.map_or(false, |(_, new_data)| new_data);
+        let (full, data_only) = self.storage.evict_after_snapshot();
+        (had_new_data, full, data_only)
+    }
+
     fn should_restore(&self) -> bool {
         self.options.storage_mode.is_some()
     }
@@ -2798,6 +2831,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
                             // Evict persisted tasks from memory to reclaim space.
                             // Like compaction, this runs after snapshot_and_persist
                             // as a separate concern.
+
                             if this.should_evict() {
                                 let evict_span = tracing::info_span!(
                                     parent: background_span.id(),

turbopack/crates/turbo-tasks-backend/src/backend/storage.rs

@@ -203,11 +203,15 @@ impl Storage {
             }
         });
 
-        // We also need to unset all the modified flags.
+        // We also need to unset the modified flags and mark data as restored
+        // (on disk and recoverable). This enables eviction of freshly created
+        // tasks that have just been snapshotted.
         for key in removed_modified {
             if let Some(mut inner) = self.map.get_mut(&key) {
                 inner.flags.set_data_modified(false);
                 inner.flags.set_meta_modified(false);
+                inner.flags.set_data_restored(true);
+                inner.flags.set_meta_restored(true);
             }
         }
 

turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs

@@ -294,7 +294,7 @@ struct TaskStorageSchema {
     #[field(storage = "auto_map", category = "transient", shrink_on_completion)]
     in_progress_cells: AutoMap<CellId, InProgressCellState>,
 
-    #[field(storage = "direct", category = "data", inline, keep_on_restore)]
+    #[field(storage = "direct", category = "data", inline)]
     pub persistent_task_type: Option<Arc<CachedTaskType>>,
 
     #[field(storage = "direct", category = "transient")]
@@ -438,6 +438,18 @@ impl TaskStorage {
             && !flags.data_modified_during_snapshot();
 
         if meta_evictable && data_evictable {
+            // Non-serializable cell data (e.g. process pool handles) cannot be restored from
+            // disk. Full eviction would permanently lose it. Downgrade to data-only eviction
+            // which preserves transient fields.
+            if self.transient_cell_data().is_some_and(|m| !m.is_empty()) {
+                return Evictability::DataOnly;
+            }
+            // Session-dependent tasks have transient `current_session_clean` state that cannot
+            // be restored from disk. Losing it would make the task appear dirty in the current
+            // session, causing redundant re-execution. Downgrade to data-only eviction.
+            if matches!(self.get_dirty(), Some(Dirtyness::SessionDependent)) {
+                return Evictability::DataOnly;
+            }
             return Evictability::Full;
         }
 

turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs

@@ -246,7 +246,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
 
         {
             let _span = tracing::trace_span!("update task data").entered();
-            let counts: (usize, usize) =
+            let (meta_count, data_count) =
                 parallel::map_collect_owned::<_, _, Result<Vec<_>>>(snapshots, |tasks| {
                     let mut local_meta = 0usize;
                     let mut local_data = 0usize;
@@ -280,8 +280,8 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
                 .into_iter()
                 .fold((0, 0), |(am, ad), (m, d)| (am + m, ad + d));
 
-            span.record("meta", counts.0);
-            span.record("data", counts.1);
+            span.record("meta", meta_count);
+            span.record("data", data_count);
             let flush_span = tracing::trace_span!("flush task data").entered();
             parallel::try_for_each(&[KeySpace::TaskMeta, KeySpace::TaskData], |&key_space| {
                 let _span = flush_span.clone().entered();