From 3007bcc68f36d942ace267939766756e19468b60 Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Sat, 18 Apr 2026 15:08:57 -0700 Subject: [PATCH 1/9] Unify Cell Storage Add new serializability options. The key thing we want to distinguish is 'this cell is not serializable and it is important' vs this cell isn't _worth_ serializing Currently there isn't a reason to distinguish, but with eviction there is. In theory we should implement a priority system * unevictable (e.g. DiskFileSystemInner). not serialiable fundamentally, evicting this breaks the watcher * expensive (e.g. the WorkerThreadPool). not serializable fundamentally, evicting this would be a perf hit * cheaply derivable: (e.g. EcmascriptChunkItemContent). serializable, but not worth it * regular: everthing else But for now there are just two levels unevictable and everything else, if some of the 'everything else' isn't serializable that is fine. --- .../src/backend/cell_data.rs | 130 ++++++++++++++++++ .../turbo-tasks-backend/src/backend/mod.rs | 50 +++---- .../src/backend/operation/mod.rs | 66 ++------- .../src/backend/operation/update_cell.rs | 93 +++++++------ .../src/backend/storage_schema.rs | 33 +++-- .../tests/derivable_cell.rs | 85 ++++++++++++ .../src/derive/task_storage_macro.rs | 38 ++++- .../turbo-tasks-macros/src/primitive_macro.rs | 2 +- .../turbo-tasks-macros/src/value_macro.rs | 41 ++++-- .../crates/turbo-tasks-testing/src/lib.rs | 1 - turbopack/crates/turbo-tasks/src/backend.rs | 11 +- turbopack/crates/turbo-tasks/src/lib.rs | 2 +- turbopack/crates/turbo-tasks/src/manager.rs | 14 +- turbopack/crates/turbo-tasks/src/raw_vc.rs | 30 +--- .../crates/turbo-tasks/src/read_options.rs | 1 - .../turbo-tasks/src/task/shared_reference.rs | 10 +- .../crates/turbo-tasks/src/value_type.rs | 68 +++++++-- turbopack/crates/turbo-tasks/src/vc/mod.rs | 27 ++-- .../crates/turbo-tasks/src/vc/operation.rs | 6 +- 19 files changed, 462 insertions(+), 246 deletions(-) create mode 100644 turbopack/crates/turbo-tasks-backend/src/backend/cell_data.rs create mode 100644 turbopack/crates/turbo-tasks-backend/tests/derivable_cell.rs diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/cell_data.rs b/turbopack/crates/turbo-tasks-backend/src/backend/cell_data.rs new file mode 100644 index 000000000000..bbb1b3f09308 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/backend/cell_data.rs @@ -0,0 +1,130 @@ +//! Unified cell storage. +//! +//! Every task cell — whether its value type is bincode-serializable, hash-only, +//! derivable, or non-reconstructible — lives in a single `CellData` map keyed +//! by [`CellId`]. The map's bincode impl decides at encode time which entries +//! to persist, by consulting the global [`ValueType`] registry: entries whose +//! value type has no bincode function are omitted from the serialized output. +//! +//! This replaces the older split of `persistent_cell_data` / +//! `transient_cell_data` fields which routed every cell write through an +//! `is_serializable_cell_content: bool` that threaded through ~14 call sites. +//! By keying the bincode decision on the value type itself, the routing +//! collapses to an unconditional insert. +//! +//! The inner value is stored as [`SharedReference`] rather than +//! [`TypedSharedReference`] because the `CellId` key already carries the +//! [`ValueTypeId`] — duplicating it in each map entry would waste memory. +//! Encode / decode recover the value type from the key. + +use std::{ + hash::BuildHasherDefault, + ops::{Deref, DerefMut}, +}; + +use auto_hash_map::AutoMap; +use bincode::{ + Decode, Encode, + error::{DecodeError, EncodeError}, +}; +use rustc_hash::FxHasher; +use turbo_bincode::{ + TurboBincodeDecode, TurboBincodeDecoder, TurboBincodeEncode, TurboBincodeEncoder, + impl_decode_for_turbo_bincode_decode, impl_encode_for_turbo_bincode_encode, +}; +use turbo_tasks::{CellId, SharedReference, ShrinkToFit, ValueTypePersistence, registry}; + +type InnerMap = AutoMap, 1>; + +/// Map of cell id → shared reference, with bincode that filters out entries +/// whose value type has no bincode function. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct CellData(InnerMap); + +impl CellData { + pub fn new() -> Self { + Self::default() + } +} + +impl Deref for CellData { + type Target = InnerMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for CellData { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl ShrinkToFit for CellData { + fn shrink_to_fit(&mut self) { + self.0.shrink_to_fit(); + } +} + +impl TurboBincodeEncode for CellData { + /// Writes `count-of-bincodable-entries` followed by each bincodable + /// `(CellId, encoded-value)`. Entries whose value type is `Derivable` or + /// `SessionStateful` (no bincode) are skipped; they will be reconstructed + /// on the next task execution after restore. + fn encode(&self, encoder: &mut TurboBincodeEncoder) -> Result<(), EncodeError> { + // First pass: count bincodable entries. One extra O(N) iteration over + // the registry — cold path (snapshot time only) and the registry is a + // static array indexed by ValueTypeId, so each lookup is cheap. + let count = self + .0 + .iter() + .filter(|(cell, _)| { + matches!( + registry::get_value_type(cell.type_id).persistence, + ValueTypePersistence::Bincodable(_, _), + ) + }) + .count(); + count.encode(encoder)?; + for (cell_id, reference) in self.0.iter() { + let value_type = registry::get_value_type(cell_id.type_id); + let ValueTypePersistence::Bincodable(encode_fn, _) = value_type.persistence else { + continue; + }; + cell_id.encode(encoder)?; + encode_fn(&*reference.0, encoder)?; + } + Ok(()) + } +} + +impl TurboBincodeDecode for CellData { + /// Reads the count written by [`CellData::encode`] and decodes each + /// `(CellId, SharedReference)` entry by looking up the value type's + /// bincode decode function. + /// + /// Missing cell types — or cells whose value type isn't `Bincodable` — + /// are a decode error: the encoder filters them out, so they should not + /// appear on the wire. + fn decode(decoder: &mut TurboBincodeDecoder) -> Result { + let count = usize::decode(decoder)?; + let mut map = InnerMap::with_capacity_and_hasher(count, BuildHasherDefault::default()); + for _ in 0..count { + let cell = CellId::decode(decoder)?; + let value_type = registry::get_value_type(cell.type_id); + let ValueTypePersistence::Bincodable(_, decode_fn) = value_type.persistence else { + return Err(DecodeError::OtherString(format!( + "cell of type {} has no bincode decoder", + value_type.ty.global_name + ))); + }; + let reference = decode_fn(decoder)?; + map.insert(cell, reference); + } + Ok(Self(map)) + } +} + +impl_encode_for_turbo_bincode_encode!(CellData); +impl_decode_for_turbo_bincode_decode!(CellData); diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 3954b6ed06ad..9ba72ef6f5ae 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -1,3 +1,4 @@ +mod cell_data; mod counter_map; mod operation; mod storage; @@ -857,7 +858,6 @@ impl TurboTasksBackendInner { } let ReadCellOptions { - is_serializable_cell_content, tracking, final_read_hint, } = options; @@ -878,9 +878,9 @@ impl TurboTasksBackendInner { }; let content = if final_read_hint { - task.remove_cell_data(is_serializable_cell_content, cell) + task.remove_cell_data(&cell) } else { - task.get_cell_data(is_serializable_cell_content, cell) + task.get_cell_data(&cell).cloned() }; if let Some(content) = content { if tracking.should_track(false) { @@ -888,7 +888,7 @@ impl TurboTasksBackendInner { } return Ok(Ok(TypedCellContent( cell.type_id, - CellContent(Some(content.reference)), + CellContent(Some(content)), ))); } @@ -2689,9 +2689,12 @@ impl TurboTasksBackendInner { // Note: We do not mark the tasks as dirty here, as these tasks are unused or stale // anyway and we want to avoid needless re-executions. When the cells become // used again, they are invalidated from the update cell operation. - // Remove cell data for cells that no longer exist - let to_remove_persistent: Vec<_> = task - .iter_persistent_cell_data() + // Remove cell data for cells that no longer exist. Both + // bincode-able and non-bincode-able cells live in the single + // `cell_data` map; identifying stale entries is purely a CellId + // index check. + let to_remove: Vec<_> = task + .iter_cell_data() .filter_map(|(cell, _)| { cell_counters .get(&cell.type_id) @@ -2699,25 +2702,9 @@ impl TurboTasksBackendInner { .then_some(*cell) }) .collect(); - - // Remove transient cell data for cells that no longer exist - let to_remove_transient: Vec<_> = task - .iter_transient_cell_data() - .filter_map(|(cell, _)| { - cell_counters - .get(&cell.type_id) - .is_none_or(|start_index| cell.index >= *start_index) - .then_some(*cell) - }) - .collect(); - removed_cell_data.reserve_exact(to_remove_persistent.len() + to_remove_transient.len()); - for cell in to_remove_persistent { - if let Some(data) = task.remove_persistent_cell_data(&cell) { - removed_cell_data.push(data.into_untyped()); - } - } - for cell in to_remove_transient { - if let Some(data) = task.remove_transient_cell_data(&cell) { + removed_cell_data.reserve_exact(to_remove.len()); + for cell in to_remove { + if let Some(data) = task.remove_cell_data(&cell) { removed_cell_data.push(data); } } @@ -2904,14 +2891,13 @@ impl TurboTasksBackendInner { &self, task_id: TaskId, cell: CellId, - options: ReadCellOptions, + _options: ReadCellOptions, turbo_tasks: &dyn TurboTasksBackendApi>, ) -> Result { let mut ctx = self.execute_context(turbo_tasks); let task = ctx.task(task_id, TaskDataCategory::Data); - if let Some(content) = task.get_cell_data(options.is_serializable_cell_content, cell) { - debug_assert!(content.type_id == cell.type_id, "Cell type ID mismatch"); - Ok(CellContent(Some(content.reference)).into_typed(cell.type_id)) + if let Some(content) = task.get_cell_data(&cell).cloned() { + Ok(CellContent(Some(content)).into_typed(cell.type_id)) } else { Ok(CellContent(None).into_typed(cell.type_id)) } @@ -3041,7 +3027,6 @@ impl TurboTasksBackendInner { &self, task_id: TaskId, cell: CellId, - is_serializable_cell_content: bool, content: CellContent, updated_key_hashes: Option>, content_hash: Option, @@ -3052,7 +3037,6 @@ impl TurboTasksBackendInner { task_id, cell, content, - is_serializable_cell_content, updated_key_hashes, content_hash, verification_mode, @@ -3598,7 +3582,6 @@ impl Backend for TurboTasksBackend { &self, task_id: TaskId, cell: CellId, - is_serializable_cell_content: bool, content: CellContent, updated_key_hashes: Option>, content_hash: Option, @@ -3608,7 +3591,6 @@ impl Backend for TurboTasksBackend { self.0.update_task_cell( task_id, cell, - is_serializable_cell_content, content, updated_key_hashes, content_hash, diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index 6b31c879a9f7..dd1ed98a2ed5 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -18,8 +18,8 @@ use tracing::info_span; #[cfg(feature = "trace_prepare_tasks")] use tracing::trace_span; use turbo_tasks::{ - CellId, DynTaskInputs, FxIndexMap, RawVc, TaskExecutionReason, TaskId, TaskPriority, - TurboTasksBackendApi, TurboTasksCallApi, TypedSharedReference, backend::CachedTaskType, + CellId, DynTaskInputs, FxIndexMap, RawVc, SharedReference, TaskExecutionReason, TaskId, + TaskPriority, TurboTasksBackendApi, TurboTasksCallApi, backend::CachedTaskType, macro_helpers::NativeFunction, }; @@ -1250,60 +1250,14 @@ pub trait TaskGuard: Debug + TaskStorageAccessors { .unwrap_or_default(); dirty_count > clean_count } - fn remove_cell_data( - &mut self, - is_serializable_cell_content: bool, - cell: CellId, - ) -> Option { - if is_serializable_cell_content { - self.remove_persistent_cell_data(&cell) - } else { - self.remove_transient_cell_data(&cell) - .map(|sr| sr.into_typed(cell.type_id)) - } - } - fn get_cell_data( - &self, - is_serializable_cell_content: bool, - cell: CellId, - ) -> Option { - if is_serializable_cell_content { - self.get_persistent_cell_data(&cell).cloned() - } else { - self.get_transient_cell_data(&cell) - .map(|sr| sr.clone().into_typed(cell.type_id)) - } - } - fn has_cell_data(&self, is_serializable_cell_content: bool, cell: CellId) -> bool { - if is_serializable_cell_content { - self.persistent_cell_data_contains(&cell) - } else { - self.transient_cell_data_contains(&cell) - } - } - /// Set cell data, returning the old value if any. - fn set_cell_data( - &mut self, - is_serializable_cell_content: bool, - cell: CellId, - value: TypedSharedReference, - ) -> Option { - if is_serializable_cell_content { - self.insert_persistent_cell_data(cell, value) - } else { - self.insert_transient_cell_data(cell, value.into_untyped()) - .map(|sr| sr.into_typed(cell.type_id)) - } - } - - /// Add new cell data (asserts that the cell is new and didn't exist before). - fn add_cell_data( - &mut self, - is_serializable_cell_content: bool, - cell: CellId, - value: TypedSharedReference, - ) { - let old = self.set_cell_data(is_serializable_cell_content, cell, value); + /// Add new cell data. Panics if the cell already had a value. + /// + /// The value type's serialization mode (including whether it's + /// bincode-able) is determined by `cell.type_id` via the `ValueType` + /// registry, not by a threaded bool — the `CellData` encoder filters + /// non-bincodable entries at snapshot time. + fn add_cell_data(&mut self, cell: CellId, value: SharedReference) { + let old = self.insert_cell_data(cell, value); assert!(old.is_none(), "Cell data already exists for {cell:?}"); } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs index d14e089c9e47..782217ab310b 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs @@ -27,7 +27,6 @@ use crate::{ #[allow(clippy::large_enum_variant)] pub enum UpdateCellOperation { InvalidateWhenCellDependency { - is_serializable_cell_content: bool, cell_ref: CellRef, #[bincode(with = "turbo_bincode::indexmap")] dependent_tasks: FxIndexMap; 2]>>, @@ -37,7 +36,6 @@ pub enum UpdateCellOperation { queue: AggregationUpdateQueue, }, FinalCellChange { - is_serializable_cell_content: bool, cell_ref: CellRef, content: Option, queue: AggregationUpdateQueue, @@ -54,17 +52,19 @@ impl UpdateCellOperation { task_id: TaskId, cell: CellId, content: CellContent, - is_serializable_cell_content: bool, updated_key_hashes: Option>, content_hash: Option, #[cfg(feature = "verify_determinism")] verification_mode: VerificationMode, #[cfg(not(feature = "verify_determinism"))] _verification_mode: VerificationMode, mut ctx: impl ExecuteContext<'_>, ) { - // content_hash is only meaningful for transient (non-serializable) cells + // Serializability is a property of the cell's value type — derive it + // from the registry rather than threading a redundant bool. + let is_bincodable_cell_content = is_bincodable(cell); + // content_hash is only meaningful for non-bincodable cells debug_assert!( - !is_serializable_cell_content || content_hash.is_none(), - "content_hash must be None for serializable cell content" + !is_bincodable_cell_content || content_hash.is_none(), + "content_hash must be None for bincodable cell content" ); let content = if let CellContent(Some(new_content)) = content { @@ -81,7 +81,7 @@ impl UpdateCellOperation { let assume_unchanged = !ctx.should_track_dependencies() || !task.has_dirty(); if assume_unchanged { - let has_old_content = task.has_cell_data(is_serializable_cell_content, cell); + let has_old_content = task.cell_data_contains(&cell); if has_old_content { // Never update cells when recomputing if they already have a value. // It's not expected that content changes during recomputation. @@ -93,7 +93,11 @@ impl UpdateCellOperation { verification_mode, turbo_tasks::backend::VerificationMode::EqualityCheck ) - && content != task.get_cell_data(is_serializable_cell_content, cell) + && content + != task + .get_cell_data(&cell) + .cloned() + .map(|r| r.into_typed(cell.type_id)) { let task_description = task.get_task_description(); let cell_type = turbo_tasks::registry::get_value_type(cell.type_id) @@ -116,8 +120,8 @@ impl UpdateCellOperation { // For transient cells without available content, use hash-based comparison to // detect whether the value actually changed—avoiding unnecessary invalidation. - let skip_invalidation = !is_serializable_cell_content && { - let has_old_content = task.has_cell_data(false, cell); + let skip_invalidation = !is_bincodable_cell_content && { + let has_old_content = task.cell_data_contains(&cell); if !has_old_content { match (content_hash, task.get_cell_data_hash(&cell)) { (Some(new_hash), Some(old_hash)) => new_hash == *old_hash, @@ -170,10 +174,10 @@ impl UpdateCellOperation { // tasks and after that set the new cell content. When the cell content is unset, // readers will wait for it to be set via InProgressCell. - let old_content = task.remove_cell_data(is_serializable_cell_content, cell); + let old_content = task.remove_cell_data(&cell); // Update cell_data_hash before dropping the task lock - update_cell_data_hash(&mut task, &cell, is_serializable_cell_content, content_hash); + update_cell_data_hash(&mut task, &cell, content_hash); drop(task); drop(old_content); @@ -186,7 +190,6 @@ impl UpdateCellOperation { ); UpdateCellOperation::InvalidateWhenCellDependency { - is_serializable_cell_content, cell_ref: CellRef { task: task_id, cell, @@ -206,13 +209,13 @@ impl UpdateCellOperation { // So we can just update the cell content. let old_content = if let Some(new_content) = content { - task.set_cell_data(is_serializable_cell_content, cell, new_content) + task.insert_cell_data(cell, new_content.into_untyped()) } else { - task.remove_cell_data(is_serializable_cell_content, cell) + task.remove_cell_data(&cell) }; - // Update cell_data_hash for non-serializable cells. - update_cell_data_hash(&mut task, &cell, is_serializable_cell_content, content_hash); + // Update cell_data_hash for non-bincodable cells. + update_cell_data_hash(&mut task, &cell, content_hash); let in_progress_cell = task.remove_in_progress_cells(&cell); @@ -224,38 +227,43 @@ impl UpdateCellOperation { } } + /// Whether this operation's mid-flight state can safely be persisted to + /// the operation suspend log. True iff the cell's value type has bincode — + /// non-bincodable values cannot be recovered across restart, so we don't + /// write a suspend point for them. fn is_serializable(&self) -> bool { match self { - UpdateCellOperation::InvalidateWhenCellDependency { - is_serializable_cell_content, - .. - } => *is_serializable_cell_content, - UpdateCellOperation::FinalCellChange { - is_serializable_cell_content, - .. - } => *is_serializable_cell_content, + UpdateCellOperation::InvalidateWhenCellDependency { cell_ref, .. } + | UpdateCellOperation::FinalCellChange { cell_ref, .. } => is_bincodable(cell_ref.cell), UpdateCellOperation::AggregationUpdate { .. } => true, UpdateCellOperation::Done => true, } } } -/// Updates the stored cell_data_hash for a non-serializable cell. +/// Returns `true` if cells of this type go through bincode on persist — +/// equivalently, the value type is `SerializationMode::Auto` or `Custom`. +fn is_bincodable(cell: CellId) -> bool { + matches!( + turbo_tasks::registry::get_value_type(cell.type_id).persistence, + turbo_tasks::ValueTypePersistence::Bincodable(_, _), + ) +} + +/// Updates the stored cell_data_hash for a non-bincodable cell. /// Skips the update if the hash hasn't changed to avoid unnecessary writes. -fn update_cell_data_hash( - task: &mut impl TaskGuard, - cell: &CellId, - is_serializable_cell_content: bool, - content_hash: Option, -) { - if !is_serializable_cell_content { - let old_hash = task.get_cell_data_hash(cell).copied(); - if old_hash != content_hash { - if let Some(hash) = content_hash { - task.insert_cell_data_hash(*cell, hash); - } else { - task.remove_cell_data_hash(cell); - } +/// Bincodable cells don't need a separate hash — their content is already on +/// disk for change detection after eviction. +fn update_cell_data_hash(task: &mut impl TaskGuard, cell: &CellId, content_hash: Option) { + if is_bincodable(*cell) { + return; + } + let old_hash = task.get_cell_data_hash(cell).copied(); + if old_hash != content_hash { + if let Some(hash) = content_hash { + task.insert_cell_data_hash(*cell, hash); + } else { + task.remove_cell_data_hash(cell); } } } @@ -268,7 +276,6 @@ impl Operation for UpdateCellOperation { } match self { UpdateCellOperation::InvalidateWhenCellDependency { - is_serializable_cell_content, cell_ref, ref mut dependent_tasks, #[cfg(feature = "trace_task_dirty")] @@ -311,7 +318,6 @@ impl Operation for UpdateCellOperation { } if dependent_tasks.is_empty() { self = UpdateCellOperation::FinalCellChange { - is_serializable_cell_content, cell_ref, content: take(content), queue: take(queue), @@ -319,7 +325,6 @@ impl Operation for UpdateCellOperation { } } UpdateCellOperation::FinalCellChange { - is_serializable_cell_content, cell_ref: CellRef { task, cell }, content, ref mut queue, @@ -327,7 +332,7 @@ impl Operation for UpdateCellOperation { let mut task = ctx.task(task, TaskDataCategory::Data); if let Some(content) = content { - task.add_cell_data(is_serializable_cell_content, cell, content); + task.add_cell_data(cell, content.into_untyped()); } let in_progress_cell = task.remove_in_progress_cells(&cell); diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs index a141195f4b84..54a8ca627499 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs @@ -21,15 +21,14 @@ use std::sync::Arc; use parking_lot::Mutex; use turbo_tasks::{ - CellId, SharedReference, TaskExecutionReason, TaskId, TraitTypeId, TypedSharedReference, - ValueTypeId, + CellId, SharedReference, TaskExecutionReason, TaskId, TraitTypeId, ValueTypeId, backend::{CachedTaskType, CellHash, TransientTaskType}, event::Event, task_storage, }; use crate::{ - backend::counter_map::CounterMap, + backend::{cell_data::CellData, counter_map::CounterMap}, data::{ ActivenessState, AggregationNumber, CellRef, CollectibleRef, CollectiblesRef, Dirtyness, InProgressCellState, InProgressState, LeafDistance, OutputValue, RootType, TransientTask, @@ -286,13 +285,27 @@ struct TaskStorageSchema { // ========================================================================= // CELL DATA (data) // ========================================================================= - /// Persistent cell data (serializable). - #[field(storage = "auto_map", category = "data", shrink_on_completion)] - persistent_cell_data: AutoMap, - - /// Transient cell data (not serializable). - #[field(storage = "auto_map", category = "transient", shrink_on_completion)] - transient_cell_data: AutoMap, + /// Cell data for all cells, regardless of serialization mode. + /// + /// `CellData` is a newtype over `AutoMap` whose + /// bincode impl filters out entries whose value type has no bincode fn + /// (`SerializationMode::None`, `Hash`, or `Derivable`) at encode time. + /// Those entries stay in memory but are not persisted — on restore the + /// next read triggers the "cell index in range but data missing" recompute + /// path. Sticky value types (non-reconstructible — `SerializationMode::None`) + /// are identified by `ValueType::sticky` for future eviction handling. + /// + /// Collapses the previous `persistent_cell_data` / `transient_cell_data` + /// split — routing every cell through a single map keyed by value type + /// deletes the `is_serializable_cell_content` bool that used to thread + /// through the read/write API. + #[field( + storage = "auto_map", + category = "data", + shrink_on_completion, + inner_type = "AutoMap" + )] + cell_data: CellData, /// Hash of transient cell data, persisted for hash-based change detection when /// transient data has been evicted from memory. diff --git a/turbopack/crates/turbo-tasks-backend/tests/derivable_cell.rs b/turbopack/crates/turbo-tasks-backend/tests/derivable_cell.rs new file mode 100644 index 000000000000..f3f29c8fcea0 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/tests/derivable_cell.rs @@ -0,0 +1,85 @@ +//! Verifies that `#[turbo_tasks::value(serialization = "...")]` maps to the +//! right [`ValueTypePersistence`] variant: +//! +//! - `"derivable"` / `"hash"` → `Derivable` (evictable, no bincode). +//! - `"none"` → `SessionStateful` (not evictable, no bincode). +//! - `"auto"` / `"custom"` → `Bincodable(_, _)` (evictable, restored from disk). +//! +//! The runtime behavior (reading/writing cells of each mode) is covered +//! transitively by every other test: the storage layer routes all modes +//! through the unified `CellData` map with identical semantics. Only the +//! persistence variant (and the macro's trait impls) differs. + +#![feature(arbitrary_self_types)] +#![feature(arbitrary_self_types_pointers)] +#![allow(clippy::needless_return)] + +use turbo_tasks::{ValueTypePersistence, VcValueType, registry}; +use turbo_tasks_testing::{Registration, register}; + +static REGISTRATION: Registration = register!(); + +#[turbo_tasks::value(serialization = "derivable")] +struct DerivedSum(u32); + +#[turbo_tasks::value(serialization = "none", cell = "new", eq = "manual")] +struct StickyHandle; + +#[turbo_tasks::value] +struct PersistedValue(u32); + +/// Trigger registration of every value type in this test file by constructing +/// a turbo_tasks instance. The global registry is populated by the +/// `#[turbo_tasks::value]` macro expansion's `register!()`-driven init. +fn ensure_registered() { + let _ = REGISTRATION.create_turbo_tasks("derivable_cell_test", true); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn derivable_value_type_maps_to_derivable_variant() { + ensure_registered(); + + let type_id = DerivedSum::get_value_type_id(); + let value_type = registry::get_value_type(type_id); + + assert!( + matches!(value_type.persistence, ValueTypePersistence::Derivable), + "Derivable serialization must map to ValueTypePersistence::Derivable" + ); + assert!( + !DerivedSum::has_serialization(), + "Derivable must report has_serialization() == false" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn none_value_type_maps_to_session_stateful_variant() { + ensure_registered(); + + let type_id = StickyHandle::get_value_type_id(); + let value_type = registry::get_value_type(type_id); + + assert!( + matches!( + value_type.persistence, + ValueTypePersistence::SessionStateful + ), + "None serialization must map to ValueTypePersistence::SessionStateful" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn auto_value_type_maps_to_bincodable_variant() { + ensure_registered(); + + let type_id = PersistedValue::get_value_type_id(); + let value_type = registry::get_value_type(type_id); + + assert!( + matches!( + value_type.persistence, + ValueTypePersistence::Bincodable(_, _) + ), + "Auto serialization must map to ValueTypePersistence::Bincodable" + ); +} diff --git a/turbopack/crates/turbo-tasks-macros/src/derive/task_storage_macro.rs b/turbopack/crates/turbo-tasks-macros/src/derive/task_storage_macro.rs index 399d649c0757..eb7f44614283 100644 --- a/turbopack/crates/turbo-tasks-macros/src/derive/task_storage_macro.rs +++ b/turbopack/crates/turbo-tasks-macros/src/derive/task_storage_macro.rs @@ -69,6 +69,16 @@ struct FieldInfo { /// If true, drop this field entirely after execution completes if the task is immutable. /// Immutable tasks don't re-execute, so dependency tracking fields are not needed. drop_on_completion_if_immutable: bool, + /// Optional override for the underlying map type, used when the field is a + /// newtype wrapping `AutoMap` (or similar) so callers can inject + /// custom bincode / accessor behavior while the macro still generates map + /// accessors with the right key/value types. + /// + /// The newtype must `Deref`/`DerefMut` to the inner map so the generated + /// accessors (which call `.iter()`, `.insert()`, etc.) keep working. + /// + /// When absent, the macro parses the outer field type directly. + inner_type: Option, } impl FieldInfo { @@ -363,6 +373,7 @@ fn parse_field_storage_attributes(field: &syn::Field) -> FieldInfo { let mut use_default = false; let mut shrink_on_completion = false; let mut drop_on_completion_if_immutable = false; + let mut inner_type: Option = None; // Find and parse the field attribute if let Some(attr) = field.attrs.iter().find(|attr| { @@ -437,11 +448,28 @@ fn parse_field_storage_attributes(field: &syn::Field) -> FieldInfo { }); } } + "inner_type" => { + if let Some(lit_str) = expect_string_literal(&nv.value, "inner_type") { + match syn::parse_str::(&lit_str.value()) { + Ok(ty) => inner_type = Some(ty), + Err(err) => { + lit_str + .span() + .unwrap() + .error(format!( + "`inner_type` must parse as a Rust type: {err}" + )) + .emit(); + } + } + } + } other => { meta.span() .unwrap() .error(format!( - "unknown attribute `{other}`, expected `storage` or `category`" + "unknown attribute `{other}`, expected `storage`, `category`, \ + or `inner_type`" )) .emit(); } @@ -581,6 +609,7 @@ fn parse_field_storage_attributes(field: &syn::Field) -> FieldInfo { use_default, shrink_on_completion, drop_on_completion_if_immutable, + inner_type, } } @@ -2302,7 +2331,12 @@ fn generate_countermap_ops(field: &FieldInfo) -> TokenStream { fn generate_automap_ops(field: &FieldInfo) -> TokenStream { let field_type = &field.field_type; - let Some((key_type, value_type)) = extract_map_types(field_type, "AutoMap") else { + // If the field uses a newtype wrapper, `inner_type` gives us the actual + // `AutoMap` to extract key/value types from. Otherwise parse the + // declared field type directly. + let map_ty = field.inner_type.as_ref().unwrap_or(field_type); + + let Some((key_type, value_type)) = extract_map_types(map_ty, "AutoMap") else { return quote! {}; }; diff --git a/turbopack/crates/turbo-tasks-macros/src/primitive_macro.rs b/turbopack/crates/turbo-tasks-macros/src/primitive_macro.rs index 4ab633e07b83..e1962bea9272 100644 --- a/turbopack/crates/turbo-tasks-macros/src/primitive_macro.rs +++ b/turbopack/crates/turbo-tasks-macros/src/primitive_macro.rs @@ -57,7 +57,7 @@ pub fn primitive(input: TokenStream) -> TokenStream { } } else { quote! { - turbo_tasks::ValueType::new_with_bincode::<#ty>(#name) + turbo_tasks::ValueType::bincodable::<#ty>(#name) } }; diff --git a/turbopack/crates/turbo-tasks-macros/src/value_macro.rs b/turbopack/crates/turbo-tasks-macros/src/value_macro.rs index a1560f097811..504745d66946 100644 --- a/turbopack/crates/turbo-tasks-macros/src/value_macro.rs +++ b/turbopack/crates/turbo-tasks-macros/src/value_macro.rs @@ -44,11 +44,21 @@ impl TryFrom for CellMode { } enum SerializationMode { + /// No bincode, **sticky** — cells of this type hold session-unique identity + /// (file system handles, worker pool handles, plugin DSOs, etc.) and cannot + /// be reconstructed by re-executing the producing task. The storage layer + /// keeps them in memory across eviction. None, /// Like `None` (no bincode serialization), but also stores a hash of the cell value so that /// changes can be detected even when the transient cell data has been evicted from memory. /// Only valid with `cell = "compare"` (or the default). Hash, + /// No bincode, **not sticky** — cells of this type can be freely dropped on + /// eviction because they are derivable: re-executing the producing task + /// from persistent inputs reproduces the same value. Use for outputs whose + /// in-memory form (SWC ASTs, codegen Ropes, etc.) isn't worth serializing + /// but is re-derivable. + Derivable, Auto, Custom, } @@ -67,11 +77,12 @@ impl TryFrom for SerializationMode { match lit.value().as_str() { "none" => Ok(SerializationMode::None), "hash" => Ok(SerializationMode::Hash), + "derivable" => Ok(SerializationMode::Derivable), "auto" => Ok(SerializationMode::Auto), "custom" => Ok(SerializationMode::Custom), _ => Err(Error::new_spanned( &lit, - "expected \"none\", \"hash\", \"auto\", or \"custom\"", + "expected \"none\", \"hash\", \"derivable\", \"auto\", or \"custom\"", )), } } @@ -363,7 +374,10 @@ pub fn value(args: TokenStream, input: TokenStream) -> TokenStream { #[bincode(crate = "turbo_tasks::macro_helpers::bincode")] }); } - SerializationMode::None | SerializationMode::Hash | SerializationMode::Custom => {} + SerializationMode::None + | SerializationMode::Hash + | SerializationMode::Derivable + | SerializationMode::Custom => {} }; if inner_type.is_some() { // Transparent structs have their own manual `ValueDebug` implementation. @@ -394,18 +408,25 @@ pub fn value(args: TokenStream, input: TokenStream) -> TokenStream { } let name = global_name_for_type(ident); + // Dispatch to the constructor whose name reflects the persistence mode. + // `Hash` and `Derivable` both map to `derivable` — hash-mode change + // detection is handled independently by the caller supplying a + // `content_hash`, not by a distinct persistence variant. let new_value_type = match serialization_mode { - SerializationMode::None | SerializationMode::Hash => quote! { - turbo_tasks::ValueType::new::<#ident>(#name) + SerializationMode::None => quote! { + turbo_tasks::ValueType::session_stateful::<#ident>(#name) + }, + SerializationMode::Hash | SerializationMode::Derivable => quote! { + turbo_tasks::ValueType::derivable::<#ident>(#name) + }, + SerializationMode::Auto | SerializationMode::Custom => quote! { + turbo_tasks::ValueType::bincodable::<#ident>(#name) }, - SerializationMode::Auto | SerializationMode::Custom => { - quote! { - turbo_tasks::ValueType::new_with_bincode::<#ident>(#name) - } - } }; let has_serialization = match serialization_mode { - SerializationMode::None | SerializationMode::Hash => quote! { false }, + SerializationMode::None | SerializationMode::Hash | SerializationMode::Derivable => { + quote! { false } + } SerializationMode::Auto | SerializationMode::Custom => quote! { true }, }; diff --git a/turbopack/crates/turbo-tasks-testing/src/lib.rs b/turbopack/crates/turbo-tasks-testing/src/lib.rs index f627ffa329e9..0d6555cb6f9b 100644 --- a/turbopack/crates/turbo-tasks-testing/src/lib.rs +++ b/turbopack/crates/turbo-tasks-testing/src/lib.rs @@ -277,7 +277,6 @@ impl TurboTasksApi for VcStorage { &self, task: TaskId, index: CellId, - _is_serializable_cell_content: bool, content: CellContent, _updated_key_hashes: Option>, _content_hash: Option<[u8; 16]>, diff --git a/turbopack/crates/turbo-tasks/src/backend.rs b/turbopack/crates/turbo-tasks/src/backend.rs index c6fca8c3939a..c30ec343770b 100644 --- a/turbopack/crates/turbo-tasks/src/backend.rs +++ b/turbopack/crates/turbo-tasks/src/backend.rs @@ -30,7 +30,7 @@ use turbo_tasks_hash::DeterministicHasher; use crate::{ RawVc, ReadCellOptions, ReadOutputOptions, ReadRef, SharedReference, TaskId, TaskIdSet, TaskPriority, TraitRef, TraitTypeId, TurboTasksCallApi, TurboTasksPanic, ValueTypeId, - VcValueTrait, VcValueType, + ValueTypePersistence, VcValueTrait, VcValueType, dyn_task_inputs::{DynTaskInputs, StackDynTaskInputs}, event::EventListener, macro_helpers::NativeFunction, @@ -258,10 +258,10 @@ impl TypedCellContent { let Self(type_id, content) = self; let value_type = registry::get_value_type(*type_id); type_id.encode(enc)?; - if let Some(bincode) = value_type.bincode { + if let ValueTypePersistence::Bincodable(encode_fn, _) = value_type.persistence { if let Some(reference) = &content.0 { true.encode(enc)?; - bincode.0(&*reference.0, enc)?; + encode_fn(&*reference.0, enc)?; Ok(()) } else { false.encode(enc)?; @@ -275,10 +275,10 @@ impl TypedCellContent { pub fn decode(dec: &mut TurboBincodeDecoder) -> Result { let type_id = ValueTypeId::decode(dec)?; let value_type = registry::get_value_type(type_id); - if let Some(bincode) = value_type.bincode { + if let ValueTypePersistence::Bincodable(_, decode_fn) = value_type.persistence { let is_some = bool::decode(dec)?; if is_some { - let reference = bincode.1(dec)?; + let reference = decode_fn(dec)?; return Ok(TypedCellContent(type_id, CellContent(Some(reference)))); } } @@ -643,7 +643,6 @@ pub trait Backend: Sync + Send { &self, task: TaskId, index: CellId, - is_serializable_cell_content: bool, content: CellContent, updated_key_hashes: Option>, content_hash: Option, diff --git a/turbopack/crates/turbo-tasks/src/lib.rs b/turbopack/crates/turbo-tasks/src/lib.rs index eb7d8149d67f..ac9e5031b26d 100644 --- a/turbopack/crates/turbo-tasks/src/lib.rs +++ b/turbopack/crates/turbo-tasks/src/lib.rs @@ -109,7 +109,7 @@ pub use crate::{ task_execution_reason::TaskExecutionReason, trait_ref::TraitRef, value::{TransientInstance, TransientValue}, - value_type::{TraitMethod, TraitType, ValueType}, + value_type::{TraitMethod, TraitType, ValueType, ValueTypePersistence}, vc::{ Dynamic, NonLocalValue, OperationValue, OperationVc, OptionVcExt, ReadVcFuture, ResolveOperationVcFuture, ResolveVcFuture, ResolvedVc, ToResolvedVcFuture, Upcast, diff --git a/turbopack/crates/turbo-tasks/src/manager.rs b/turbopack/crates/turbo-tasks/src/manager.rs index 4abe849cf68f..4c8eeb7fa966 100644 --- a/turbopack/crates/turbo-tasks/src/manager.rs +++ b/turbopack/crates/turbo-tasks/src/manager.rs @@ -172,7 +172,6 @@ pub trait TurboTasksApi: TurboTasksCallApi + Sync + Send { &self, task: TaskId, index: CellId, - is_serializable_cell_content: bool, content: CellContent, updated_key_hashes: Option>, content_hash: Option, @@ -1570,7 +1569,6 @@ impl TurboTasksApi for TurboTasks { &self, task: TaskId, index: CellId, - is_serializable_cell_content: bool, content: CellContent, updated_key_hashes: Option>, content_hash: Option, @@ -1579,7 +1577,6 @@ impl TurboTasksApi for TurboTasks { self.backend.update_task_cell( task, index, - is_serializable_cell_content, content, updated_key_hashes, content_hash, @@ -2041,7 +2038,6 @@ pub(crate) async fn read_task_output( pub struct CurrentCellRef { current_task: TaskId, index: CellId, - is_serializable_cell_content: bool, } type VcReadTarget = <::Read as VcRead>::Target; @@ -2084,7 +2080,6 @@ impl CurrentCellRef { ReadCellOptions { // INVALIDATION: Reading our own cell must be untracked tracking: ReadCellTracking::Untracked, - is_serializable_cell_content: self.is_serializable_cell_content, final_read_hint: false, }, ) @@ -2094,7 +2089,6 @@ impl CurrentCellRef { tt.update_own_task_cell( self.current_task, self.index, - self.is_serializable_cell_content, CellContent(Some(update)), updated_key_hashes, content_hash, @@ -2286,7 +2280,6 @@ impl CurrentCellRef { tt.update_own_task_cell( self.current_task, self.index, - self.is_serializable_cell_content, CellContent(Some(SharedReference::new(triomphe::Arc::new(new_value)))), None, None, @@ -2315,7 +2308,6 @@ impl CurrentCellRef { ReadCellOptions { // INVALIDATION: Reading our own cell must be untracked tracking: ReadCellTracking::Untracked, - is_serializable_cell_content: self.is_serializable_cell_content, final_read_hint: false, }, ) @@ -2333,7 +2325,6 @@ impl CurrentCellRef { tt.update_own_task_cell( self.current_task, self.index, - self.is_serializable_cell_content, CellContent(Some(shared_ref)), None, None, @@ -2355,10 +2346,10 @@ fn extract_sr_value(sr: &SharedReference) -> &T { } pub fn find_cell_by_type() -> CurrentCellRef { - find_cell_by_id(T::get_value_type_id(), T::has_serialization()) + find_cell_by_id(T::get_value_type_id()) } -pub fn find_cell_by_id(ty: ValueTypeId, is_serializable_cell_content: bool) -> CurrentCellRef { +pub fn find_cell_by_id(ty: ValueTypeId) -> CurrentCellRef { CURRENT_TASK_STATE.with(|ts| { let current_task = current_task("celling turbo_tasks values"); let mut ts = ts.write().unwrap(); @@ -2369,7 +2360,6 @@ pub fn find_cell_by_id(ty: ValueTypeId, is_serializable_cell_content: bool) -> C CurrentCellRef { current_task, index: CellId { type_id: ty, index }, - is_serializable_cell_content, } }) } diff --git a/turbopack/crates/turbo-tasks/src/raw_vc.rs b/turbopack/crates/turbo-tasks/src/raw_vc.rs index 9372d9e51d91..df977835b211 100644 --- a/turbopack/crates/turbo-tasks/src/raw_vc.rs +++ b/turbopack/crates/turbo-tasks/src/raw_vc.rs @@ -135,16 +135,10 @@ impl RawVc { } } - pub(crate) fn into_read(self, is_serializable_cell_content: bool) -> ReadRawVcFuture { + pub(crate) fn into_read(self) -> ReadRawVcFuture { // returns a custom future to have something concrete and sized // this avoids boxing in IntoFuture - ReadRawVcFuture::new(self, Some(is_serializable_cell_content)) - } - - pub(crate) fn into_read_with_unknown_is_serializable_cell_content(self) -> ReadRawVcFuture { - // returns a custom future to have something concrete and sized - // this avoids boxing in IntoFuture - ReadRawVcFuture::new(self, None) + ReadRawVcFuture::new(self) } /// See [`crate::Vc::to_resolved`]. @@ -396,10 +390,6 @@ pub struct ReadRawVcFuture { resolve: ResolveRawVcFuture, /// Phase 2: options for the cell read once we have a [`RawVc::TaskCell`]. read_cell_options: ReadCellOptions, - /// If `true`, the `is_serializable_cell_content` flag in `read_cell_options` is unknown at - /// construction time and must be determined lazily from the type registry once we reach the - /// [`RawVc::TaskCell`]. - is_serializable_cell_content_unknown: bool, /// Phase 2: the resolved task and cell identity, set when phase 1 completes. resolved: Option<(TaskId, CellId)>, /// Phase 2: listener for the cell read wait. @@ -407,14 +397,10 @@ pub struct ReadRawVcFuture { } impl ReadRawVcFuture { - pub(crate) fn new(vc: RawVc, is_serializable_cell_content: Option) -> Self { + pub(crate) fn new(vc: RawVc) -> Self { ReadRawVcFuture { resolve: ResolveRawVcFuture::new(vc), - read_cell_options: ReadCellOptions { - is_serializable_cell_content: is_serializable_cell_content.unwrap_or(false), - ..Default::default() - }, - is_serializable_cell_content_unknown: is_serializable_cell_content.is_none(), + read_cell_options: ReadCellOptions::default(), resolved: None, listener: None, } @@ -478,14 +464,6 @@ impl Future for ReadRawVcFuture { // At this point `this.resolved` is `Some((task, index))`. let (task, index) = this.resolved.unwrap(); - // Lazily resolve `is_serializable_cell_content` from the type registry on the first - // entry into phase 2, then clear the flag so subsequent polls skip this lookup. - if this.is_serializable_cell_content_unknown { - this.read_cell_options.is_serializable_cell_content = - get_value_type(index.type_id).bincode.is_some(); - this.is_serializable_cell_content_unknown = false; - } - let poll_fn = |tt: &Arc| -> Poll { loop { ready!(poll_listener(&mut this.listener, cx)); diff --git a/turbopack/crates/turbo-tasks/src/read_options.rs b/turbopack/crates/turbo-tasks/src/read_options.rs index 33a1e14115ff..3be979bfa0bb 100644 --- a/turbopack/crates/turbo-tasks/src/read_options.rs +++ b/turbopack/crates/turbo-tasks/src/read_options.rs @@ -3,7 +3,6 @@ use crate::{ReadConsistency, ReadTracking, manager::ReadCellTracking}; #[derive(Clone, Copy, Debug, Default)] pub struct ReadCellOptions { pub tracking: ReadCellTracking, - pub is_serializable_cell_content: bool, pub final_read_hint: bool, } diff --git a/turbopack/crates/turbo-tasks/src/task/shared_reference.rs b/turbopack/crates/turbo-tasks/src/task/shared_reference.rs index e7e93f783b0a..58655064ebe3 100644 --- a/turbopack/crates/turbo-tasks/src/task/shared_reference.rs +++ b/turbopack/crates/turbo-tasks/src/task/shared_reference.rs @@ -18,7 +18,7 @@ use turbo_bincode::{ use unsize::CoerceUnsize; use crate::{ - ValueType, ValueTypeId, registry, + ValueType, ValueTypeId, ValueTypePersistence, registry, triomphe_utils::{coerce_to_any_send_sync, downcast_triomphe_arc}, }; @@ -69,9 +69,9 @@ impl TurboBincodeEncode for TypedSharedReference { fn encode(&self, encoder: &mut TurboBincodeEncoder) -> Result<(), EncodeError> { let Self { type_id, reference } = self; let value_type = registry::get_value_type(*type_id); - if let Some(bincode) = value_type.bincode { + if let ValueTypePersistence::Bincodable(encode_fn, _) = value_type.persistence { type_id.encode(encoder)?; - bincode.0(&*reference.0, encoder)?; + encode_fn(&*reference.0, encoder)?; Ok(()) } else { Err(EncodeError::OtherString(format!( @@ -86,8 +86,8 @@ impl TurboBincodeDecode for TypedSharedReference { fn decode(decoder: &mut TurboBincodeDecoder) -> Result { let type_id = ValueTypeId::decode(decoder)?; let value_type = registry::get_value_type(type_id); - if let Some(bincode) = value_type.bincode { - let reference = bincode.1(decoder)?; + if let ValueTypePersistence::Bincodable(_, decode_fn) = value_type.persistence { + let reference = decode_fn(decoder)?; Ok(Self { type_id, reference }) } else { #[cold] diff --git a/turbopack/crates/turbo-tasks/src/value_type.rs b/turbopack/crates/turbo-tasks/src/value_type.rs index 396fc89a8e50..cd820268efd7 100644 --- a/turbopack/crates/turbo-tasks/src/value_type.rs +++ b/turbopack/crates/turbo-tasks/src/value_type.rs @@ -28,14 +28,36 @@ type Vtable = &'static [&'static NativeFunction]; // That's also needed in a distributed world, where the function might be only // available on a remote instance. +/// Cell-persistence behavior of a [`ValueType`]. +/// +/// The three variants correspond to mutually exclusive storage semantics, so +/// consumers can rely on a single match instead of checking multiple bits. +/// Adding a payload to `Derivable` later (e.g. a `DeriveCost` hint an eviction +/// policy can consult) is forwards-compatible. +pub enum ValueTypePersistence { + /// Bincode round-trips. Cells are evictable and restored from disk on next + /// access. Maps to `SerializationMode::Auto | Custom`. + Bincodable(AnyEncodeFn, AnyDecodeFn), + /// No bincode, but recomputable — the next reader after eviction triggers + /// a recompute from the task's inputs. Maps to + /// `SerializationMode::Derivable | Hash`. The hash-based change detection + /// for `Hash` is handled by the caller supplying a `content_hash`, not by + /// a distinct persistence variant. + Derivable, + /// No bincode, not recomputable — holds per-session identity (file system + /// handles, worker pools, plugin DSOs, transient env). Cells of this type + /// must stay in memory across eviction. Maps to `SerializationMode::None`. + SessionStateful, +} + /// A definition of a type of data. /// /// Contains a list of traits and trait methods that are available on that type. pub struct ValueType { pub ty: RegistryType, - /// Functions to convert to write the type to a buffer or read it from a buffer. - pub bincode: Option<(AnyEncodeFn, AnyDecodeFn)>, + /// How cells of this type participate in the persistent cache. + pub persistence: ValueTypePersistence, /// An implementation of /// [`VcCellMode::raw_cell`][crate::vc::VcCellMode::raw_cell]. @@ -86,18 +108,38 @@ pub trait ManualDecodeWrapper: Decode<()> { } impl ValueType { - /// This is internally used by [`#[turbo_tasks::value]`][crate::value]. - pub const fn new(global_name: &'static str) -> Self { - Self::new_inner::(global_name, None) + /// Construct a `ValueType` for a value that can be recomputed from its + /// task's inputs. Cells are evictable; the next reader after eviction + /// triggers a recompute. + /// + /// This is internally used by [`#[turbo_tasks::value]`][crate::value] for + /// `serialization = "derivable"` and `serialization = "hash"`. + pub const fn derivable(global_name: &'static str) -> Self { + Self::new_inner::(global_name, ValueTypePersistence::Derivable) } - /// This is internally used by [`#[turbo_tasks::value]`][crate::value]. - pub const fn new_with_bincode>( + /// Construct a `ValueType` whose cells cannot be reconstructed by + /// re-executing the task — they hold per-session identity (file system + /// handles, worker pools, plugin DSOs). The storage layer must keep them + /// in memory across eviction. + /// + /// This is internally used by [`#[turbo_tasks::value]`][crate::value] for + /// `serialization = "none"`. + pub const fn session_stateful(global_name: &'static str) -> Self { + Self::new_inner::(global_name, ValueTypePersistence::SessionStateful) + } + + /// Construct a `ValueType` whose cells bincode-round-trip. Cells are + /// evictable and restored from disk on next access. + /// + /// This is internally used by [`#[turbo_tasks::value]`][crate::value] for + /// `serialization = "auto"` and `serialization = "custom"`. + pub const fn bincodable>( global_name: &'static str, ) -> Self { Self::new_inner::( global_name, - Some(( + ValueTypePersistence::Bincodable( |this, enc| { T::encode(any_as_encode::(this), enc)?; Ok(()) @@ -106,7 +148,7 @@ impl ValueType { let val = T::decode(dec)?; Ok(SharedReference::new(triomphe::Arc::new(val))) }, - )), + ), ) } @@ -126,7 +168,7 @@ impl ValueType { ) -> Self { Self::new_inner::( global_name, - Some(( + ValueTypePersistence::Bincodable( |this, enc| { E::new(any_as_encode::(this)).encode(enc)?; Ok(()) @@ -135,18 +177,18 @@ impl ValueType { let val = D::inner(D::decode(dec)?); Ok(SharedReference::new(triomphe::Arc::new(val))) }, - )), + ), ) } // Helper for other constructor functions const fn new_inner( global_name: &'static str, - bincode: Option<(AnyEncodeFn, AnyDecodeFn)>, + persistence: ValueTypePersistence, ) -> Self { Self { ty: RegistryType::new::(std::any::type_name::(), global_name), - bincode, + persistence, raw_cell: >::raw_cell, traits: SyncUnsafeCell::new(ValueTypeTraits { traits: None }), } diff --git a/turbopack/crates/turbo-tasks/src/vc/mod.rs b/turbopack/crates/turbo-tasks/src/vc/mod.rs index 15a717f6e81b..4b2d6ecb522f 100644 --- a/turbopack/crates/turbo-tasks/src/vc/mod.rs +++ b/turbopack/crates/turbo-tasks/src/vc/mod.rs @@ -500,7 +500,7 @@ macro_rules! into_future { type Output = as Future>::Output; type IntoFuture = ReadVcFuture; fn into_future(self) -> Self::IntoFuture { - self.node.into_read(T::has_serialization()).into() + self.node.into_read().into() } } }; @@ -517,28 +517,19 @@ where /// Do not use this: Use [`OperationVc::read_strongly_consistent`] instead. #[cfg(feature = "non_operation_vc_strongly_consistent")] pub fn strongly_consistent(self) -> ReadVcFuture { - self.node - .into_read(T::has_serialization()) - .strongly_consistent() - .into() + self.node.into_read().strongly_consistent().into() } /// Returns a untracked read of the value. This will not invalidate the current function when /// the read value changed. pub fn untracked(self) -> ReadVcFuture { - self.node - .into_read(T::has_serialization()) - .untracked() - .into() + self.node.into_read().untracked().into() } /// Read the value with the hint that this is the final read of the value. This might drop the /// cell content. Future reads might need to recompute the value. pub fn final_read_hint(self) -> ReadVcFuture { - self.node - .into_read(T::has_serialization()) - .final_read_hint() - .into() + self.node.into_read().final_read_hint().into() } } @@ -549,7 +540,7 @@ where { /// Read the value and returns a owned version of it. It might clone the value. pub fn owned(self) -> ReadOwnedVcFuture { - let future: ReadVcFuture = self.node.into_read(T::has_serialization()).into(); + let future: ReadVcFuture = self.node.into_read().into(); future.owned() } } @@ -566,7 +557,7 @@ where Q: Hash + ?Sized, VcReadTarget: KeyedAccess, { - let future: ReadVcFuture = self.node.into_read(T::has_serialization()).into(); + let future: ReadVcFuture = self.node.into_read().into(); future.get(key) } @@ -577,7 +568,7 @@ where Q: Hash + ?Sized, VcReadTarget: KeyedAccess, { - let future: ReadVcFuture = self.node.into_read(T::has_serialization()).into(); + let future: ReadVcFuture = self.node.into_read().into(); future.contains_key(key) } } @@ -594,9 +585,7 @@ where /// have the same future-like semantics as value vcs when it comes to producing refs. This /// behavior is rarely needed, so in most cases, `.await`ing a trait vc is a mistake. pub fn into_trait_ref(self) -> ReadVcFuture> { - self.node - .into_read_with_unknown_is_serializable_cell_content() - .into() + self.node.into_read().into() } } diff --git a/turbopack/crates/turbo-tasks/src/vc/operation.rs b/turbopack/crates/turbo-tasks/src/vc/operation.rs index 71cc472c7c32..8a02be6fe720 100644 --- a/turbopack/crates/turbo-tasks/src/vc/operation.rs +++ b/turbopack/crates/turbo-tasks/src/vc/operation.rs @@ -171,11 +171,7 @@ impl OperationVc { where T: VcValueType, { - self.connect() - .node - .into_read(T::has_serialization()) - .strongly_consistent() - .into() + self.connect().node.into_read().strongly_consistent().into() } /// [Connects the `OperationVc`][Self::connect] and returns a [strongly From f47bdcbe737450f496ca1c42ae5a8229bc03b207 Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Sat, 18 Apr 2026 15:37:00 -0700 Subject: [PATCH 2/9] simplificationss --- .../turbo-tasks-backend/src/backend/mod.rs | 5 +-- .../src/backend/operation/update_cell.rs | 12 ++--- .../crates/turbo-tasks-testing/src/lib.rs | 10 +---- turbopack/crates/turbo-tasks/src/backend.rs | 8 +--- turbopack/crates/turbo-tasks/src/manager.rs | 44 +++---------------- 5 files changed, 14 insertions(+), 65 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 9ba72ef6f5ae..da81c0f9a89c 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -2891,7 +2891,6 @@ impl TurboTasksBackendInner { &self, task_id: TaskId, cell: CellId, - _options: ReadCellOptions, turbo_tasks: &dyn TurboTasksBackendApi>, ) -> Result { let mut ctx = self.execute_context(turbo_tasks); @@ -3537,11 +3536,9 @@ impl Backend for TurboTasksBackend { &self, task_id: TaskId, cell: CellId, - options: ReadCellOptions, turbo_tasks: &dyn TurboTasksBackendApi, ) -> Result { - self.0 - .try_read_own_task_cell(task_id, cell, options, turbo_tasks) + self.0.try_read_own_task_cell(task_id, cell, turbo_tasks) } fn read_task_collectibles( diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs index 782217ab310b..f0c2b85fe9cb 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs @@ -68,7 +68,7 @@ impl UpdateCellOperation { ); let content = if let CellContent(Some(new_content)) = content { - Some(new_content.into_typed(cell.type_id)) + Some(new_content) } else { None }; @@ -93,11 +93,7 @@ impl UpdateCellOperation { verification_mode, turbo_tasks::backend::VerificationMode::EqualityCheck ) - && content - != task - .get_cell_data(&cell) - .cloned() - .map(|r| r.into_typed(cell.type_id)) + && content.as_ref() != task.get_cell_data(&cell) { let task_description = task.get_task_description(); let cell_type = turbo_tasks::registry::get_value_type(cell.type_id) @@ -197,7 +193,7 @@ impl UpdateCellOperation { dependent_tasks, #[cfg(feature = "trace_task_dirty")] has_updated_key_hashes, - content, + content: content.map(|r| r.into_typed(cell.type_id)), queue: AggregationUpdateQueue::new(), } .execute(&mut ctx); @@ -209,7 +205,7 @@ impl UpdateCellOperation { // So we can just update the cell content. let old_content = if let Some(new_content) = content { - task.insert_cell_data(cell, new_content.into_untyped()) + task.insert_cell_data(cell, new_content) } else { task.remove_cell_data(&cell) }; diff --git a/turbopack/crates/turbo-tasks-testing/src/lib.rs b/turbopack/crates/turbo-tasks-testing/src/lib.rs index 0d6555cb6f9b..5409ccf5a072 100644 --- a/turbopack/crates/turbo-tasks-testing/src/lib.rs +++ b/turbopack/crates/turbo-tasks-testing/src/lib.rs @@ -220,9 +220,8 @@ impl TurboTasksApi for VcStorage { &self, current_task: TaskId, index: CellId, - options: ReadCellOptions, ) -> Result { - self.read_own_task_cell(current_task, index, options) + self.read_own_task_cell(current_task, index) } fn try_read_local_output( @@ -258,12 +257,7 @@ impl TurboTasksApi for VcStorage { unimplemented!() } - fn read_own_task_cell( - &self, - task: TaskId, - index: CellId, - _options: ReadCellOptions, - ) -> Result { + fn read_own_task_cell(&self, task: TaskId, index: CellId) -> Result { let map = self.cells.lock().unwrap(); Ok(if let Some(cell) = map.get(&(task, index)) { cell.to_owned() diff --git a/turbopack/crates/turbo-tasks/src/backend.rs b/turbopack/crates/turbo-tasks/src/backend.rs index c30ec343770b..6b83c5e9694b 100644 --- a/turbopack/crates/turbo-tasks/src/backend.rs +++ b/turbopack/crates/turbo-tasks/src/backend.rs @@ -603,14 +603,8 @@ pub trait Backend: Sync + Send { &self, current_task: TaskId, index: CellId, - options: ReadCellOptions, turbo_tasks: &dyn TurboTasksBackendApi, - ) -> Result { - match self.try_read_task_cell(current_task, index, None, options, turbo_tasks)? { - Ok(content) => Ok(content), - Err(_) => Ok(TypedCellContent(index.type_id, CellContent(None))), - } - } + ) -> Result; /// INVALIDATION: Be careful with this, when reader is None, it will not track dependencies, so /// using it could break cache invalidation. diff --git a/turbopack/crates/turbo-tasks/src/manager.rs b/turbopack/crates/turbo-tasks/src/manager.rs index 4c8eeb7fa966..40b10237c98d 100644 --- a/turbopack/crates/turbo-tasks/src/manager.rs +++ b/turbopack/crates/turbo-tasks/src/manager.rs @@ -159,15 +159,9 @@ pub trait TurboTasksApi: TurboTasksCallApi + Sync + Send { &self, current_task: TaskId, index: CellId, - options: ReadCellOptions, ) -> Result; - fn read_own_task_cell( - &self, - task: TaskId, - index: CellId, - options: ReadCellOptions, - ) -> Result; + fn read_own_task_cell(&self, task: TaskId, index: CellId) -> Result; fn update_own_task_cell( &self, task: TaskId, @@ -1483,10 +1477,9 @@ impl TurboTasksApi for TurboTasks { &self, current_task: TaskId, index: CellId, - options: ReadCellOptions, ) -> Result { self.backend - .try_read_own_task_cell(current_task, index, options, self) + .try_read_own_task_cell(current_task, index, self) } #[track_caller] @@ -1556,13 +1549,8 @@ impl TurboTasksApi for TurboTasks { } } - fn read_own_task_cell( - &self, - task: TaskId, - index: CellId, - options: ReadCellOptions, - ) -> Result { - self.try_read_own_task_cell(task, index, options) + fn read_own_task_cell(&self, task: TaskId, index: CellId) -> Result { + self.try_read_own_task_cell(task, index) } fn update_own_task_cell( @@ -2073,17 +2061,7 @@ impl CurrentCellRef { )>, ) { let tt = turbo_tasks(); - let cell_content = tt - .read_own_task_cell( - self.current_task, - self.index, - ReadCellOptions { - // INVALIDATION: Reading our own cell must be untracked - tracking: ReadCellTracking::Untracked, - final_read_hint: false, - }, - ) - .ok(); + let cell_content = tt.read_own_task_cell(self.current_task, self.index).ok(); let update = functor(cell_content.as_ref().and_then(|cc| cc.1.0.as_ref())); if let Some((update, updated_key_hashes, content_hash)) = update { tt.update_own_task_cell( @@ -2301,17 +2279,7 @@ impl CurrentCellRef { ) { let tt = turbo_tasks(); let update = if matches!(verification_mode, VerificationMode::EqualityCheck) { - let content = tt - .read_own_task_cell( - self.current_task, - self.index, - ReadCellOptions { - // INVALIDATION: Reading our own cell must be untracked - tracking: ReadCellTracking::Untracked, - final_read_hint: false, - }, - ) - .ok(); + let content = tt.read_own_task_cell(self.current_task, self.index).ok(); if let Some(TypedCellContent(_, CellContent(Some(shared_ref_exp)))) = content { // pointer equality (not value equality) shared_ref_exp != shared_ref From 8ca74e1a7b01d731ca2485aa4beb2705fe17a384 Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Sat, 18 Apr 2026 19:10:35 -0700 Subject: [PATCH 3/9] rename annotations, annotate everything --- crates/next-build-test/src/lib.rs | 2 +- .../src/next_api/analyze.rs | 2 +- .../src/next_api/endpoint.rs | 4 +- .../src/next_api/project.rs | 10 +- .../src/backend/cell_data.rs | 6 +- .../src/backend/storage_schema.rs | 12 +-- .../tests/derivable_cell.rs | 85 ---------------- .../tests/read_ref_cell.rs | 2 +- .../tests/trait_ref_cell.rs | 2 +- turbopack/crates/turbo-tasks-env/src/lib.rs | 4 +- .../crates/turbo-tasks-fs/src/embed/fs.rs | 2 +- turbopack/crates/turbo-tasks-fs/src/lib.rs | 4 +- .../turbo-tasks-macros/src/value_macro.rs | 50 +++++----- turbopack/crates/turbo-tasks/src/effect.rs | 6 +- .../crates/turbo-tasks/src/value_type.rs | 98 ++++++++++++++----- .../src/ecmascript/content.rs | 2 +- .../src/ecmascript/list/version.rs | 2 +- .../src/ecmascript/merged/content.rs | 2 +- .../src/ecmascript/merged/version.rs | 2 +- .../src/ecmascript/version.rs | 2 +- .../crates/turbopack-cli-utils/src/issue.rs | 2 +- .../src/chunk/available_modules.rs | 2 +- .../turbopack-core/src/diagnostics/mod.rs | 2 +- .../crates/turbopack-core/src/environment.rs | 2 +- .../crates/turbopack-core/src/issue/mod.rs | 8 +- .../turbopack-core/src/module_graph/mod.rs | 6 +- .../crates/turbopack-core/src/package_json.rs | 2 +- .../crates/turbopack-core/src/version.rs | 4 +- .../crates/turbopack-css/src/code_gen.rs | 2 +- turbopack/crates/turbopack-css/src/process.rs | 6 +- .../turbopack-css/src/references/import.rs | 2 +- .../crates/turbopack-dev-server/src/http.rs | 4 +- .../crates/turbopack-dev-server/src/lib.rs | 2 +- .../src/source/asset_graph.rs | 2 +- .../src/source/resolve.rs | 2 +- .../turbopack-dev-server/src/update/stream.rs | 4 +- .../transform/swc_ecma_transform_plugins.rs | 7 +- .../src/chunk/code_and_ids.rs | 4 +- .../turbopack-ecmascript/src/chunk/item.rs | 2 +- .../crates/turbopack-ecmascript/src/parse.rs | 2 +- .../turbopack-ecmascript/src/transform/mod.rs | 7 +- .../src/tree_shake/mod.rs | 2 +- .../turbopack-ecmascript/src/webpack/parse.rs | 2 +- .../crates/turbopack-node/src/evaluate.rs | 7 +- .../turbopack-node/src/process_pool/mod.rs | 7 +- .../turbopack-node/src/worker_pool/mod.rs | 7 +- .../src/worker_pool/operation.rs | 7 +- .../src/ecmascript/node/version.rs | 2 +- .../crates/turbopack-tests/tests/execution.rs | 2 +- .../tests/node-file-trace.rs | 2 +- 50 files changed, 205 insertions(+), 206 deletions(-) delete mode 100644 turbopack/crates/turbo-tasks-backend/tests/derivable_cell.rs diff --git a/crates/next-build-test/src/lib.rs b/crates/next-build-test/src/lib.rs index f60b5cfe9e31..e5e0d7a71acb 100644 --- a/crates/next-build-test/src/lib.rs +++ b/crates/next-build-test/src/lib.rs @@ -254,7 +254,7 @@ async fn endpoint_write_to_disk_with_apply( endpoint_write_to_disk(*endpoint) } - #[turbo_tasks::value(serialization = "none")] + #[turbo_tasks::value(serialization = "skip")] struct WithEffects { output_paths: ReadRef, effects: Effects, diff --git a/crates/next-napi-bindings/src/next_api/analyze.rs b/crates/next-napi-bindings/src/next_api/analyze.rs index 5151c73fd0e2..b9f5a42f4d50 100644 --- a/crates/next-napi-bindings/src/next_api/analyze.rs +++ b/crates/next-napi-bindings/src/next_api/analyze.rs @@ -15,7 +15,7 @@ use turbopack_core::{ use crate::next_api::utils::strongly_consistent_catch_collectables; -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] pub struct WriteAnalyzeResult { pub issues: Arc>>, pub diagnostics: Arc>>, diff --git a/crates/next-napi-bindings/src/next_api/endpoint.rs b/crates/next-napi-bindings/src/next_api/endpoint.rs index a46de5c045b7..c9f618fd694c 100644 --- a/crates/next-napi-bindings/src/next_api/endpoint.rs +++ b/crates/next-napi-bindings/src/next_api/endpoint.rs @@ -115,7 +115,7 @@ async fn issue_filter_from_endpoint( } } -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] struct WrittenEndpointWithIssues { written: Option>, issues: Arc>>, @@ -215,7 +215,7 @@ pub fn endpoint_server_changed_subscribe( ) } -#[turbo_tasks::value(shared, serialization = "none", eq = "manual")] +#[turbo_tasks::value(shared, serialization = "skip", eq = "manual")] struct EndpointIssuesAndDiags { changed: Option>, issues: Arc>>, diff --git a/crates/next-napi-bindings/src/next_api/project.rs b/crates/next-napi-bindings/src/next_api/project.rs index a55de27d4d68..5a14c6d37ec3 100644 --- a/crates/next-napi-bindings/src/next_api/project.rs +++ b/crates/next-napi-bindings/src/next_api/project.rs @@ -965,7 +965,7 @@ impl NapiEntrypoints { } } -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] struct EntrypointsWithIssues { entrypoints: Option>, issues: Arc>>, @@ -1000,14 +1000,14 @@ fn project_container_entrypoints_operation( container.entrypoints() } -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] struct OperationResult { issues: Arc>>, diagnostics: Arc>>, effects: Arc, } -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] struct AllWrittenEntrypointsWithIssues { entrypoints: Option>, issues: Arc>>, @@ -1800,7 +1800,7 @@ pub fn project_entrypoints_subscribe( ) } -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] struct HmrUpdateWithIssues { update: ReadRef, issues: Arc>>, @@ -1945,7 +1945,7 @@ struct HmrChunkNames { pub chunk_names: Vec, } -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] struct HmrChunkNamesWithIssues { chunk_names: ReadRef>, issues: Arc>>, diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/cell_data.rs b/turbopack/crates/turbo-tasks-backend/src/backend/cell_data.rs index bbb1b3f09308..069cf4aa94e8 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/cell_data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/cell_data.rs @@ -69,9 +69,9 @@ impl ShrinkToFit for CellData { impl TurboBincodeEncode for CellData { /// Writes `count-of-bincodable-entries` followed by each bincodable - /// `(CellId, encoded-value)`. Entries whose value type is `Derivable` or - /// `SessionStateful` (no bincode) are skipped; they will be reconstructed - /// on the next task execution after restore. + /// `(CellId, encoded-value)`. Entries whose value type is `SkipPersist` + /// or `SessionStateful` (no bincode) are skipped; they will be + /// reconstructed on the next task execution after restore. fn encode(&self, encoder: &mut TurboBincodeEncoder) -> Result<(), EncodeError> { // First pass: count bincodable entries. One extra O(N) iteration over // the registry — cold path (snapshot time only) and the registry is a diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs index 54a8ca627499..62b374b6648e 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs @@ -288,12 +288,12 @@ struct TaskStorageSchema { /// Cell data for all cells, regardless of serialization mode. /// /// `CellData` is a newtype over `AutoMap` whose - /// bincode impl filters out entries whose value type has no bincode fn - /// (`SerializationMode::None`, `Hash`, or `Derivable`) at encode time. - /// Those entries stay in memory but are not persisted — on restore the - /// next read triggers the "cell index in range but data missing" recompute - /// path. Sticky value types (non-reconstructible — `SerializationMode::None`) - /// are identified by `ValueType::sticky` for future eviction handling. + /// bincode impl filters out entries whose value type is not + /// `ValueTypePersistence::Bincodable` at encode time (i.e. `SkipPersist` + /// or `SessionStateful`). Those entries stay in memory but are not + /// persisted — on restore the next read triggers the "cell index in range + /// but data missing" recompute path. `SessionStateful` value types are + /// identified on `ValueType::persistence` for future eviction handling. /// /// Collapses the previous `persistent_cell_data` / `transient_cell_data` /// split — routing every cell through a single map keyed by value type diff --git a/turbopack/crates/turbo-tasks-backend/tests/derivable_cell.rs b/turbopack/crates/turbo-tasks-backend/tests/derivable_cell.rs deleted file mode 100644 index f3f29c8fcea0..000000000000 --- a/turbopack/crates/turbo-tasks-backend/tests/derivable_cell.rs +++ /dev/null @@ -1,85 +0,0 @@ -//! Verifies that `#[turbo_tasks::value(serialization = "...")]` maps to the -//! right [`ValueTypePersistence`] variant: -//! -//! - `"derivable"` / `"hash"` → `Derivable` (evictable, no bincode). -//! - `"none"` → `SessionStateful` (not evictable, no bincode). -//! - `"auto"` / `"custom"` → `Bincodable(_, _)` (evictable, restored from disk). -//! -//! The runtime behavior (reading/writing cells of each mode) is covered -//! transitively by every other test: the storage layer routes all modes -//! through the unified `CellData` map with identical semantics. Only the -//! persistence variant (and the macro's trait impls) differs. - -#![feature(arbitrary_self_types)] -#![feature(arbitrary_self_types_pointers)] -#![allow(clippy::needless_return)] - -use turbo_tasks::{ValueTypePersistence, VcValueType, registry}; -use turbo_tasks_testing::{Registration, register}; - -static REGISTRATION: Registration = register!(); - -#[turbo_tasks::value(serialization = "derivable")] -struct DerivedSum(u32); - -#[turbo_tasks::value(serialization = "none", cell = "new", eq = "manual")] -struct StickyHandle; - -#[turbo_tasks::value] -struct PersistedValue(u32); - -/// Trigger registration of every value type in this test file by constructing -/// a turbo_tasks instance. The global registry is populated by the -/// `#[turbo_tasks::value]` macro expansion's `register!()`-driven init. -fn ensure_registered() { - let _ = REGISTRATION.create_turbo_tasks("derivable_cell_test", true); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn derivable_value_type_maps_to_derivable_variant() { - ensure_registered(); - - let type_id = DerivedSum::get_value_type_id(); - let value_type = registry::get_value_type(type_id); - - assert!( - matches!(value_type.persistence, ValueTypePersistence::Derivable), - "Derivable serialization must map to ValueTypePersistence::Derivable" - ); - assert!( - !DerivedSum::has_serialization(), - "Derivable must report has_serialization() == false" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn none_value_type_maps_to_session_stateful_variant() { - ensure_registered(); - - let type_id = StickyHandle::get_value_type_id(); - let value_type = registry::get_value_type(type_id); - - assert!( - matches!( - value_type.persistence, - ValueTypePersistence::SessionStateful - ), - "None serialization must map to ValueTypePersistence::SessionStateful" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn auto_value_type_maps_to_bincodable_variant() { - ensure_registered(); - - let type_id = PersistedValue::get_value_type_id(); - let value_type = registry::get_value_type(type_id); - - assert!( - matches!( - value_type.persistence, - ValueTypePersistence::Bincodable(_, _) - ), - "Auto serialization must map to ValueTypePersistence::Bincodable" - ); -} diff --git a/turbopack/crates/turbo-tasks-backend/tests/read_ref_cell.rs b/turbopack/crates/turbo-tasks-backend/tests/read_ref_cell.rs index 0b8b6881e79a..ea92638c61ed 100644 --- a/turbopack/crates/turbo-tasks-backend/tests/read_ref_cell.rs +++ b/turbopack/crates/turbo-tasks-backend/tests/read_ref_cell.rs @@ -56,7 +56,7 @@ async fn test_read_ref() { #[turbo_tasks::value(transparent)] struct CounterValue(usize); -#[turbo_tasks::value(serialization = "none", cell = "new", eq = "manual")] +#[turbo_tasks::value(serialization = "session_stateful", cell = "new", eq = "manual")] struct Counter { #[turbo_tasks(debug_ignore, trace_ignore)] value: Mutex<(usize, HashSet)>, diff --git a/turbopack/crates/turbo-tasks-backend/tests/trait_ref_cell.rs b/turbopack/crates/turbo-tasks-backend/tests/trait_ref_cell.rs index 93c692885185..0c0303dfb66d 100644 --- a/turbopack/crates/turbo-tasks-backend/tests/trait_ref_cell.rs +++ b/turbopack/crates/turbo-tasks-backend/tests/trait_ref_cell.rs @@ -65,7 +65,7 @@ async fn trait_ref() { #[derive(Copy, Clone)] struct CounterValue(usize); -#[turbo_tasks::value(serialization = "none", cell = "new", eq = "manual")] +#[turbo_tasks::value(serialization = "session_stateful", cell = "new", eq = "manual")] struct Counter { #[turbo_tasks(debug_ignore, trace_ignore)] value: Mutex<(usize, HashSet)>, diff --git a/turbopack/crates/turbo-tasks-env/src/lib.rs b/turbopack/crates/turbo-tasks-env/src/lib.rs index 9c945251ff65..54e941ef7c9c 100644 --- a/turbopack/crates/turbo-tasks-env/src/lib.rs +++ b/turbopack/crates/turbo-tasks-env/src/lib.rs @@ -17,9 +17,9 @@ pub use self::{ filter::FilterProcessEnv, }; -/// Like [`EnvMap`], but with `serialization = "none"` to avoid storing +/// Like [`EnvMap`], but with `serialization = "skip"` to avoid storing /// environment variables (which may contain secrets) in the persistent cache. -#[turbo_tasks::value(transparent, serialization = "none")] +#[turbo_tasks::value(transparent, serialization = "skip")] pub struct TransientEnvMap(#[turbo_tasks(trace_ignore)] FxIndexMap); #[turbo_tasks::value_impl] diff --git a/turbopack/crates/turbo-tasks-fs/src/embed/fs.rs b/turbopack/crates/turbo-tasks-fs/src/embed/fs.rs index 705573490115..4d05303adccb 100644 --- a/turbopack/crates/turbo-tasks-fs/src/embed/fs.rs +++ b/turbopack/crates/turbo-tasks-fs/src/embed/fs.rs @@ -11,7 +11,7 @@ use crate::{ #[derive(ValueToString)] #[value_to_string(self.name)] -#[turbo_tasks::value(serialization = "none", cell = "new", eq = "manual")] +#[turbo_tasks::value(serialization = "session_stateful", cell = "new", eq = "manual")] pub struct EmbeddedFileSystem { name: RcStr, #[turbo_tasks(trace_ignore)] diff --git a/turbopack/crates/turbo-tasks-fs/src/lib.rs b/turbopack/crates/turbo-tasks-fs/src/lib.rs index 91c52c0f43a9..f422275d6f4d 100644 --- a/turbopack/crates/turbo-tasks-fs/src/lib.rs +++ b/turbopack/crates/turbo-tasks-fs/src/lib.rs @@ -2478,7 +2478,7 @@ impl FileContent { } /// A file's content interpreted as a JSON value. -#[turbo_tasks::value(shared, serialization = "none")] +#[turbo_tasks::value(shared, serialization = "skip")] pub enum FileJsonContent { Content(Value), Unparsable(Box), @@ -2549,7 +2549,7 @@ impl FileLine { } } -#[turbo_tasks::value(shared, serialization = "none")] +#[turbo_tasks::value(shared, serialization = "skip")] pub enum FileLinesContent { Lines(#[turbo_tasks(trace_ignore)] Vec), Unparsable, diff --git a/turbopack/crates/turbo-tasks-macros/src/value_macro.rs b/turbopack/crates/turbo-tasks-macros/src/value_macro.rs index 504745d66946..c7df1c9ad05c 100644 --- a/turbopack/crates/turbo-tasks-macros/src/value_macro.rs +++ b/turbopack/crates/turbo-tasks-macros/src/value_macro.rs @@ -44,21 +44,21 @@ impl TryFrom for CellMode { } enum SerializationMode { - /// No bincode, **sticky** — cells of this type hold session-unique identity - /// (file system handles, worker pool handles, plugin DSOs, etc.) and cannot - /// be reconstructed by re-executing the producing task. The storage layer - /// keeps them in memory across eviction. - None, - /// Like `None` (no bincode serialization), but also stores a hash of the cell value so that - /// changes can be detected even when the transient cell data has been evicted from memory. + /// No bincode: cells of this type hold session-scoped state + /// (file system handles, worker pool handles, plugin DSOs, `State<>` + /// interior mutability, etc.) and cannot be reconstructed by re-executing + /// the producing task. The storage layer keeps them in memory across + /// eviction. + SessionStateful, + /// Like `SessionStateful` (no bincode serialization), but also stores a hash of the cell value + /// so that changes can be detected even when the cell data has been evicted from memory. /// Only valid with `cell = "compare"` (or the default). Hash, - /// No bincode, **not sticky** — cells of this type can be freely dropped on - /// eviction because they are derivable: re-executing the producing task - /// from persistent inputs reproduces the same value. Use for outputs whose - /// in-memory form (SWC ASTs, codegen Ropes, etc.) isn't worth serializing - /// but is re-derivable. - Derivable, + /// No bincode, **but evictable** — skip persisting the cell content because + /// re-running the producing task to reproduce it is cheaper/simpler than + /// bincoding the in-memory form. Use for outputs whose in-memory form + /// (SWC ASTs, codegen Ropes, etc.) isn't worth serializing. + Skip, Auto, Custom, } @@ -75,14 +75,14 @@ impl TryFrom for SerializationMode { fn try_from(lit: LitStr) -> Result { match lit.value().as_str() { - "none" => Ok(SerializationMode::None), + "session_stateful" => Ok(SerializationMode::SessionStateful), "hash" => Ok(SerializationMode::Hash), - "derivable" => Ok(SerializationMode::Derivable), + "skip" => Ok(SerializationMode::Skip), "auto" => Ok(SerializationMode::Auto), "custom" => Ok(SerializationMode::Custom), _ => Err(Error::new_spanned( &lit, - "expected \"none\", \"hash\", \"derivable\", \"auto\", or \"custom\"", + "expected \"session_stateful\", \"hash\", \"skip\", \"auto\", or \"custom\"", )), } } @@ -374,9 +374,9 @@ pub fn value(args: TokenStream, input: TokenStream) -> TokenStream { #[bincode(crate = "turbo_tasks::macro_helpers::bincode")] }); } - SerializationMode::None + SerializationMode::SessionStateful | SerializationMode::Hash - | SerializationMode::Derivable + | SerializationMode::Skip | SerializationMode::Custom => {} }; if inner_type.is_some() { @@ -409,22 +409,22 @@ pub fn value(args: TokenStream, input: TokenStream) -> TokenStream { let name = global_name_for_type(ident); // Dispatch to the constructor whose name reflects the persistence mode. - // `Hash` and `Derivable` both map to `derivable` — hash-mode change - // detection is handled independently by the caller supplying a - // `content_hash`, not by a distinct persistence variant. + // `Hash` and `Skip` both map to `skip_persist` — hash-mode change detection + // is handled independently by the caller supplying a `content_hash`, not by + // a distinct persistence variant. let new_value_type = match serialization_mode { - SerializationMode::None => quote! { + SerializationMode::SessionStateful => quote! { turbo_tasks::ValueType::session_stateful::<#ident>(#name) }, - SerializationMode::Hash | SerializationMode::Derivable => quote! { - turbo_tasks::ValueType::derivable::<#ident>(#name) + SerializationMode::Hash | SerializationMode::Skip => quote! { + turbo_tasks::ValueType::skip_persist::<#ident>(#name) }, SerializationMode::Auto | SerializationMode::Custom => quote! { turbo_tasks::ValueType::bincodable::<#ident>(#name) }, }; let has_serialization = match serialization_mode { - SerializationMode::None | SerializationMode::Hash | SerializationMode::Derivable => { + SerializationMode::SessionStateful | SerializationMode::Hash | SerializationMode::Skip => { quote! { false } } SerializationMode::Auto | SerializationMode::Custom => quote! { true }, diff --git a/turbopack/crates/turbo-tasks/src/effect.rs b/turbopack/crates/turbo-tasks/src/effect.rs index 5432c57fa32a..b2a21574cd57 100644 --- a/turbopack/crates/turbo-tasks/src/effect.rs +++ b/turbopack/crates/turbo-tasks/src/effect.rs @@ -143,7 +143,7 @@ type DynEffectApplyFuture<'a> = Pin> + Send + trait EffectCollectible {} /// The Effect instance collectible that is emitted for effects. -#[turbo_tasks::value(serialization = "none", cell = "new", eq = "manual")] +#[turbo_tasks::value(serialization = "session_stateful", cell = "new", eq = "manual")] struct EffectInstance { #[turbo_tasks(debug_ignore)] inner: Box, @@ -199,7 +199,7 @@ pub fn emit_effect(effect: impl Effect) { /// # #[turbo_tasks::function(operation)] /// # fn some_turbo_tasks_operation(_args: Args) {} /// # -/// #[turbo_tasks::value(serialization = "none")] +/// #[turbo_tasks::value(serialization = "session_stateful")] /// struct OutputWithEffects { /// output: ReadRef, /// effects: Effects, @@ -255,7 +255,7 @@ type UniqueEffectIndices = Result)>, String>; /// Captured effects from an operation. This struct can be used to return Effects from a turbo-tasks /// function and apply them later. #[derive(Default)] -#[turbo_tasks::value(shared, eq = "manual", serialization = "none")] +#[turbo_tasks::value(shared, eq = "manual", serialization = "session_stateful")] pub struct Effects { #[turbo_tasks(debug_ignore)] effects: Vec>, diff --git a/turbopack/crates/turbo-tasks/src/value_type.rs b/turbopack/crates/turbo-tasks/src/value_type.rs index cd820268efd7..68fe969b6408 100644 --- a/turbopack/crates/turbo-tasks/src/value_type.rs +++ b/turbopack/crates/turbo-tasks/src/value_type.rs @@ -32,21 +32,24 @@ type Vtable = &'static [&'static NativeFunction]; /// /// The three variants correspond to mutually exclusive storage semantics, so /// consumers can rely on a single match instead of checking multiple bits. -/// Adding a payload to `Derivable` later (e.g. a `DeriveCost` hint an eviction -/// policy can consult) is forwards-compatible. +/// Adding a payload to `SkipPersist` later (e.g. a `DeriveCost` hint an +/// eviction policy can consult) is forwards-compatible. pub enum ValueTypePersistence { /// Bincode round-trips. Cells are evictable and restored from disk on next - /// access. Maps to `SerializationMode::Auto | Custom`. + /// access. Maps to `serialization = "auto" | "custom"`. Bincodable(AnyEncodeFn, AnyDecodeFn), - /// No bincode, but recomputable — the next reader after eviction triggers - /// a recompute from the task's inputs. Maps to - /// `SerializationMode::Derivable | Hash`. The hash-based change detection - /// for `Hash` is handled by the caller supplying a `content_hash`, not by - /// a distinct persistence variant. - Derivable, - /// No bincode, not recomputable — holds per-session identity (file system - /// handles, worker pools, plugin DSOs, transient env). Cells of this type - /// must stay in memory across eviction. Maps to `SerializationMode::None`. + /// No bincode: the value type opts out of being persisted because + /// re-running the producing task to reproduce the cell is cheaper/simpler + /// than bincoding the in-memory form. Cells are evictable; the next reader + /// after eviction triggers a recompute from the task's inputs. Maps to + /// `serialization = "skip" | "hash"`. The hash-based change detection for + /// `"hash"` is handled by the caller supplying a `content_hash`, not by a + /// distinct persistence variant. + SkipPersist, + /// No bincode, not reconstructible — holds session-scoped state (file + /// system handles, worker pools, plugin DSOs, `State<>` interior + /// mutability). Cells of this type must stay in memory across eviction. + /// Maps to `serialization = "session_stateful"`. SessionStateful, } @@ -108,23 +111,23 @@ pub trait ManualDecodeWrapper: Decode<()> { } impl ValueType { - /// Construct a `ValueType` for a value that can be recomputed from its - /// task's inputs. Cells are evictable; the next reader after eviction - /// triggers a recompute. + /// Construct a `ValueType` that opts out of being persisted. Cells are + /// evictable; the next reader after eviction triggers a recompute from + /// the task's inputs. /// /// This is internally used by [`#[turbo_tasks::value]`][crate::value] for - /// `serialization = "derivable"` and `serialization = "hash"`. - pub const fn derivable(global_name: &'static str) -> Self { - Self::new_inner::(global_name, ValueTypePersistence::Derivable) + /// `serialization = "skip"` and `serialization = "hash"`. + pub const fn skip_persist(global_name: &'static str) -> Self { + Self::new_inner::(global_name, ValueTypePersistence::SkipPersist) } /// Construct a `ValueType` whose cells cannot be reconstructed by - /// re-executing the task — they hold per-session identity (file system - /// handles, worker pools, plugin DSOs). The storage layer must keep them - /// in memory across eviction. + /// re-executing the task — they hold session-scoped state (file system + /// handles, worker pools, plugin DSOs, `State<>` interior mutability). + /// The storage layer must keep them in memory across eviction. /// /// This is internally used by [`#[turbo_tasks::value]`][crate::value] for - /// `serialization = "none"`. + /// `serialization = "session_stateful"`. pub const fn session_stateful(global_name: &'static str) -> Self { Self::new_inner::(global_name, ValueTypePersistence::SessionStateful) } @@ -387,3 +390,54 @@ pub const fn build_trait_vtable( } methods } + +#[cfg(test)] +mod tests { + //! Asserts that each `serialization = "..."` annotation lands on the right + //! `ValueTypePersistence` variant. These are purely compile-time / + //! macro-expansion properties of the value types, so no turbo_tasks runtime + //! is needed — we read the registered `ValueType` via `registry::get_value_type` + //! and match on `persistence`. + use super::ValueTypePersistence; + use crate::{self as turbo_tasks, VcValueType, registry}; + + #[turbo_tasks::value(serialization = "skip")] + struct SkipValue(#[turbo_tasks(trace_ignore)] u32); + + #[turbo_tasks::value(serialization = "session_stateful", cell = "new", eq = "manual")] + struct SessionStatefulValue; + + #[turbo_tasks::value] + struct BincodableValue(u32); + + #[test] + fn skip_maps_to_skip_persist() { + let vt = registry::get_value_type(SkipValue::get_value_type_id()); + assert!( + matches!(vt.persistence, ValueTypePersistence::SkipPersist), + "`serialization = \"skip\"` must map to ValueTypePersistence::SkipPersist" + ); + assert!(!SkipValue::has_serialization()); + } + + #[test] + fn session_stateful_maps_to_session_stateful() { + let vt = registry::get_value_type(SessionStatefulValue::get_value_type_id()); + assert!( + matches!(vt.persistence, ValueTypePersistence::SessionStateful), + "`serialization = \"session_stateful\"` must map to \ + ValueTypePersistence::SessionStateful" + ); + assert!(!SessionStatefulValue::has_serialization()); + } + + #[test] + fn default_maps_to_bincodable() { + let vt = registry::get_value_type(BincodableValue::get_value_type_id()); + assert!( + matches!(vt.persistence, ValueTypePersistence::Bincodable(_, _)), + "default (auto) serialization must map to ValueTypePersistence::Bincodable" + ); + assert!(BincodableValue::has_serialization()); + } +} diff --git a/turbopack/crates/turbopack-browser/src/ecmascript/content.rs b/turbopack/crates/turbopack-browser/src/ecmascript/content.rs index 43f4114ca815..d64336dbef3b 100644 --- a/turbopack/crates/turbopack-browser/src/ecmascript/content.rs +++ b/turbopack/crates/turbopack-browser/src/ecmascript/content.rs @@ -24,7 +24,7 @@ use crate::{ chunking_context::{CURRENT_CHUNK_METHOD_DOCUMENT_CURRENT_SCRIPT_EXPR, CurrentChunkMethod}, }; -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] pub struct EcmascriptBrowserChunkContent { pub(super) chunking_context: ResolvedVc, pub(super) chunk: ResolvedVc, diff --git a/turbopack/crates/turbopack-browser/src/ecmascript/list/version.rs b/turbopack/crates/turbopack-browser/src/ecmascript/list/version.rs index ae51b37d9ed0..e9e5f5fbba00 100644 --- a/turbopack/crates/turbopack-browser/src/ecmascript/list/version.rs +++ b/turbopack/crates/turbopack-browser/src/ecmascript/list/version.rs @@ -9,7 +9,7 @@ type VersionTraitRef = TraitRef>; /// The version of a [`EcmascriptDevChunkListContent`]. /// /// [`EcmascriptDevChunkListContent`]: super::content::EcmascriptDevChunkListContent -#[turbo_tasks::value(serialization = "none", shared)] +#[turbo_tasks::value(serialization = "skip", shared)] pub(super) struct EcmascriptDevChunkListVersion { /// A map from chunk path to its version. #[turbo_tasks(trace_ignore)] diff --git a/turbopack/crates/turbopack-browser/src/ecmascript/merged/content.rs b/turbopack/crates/turbopack-browser/src/ecmascript/merged/content.rs index 13e51f3fdfcf..1081901f816d 100644 --- a/turbopack/crates/turbopack-browser/src/ecmascript/merged/content.rs +++ b/turbopack/crates/turbopack-browser/src/ecmascript/merged/content.rs @@ -15,7 +15,7 @@ use super::{ /// [`EcmascriptChunkContentMerger`]. /// /// [`EcmascriptChunkContentMerger`]: super::merger::EcmascriptChunkContentMerger -#[turbo_tasks::value(serialization = "none", shared)] +#[turbo_tasks::value(serialization = "skip", shared)] pub(super) struct EcmascriptBrowserMergedChunkContent { pub contents: Vec>, } diff --git a/turbopack/crates/turbopack-browser/src/ecmascript/merged/version.rs b/turbopack/crates/turbopack-browser/src/ecmascript/merged/version.rs index 435cc7a4ebf9..5a3d709ee652 100644 --- a/turbopack/crates/turbopack-browser/src/ecmascript/merged/version.rs +++ b/turbopack/crates/turbopack-browser/src/ecmascript/merged/version.rs @@ -8,7 +8,7 @@ use super::super::version::EcmascriptBrowserChunkVersion; /// The version of a [`super::content::EcmascriptMergedChunkContent`]. This is /// essentially a composite [`EcmascriptChunkVersion`]. -#[turbo_tasks::value(serialization = "none", shared)] +#[turbo_tasks::value(serialization = "skip", shared)] pub(super) struct EcmascriptBrowserMergedChunkVersion { #[turbo_tasks(trace_ignore)] pub(super) versions: Vec>, diff --git a/turbopack/crates/turbopack-browser/src/ecmascript/version.rs b/turbopack/crates/turbopack-browser/src/ecmascript/version.rs index d13b135ac71e..777dbfca6310 100644 --- a/turbopack/crates/turbopack-browser/src/ecmascript/version.rs +++ b/turbopack/crates/turbopack-browser/src/ecmascript/version.rs @@ -8,7 +8,7 @@ use turbopack_ecmascript::chunk::EcmascriptChunkContent; use super::content_entry::EcmascriptBrowserChunkContentEntries; -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] pub(super) struct EcmascriptBrowserChunkVersion { pub(super) chunk_path: String, pub(super) entries_hashes: FxIndexMap, diff --git a/turbopack/crates/turbopack-cli-utils/src/issue.rs b/turbopack/crates/turbopack-cli-utils/src/issue.rs index b8ca54512715..01d47668bec6 100644 --- a/turbopack/crates/turbopack-cli-utils/src/issue.rs +++ b/turbopack/crates/turbopack-cli-utils/src/issue.rs @@ -349,7 +349,7 @@ impl SeenIssues { /// /// The ConsoleUi can be shared and capture issues from multiple sources, with deduplication /// operating across all issues. -#[turbo_tasks::value(shared, serialization = "none", eq = "manual")] +#[turbo_tasks::value(shared, serialization = "session_stateful", eq = "manual")] #[derive(Clone)] pub struct ConsoleUi { options: LogOptions, diff --git a/turbopack/crates/turbopack-core/src/chunk/available_modules.rs b/turbopack/crates/turbopack-core/src/chunk/available_modules.rs index ba430c840dc8..152f433260bd 100644 --- a/turbopack/crates/turbopack-core/src/chunk/available_modules.rs +++ b/turbopack/crates/turbopack-core/src/chunk/available_modules.rs @@ -140,7 +140,7 @@ impl AvailableModules { } } -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] #[derive(Debug, Clone)] pub struct AvailableModulesSnapshot { parent: Option>, diff --git a/turbopack/crates/turbopack-core/src/diagnostics/mod.rs b/turbopack/crates/turbopack-core/src/diagnostics/mod.rs index cfc82df59b71..9f2a440aa76f 100644 --- a/turbopack/crates/turbopack-core/src/diagnostics/mod.rs +++ b/turbopack/crates/turbopack-core/src/diagnostics/mod.rs @@ -6,7 +6,7 @@ use auto_hash_map::AutoSet; use turbo_rcstr::RcStr; use turbo_tasks::{CollectiblesSource, FxIndexMap, ResolvedVc, Upcast, Vc, emit}; -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] #[derive(Clone, Debug)] pub struct PlainDiagnostic { pub category: RcStr, diff --git a/turbopack/crates/turbopack-core/src/environment.rs b/turbopack/crates/turbopack-core/src/environment.rs index 4d9e021862eb..663d18b28d0a 100644 --- a/turbopack/crates/turbopack-core/src/environment.rs +++ b/turbopack/crates/turbopack-core/src/environment.rs @@ -354,7 +354,7 @@ impl EdgeWorkerEnvironment { // TODO preset_env_base::Version implements Serialize/Deserialize incorrectly #[derive(Debug)] -#[turbo_tasks::value(transparent, serialization = "none")] +#[turbo_tasks::value(transparent, serialization = "skip")] pub struct RuntimeVersions(#[turbo_tasks(trace_ignore)] pub Versions); #[turbo_tasks::value_impl] diff --git a/turbopack/crates/turbopack-core/src/issue/mod.rs b/turbopack/crates/turbopack-core/src/issue/mod.rs index a90e702d7d3c..47a39e9ec401 100644 --- a/turbopack/crates/turbopack-core/src/issue/mod.rs +++ b/turbopack/crates/turbopack-core/src/issue/mod.rs @@ -892,7 +892,7 @@ impl Display for IssueStage { } } -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] #[derive(Clone, Debug, PartialOrd, Ord)] pub struct PlainIssue { pub severity: IssueSeverity, @@ -910,7 +910,7 @@ pub struct PlainIssue { pub import_traces: Vec, } -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] #[derive(Clone, Debug, PartialOrd, Ord)] pub struct PlainAdditionalIssueSource { pub description: RcStr, @@ -1014,14 +1014,14 @@ impl PlainIssue { } } -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] #[derive(Clone, Debug, PartialOrd, Ord)] pub struct PlainIssueSource { pub asset: ReadRef, pub range: Option<(SourcePos, SourcePos)>, } -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] #[derive(Clone, Debug, PartialOrd, Ord)] pub struct PlainSource { pub ident: ReadRef, diff --git a/turbopack/crates/turbopack-core/src/module_graph/mod.rs b/turbopack/crates/turbopack-core/src/module_graph/mod.rs index 09785999d2e3..95f4be07d799 100644 --- a/turbopack/crates/turbopack-core/src/module_graph/mod.rs +++ b/turbopack/crates/turbopack-core/src/module_graph/mod.rs @@ -688,7 +688,7 @@ impl ImportTracer for ModuleGraphImportTracer { /// The ReadRef version of ModuleGraphBase. This is better for eventual consistency, as the graphs /// aren't awaited multiple times within the same task. -#[turbo_tasks::value(shared, serialization = "none", eq = "manual", cell = "new")] +#[turbo_tasks::value(shared, serialization = "skip", eq = "manual", cell = "new")] pub struct ModuleGraph { input_graphs: Vec>, input_binding_usage: Option>, @@ -858,7 +858,7 @@ impl Deref for ModuleGraph { } } -#[turbo_tasks::value(shared, serialization = "none", eq = "manual", cell = "new")] +#[turbo_tasks::value(shared, serialization = "skip", eq = "manual", cell = "new")] pub struct ModuleGraphLayer { snapshot: ModuleGraphSnapshot, } @@ -2310,7 +2310,7 @@ pub mod tests { + Send + 'static, ) { - #[turbo_tasks::value(serialization = "none", eq = "manual", cell = "new")] + #[turbo_tasks::value(serialization = "skip", eq = "manual", cell = "new")] struct SetupGraph { module_graph: ReadRef, entry_modules: Vec>>, diff --git a/turbopack/crates/turbopack-core/src/package_json.rs b/turbopack/crates/turbopack-core/src/package_json.rs index a7d98992224d..5188780c6f2e 100644 --- a/turbopack/crates/turbopack-core/src/package_json.rs +++ b/turbopack/crates/turbopack-core/src/package_json.rs @@ -32,7 +32,7 @@ impl Deref for PackageJson { } } -#[turbo_tasks::value(transparent, serialization = "none")] +#[turbo_tasks::value(transparent, serialization = "skip")] pub struct OptionPackageJson(Option); /// Reads a package.json file (if it exists). If the file is unparsable, it diff --git a/turbopack/crates/turbopack-core/src/version.rs b/turbopack/crates/turbopack-core/src/version.rs index 4583bc4de878..b637e0739e18 100644 --- a/turbopack/crates/turbopack-core/src/version.rs +++ b/turbopack/crates/turbopack-core/src/version.rs @@ -174,7 +174,7 @@ impl Version for NotFoundVersion { } /// Describes an update to a versioned object. -#[turbo_tasks::value(serialization = "none", shared)] +#[turbo_tasks::value(serialization = "skip", shared)] #[derive(Debug)] pub enum Update { /// The asset can't be meaningfully updated while the app is running, so the @@ -260,7 +260,7 @@ struct VersionRef( #[turbo_tasks(trace_ignore)] TraitRef>, ); -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "session_stateful")] pub struct VersionState { version: State, } diff --git a/turbopack/crates/turbopack-css/src/code_gen.rs b/turbopack/crates/turbopack-css/src/code_gen.rs index 284811bdb2f7..c94bc54d254d 100644 --- a/turbopack/crates/turbopack-css/src/code_gen.rs +++ b/turbopack/crates/turbopack-css/src/code_gen.rs @@ -5,7 +5,7 @@ use crate::chunk::CssImport; /// impl of code generation inferred from a ModuleReference. /// This is rust only and can't be implemented by non-rust plugins. -#[turbo_tasks::value(shared, serialization = "none", eq = "manual", cell = "new")] +#[turbo_tasks::value(shared, serialization = "skip", eq = "manual", cell = "new")] pub struct CodeGeneration { #[turbo_tasks(debug_ignore, trace_ignore)] pub imports: Vec, diff --git a/turbopack/crates/turbopack-css/src/process.rs b/turbopack/crates/turbopack-css/src/process.rs index 26b1c247c992..302d03f03eae 100644 --- a/turbopack/crates/turbopack-css/src/process.rs +++ b/turbopack/crates/turbopack-css/src/process.rs @@ -149,7 +149,7 @@ async fn stylesheet_to_css( #[turbo_tasks::value(transparent)] pub struct UnresolvedUrlReferences(pub Vec<(String, ResolvedVc)>); -#[turbo_tasks::value(shared, serialization = "none", eq = "manual", cell = "new")] +#[turbo_tasks::value(shared, serialization = "skip", eq = "manual", cell = "new")] #[allow(clippy::large_enum_variant)] // This is a turbo-tasks value pub enum ParseCssResult { Ok { @@ -169,7 +169,7 @@ pub enum ParseCssResult { NotFound, } -#[turbo_tasks::value(shared, serialization = "none", eq = "manual", cell = "new")] +#[turbo_tasks::value(shared, serialization = "skip", eq = "manual", cell = "new")] pub enum CssWithPlaceholderResult { Ok { parse_result: ResolvedVc, @@ -188,7 +188,7 @@ pub enum CssWithPlaceholderResult { NotFound, } -#[turbo_tasks::value(shared, serialization = "none")] +#[turbo_tasks::value(shared, serialization = "skip")] pub enum FinalCssResult { Ok { #[turbo_tasks(trace_ignore)] diff --git a/turbopack/crates/turbopack-css/src/references/import.rs b/turbopack/crates/turbopack-css/src/references/import.rs index fb5db849c2f9..e688f2c63e37 100644 --- a/turbopack/crates/turbopack-css/src/references/import.rs +++ b/turbopack/crates/turbopack-css/src/references/import.rs @@ -21,7 +21,7 @@ use crate::{ references::css_resolve, }; -#[turbo_tasks::value(eq = "manual", serialization = "none", shared)] +#[turbo_tasks::value(eq = "manual", serialization = "skip", shared)] #[derive(PartialEq)] pub enum ImportAttributes { LightningCss { diff --git a/turbopack/crates/turbopack-dev-server/src/http.rs b/turbopack/crates/turbopack-dev-server/src/http.rs index 5749499973e4..58b5565c51b4 100644 --- a/turbopack/crates/turbopack-dev-server/src/http.rs +++ b/turbopack/crates/turbopack-dev-server/src/http.rs @@ -28,7 +28,7 @@ use crate::source::{ resolve::{ResolveSourceRequestResult, resolve_source_request}, }; -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] enum GetFromSourceResult { Static { content: ReadRef, @@ -71,7 +71,7 @@ async fn get_from_source_operation( ) } -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] struct GetFromSourceResultWithCollectibles { result: ReadRef, effects: Effects, diff --git a/turbopack/crates/turbopack-dev-server/src/lib.rs b/turbopack/crates/turbopack-dev-server/src/lib.rs index 639788c3e6a5..132001a29ce1 100644 --- a/turbopack/crates/turbopack-dev-server/src/lib.rs +++ b/turbopack/crates/turbopack-dev-server/src/lib.rs @@ -55,7 +55,7 @@ where } } -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] struct ContentSourceWithIssues { source_op: OperationVc>, effects: Effects, diff --git a/turbopack/crates/turbopack-dev-server/src/source/asset_graph.rs b/turbopack/crates/turbopack-dev-server/src/source/asset_graph.rs index 413e772692de..26a5f3bc4461 100644 --- a/turbopack/crates/turbopack-dev-server/src/source/asset_graph.rs +++ b/turbopack/crates/turbopack-dev-server/src/source/asset_graph.rs @@ -28,7 +28,7 @@ struct OutputAssetsMap( type ExpandedState = State>; -#[turbo_tasks::value(serialization = "none", eq = "manual", cell = "new")] +#[turbo_tasks::value(serialization = "skip", eq = "manual", cell = "new")] pub struct AssetGraphContentSource { root_path: FileSystemPath, root_assets: ResolvedVc, diff --git a/turbopack/crates/turbopack-dev-server/src/source/resolve.rs b/turbopack/crates/turbopack-dev-server/src/source/resolve.rs index affd848b21a7..e27d034aa327 100644 --- a/turbopack/crates/turbopack-dev-server/src/source/resolve.rs +++ b/turbopack/crates/turbopack-dev-server/src/source/resolve.rs @@ -23,7 +23,7 @@ use crate::source::{ /// The result of [`resolve_source_request`]. Similar to a [`ContentSourceContent`], but without the /// [`Rewrite`][ContentSourceContent::Rewrite] variant, as this is taken care in the function. -#[turbo_tasks::value(serialization = "none", shared)] +#[turbo_tasks::value(serialization = "skip", shared)] pub enum ResolveSourceRequestResult { NotFound, Static(ResolvedVc, ResolvedVc), diff --git a/turbopack/crates/turbopack-dev-server/src/update/stream.rs b/turbopack/crates/turbopack-dev-server/src/update/stream.rs index 40d964b92720..b9b67a1bc622 100644 --- a/turbopack/crates/turbopack-dev-server/src/update/stream.rs +++ b/turbopack/crates/turbopack-dev-server/src/update/stream.rs @@ -359,7 +359,7 @@ impl Stream for UpdateStream { } } -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] #[derive(Debug)] pub enum UpdateStreamItem { NotFound, @@ -369,7 +369,7 @@ pub enum UpdateStreamItem { }, } -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] struct FatalStreamIssue { description: ResolvedVc, resource: RcStr, diff --git a/turbopack/crates/turbopack-ecmascript-plugins/src/transform/swc_ecma_transform_plugins.rs b/turbopack/crates/turbopack-ecmascript-plugins/src/transform/swc_ecma_transform_plugins.rs index e98e2030fca0..72750996310e 100644 --- a/turbopack/crates/turbopack-ecmascript-plugins/src/transform/swc_ecma_transform_plugins.rs +++ b/turbopack/crates/turbopack-ecmascript-plugins/src/transform/swc_ecma_transform_plugins.rs @@ -16,7 +16,12 @@ use turbopack_ecmascript::{CustomTransformer, TransformContext}; /// Internally this contains a `CompiledPluginModuleBytes`, which points to the /// compiled, serialized WASM module instead of raw file bytes to reduce the /// cost of the compilation. -#[turbo_tasks::value(serialization = "none", eq = "manual", cell = "new", shared)] +#[turbo_tasks::value( + serialization = "session_stateful", + eq = "manual", + cell = "new", + shared +)] pub struct SwcPluginModule { pub name: RcStr, #[turbo_tasks(trace_ignore, debug_ignore)] diff --git a/turbopack/crates/turbopack-ecmascript/src/chunk/code_and_ids.rs b/turbopack/crates/turbopack-ecmascript/src/chunk/code_and_ids.rs index 4e9594a1d647..a3818954bb4f 100644 --- a/turbopack/crates/turbopack-ecmascript/src/chunk/code_and_ids.rs +++ b/turbopack/crates/turbopack-ecmascript/src/chunk/code_and_ids.rs @@ -12,10 +12,10 @@ use crate::chunk::{ EcmascriptChunkItemWithAsyncInfo, }; -#[turbo_tasks::value(transparent, serialization = "none")] +#[turbo_tasks::value(transparent, serialization = "skip")] pub struct CodeAndIds(SmallVec<[(ModuleId, ReadRef); 1]>); -#[turbo_tasks::value(transparent, serialization = "none")] +#[turbo_tasks::value(transparent, serialization = "skip")] pub struct BatchGroupCodeAndIds( FxHashMap>, ); diff --git a/turbopack/crates/turbopack-ecmascript/src/chunk/item.rs b/turbopack/crates/turbopack-ecmascript/src/chunk/item.rs index fb046061f37e..7d382eaa6472 100644 --- a/turbopack/crates/turbopack-ecmascript/src/chunk/item.rs +++ b/turbopack/crates/turbopack-ecmascript/src/chunk/item.rs @@ -53,7 +53,7 @@ pub enum RewriteSourcePath { // Note we don't want to persist this as `module_factory_with_code_generation_issue` is already // persisted and we want to avoid duplicating it. -#[turbo_tasks::value(shared, serialization = "none")] +#[turbo_tasks::value(shared, serialization = "skip")] #[derive(Default, Clone)] pub struct EcmascriptChunkItemContent { pub inner_code: Rope, diff --git a/turbopack/crates/turbopack-ecmascript/src/parse.rs b/turbopack/crates/turbopack-ecmascript/src/parse.rs index 5c7755eb61a9..d54a27bbc0d3 100644 --- a/turbopack/crates/turbopack-ecmascript/src/parse.rs +++ b/turbopack/crates/turbopack-ecmascript/src/parse.rs @@ -144,7 +144,7 @@ impl Visit for IdentCollector { } } -#[turbo_tasks::value(shared, serialization = "none", eq = "manual", cell = "new")] +#[turbo_tasks::value(shared, serialization = "skip", eq = "manual", cell = "new")] #[allow(clippy::large_enum_variant)] pub enum ParseResult { // Note: Ok must not contain any Vc as it's snapshot by failsafe_parse diff --git a/turbopack/crates/turbopack-ecmascript/src/transform/mod.rs b/turbopack/crates/turbopack-ecmascript/src/transform/mod.rs index 53ad88cb617f..3a47053b6f72 100644 --- a/turbopack/crates/turbopack-ecmascript/src/transform/mod.rs +++ b/turbopack/crates/turbopack-ecmascript/src/transform/mod.rs @@ -93,7 +93,12 @@ pub trait CustomTransformer: Debug { /// A wrapper around a TransformPlugin instance, allowing it to operate with /// the turbo_task caching requirements. -#[turbo_tasks::value(transparent, serialization = "none", eq = "manual", cell = "new")] +#[turbo_tasks::value( + transparent, + serialization = "session_stateful", + eq = "manual", + cell = "new" +)] #[derive(Debug)] pub struct TransformPlugin(#[turbo_tasks(trace_ignore)] Box); diff --git a/turbopack/crates/turbopack-ecmascript/src/tree_shake/mod.rs b/turbopack/crates/turbopack-ecmascript/src/tree_shake/mod.rs index c35f0b219a5a..c85e7de426f2 100644 --- a/turbopack/crates/turbopack-ecmascript/src/tree_shake/mod.rs +++ b/turbopack/crates/turbopack-ecmascript/src/tree_shake/mod.rs @@ -437,7 +437,7 @@ async fn get_part_id(result: &SplitResult, part: &ModulePart) -> Result { ) } -#[turbo_tasks::value(shared, serialization = "none", eq = "manual")] +#[turbo_tasks::value(shared, serialization = "skip", eq = "manual")] pub(crate) enum SplitResult { Ok { asset_ident: ResolvedVc, diff --git a/turbopack/crates/turbopack-ecmascript/src/webpack/parse.rs b/turbopack/crates/turbopack-ecmascript/src/webpack/parse.rs index 74ed520b362b..7c411f1c0feb 100644 --- a/turbopack/crates/turbopack-ecmascript/src/webpack/parse.rs +++ b/turbopack/crates/turbopack-ecmascript/src/webpack/parse.rs @@ -24,7 +24,7 @@ use crate::{ utils::unparen, }; -#[turbo_tasks::value(shared, serialization = "none")] +#[turbo_tasks::value(shared, serialization = "skip")] #[derive(Debug)] pub enum WebpackRuntime { Webpack5 { diff --git a/turbopack/crates/turbopack-node/src/evaluate.rs b/turbopack/crates/turbopack-node/src/evaluate.rs index e0d34bc4d476..cafb38424772 100644 --- a/turbopack/crates/turbopack-node/src/evaluate.rs +++ b/turbopack/crates/turbopack-node/src/evaluate.rs @@ -67,7 +67,12 @@ enum EvalJavaScriptIncomingMessage { Error(StructuredError), } -#[turbo_tasks::value(cell = "new", serialization = "none", eq = "manual", shared)] +#[turbo_tasks::value( + cell = "new", + serialization = "session_stateful", + eq = "manual", + shared +)] pub struct EvaluatePool { #[turbo_tasks(trace_ignore, debug_ignore)] pool: Box, diff --git a/turbopack/crates/turbopack-node/src/process_pool/mod.rs b/turbopack/crates/turbopack-node/src/process_pool/mod.rs index 9f77f9101e90..5f3eaed2accc 100644 --- a/turbopack/crates/turbopack-node/src/process_pool/mod.rs +++ b/turbopack/crates/turbopack-node/src/process_pool/mod.rs @@ -531,7 +531,12 @@ impl ProcessArgs { /// /// The worker will *not* use the `env` of the parent process by default. All environment variables /// need to be provided to make the execution as pure as possible. -#[turbo_tasks::value(cell = "new", serialization = "none", eq = "manual", shared)] +#[turbo_tasks::value( + cell = "new", + serialization = "session_stateful", + eq = "manual", + shared +)] pub struct ChildProcessPool { cwd: PathBuf, entrypoint: PathBuf, diff --git a/turbopack/crates/turbopack-node/src/worker_pool/mod.rs b/turbopack/crates/turbopack-node/src/worker_pool/mod.rs index 717d89aa2bfd..0fe97c488659 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/mod.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/mod.rs @@ -36,7 +36,12 @@ mod worker_thread; static OPERATION_TASK_ID: AtomicU32 = AtomicU32::new(1); -#[turbo_tasks::value(cell = "new", serialization = "none", eq = "manual", shared)] +#[turbo_tasks::value( + cell = "new", + serialization = "session_stateful", + eq = "manual", + shared +)] pub(crate) struct WorkerThreadPool { worker_options: Arc, concurrency: usize, diff --git a/turbopack/crates/turbopack-node/src/worker_pool/operation.rs b/turbopack/crates/turbopack-node/src/worker_pool/operation.rs index 9060e585a1d3..8b82012d061c 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/operation.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/operation.rs @@ -57,7 +57,12 @@ pub(crate) struct PoolState { pub(crate) waiters: Mutex>>, } -#[turbo_tasks::value(cell = "new", serialization = "none", eq = "manual", shared)] +#[turbo_tasks::value( + cell = "new", + serialization = "session_stateful", + eq = "manual", + shared +)] #[derive(Clone, PartialEq, Eq, Hash)] pub(super) struct WorkerOptions { pub(super) filename: RcStr, diff --git a/turbopack/crates/turbopack-nodejs/src/ecmascript/node/version.rs b/turbopack/crates/turbopack-nodejs/src/ecmascript/node/version.rs index 2b124093557d..62d127481d3f 100644 --- a/turbopack/crates/turbopack-nodejs/src/ecmascript/node/version.rs +++ b/turbopack/crates/turbopack-nodejs/src/ecmascript/node/version.rs @@ -9,7 +9,7 @@ use turbopack_core::{ }; use turbopack_ecmascript::chunk::{CodeAndIds, EcmascriptChunkContent}; -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "skip")] pub(super) struct EcmascriptBuildNodeChunkVersion { pub(super) chunk_path: String, pub(super) chunk_items: Vec>, diff --git a/turbopack/crates/turbopack-tests/tests/execution.rs b/turbopack/crates/turbopack-tests/tests/execution.rs index 29002c8defbe..d04b96106dd5 100644 --- a/turbopack/crates/turbopack-tests/tests/execution.rs +++ b/turbopack/crates/turbopack-tests/tests/execution.rs @@ -213,7 +213,7 @@ async fn run(resource: PathBuf, snapshot_mode: IssueSnapshotMode) -> Result, effects: Effects, diff --git a/turbopack/crates/turbopack-tracing/tests/node-file-trace.rs b/turbopack/crates/turbopack-tracing/tests/node-file-trace.rs index f5ea4f931822..79d37f3dd3bb 100644 --- a/turbopack/crates/turbopack-tracing/tests/node-file-trace.rs +++ b/turbopack/crates/turbopack-tracing/tests/node-file-trace.rs @@ -346,7 +346,7 @@ fn bench_against_node_nft_inner(input: CaseInput) { }); } -#[turbo_tasks::value(serialization = "none")] +#[turbo_tasks::value(serialization = "session_stateful")] struct NodeFileTraceResult { rebased: ResolvedVc, effects: Effects, From 552d6c71d551fdecd370eed80e389860d5823104 Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Sun, 19 Apr 2026 09:08:46 -0700 Subject: [PATCH 4/9] comment --- .../src/transform/swc_ecma_transform_plugins.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/turbopack/crates/turbopack-ecmascript-plugins/src/transform/swc_ecma_transform_plugins.rs b/turbopack/crates/turbopack-ecmascript-plugins/src/transform/swc_ecma_transform_plugins.rs index 72750996310e..3f7be884dcfe 100644 --- a/turbopack/crates/turbopack-ecmascript-plugins/src/transform/swc_ecma_transform_plugins.rs +++ b/turbopack/crates/turbopack-ecmascript-plugins/src/transform/swc_ecma_transform_plugins.rs @@ -16,6 +16,9 @@ use turbopack_ecmascript::{CustomTransformer, TransformContext}; /// Internally this contains a `CompiledPluginModuleBytes`, which points to the /// compiled, serialized WASM module instead of raw file bytes to reduce the /// cost of the compilation. +/// +/// This is tagged as `session_stateful` to avoid evicting compiled modules from RAM on the theory +/// that there simply are not very many of them #[turbo_tasks::value( serialization = "session_stateful", eq = "manual", From f0b16c1a77111a4f038b977f6f2d1f6219275986 Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Sun, 19 Apr 2026 09:36:20 -0700 Subject: [PATCH 5/9] downgrades --- turbopack/crates/turbo-tasks-fs/src/embed/fs.rs | 2 +- turbopack/crates/turbopack-ecmascript/src/transform/mod.rs | 7 +------ .../crates/turbopack-node/src/worker_pool/operation.rs | 7 +------ 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/turbopack/crates/turbo-tasks-fs/src/embed/fs.rs b/turbopack/crates/turbo-tasks-fs/src/embed/fs.rs index 4d05303adccb..12136726ab6f 100644 --- a/turbopack/crates/turbo-tasks-fs/src/embed/fs.rs +++ b/turbopack/crates/turbo-tasks-fs/src/embed/fs.rs @@ -11,7 +11,7 @@ use crate::{ #[derive(ValueToString)] #[value_to_string(self.name)] -#[turbo_tasks::value(serialization = "session_stateful", cell = "new", eq = "manual")] +#[turbo_tasks::value(serialization = "skip", cell = "new", eq = "manual")] pub struct EmbeddedFileSystem { name: RcStr, #[turbo_tasks(trace_ignore)] diff --git a/turbopack/crates/turbopack-ecmascript/src/transform/mod.rs b/turbopack/crates/turbopack-ecmascript/src/transform/mod.rs index 3a47053b6f72..6c4ca8a238a5 100644 --- a/turbopack/crates/turbopack-ecmascript/src/transform/mod.rs +++ b/turbopack/crates/turbopack-ecmascript/src/transform/mod.rs @@ -93,12 +93,7 @@ pub trait CustomTransformer: Debug { /// A wrapper around a TransformPlugin instance, allowing it to operate with /// the turbo_task caching requirements. -#[turbo_tasks::value( - transparent, - serialization = "session_stateful", - eq = "manual", - cell = "new" -)] +#[turbo_tasks::value(transparent, serialization = "skip", eq = "manual", cell = "new")] #[derive(Debug)] pub struct TransformPlugin(#[turbo_tasks(trace_ignore)] Box); diff --git a/turbopack/crates/turbopack-node/src/worker_pool/operation.rs b/turbopack/crates/turbopack-node/src/worker_pool/operation.rs index 8b82012d061c..b3d462070649 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/operation.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/operation.rs @@ -57,12 +57,7 @@ pub(crate) struct PoolState { pub(crate) waiters: Mutex>>, } -#[turbo_tasks::value( - cell = "new", - serialization = "session_stateful", - eq = "manual", - shared -)] +#[turbo_tasks::value(cell = "new", serialization = "skip", eq = "manual", shared)] #[derive(Clone, PartialEq, Eq, Hash)] pub(super) struct WorkerOptions { pub(super) filename: RcStr, From e13e9294cc3d133f5d02444c1301a837426c7342 Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Sun, 19 Apr 2026 11:07:17 -0700 Subject: [PATCH 6/9] improve terminology --- .../src/backend/cell_data.rs | 13 ++-- .../turbo-tasks-backend/src/backend/mod.rs | 4 -- .../src/backend/operation/mod.rs | 5 -- .../src/backend/operation/update_cell.rs | 66 ++++++++++++------- .../src/backend/storage_schema.rs | 7 +- .../turbo-tasks-macros/src/primitive_macro.rs | 2 +- .../turbo-tasks-macros/src/value_macro.rs | 2 +- turbopack/crates/turbo-tasks/src/backend.rs | 4 +- .../turbo-tasks/src/task/shared_reference.rs | 4 +- .../crates/turbo-tasks/src/value_type.rs | 51 +++++++------- 10 files changed, 80 insertions(+), 78 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/cell_data.rs b/turbopack/crates/turbo-tasks-backend/src/backend/cell_data.rs index 069cf4aa94e8..40811eb94696 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/cell_data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/cell_data.rs @@ -68,12 +68,12 @@ impl ShrinkToFit for CellData { } impl TurboBincodeEncode for CellData { - /// Writes `count-of-bincodable-entries` followed by each bincodable + /// Writes `count-of-persistable-entries` followed by each persistable /// `(CellId, encoded-value)`. Entries whose value type is `SkipPersist` /// or `SessionStateful` (no bincode) are skipped; they will be /// reconstructed on the next task execution after restore. fn encode(&self, encoder: &mut TurboBincodeEncoder) -> Result<(), EncodeError> { - // First pass: count bincodable entries. One extra O(N) iteration over + // First pass: count persistable entries. One extra O(N) iteration over // the registry — cold path (snapshot time only) and the registry is a // static array indexed by ValueTypeId, so each lookup is cheap. let count = self @@ -82,14 +82,15 @@ impl TurboBincodeEncode for CellData { .filter(|(cell, _)| { matches!( registry::get_value_type(cell.type_id).persistence, - ValueTypePersistence::Bincodable(_, _), + ValueTypePersistence::Persistable(_, _), ) }) .count(); count.encode(encoder)?; + // TODO: consider sorting by type_id and delta encoding indices to reduce serialized size for (cell_id, reference) in self.0.iter() { let value_type = registry::get_value_type(cell_id.type_id); - let ValueTypePersistence::Bincodable(encode_fn, _) = value_type.persistence else { + let ValueTypePersistence::Persistable(encode_fn, _) = value_type.persistence else { continue; }; cell_id.encode(encoder)?; @@ -104,7 +105,7 @@ impl TurboBincodeDecode for CellData { /// `(CellId, SharedReference)` entry by looking up the value type's /// bincode decode function. /// - /// Missing cell types — or cells whose value type isn't `Bincodable` — + /// Missing cell types — or cells whose value type isn't `Persistable` — /// are a decode error: the encoder filters them out, so they should not /// appear on the wire. fn decode(decoder: &mut TurboBincodeDecoder) -> Result { @@ -113,7 +114,7 @@ impl TurboBincodeDecode for CellData { for _ in 0..count { let cell = CellId::decode(decoder)?; let value_type = registry::get_value_type(cell.type_id); - let ValueTypePersistence::Bincodable(_, decode_fn) = value_type.persistence else { + let ValueTypePersistence::Persistable(_, decode_fn) = value_type.persistence else { return Err(DecodeError::OtherString(format!( "cell of type {} has no bincode decoder", value_type.ty.global_name diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index da81c0f9a89c..8e05f9930493 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -2689,10 +2689,6 @@ impl TurboTasksBackendInner { // Note: We do not mark the tasks as dirty here, as these tasks are unused or stale // anyway and we want to avoid needless re-executions. When the cells become // used again, they are invalidated from the update cell operation. - // Remove cell data for cells that no longer exist. Both - // bincode-able and non-bincode-able cells live in the single - // `cell_data` map; identifying stale entries is purely a CellId - // index check. let to_remove: Vec<_> = task .iter_cell_data() .filter_map(|(cell, _)| { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index dd1ed98a2ed5..6993bfe0aaaa 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -1251,11 +1251,6 @@ pub trait TaskGuard: Debug + TaskStorageAccessors { dirty_count > clean_count } /// Add new cell data. Panics if the cell already had a value. - /// - /// The value type's serialization mode (including whether it's - /// bincode-able) is determined by `cell.type_id` via the `ValueType` - /// registry, not by a threaded bool — the `CellData` encoder filters - /// non-bincodable entries at snapshot time. fn add_cell_data(&mut self, cell: CellId, value: SharedReference) { let old = self.insert_cell_data(cell, value); assert!(old.is_none(), "Cell data already exists for {cell:?}"); diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs index f0c2b85fe9cb..9c2847371090 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs @@ -5,7 +5,7 @@ use once_cell::unsync::Lazy; use rustc_hash::FxHashSet; use smallvec::SmallVec; use turbo_tasks::{ - CellId, FxIndexMap, TaskId, TypedSharedReference, + CellId, FxIndexMap, TaskId, TypedSharedReference, ValueTypePersistence, backend::{CellContent, CellHash, VerificationMode}, }; @@ -58,13 +58,13 @@ impl UpdateCellOperation { #[cfg(not(feature = "verify_determinism"))] _verification_mode: VerificationMode, mut ctx: impl ExecuteContext<'_>, ) { - // Serializability is a property of the cell's value type — derive it - // from the registry rather than threading a redundant bool. - let is_bincodable_cell_content = is_bincodable(cell); - // content_hash is only meaningful for non-bincodable cells + // content_hash is only meaningful for `serialization = "hash"` cells — + // other modes pass `None`. This invariant lets the hash-based + // invalidation-skip below fall through to `false` for any non-hash + // cell without an explicit persistable guard. debug_assert!( - !is_bincodable_cell_content || content_hash.is_none(), - "content_hash must be None for bincodable cell content" + !is_persistable(cell) || content_hash.is_none(), + "content_hash must be None for persistable cell content" ); let content = if let CellContent(Some(new_content)) = content { @@ -114,9 +114,12 @@ impl UpdateCellOperation { // When not recomputing, we need to notify dependent tasks if the content actually // changes. - // For transient cells without available content, use hash-based comparison to - // detect whether the value actually changed—avoiding unnecessary invalidation. - let skip_invalidation = !is_bincodable_cell_content && { + // For cells without available content, use hash-based comparison + // to detect whether the value actually changed—avoiding + // unnecessary invalidation. Only `serialization = "hash"` cells + // supply a `content_hash`; for all other modes `content_hash` is + // `None` and this falls through to `false`. + let skip_invalidation = { let has_old_content = task.cell_data_contains(&cell); if !has_old_content { match (content_hash, task.get_cell_data_hash(&cell)) { @@ -210,7 +213,7 @@ impl UpdateCellOperation { task.remove_cell_data(&cell) }; - // Update cell_data_hash for non-bincodable cells. + // Update cell_data_hash for non-persistable cells. update_cell_data_hash(&mut task, &cell, content_hash); let in_progress_cell = task.remove_in_progress_cells(&cell); @@ -225,33 +228,46 @@ impl UpdateCellOperation { /// Whether this operation's mid-flight state can safely be persisted to /// the operation suspend log. True iff the cell's value type has bincode — - /// non-bincodable values cannot be recovered across restart, so we don't + /// non-persistable values cannot be recovered across restart, so we don't /// write a suspend point for them. fn is_serializable(&self) -> bool { match self { UpdateCellOperation::InvalidateWhenCellDependency { cell_ref, .. } - | UpdateCellOperation::FinalCellChange { cell_ref, .. } => is_bincodable(cell_ref.cell), + | UpdateCellOperation::FinalCellChange { cell_ref, .. } => { + is_persistable(cell_ref.cell) + } UpdateCellOperation::AggregationUpdate { .. } => true, UpdateCellOperation::Done => true, } } } -/// Returns `true` if cells of this type go through bincode on persist — -/// equivalently, the value type is `SerializationMode::Auto` or `Custom`. -fn is_bincodable(cell: CellId) -> bool { - matches!( - turbo_tasks::registry::get_value_type(cell.type_id).persistence, - turbo_tasks::ValueTypePersistence::Bincodable(_, _), - ) +fn is_persistable(cell: CellId) -> bool { + matches!(persistence(cell), ValueTypePersistence::Persistable(_, _),) +} + +/// Returns `true` if cells of this type need a stored `cell_data_hash`. +fn needs_content_hash(cell: CellId) -> bool { + matches!(persistence(cell), ValueTypePersistence::SkipPersist,) +} + +fn persistence(cell: CellId) -> &'static ValueTypePersistence { + &turbo_tasks::registry::get_value_type(cell.type_id).persistence } -/// Updates the stored cell_data_hash for a non-bincodable cell. -/// Skips the update if the hash hasn't changed to avoid unnecessary writes. -/// Bincodable cells don't need a separate hash — their content is already on -/// disk for change detection after eviction. +/// Updates the stored cell_data_hash, which only `serialization = "hash"` +/// cells consult (on eviction + recompute). Skips the update for all other +/// persistence modes and when the hash hasn't changed. +/// +/// `Persistable` cells store content directly; `SessionStateful` cells are +/// pinned in memory. Neither ever reads a hash back, so writing one is waste. +/// The callers of this function always pass `content_hash = None` for those +/// modes (the hash-producing `hashed_compare_and_update` API is only emitted +/// by the `"hash"` macro path), so the function would behave identically +/// without the gate — keeping it for clarity and to skip a map access on the +/// hot write path. fn update_cell_data_hash(task: &mut impl TaskGuard, cell: &CellId, content_hash: Option) { - if is_bincodable(*cell) { + if !needs_content_hash(*cell) { return; } let old_hash = task.get_cell_data_hash(cell).copied(); diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs index 62b374b6648e..bb42a72f52c5 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs @@ -289,16 +289,11 @@ struct TaskStorageSchema { /// /// `CellData` is a newtype over `AutoMap` whose /// bincode impl filters out entries whose value type is not - /// `ValueTypePersistence::Bincodable` at encode time (i.e. `SkipPersist` + /// `ValueTypePersistence::Persistable` at encode time (i.e. `SkipPersist` /// or `SessionStateful`). Those entries stay in memory but are not /// persisted — on restore the next read triggers the "cell index in range /// but data missing" recompute path. `SessionStateful` value types are /// identified on `ValueType::persistence` for future eviction handling. - /// - /// Collapses the previous `persistent_cell_data` / `transient_cell_data` - /// split — routing every cell through a single map keyed by value type - /// deletes the `is_serializable_cell_content` bool that used to thread - /// through the read/write API. #[field( storage = "auto_map", category = "data", diff --git a/turbopack/crates/turbo-tasks-macros/src/primitive_macro.rs b/turbopack/crates/turbo-tasks-macros/src/primitive_macro.rs index e1962bea9272..a0668cfe507c 100644 --- a/turbopack/crates/turbo-tasks-macros/src/primitive_macro.rs +++ b/turbopack/crates/turbo-tasks-macros/src/primitive_macro.rs @@ -57,7 +57,7 @@ pub fn primitive(input: TokenStream) -> TokenStream { } } else { quote! { - turbo_tasks::ValueType::bincodable::<#ty>(#name) + turbo_tasks::ValueType::persistable::<#ty>(#name) } }; diff --git a/turbopack/crates/turbo-tasks-macros/src/value_macro.rs b/turbopack/crates/turbo-tasks-macros/src/value_macro.rs index c7df1c9ad05c..19daacc68a68 100644 --- a/turbopack/crates/turbo-tasks-macros/src/value_macro.rs +++ b/turbopack/crates/turbo-tasks-macros/src/value_macro.rs @@ -420,7 +420,7 @@ pub fn value(args: TokenStream, input: TokenStream) -> TokenStream { turbo_tasks::ValueType::skip_persist::<#ident>(#name) }, SerializationMode::Auto | SerializationMode::Custom => quote! { - turbo_tasks::ValueType::bincodable::<#ident>(#name) + turbo_tasks::ValueType::persistable::<#ident>(#name) }, }; let has_serialization = match serialization_mode { diff --git a/turbopack/crates/turbo-tasks/src/backend.rs b/turbopack/crates/turbo-tasks/src/backend.rs index 6b83c5e9694b..0bb836768601 100644 --- a/turbopack/crates/turbo-tasks/src/backend.rs +++ b/turbopack/crates/turbo-tasks/src/backend.rs @@ -258,7 +258,7 @@ impl TypedCellContent { let Self(type_id, content) = self; let value_type = registry::get_value_type(*type_id); type_id.encode(enc)?; - if let ValueTypePersistence::Bincodable(encode_fn, _) = value_type.persistence { + if let ValueTypePersistence::Persistable(encode_fn, _) = value_type.persistence { if let Some(reference) = &content.0 { true.encode(enc)?; encode_fn(&*reference.0, enc)?; @@ -275,7 +275,7 @@ impl TypedCellContent { pub fn decode(dec: &mut TurboBincodeDecoder) -> Result { let type_id = ValueTypeId::decode(dec)?; let value_type = registry::get_value_type(type_id); - if let ValueTypePersistence::Bincodable(_, decode_fn) = value_type.persistence { + if let ValueTypePersistence::Persistable(_, decode_fn) = value_type.persistence { let is_some = bool::decode(dec)?; if is_some { let reference = decode_fn(dec)?; diff --git a/turbopack/crates/turbo-tasks/src/task/shared_reference.rs b/turbopack/crates/turbo-tasks/src/task/shared_reference.rs index 58655064ebe3..56609f4f2238 100644 --- a/turbopack/crates/turbo-tasks/src/task/shared_reference.rs +++ b/turbopack/crates/turbo-tasks/src/task/shared_reference.rs @@ -69,7 +69,7 @@ impl TurboBincodeEncode for TypedSharedReference { fn encode(&self, encoder: &mut TurboBincodeEncoder) -> Result<(), EncodeError> { let Self { type_id, reference } = self; let value_type = registry::get_value_type(*type_id); - if let ValueTypePersistence::Bincodable(encode_fn, _) = value_type.persistence { + if let ValueTypePersistence::Persistable(encode_fn, _) = value_type.persistence { type_id.encode(encoder)?; encode_fn(&*reference.0, encoder)?; Ok(()) @@ -86,7 +86,7 @@ impl TurboBincodeDecode for TypedSharedReference { fn decode(decoder: &mut TurboBincodeDecoder) -> Result { let type_id = ValueTypeId::decode(decoder)?; let value_type = registry::get_value_type(type_id); - if let ValueTypePersistence::Bincodable(_, decode_fn) = value_type.persistence { + if let ValueTypePersistence::Persistable(_, decode_fn) = value_type.persistence { let reference = decode_fn(decoder)?; Ok(Self { type_id, reference }) } else { diff --git a/turbopack/crates/turbo-tasks/src/value_type.rs b/turbopack/crates/turbo-tasks/src/value_type.rs index 68fe969b6408..4e44e16d1268 100644 --- a/turbopack/crates/turbo-tasks/src/value_type.rs +++ b/turbopack/crates/turbo-tasks/src/value_type.rs @@ -30,24 +30,23 @@ type Vtable = &'static [&'static NativeFunction]; /// Cell-persistence behavior of a [`ValueType`]. /// -/// The three variants correspond to mutually exclusive storage semantics, so -/// consumers can rely on a single match instead of checking multiple bits. -/// Adding a payload to `SkipPersist` later (e.g. a `DeriveCost` hint an -/// eviction policy can consult) is forwards-compatible. +/// Carries the serializer/deserializer pair for `Persistable` values — today +/// that's bincode, but the enum name is neutral so the choice of mechanism can +/// evolve without a cascade of rename work. pub enum ValueTypePersistence { - /// Bincode round-trips. Cells are evictable and restored from disk on next - /// access. Maps to `serialization = "auto" | "custom"`. - Bincodable(AnyEncodeFn, AnyDecodeFn), - /// No bincode: the value type opts out of being persisted because - /// re-running the producing task to reproduce the cell is cheaper/simpler - /// than bincoding the in-memory form. Cells are evictable; the next reader + /// Cells are serialized to the persistent cache and restored on next + /// access after eviction. Maps to `serialization = "auto" | "custom"`. + Persistable(AnyEncodeFn, AnyDecodeFn), + /// The value type opts out of being persisted because re-running the + /// producing task to reproduce the cell is cheaper/simpler than + /// serializing the in-memory form. Cells are evictable; the next reader /// after eviction triggers a recompute from the task's inputs. Maps to - /// `serialization = "skip" | "hash"`. The hash-based change detection for - /// `"hash"` is handled by the caller supplying a `content_hash`, not by a - /// distinct persistence variant. + /// `serialization = "skip" | "hash"`. The hash-based change detection + /// for `"hash"` is handled by the caller supplying a `content_hash`, not + /// by a distinct persistence variant. SkipPersist, - /// No bincode, not reconstructible — holds session-scoped state (file - /// system handles, worker pools, plugin DSOs, `State<>` interior + /// Not persistable, not reconstructible — holds session-scoped state + /// (file system handles, worker pools, plugin DSOs, `State<>` interior /// mutability). Cells of this type must stay in memory across eviction. /// Maps to `serialization = "session_stateful"`. SessionStateful, @@ -132,17 +131,17 @@ impl ValueType { Self::new_inner::(global_name, ValueTypePersistence::SessionStateful) } - /// Construct a `ValueType` whose cells bincode-round-trip. Cells are - /// evictable and restored from disk on next access. + /// Construct a `ValueType` whose cells round-trip through the persistent + /// cache. Cells are evictable and restored from disk on next access. /// /// This is internally used by [`#[turbo_tasks::value]`][crate::value] for /// `serialization = "auto"` and `serialization = "custom"`. - pub const fn bincodable>( + pub const fn persistable>( global_name: &'static str, ) -> Self { Self::new_inner::( global_name, - ValueTypePersistence::Bincodable( + ValueTypePersistence::Persistable( |this, enc| { T::encode(any_as_encode::(this), enc)?; Ok(()) @@ -171,7 +170,7 @@ impl ValueType { ) -> Self { Self::new_inner::( global_name, - ValueTypePersistence::Bincodable( + ValueTypePersistence::Persistable( |this, enc| { E::new(any_as_encode::(this)).encode(enc)?; Ok(()) @@ -408,7 +407,7 @@ mod tests { struct SessionStatefulValue; #[turbo_tasks::value] - struct BincodableValue(u32); + struct PersistableValue(u32); #[test] fn skip_maps_to_skip_persist() { @@ -432,12 +431,12 @@ mod tests { } #[test] - fn default_maps_to_bincodable() { - let vt = registry::get_value_type(BincodableValue::get_value_type_id()); + fn default_maps_to_persistable() { + let vt = registry::get_value_type(PersistableValue::get_value_type_id()); assert!( - matches!(vt.persistence, ValueTypePersistence::Bincodable(_, _)), - "default (auto) serialization must map to ValueTypePersistence::Bincodable" + matches!(vt.persistence, ValueTypePersistence::Persistable(_, _)), + "default (auto) serialization must map to ValueTypePersistence::Persistable" ); - assert!(BincodableValue::has_serialization()); + assert!(PersistableValue::has_serialization()); } } From af9d40e2c5e2f0b591d5fc189ecf7cabec49e239 Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Sun, 19 Apr 2026 11:24:50 -0700 Subject: [PATCH 7/9] rename inner_type --- .../src/backend/storage_schema.rs | 2 +- .../src/derive/task_storage_macro.rs | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs index bb42a72f52c5..8dc1a8fe47ba 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs @@ -298,7 +298,7 @@ struct TaskStorageSchema { storage = "auto_map", category = "data", shrink_on_completion, - inner_type = "AutoMap" + as_type = "AutoMap" )] cell_data: CellData, diff --git a/turbopack/crates/turbo-tasks-macros/src/derive/task_storage_macro.rs b/turbopack/crates/turbo-tasks-macros/src/derive/task_storage_macro.rs index eb7f44614283..efbbae9d01d6 100644 --- a/turbopack/crates/turbo-tasks-macros/src/derive/task_storage_macro.rs +++ b/turbopack/crates/turbo-tasks-macros/src/derive/task_storage_macro.rs @@ -78,7 +78,7 @@ struct FieldInfo { /// accessors (which call `.iter()`, `.insert()`, etc.) keep working. /// /// When absent, the macro parses the outer field type directly. - inner_type: Option, + as_type: Option, } impl FieldInfo { @@ -373,7 +373,7 @@ fn parse_field_storage_attributes(field: &syn::Field) -> FieldInfo { let mut use_default = false; let mut shrink_on_completion = false; let mut drop_on_completion_if_immutable = false; - let mut inner_type: Option = None; + let mut as_type: Option = None; // Find and parse the field attribute if let Some(attr) = field.attrs.iter().find(|attr| { @@ -448,16 +448,16 @@ fn parse_field_storage_attributes(field: &syn::Field) -> FieldInfo { }); } } - "inner_type" => { - if let Some(lit_str) = expect_string_literal(&nv.value, "inner_type") { + "as_type" => { + if let Some(lit_str) = expect_string_literal(&nv.value, "as_type") { match syn::parse_str::(&lit_str.value()) { - Ok(ty) => inner_type = Some(ty), + Ok(ty) => as_type = Some(ty), Err(err) => { lit_str .span() .unwrap() .error(format!( - "`inner_type` must parse as a Rust type: {err}" + "`as_type` must parse as a Rust type: {err}" )) .emit(); } @@ -469,7 +469,7 @@ fn parse_field_storage_attributes(field: &syn::Field) -> FieldInfo { .unwrap() .error(format!( "unknown attribute `{other}`, expected `storage`, `category`, \ - or `inner_type`" + or `as_type`" )) .emit(); } @@ -609,7 +609,7 @@ fn parse_field_storage_attributes(field: &syn::Field) -> FieldInfo { use_default, shrink_on_completion, drop_on_completion_if_immutable, - inner_type, + as_type, } } @@ -2331,10 +2331,10 @@ fn generate_countermap_ops(field: &FieldInfo) -> TokenStream { fn generate_automap_ops(field: &FieldInfo) -> TokenStream { let field_type = &field.field_type; - // If the field uses a newtype wrapper, `inner_type` gives us the actual + // If the field uses a newtype wrapper, `as_type` gives us the actual // `AutoMap` to extract key/value types from. Otherwise parse the // declared field type directly. - let map_ty = field.inner_type.as_ref().unwrap_or(field_type); + let map_ty = field.as_type.as_ref().unwrap_or(field_type); let Some((key_type, value_type)) = extract_map_types(map_ty, "AutoMap") else { return quote! {}; From 7e5227918f67b3fc96bfc111407d4d1f4ccfea60 Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Sun, 19 Apr 2026 16:28:30 -0700 Subject: [PATCH 8/9] reduce hash reads --- .../src/backend/operation/update_cell.rs | 92 ++++++---------- .../turbo-tasks-macros/src/value_macro.rs | 27 +++-- turbopack/crates/turbo-tasks/src/effect.rs | 6 +- .../crates/turbo-tasks/src/value_type.rs | 100 +++++++++++++++--- .../transform/swc_ecma_transform_plugins.rs | 11 +- .../crates/turbopack-node/src/evaluate.rs | 7 +- .../turbopack-node/src/process_pool/mod.rs | 7 +- .../turbopack-node/src/worker_pool/mod.rs | 7 +- 8 files changed, 147 insertions(+), 110 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs index 9c2847371090..a182c27d9e41 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs @@ -7,6 +7,7 @@ use smallvec::SmallVec; use turbo_tasks::{ CellId, FxIndexMap, TaskId, TypedSharedReference, ValueTypePersistence, backend::{CellContent, CellHash, VerificationMode}, + registry, }; #[cfg(feature = "trace_task_dirty")] @@ -58,20 +59,18 @@ impl UpdateCellOperation { #[cfg(not(feature = "verify_determinism"))] _verification_mode: VerificationMode, mut ctx: impl ExecuteContext<'_>, ) { - // content_hash is only meaningful for `serialization = "hash"` cells — - // other modes pass `None`. This invariant lets the hash-based - // invalidation-skip below fall through to `false` for any non-hash - // cell without an explicit persistable guard. + let value_type = registry::get_value_type(cell.type_id); + // `content_hash` is only ever supplied for `HashOnly` cells — only the + // `"hash"`-mode write path emits a hash, and no other mode consumes + // it. (It can still be `None` for `HashOnly` when the cell is being + // cleared.) debug_assert!( - !is_persistable(cell) || content_hash.is_none(), - "content_hash must be None for persistable cell content" + content_hash.is_none() + || matches!(value_type.persistence, ValueTypePersistence::HashOnly), + "content_hash must only be supplied for HashOnly cells" ); - let content = if let CellContent(Some(new_content)) = content { - Some(new_content) - } else { - None - }; + let content = content.0; let mut task = ctx.task(task_id, TaskDataCategory::All); @@ -96,9 +95,7 @@ impl UpdateCellOperation { && content.as_ref() != task.get_cell_data(&cell) { let task_description = task.get_task_description(); - let cell_type = turbo_tasks::registry::get_value_type(cell.type_id) - .ty - .global_name; + let cell_type = value_type.ty.global_name; eprintln!( "Task {} updated cell #{} (type: {}) while recomputing", task_description, cell.index, cell_type @@ -114,22 +111,20 @@ impl UpdateCellOperation { // When not recomputing, we need to notify dependent tasks if the content actually // changes. - // For cells without available content, use hash-based comparison - // to detect whether the value actually changed—avoiding - // unnecessary invalidation. Only `serialization = "hash"` cells - // supply a `content_hash`; for all other modes `content_hash` is - // `None` and this falls through to `false`. - let skip_invalidation = { - let has_old_content = task.cell_data_contains(&cell); - if !has_old_content { - match (content_hash, task.get_cell_data_hash(&cell)) { - (Some(new_hash), Some(old_hash)) => new_hash == *old_hash, - _ => false, + // For HashOnly cells without available content, use hash-based comparison to + // detect whether the value actually changed—avoiding unnecessary invalidation. + let skip_invalidation = + matches!(value_type.persistence, ValueTypePersistence::HashOnly) && { + let has_old_content = task.cell_data_contains(&cell); + if !has_old_content { + match (content_hash, task.get_cell_data_hash(&cell)) { + (Some(new_hash), Some(old_hash)) => new_hash == *old_hash, + _ => false, + } + } else { + false } - } else { - false - } - }; + }; #[cfg(feature = "trace_task_dirty")] let has_updated_key_hashes = updated_key_hashes.is_some(); @@ -176,7 +171,9 @@ impl UpdateCellOperation { let old_content = task.remove_cell_data(&cell); // Update cell_data_hash before dropping the task lock - update_cell_data_hash(&mut task, &cell, content_hash); + if matches!(value_type.persistence, ValueTypePersistence::HashOnly) { + update_cell_data_hash(&mut task, &cell, content_hash); + } drop(task); drop(old_content); @@ -213,8 +210,10 @@ impl UpdateCellOperation { task.remove_cell_data(&cell) }; - // Update cell_data_hash for non-persistable cells. - update_cell_data_hash(&mut task, &cell, content_hash); + // Update cell_data_hash for non-hashonly cells. + if matches!(value_type.persistence, ValueTypePersistence::HashOnly) { + update_cell_data_hash(&mut task, &cell, content_hash); + } let in_progress_cell = task.remove_in_progress_cells(&cell); @@ -234,7 +233,10 @@ impl UpdateCellOperation { match self { UpdateCellOperation::InvalidateWhenCellDependency { cell_ref, .. } | UpdateCellOperation::FinalCellChange { cell_ref, .. } => { - is_persistable(cell_ref.cell) + matches!( + registry::get_value_type(cell_ref.cell.type_id).persistence, + ValueTypePersistence::Persistable(_, _), + ) } UpdateCellOperation::AggregationUpdate { .. } => true, UpdateCellOperation::Done => true, @@ -242,34 +244,10 @@ impl UpdateCellOperation { } } -fn is_persistable(cell: CellId) -> bool { - matches!(persistence(cell), ValueTypePersistence::Persistable(_, _),) -} - -/// Returns `true` if cells of this type need a stored `cell_data_hash`. -fn needs_content_hash(cell: CellId) -> bool { - matches!(persistence(cell), ValueTypePersistence::SkipPersist,) -} - -fn persistence(cell: CellId) -> &'static ValueTypePersistence { - &turbo_tasks::registry::get_value_type(cell.type_id).persistence -} - /// Updates the stored cell_data_hash, which only `serialization = "hash"` /// cells consult (on eviction + recompute). Skips the update for all other /// persistence modes and when the hash hasn't changed. -/// -/// `Persistable` cells store content directly; `SessionStateful` cells are -/// pinned in memory. Neither ever reads a hash back, so writing one is waste. -/// The callers of this function always pass `content_hash = None` for those -/// modes (the hash-producing `hashed_compare_and_update` API is only emitted -/// by the `"hash"` macro path), so the function would behave identically -/// without the gate — keeping it for clarity and to skip a map access on the -/// hot write path. fn update_cell_data_hash(task: &mut impl TaskGuard, cell: &CellId, content_hash: Option) { - if !needs_content_hash(*cell) { - return; - } let old_hash = task.get_cell_data_hash(cell).copied(); if old_hash != content_hash { if let Some(hash) = content_hash { diff --git a/turbopack/crates/turbo-tasks-macros/src/value_macro.rs b/turbopack/crates/turbo-tasks-macros/src/value_macro.rs index 19daacc68a68..d9aa5961dc57 100644 --- a/turbopack/crates/turbo-tasks-macros/src/value_macro.rs +++ b/turbopack/crates/turbo-tasks-macros/src/value_macro.rs @@ -59,6 +59,10 @@ enum SerializationMode { /// bincoding the in-memory form. Use for outputs whose in-memory form /// (SWC ASTs, codegen Ropes, etc.) isn't worth serializing. Skip, + /// Like `Skip` (no bincode, evictable), but the cell is expensive to + /// re-derive (e.g. WASM compile, Node process spawn). Eviction policy + /// may prefer evicting cheap cells first. + SkipExpensive, Auto, Custom, } @@ -78,11 +82,13 @@ impl TryFrom for SerializationMode { "session_stateful" => Ok(SerializationMode::SessionStateful), "hash" => Ok(SerializationMode::Hash), "skip" => Ok(SerializationMode::Skip), + "skip_expensive" => Ok(SerializationMode::SkipExpensive), "auto" => Ok(SerializationMode::Auto), "custom" => Ok(SerializationMode::Custom), _ => Err(Error::new_spanned( &lit, - "expected \"session_stateful\", \"hash\", \"skip\", \"auto\", or \"custom\"", + "expected \"session_stateful\", \"hash\", \"skip\", \"skip_expensive\", \"auto\", \ + or \"custom\"", )), } } @@ -377,6 +383,7 @@ pub fn value(args: TokenStream, input: TokenStream) -> TokenStream { SerializationMode::SessionStateful | SerializationMode::Hash | SerializationMode::Skip + | SerializationMode::SkipExpensive | SerializationMode::Custom => {} }; if inner_type.is_some() { @@ -409,24 +416,28 @@ pub fn value(args: TokenStream, input: TokenStream) -> TokenStream { let name = global_name_for_type(ident); // Dispatch to the constructor whose name reflects the persistence mode. - // `Hash` and `Skip` both map to `skip_persist` — hash-mode change detection - // is handled independently by the caller supplying a `content_hash`, not by - // a distinct persistence variant. let new_value_type = match serialization_mode { SerializationMode::SessionStateful => quote! { turbo_tasks::ValueType::session_stateful::<#ident>(#name) }, - SerializationMode::Hash | SerializationMode::Skip => quote! { + SerializationMode::Skip => quote! { turbo_tasks::ValueType::skip_persist::<#ident>(#name) }, + SerializationMode::Hash => quote! { + turbo_tasks::ValueType::hash_only::<#ident>(#name) + }, + SerializationMode::SkipExpensive => quote! { + turbo_tasks::ValueType::skip_persist_expensive::<#ident>(#name) + }, SerializationMode::Auto | SerializationMode::Custom => quote! { turbo_tasks::ValueType::persistable::<#ident>(#name) }, }; let has_serialization = match serialization_mode { - SerializationMode::SessionStateful | SerializationMode::Hash | SerializationMode::Skip => { - quote! { false } - } + SerializationMode::SessionStateful + | SerializationMode::Hash + | SerializationMode::Skip + | SerializationMode::SkipExpensive => quote! { false }, SerializationMode::Auto | SerializationMode::Custom => quote! { true }, }; diff --git a/turbopack/crates/turbo-tasks/src/effect.rs b/turbopack/crates/turbo-tasks/src/effect.rs index b2a21574cd57..9e0ae0c6e063 100644 --- a/turbopack/crates/turbo-tasks/src/effect.rs +++ b/turbopack/crates/turbo-tasks/src/effect.rs @@ -143,7 +143,7 @@ type DynEffectApplyFuture<'a> = Pin> + Send + trait EffectCollectible {} /// The Effect instance collectible that is emitted for effects. -#[turbo_tasks::value(serialization = "session_stateful", cell = "new", eq = "manual")] +#[turbo_tasks::value(serialization = "skip_expensive", cell = "new", eq = "manual")] struct EffectInstance { #[turbo_tasks(debug_ignore)] inner: Box, @@ -199,7 +199,7 @@ pub fn emit_effect(effect: impl Effect) { /// # #[turbo_tasks::function(operation)] /// # fn some_turbo_tasks_operation(_args: Args) {} /// # -/// #[turbo_tasks::value(serialization = "session_stateful")] +/// #[turbo_tasks::value(serialization = "skip_expensive")] /// struct OutputWithEffects { /// output: ReadRef, /// effects: Effects, @@ -255,7 +255,7 @@ type UniqueEffectIndices = Result)>, String>; /// Captured effects from an operation. This struct can be used to return Effects from a turbo-tasks /// function and apply them later. #[derive(Default)] -#[turbo_tasks::value(shared, eq = "manual", serialization = "session_stateful")] +#[turbo_tasks::value(shared, eq = "manual", serialization = "skip_expensive")] pub struct Effects { #[turbo_tasks(debug_ignore)] effects: Vec>, diff --git a/turbopack/crates/turbo-tasks/src/value_type.rs b/turbopack/crates/turbo-tasks/src/value_type.rs index 4e44e16d1268..e085da0ccb5d 100644 --- a/turbopack/crates/turbo-tasks/src/value_type.rs +++ b/turbopack/crates/turbo-tasks/src/value_type.rs @@ -37,18 +37,28 @@ pub enum ValueTypePersistence { /// Cells are serialized to the persistent cache and restored on next /// access after eviction. Maps to `serialization = "auto" | "custom"`. Persistable(AnyEncodeFn, AnyDecodeFn), - /// The value type opts out of being persisted because re-running the - /// producing task to reproduce the cell is cheaper/simpler than - /// serializing the in-memory form. Cells are evictable; the next reader - /// after eviction triggers a recompute from the task's inputs. Maps to - /// `serialization = "skip" | "hash"`. The hash-based change detection - /// for `"hash"` is handled by the caller supplying a `content_hash`, not - /// by a distinct persistence variant. - SkipPersist, - /// Not persistable, not reconstructible — holds session-scoped state - /// (file system handles, worker pools, plugin DSOs, `State<>` interior - /// mutability). Cells of this type must stay in memory across eviction. - /// Maps to `serialization = "session_stateful"`. + /// The value type opts out of being persisted: re-running the producing + /// task to reproduce the cell is preferred over serializing the in-memory + /// form. Cells are evictable; the next reader after eviction triggers a + /// recompute from the task's inputs. Maps to + /// `serialization = "skip" | "skip_expensive"`. + SkipPersist { + /// Whether re-deriving this cell is non-trivial (e.g. WASM compile, + /// spawning a Node process pool). Eviction policy may prefer + /// evicting cheap cells first. True iff declared with + /// `serialization = "skip_expensive"`. + expensive: bool, + }, + /// The value type is not persisted, but the macro emitted a + /// `DeterministicHash` derive and the write path stashes a `content_hash` + /// into `cell_data_hash` so post-eviction reads can detect unchanged + /// content and skip invalidation. Maps to `serialization = "hash"`. + HashOnly, + /// Not persistable, not reconstructible — holds interior-mutable state + /// that accumulates across the session (`State<>` cells, `Arc>` + /// dedup histories). Re-running the producing task would lose the + /// accumulated state, so cells of this type must stay in memory across + /// eviction. Maps to `serialization = "session_stateful"`. SessionStateful, } @@ -115,9 +125,35 @@ impl ValueType { /// the task's inputs. /// /// This is internally used by [`#[turbo_tasks::value]`][crate::value] for - /// `serialization = "skip"` and `serialization = "hash"`. + /// `serialization = "skip"`. pub const fn skip_persist(global_name: &'static str) -> Self { - Self::new_inner::(global_name, ValueTypePersistence::SkipPersist) + Self::new_inner::( + global_name, + ValueTypePersistence::SkipPersist { expensive: false }, + ) + } + + /// Construct a `ValueType` that opts out of being persisted and is marked + /// as expensive to re-derive (e.g. WASM compile, Node process spawn). The + /// eviction policy may prefer evicting cheaper cells first. + /// + /// This is internally used by [`#[turbo_tasks::value]`][crate::value] for + /// `serialization = "skip_expensive"`. + pub const fn skip_persist_expensive(global_name: &'static str) -> Self { + Self::new_inner::( + global_name, + ValueTypePersistence::SkipPersist { expensive: true }, + ) + } + + /// Construct a `ValueType` that opts out of being persisted but stashes a + /// `content_hash` on each write so post-eviction reads can detect + /// unchanged content and skip invalidation. + /// + /// This is internally used by [`#[turbo_tasks::value]`][crate::value] for + /// `serialization = "hash"`. + pub const fn hash_only(global_name: &'static str) -> Self { + Self::new_inner::(global_name, ValueTypePersistence::HashOnly) } /// Construct a `ValueType` whose cells cannot be reconstructed by @@ -403,6 +439,12 @@ mod tests { #[turbo_tasks::value(serialization = "skip")] struct SkipValue(#[turbo_tasks(trace_ignore)] u32); + #[turbo_tasks::value(serialization = "hash")] + struct HashValue(u32); + + #[turbo_tasks::value(serialization = "skip_expensive")] + struct SkipExpensiveValue(#[turbo_tasks(trace_ignore)] u32); + #[turbo_tasks::value(serialization = "session_stateful", cell = "new", eq = "manual")] struct SessionStatefulValue; @@ -413,12 +455,38 @@ mod tests { fn skip_maps_to_skip_persist() { let vt = registry::get_value_type(SkipValue::get_value_type_id()); assert!( - matches!(vt.persistence, ValueTypePersistence::SkipPersist), - "`serialization = \"skip\"` must map to ValueTypePersistence::SkipPersist" + matches!( + vt.persistence, + ValueTypePersistence::SkipPersist { expensive: false }, + ), + "`serialization = \"skip\"` must map to SkipPersist {{ expensive: false }}" ); assert!(!SkipValue::has_serialization()); } + #[test] + fn hash_maps_to_hash_only() { + let vt = registry::get_value_type(HashValue::get_value_type_id()); + assert!( + matches!(vt.persistence, ValueTypePersistence::HashOnly), + "`serialization = \"hash\"` must map to HashOnly" + ); + assert!(!HashValue::has_serialization()); + } + + #[test] + fn skip_expensive_maps_to_skip_persist_expensive() { + let vt = registry::get_value_type(SkipExpensiveValue::get_value_type_id()); + assert!( + matches!( + vt.persistence, + ValueTypePersistence::SkipPersist { expensive: true }, + ), + "`serialization = \"skip_expensive\"` must map to SkipPersist {{ expensive: true }}" + ); + assert!(!SkipExpensiveValue::has_serialization()); + } + #[test] fn session_stateful_maps_to_session_stateful() { let vt = registry::get_value_type(SessionStatefulValue::get_value_type_id()); diff --git a/turbopack/crates/turbopack-ecmascript-plugins/src/transform/swc_ecma_transform_plugins.rs b/turbopack/crates/turbopack-ecmascript-plugins/src/transform/swc_ecma_transform_plugins.rs index 3f7be884dcfe..5d57d8e695f6 100644 --- a/turbopack/crates/turbopack-ecmascript-plugins/src/transform/swc_ecma_transform_plugins.rs +++ b/turbopack/crates/turbopack-ecmascript-plugins/src/transform/swc_ecma_transform_plugins.rs @@ -17,14 +17,9 @@ use turbopack_ecmascript::{CustomTransformer, TransformContext}; /// compiled, serialized WASM module instead of raw file bytes to reduce the /// cost of the compilation. /// -/// This is tagged as `session_stateful` to avoid evicting compiled modules from RAM on the theory -/// that there simply are not very many of them -#[turbo_tasks::value( - serialization = "session_stateful", - eq = "manual", - cell = "new", - shared -)] +/// Tagged `skip_expensive` so eviction prefers evicting cheaper cells first — +/// re-deriving a compiled module is pure but pays a non-trivial WASM compile. +#[turbo_tasks::value(serialization = "skip_expensive", eq = "manual", cell = "new", shared)] pub struct SwcPluginModule { pub name: RcStr, #[turbo_tasks(trace_ignore, debug_ignore)] diff --git a/turbopack/crates/turbopack-node/src/evaluate.rs b/turbopack/crates/turbopack-node/src/evaluate.rs index cafb38424772..f4cf737f2c64 100644 --- a/turbopack/crates/turbopack-node/src/evaluate.rs +++ b/turbopack/crates/turbopack-node/src/evaluate.rs @@ -67,12 +67,7 @@ enum EvalJavaScriptIncomingMessage { Error(StructuredError), } -#[turbo_tasks::value( - cell = "new", - serialization = "session_stateful", - eq = "manual", - shared -)] +#[turbo_tasks::value(cell = "new", serialization = "skip_expensive", eq = "manual", shared)] pub struct EvaluatePool { #[turbo_tasks(trace_ignore, debug_ignore)] pool: Box, diff --git a/turbopack/crates/turbopack-node/src/process_pool/mod.rs b/turbopack/crates/turbopack-node/src/process_pool/mod.rs index 5f3eaed2accc..9a459408c22f 100644 --- a/turbopack/crates/turbopack-node/src/process_pool/mod.rs +++ b/turbopack/crates/turbopack-node/src/process_pool/mod.rs @@ -531,12 +531,7 @@ impl ProcessArgs { /// /// The worker will *not* use the `env` of the parent process by default. All environment variables /// need to be provided to make the execution as pure as possible. -#[turbo_tasks::value( - cell = "new", - serialization = "session_stateful", - eq = "manual", - shared -)] +#[turbo_tasks::value(cell = "new", serialization = "skip_expensive", eq = "manual", shared)] pub struct ChildProcessPool { cwd: PathBuf, entrypoint: PathBuf, diff --git a/turbopack/crates/turbopack-node/src/worker_pool/mod.rs b/turbopack/crates/turbopack-node/src/worker_pool/mod.rs index 0fe97c488659..ebee35a4f735 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/mod.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/mod.rs @@ -36,12 +36,7 @@ mod worker_thread; static OPERATION_TASK_ID: AtomicU32 = AtomicU32::new(1); -#[turbo_tasks::value( - cell = "new", - serialization = "session_stateful", - eq = "manual", - shared -)] +#[turbo_tasks::value(cell = "new", serialization = "skip_expensive", eq = "manual", shared)] pub(crate) struct WorkerThreadPool { worker_options: Arc, concurrency: usize, From a23b6b4d133665f8677d301fc0de8c7c250629a9 Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Mon, 20 Apr 2026 14:01:08 -0700 Subject: [PATCH 9/9] simplify terminology --- .../tests/read_ref_cell.rs | 2 +- .../tests/trait_ref_cell.rs | 2 +- .../turbo-tasks-macros/src/value_macro.rs | 147 ++++++++++++------ turbopack/crates/turbo-tasks/src/effect.rs | 6 +- .../crates/turbo-tasks/src/value_type.rs | 19 +-- .../crates/turbopack-cli-utils/src/issue.rs | 2 +- .../crates/turbopack-core/src/version.rs | 2 +- .../transform/swc_ecma_transform_plugins.rs | 10 +- .../crates/turbopack-node/src/evaluate.rs | 8 +- .../turbopack-node/src/process_pool/mod.rs | 8 +- .../turbopack-node/src/worker_pool/mod.rs | 8 +- .../crates/turbopack-tests/tests/execution.rs | 2 +- .../tests/node-file-trace.rs | 2 +- 13 files changed, 148 insertions(+), 70 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/tests/read_ref_cell.rs b/turbopack/crates/turbo-tasks-backend/tests/read_ref_cell.rs index ea92638c61ed..cdbcbbdd6841 100644 --- a/turbopack/crates/turbo-tasks-backend/tests/read_ref_cell.rs +++ b/turbopack/crates/turbo-tasks-backend/tests/read_ref_cell.rs @@ -56,7 +56,7 @@ async fn test_read_ref() { #[turbo_tasks::value(transparent)] struct CounterValue(usize); -#[turbo_tasks::value(serialization = "session_stateful", cell = "new", eq = "manual")] +#[turbo_tasks::value(serialization = "skip", evict = "never", cell = "new", eq = "manual")] struct Counter { #[turbo_tasks(debug_ignore, trace_ignore)] value: Mutex<(usize, HashSet)>, diff --git a/turbopack/crates/turbo-tasks-backend/tests/trait_ref_cell.rs b/turbopack/crates/turbo-tasks-backend/tests/trait_ref_cell.rs index 0c0303dfb66d..b5df9862c6fd 100644 --- a/turbopack/crates/turbo-tasks-backend/tests/trait_ref_cell.rs +++ b/turbopack/crates/turbo-tasks-backend/tests/trait_ref_cell.rs @@ -65,7 +65,7 @@ async fn trait_ref() { #[derive(Copy, Clone)] struct CounterValue(usize); -#[turbo_tasks::value(serialization = "session_stateful", cell = "new", eq = "manual")] +#[turbo_tasks::value(serialization = "skip", evict = "never", cell = "new", eq = "manual")] struct Counter { #[turbo_tasks(debug_ignore, trace_ignore)] value: Mutex<(usize, HashSet)>, diff --git a/turbopack/crates/turbo-tasks-macros/src/value_macro.rs b/turbopack/crates/turbo-tasks-macros/src/value_macro.rs index d9aa5961dc57..ef0892efad09 100644 --- a/turbopack/crates/turbo-tasks-macros/src/value_macro.rs +++ b/turbopack/crates/turbo-tasks-macros/src/value_macro.rs @@ -43,28 +43,20 @@ impl TryFrom for CellMode { } } +/// How a value type's cells are persisted across restarts. enum SerializationMode { - /// No bincode: cells of this type hold session-scoped state - /// (file system handles, worker pool handles, plugin DSOs, `State<>` - /// interior mutability, etc.) and cannot be reconstructed by re-executing - /// the producing task. The storage layer keeps them in memory across - /// eviction. - SessionStateful, - /// Like `SessionStateful` (no bincode serialization), but also stores a hash of the cell value - /// so that changes can be detected even when the cell data has been evicted from memory. - /// Only valid with `cell = "compare"` (or the default). - Hash, - /// No bincode, **but evictable** — skip persisting the cell content because - /// re-running the producing task to reproduce it is cheaper/simpler than - /// bincoding the in-memory form. Use for outputs whose in-memory form - /// (SWC ASTs, codegen Ropes, etc.) isn't worth serializing. - Skip, - /// Like `Skip` (no bincode, evictable), but the cell is expensive to - /// re-derive (e.g. WASM compile, Node process spawn). Eviction policy - /// may prefer evicting cheap cells first. - SkipExpensive, + /// Round-trip through bincode via auto-derived `Encode` / `Decode`. Auto, + /// Round-trip through bincode via a manual `Encode` / `Decode` impl + /// supplied by the value type. Custom, + /// No persistence of the value itself. Eviction policy is controlled + /// separately via the `evict` attribute. + Skip, + /// Persist only a hash of the value so post-eviction reads can detect + /// unchanged content and skip invalidation. Only valid with + /// `cell = "compare"` (or the default). + Hash, } impl Parse for SerializationMode { @@ -79,16 +71,53 @@ impl TryFrom for SerializationMode { fn try_from(lit: LitStr) -> Result { match lit.value().as_str() { - "session_stateful" => Ok(SerializationMode::SessionStateful), - "hash" => Ok(SerializationMode::Hash), - "skip" => Ok(SerializationMode::Skip), - "skip_expensive" => Ok(SerializationMode::SkipExpensive), "auto" => Ok(SerializationMode::Auto), "custom" => Ok(SerializationMode::Custom), + "skip" => Ok(SerializationMode::Skip), + "hash" => Ok(SerializationMode::Hash), + _ => Err(Error::new_spanned( + &lit, + "expected \"auto\", \"custom\", \"skip\", or \"hash\"", + )), + } + } +} + +/// Eviction policy for a `serialization = "skip"` value type. Ignored for +/// other serialization modes (the macro rejects non-`Always` values in that +/// case). +enum EvictMode { + /// Evictable freely. The next reader after eviction triggers a recompute + /// from the task's inputs. This is the default when `evict` is omitted. + Always, + /// Evictable, but re-deriving is non-trivial (e.g. WASM compile, + /// spawning a Node process pool). Eviction policy should prefer + /// evicting cheaper cells first. + Last, + /// Not evictable: the value holds interior-mutable state that + /// accumulates across the session (`State<>` cells, `Arc>` + /// dedup histories) and must stay in memory. + Never, +} + +impl Parse for EvictMode { + fn parse(input: ParseStream) -> syn::Result { + let ident = input.parse::()?; + Self::try_from(ident) + } +} + +impl TryFrom for EvictMode { + type Error = Error; + + fn try_from(lit: LitStr) -> Result { + match lit.value().as_str() { + "always" => Ok(EvictMode::Always), + "last" => Ok(EvictMode::Last), + "never" => Ok(EvictMode::Never), _ => Err(Error::new_spanned( &lit, - "expected \"session_stateful\", \"hash\", \"skip\", \"skip_expensive\", \"auto\", \ - or \"custom\"", + "expected \"always\", \"last\", or \"never\"", )), } } @@ -96,6 +125,7 @@ impl TryFrom for SerializationMode { struct ValueArguments { serialization_mode: SerializationMode, + evict_mode: EvictMode, shared: bool, cell_mode: CellMode, manual_eq: bool, @@ -109,6 +139,7 @@ impl Parse for ValueArguments { fn parse(input: ParseStream) -> syn::Result { let mut result = ValueArguments { serialization_mode: SerializationMode::Auto, + evict_mode: EvictMode::Always, shared: false, cell_mode: CellMode::Compare, manual_eq: false, @@ -141,6 +172,18 @@ impl Parse for ValueArguments { ) => { result.serialization_mode = SerializationMode::try_from(str)?; } + ( + "evict", + Meta::NameValue(MetaNameValue { + value: + Expr::Lit(ExprLit { + lit: Lit::Str(str), .. + }), + .. + }), + ) => { + result.evict_mode = EvictMode::try_from(str)?; + } ( "cell", Meta::NameValue(MetaNameValue { @@ -196,8 +239,8 @@ impl Parse for ValueArguments { &meta, format!( "unexpected {meta:?}, expected \"shared\", \"into\", \ - \"serialization\", \"cell\", \"eq\", \"hash\", \"transparent\", or \ - \"operation\"" + \"serialization\", \"evict\", \"cell\", \"eq\", \"hash\", \ + \"transparent\", or \"operation\"" ), )); } @@ -212,6 +255,7 @@ pub fn value(args: TokenStream, input: TokenStream) -> TokenStream { let item = parse_macro_input!(input as Item); let ValueArguments { serialization_mode, + evict_mode, shared, cell_mode, manual_eq, @@ -242,6 +286,20 @@ pub fn value(args: TokenStream, input: TokenStream) -> TokenStream { .into(); } + // `evict = "last" | "never"` is only valid when `serialization = "skip"`; + // other persistence modes have their own eviction semantics fixed by the + // backend (Persistable: evict-and-restore, HashOnly: evict-with-hash-gate). + if !matches!(evict_mode, EvictMode::Always) + && !matches!(serialization_mode, SerializationMode::Skip) + { + return syn::Error::new( + proc_macro2::Span::call_site(), + "evict = \"last\" | \"never\" is only valid with serialization = \"skip\"", + ) + .to_compile_error() + .into(); + } + let mut struct_attributes = vec![quote! { #[derive( turbo_tasks::ShrinkToFit, @@ -380,11 +438,7 @@ pub fn value(args: TokenStream, input: TokenStream) -> TokenStream { #[bincode(crate = "turbo_tasks::macro_helpers::bincode")] }); } - SerializationMode::SessionStateful - | SerializationMode::Hash - | SerializationMode::Skip - | SerializationMode::SkipExpensive - | SerializationMode::Custom => {} + SerializationMode::Custom | SerializationMode::Skip | SerializationMode::Hash => {} }; if inner_type.is_some() { // Transparent structs have their own manual `ValueDebug` implementation. @@ -415,29 +469,28 @@ pub fn value(args: TokenStream, input: TokenStream) -> TokenStream { } let name = global_name_for_type(ident); - // Dispatch to the constructor whose name reflects the persistence mode. - let new_value_type = match serialization_mode { - SerializationMode::SessionStateful => quote! { - turbo_tasks::ValueType::session_stateful::<#ident>(#name) - }, - SerializationMode::Skip => quote! { - turbo_tasks::ValueType::skip_persist::<#ident>(#name) + // Dispatch to the constructor whose name reflects the persistence + + // eviction combo. `evict` is only read when `serialization = Skip`; + // other modes ignore it (and the parser rejects non-Always values). + let new_value_type = match (&serialization_mode, &evict_mode) { + (SerializationMode::Auto | SerializationMode::Custom, _) => quote! { + turbo_tasks::ValueType::persistable::<#ident>(#name) }, - SerializationMode::Hash => quote! { + (SerializationMode::Hash, _) => quote! { turbo_tasks::ValueType::hash_only::<#ident>(#name) }, - SerializationMode::SkipExpensive => quote! { + (SerializationMode::Skip, EvictMode::Always) => quote! { + turbo_tasks::ValueType::skip_persist::<#ident>(#name) + }, + (SerializationMode::Skip, EvictMode::Last) => quote! { turbo_tasks::ValueType::skip_persist_expensive::<#ident>(#name) }, - SerializationMode::Auto | SerializationMode::Custom => quote! { - turbo_tasks::ValueType::persistable::<#ident>(#name) + (SerializationMode::Skip, EvictMode::Never) => quote! { + turbo_tasks::ValueType::session_stateful::<#ident>(#name) }, }; let has_serialization = match serialization_mode { - SerializationMode::SessionStateful - | SerializationMode::Hash - | SerializationMode::Skip - | SerializationMode::SkipExpensive => quote! { false }, + SerializationMode::Skip | SerializationMode::Hash => quote! { false }, SerializationMode::Auto | SerializationMode::Custom => quote! { true }, }; diff --git a/turbopack/crates/turbo-tasks/src/effect.rs b/turbopack/crates/turbo-tasks/src/effect.rs index 9e0ae0c6e063..dfe3b57eff48 100644 --- a/turbopack/crates/turbo-tasks/src/effect.rs +++ b/turbopack/crates/turbo-tasks/src/effect.rs @@ -143,7 +143,7 @@ type DynEffectApplyFuture<'a> = Pin> + Send + trait EffectCollectible {} /// The Effect instance collectible that is emitted for effects. -#[turbo_tasks::value(serialization = "skip_expensive", cell = "new", eq = "manual")] +#[turbo_tasks::value(serialization = "skip", evict = "last", cell = "new", eq = "manual")] struct EffectInstance { #[turbo_tasks(debug_ignore)] inner: Box, @@ -199,7 +199,7 @@ pub fn emit_effect(effect: impl Effect) { /// # #[turbo_tasks::function(operation)] /// # fn some_turbo_tasks_operation(_args: Args) {} /// # -/// #[turbo_tasks::value(serialization = "skip_expensive")] +/// #[turbo_tasks::value(serialization = "skip", evict = "last")] /// struct OutputWithEffects { /// output: ReadRef, /// effects: Effects, @@ -255,7 +255,7 @@ type UniqueEffectIndices = Result)>, String>; /// Captured effects from an operation. This struct can be used to return Effects from a turbo-tasks /// function and apply them later. #[derive(Default)] -#[turbo_tasks::value(shared, eq = "manual", serialization = "skip_expensive")] +#[turbo_tasks::value(shared, eq = "manual", serialization = "skip", evict = "last")] pub struct Effects { #[turbo_tasks(debug_ignore)] effects: Vec>, diff --git a/turbopack/crates/turbo-tasks/src/value_type.rs b/turbopack/crates/turbo-tasks/src/value_type.rs index e085da0ccb5d..9be43b3b020c 100644 --- a/turbopack/crates/turbo-tasks/src/value_type.rs +++ b/turbopack/crates/turbo-tasks/src/value_type.rs @@ -41,12 +41,12 @@ pub enum ValueTypePersistence { /// task to reproduce the cell is preferred over serializing the in-memory /// form. Cells are evictable; the next reader after eviction triggers a /// recompute from the task's inputs. Maps to - /// `serialization = "skip" | "skip_expensive"`. + /// `serialization = "skip"` (plus an optional `evict` attribute). SkipPersist { /// Whether re-deriving this cell is non-trivial (e.g. WASM compile, /// spawning a Node process pool). Eviction policy may prefer /// evicting cheap cells first. True iff declared with - /// `serialization = "skip_expensive"`. + /// `serialization = "skip", evict = "last"`. expensive: bool, }, /// The value type is not persisted, but the macro emitted a @@ -58,7 +58,7 @@ pub enum ValueTypePersistence { /// that accumulates across the session (`State<>` cells, `Arc>` /// dedup histories). Re-running the producing task would lose the /// accumulated state, so cells of this type must stay in memory across - /// eviction. Maps to `serialization = "session_stateful"`. + /// eviction. Maps to `serialization = "skip", evict = "never"`. SessionStateful, } @@ -138,7 +138,7 @@ impl ValueType { /// eviction policy may prefer evicting cheaper cells first. /// /// This is internally used by [`#[turbo_tasks::value]`][crate::value] for - /// `serialization = "skip_expensive"`. + /// `serialization = "skip", evict = "last"`. pub const fn skip_persist_expensive(global_name: &'static str) -> Self { Self::new_inner::( global_name, @@ -162,7 +162,7 @@ impl ValueType { /// The storage layer must keep them in memory across eviction. /// /// This is internally used by [`#[turbo_tasks::value]`][crate::value] for - /// `serialization = "session_stateful"`. + /// `serialization = "skip", evict = "never"`. pub const fn session_stateful(global_name: &'static str) -> Self { Self::new_inner::(global_name, ValueTypePersistence::SessionStateful) } @@ -442,10 +442,10 @@ mod tests { #[turbo_tasks::value(serialization = "hash")] struct HashValue(u32); - #[turbo_tasks::value(serialization = "skip_expensive")] + #[turbo_tasks::value(serialization = "skip", evict = "last")] struct SkipExpensiveValue(#[turbo_tasks(trace_ignore)] u32); - #[turbo_tasks::value(serialization = "session_stateful", cell = "new", eq = "manual")] + #[turbo_tasks::value(serialization = "skip", evict = "never", cell = "new", eq = "manual")] struct SessionStatefulValue; #[turbo_tasks::value] @@ -482,7 +482,8 @@ mod tests { vt.persistence, ValueTypePersistence::SkipPersist { expensive: true }, ), - "`serialization = \"skip_expensive\"` must map to SkipPersist {{ expensive: true }}" + "`serialization = \"skip\", evict = \"last\"` must map to SkipPersist {{ expensive: \ + true }}" ); assert!(!SkipExpensiveValue::has_serialization()); } @@ -492,7 +493,7 @@ mod tests { let vt = registry::get_value_type(SessionStatefulValue::get_value_type_id()); assert!( matches!(vt.persistence, ValueTypePersistence::SessionStateful), - "`serialization = \"session_stateful\"` must map to \ + "`serialization = \"skip\", evict = \"never\"` must map to \ ValueTypePersistence::SessionStateful" ); assert!(!SessionStatefulValue::has_serialization()); diff --git a/turbopack/crates/turbopack-cli-utils/src/issue.rs b/turbopack/crates/turbopack-cli-utils/src/issue.rs index 01d47668bec6..233d6a556268 100644 --- a/turbopack/crates/turbopack-cli-utils/src/issue.rs +++ b/turbopack/crates/turbopack-cli-utils/src/issue.rs @@ -349,7 +349,7 @@ impl SeenIssues { /// /// The ConsoleUi can be shared and capture issues from multiple sources, with deduplication /// operating across all issues. -#[turbo_tasks::value(shared, serialization = "session_stateful", eq = "manual")] +#[turbo_tasks::value(shared, serialization = "skip", evict = "never", eq = "manual")] #[derive(Clone)] pub struct ConsoleUi { options: LogOptions, diff --git a/turbopack/crates/turbopack-core/src/version.rs b/turbopack/crates/turbopack-core/src/version.rs index b637e0739e18..bb1d422adc9c 100644 --- a/turbopack/crates/turbopack-core/src/version.rs +++ b/turbopack/crates/turbopack-core/src/version.rs @@ -260,7 +260,7 @@ struct VersionRef( #[turbo_tasks(trace_ignore)] TraitRef>, ); -#[turbo_tasks::value(serialization = "session_stateful")] +#[turbo_tasks::value(serialization = "skip", evict = "never")] pub struct VersionState { version: State, } diff --git a/turbopack/crates/turbopack-ecmascript-plugins/src/transform/swc_ecma_transform_plugins.rs b/turbopack/crates/turbopack-ecmascript-plugins/src/transform/swc_ecma_transform_plugins.rs index 5d57d8e695f6..bc4e9ee94e63 100644 --- a/turbopack/crates/turbopack-ecmascript-plugins/src/transform/swc_ecma_transform_plugins.rs +++ b/turbopack/crates/turbopack-ecmascript-plugins/src/transform/swc_ecma_transform_plugins.rs @@ -17,9 +17,15 @@ use turbopack_ecmascript::{CustomTransformer, TransformContext}; /// compiled, serialized WASM module instead of raw file bytes to reduce the /// cost of the compilation. /// -/// Tagged `skip_expensive` so eviction prefers evicting cheaper cells first — +/// Tagged `evict = "last"` so eviction prefers evicting cheaper cells first — /// re-deriving a compiled module is pure but pays a non-trivial WASM compile. -#[turbo_tasks::value(serialization = "skip_expensive", eq = "manual", cell = "new", shared)] +#[turbo_tasks::value( + serialization = "skip", + evict = "last", + eq = "manual", + cell = "new", + shared +)] pub struct SwcPluginModule { pub name: RcStr, #[turbo_tasks(trace_ignore, debug_ignore)] diff --git a/turbopack/crates/turbopack-node/src/evaluate.rs b/turbopack/crates/turbopack-node/src/evaluate.rs index f4cf737f2c64..1728899d5161 100644 --- a/turbopack/crates/turbopack-node/src/evaluate.rs +++ b/turbopack/crates/turbopack-node/src/evaluate.rs @@ -67,7 +67,13 @@ enum EvalJavaScriptIncomingMessage { Error(StructuredError), } -#[turbo_tasks::value(cell = "new", serialization = "skip_expensive", eq = "manual", shared)] +#[turbo_tasks::value( + cell = "new", + serialization = "skip", + evict = "last", + eq = "manual", + shared +)] pub struct EvaluatePool { #[turbo_tasks(trace_ignore, debug_ignore)] pool: Box, diff --git a/turbopack/crates/turbopack-node/src/process_pool/mod.rs b/turbopack/crates/turbopack-node/src/process_pool/mod.rs index 9a459408c22f..753e2a13a7b5 100644 --- a/turbopack/crates/turbopack-node/src/process_pool/mod.rs +++ b/turbopack/crates/turbopack-node/src/process_pool/mod.rs @@ -531,7 +531,13 @@ impl ProcessArgs { /// /// The worker will *not* use the `env` of the parent process by default. All environment variables /// need to be provided to make the execution as pure as possible. -#[turbo_tasks::value(cell = "new", serialization = "skip_expensive", eq = "manual", shared)] +#[turbo_tasks::value( + cell = "new", + serialization = "skip", + evict = "last", + eq = "manual", + shared +)] pub struct ChildProcessPool { cwd: PathBuf, entrypoint: PathBuf, diff --git a/turbopack/crates/turbopack-node/src/worker_pool/mod.rs b/turbopack/crates/turbopack-node/src/worker_pool/mod.rs index ebee35a4f735..27e2788b6762 100644 --- a/turbopack/crates/turbopack-node/src/worker_pool/mod.rs +++ b/turbopack/crates/turbopack-node/src/worker_pool/mod.rs @@ -36,7 +36,13 @@ mod worker_thread; static OPERATION_TASK_ID: AtomicU32 = AtomicU32::new(1); -#[turbo_tasks::value(cell = "new", serialization = "skip_expensive", eq = "manual", shared)] +#[turbo_tasks::value( + cell = "new", + serialization = "skip", + evict = "last", + eq = "manual", + shared +)] pub(crate) struct WorkerThreadPool { worker_options: Arc, concurrency: usize, diff --git a/turbopack/crates/turbopack-tests/tests/execution.rs b/turbopack/crates/turbopack-tests/tests/execution.rs index d04b96106dd5..a63ab48c95f5 100644 --- a/turbopack/crates/turbopack-tests/tests/execution.rs +++ b/turbopack/crates/turbopack-tests/tests/execution.rs @@ -213,7 +213,7 @@ async fn run(resource: PathBuf, snapshot_mode: IssueSnapshotMode) -> Result, effects: Effects, diff --git a/turbopack/crates/turbopack-tracing/tests/node-file-trace.rs b/turbopack/crates/turbopack-tracing/tests/node-file-trace.rs index 79d37f3dd3bb..71bddfd304f2 100644 --- a/turbopack/crates/turbopack-tracing/tests/node-file-trace.rs +++ b/turbopack/crates/turbopack-tracing/tests/node-file-trace.rs @@ -346,7 +346,7 @@ fn bench_against_node_nft_inner(input: CaseInput) { }); } -#[turbo_tasks::value(serialization = "session_stateful")] +#[turbo_tasks::value(serialization = "skip", evict = "never")] struct NodeFileTraceResult { rebased: ResolvedVc, effects: Effects,