From 9e97759ab8fc876dca1fbfc3e9746d34d113b0d8 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 23 Jan 2026 09:02:10 +0100 Subject: [PATCH 01/10] Turbopack: refactor data storage to avoid reverse task cache (#88492) ### What? * task type is stored as CachedDataItem * reverse task cache is removed --- .../turbo-tasks-backend/src/backend/mod.rs | 420 +++++++++--------- .../backend/operation/aggregation_update.rs | 72 ++- .../src/backend/operation/connect_child.rs | 60 ++- .../src/backend/operation/invalidate.rs | 63 ++- .../src/backend/operation/mod.rs | 141 ++++-- .../src/backend/operation/update_cell.rs | 26 +- .../src/backend/storage_schema.rs | 105 +++-- .../src/backing_storage.rs | 27 -- .../crates/turbo-tasks-backend/src/data.rs | 100 +++-- .../src/database/by_key_space.rs | 15 +- .../src/database/key_value_database.rs | 3 +- .../src/database/lmdb/mod.rs | 19 +- .../src/database/startup_cache.rs | 9 +- .../src/database/turbo/mod.rs | 2 +- .../src/kv_backing_storage.rs | 53 +-- .../src/utils/arc_or_owned.rs | 87 ++++ .../turbo-tasks-backend/src/utils/bi_map.rs | 68 --- .../src/utils/dash_map_raw_entry.rs | 129 ++++++ .../turbo-tasks-backend/src/utils/mod.rs | 3 +- .../src/derive/task_storage_macro.rs | 52 ++- turbopack/crates/turbo-tasks/src/backend.rs | 2 - turbopack/crates/turbo-tasks/src/event.rs | 68 ++- 22 files changed, 906 insertions(+), 618 deletions(-) create mode 100644 turbopack/crates/turbo-tasks-backend/src/utils/arc_or_owned.rs delete mode 100644 turbopack/crates/turbo-tasks-backend/src/utils/bi_map.rs create mode 100644 turbopack/crates/turbo-tasks-backend/src/utils/dash_map_raw_entry.rs diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index bc7af478d89526..cedd4969b51d55 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -29,10 +29,10 @@ use turbo_tasks::{ ReadOutputOptions, ReadTracking, SharedReference, TRANSIENT_TASK_BIT, TaskExecutionReason, TaskId, TaskPriority, TraitTypeId, TurboTasksBackendApi, ValueTypeId, backend::{ - Backend, CachedTaskType, CellContent, TaskExecutionSpec, TransientTaskRoot, - TransientTaskType, TurboTasksExecutionError, TypedCellContent, VerificationMode, + Backend, CachedTaskType, CellContent, TaskExecutionSpec, TransientTaskType, + TurboTasksExecutionError, TypedCellContent, VerificationMode, }, - event::{Event, EventListener}, + event::{Event, EventDescription, EventListener}, message_queue::TimingEvent, registry::get_value_type, scope::scope_and_block, @@ -54,8 +54,8 @@ use crate::{ AggregationUpdateJob, AggregationUpdateQueue, ChildExecuteContext, CleanupOldEdgesOperation, ComputeDirtyAndCleanUpdate, ConnectChildOperation, ExecuteContext, ExecuteContextImpl, LeafDistanceUpdateQueue, Operation, OutdatedEdge, - TaskGuard, connect_children, get_aggregation_number, get_uppers, is_root_node, - make_task_dirty_internal, prepare_new_children, + TaskGuard, TaskType, connect_children, get_aggregation_number, get_uppers, + is_root_node, make_task_dirty_internal, prepare_new_children, }, storage::Storage, storage_schema::{TaskStorage, TaskStorageAccessors}, @@ -63,11 +63,17 @@ use crate::{ backing_storage::BackingStorage, data::{ ActivenessState, CellRef, CollectibleRef, CollectiblesRef, Dirtyness, InProgressCellState, - InProgressState, InProgressStateInner, OutputValue, RootType, + InProgressState, InProgressStateInner, OutputValue, TransientTask, }, utils::{ - bi_map::BiMap, chunked_vec::ChunkedVec, dash_map_drop_contents::drop_contents, - ptr_eq_arc::PtrEqArc, shard_amount::compute_shard_amount, sharded::Sharded, swap_retain, + arc_or_owned::ArcOrOwned, + chunked_vec::ChunkedVec, + dash_map_drop_contents::drop_contents, + dash_map_raw_entry::{RawEntry, raw_entry}, + ptr_eq_arc::PtrEqArc, + shard_amount::compute_shard_amount, + sharded::Sharded, + swap_retain, }, }; @@ -102,28 +108,6 @@ impl SnapshotRequest { } } -type TransientTaskOnce = - Mutex> + Send + 'static>>>>; - -pub enum TransientTask { - /// A root task that will track dependencies and re-execute when - /// dependencies change. Task will eventually settle to the correct - /// execution. - /// - /// Always active. Automatically scheduled. - Root(TransientTaskRoot), - - // TODO implement these strongly consistency - /// A single root task execution. It won't track dependencies. - /// Task will definitely include all invalidations that happened before the - /// start of the task. It may or may not include invalidations that - /// happened after that. It may see these invalidations partially - /// applied. - /// - /// Active until done. Automatically scheduled. - Once(TransientTaskOnce), -} - pub enum StorageMode { /// Queries the storage for cache entries that don't exist locally. ReadOnly, @@ -190,8 +174,7 @@ struct TurboTasksBackendInner { transient_task_id_factory: IdFactoryWithReuse, persisted_task_cache_log: Option, - task_cache: BiMap, TaskId>, - transient_tasks: FxDashMap>, + task_cache: FxDashMap, TaskId>, storage: Storage, @@ -270,8 +253,7 @@ impl TurboTasksBackendInner { TaskId::MAX, ), persisted_task_cache_log: need_log.then(|| Sharded::new(shard_amount)), - task_cache: BiMap::new(), - transient_tasks: FxDashMap::default(), + task_cache: FxDashMap::default(), local_is_partial: AtomicBool::new(next_task_id != TaskId::MIN), storage: Storage::new(shard_amount, small_preallocation), in_progress_operations: AtomicUsize::new(0), @@ -443,9 +425,10 @@ impl TurboTasksBackendInner { tx: Option<&'l B::ReadTransaction<'tx>>, parent_task: Option, child_task: TaskId, + task_type: Option>, turbo_tasks: &'l dyn TurboTasksBackendApi>, ) { - operation::ConnectChildOperation::run(parent_task, child_task, unsafe { + operation::ConnectChildOperation::run(parent_task, child_task, task_type, unsafe { self.execute_context_with_tx(tx, turbo_tasks) }); } @@ -454,11 +437,13 @@ impl TurboTasksBackendInner { &self, parent_task: Option, child_task: TaskId, + task_type: Option>, turbo_tasks: &dyn TurboTasksBackendApi>, ) { operation::ConnectChildOperation::run( parent_task, child_task, + task_type, self.execute_context(turbo_tasks), ); } @@ -493,17 +478,18 @@ impl TurboTasksBackendInner { (ctx.task(task_id, TaskDataCategory::All), None) }; - fn listen_to_done_event( - this: &TurboTasksBackendInner, - reader: Option, + fn listen_to_done_event( + reader_description: Option, tracking: ReadTracking, done_event: &Event, ) -> EventListener { done_event.listen_with_note(move || { - let reader_desc = reader.map(|r| this.get_task_desc_fn(r)); move || { - if let Some(reader_desc) = reader_desc.as_ref() { - format!("try_read_task_output from {} ({})", reader_desc(), tracking) + if let Some(reader_description) = reader_description.as_ref() { + format!( + "try_read_task_output from {} ({})", + reader_description, tracking + ) } else { format!("try_read_task_output ({})", tracking) } @@ -511,26 +497,26 @@ impl TurboTasksBackendInner { }) } - fn check_in_progress( - this: &TurboTasksBackendInner, + fn check_in_progress( task: &impl TaskGuard, - reader: Option, + reader_description: Option, tracking: ReadTracking, - ctx: &impl ExecuteContext<'_>, ) -> Option, anyhow::Error>> { match task.get_in_progress() { Some(InProgressState::Scheduled { done_event, .. }) => Some(Ok(Err( - listen_to_done_event(this, reader, tracking, done_event), + listen_to_done_event(reader_description, tracking, done_event), ))), Some(InProgressState::InProgress(box InProgressStateInner { done_event, .. })) => Some(Ok(Err(listen_to_done_event( - this, reader, tracking, done_event, + reader_description, + tracking, + done_event, )))), Some(InProgressState::Canceled) => Some(Err(anyhow::anyhow!( "{} was canceled", - ctx.get_task_description(task.id()) + task.get_task_description() ))), None => None, } @@ -640,7 +626,7 @@ impl TurboTasksBackendInner { // Check the dirty count of the root node let has_dirty_containers = task.has_dirty_containers(); - let task_description = ctx.get_task_description(task_id); + let task_description = task.get_task_description(); let is_dirty_label = if let Some(parent_priority) = is_dirty { format!(", dirty({parent_priority})") } else { @@ -672,7 +658,9 @@ impl TurboTasksBackendInner { writeln!(info, "\n dirty tasks:").unwrap(); for (child_task_id, count) in children { - let task_description = ctx.get_task_description(child_task_id); + let task_description = ctx + .task(child_task_id, TaskDataCategory::Data) + .get_task_description(); if visited.insert(child_task_id) { let child_info = get_info( ctx, @@ -714,7 +702,11 @@ impl TurboTasksBackendInner { } } - if let Some(value) = check_in_progress(self, &task, reader, options.tracking, &ctx) { + let reader_description = reader_task + .as_ref() + .map(|r| EventDescription::new(|| r.get_task_desc_fn())); + if let Some(value) = check_in_progress(&task, reader_description.clone(), options.tracking) + { return value; } @@ -723,7 +715,7 @@ impl TurboTasksBackendInner { OutputValue::Cell(cell) => Ok(Ok(RawVc::TaskCell(cell.task, cell.cell))), OutputValue::Output(task) => Ok(Ok(RawVc::TaskOutput(*task))), OutputValue::Error(error) => Err(error - .with_task_context(ctx.get_task_description(task_id), Some(task_id)) + .with_task_context(task.get_task_type(), Some(task_id)) .into()), }; if let Some(mut reader_task) = reader_task @@ -772,23 +764,23 @@ impl TurboTasksBackendInner { } drop(reader_task); - let note = move || { - let reader_desc = reader.map(|r| self.get_task_desc_fn(r)); + let note = EventDescription::new(|| { move || { - if let Some(reader_desc) = reader_desc.as_ref() { - format!("try_read_task_output (recompute) from {}", (reader_desc)()) + if let Some(reader) = reader_description.as_ref() { + format!("try_read_task_output (recompute) from {reader}",) } else { "try_read_task_output (recompute, untracked)".to_string() } } - }; + }); // Output doesn't exist. We need to schedule the task to compute it. let (in_progress_state, listener) = InProgressState::new_scheduled_with_listener( TaskExecutionReason::OutputNotAvailable, - || self.get_task_desc_fn(task_id), + EventDescription::new(|| task.get_task_desc_fn()), note, ); + // It's not possible that the task is InProgress at this point. If it is InProgress { // done: true } it must have Output and would early return. let old = task.set_in_progress(in_progress_state); @@ -880,37 +872,37 @@ impl TurboTasksBackendInner { in_progress, Some(InProgressState::InProgress(..) | InProgressState::Scheduled { .. }) ) { - return Ok(Err(self.listen_to_cell(&mut task, task_id, reader, cell).0)); + return Ok(Err(self + .listen_to_cell(&mut task, task_id, &reader_task, cell) + .0)); } let is_cancelled = matches!(in_progress, Some(InProgressState::Canceled)); // Check cell index range (cell might not exist at all) let max_id = task.get_cell_type_max_index(&cell.type_id).copied(); let Some(max_id) = max_id else { + let task_desc = task.get_task_description(); if tracking.should_track(true) { add_cell_dependency(task_id, task, reader, reader_task, cell, tracking.key()); } bail!( - "Cell {cell:?} no longer exists in task {} (no cell of this type exists)", - ctx.get_task_description(task_id) + "Cell {cell:?} no longer exists in task {task_desc} (no cell of this type exists)", ); }; if cell.index >= max_id { + let task_desc = task.get_task_description(); if tracking.should_track(true) { add_cell_dependency(task_id, task, reader, reader_task, cell, tracking.key()); } - bail!( - "Cell {cell:?} no longer exists in task {} (index out of bounds)", - ctx.get_task_description(task_id) - ); + bail!("Cell {cell:?} no longer exists in task {task_desc} (index out of bounds)"); } - drop(reader_task); // Cell should exist, but data was dropped or is not serializable. We need to recompute the // task the get the cell content. // Listen to the cell and potentially schedule the task - let (listener, new_listener) = self.listen_to_cell(&mut task, task_id, reader, cell); + let (listener, new_listener) = self.listen_to_cell(&mut task, task_id, &reader_task, cell); + drop(reader_task); if !new_listener { return Ok(Err(listener)); } @@ -924,12 +916,14 @@ impl TurboTasksBackendInner { // Schedule the task, if not already scheduled if is_cancelled { - bail!("{} was canceled", ctx.get_task_description(task_id)); + bail!("{} was canceled", task.get_task_description()); } - let _ = task.add_scheduled(TaskExecutionReason::CellNotAvailable, || { - self.get_task_desc_fn(task_id) - }); - ctx.schedule_task(task, TaskPriority::initial()); + + let _ = task.add_scheduled( + TaskExecutionReason::CellNotAvailable, + EventDescription::new(|| task.get_task_desc_fn()), + ); + ctx.schedule_task(task, TaskPriority::Initial); Ok(Err(listener)) } @@ -938,11 +932,11 @@ impl TurboTasksBackendInner { &self, task: &mut impl TaskGuard, task_id: TaskId, - reader: Option, + reader_task: &Option, cell: CellId, ) -> (EventListener, bool) { - let note = move || { - let reader_desc = reader.map(|r| self.get_task_desc_fn(r)); + let note = || { + let reader_desc = reader_task.as_ref().map(|r| r.get_task_desc_fn()); move || { if let Some(reader_desc) = reader_desc.as_ref() { format!("try_read_task_cell (in progress) from {}", (reader_desc)()) @@ -963,35 +957,6 @@ impl TurboTasksBackendInner { (listener, true) } - fn lookup_task_type(&self, task_id: TaskId) -> Option> { - if let Some(task_type) = self.task_cache.lookup_reverse(&task_id) { - return Some(task_type); - } - if self.should_restore() - && self.local_is_partial.load(Ordering::Acquire) - && !task_id.is_transient() - && let Some(task_type) = unsafe { - self.backing_storage - .reverse_lookup_task_cache(None, task_id) - .expect("Failed to lookup task type") - } - { - let _ = self.task_cache.try_insert(task_type.clone(), task_id); - return Some(task_type); - } - None - } - - fn get_task_desc_fn(&self, task_id: TaskId) -> impl Fn() -> String + Send + Sync + 'static { - let task_type = self.lookup_task_type(task_id); - move || { - task_type.as_ref().map_or_else( - || format!("{task_id:?} transient"), - |task_type| format!("{task_id:?} {task_type}"), - ) - } - } - fn snapshot_and_persist( &self, parent_span: Option, @@ -1144,7 +1109,7 @@ impl TurboTasksBackendInner { #[cfg(feature = "print_cache_item_size")] task_cache_stats .lock() - .entry(self.get_task_description(task_id)) + .entry(self.debug_get_task_description(task_id)) .or_default() .add_meta(&meta); Some(meta) @@ -1153,7 +1118,7 @@ impl TurboTasksBackendInner { Some(Err(err)) => { println!( "Serializing task {} failed (meta): {:?}", - self.get_task_description(task_id), + self.debug_get_task_description(task_id), err ); None @@ -1164,7 +1129,7 @@ impl TurboTasksBackendInner { #[cfg(feature = "print_cache_item_size")] task_cache_stats .lock() - .entry(self.get_task_description(task_id)) + .entry(self.debug_get_task_description(task_id)) .or_default() .add_data(&data); Some(data) @@ -1173,7 +1138,7 @@ impl TurboTasksBackendInner { Some(Err(err)) => { println!( "Serializing task {} failed (data): {:?}", - self.get_task_description(task_id), + self.debug_get_task_description(task_id), err ); None @@ -1305,8 +1270,7 @@ impl TurboTasksBackendInner { if self.should_persist() { self.snapshot_and_persist(Span::current().into(), "stop"); } - self.task_cache.drop_contents(); - drop_contents(&self.transient_tasks); + drop_contents(&self.task_cache); self.storage.drop_contents(); if let Err(err) = self.backing_storage.shutdown() { println!("Shutting down failed: {err}"); @@ -1353,9 +1317,16 @@ impl TurboTasksBackendInner { parent_task: Option, turbo_tasks: &dyn TurboTasksBackendApi>, ) -> TaskId { - if let Some(task_id) = self.task_cache.lookup_forward(&task_type) { + // First check if the task exists in the cache which only uses a read lock + if let Some(task_id) = self.task_cache.get(&task_type) { + let task_id = *task_id; self.track_cache_hit(&task_type); - self.connect_child(parent_task, task_id, turbo_tasks); + self.connect_child( + parent_task, + task_id, + Some(ArcOrOwned::Owned(task_type)), + turbo_tasks, + ); return task_id; } @@ -1364,7 +1335,7 @@ impl TurboTasksBackendInner { let tx = check_backing_storage .then(|| self.backing_storage.start_read_transaction()) .flatten(); - let task_id = { + let (task_id, task_type) = { // Safety: `tx` is a valid transaction from `self.backend.backing_storage`. if let Some(task_id) = unsafe { check_backing_storage @@ -1375,34 +1346,55 @@ impl TurboTasksBackendInner { }) .flatten() } { + // Task exists in backing storage + // So we only need to insert it into the in-memory cahce self.track_cache_hit(&task_type); - let _ = self.task_cache.try_insert(Arc::new(task_type), task_id); - task_id + let task_type = match raw_entry(&self.task_cache, &task_type) { + RawEntry::Occupied(_) => ArcOrOwned::Owned(task_type), + RawEntry::Vacant(e) => { + let task_type = Arc::new(task_type); + e.insert(task_type.clone(), task_id); + ArcOrOwned::Arc(task_type) + } + }; + (task_id, task_type) } else { - let task_type = Arc::new(task_type); - let task_id = self.persisted_task_id_factory.get(); - let task_id = if let Err(existing_task_id) = - self.task_cache.try_insert(task_type.clone(), task_id) - { - self.track_cache_hit(&task_type); - // Safety: We just created the id and failed to insert it. - unsafe { - self.persisted_task_id_factory.reuse(task_id); + // Task doesn't exist in memory cache or backing storage + // So we might need to create a new task + let (task_id, mut task_type) = match raw_entry(&self.task_cache, &task_type) { + RawEntry::Occupied(e) => { + let task_id = *e.get(); + drop(e); + self.track_cache_hit(&task_type); + (task_id, ArcOrOwned::Owned(task_type)) + } + RawEntry::Vacant(e) => { + let task_type = Arc::new(task_type); + let task_id = self.persisted_task_id_factory.get(); + e.insert(task_type.clone(), task_id); + self.track_cache_miss(&task_type); + (task_id, ArcOrOwned::Arc(task_type)) } - existing_task_id - } else { - self.track_cache_miss(&task_type); - task_id }; if let Some(log) = &self.persisted_task_cache_log { - log.lock(task_id).push((task_type, task_id)); + let task_type_arc: Arc = Arc::from(task_type); + log.lock(task_id).push((task_type_arc.clone(), task_id)); + task_type = ArcOrOwned::Arc(task_type_arc); } - task_id + (task_id, task_type) } }; // Safety: `tx` is a valid transaction from `self.backend.backing_storage`. - unsafe { self.connect_child_with_tx(tx.as_ref(), parent_task, task_id, turbo_tasks) }; + unsafe { + self.connect_child_with_tx( + tx.as_ref(), + parent_task, + task_id, + Some(task_type), + turbo_tasks, + ) + }; task_id } @@ -1417,33 +1409,52 @@ impl TurboTasksBackendInner { && !parent_task.is_transient() { self.panic_persistent_calling_transient( - self.lookup_task_type(parent_task).as_deref(), + self.debug_get_task_description(parent_task), Some(&task_type), /* cell_id */ None, ); } - if let Some(task_id) = self.task_cache.lookup_forward(&task_type) { + // First check if the task exists in the cache which only uses a read lock + if let Some(task_id) = self.task_cache.get(&task_type) { + let task_id = *task_id; self.track_cache_hit(&task_type); - self.connect_child(parent_task, task_id, turbo_tasks); + self.connect_child( + parent_task, + task_id, + Some(ArcOrOwned::Owned(task_type)), + turbo_tasks, + ); return task_id; } + // If not, acquire a write lock and double check / insert + match raw_entry(&self.task_cache, &task_type) { + RawEntry::Occupied(e) => { + let task_id = *e.get(); + drop(e); + self.track_cache_hit(&task_type); + self.connect_child( + parent_task, + task_id, + Some(ArcOrOwned::Owned(task_type)), + turbo_tasks, + ); + task_id + } + RawEntry::Vacant(e) => { + let task_type = Arc::new(task_type); + let task_id = self.transient_task_id_factory.get(); + e.insert(task_type.clone(), task_id); + self.track_cache_miss(&task_type); + self.connect_child( + parent_task, + task_id, + Some(ArcOrOwned::Arc(task_type)), + turbo_tasks, + ); - let task_type = Arc::new(task_type); - let task_id = self.transient_task_id_factory.get(); - if let Err(existing_task_id) = self.task_cache.try_insert(task_type.clone(), task_id) { - self.track_cache_hit(&task_type); - // Safety: We just created the id and failed to insert it. - unsafe { - self.transient_task_id_factory.reuse(task_id); + task_id } - self.connect_child(parent_task, existing_task_id, turbo_tasks); - return existing_task_id; } - - self.track_cache_miss(&task_type); - self.connect_child(parent_task, task_id, turbo_tasks); - - task_id } /// Generate an object that implements [`fmt::Display`] explaining why the given @@ -1461,7 +1472,7 @@ impl TurboTasksBackendInner { cell_type_id: Option, visited_set: &mut FxHashSet, ) -> DebugTraceTransientTask { - if let Some(task_type) = backend.lookup_task_type(task_id) { + if let Some(task_type) = backend.debug_get_cached_task_type(task_id) { if visited_set.contains(&task_id) { let task_name = task_type.get_name(); DebugTraceTransientTask::Collapsed { @@ -1598,11 +1609,20 @@ impl TurboTasksBackendInner { task.invalidate_serialization(); } - fn get_task_description(&self, task_id: TaskId) -> String { - self.lookup_task_type(task_id).map_or_else( - || format!("{task_id:?} transient"), - |task_type| task_type.to_string(), - ) + fn debug_get_task_description(&self, task_id: TaskId) -> String { + let task = self.storage.access_mut(task_id); + if let Some(value) = task.get_persistent_task_type() { + format!("{task_id:?} {}", value) + } else if let Some(value) = task.get_transient_task_type() { + format!("{task_id:?} {}", value) + } else { + format!("{task_id:?} unknown") + } + } + + fn debug_get_cached_task_type(&self, task_id: TaskId) -> Option> { + let task = self.storage.access_mut(task_id); + task.get_persistent_task_type().cloned() } fn task_execution_canceled( @@ -1634,24 +1654,13 @@ impl TurboTasksBackendInner { priority: TaskPriority, turbo_tasks: &dyn TurboTasksBackendApi>, ) -> Option> { - enum TaskType { - Cached(Arc), - Transient(Arc), - } - let (task_type, once_task) = if let Some(task_type) = self.lookup_task_type(task_id) { - (TaskType::Cached(task_type), false) - } else if let Some(task_type) = self.transient_tasks.get(&task_id) { - ( - TaskType::Transient(task_type.clone()), - matches!(**task_type, TransientTask::Once(_)), - ) - } else { - return None; - }; let execution_reason; + let task_type; { let mut ctx = self.execute_context(turbo_tasks); let mut task = ctx.task(task_id, TaskDataCategory::All); + task_type = task.get_task_type().to_owned(); + let once_task = matches!(task_type, TaskType::Transient(ref tt) if matches!(&**tt, TransientTask::Once(_))); if let Some(tasks) = task.prefetch() { drop(task); ctx.prepare_tasks(tasks); @@ -1895,6 +1904,7 @@ impl TurboTasksBackendInner { stale, ref mut new_children, session_dependent, + once_task: is_once_task, .. }) = in_progress else { @@ -1903,7 +1913,7 @@ impl TurboTasksBackendInner { // If the task is stale, reschedule it #[cfg(not(feature = "no_fast_stale"))] - if stale { + if stale && !is_once_task { let Some(InProgressState::InProgress(box InProgressStateInner { done_event, mut new_children, @@ -2137,14 +2147,15 @@ impl TurboTasksBackendInner { result = tracing::field::Empty, ) .entered(); - if ctx.is_once_task(dependent_task_id) { + let mut make_stale = true; + let dependent = ctx.task(dependent_task_id, TaskDataCategory::All); + let transient_task_type = dependent.get_transient_task_type(); + if transient_task_type.is_some_and(|tt| matches!(&**tt, TransientTask::Once(_))) { // once tasks are never invalidated #[cfg(feature = "trace_task_output_dependencies")] span.record("result", "once task"); return; } - let mut make_stale = true; - let dependent = ctx.task(dependent_task_id, TaskDataCategory::All); if dependent.outdated_output_dependencies_contains(&task_id) { #[cfg(feature = "trace_task_output_dependencies")] span.record("result", "outdated dependency"); @@ -2211,7 +2222,7 @@ impl TurboTasksBackendInner { debug_assert!(!new_children.is_empty()); let mut queue = AggregationUpdateQueue::new(); - ctx.for_each_task_meta(new_children.iter().copied(), |child_task, ctx| { + ctx.for_each_task_all(new_children.iter().copied(), |child_task, ctx| { if !child_task.has_output() { let child_id = child_task.id(); make_task_dirty_internal( @@ -2248,6 +2259,7 @@ impl TurboTasksBackendInner { let InProgressState::InProgress(box InProgressStateInner { #[cfg(not(feature = "no_fast_stale"))] stale, + once_task: is_once_task, .. }) = in_progress else { @@ -2256,7 +2268,7 @@ impl TurboTasksBackendInner { // If the task is stale, reschedule it #[cfg(not(feature = "no_fast_stale"))] - if *stale { + if *stale && !is_once_task { let Some(InProgressState::InProgress(box InProgressStateInner { done_event, .. })) = task.take_in_progress() else { @@ -2319,7 +2331,7 @@ impl TurboTasksBackendInner { } let InProgressState::InProgress(box InProgressStateInner { done_event, - once_task: _, + once_task: is_once_task, stale, session_dependent, marked_as_completed: _, @@ -2331,7 +2343,7 @@ impl TurboTasksBackendInner { debug_assert!(new_children.is_empty()); // If the task is stale, reschedule it - if stale { + if stale && !is_once_task { let old = task.set_in_progress(InProgressState::Scheduled { done_event, reason: TaskExecutionReason::Stale, @@ -2437,7 +2449,8 @@ impl TurboTasksBackendInner { }; #[cfg(feature = "verify_determinism")] - let reschedule = (dirty_changed || no_output_set) && !task_id.is_transient(); + let reschedule = + (dirty_changed || no_output_set) && !task_id.is_transient() && !is_once_task; #[cfg(not(feature = "verify_determinism"))] let reschedule = false; if reschedule { @@ -2864,25 +2877,14 @@ impl TurboTasksBackendInner { turbo_tasks: &dyn TurboTasksBackendApi>, ) { self.assert_not_persistent_calling_transient(parent_task, task, None); - ConnectChildOperation::run(parent_task, task, self.execute_context(turbo_tasks)); + ConnectChildOperation::run(parent_task, task, None, self.execute_context(turbo_tasks)); } fn create_transient_task(&self, task_type: TransientTaskType) -> TaskId { let task_id = self.transient_task_id_factory.get(); - let root_type = match task_type { - TransientTaskType::Root(_) => RootType::RootTask, - TransientTaskType::Once(_) => RootType::OnceTask, - }; - self.transient_tasks.insert( - task_id, - Arc::new(match task_type { - TransientTaskType::Root(f) => TransientTask::Root(f), - TransientTaskType::Once(f) => TransientTask::Once(Mutex::new(Some(f))), - }), - ); { let mut task = self.storage.access_mut(task_id); - (*task).init_transient_task(task_id, root_type, self.should_track_activeness()); + task.init_transient_task(task_id, task_type, self.should_track_activeness()); } #[cfg(feature = "verify_aggregation_graph")] self.root_tasks.lock().insert(task_id); @@ -3109,12 +3111,13 @@ impl TurboTasksBackendInner { child_id: TaskId, cell_id: Option, ) { - if !parent_id.is_none_or(|id| id.is_transient()) && child_id.is_transient() { + if let Some(parent_id) = parent_id + && !parent_id.is_transient() + && child_id.is_transient() + { self.panic_persistent_calling_transient( - parent_id - .and_then(|id| self.lookup_task_type(id)) - .as_deref(), - self.lookup_task_type(child_id).as_deref(), + self.debug_get_task_description(parent_id), + self.debug_get_cached_task_type(child_id).as_deref(), cell_id, ); } @@ -3122,7 +3125,7 @@ impl TurboTasksBackendInner { fn panic_persistent_calling_transient( &self, - parent: Option<&CachedTaskType>, + parent: String, child: Option<&CachedTaskType>, cell_id: Option, ) { @@ -3136,7 +3139,7 @@ impl TurboTasksBackendInner { }; panic!( "Persistent task {} is not allowed to call, read, or connect to transient tasks {}.{}", - parent.map_or("unknown", |t| t.get_name()), + parent, child.map_or("unknown", |t| t.get_name()), transient_reason, ); @@ -3148,7 +3151,7 @@ impl TurboTasksBackendInner { // This should never happen: The collectible APIs use ResolvedVc let task_info = if let Some(col_task_ty) = collectible .try_get_task_id() - .and_then(|t| self.lookup_task_type(t)) + .map(|t| self.debug_get_task_description(t)) { Cow::Owned(format!(" (return type of {col_task_ty})")) } else { @@ -3157,14 +3160,15 @@ impl TurboTasksBackendInner { panic!("Collectible{task_info} must be a ResolvedVc") }; if col_task_id.is_transient() && !task_id.is_transient() { - let transient_reason = if let Some(col_task_ty) = self.lookup_task_type(col_task_id) { - Cow::Owned(format!( - ". The collectible is transient because it depends on:\n{}", - self.debug_trace_transient_task(&col_task_ty, Some(col_cell_id)), - )) - } else { - Cow::Borrowed("") - }; + let transient_reason = + if let Some(col_task_ty) = self.debug_get_cached_task_type(col_task_id) { + Cow::Owned(format!( + ". The collectible is transient because it depends on:\n{}", + self.debug_trace_transient_task(&col_task_ty, Some(col_cell_id)), + )) + } else { + Cow::Borrowed("") + }; // this should never happen: How would a persistent function get a transient Vc? panic!( "Collectible is transient, transient collectibles cannot be emitted from \ @@ -3239,10 +3243,6 @@ impl Backend for TurboTasksBackend { self.0.invalidate_serialization(task_id, turbo_tasks); } - fn get_task_description(&self, task: TaskId) -> String { - self.0.get_task_description(task) - } - fn task_execution_canceled(&self, task: TaskId, turbo_tasks: &dyn TurboTasksBackendApi) { self.0.task_execution_canceled(task, turbo_tasks) } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index c5311761f04a6b..12e01392391615 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -5,6 +5,7 @@ use std::{ mem::take, num::NonZeroU32, ops::{ControlFlow, Deref}, + sync::Arc, thread::yield_now, time::{Duration, Instant}, }; @@ -21,8 +22,10 @@ use tracing::span::Span; feature = "trace_aggregation_update", feature = "trace_find_and_schedule" ))] -use tracing::{span::Span, trace_span}; -use turbo_tasks::{FxIndexMap, TaskExecutionReason, TaskId}; +use tracing::trace_span; +use turbo_tasks::{ + FxIndexMap, TaskExecutionReason, TaskId, backend::CachedTaskType, event::EventDescription, +}; #[cfg(feature = "trace_task_dirty")] use crate::backend::operation::invalidate::TaskDirtyCause; @@ -264,6 +267,8 @@ pub enum AggregationUpdateJob { // upon attempted serialization) similar to #[serde(skip)] on variants #[bincode(skip, default = "unreachable_decode")] task: TaskId, + /// Set the task type if not already set + task_type: Option>, }, /// Increases the active counters of the tasks IncreaseActiveCounts { @@ -1153,12 +1158,12 @@ impl AggregationUpdateQueue { } } } - AggregationUpdateJob::IncreaseActiveCount { task } => { - self.increase_active_count(ctx, task); + AggregationUpdateJob::IncreaseActiveCount { task, task_type } => { + self.increase_active_count(ctx, task, task_type); } AggregationUpdateJob::IncreaseActiveCounts { mut task_ids } => { if let Some(task_id) = task_ids.pop() { - self.increase_active_count(ctx, task_id); + self.increase_active_count(ctx, task_id, None); if !task_ids.is_empty() { self.jobs.push_front(AggregationUpdateJobItem::new( AggregationUpdateJob::IncreaseActiveCounts { task_ids }, @@ -1371,6 +1376,7 @@ impl AggregationUpdateQueue { if has_active_count { self.push(AggregationUpdateJob::IncreaseActiveCount { task: task_id, + task_type: None, }); } } @@ -1476,11 +1482,12 @@ impl AggregationUpdateQueue { activeness_state.set_active_until_clean(); } } - if let Some((reason, parent_priority)) = should_schedule - && task.add_scheduled(reason, || ctx.get_task_desc_fn(task_id)) - { - drop(task); - ctx.schedule(task_id, parent_priority); + if let Some((reason, parent_priority)) = should_schedule { + let description = EventDescription::new(|| task.get_task_desc_fn()); + if task.add_scheduled(reason, description) { + drop(task); + ctx.schedule(task_id, parent_priority); + } } } @@ -1640,10 +1647,15 @@ impl AggregationUpdateQueue { "inner_of_uppers_lost_follower is not able to remove follower \ {lost_follower_id} ({}) from {} as they don't exist as upper or follower \ edges", - ctx.get_task_description(lost_follower_id), + ctx.task(lost_follower_id, TaskDataCategory::Data) + .get_task_description(), upper_ids .iter() - .map(|id| format!("{} ({})", id, ctx.get_task_description(*id))) + .map(|id| format!( + "{} ({})", + id, + ctx.task(*id, TaskDataCategory::Data).get_task_description() + )) .collect::>() .join(", ") ); @@ -1789,10 +1801,15 @@ impl AggregationUpdateQueue { {upper_id} ({}) as they don't exist as upper or follower edges", lost_follower_ids .iter() - .map(|id| format!("{} ({})", id, ctx.get_task_description(*id))) + .map(|id| format!( + "{} ({})", + id, + ctx.task(*id, TaskDataCategory::Data).get_task_description() + )) .collect::>() .join(", "), - ctx.get_task_description(upper_id), + ctx.task(upper_id, TaskDataCategory::Data) + .get_task_description() ) } self.push(AggregationUpdateJob::InnerOfUpperLostFollowers { @@ -1987,7 +2004,7 @@ impl AggregationUpdateQueue { fn inner_of_upper_has_new_followers( &mut self, - ctx: &mut impl ExecuteContext, + ctx: &mut impl ExecuteContext<'_>, new_follower_ids: SmallVec<[T; N]>, upper_id: TaskId, ) { @@ -2252,6 +2269,7 @@ impl AggregationUpdateQueue { if has_active_count { self.push(AggregationUpdateJob::IncreaseActiveCount { task: new_follower_id, + task_type: None, }); } // notify uppers about new follower @@ -2337,7 +2355,7 @@ impl AggregationUpdateQueue { /// Decreases the active count of a task. /// /// Only used when activeness is tracked. - fn decrease_active_count(&mut self, ctx: &mut impl ExecuteContext, task_id: TaskId) { + fn decrease_active_count(&mut self, ctx: &mut impl ExecuteContext<'_>, task_id: TaskId) { #[cfg(feature = "trace_aggregation_update")] let _span = trace_span!("decrease active count").entered(); @@ -2381,15 +2399,29 @@ impl AggregationUpdateQueue { /// Increases the active count of a task. /// /// Only used when activeness is tracked. - fn increase_active_count(&mut self, ctx: &mut impl ExecuteContext, task_id: TaskId) { + fn increase_active_count( + &mut self, + ctx: &mut impl ExecuteContext, + task_id: TaskId, + task_type: Option>, + ) { #[cfg(feature = "trace_aggregation_update")] let _span = trace_span!("increase active count").entered(); let mut task = ctx.task( task_id, - // For performance reasons this should stay `Meta` and not `All` - TaskDataCategory::Meta, + if task_type.is_some() { + TaskDataCategory::All + } else { + // For performance reasons this should stay `Meta` and not `All` + TaskDataCategory::Meta + }, ); + if let Some(task_type) = task_type + && !task.has_persistent_task_type() + { + let _ = task.set_persistent_task_type(task_type); + } let state = task.get_activeness_mut_or_insert_with(|| ActivenessState::new(task_id)); let is_new = state.is_empty(); let is_positive_now = state.increment_active_counter(); @@ -2423,7 +2455,7 @@ impl AggregationUpdateQueue { fn update_aggregation_number( &mut self, - ctx: &mut impl ExecuteContext, + ctx: &mut impl ExecuteContext<'_>, task_id: TaskId, base_effective_distance: Option>, base_aggregation_number: u32, diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index 16424953d73fed..b16746f9eb713c 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -1,5 +1,7 @@ +use std::sync::Arc; + use bincode::{Decode, Encode}; -use turbo_tasks::{TaskExecutionReason, TaskId}; +use turbo_tasks::{TaskExecutionReason, TaskId, backend::CachedTaskType, event::EventDescription}; use crate::{ backend::{ @@ -11,6 +13,7 @@ use crate::{ storage_schema::TaskStorageAccessors, }, data::{InProgressState, InProgressStateInner}, + utils::arc_or_owned::ArcOrOwned, }; #[derive(Encode, Decode, Clone, Default)] @@ -27,24 +30,36 @@ impl ConnectChildOperation { pub fn run( parent_task_id: Option, child_task_id: TaskId, + child_task_type: Option>, mut ctx: impl ExecuteContext<'_>, ) { if let Some(parent_task_id) = parent_task_id { - let mut parent_task = ctx.task(parent_task_id, TaskDataCategory::All); + let mut parent_task = ctx.task(parent_task_id, TaskDataCategory::Meta); let Some(InProgressState::InProgress(box InProgressStateInner { new_children, .. - })) = parent_task.get_in_progress_mut() + })) = parent_task.get_in_progress() else { panic!("Task is not in progress while calling another task: {parent_task:?}"); }; // Quick skip if the child was already connected before - if !new_children.insert(child_task_id) { + // We can't call insert here as this would skip the mandatory task type update below + // Instead we only add it after updating the child task type + if new_children.contains(&child_task_id) { return; } if parent_task.children_contains(&child_task_id) { // It is already connected, we can skip the rest + // but we still need to update the new_children set + let Some(InProgressState::InProgress(box InProgressStateInner { + new_children, + .. + })) = parent_task.get_in_progress_mut() + else { + unreachable!(); + }; + new_children.insert(child_task_id); return; } } @@ -63,14 +78,21 @@ impl ConnectChildOperation { if ctx.should_track_activeness() && parent_task_id.is_some() { queue.push(AggregationUpdateJob::IncreaseActiveCount { task: child_task_id, + task_type: child_task_type.map(Arc::from), }); } else { let mut child_task = ctx.task(child_task_id, TaskDataCategory::All); + if let Some(child_task_type) = child_task_type + && !child_task.has_persistent_task_type() + { + child_task.set_persistent_task_type(child_task_type.into()); + } if !child_task.has_output() - && child_task.add_scheduled(TaskExecutionReason::Connect, || { - ctx.get_task_desc_fn(child_task_id) - }) + && child_task.add_scheduled( + TaskExecutionReason::Connect, + EventDescription::new(|| child_task.get_task_desc_fn()), + ) { ctx.schedule_task(child_task, ctx.get_current_task_priority()); } @@ -80,6 +102,30 @@ impl ConnectChildOperation { aggregation_update: queue, } .execute(&mut ctx); + + if let Some(parent_task_id) = parent_task_id { + let mut parent_task = ctx.task(parent_task_id, TaskDataCategory::Meta); + let Some(InProgressState::InProgress(box InProgressStateInner { + new_children, .. + })) = parent_task.get_in_progress_mut() + else { + panic!("Task is not in progress while calling another task: {parent_task:?}"); + }; + + // Really add the child to the new children set + if !new_children.insert(child_task_id) { + drop(parent_task); + + // There was a concurrent connect child operation, + // so we need to undo the active count update. + AggregationUpdateQueue::run( + AggregationUpdateJob::DecreaseActiveCount { + task: child_task_id, + }, + &mut ctx, + ); + } + } } } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs index 72c3876fcb877f..48df6e1ff4ed54 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs @@ -1,6 +1,6 @@ use bincode::{Decode, Encode}; use smallvec::SmallVec; -use turbo_tasks::{TaskExecutionReason, TaskId}; +use turbo_tasks::{TaskExecutionReason, TaskId, event::EventDescription}; use crate::{ backend::{ @@ -107,25 +107,28 @@ pub enum TaskDirtyCause { } #[cfg(feature = "trace_task_dirty")] -struct TaskDirtyCauseInContext<'l, 'e, E: ExecuteContext<'e>> { +struct TaskDirtyCauseInContext<'l> { cause: &'l TaskDirtyCause, - ctx: &'l E, - _phantom: std::marker::PhantomData<&'e ()>, + task_description: String, } #[cfg(feature = "trace_task_dirty")] -impl<'l, 'e, E: ExecuteContext<'e>> TaskDirtyCauseInContext<'l, 'e, E> { - fn new(cause: &'l TaskDirtyCause, ctx: &'l E) -> Self { +impl<'l> TaskDirtyCauseInContext<'l> { + fn new(cause: &'l TaskDirtyCause, ctx: &'l mut impl ExecuteContext<'_>) -> Self { Self { cause, - ctx, - _phantom: Default::default(), + task_description: match cause { + TaskDirtyCause::OutputChange { task_id } => ctx + .task(*task_id, TaskDataCategory::Data) + .get_task_description(), + _ => String::new(), + }, } } } #[cfg(feature = "trace_task_dirty")] -impl<'e, E: ExecuteContext<'e>> std::fmt::Display for TaskDirtyCauseInContext<'_, 'e, E> { +impl std::fmt::Display for TaskDirtyCauseInContext<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self.cause { TaskDirtyCause::InitialDirty => write!(f, "initial dirty"), @@ -158,12 +161,8 @@ impl<'e, E: ExecuteContext<'e>> std::fmt::Display for TaskDirtyCauseInContext<'_ turbo_tasks::registry::get_value_type(*value_type).name ) } - TaskDirtyCause::OutputChange { task_id } => { - write!( - f, - "task {} output changed", - self.ctx.get_task_description(*task_id) - ) + TaskDirtyCause::OutputChange { .. } => { + write!(f, "task {} output changed", self.task_description) } TaskDirtyCause::CollectiblesChange { collectible_type } => { write!( @@ -184,12 +183,7 @@ pub fn make_task_dirty( queue: &mut AggregationUpdateQueue, ctx: &mut impl ExecuteContext<'_>, ) { - if ctx.is_once_task(task_id) { - return; - } - - let task = ctx.task(task_id, TaskDataCategory::Meta); - + let task = ctx.task(task_id, TaskDataCategory::All); make_task_dirty_internal( task, task_id, @@ -224,10 +218,12 @@ pub fn make_task_dirty_internal( panic!( "Task {} is immutable, but was made dirty. This should not happen and is a \ bug.{extra_info}", - ctx.get_task_description(task_id), + task.get_task_description(), ); } + #[cfg(feature = "trace_task_dirty")] + let task_name = task.get_task_name(); if make_stale && let Some(InProgressState::InProgress(box InProgressStateInner { stale, .. })) = task.get_in_progress_mut() @@ -237,7 +233,7 @@ pub fn make_task_dirty_internal( let _span = tracing::trace_span!( "make task stale", task_id = display(task_id), - name = ctx.get_task_description(task_id), + name = task_name, cause = %TaskDirtyCauseInContext::new(&cause, ctx) ) .entered(); @@ -250,7 +246,7 @@ pub fn make_task_dirty_internal( let _span = tracing::trace_span!( "task already dirty", task_id = display(task_id), - name = ctx.get_task_description(task_id), + name = task_name, cause = %TaskDirtyCauseInContext::new(&cause, ctx) ) .entered(); @@ -276,7 +272,7 @@ pub fn make_task_dirty_internal( #[cfg(feature = "trace_task_dirty")] let _span = tracing::trace_span!( "session-dependent task already dirty", - name = ctx.get_task_description(task_id), + name = task_name, cause = %TaskDirtyCauseInContext::new(&cause, ctx) ) .entered(); @@ -308,7 +304,7 @@ pub fn make_task_dirty_internal( let _span = tracing::trace_span!( "make task dirty", task_id = display(task_id), - name = ctx.get_task_description(task_id), + name = task_name, cause = %TaskDirtyCauseInContext::new(&cause, ctx) ) .entered(); @@ -334,13 +330,12 @@ pub fn make_task_dirty_internal( let should_schedule = !ctx.should_track_activeness() || task.has_activeness(); - if should_schedule - && task.add_scheduled(TaskExecutionReason::Invalidated, || { - ctx.get_task_desc_fn(task_id) - }) - { - drop(task); - let task = ctx.task(task_id, TaskDataCategory::All); - ctx.schedule_task(task, parent_priority); + if should_schedule { + let description = EventDescription::new(|| task.get_task_desc_fn()); + if task.add_scheduled(TaskExecutionReason::Invalidated, description) { + drop(task); + let task = ctx.task(task_id, TaskDataCategory::All); + ctx.schedule_task(task, parent_priority); + } } } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index e59f0dc4d83765..4cb72b8d19b84d 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -7,32 +7,30 @@ mod leaf_distance_update; mod prepare_new_children; mod update_cell; mod update_collectible; - use std::{ - fmt::{Debug, Formatter}, + fmt::{Debug, Display, Formatter}, mem::transmute, - sync::atomic::Ordering, + sync::{Arc, atomic::Ordering}, }; use bincode::{Decode, Encode}; use turbo_tasks::{ CellId, FxIndexMap, TaskExecutionReason, TaskId, TaskPriority, TurboTasksBackendApi, - TypedSharedReference, + TypedSharedReference, backend::CachedTaskType, }; use crate::{ backend::{ - OperationGuard, TaskDataCategory, TransientTask, TurboTasksBackend, TurboTasksBackendInner, + EventDescription, OperationGuard, TaskDataCategory, TurboTasksBackend, + TurboTasksBackendInner, storage::{SpecificTaskDataCategory, StorageWriteGuard}, storage_schema::{TaskStorage, TaskStorageAccessors}, }, backing_storage::{BackingStorage, BackingStorageSealed}, - data::{ActivenessState, CollectibleRef, Dirtyness, InProgressState}, + data::{ActivenessState, CollectibleRef, Dirtyness, InProgressState, TransientTask}, }; -pub trait Operation: - Encode + Decode<()> + Default + TryFrom + Into -{ +pub trait Operation: Encode + Decode<()> + Default + TryFrom { fn execute(self, ctx: &mut impl ExecuteContext<'_>); } @@ -70,7 +68,16 @@ pub trait ExecuteContext<'e>: Sized { func, ) } - fn is_once_task(&self, task_id: TaskId) -> bool; + fn for_each_task_all( + &mut self, + task_ids: impl IntoIterator, + func: impl FnMut(Self::TaskGuardImpl, &mut Self), + ) { + self.for_each_task( + task_ids.into_iter().map(|id| (id, TaskDataCategory::All)), + func, + ) + } fn task_pair( &mut self, task_id1: TaskId, @@ -84,8 +91,6 @@ pub trait ExecuteContext<'e>: Sized { where T: Clone + Into; fn suspending_requested(&self) -> bool; - fn get_task_desc_fn(&self, task_id: TaskId) -> impl Fn() -> String + Send + Sync + 'static; - fn get_task_description(&self, task_id: TaskId) -> String; fn should_track_dependencies(&self) -> bool; fn should_track_activeness(&self) -> bool; } @@ -178,10 +183,9 @@ where match result { Ok(()) => storage, Err(e) => { - let task_name = self.backend.get_task_description(task_id); panic!( "Failed to restore task data (corrupted database or bug): {:?}", - e.context(format!("{category:?} for {task_name} ({task_id}))")) + e.context(format!("{category:?} for {task_id})")) ) } } @@ -373,12 +377,14 @@ where ); } + let mut task_type = None; let mut task = self.backend.storage.access_mut(task_id); if let Some(storage) = storage_for_data && !task.flags.is_restored(TaskDataCategory::Data) { task.restore_from(storage, TaskDataCategory::Data); task.flags.set_restored(TaskDataCategory::Data); + task_type = task.get_persistent_task_type().cloned() } if let Some(storage) = storage_for_meta && !task.flags.is_restored(TaskDataCategory::Meta) @@ -389,6 +395,10 @@ where prepared_task_callback(self, task_id, category, task); #[cfg(debug_assertions)] self.active_task_locks.fetch_sub(1, Ordering::AcqRel); + if let Some(task_type) = task_type { + // Insert into the task cache to avoid future lookups + self.backend.task_cache.entry(task_type).or_insert(task_id); + } } } } @@ -460,7 +470,7 @@ where TaskGuardImpl { task, task_id, - backend: self.backend, + _backend: self.backend, #[cfg(debug_assertions)] category, #[cfg(debug_assertions)] @@ -490,7 +500,7 @@ where let guard: TaskGuardImpl<'_, B> = TaskGuardImpl { task, task_id, - backend, + _backend: backend, #[cfg(debug_assertions)] category: _category, #[cfg(debug_assertions)] @@ -500,17 +510,6 @@ where }); } - fn is_once_task(&self, task_id: TaskId) -> bool { - if !task_id.is_transient() { - return false; - } - if let Some(ty) = self.backend.transient_tasks.get(&task_id) { - matches!(**ty, TransientTask::Once(_)) - } else { - false - } - } - fn task_pair( &mut self, task_id1: TaskId, @@ -586,7 +585,7 @@ where TaskGuardImpl { task: task1, task_id: task_id1, - backend: self.backend, + _backend: self.backend, #[cfg(debug_assertions)] category, #[cfg(debug_assertions)] @@ -595,7 +594,7 @@ where TaskGuardImpl { task: task2, task_id: task_id2, - backend: self.backend, + _backend: self.backend, #[cfg(debug_assertions)] category, #[cfg(debug_assertions)] @@ -636,14 +635,6 @@ where self.backend.suspending_requested() } - fn get_task_desc_fn(&self, task_id: TaskId) -> impl Fn() -> String + Send + Sync + 'static { - self.backend.get_task_desc_fn(task_id) - } - - fn get_task_description(&self, task_id: TaskId) -> String { - self.backend.get_task_description(task_id) - } - fn should_track_dependencies(&self) -> bool { self.backend.should_track_dependencies() } @@ -671,6 +662,43 @@ impl<'e, B: BackingStorage> ChildExecuteContext<'e> for ChildExecuteContextImpl< } } +pub enum TaskTypeRef<'l> { + Cached(&'l Arc), + Transient(&'l Arc), +} + +impl TaskTypeRef<'_> { + pub fn to_owned(&self) -> TaskType { + match self { + TaskTypeRef::Cached(ty) => TaskType::Cached(Arc::clone(ty)), + TaskTypeRef::Transient(ty) => TaskType::Transient(Arc::clone(ty)), + } + } +} + +impl Display for TaskTypeRef<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + TaskTypeRef::Cached(ty) => write!(f, "{}", ty), + TaskTypeRef::Transient(ty) => write!(f, "{}", ty), + } + } +} + +pub enum TaskType { + Cached(Arc), + Transient(Arc), +} + +impl Display for TaskType { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + TaskType::Cached(ty) => write!(f, "{}", ty), + TaskType::Transient(ty) => write!(f, "{}", ty), + } + } +} + pub trait TaskGuard: Debug + TaskStorageAccessors { fn id(&self) -> TaskId; @@ -848,14 +876,11 @@ pub trait TaskGuard: Debug + TaskStorageAccessors { /// Add a scheduled task item. Returns true if the task was successfully added (wasn't already /// present). #[must_use] - fn add_scheduled( + fn add_scheduled( &mut self, reason: TaskExecutionReason, - description: impl FnOnce() -> InnerFn, - ) -> bool - where - InnerFn: Fn() -> String + Sync + Send + 'static, - { + description: EventDescription, + ) -> bool { if self.has_in_progress() { false } else { @@ -864,8 +889,6 @@ pub trait TaskGuard: Debug + TaskStorageAccessors { } } - // ============ Collectible APIs ============ - /// Insert an outdated collectible with count. Returns true if it was newly inserted. #[must_use] fn insert_outdated_collectible(&mut self, collectible: CollectibleRef, value: i32) -> bool { @@ -877,12 +900,35 @@ pub trait TaskGuard: Debug + TaskStorageAccessors { self.add_outdated_collectibles(collectible, value); true } + fn get_task_type(&self) -> TaskTypeRef<'_> { + if let Some(task_type) = self.get_persistent_task_type() { + TaskTypeRef::Cached(task_type) + } else if let Some(task_type) = self.get_transient_task_type() { + TaskTypeRef::Transient(task_type) + } else { + panic!("Every task must have a task type {self:?}"); + } + } + fn get_task_desc_fn(&self) -> impl Fn() -> String + Send + Sync + 'static { + let task_type = self.get_task_type().to_owned(); + let task_id = self.id(); + move || format!("{task_id:?} {task_type}") + } + fn get_task_description(&self) -> String { + let task_type = self.get_task_type().to_owned(); + let task_id = self.id(); + format!("{task_id:?} {task_type}") + } + fn get_task_name(&self) -> String { + let task_type = self.get_task_type().to_owned(); + format!("{task_type}") + } } pub struct TaskGuardImpl<'a, B: BackingStorage> { task_id: TaskId, task: StorageWriteGuard<'a>, - backend: &'a TurboTasksBackendInner, + _backend: &'a TurboTasksBackendInner, #[cfg(debug_assertions)] category: TaskDataCategory, #[cfg(debug_assertions)] @@ -933,9 +979,6 @@ impl Debug for TaskGuardImpl<'_, B> { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { let mut d = f.debug_struct("TaskGuard"); d.field("task_id", &self.task_id); - if let Some(task_type) = self.backend.task_cache.lookup_reverse(&self.task_id) { - d.field("task_type", &task_type); - }; d.field("storage", &*self.task); d.finish() } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs index e9ccec3a7b5e31..73fbec9b55c2cc 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs @@ -107,21 +107,17 @@ impl UpdateCellOperation { Lazy::new(|| updated_key_hashes.into_iter().collect::>()) }); - let tasks_with_keys = task - .iter_cell_dependents() - .filter(|&(dependent_cell, key, _)| { - dependent_cell == cell - && key.is_none_or(|key_hash| { - updated_key_hashes_set - .as_ref() - .is_none_or(|set| set.contains(&key_hash)) - }) - }) - .map(|(_, key, task)| (task, key)) - .filter(|&(dependent_task_id, _)| { - // once tasks are never invalidated - !ctx.is_once_task(dependent_task_id) - }); + let tasks_with_keys = + task.iter_cell_dependents() + .filter_map(|(dependent_cell, key, task)| { + (dependent_cell == cell + && key.is_none_or(|key_hash| { + updated_key_hashes_set + .as_ref() + .is_none_or(|set| set.contains(&key_hash)) + })) + .then_some((task, key)) + }); let mut dependent_tasks: FxIndexMap; 2]>> = FxIndexMap::default(); for (task, key) in tasks_with_keys { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs index 71f4beff39416f..13368617b40b04 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs @@ -17,16 +17,22 @@ //! - `data` - Frequently changed bulk data (dependencies, cell data) //! - `meta` - Rarely changed metadata (output, aggregation, flags) //! - `transient` - Not serialized, only exists in memory +use std::sync::Arc; + +use parking_lot::Mutex; use turbo_tasks::{ CellId, SharedReference, TaskExecutionReason, TaskId, TraitTypeId, TypedSharedReference, - ValueTypeId, event::Event, task_storage, + ValueTypeId, + backend::{CachedTaskType, TransientTaskType}, + event::Event, + task_storage, }; use crate::{ backend::counter_map::CounterMap, data::{ ActivenessState, AggregationNumber, CellRef, CollectibleRef, CollectiblesRef, Dirtyness, - InProgressCellState, InProgressState, LeafDistance, OutputValue, RootType, + InProgressCellState, InProgressState, LeafDistance, OutputValue, RootType, TransientTask, }, }; @@ -58,42 +64,42 @@ struct TaskStorageSchema { // ========================================================================= /// The task's distance for prioritizing invalidation execution #[field(storage = "direct", category = "data", inline, default)] - pub leaf_distance: LeafDistance, + leaf_distance: LeafDistance, /// The task's aggregation number for the aggregation tree. /// Uses Default::default() semantics - a zero aggregation number means "not set". #[field(storage = "direct", category = "meta", inline, default)] - pub aggregation_number: AggregationNumber, + aggregation_number: AggregationNumber, /// Tasks that depend on this task's output. #[field(storage = "auto_set", category = "data", inline, filter_transient)] - pub output_dependent: AutoSet, + output_dependent: AutoSet, /// The task's output value. /// Filtered during serialization to skip transient outputs (referencing transient tasks). #[field(storage = "direct", category = "meta", inline, filter_transient)] - pub output: Option, + output: Option, /// Upper nodes in the aggregation tree (reference counted). #[field(storage = "counter_map", category = "meta", inline, filter_transient)] - pub upper: CounterMap, + upper: CounterMap, // ========================================================================= // COLLECTIBLES (meta) // ========================================================================= /// Collectibles emitted by this task (reference counted). #[field(storage = "counter_map", category = "meta", filter_transient)] - pub collectibles: CounterMap, + collectibles: CounterMap, /// Aggregated collectibles from the subgraph. #[field(storage = "counter_map", category = "meta", filter_transient)] - pub aggregated_collectibles: CounterMap, + aggregated_collectibles: CounterMap, /// Outdated collectibles to be cleaned up (transient). #[field(storage = "counter_map", category = "transient")] - pub outdated_collectibles: CounterMap, + outdated_collectibles: CounterMap, // ========================================================================= // STATE FIELDS (meta) @@ -102,25 +108,25 @@ struct TaskStorageSchema { /// Whether the task is dirty (needs re-execution). /// Absent = clean, present = dirty with the specified Dirtyness state. #[field(storage = "direct", category = "meta")] - pub dirty: Dirtyness, + dirty: Dirtyness, /// Count of dirty containers in the aggregated subgraph. /// Absent = 0, present = actual count. #[field(storage = "direct", category = "meta")] - pub aggregated_dirty_container_count: i32, + aggregated_dirty_container_count: i32, /// Individual dirty containers in the aggregated subgraph. #[field(storage = "counter_map", category = "meta", filter_transient)] - pub aggregated_dirty_containers: CounterMap, + aggregated_dirty_containers: CounterMap, /// Count of clean containers in current session (transient). /// Absent = 0, present = actual count. #[field(storage = "direct", category = "transient")] - pub aggregated_current_session_clean_container_count: i32, + aggregated_current_session_clean_container_count: i32, /// Individual clean containers in current session (transient). #[field(storage = "counter_map", category = "transient")] - pub aggregated_current_session_clean_containers: CounterMap, + aggregated_current_session_clean_containers: CounterMap, // ========================================================================= // FLAGS (meta) - Boolean flags stored in TaskFlags bitfield @@ -128,15 +134,15 @@ struct TaskStorageSchema { // ========================================================================= /// Whether the task has an invalidator. #[field(storage = "flag", category = "meta")] - pub invalidator: bool, + invalidator: bool, /// Whether the task output is immutable (persisted). #[field(storage = "flag", category = "meta")] - pub immutable: bool, + immutable: bool, /// Whether clean in current session (transient flag). #[field(storage = "flag", category = "transient")] - pub current_session_clean: bool, + current_session_clean: bool, // ========================================================================= // INTERNAL STATE FLAGS (transient) - Replaces InnerStorageState @@ -144,110 +150,116 @@ struct TaskStorageSchema { // ========================================================================= /// Whether meta data has been restored from persistent storage. #[field(storage = "flag", category = "transient")] - pub meta_restored: bool, + meta_restored: bool, /// Whether data has been restored from persistent storage. #[field(storage = "flag", category = "transient")] - pub data_restored: bool, + data_restored: bool, /// Whether meta was modified before snapshot mode was entered. #[field(storage = "flag", category = "transient")] - pub meta_modified: bool, + meta_modified: bool, /// Whether data was modified before snapshot mode was entered. #[field(storage = "flag", category = "transient")] - pub data_modified: bool, + data_modified: bool, /// Whether meta was modified after snapshot mode was entered (snapshot taken). #[field(storage = "flag", category = "transient")] - pub meta_snapshot: bool, + meta_snapshot: bool, /// Whether data was modified after snapshot mode was entered (snapshot taken). #[field(storage = "flag", category = "transient")] - pub data_snapshot: bool, + data_snapshot: bool, /// Whether dependencies have been prefetched. #[field(storage = "flag", category = "transient")] - pub prefetched: bool, + prefetched: bool, // ========================================================================= // CHILDREN & AGGREGATION (meta) // ========================================================================= /// Child tasks of this task. #[field(storage = "auto_set", category = "meta", filter_transient)] - pub children: AutoSet, + children: AutoSet, /// Follower nodes in the aggregation tree (reference counted). #[field(storage = "counter_map", category = "meta", filter_transient)] - pub followers: CounterMap, + followers: CounterMap, // ========================================================================= // DEPENDENCIES (data) // ========================================================================= #[field(storage = "auto_set", category = "data", filter_transient)] - pub output_dependencies: AutoSet, + output_dependencies: AutoSet, /// Cells this task depends on. #[field(storage = "auto_set", category = "data", filter_transient)] - pub cell_dependencies: AutoSet<(CellRef, Option)>, + cell_dependencies: AutoSet<(CellRef, Option)>, /// Collectibles this task depends on. #[field(storage = "auto_set", category = "data", filter_transient)] - pub collectibles_dependencies: AutoSet, + collectibles_dependencies: AutoSet, /// Outdated output dependencies to be cleaned up (transient). #[field(storage = "auto_set", category = "transient")] - pub outdated_output_dependencies: AutoSet, + outdated_output_dependencies: AutoSet, /// Outdated cell dependencies to be cleaned up (transient). #[field(storage = "auto_set", category = "transient")] - pub outdated_cell_dependencies: AutoSet<(CellRef, Option)>, + outdated_cell_dependencies: AutoSet<(CellRef, Option)>, /// Outdated collectibles dependencies to be cleaned up (transient). #[field(storage = "auto_set", category = "transient")] - pub outdated_collectibles_dependencies: AutoSet, + outdated_collectibles_dependencies: AutoSet, // ========================================================================= // DEPENDENTS - Tasks that depend on this task's cells // ========================================================================= #[field(storage = "auto_set", category = "data", filter_transient)] - pub cell_dependents: AutoSet<(CellId, Option, TaskId)>, + cell_dependents: AutoSet<(CellId, Option, TaskId)>, /// Tasks that depend on collectibles of a specific type from this task. /// Maps TraitTypeId -> Set #[field(storage = "auto_set", category = "meta", filter_transient)] - pub collectibles_dependents: AutoSet<(TraitTypeId, TaskId)>, + collectibles_dependents: AutoSet<(TraitTypeId, TaskId)>, // ========================================================================= // CELL DATA (data) // ========================================================================= /// Persistent cell data (serializable). #[field(storage = "auto_map", category = "data")] - pub persistent_cell_data: AutoMap, + persistent_cell_data: AutoMap, /// Transient cell data (not serializable). #[field(storage = "auto_map", category = "transient")] - pub transient_cell_data: AutoMap, + transient_cell_data: AutoMap, /// Maximum cell index per cell type. #[field(storage = "auto_map", category = "data")] - pub cell_type_max_index: AutoMap, + cell_type_max_index: AutoMap, // ========================================================================= // TRANSIENT EXECUTION STATE (transient) // ========================================================================= /// Activeness state for root/once tasks (transient). #[field(storage = "direct", category = "transient")] - pub activeness: ActivenessState, + activeness: ActivenessState, /// In-progress execution state (transient). #[field(storage = "direct", category = "transient")] - pub in_progress: InProgressState, + in_progress: InProgressState, /// In-progress cell state for cells being computed (transient). #[field(storage = "auto_map", category = "transient")] - pub in_progress_cells: AutoMap, + in_progress_cells: AutoMap, + + #[field(storage = "direct", category = "data")] + pub persistent_task_type: Arc, + + #[field(storage = "direct", category = "transient")] + pub transient_task_type: Arc, } // ============================================================================= @@ -457,7 +469,7 @@ impl TaskStorage { pub fn init_transient_task( &mut self, task_id: TaskId, - root_type: RootType, + task_type: TransientTaskType, should_track_activeness: bool, ) { // Mark as fully restored since transient tasks don't need restoration from disk @@ -469,7 +481,10 @@ impl TaskStorage { distance: 0, effective: u32::MAX, }; - + let root_type = match task_type { + TransientTaskType::Root(_) => RootType::RootTask, + TransientTaskType::Once(_) => RootType::OnceTask, + }; if should_track_activeness { let activeness = ActivenessState::new_root(root_type, task_id); self.lazy.push(LazyField::Activeness(activeness)); @@ -482,6 +497,10 @@ impl TaskStorage { RootType::OnceTask => "Once Task".to_string(), } }); + self.set_transient_task_type(Arc::new(match task_type { + TransientTaskType::Root(f) => TransientTask::Root(f), + TransientTaskType::Once(f) => TransientTask::Once(Mutex::new(Some(f))), + })); self.set_in_progress(InProgressState::Scheduled { done_event, reason: TaskExecutionReason::Initial, diff --git a/turbopack/crates/turbo-tasks-backend/src/backing_storage.rs b/turbopack/crates/turbo-tasks-backend/src/backing_storage.rs index d52501947ab7dd..c927ee44018c93 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backing_storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backing_storage.rs @@ -77,16 +77,6 @@ pub trait BackingStorageSealed: 'static + Send + Sync { /// # Safety /// /// `tx` must be a transaction from this BackingStorage instance. - unsafe fn reverse_lookup_task_cache( - &self, - tx: Option<&Self::ReadTransaction<'_>>, - task_id: TaskId, - ) -> Result>>; - - /// Lookup and decode fields directly into TaskStorage. - /// # Safety - /// - /// `tx` must be a transaction from this BackingStorage instance. unsafe fn lookup_data( &self, tx: Option<&Self::ReadTransaction<'_>>, @@ -191,23 +181,6 @@ where } } - unsafe fn reverse_lookup_task_cache( - &self, - tx: Option<&Self::ReadTransaction<'_>>, - task_id: TaskId, - ) -> Result>> { - match self { - Either::Left(this) => { - let tx = tx.map(|tx| read_transaction_left_or_panic(tx.as_ref())); - unsafe { this.reverse_lookup_task_cache(tx, task_id) } - } - Either::Right(this) => { - let tx = tx.map(|tx| read_transaction_right_or_panic(tx.as_ref())); - unsafe { this.reverse_lookup_task_cache(tx, task_id) } - } - } - } - unsafe fn lookup_data( &self, tx: Option<&Self::ReadTransaction<'_>>, diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index 9a254c12cc8536..670c0ccb8bad7f 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -1,9 +1,16 @@ +use std::{ + fmt::{self, Debug, Display}, + pin::Pin, +}; + +use anyhow::Result; use bincode::{Decode, Encode}; +use parking_lot::Mutex; use rustc_hash::FxHashSet; use turbo_tasks::{ - CellId, TaskExecutionReason, TaskId, TaskPriority, TraitTypeId, - backend::TurboTasksExecutionError, - event::{Event, EventListener}, + CellId, RawVc, TaskExecutionReason, TaskId, TaskPriority, TraitTypeId, + backend::{TransientTaskRoot, TurboTasksExecutionError}, + event::{Event, EventDescription, EventListener}, }; // this traits are needed for the transient variants of `CachedDataItem` @@ -23,6 +30,8 @@ macro_rules! transient_traits { panic!(concat!(stringify!($name), " cannot be compared")); } } + + impl Eq for $name {} }; } @@ -157,7 +166,47 @@ impl ActivenessState { transient_traits!(ActivenessState); -impl Eq for ActivenessState {} +type TransientTaskOnce = + Mutex> + Send + 'static>>>>; + +pub enum TransientTask { + /// A root task that will track dependencies and re-execute when + /// dependencies change. Task will eventually settle to the correct + /// execution. + /// + /// Always active. Automatically scheduled. + Root(TransientTaskRoot), + + // TODO implement these strongly consistency + /// A single root task execution. It won't track dependencies. + /// Task will definitely include all invalidations that happened before the + /// start of the task. It may or may not include invalidations that + /// happened after that. It may see these invalidations partially + /// applied. + /// + /// Active until done. Automatically scheduled. + Once(TransientTaskOnce), +} + +impl Debug for TransientTask { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + TransientTask::Root(_) => f.write_str("TransientTask::Root"), + TransientTask::Once(_) => f.write_str("TransientTask::Once"), + } + } +} + +impl Display for TransientTask { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + TransientTask::Root(_) => f.write_str("Root Task"), + TransientTask::Once(_) => f.write_str("Once Task"), + } + } +} + +transient_traits!(TransientTask); #[derive(Debug, Clone, Copy, Encode, Decode, PartialEq, Eq)] pub enum Dirtyness { @@ -171,6 +220,15 @@ pub enum RootType { OnceTask, } +impl Display for RootType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + RootType::RootTask => f.write_str("Root Task"), + RootType::OnceTask => f.write_str("Once Task"), + } + } +} + #[derive(Debug)] pub struct InProgressStateInner { pub stale: bool, @@ -203,8 +261,6 @@ pub enum InProgressState { transient_traits!(InProgressState); -impl Eq for InProgressState {} - #[derive(Debug)] pub struct InProgressCellState { pub event: Event, @@ -212,8 +268,6 @@ pub struct InProgressCellState { transient_traits!(InProgressCellState); -impl Eq for InProgressCellState {} - impl InProgressCellState { pub fn new(task_id: TaskId, cell: CellId) -> Self { InProgressCellState { @@ -256,33 +310,17 @@ pub struct LeafDistance { impl InProgressState { /// Create a new scheduled state with a done event. - pub fn new_scheduled( - reason: TaskExecutionReason, - description: impl FnOnce() -> InnerFnDescription, - ) -> Self - where - InnerFnDescription: Fn() -> String + Sync + Send + 'static, - { - let done_event = Event::new(move || { - let inner = description(); - move || format!("{} done_event", inner()) - }); + pub fn new_scheduled(reason: TaskExecutionReason, description: EventDescription) -> Self { + let done_event = Event::new(move || move || format!("{description} done_event")); InProgressState::Scheduled { done_event, reason } } - pub fn new_scheduled_with_listener( + pub fn new_scheduled_with_listener( reason: TaskExecutionReason, - description: impl FnOnce() -> InnerFnDescription, - note: impl FnOnce() -> InnerFnNote, - ) -> (Self, EventListener) - where - InnerFnDescription: Fn() -> String + Sync + Send + 'static, - InnerFnNote: Fn() -> String + Sync + Send + 'static, - { - let done_event = Event::new(move || { - let inner = description(); - move || format!("{} done_event", inner()) - }); + description: EventDescription, + note: EventDescription, + ) -> (Self, EventListener) { + let done_event = Event::new(move || move || format!("{description} done_event")); let listener = done_event.listen_with_note(note); (InProgressState::Scheduled { done_event, reason }, listener) } diff --git a/turbopack/crates/turbo-tasks-backend/src/database/by_key_space.rs b/turbopack/crates/turbo-tasks-backend/src/database/by_key_space.rs index 6a0cd9caecd2e7..211fb48828fa5f 100644 --- a/turbopack/crates/turbo-tasks-backend/src/database/by_key_space.rs +++ b/turbopack/crates/turbo-tasks-backend/src/database/by_key_space.rs @@ -4,8 +4,7 @@ pub struct ByKeySpace { infra: T, task_meta: T, task_data: T, - forward_task_cache: T, - reverse_task_cache: T, + task_cache: T, } impl ByKeySpace { @@ -14,8 +13,7 @@ impl ByKeySpace { infra: factory(KeySpace::Infra), task_meta: factory(KeySpace::TaskMeta), task_data: factory(KeySpace::TaskData), - forward_task_cache: factory(KeySpace::ForwardTaskCache), - reverse_task_cache: factory(KeySpace::ReverseTaskCache), + task_cache: factory(KeySpace::TaskCache), } } @@ -24,8 +22,7 @@ impl ByKeySpace { KeySpace::Infra => &self.infra, KeySpace::TaskMeta => &self.task_meta, KeySpace::TaskData => &self.task_data, - KeySpace::ForwardTaskCache => &self.forward_task_cache, - KeySpace::ReverseTaskCache => &self.reverse_task_cache, + KeySpace::TaskCache => &self.task_cache, } } @@ -34,8 +31,7 @@ impl ByKeySpace { KeySpace::Infra => &mut self.infra, KeySpace::TaskMeta => &mut self.task_meta, KeySpace::TaskData => &mut self.task_data, - KeySpace::ForwardTaskCache => &mut self.forward_task_cache, - KeySpace::ReverseTaskCache => &mut self.reverse_task_cache, + KeySpace::TaskCache => &mut self.task_cache, } } @@ -44,8 +40,7 @@ impl ByKeySpace { (KeySpace::Infra, &self.infra), (KeySpace::TaskMeta, &self.task_meta), (KeySpace::TaskData, &self.task_data), - (KeySpace::ForwardTaskCache, &self.forward_task_cache), - (KeySpace::ReverseTaskCache, &self.reverse_task_cache), + (KeySpace::TaskCache, &self.task_cache), ] .into_iter() } diff --git a/turbopack/crates/turbo-tasks-backend/src/database/key_value_database.rs b/turbopack/crates/turbo-tasks-backend/src/database/key_value_database.rs index 3466fb94e3f174..bfb167ad80c77d 100644 --- a/turbopack/crates/turbo-tasks-backend/src/database/key_value_database.rs +++ b/turbopack/crates/turbo-tasks-backend/src/database/key_value_database.rs @@ -9,8 +9,7 @@ pub enum KeySpace { Infra = 0, TaskMeta = 1, TaskData = 2, - ForwardTaskCache = 3, - ReverseTaskCache = 4, + TaskCache = 3, } pub trait KeyValueDatabase { diff --git a/turbopack/crates/turbo-tasks-backend/src/database/lmdb/mod.rs b/turbopack/crates/turbo-tasks-backend/src/database/lmdb/mod.rs index 39dc1f3a19aa5d..9e754d6e01b3c0 100644 --- a/turbopack/crates/turbo-tasks-backend/src/database/lmdb/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/database/lmdb/mod.rs @@ -18,8 +18,7 @@ pub struct LmbdKeyValueDatabase { infra_db: Database, data_db: Database, meta_db: Database, - forward_task_cache_db: Database, - reverse_task_cache_db: Database, + task_cache_db: Database, } impl LmbdKeyValueDatabase { @@ -38,23 +37,19 @@ impl LmbdKeyValueDatabase { | EnvironmentFlags::NO_TLS, ) .set_max_readers((available_parallelism().map_or(16, |v| v.get()) * 8) as u32) - .set_max_dbs(5) + .set_max_dbs(4) .set_map_size(MAP_SIZE) .open(path)?; let infra_db = env.create_db(Some("infra"), DatabaseFlags::INTEGER_KEY)?; let data_db = env.create_db(Some("data"), DatabaseFlags::INTEGER_KEY)?; let meta_db = env.create_db(Some("meta"), DatabaseFlags::INTEGER_KEY)?; - let forward_task_cache_db = - env.create_db(Some("forward_task_cache"), DatabaseFlags::empty())?; - let reverse_task_cache_db = - env.create_db(Some("reverse_task_cache"), DatabaseFlags::INTEGER_KEY)?; + let task_cache_db = env.create_db(Some("task_cache"), DatabaseFlags::empty())?; Ok(LmbdKeyValueDatabase { env, infra_db, data_db, meta_db, - forward_task_cache_db, - reverse_task_cache_db, + task_cache_db, }) } @@ -63,8 +58,7 @@ impl LmbdKeyValueDatabase { KeySpace::Infra => self.infra_db, KeySpace::TaskMeta => self.meta_db, KeySpace::TaskData => self.data_db, - KeySpace::ForwardTaskCache => self.forward_task_cache_db, - KeySpace::ReverseTaskCache => self.reverse_task_cache_db, + KeySpace::TaskCache => self.task_cache_db, } } } @@ -91,8 +85,7 @@ impl KeyValueDatabase for LmbdKeyValueDatabase { KeySpace::Infra => self.infra_db, KeySpace::TaskMeta => self.meta_db, KeySpace::TaskData => self.data_db, - KeySpace::ForwardTaskCache => self.forward_task_cache_db, - KeySpace::ReverseTaskCache => self.reverse_task_cache_db, + KeySpace::TaskCache => self.task_cache_db, }; let value = match extended_key::get(transaction, db, key) { diff --git a/turbopack/crates/turbo-tasks-backend/src/database/startup_cache.rs b/turbopack/crates/turbo-tasks-backend/src/database/startup_cache.rs index 60fcdbcc94d278..e595f9a5ba178e 100644 --- a/turbopack/crates/turbo-tasks-backend/src/database/startup_cache.rs +++ b/turbopack/crates/turbo-tasks-backend/src/database/startup_cache.rs @@ -87,8 +87,7 @@ impl StartupCacheLayer { KeySpace::Infra => 8, KeySpace::TaskMeta => 1024 * 1024, KeySpace::TaskData => 1024 * 1024, - KeySpace::ForwardTaskCache => 1024 * 1024, - KeySpace::ReverseTaskCache => 1024 * 1024, + KeySpace::TaskCache => 1024 * 1024, }, Default::default(), ) @@ -322,8 +321,7 @@ fn write_key_value_pair( KeySpace::Infra => 0, KeySpace::TaskMeta => 1, KeySpace::TaskData => 2, - KeySpace::ForwardTaskCache => 3, - KeySpace::ReverseTaskCache => 4, + KeySpace::TaskCache => 3, })?; let key_len = key.len(); size_buffer.copy_from_slice(&(key_len as u32).to_be_bytes()); @@ -344,8 +342,7 @@ fn read_key_value_pair<'l>( 0 => KeySpace::Infra, 1 => KeySpace::TaskMeta, 2 => KeySpace::TaskData, - 3 => KeySpace::ForwardTaskCache, - 4 => KeySpace::ReverseTaskCache, + 3 => KeySpace::TaskCache, _ => return Err(anyhow::anyhow!("Invalid key space")), }; *pos += 1; diff --git a/turbopack/crates/turbo-tasks-backend/src/database/turbo/mod.rs b/turbopack/crates/turbo-tasks-backend/src/database/turbo/mod.rs index f70c1284cbe096..ea1144fdd416c4 100644 --- a/turbopack/crates/turbo-tasks-backend/src/database/turbo/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/database/turbo/mod.rs @@ -22,7 +22,7 @@ use crate::database::{ mod parallel_scheduler; /// Number of key families, see KeySpace enum for their numbers. -const FAMILIES: usize = 5; +const FAMILIES: usize = 4; const MB: u64 = 1024 * 1024; const COMPACT_CONFIG: CompactConfig = CompactConfig { diff --git a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs index 4103ade453018b..02de0c5ff9ca51 100644 --- a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs @@ -328,7 +328,7 @@ impl BackingStorageSealed batch .put( - KeySpace::ForwardTaskCache, + KeySpace::TaskCache, WriteBuffer::Borrowed(&task_type_bytes), WriteBuffer::Borrowed(&task_id.to_le_bytes()), ) @@ -337,17 +337,6 @@ impl BackingStorageSealed "Unable to write task cache {task_type:?} => {task_id}" ) })?; - batch - .put( - KeySpace::ReverseTaskCache, - WriteBuffer::Borrowed(IntKey::new(task_id).as_ref()), - WriteBuffer::Borrowed(&task_type_bytes), - ) - .with_context(|| { - format!( - "Unable to write task cache {task_id} => {task_type:?}" - ) - })?; max_task_id = max_task_id.max(task_id); } @@ -415,22 +404,13 @@ impl BackingStorageSealed batch .put( - KeySpace::ForwardTaskCache, + KeySpace::TaskCache, WriteBuffer::Borrowed(&task_type_bytes), WriteBuffer::Borrowed(&task_id.to_le_bytes()), ) .with_context(|| { format!("Unable to write task cache {task_type:?} => {task_id}") })?; - batch - .put( - KeySpace::ReverseTaskCache, - WriteBuffer::Borrowed(IntKey::new(task_id).as_ref()), - WriteBuffer::Borrowed(&task_type_bytes), - ) - .with_context(|| { - format!("Unable to write task cache {task_id} => {task_type:?}") - })?; next_task_id = next_task_id.max(task_id + 1); } } @@ -467,8 +447,7 @@ impl BackingStorageSealed ) -> Result> { let mut task_type_bytes = TurboBincodeBuffer::new(); encode_task_type(task_type, &mut task_type_bytes, None)?; - let Some(bytes) = database.get(tx, KeySpace::ForwardTaskCache, &task_type_bytes)? - else { + let Some(bytes) = database.get(tx, KeySpace::TaskCache, &task_type_bytes)? else { return Ok(None); }; let bytes = bytes.borrow().try_into()?; @@ -485,32 +464,6 @@ impl BackingStorageSealed .with_context(|| format!("Looking up task id for {task_type:?} from database failed")) } - unsafe fn reverse_lookup_task_cache( - &self, - tx: Option<&T::ReadTransaction<'_>>, - task_id: TaskId, - ) -> Result>> { - let inner = &*self.inner; - fn lookup( - database: &D, - tx: &D::ReadTransaction<'_>, - task_id: TaskId, - ) -> Result>> { - let Some(bytes) = database.get( - tx, - KeySpace::ReverseTaskCache, - IntKey::new(*task_id).as_ref(), - )? - else { - return Ok(None); - }; - Ok(Some(turbo_bincode_decode(bytes.borrow())?)) - } - inner - .with_tx(tx, |tx| lookup(&inner.database, tx, task_id)) - .with_context(|| format!("Looking up task type for {task_id} from database failed")) - } - unsafe fn lookup_data( &self, tx: Option<&T::ReadTransaction<'_>>, diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/arc_or_owned.rs b/turbopack/crates/turbo-tasks-backend/src/utils/arc_or_owned.rs new file mode 100644 index 00000000000000..c799850f5e8ed0 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/utils/arc_or_owned.rs @@ -0,0 +1,87 @@ +use std::{fmt::Debug, ops::Deref, sync::Arc}; + +use bincode::{ + Decode, Encode, + de::Decoder, + error::{DecodeError, EncodeError}, + impl_borrow_decode_with_context, +}; + +#[derive(Clone)] +pub enum ArcOrOwned { + Arc(Arc), + Owned(T), +} + +impl ArcOrOwned {} + +impl Deref for ArcOrOwned { + type Target = T; + + fn deref(&self) -> &Self::Target { + match self { + Self::Arc(arc) => arc.deref(), + Self::Owned(value) => value, + } + } +} + +impl AsRef for ArcOrOwned { + fn as_ref(&self) -> &T { + self.deref() + } +} + +impl From for ArcOrOwned { + fn from(value: T) -> Self { + Self::Owned(value) + } +} + +impl From> for ArcOrOwned { + fn from(arc: Arc) -> Self { + Self::Arc(arc) + } +} + +impl From> for Arc { + fn from(value: ArcOrOwned) -> Self { + match value { + ArcOrOwned::Arc(arc) => arc, + ArcOrOwned::Owned(value) => Arc::new(value), + } + } +} + +impl Encode for ArcOrOwned +where + T: Encode, +{ + fn encode(&self, encoder: &mut E) -> Result<(), EncodeError> { + match self { + ArcOrOwned::Arc(arc) => arc.encode(encoder), + ArcOrOwned::Owned(value) => value.encode(encoder), + } + } +} + +impl Decode for ArcOrOwned +where + T: Decode, +{ + fn decode>(decoder: &mut D) -> Result { + let value = T::decode(decoder)?; + Ok(ArcOrOwned::Owned(value)) + } +} + +impl_borrow_decode_with_context!(ArcOrOwned, Context, Context, T: Decode); + +impl Debug for ArcOrOwned +where + T: Debug, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.as_ref().fmt(f) + } +} diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/bi_map.rs b/turbopack/crates/turbo-tasks-backend/src/utils/bi_map.rs deleted file mode 100644 index bf912a248613d5..00000000000000 --- a/turbopack/crates/turbo-tasks-backend/src/utils/bi_map.rs +++ /dev/null @@ -1,68 +0,0 @@ -use std::{borrow::Borrow, hash::Hash}; - -use dashmap::mapref::entry::Entry; -use turbo_tasks::FxDashMap; - -use crate::utils::dash_map_drop_contents::drop_contents; - -/// A bidirectional [`FxDashMap`] that allows lookup by key or value. -/// -/// As keys and values are stored twice, they should be small types, such as -/// [`Arc`][`std::sync::Arc`]. -pub struct BiMap { - forward: FxDashMap, - reverse: FxDashMap, -} - -impl BiMap -where - K: Eq + Hash + Clone, - V: Eq + Hash + Clone, -{ - pub fn new() -> Self { - Self { - forward: FxDashMap::default(), - reverse: FxDashMap::default(), - } - } - - pub fn lookup_forward(&self, key: &Q) -> Option - where - K: Borrow, - Q: Hash + Eq, - { - self.forward.get(key).map(|v| v.value().clone()) - } - - pub fn lookup_reverse(&self, key: &Q) -> Option - where - V: Borrow, - Q: Hash + Eq, - { - self.reverse.get(key).map(|v| v.value().clone()) - } - - pub fn try_insert(&self, key: K, value: V) -> Result<(), V> { - match self.forward.entry(key) { - Entry::Occupied(e) => Err(e.get().clone()), - Entry::Vacant(e) => { - let e = e.insert_entry(value.clone()); - let key = e.key().clone(); - self.reverse.insert(value, key); - drop(e); - Ok(()) - } - } - } -} - -impl BiMap -where - K: Eq + Hash + Send + Sync, - V: Eq + Hash + Send + Sync, -{ - pub fn drop_contents(&self) { - drop_contents(&self.forward); - drop_contents(&self.reverse); - } -} diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_raw_entry.rs b/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_raw_entry.rs new file mode 100644 index 00000000000000..d95ba366883c94 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_raw_entry.rs @@ -0,0 +1,129 @@ +use std::{ + hash::{BuildHasher, Hash}, + ops::{Deref, DerefMut}, +}; + +use dashmap::{DashMap, RwLockWriteGuard, SharedValue}; +use hashbrown::raw::{Bucket, InsertSlot, RawTable}; + +pub fn raw_entry<'l, K: Eq + Hash + AsRef, V, Q: Eq + Hash, S: BuildHasher + Clone>( + map: &'l DashMap, + key: &Q, +) -> RawEntry<'l, K, V> { + let hasher = map.hasher(); + let hash = hasher.hash_one(key); + let shard = map.determine_shard(hash as usize); + let mut shard = map.shards()[shard].write(); + let result = shard.find_or_find_insert_slot( + hash, + |(k, _v)| k.as_ref() == key, + |(k, _v)| hasher.hash_one(k), + ); + match result { + Ok(bucket) => RawEntry::Occupied(OccupiedEntry { bucket, shard }), + Err(insert_slot) => RawEntry::Vacant(VacantEntry { + hash, + insert_slot, + shard, + }), + } +} + +pub enum RawEntry<'l, K, V> { + Occupied(OccupiedEntry<'l, K, V>), + Vacant(VacantEntry<'l, K, V>), +} + +impl<'l, K, V> RawEntry<'l, K, V> { + #[allow(dead_code)] + pub fn or_insert_with (K, V)>(self, f: F) -> RefMut<'l, K, V> { + match self { + RawEntry::Occupied(occupied) => occupied.into_mut(), + RawEntry::Vacant(vacant) => { + let (key, value) = f(); + vacant.insert(key, value) + } + } + } +} + +pub struct OccupiedEntry<'l, K, V> { + bucket: Bucket<(K, SharedValue)>, + #[allow(dead_code, reason = "kept to ensure the lock lives long enough")] + shard: RwLockWriteGuard<'l, RawTable<(K, SharedValue)>>, +} + +impl<'l, K, V> OccupiedEntry<'l, K, V> { + pub fn get(&self) -> &V { + // Safety: We have a write lock on the shard, so no other references to the value can + // exist. + unsafe { self.bucket.as_ref().1.get() } + } + + #[allow(dead_code)] + pub fn get_mut(&mut self) -> &mut V { + // Safety: We have a write lock on the shard, so no other references to the value can + // exist. + unsafe { self.bucket.as_mut().1.get_mut() } + } + + #[allow(dead_code)] + pub fn into_mut(self) -> RefMut<'l, K, V> { + // Safety: We have a write lock on the shard, so no other references to the value can + // exist. + RefMut::from_bucket(self.bucket, self.shard) + } +} + +pub struct VacantEntry<'l, K, V> { + hash: u64, + insert_slot: InsertSlot, + shard: RwLockWriteGuard<'l, RawTable<(K, SharedValue)>>, +} + +impl<'l, K, V> VacantEntry<'l, K, V> { + pub fn insert(mut self, key: K, value: V) -> RefMut<'l, K, V> { + let shared_value = SharedValue::new(value); + // Safety: The insert slot is valid and the map has not be modified since we obtained it (we + // hold the write lock). + unsafe { + let bucket = + self.shard + .insert_in_slot(self.hash, self.insert_slot, (key, shared_value)); + RefMut::from_bucket(bucket, self.shard) + } + } +} + +pub struct RefMut<'l, K, V> { + bucket: Bucket<(K, SharedValue)>, + #[allow(dead_code, reason = "kept to ensure the lock lives long enough")] + shard: RwLockWriteGuard<'l, RawTable<(K, SharedValue)>>, +} + +impl<'l, K, V> RefMut<'l, K, V> { + fn from_bucket( + bucket: Bucket<(K, SharedValue)>, + shard: RwLockWriteGuard<'l, RawTable<(K, SharedValue)>>, + ) -> Self { + Self { bucket, shard } + } +} + +impl<'l, K, V> Deref for RefMut<'l, K, V> { + type Target = V; + + fn deref(&self) -> &Self::Target { + // Safety: We have a write lock on the shard, so no other references to the value can + // exist. + unsafe { self.bucket.as_ref().1.get() } + } +} + +impl<'l, K, V> DerefMut for RefMut<'l, K, V> { + fn deref_mut(&mut self) -> &mut Self::Target { + // Safety: We have a write lock on the shard, so no other references to the value can + // exist. + unsafe { self.bucket.as_mut().1.get_mut() } + } +} diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/mod.rs b/turbopack/crates/turbo-tasks-backend/src/utils/mod.rs index 142ae2c4db1670..1d476cc64d057c 100644 --- a/turbopack/crates/turbo-tasks-backend/src/utils/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/utils/mod.rs @@ -1,7 +1,8 @@ -pub mod bi_map; +pub mod arc_or_owned; pub mod chunked_vec; pub mod dash_map_drop_contents; pub mod dash_map_multi; +pub mod dash_map_raw_entry; pub mod ptr_eq_arc; pub mod shard_amount; pub mod sharded; diff --git a/turbopack/crates/turbo-tasks-macros/src/derive/task_storage_macro.rs b/turbopack/crates/turbo-tasks-macros/src/derive/task_storage_macro.rs index 8a0934d809e9dd..5c5a5b4a5a5318 100644 --- a/turbopack/crates/turbo-tasks-macros/src/derive/task_storage_macro.rs +++ b/turbopack/crates/turbo-tasks-macros/src/derive/task_storage_macro.rs @@ -1,6 +1,9 @@ use proc_macro2::TokenStream; use quote::quote; -use syn::{Fields, Ident, ItemStruct, Meta, Token, Type, punctuated::Punctuated, spanned::Spanned}; +use syn::{ + Fields, Ident, ItemStruct, Meta, Token, Type, Visibility, punctuated::Punctuated, + spanned::Spanned, +}; /// Derives the TaskStorage trait and generates optimized storage structures. pub fn task_storage(input: proc_macro::TokenStream) -> proc_macro::TokenStream { @@ -42,6 +45,7 @@ fn task_storage_impl(input: TokenStream) -> TokenStream { /// along with pre-computed values like the PascalCase variant name. #[derive(Debug, Clone)] struct FieldInfo { + is_pub: bool, /// The field's identifier (snake_case) field_name: Ident, /// The PascalCase variant name for use in LazyField enum @@ -342,6 +346,7 @@ fn expect_string_literal<'a>(expr: &'a syn::Expr, attr_name: &str) -> Option<&'a fn parse_field_storage_attributes(field: &syn::Field) -> FieldInfo { let field_name = field.ident.as_ref().unwrap().clone(); let field_type = field.ty.clone(); + let is_pub = matches!(field.vis, Visibility::Public(_)); // Pre-compute the PascalCase variant name once let variant_name = syn::Ident::new(&to_pascal_case(&field_name.to_string()), field_name.span()); @@ -521,6 +526,7 @@ fn parse_field_storage_attributes(field: &syn::Field) -> FieldInfo { }; FieldInfo { + is_pub, field_name, variant_name, field_type, @@ -952,7 +958,7 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream #[doc = "Unified typed storage containing all task fields."] #[doc = "This is designed to be embedded in the actual InnerStorage for incremental migration."] #[automatically_derived] - #[derive(Debug, Default, PartialEq, turbo_tasks::ShrinkToFit)] + #[derive(Debug, Default, turbo_tasks::ShrinkToFit)] #[shrink_to_fit(crate = "turbo_tasks::macro_helpers::shrink_to_fit")] pub struct TaskStorage { #(#field_defs,)* @@ -1013,6 +1019,11 @@ fn generate_field_accessors(field: &FieldInfo) -> TokenStream { fn generate_direct_field_accessors(field: &FieldInfo) -> TokenStream { let field_name = &field.field_name; let field_type = &field.field_type; + let vis = if field.is_pub { + quote! {pub} + } else { + quote! {} + }; let get_name = field.get_ident(); let set_name = field.set_ident(); @@ -1022,7 +1033,7 @@ fn generate_direct_field_accessors(field: &FieldInfo) -> TokenStream { if field.is_inline() && field.use_default { // Inline with default: field is T stored directly, uses Default::default() for "empty" quote! { - fn #get_name(&self) -> Option<&#field_type> { + #vis fn #get_name(&self) -> Option<&#field_type> { if self.#field_name != #field_type::default() { Some(&self.#field_name) } else { @@ -1030,7 +1041,7 @@ fn generate_direct_field_accessors(field: &FieldInfo) -> TokenStream { } } - fn #set_name(&mut self, value: #field_type) -> Option<#field_type> { + #vis fn #set_name(&mut self, value: #field_type) -> Option<#field_type> { let old = std::mem::replace(&mut self.#field_name, value); if old != #field_type::default() { Some(old) @@ -1039,7 +1050,7 @@ fn generate_direct_field_accessors(field: &FieldInfo) -> TokenStream { } } - fn #take_name(&mut self) -> Option<#field_type> { + #vis fn #take_name(&mut self) -> Option<#field_type> { let old = std::mem::take(&mut self.#field_name); if old != #field_type::default() { Some(old) @@ -1053,15 +1064,15 @@ fn generate_direct_field_accessors(field: &FieldInfo) -> TokenStream { let inner_type = extract_option_inner_type(field_type); quote! { - fn #get_name(&self) -> Option<&#inner_type> { + #vis fn #get_name(&self) -> Option<&#inner_type> { self.#field_name.as_ref() } - fn #set_name(&mut self, value: #inner_type) -> Option<#inner_type> { + #vis fn #set_name(&mut self, value: #inner_type) -> Option<#inner_type> { self.#field_name.replace(value) } - fn #take_name(&mut self) -> Option<#inner_type> { + #vis fn #take_name(&mut self) -> Option<#inner_type> { self.#field_name.take() } } @@ -1073,16 +1084,16 @@ fn generate_direct_field_accessors(field: &FieldInfo) -> TokenStream { let constructor = field.lazy_constructor(quote! { value }); quote! { - fn #get_name(&self) -> Option<&#field_type> { + #vis fn #get_name(&self) -> Option<&#field_type> { self.find_lazy(#extractor) } #[doc = "Set the field value, returning the old value if present."] - fn #set_name(&mut self, value: #field_type) -> Option<#field_type> { + #vis fn #set_name(&mut self, value: #field_type) -> Option<#field_type> { self.set_lazy(#matches_closure, #unwrap_owned, #constructor) } - fn #take_name(&mut self) -> Option<#field_type> { + #vis fn #take_name(&mut self) -> Option<#field_type> { self.take_lazy(#matches_closure, #unwrap_owned) } @@ -1090,7 +1101,7 @@ fn generate_direct_field_accessors(field: &FieldInfo) -> TokenStream { #[doc = ""] #[doc = "Unlike `get_or_create_lazy` for collections, this does NOT allocate"] #[doc = "if the field is absent - it returns None instead."] - fn #get_mut_name(&mut self) -> Option<&mut #field_type> { + #vis fn #get_mut_name(&mut self) -> Option<&mut #field_type> { self.find_lazy_mut(#extractor) } } @@ -1106,19 +1117,24 @@ fn generate_collection_field_accessors( let ref_name = field.ref_ident(); let mut_name = field.mut_ident(); let take_name = field.take_ident(); + let vis = if field.is_pub { + quote! {pub} + } else { + quote! {} + }; if field.is_inline() { // Inline: direct field access quote! { - fn #ref_name(&self) -> &#field_type { + #vis fn #ref_name(&self) -> &#field_type { &self.#field_name } - fn #mut_name(&mut self) -> &mut #field_type { + #vis fn #mut_name(&mut self) -> &mut #field_type { &mut self.#field_name } - fn #take_name(&mut self) -> #field_type { + #vis fn #take_name(&mut self) -> #field_type { std::mem::take(&mut self.#field_name) } } @@ -1130,11 +1146,11 @@ fn generate_collection_field_accessors( let constructor = field.lazy_constructor(quote! { Default::default() }); quote! { - fn #ref_name(&self) -> Option<&#field_type> { + #vis fn #ref_name(&self) -> Option<&#field_type> { self.find_lazy(#extractor) } - fn #mut_name(&mut self) -> &mut #field_type { + #vis fn #mut_name(&mut self) -> &mut #field_type { self.get_or_create_lazy( #matches_closure, #unwrap_closure, @@ -1142,7 +1158,7 @@ fn generate_collection_field_accessors( ) } - fn #take_name(&mut self) -> Option<#field_type> { + #vis fn #take_name(&mut self) -> Option<#field_type> { self.take_lazy( #matches_closure, #unwrap_closure, diff --git a/turbopack/crates/turbo-tasks/src/backend.rs b/turbopack/crates/turbo-tasks/src/backend.rs index ece80e26d317ca..23ddbf215f45f4 100644 --- a/turbopack/crates/turbo-tasks/src/backend.rs +++ b/turbopack/crates/turbo-tasks/src/backend.rs @@ -409,8 +409,6 @@ pub trait Backend: Sync + Send { ) { } - fn get_task_description(&self, task: TaskId) -> String; - fn try_start_task_execution<'a>( &'a self, task: TaskId, diff --git a/turbopack/crates/turbo-tasks/src/event.rs b/turbopack/crates/turbo-tasks/src/event.rs index cc42fc2b1f98ed..8ed124cc1a19c3 100644 --- a/turbopack/crates/turbo-tasks/src/event.rs +++ b/turbopack/crates/turbo-tasks/src/event.rs @@ -1,5 +1,5 @@ use std::{ - fmt::{Debug, Formatter}, + fmt::{Debug, Display, Formatter}, future::Future, mem::replace, pin::Pin, @@ -14,6 +14,58 @@ use std::{ #[cfg(feature = "hanging_detection")] use tokio::time::{Timeout, timeout}; +pub trait EventDescriptor { + #[cfg(feature = "hanging_detection")] + fn get_description(self) -> Arc String + Sync + Send>; +} + +impl EventDescriptor for T +where + T: FnOnce() -> InnerFn, + InnerFn: Fn() -> String + Sync + Send + 'static, +{ + #[cfg(feature = "hanging_detection")] + fn get_description(self) -> Arc String + Sync + Send> { + Arc::new((self)()) + } +} + +#[derive(Clone)] +pub struct EventDescription { + #[cfg(feature = "hanging_detection")] + description: Arc String + Sync + Send>, +} + +impl EventDescription { + #[inline(always)] + pub fn new(#[allow(unused_variables)] description: impl FnOnce() -> InnerFn) -> Self + where + InnerFn: Fn() -> String + Sync + Send + 'static, + { + Self { + #[cfg(feature = "hanging_detection")] + description: Arc::new((description)()), + } + } +} + +impl Display for EventDescription { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + #[cfg(not(feature = "hanging_detection"))] + return write!(f, ""); + + #[cfg(feature = "hanging_detection")] + return write!(f, "{}", (self.description)()); + } +} + +impl EventDescriptor for EventDescription { + #[cfg(feature = "hanging_detection")] + fn get_description(self) -> Arc String + Sync + Send> { + self.description + } +} + pub struct Event { #[cfg(feature = "hanging_detection")] description: Arc String + Sync + Send>, @@ -35,17 +87,14 @@ impl Event { /// The outer closure allows avoiding extra lookups (e.g. task type info) that may be needed to /// capture information needed for constructing (moving into) the inner closure. #[inline(always)] - pub fn new(_description: impl FnOnce() -> InnerFn) -> Self - where - InnerFn: Fn() -> String + Sync + Send + 'static, - { + pub fn new(#[allow(unused_variables)] description: impl EventDescriptor) -> Self { #[cfg(not(feature = "hanging_detection"))] return Self { event: event_listener::Event::new(), }; #[cfg(feature = "hanging_detection")] return Self { - description: Arc::new((_description)()), + description: description.get_description(), event: event_listener::Event::new(), }; } @@ -80,10 +129,7 @@ impl Event { /// /// The outer closure allow avoiding extra lookups (e.g. task type info) that may be needed to /// capture information needed for constructing (moving into) the inner closure. - pub fn listen_with_note(&self, _note: impl FnOnce() -> InnerFn) -> EventListener - where - InnerFn: Fn() -> String + Sync + Send + 'static, - { + pub fn listen_with_note(&self, _note: impl EventDescriptor) -> EventListener { #[cfg(not(feature = "hanging_detection"))] return EventListener { listener: self.event.listen(), @@ -91,7 +137,7 @@ impl Event { #[cfg(feature = "hanging_detection")] return EventListener { description: self.description.clone(), - note: Arc::new((_note)()), + note: _note.get_description(), future: Some(Box::pin(timeout( Duration::from_secs(30), self.event.listen(), From cedfb53ae241af4540c8597a6c7ffdeb7e9fdf80 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 23 Jan 2026 09:34:57 +0100 Subject: [PATCH 02/10] Turbopack: change invalidator and immutable to data category (#88889) ### What? save a few bits --- turbopack/crates/turbo-tasks-backend/src/backend/mod.rs | 2 +- .../crates/turbo-tasks-backend/src/backend/storage_schema.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index cedd4969b51d55..588ff2fc685c01 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -2091,7 +2091,7 @@ impl TurboTasksBackendInner { if let Some(dependencies) = task_dependencies_for_immutable && dependencies .iter() - .all(|&task_id| ctx.task(task_id, TaskDataCategory::Meta).immutable()) + .all(|&task_id| ctx.task(task_id, TaskDataCategory::Data).immutable()) { is_now_immutable = true; } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs index 13368617b40b04..a9edd0b7c92371 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs @@ -133,11 +133,11 @@ struct TaskStorageSchema { // Persisted flags come first, then transient flags. // ========================================================================= /// Whether the task has an invalidator. - #[field(storage = "flag", category = "meta")] + #[field(storage = "flag", category = "data")] invalidator: bool, /// Whether the task output is immutable (persisted). - #[field(storage = "flag", category = "meta")] + #[field(storage = "flag", category = "data")] immutable: bool, /// Whether clean in current session (transient flag). From 2e2dd02c120ebe62c154043acb1fdbd8537dd1b7 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 23 Jan 2026 09:49:06 +0100 Subject: [PATCH 03/10] Turbopack: reduce cache size (#88929) ### What? do not persist EcmascriptChunkItemContent It's already stored as part of the module factory --- turbopack/crates/turbopack-ecmascript/src/chunk/item.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/turbopack/crates/turbopack-ecmascript/src/chunk/item.rs b/turbopack/crates/turbopack-ecmascript/src/chunk/item.rs index 410d9ceb058ec7..20bff5b4810ed3 100644 --- a/turbopack/crates/turbopack-ecmascript/src/chunk/item.rs +++ b/turbopack/crates/turbopack-ecmascript/src/chunk/item.rs @@ -47,7 +47,9 @@ pub enum RewriteSourcePath { None, } -#[turbo_tasks::value(shared)] +// Note we don't want to persist this as `module_factory_with_code_generation_issue` is already +// persisted and we want to avoid duplicating it. +#[turbo_tasks::value(shared, serialization = "none")] #[derive(Default, Clone)] pub struct EcmascriptChunkItemContent { pub inner_code: Rope, From c582bf8eb2e1fabb020c940f360562b70a26c6c2 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 23 Jan 2026 09:52:09 +0100 Subject: [PATCH 04/10] fix typo (#88934) typo --- turbopack/crates/turbo-tasks-backend/src/backend/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 588ff2fc685c01..819bc98664ba54 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -1347,7 +1347,7 @@ impl TurboTasksBackendInner { .flatten() } { // Task exists in backing storage - // So we only need to insert it into the in-memory cahce + // So we only need to insert it into the in-memory cache self.track_cache_hit(&task_type); let task_type = match raw_entry(&self.task_cache, &task_type) { RawEntry::Occupied(_) => ArcOrOwned::Owned(task_type), From 13c47b3ff46ec4dc323130b7b5beec4b1df9121b Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 23 Jan 2026 10:14:06 +0100 Subject: [PATCH 05/10] Turbopack: improve module type error message (#88815) ### What? improve module type error message --- turbopack/crates/turbopack/src/lib.rs | 8 ++++--- .../src/module_options/module_rule.rs | 23 +++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/turbopack/crates/turbopack/src/lib.rs b/turbopack/crates/turbopack/src/lib.rs index e938ba2aba9a99..765d3d9710dfa1 100644 --- a/turbopack/crates/turbopack/src/lib.rs +++ b/turbopack/crates/turbopack/src/lib.rs @@ -674,10 +674,12 @@ async fn process_default_internal( ModuleIssue::new( *ident, rcstr!("Invalid module type"), - rcstr!( + format!( "The module type must be Ecmascript or Typescript to add \ - Ecmascript transforms" - ), + Ecmascript transforms (got {})", + module_type + ) + .into(), Some(IssueSource::from_source_only(current_source)), ) .to_resolved() diff --git a/turbopack/crates/turbopack/src/module_options/module_rule.rs b/turbopack/crates/turbopack/src/module_options/module_rule.rs index 0f5ddaecb340be..cc166015dcc91a 100644 --- a/turbopack/crates/turbopack/src/module_options/module_rule.rs +++ b/turbopack/crates/turbopack/src/module_options/module_rule.rs @@ -1,3 +1,5 @@ +use std::fmt::Display; + use anyhow::Result; use bincode::{Decode, Encode}; use turbo_rcstr::RcStr; @@ -153,3 +155,24 @@ pub enum ModuleType { }, Custom(ResolvedVc>), } + +impl Display for ModuleType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ModuleType::Ecmascript { .. } => write!(f, "Ecmascript"), + ModuleType::Typescript { .. } => write!(f, "Typescript"), + ModuleType::TypescriptDeclaration { .. } => write!(f, "TypescriptDeclaration"), + ModuleType::EcmascriptExtensionless { .. } => write!(f, "EcmascriptExtensionless"), + ModuleType::Json => write!(f, "Json"), + ModuleType::Raw => write!(f, "Raw"), + ModuleType::NodeAddon => write!(f, "NodeAddon"), + ModuleType::CssModule => write!(f, "CssModule"), + ModuleType::Css { .. } => write!(f, "Css"), + ModuleType::StaticUrlJs { .. } => write!(f, "StaticUrlJs"), + ModuleType::StaticUrlCss { .. } => write!(f, "StaticUrlCss"), + ModuleType::InlinedBytesJs => write!(f, "InlinedBytesJs"), + ModuleType::WebAssembly { .. } => write!(f, "WebAssembly"), + ModuleType::Custom(_) => write!(f, "Custom"), + } + } +} From 088bbd8d9bc33b10431ec8480752ec7e39e210ae Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 23 Jan 2026 10:56:44 +0100 Subject: [PATCH 06/10] Turbopack: improve selective read support to allow `Equivalent` keys (#88760) ### What? Allow keys for selective reads to be something equivalent to the key instead of only exactly a reference of the key. --- turbopack/crates/turbo-tasks/src/keyed.rs | 71 ++++++++++++------- turbopack/crates/turbo-tasks/src/manager.rs | 10 +-- .../crates/turbo-tasks/src/mapped_read_ref.rs | 2 - .../crates/turbo-tasks/src/vc/cell_mode.rs | 6 +- turbopack/crates/turbo-tasks/src/vc/mod.rs | 20 +++--- turbopack/crates/turbo-tasks/src/vc/read.rs | 48 ++++++++----- 6 files changed, 95 insertions(+), 62 deletions(-) diff --git a/turbopack/crates/turbo-tasks/src/keyed.rs b/turbopack/crates/turbo-tasks/src/keyed.rs index 3824f7d9254f00..e792bddae6330c 100644 --- a/turbopack/crates/turbo-tasks/src/keyed.rs +++ b/turbopack/crates/turbo-tasks/src/keyed.rs @@ -1,31 +1,35 @@ use std::{ + borrow::Borrow, collections::{HashMap, HashSet}, hash::{BuildHasher, Hash}, }; -use indexmap::{IndexMap, IndexSet}; +use indexmap::{Equivalent, IndexMap, IndexSet}; use smallvec::SmallVec; -pub trait Keyed { +pub trait KeyedEq { type Key; - type Value; fn different_keys<'l>(&'l self, other: &'l Self) -> SmallVec<[&'l Self::Key; 2]>; - fn get(&self, key: &Self::Key) -> Option<&Self::Value>; - fn contains_key(&self, key: &Self::Key) -> bool { +} + +pub trait KeyedAccess { + type Value; + + fn get(&self, key: &Q) -> Option<&Self::Value>; + fn contains_key(&self, key: &Q) -> bool { self.get(key).is_some() } } -impl Keyed for HashMap { +impl KeyedEq for HashMap { type Key = K; - type Value = V; fn different_keys<'l>(&'l self, other: &'l Self) -> SmallVec<[&'l Self::Key; 2]> { let mut different_keys = SmallVec::new(); for (key, value) in self.iter() { - if let Some(other_value) = other.get(key) { + if let Some(other_value) = HashMap::get(other, key) { if value != other_value { different_keys.push(key); } @@ -44,19 +48,24 @@ impl Keyed for HashMap { different_keys } +} - fn get(&self, key: &Self::Key) -> Option<&Self::Value> { - self.get(key) +impl, V, H: BuildHasher, Q: Eq + Hash + ?Sized> KeyedAccess + for HashMap +{ + type Value = V; + + fn get(&self, key: &Q) -> Option<&Self::Value> { + HashMap::get(self, key) } - fn contains_key(&self, key: &Self::Key) -> bool { - self.contains_key(key) + fn contains_key(&self, key: &Q) -> bool { + HashMap::contains_key(self, key) } } -impl Keyed for IndexMap { +impl KeyedEq for IndexMap { type Key = K; - type Value = V; fn different_keys<'l>(&'l self, other: &'l Self) -> SmallVec<[&'l Self::Key; 2]> { let mut different_keys = SmallVec::new(); @@ -81,19 +90,22 @@ impl Keyed for IndexMap { different_keys } +} + +impl + Hash + ?Sized> KeyedAccess for IndexMap { + type Value = V; - fn get(&self, key: &Self::Key) -> Option<&Self::Value> { + fn get(&self, key: &Q) -> Option<&Self::Value> { self.get(key) } - fn contains_key(&self, key: &Self::Key) -> bool { + fn contains_key(&self, key: &Q) -> bool { self.contains_key(key) } } -impl Keyed for HashSet { +impl KeyedEq for HashSet { type Key = K; - type Value = (); fn different_keys<'l>(&'l self, other: &'l Self) -> SmallVec<[&'l Self::Key; 2]> { let mut different_keys = SmallVec::new(); @@ -114,19 +126,24 @@ impl Keyed for HashSet { different_keys } +} - fn get(&self, key: &Self::Key) -> Option<&Self::Value> { +impl, Q: Eq + Hash + ?Sized, H: BuildHasher> KeyedAccess + for HashSet +{ + type Value = (); + + fn get(&self, key: &Q) -> Option<&Self::Value> { if self.contains(key) { Some(&()) } else { None } } - fn contains_key(&self, key: &Self::Key) -> bool { + fn contains_key(&self, key: &Q) -> bool { self.contains(key) } } -impl Keyed for IndexSet { +impl KeyedEq for IndexSet { type Key = K; - type Value = (); fn different_keys<'l>(&'l self, other: &'l Self) -> SmallVec<[&'l Self::Key; 2]> { let mut different_keys = SmallVec::new(); @@ -147,12 +164,16 @@ impl Keyed for IndexSet { different_keys } +} + +impl + Hash + ?Sized, H: BuildHasher> KeyedAccess for IndexSet { + type Value = (); - fn get(&self, key: &Self::Key) -> Option<&Self::Value> { + fn get(&self, key: &Q) -> Option<&Self::Value> { if self.contains(key) { Some(&()) } else { None } } - fn contains_key(&self, key: &Self::Key) -> bool { + fn contains_key(&self, key: &Q) -> bool { self.contains(key) } } @@ -181,7 +202,7 @@ mod tests { [("e", 5), ("d", 4), ("c", 3), ("b", 2), ("a", 1)] } - fn assert_diff(a: &T, b: &T, expected: &[&T::Key]) + fn assert_diff(a: &T, b: &T, expected: &[&T::Key]) where T::Key: std::fmt::Debug + PartialEq, { diff --git a/turbopack/crates/turbo-tasks/src/manager.rs b/turbopack/crates/turbo-tasks/src/manager.rs index 1f176a34532aa6..c4c87f0dfd0422 100644 --- a/turbopack/crates/turbo-tasks/src/manager.rs +++ b/turbopack/crates/turbo-tasks/src/manager.rs @@ -35,7 +35,7 @@ use crate::{ event::{Event, EventListener}, id::{ExecutionId, LocalTaskId, TRANSIENT_TASK_BIT, TraitTypeId}, id_factory::IdFactoryWithReuse, - keyed::Keyed, + keyed::KeyedEq, macro_helpers::NativeFunction, magic_any::MagicAny, message_queue::{CompilationEvent, CompilationEventQueue}, @@ -2040,8 +2040,8 @@ impl CurrentCellRef { pub fn keyed_compare_and_update(&self, new_value: T) where T: PartialEq + VcValueType, - VcReadTarget: Keyed, - as Keyed>::Key: std::hash::Hash, + VcReadTarget: KeyedEq, + as KeyedEq>::Key: std::hash::Hash, { self.conditional_update(|old_value| { let Some(old_value) = old_value else { @@ -2069,8 +2069,8 @@ impl CurrentCellRef { new_shared_reference: SharedReference, ) where T: VcValueType + PartialEq, - VcReadTarget: Keyed, - as Keyed>::Key: std::hash::Hash, + VcReadTarget: KeyedEq, + as KeyedEq>::Key: std::hash::Hash, { self.conditional_update_with_shared_reference(|old_sr| { let Some(old_sr) = old_sr else { diff --git a/turbopack/crates/turbo-tasks/src/mapped_read_ref.rs b/turbopack/crates/turbo-tasks/src/mapped_read_ref.rs index 84fd99da97a617..80a0911c40b37d 100644 --- a/turbopack/crates/turbo-tasks/src/mapped_read_ref.rs +++ b/turbopack/crates/turbo-tasks/src/mapped_read_ref.rs @@ -4,7 +4,6 @@ use serde::Serialize; use crate::{ debug::{ValueDebugFormat, ValueDebugFormatString}, - keyed::Keyed, trace::{TraceRawVcs, TraceRawVcsContext}, }; @@ -128,7 +127,6 @@ where impl> IntoIterator for &MappedReadRef where - T: Keyed, for<'b> &'b T: IntoIterator, { type Item = I; diff --git a/turbopack/crates/turbo-tasks/src/vc/cell_mode.rs b/turbopack/crates/turbo-tasks/src/vc/cell_mode.rs index 4faba1e82a1230..7991aaea1ae382 100644 --- a/turbopack/crates/turbo-tasks/src/vc/cell_mode.rs +++ b/turbopack/crates/turbo-tasks/src/vc/cell_mode.rs @@ -2,7 +2,7 @@ use std::{any::type_name, marker::PhantomData}; use super::{read::VcRead, traits::VcValueType}; use crate::{ - RawVc, Vc, backend::VerificationMode, keyed::Keyed, manager::find_cell_by_type, + RawVc, Vc, backend::VerificationMode, keyed::KeyedEq, manager::find_cell_by_type, task::shared_reference::TypedSharedReference, }; @@ -93,8 +93,8 @@ pub struct VcCellKeyedCompareMode { impl VcCellMode for VcCellKeyedCompareMode where T: VcValueType + PartialEq, - VcReadTarget: Keyed, - as Keyed>::Key: std::hash::Hash, + VcReadTarget: KeyedEq, + as KeyedEq>::Key: std::hash::Hash, { fn cell(inner: VcReadTarget) -> Vc { let cell = find_cell_by_type::(); diff --git a/turbopack/crates/turbo-tasks/src/vc/mod.rs b/turbopack/crates/turbo-tasks/src/vc/mod.rs index 52e4f4ba9c7691..10565b3277b8c5 100644 --- a/turbopack/crates/turbo-tasks/src/vc/mod.rs +++ b/turbopack/crates/turbo-tasks/src/vc/mod.rs @@ -34,7 +34,7 @@ pub use self::{ use crate::{ CellId, RawVc, ResolveTypeError, debug::{ValueDebug, ValueDebugFormat, ValueDebugFormatString}, - keyed::Keyed, + keyed::{KeyedAccess, KeyedEq}, registry, trace::{TraceRawVcs, TraceRawVcsContext}, vc::read::{ReadContainsKeyedVcFuture, ReadKeyedVcFuture}, @@ -674,22 +674,26 @@ where impl Vc where T: VcValueType, - VcReadTarget: Keyed, - as Keyed>::Key: Hash, + VcReadTarget: KeyedEq, { /// Read the value and selects a keyed value from it. Only depends on the used key instead of /// the full value. - pub fn get<'l>(self, key: &'l as Keyed>::Key) -> ReadKeyedVcFuture<'l, T> { + pub fn get<'l, Q>(self, key: &'l Q) -> ReadKeyedVcFuture<'l, T, Q> + where + Q: Hash + ?Sized, + VcReadTarget: KeyedAccess, + { let future: ReadVcFuture = self.node.into_read(T::has_serialization()).into(); future.get(key) } /// Read the value and checks if it contains the given key. Only depends on the used key instead /// of the full value. - pub fn contains_key<'l>( - self, - key: &'l as Keyed>::Key, - ) -> ReadContainsKeyedVcFuture<'l, T> { + pub fn contains_key<'l, Q>(self, key: &'l Q) -> ReadContainsKeyedVcFuture<'l, T, Q> + where + Q: Hash + ?Sized, + VcReadTarget: KeyedAccess, + { let future: ReadVcFuture = self.node.into_read(T::has_serialization()).into(); future.contains_key(key) } diff --git a/turbopack/crates/turbo-tasks/src/vc/read.rs b/turbopack/crates/turbo-tasks/src/vc/read.rs index 5c9de09ae171e1..3be5c6df2e36c6 100644 --- a/turbopack/crates/turbo-tasks/src/vc/read.rs +++ b/turbopack/crates/turbo-tasks/src/vc/read.rs @@ -15,7 +15,8 @@ use rustc_hash::FxBuildHasher; use super::traits::VcValueType; use crate::{ MappedReadRef, ReadRawVcFuture, ReadRef, VcCast, VcValueTrait, VcValueTraitCast, - VcValueTypeCast, keyed::Keyed, + VcValueTypeCast, + keyed::{KeyedAccess, KeyedEq}, }; type VcReadTarget = <::Read as VcRead>::Target; @@ -193,12 +194,15 @@ where impl ReadVcFuture> where T: VcValueType, - VcReadTarget: Keyed, - as Keyed>::Key: Hash, + VcReadTarget: KeyedEq, { /// Read the value and selects a keyed value from it. Only depends on the used key instead of /// the full value. - pub fn get<'l>(mut self, key: &'l as Keyed>::Key) -> ReadKeyedVcFuture<'l, T> { + pub fn get<'l, Q>(mut self, key: &'l Q) -> ReadKeyedVcFuture<'l, T, Q> + where + Q: Hash + ?Sized, + VcReadTarget: KeyedAccess, + { self.raw = self.raw.track_with_key(FxBuildHasher.hash_one(key)); ReadKeyedVcFuture { future: self, key } } @@ -208,10 +212,11 @@ where /// /// Note: This is also invalidated when the value of the key changes, not only when the presence /// of the key changes. - pub fn contains_key<'l>( - mut self, - key: &'l as Keyed>::Key, - ) -> ReadContainsKeyedVcFuture<'l, T> { + pub fn contains_key<'l, Q>(mut self, key: &'l Q) -> ReadContainsKeyedVcFuture<'l, T, Q> + where + Q: Hash + ?Sized, + VcReadTarget: KeyedAccess, + { self.raw = self.raw.track_with_key(FxBuildHasher.hash_one(key)); ReadContainsKeyedVcFuture { future: self, key } } @@ -284,23 +289,26 @@ where } pin_project! { - pub struct ReadKeyedVcFuture<'l, T> + pub struct ReadKeyedVcFuture<'l, T, Q> where T: VcValueType, - VcReadTarget: Keyed, + Q: ?Sized, + VcReadTarget: KeyedAccess, + { #[pin] future: ReadVcFuture>, - key: &'l as Keyed>::Key, + key: &'l Q, } } -impl<'l, T> Future for ReadKeyedVcFuture<'l, T> +impl<'l, T, Q> Future for ReadKeyedVcFuture<'l, T, Q> where T: VcValueType, - VcReadTarget: Keyed, + Q: ?Sized, + VcReadTarget: KeyedAccess, { - type Output = Result as Keyed>::Value>>>; + type Output = Result as KeyedAccess>::Value>>>; fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { // Safety: We never move the contents of `self` @@ -322,21 +330,23 @@ where } pin_project! { - pub struct ReadContainsKeyedVcFuture<'l, T> + pub struct ReadContainsKeyedVcFuture<'l, T, Q> where T: VcValueType, - VcReadTarget: Keyed, + Q: ?Sized, + VcReadTarget: KeyedAccess, { #[pin] future: ReadVcFuture>, - key: &'l as Keyed>::Key, + key: &'l Q, } } -impl<'l, T> Future for ReadContainsKeyedVcFuture<'l, T> +impl<'l, T, Q> Future for ReadContainsKeyedVcFuture<'l, T, Q> where T: VcValueType, - VcReadTarget: Keyed, + Q: ?Sized, + VcReadTarget: KeyedAccess, { type Output = Result; From ea81858027f401aefe5adc5b182a7c80f6748657 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 23 Jan 2026 11:27:36 +0100 Subject: [PATCH 07/10] Turbopack: add indirection layer for better caching during resolving (#80062) ### What? Cache the before plugin matching --- .../crates/turbopack-core/src/resolve/mod.rs | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/turbopack/crates/turbopack-core/src/resolve/mod.rs b/turbopack/crates/turbopack-core/src/resolve/mod.rs index c531ca48df6e88..8fa0f5e0423170 100644 --- a/turbopack/crates/turbopack-core/src/resolve/mod.rs +++ b/turbopack/crates/turbopack-core/src/resolve/mod.rs @@ -1700,6 +1700,24 @@ pub async fn url_resolve( .await } +#[turbo_tasks::value(transparent)] +struct MatchingBeforeResolvePlugins(Vec>>); + +#[turbo_tasks::function] +async fn get_matching_before_resolve_plugins( + options: Vc, + request: Vc, +) -> Result> { + let mut matching_plugins = Vec::new(); + for &plugin in &options.await?.before_resolve_plugins { + let condition = plugin.before_resolve_condition().resolve().await?; + if *condition.matches(request).await? { + matching_plugins.push(plugin); + } + } + Ok(Vc::cell(matching_plugins)) +} + #[tracing::instrument(level = "trace", skip_all)] async fn handle_before_resolve_plugins( lookup_path: FileSystemPath, @@ -1707,14 +1725,7 @@ async fn handle_before_resolve_plugins( request: Vc, options: Vc, ) -> Result>> { - let options_value = options.await?; - - for plugin in &options_value.before_resolve_plugins { - let condition = plugin.before_resolve_condition().resolve().await?; - if !*condition.matches(request).await? { - continue; - } - + for plugin in get_matching_before_resolve_plugins(options, request).await? { if let Some(result) = *plugin .before_resolve(lookup_path.clone(), reference_type.clone(), request) .await? From b08e16b21a2d2a634134469977c58c6a628d1f53 Mon Sep 17 00:00:00 2001 From: "Sebastian \"Sebbie\" Silbermann" Date: Fri, 23 Jan 2026 12:22:37 +0100 Subject: [PATCH 08/10] Revert "[prebuilt-skew-protection] feat: adding in automatic deploymentId" (#88942) --- errors/deploymentid-invalid-characters.mdx | 71 ---- errors/deploymentid-not-a-string.mdx | 34 -- errors/deploymentid-too-long.mdx | 51 --- packages/next/errors.json | 6 +- .../next/src/build/adapter/build-complete.ts | 8 - packages/next/src/build/analyze/index.ts | 6 +- packages/next/src/build/build-context.ts | 2 - packages/next/src/build/define-env.ts | 4 +- .../next/src/build/generate-deployment-id.ts | 99 ------ .../src/build/generate-routes-manifest.ts | 3 - packages/next/src/build/index.ts | 85 +---- packages/next/src/build/swc/index.ts | 4 - .../next/src/build/turbopack-build/impl.ts | 9 +- packages/next/src/build/webpack-config.ts | 12 +- packages/next/src/export/index.ts | 3 +- packages/next/src/lib/inline-static-env.ts | 7 +- packages/next/src/server/base-server.ts | 3 +- packages/next/src/server/config-shared.ts | 3 +- packages/next/src/server/config.ts | 18 +- .../src/server/dev/hot-reloader-turbopack.ts | 10 +- .../next/src/server/evaluate-deployment-id.ts | 29 -- .../src/server/route-modules/route-module.ts | 7 +- .../shared/lib/turbopack/manifest-loader.ts | 9 - .../deployment-id-function.test.ts | 328 ------------------ .../tsconfig.json | 8 +- .../deployment-id-handling/app/next.config.js | 9 +- .../deterministic-build/deployment-id.test.ts | 19 - test/unit/generate-deployment-id.test.ts | 263 -------------- 28 files changed, 39 insertions(+), 1071 deletions(-) delete mode 100644 errors/deploymentid-invalid-characters.mdx delete mode 100644 errors/deploymentid-not-a-string.mdx delete mode 100644 errors/deploymentid-too-long.mdx delete mode 100644 packages/next/src/build/generate-deployment-id.ts delete mode 100644 packages/next/src/server/evaluate-deployment-id.ts delete mode 100644 test/e2e/deployment-id-function/deployment-id-function.test.ts delete mode 100644 test/unit/generate-deployment-id.test.ts diff --git a/errors/deploymentid-invalid-characters.mdx b/errors/deploymentid-invalid-characters.mdx deleted file mode 100644 index 1aa52e4cb67aa6..00000000000000 --- a/errors/deploymentid-invalid-characters.mdx +++ /dev/null @@ -1,71 +0,0 @@ ---- -title: '`deploymentId` contains invalid characters' ---- - -## Why This Error Occurred - -The `deploymentId` in your `next.config.js` contains characters that are not allowed. Only alphanumeric characters (a-z, A-Z, 0-9), hyphens (-), and underscores (\_) are permitted. - -## Possible Ways to Fix It - -### Option 1: Remove Invalid Characters - -Remove or replace any characters that are not alphanumeric, hyphens, or underscores: - -```js -// ✅ Correct -module.exports = { - deploymentId: 'my-deployment-123', // Only alphanumeric, hyphens, underscores -} - -// ❌ Incorrect -module.exports = { - deploymentId: 'my deployment 123', // Contains spaces - deploymentId: 'my.deployment.123', // Contains dots - deploymentId: 'my/deployment/123', // Contains slashes - deploymentId: 'my@deployment#123', // Contains special characters -} -``` - -### Option 2: Sanitize the Deployment ID - -If you're generating the ID from user input or other sources, sanitize it to remove invalid characters: - -```js -// next.config.js -module.exports = { - deploymentId: () => { - const rawId = process.env.DEPLOYMENT_ID || 'default-id' - // Remove all characters that are not alphanumeric, hyphens, or underscores - return rawId.replace(/[^a-zA-Z0-9_-]/g, '') - }, -} -``` - -### Option 3: Use a Valid Format - -Common valid formats include: - -```js -// next.config.js -module.exports = { - // Using hyphens - deploymentId: 'my-deployment-id', - - // Using underscores - deploymentId: 'my_deployment_id', - - // Alphanumeric only - deploymentId: 'mydeployment123', - - // Mixed format - deploymentId: 'my-deployment_123', -} -``` - -## Additional Information - -- The deployment ID is used for skew protection and asset versioning -- Invalid characters can cause issues with URL encoding and routing -- Keep the ID URL-friendly by using only the allowed character set -- The validation ensures compatibility across different systems and environments diff --git a/errors/deploymentid-not-a-string.mdx b/errors/deploymentid-not-a-string.mdx deleted file mode 100644 index 277ac409b42ff3..00000000000000 --- a/errors/deploymentid-not-a-string.mdx +++ /dev/null @@ -1,34 +0,0 @@ ---- -title: '`deploymentId` function must return a string' ---- - -## Why This Error Occurred - -The `deploymentId` option in your `next.config.js` is defined as a function, but it did not return a string value. - -## Possible Ways to Fix It - -Always return a string from your `deploymentId` function: - -```js -// ✅ Correct -module.exports = { - deploymentId: () => { - return process.env.GIT_HASH || Date.now().toString() - }, -} - -// ❌ Incorrect -module.exports = { - deploymentId: () => { - // Missing return statement or returning non-string - return null - }, -} -``` - -The `deploymentId` can be: - -- A string: `deploymentId: 'my-deployment-123'` -- A function that returns a string: `deploymentId: () => process.env.GIT_HASH || ''` -- `undefined` (will be empty string, or use `NEXT_DEPLOYMENT_ID` environment variable if set) diff --git a/errors/deploymentid-too-long.mdx b/errors/deploymentid-too-long.mdx deleted file mode 100644 index 8deb3901ea6c59..00000000000000 --- a/errors/deploymentid-too-long.mdx +++ /dev/null @@ -1,51 +0,0 @@ -# Deployment ID Too Long - -The `deploymentId` in your `next.config.js` exceeds the maximum length of 32 characters. - -## Why This Error Occurred - -The `deploymentId` configuration option has a maximum length of 32 characters to ensure compatibility with various systems and constraints. - -## Possible Ways to Fix It - -### Option 1: Shorten Your Deployment ID - -Reduce the length of your `deploymentId` to 32 characters or less: - -```js -// next.config.js -module.exports = { - deploymentId: 'my-short-id', // ✅ 12 characters -} -``` - -### Option 2: Use a Function to Generate a Shorter ID - -If you're generating the ID dynamically, ensure it doesn't exceed 32 characters: - -```js -// next.config.js -module.exports = { - deploymentId: () => { - // Generate a shorter ID (e.g., hash or truncate) - return process.env.GIT_COMMIT_SHA?.substring(0, 32) || 'default-id' - }, -} -``` - -### Option 3: Truncate Environment Variables - -If using environment variables, ensure the value is truncated to 32 characters: - -```js -// next.config.js -module.exports = { - deploymentId: process.env.DEPLOYMENT_ID?.substring(0, 32), -} -``` - -## Additional Information - -- The deployment ID is used for skew protection and asset versioning -- Keep it concise but meaningful for your use case -- Consider using hashes or shortened identifiers if you need unique values diff --git a/packages/next/errors.json b/packages/next/errors.json index d56ec2ec58fc6a..32c6553b435829 100644 --- a/packages/next/errors.json +++ b/packages/next/errors.json @@ -980,9 +980,5 @@ "979": "invariant: expected %s bytes of postponed state but only received %s bytes", "980": "Failed to load client middleware manifest", "981": "resolvedPathname must be set in request metadata", - "982": "`serializeResumeDataCache` should not be called in edge runtime.", - "983": "deploymentId function must return a string. https://nextjs.org/docs/messages/deploymentid-not-a-string", - "984": "The deploymentId \"%s\" cannot start with the \"dpl_\" prefix. Please choose a different deploymentId in your next.config.js. https://vercel.com/docs/skew-protection#custom-skew-protection-deployment-id", - "985": "The deploymentId \"%s\" exceeds the maximum length of 32 characters. Please choose a shorter deploymentId. https://nextjs.org/docs/messages/deploymentid-too-long", - "986": "The deploymentId \"%s\" contains invalid characters. Only alphanumeric characters (a-z, A-Z, 0-9), hyphens (-), and underscores (_) are allowed. https://nextjs.org/docs/messages/deploymentid-invalid-characters" + "982": "`serializeResumeDataCache` should not be called in edge runtime." } diff --git a/packages/next/src/build/adapter/build-complete.ts b/packages/next/src/build/adapter/build-complete.ts index 173efaaab9a505..1af503c507551d 100644 --- a/packages/next/src/build/adapter/build-complete.ts +++ b/packages/next/src/build/adapter/build-complete.ts @@ -386,11 +386,6 @@ export interface NextAdapter { * influenced by NextConfig.generateBuildId */ buildId: string - /** - * deploymentId is the current deployment ID, this can be - * influenced by NextConfig.deploymentId or NEXT_DEPLOYMENT_ID environment variable - */ - deploymentId: string }) => Promise | void } @@ -420,7 +415,6 @@ export async function handleBuildComplete({ config, appType, buildId, - deploymentId, configOutDir, distDir, pageKeys, @@ -444,7 +438,6 @@ export async function handleBuildComplete({ appType: 'app' | 'pages' | 'hybrid' distDir: string buildId: string - deploymentId: string configOutDir: string adapterPath: string tracingRoot: string @@ -1863,7 +1856,6 @@ export async function handleBuildComplete({ config, distDir, buildId, - deploymentId, nextVersion, projectDir: dir, repoRoot: tracingRoot, diff --git a/packages/next/src/build/analyze/index.ts b/packages/next/src/build/analyze/index.ts index bccff24fea6b81..bc9b099743544e 100644 --- a/packages/next/src/build/analyze/index.ts +++ b/packages/next/src/build/analyze/index.ts @@ -21,7 +21,6 @@ import loadCustomRoutes from '../../lib/load-custom-routes' import { generateRoutesManifest } from '../generate-routes-manifest' import { checkIsAppPPREnabled } from '../../server/lib/experimental/ppr' import { normalizeAppPath } from '../../shared/lib/router/utils/app-paths' -import { resolveAndSetDeploymentId } from '../generate-deployment-id' import http from 'node:http' // @ts-expect-error types are in @types/serve-handler @@ -54,10 +53,7 @@ export default async function analyze({ reactProductionProfiling, }) - config.deploymentId = resolveAndSetDeploymentId( - config.deploymentId, - config.deploymentId != null ? 'user-config' : 'env-var' - ) + process.env.NEXT_DEPLOYMENT_ID = config.deploymentId || '' const distDir = path.join(dir, '.next') const telemetry = new Telemetry({ distDir }) diff --git a/packages/next/src/build/build-context.ts b/packages/next/src/build/build-context.ts index 8710de0f7ed90d..0eff06e1194cb7 100644 --- a/packages/next/src/build/build-context.ts +++ b/packages/next/src/build/build-context.ts @@ -49,7 +49,6 @@ export interface MappedPages { // to pass it through function arguments. // Not exhaustive, but should be extended to as needed whilst refactoring export const NextBuildContext: Partial<{ - deploymentId?: string | undefined compilerIdx?: number pluginState: Record // core fields @@ -97,7 +96,6 @@ export const NextBuildContext: Partial<{ isCompileMode?: boolean debugPrerender: boolean analyze: boolean - preservedDeploymentId?: string debugBuildPaths?: { app: string[] pages: string[] diff --git a/packages/next/src/build/define-env.ts b/packages/next/src/build/define-env.ts index 786089eb21c426..a2cafd4d8260b7 100644 --- a/packages/next/src/build/define-env.ts +++ b/packages/next/src/build/define-env.ts @@ -46,7 +46,6 @@ const DEFINE_ENV_EXPRESSION = Symbol('DEFINE_ENV_EXPRESSION') interface DefineEnv { [key: string]: | string - | (() => string) | string[] | boolean | { [DEFINE_ENV_EXPRESSION]: string } @@ -171,8 +170,7 @@ export function getDefineEnv({ 'process.env.__NEXT_CACHE_COMPONENTS': isCacheComponentsEnabled, 'process.env.__NEXT_USE_CACHE': isUseCacheEnabled, - ...(config.experimental?.useSkewCookie || - (!config.deploymentId && !config.experimental?.runtimeServerDeploymentId) + ...(config.experimental?.useSkewCookie || !config.deploymentId ? { 'process.env.NEXT_DEPLOYMENT_ID': false, } diff --git a/packages/next/src/build/generate-deployment-id.ts b/packages/next/src/build/generate-deployment-id.ts deleted file mode 100644 index 8ba1af85774832..00000000000000 --- a/packages/next/src/build/generate-deployment-id.ts +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Generates a deployment ID from a user-provided function or string. - * Similar to generateBuildId, but for deploymentId. - */ -export function generateDeploymentId( - deploymentId: string | (() => string) | undefined -): string | undefined { - if (typeof deploymentId === 'function') { - const result = deploymentId() - if (typeof result !== 'string') { - throw new Error( - 'deploymentId function must return a string. https://nextjs.org/docs/messages/deploymentid-not-a-string' - ) - } - return result - } - - if (typeof deploymentId === 'string') { - return deploymentId - } - - return undefined -} - -type DeploymentIdSource = 'user-config' | 'env-var' - -/** - * Resolves and sets the deployment ID from config, handling precedence and ensuring function is only evaluated once. - * User-configured deploymentId always takes precedence over NEXT_DEPLOYMENT_ID. - * - * @param configDeploymentId - The deploymentId from config (can be string, function, or undefined) - * @param source - Source indicator: 'user-config' treats as user-configured (validates), 'env-var' uses NEXT_DEPLOYMENT_ID - * @param fallbackDeploymentId - Optional fallback deployment ID to use if process.env.NEXT_DEPLOYMENT_ID is empty - * @returns The resolved deploymentId string to use - */ -export function resolveAndSetDeploymentId( - configDeploymentId: string | (() => string) | undefined, - source: DeploymentIdSource, - fallbackDeploymentId?: string -): string { - if (source === 'env-var') { - // Prefer fallbackDeploymentId (from combinedEnv) over process.env since - // loadEnvConfig may have reset process.env - let envDeploymentId = - fallbackDeploymentId || process.env['NEXT_DEPLOYMENT_ID'] || '' - - if ( - envDeploymentId && - envDeploymentId !== process.env['NEXT_DEPLOYMENT_ID'] - ) { - process.env['NEXT_DEPLOYMENT_ID'] = envDeploymentId - } - if (envDeploymentId.length > 0) { - if (envDeploymentId.length > 32) { - throw new Error( - `The deploymentId "${envDeploymentId}" exceeds the maximum length of 32 characters. Please choose a shorter deploymentId. https://nextjs.org/docs/messages/deploymentid-too-long` - ) - } - const validCharacterPattern = /^[a-zA-Z0-9_-]+$/ - if (!validCharacterPattern.test(envDeploymentId)) { - throw new Error( - `The deploymentId "${envDeploymentId}" contains invalid characters. Only alphanumeric characters (a-z, A-Z, 0-9), hyphens (-), and underscores (_) are allowed. https://nextjs.org/docs/messages/deploymentid-invalid-characters` - ) - } - process.env['NEXT_DEPLOYMENT_ID'] = envDeploymentId - return envDeploymentId - } - return '' - } - - let userConfiguredDeploymentId: string | undefined - if (typeof configDeploymentId === 'string') { - userConfiguredDeploymentId = configDeploymentId - } else if (typeof configDeploymentId === 'function') { - userConfiguredDeploymentId = generateDeploymentId(configDeploymentId) - } - - if (userConfiguredDeploymentId !== undefined) { - if (userConfiguredDeploymentId.length === 0) { - return process.env['NEXT_DEPLOYMENT_ID'] || '' - } - - if (userConfiguredDeploymentId.length > 32) { - throw new Error( - `The deploymentId "${userConfiguredDeploymentId}" exceeds the maximum length of 32 characters. Please choose a shorter deploymentId. https://nextjs.org/docs/messages/deploymentid-too-long` - ) - } - const validCharacterPattern = /^[a-zA-Z0-9_-]+$/ - if (!validCharacterPattern.test(userConfiguredDeploymentId)) { - throw new Error( - `The deploymentId "${userConfiguredDeploymentId}" contains invalid characters. Only alphanumeric characters (a-z, A-Z, 0-9), hyphens (-), and underscores (_) are allowed. https://nextjs.org/docs/messages/deploymentid-invalid-characters` - ) - } - process.env['NEXT_DEPLOYMENT_ID'] = userConfiguredDeploymentId - return userConfiguredDeploymentId - } - - return process.env['NEXT_DEPLOYMENT_ID'] || '' -} diff --git a/packages/next/src/build/generate-routes-manifest.ts b/packages/next/src/build/generate-routes-manifest.ts index dc7c7559bf442d..22e264f8a794df 100644 --- a/packages/next/src/build/generate-routes-manifest.ts +++ b/packages/next/src/build/generate-routes-manifest.ts @@ -35,7 +35,6 @@ export interface GenerateRoutesManifestOptions { restrictedRedirectPaths: string[] isAppPPREnabled: boolean appType: 'pages' | 'app' | 'hybrid' - deploymentId?: string } export interface GenerateRoutesManifestResult { @@ -61,7 +60,6 @@ export function generateRoutesManifest( restrictedRedirectPaths, isAppPPREnabled, appType, - deploymentId, } = options const sortedRoutes = sortPages([...pageKeys.pages, ...(pageKeys.app ?? [])]) @@ -136,7 +134,6 @@ export function generateRoutesManifest( queryHeader: NEXT_REWRITTEN_QUERY_HEADER, }, skipProxyUrlNormalize: config.skipProxyUrlNormalize, - deploymentId: deploymentId || undefined, ppr: isAppPPREnabled ? { chain: { diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 7681e1d28e7001..5346895e2687e4 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -10,12 +10,7 @@ import type { CacheControl, Revalidate } from '../server/lib/cache-control' import '../lib/setup-exception-listeners' -import { - loadEnvConfig, - type LoadedEnvFiles, - initialEnv, - updateInitialEnv, -} from '@next/env' +import { loadEnvConfig, type LoadedEnvFiles } from '@next/env' import { bold, yellow } from '../lib/picocolors' import { makeRe } from 'next/dist/compiled/picomatch' import { existsSync, promises as fs } from 'fs' @@ -133,8 +128,6 @@ import { sortByPageExts } from './sort-by-page-exts' import { getStaticInfoIncludingLayouts } from './get-static-info-including-layouts' import { PAGE_TYPES } from '../lib/page-types' import { generateBuildId } from './generate-build-id' -import { resolveAndSetDeploymentId } from './generate-deployment-id' -import { evaluateDeploymentId } from '../server/evaluate-deployment-id' import { isWriteable } from './is-writeable' import * as Log from './output/log' import createSpinner from './spinner' @@ -495,13 +488,6 @@ export type RoutesManifest = { } skipProxyUrlNormalize?: boolean caseSensitive?: boolean - /** - * User-configured deployment ID for skew protection. - * This allows users to specify a custom deployment identifier - * in their next.config.js that will be used for version skew protection - * with pre-built deployments. - */ - deploymentId?: string /** * Configuration related to Partial Prerendering. */ @@ -938,11 +924,6 @@ export default async function build( let loadedConfig: NextConfigComplete | undefined let staticWorker: StaticWorker try { - const preservedDeploymentIdAtStart = process.env.NEXT_DEPLOYMENT_ID - if (preservedDeploymentIdAtStart) { - NextBuildContext.preservedDeploymentId = preservedDeploymentIdAtStart - } - const nextBuildSpan = trace('next-build', undefined, { buildMode: experimentalBuildMode, version: process.env.__NEXT_VERSION as string, @@ -957,35 +938,12 @@ export default async function build( NextBuildContext.debugBuildPaths = debugBuildPaths await nextBuildSpan.traceAsyncFn(async () => { - // Check process.env first to get the latest value (e.g., from next.env in tests) - // Then fall back to NextBuildContext which might have a stale value from previous build - const preservedDeploymentId = - process.env.NEXT_DEPLOYMENT_ID || NextBuildContext.preservedDeploymentId - if (preservedDeploymentId) { - NextBuildContext.preservedDeploymentId = preservedDeploymentId - // Ensure it's in process.env so loadEnvConfig can see it - process.env.NEXT_DEPLOYMENT_ID = preservedDeploymentId - // Update initialEnv so loadEnvConfig includes it when resetting process.env - updateInitialEnv({ NEXT_DEPLOYMENT_ID: preservedDeploymentId }) - } - - const { loadedEnvFiles, combinedEnv } = nextBuildSpan + // attempt to load global env values so they are available in next.config.js + const { loadedEnvFiles } = nextBuildSpan .traceChild('load-dotenv') .traceFn(() => loadEnvConfig(dir, false, Log)) NextBuildContext.loadedEnvFiles = loadedEnvFiles - // Restore NEXT_DEPLOYMENT_ID after loadEnvConfig resets process.env - // Priority: preservedDeploymentId > process.env (after loadEnvConfig) > combinedEnv - if (preservedDeploymentId) { - process.env.NEXT_DEPLOYMENT_ID = preservedDeploymentId - NextBuildContext.preservedDeploymentId = preservedDeploymentId - } else if (process.env.NEXT_DEPLOYMENT_ID) { - NextBuildContext.preservedDeploymentId = process.env.NEXT_DEPLOYMENT_ID - } else if (combinedEnv.NEXT_DEPLOYMENT_ID) { - process.env.NEXT_DEPLOYMENT_ID = combinedEnv.NEXT_DEPLOYMENT_ID - NextBuildContext.preservedDeploymentId = combinedEnv.NEXT_DEPLOYMENT_ID - } - const turborepoAccessTraceResult = new TurborepoAccessTraceResult() let experimentalFeatures: ConfiguredExperimentalFeature[] = [] const config: NextConfigComplete = await nextBuildSpan @@ -1009,32 +967,13 @@ export default async function build( ) loadedConfig = config + // Reading the config can modify environment variables that influence the bundler selection. bundler = finalizeBundlerFromConfig(bundler) nextBuildSpan.setAttribute('bundler', getBundlerForTelemetry(bundler)) + // Install the native bindings early so we can have synchronous access later. await installBindings(config.experimental?.useWasmBinary) - // Collect all possible sources of deployment ID - let currentDeploymentId = - NextBuildContext.preservedDeploymentId || - preservedDeploymentId || - process.env.NEXT_DEPLOYMENT_ID || - initialEnv?.NEXT_DEPLOYMENT_ID || - combinedEnv.NEXT_DEPLOYMENT_ID || - '' - - // Ensure process.env has the deployment ID before resolving - if (currentDeploymentId) { - process.env['NEXT_DEPLOYMENT_ID'] = currentDeploymentId - } - - config.deploymentId = resolveAndSetDeploymentId( - config.deploymentId, - config.deploymentId != null ? 'user-config' : 'env-var', - currentDeploymentId || undefined - ) - if (config.deploymentId) { - process.env['NEXT_DEPLOYMENT_ID'] = config.deploymentId - } + process.env.NEXT_DEPLOYMENT_ID = config.deploymentId || '' NextBuildContext.config = config let configOutDir = 'out' @@ -1054,7 +993,6 @@ export default async function build( config ) NextBuildContext.buildId = buildId - NextBuildContext.deploymentId = config.deploymentId if (experimentalBuildMode === 'generate-env') { if (bundler === Bundler.Turbopack) { @@ -1080,7 +1018,7 @@ export default async function build( // when using compile mode static env isn't inlined so we // need to populate in normal runtime env if (isCompileMode || isGenerateMode) { - populateStaticEnv(config, config.deploymentId || '') + populateStaticEnv(config, config.deploymentId) } const customRoutes: CustomRoutes = await nextBuildSpan @@ -1699,7 +1637,6 @@ export default async function build( rewrites, restrictedRedirectPaths, isAppPPREnabled, - deploymentId: evaluateDeploymentId(config.deploymentId), }) ) @@ -2777,10 +2714,7 @@ export default async function build( 2 )};self.__MIDDLEWARE_MATCHERS_CB && self.__MIDDLEWARE_MATCHERS_CB()` - const evaluatedDeploymentId = evaluateDeploymentId( - config.deploymentId - ) - let clientMiddlewareManifestPath = evaluatedDeploymentId + let clientMiddlewareManifestPath = config.deploymentId ? path.join( CLIENT_STATIC_FILES_PATH, TURBOPACK_CLIENT_MIDDLEWARE_MANIFEST @@ -4140,7 +4074,7 @@ export default async function build( distDir, buildId, locales: config.i18n?.locales, - deploymentId: evaluateDeploymentId(config.deploymentId), + deploymentId: config.deploymentId, }) } else { await writePrerenderManifest(distDir, { @@ -4272,7 +4206,6 @@ export default async function build( config, appType, buildId, - deploymentId: (config.deploymentId as string) || '', configOutDir: path.join(dir, configOutDir), staticPages, serverPropsPages, diff --git a/packages/next/src/build/swc/index.ts b/packages/next/src/build/swc/index.ts index 3cde6b9cabba86..257314bcb04a13 100644 --- a/packages/next/src/build/swc/index.ts +++ b/packages/next/src/build/swc/index.ts @@ -854,10 +854,6 @@ function bindingToApi( ): Promise { // Avoid mutating the existing `nextConfig` object. NOTE: This is only a shallow clone. let nextConfigSerializable: Record = { ...nextConfig } - // deploymentId is already evaluated to a string in config.ts, ensure it's typed correctly - if (nextConfigSerializable.deploymentId != null) { - nextConfigSerializable.deploymentId = nextConfig.deploymentId as string - } // These values are never read by Turbopack and are potentially non-serializable. nextConfigSerializable.exportPathMap = {} diff --git a/packages/next/src/build/turbopack-build/impl.ts b/packages/next/src/build/turbopack-build/impl.ts index 9b09cb7e4ab0f2..bc3ba062e4f565 100644 --- a/packages/next/src/build/turbopack-build/impl.ts +++ b/packages/next/src/build/turbopack-build/impl.ts @@ -14,8 +14,6 @@ import { TurbopackManifestLoader } from '../../shared/lib/turbopack/manifest-loa import { promises as fs } from 'fs' import { PHASE_PRODUCTION_BUILD } from '../../shared/lib/constants' import loadConfig from '../../server/config' -import type { NextConfigComplete } from '../../server/config-shared' -import { evaluateDeploymentId } from '../../server/evaluate-deployment-id' import { hasCustomExportOutput } from '../../export/utils' import { Telemetry } from '../../telemetry/storage' import { setGlobal } from '../../trace' @@ -59,9 +57,7 @@ export async function turbopackBuild(): Promise<{ rootPath: config.turbopack?.root || config.outputFileTracingRoot || dir, projectPath: normalizePath(path.relative(rootPath, dir) || '.'), distDir, - nextConfig: config as NextConfigComplete & { - deploymentId?: string - }, + nextConfig: config, watch: { enable: false, }, @@ -142,8 +138,7 @@ export async function turbopackBuild(): Promise<{ distDir, encryptionKey, dev: false, - deploymentId: evaluateDeploymentId(config.deploymentId), - runtimeServerDeploymentId: config.experimental.runtimeServerDeploymentId, + deploymentId: config.deploymentId, }) const currentEntrypoints = await rawEntrypointsToEntrypoints( diff --git a/packages/next/src/build/webpack-config.ts b/packages/next/src/build/webpack-config.ts index 2f371e72319a91..6be28f15f2c6e2 100644 --- a/packages/next/src/build/webpack-config.ts +++ b/packages/next/src/build/webpack-config.ts @@ -32,7 +32,6 @@ import { import type { CompilerNameValues } from '../shared/lib/constants' import { execOnce } from '../shared/lib/utils' import type { NextConfigComplete } from '../server/config-shared' -import { evaluateDeploymentId } from '../server/evaluate-deployment-id' import { finalizeEntrypoint } from './entries' import * as Log from './output/log' import { buildConfiguration } from './webpack/config' @@ -564,9 +563,7 @@ export default async function getBaseWebpackConfig( pagesDir, appDir, hasReactRefresh: dev && isClient, - nextConfig: config as NextConfigComplete & { - deploymentId?: string - }, + nextConfig: config, jsConfig, transpilePackages: finalTranspilePackages, supportedBrowsers, @@ -2113,18 +2110,13 @@ export default async function getBaseWebpackConfig( __NEXT_PREVIEW_MODE_SIGNING_KEY: previewProps.previewModeSigningKey, __NEXT_PREVIEW_MODE_ENCRYPTION_KEY: previewProps.previewModeEncryptionKey, - ...(config.experimental.runtimeServerDeploymentId - ? { - NEXT_DEPLOYMENT_ID: process.env.NEXT_DEPLOYMENT_ID, - } - : {}), }, }), isClient && new BuildManifestPlugin({ buildId, dev, - deploymentId: evaluateDeploymentId(config.deploymentId), + deploymentId: config.deploymentId, rewrites, isDevFallback, appDirEnabled: hasAppDir, diff --git a/packages/next/src/export/index.ts b/packages/next/src/export/index.ts index 1ffd15f698af6c..50ab3db0c53637 100644 --- a/packages/next/src/export/index.ts +++ b/packages/next/src/export/index.ts @@ -53,7 +53,6 @@ import { Telemetry } from '../telemetry/storage' import { normalizePagePath } from '../shared/lib/page-path/normalize-page-path' import { denormalizePagePath } from '../shared/lib/page-path/denormalize-page-path' import { loadEnvConfig } from '@next/env' -import { evaluateDeploymentId } from '../server/evaluate-deployment-id' import { isAPIRoute } from '../lib/is-api-route' import { getPagePath } from '../server/require' import type { Span } from '../trace' @@ -713,7 +712,7 @@ async function exportAppImpl( batches.map(async (batch) => worker.exportPages({ buildId, - deploymentId: evaluateDeploymentId(nextConfig.deploymentId) || '', + deploymentId: nextConfig.deploymentId, exportPaths: batch, parentSpanId: span.getId(), pagesDataDir, diff --git a/packages/next/src/lib/inline-static-env.ts b/packages/next/src/lib/inline-static-env.ts index cf4d25c6d65fe8..057113326d2d85 100644 --- a/packages/next/src/lib/inline-static-env.ts +++ b/packages/next/src/lib/inline-static-env.ts @@ -6,7 +6,6 @@ import globOriginal from 'next/dist/compiled/glob' import { Sema } from 'next/dist/compiled/async-sema' import type { NextConfigComplete } from '../server/config-shared' import { getNextConfigEnv, getStaticEnv } from './static-env' -import { evaluateDeploymentId } from '../server/evaluate-deployment-id' const glob = promisify(globOriginal) @@ -18,11 +17,7 @@ export async function inlineStaticEnv({ config: NextConfigComplete }) { const nextConfigEnv = getNextConfigEnv(config) - // User-configured deploymentId takes precedence over NEXT_DEPLOYMENT_ID - const deploymentId = evaluateDeploymentId( - config.deploymentId || process.env.NEXT_DEPLOYMENT_ID - ) - const staticEnv = getStaticEnv(config, deploymentId) + const staticEnv = getStaticEnv(config, config.deploymentId) const serverDir = path.join(distDir, 'server') const serverChunks = await glob('**/*.{js,json,js.map}', { diff --git a/packages/next/src/server/base-server.ts b/packages/next/src/server/base-server.ts index 12105f7ce36a09..e9f4768126ccf9 100644 --- a/packages/next/src/server/base-server.ts +++ b/packages/next/src/server/base-server.ts @@ -182,7 +182,6 @@ export type RouteHandler< * the rewrites normalized to the object shape that the router expects. */ export type NormalizedRouteManifest = { - readonly deploymentId?: string | undefined readonly dynamicRoutes: ReadonlyArray readonly rewrites: { readonly beforeFiles: ReadonlyArray @@ -466,7 +465,7 @@ export default abstract class Server< } else { let id = this.nextConfig.experimental.useSkewCookie ? '' - : (this.nextConfig.deploymentId as string) || '' + : this.nextConfig.deploymentId || '' this.deploymentId = id process.env.NEXT_DEPLOYMENT_ID = id diff --git a/packages/next/src/server/config-shared.ts b/packages/next/src/server/config-shared.ts index 58af5a3198649d..f379b71275bb2a 100644 --- a/packages/next/src/server/config-shared.ts +++ b/packages/next/src/server/config-shared.ts @@ -1149,9 +1149,8 @@ export interface NextConfig { /** * A unique identifier for a deployment that will be included in each request's query string or header. - * Can be a string or a function that returns a string. */ - deploymentId?: string | (() => string) + deploymentId?: string /** * Deploy a Next.js application under a sub-path of a domain diff --git a/packages/next/src/server/config.ts b/packages/next/src/server/config.ts index 3e59370c6c460f..8fe9a9cf6d188a 100644 --- a/packages/next/src/server/config.ts +++ b/packages/next/src/server/config.ts @@ -45,7 +45,6 @@ import { djb2Hash } from '../shared/lib/hash' import type { NextAdapter } from '../build/adapter/build-complete' import { HardDeprecatedConfigError } from '../shared/lib/errors/hard-deprecated-config-error' import { NextInstanceErrorState } from './mcp/tools/next-instance-error-state' -import { evaluateDeploymentId } from './evaluate-deployment-id' export { normalizeConfig } from './config-shared' export type { DomainLocale, NextConfig } from './config-shared' @@ -956,19 +955,28 @@ function assignDefaultsAndValidate( } } - if (result.deploymentId != null) { - result.deploymentId = evaluateDeploymentId(result.deploymentId) - } - if ( result.experimental.runtimeServerDeploymentId == null && phase === PHASE_PRODUCTION_BUILD && ciEnvironment.hasNextSupport && process.env.NEXT_DEPLOYMENT_ID ) { + if ( + result.deploymentId != null && + result.deploymentId !== process.env.NEXT_DEPLOYMENT_ID + ) { + throw new Error( + `The NEXT_DEPLOYMENT_ID environment variable value "${process.env.NEXT_DEPLOYMENT_ID}" does not match the provided deploymentId "${result.deploymentId}" in the config.` + ) + } result.experimental.runtimeServerDeploymentId = true } + // only leverage deploymentId + if (process.env.NEXT_DEPLOYMENT_ID) { + result.deploymentId = process.env.NEXT_DEPLOYMENT_ID + } + const tracingRoot = result?.outputFileTracingRoot const turbopackRoot = result?.turbopack?.root diff --git a/packages/next/src/server/dev/hot-reloader-turbopack.ts b/packages/next/src/server/dev/hot-reloader-turbopack.ts index 8bea4d09747730..e4846cb83ba7f8 100644 --- a/packages/next/src/server/dev/hot-reloader-turbopack.ts +++ b/packages/next/src/server/dev/hot-reloader-turbopack.ts @@ -77,8 +77,6 @@ import { } from './messages' import { generateEncryptionKeyBase64 } from '../app-render/encryption-utils-server' import { isAppPageRouteDefinition } from '../route-definitions/app-page-route-definition' -import type { NextConfigComplete } from '../config-shared' -import { evaluateDeploymentId } from '../evaluate-deployment-id' import { normalizeAppPath } from '../../shared/lib/router/utils/app-paths' import type { ModernSourceMapPayload } from '../lib/source-maps' import { isMetadataRouteFile } from '../../lib/metadata/is-metadata-route' @@ -256,9 +254,7 @@ export async function createHotReloaderTurbopack( rootPath, projectPath: normalizePath(relative(rootPath, projectPath) || '.'), distDir, - nextConfig: opts.nextConfig as NextConfigComplete & { - deploymentId?: string - }, + nextConfig: opts.nextConfig, watch: { enable: dev, pollIntervalMs: nextConfig.watchOptions?.pollIntervalMs, @@ -328,9 +324,7 @@ export async function createHotReloaderTurbopack( distDir, encryptionKey, dev: true, - deploymentId: evaluateDeploymentId(nextConfig.deploymentId), - runtimeServerDeploymentId: - nextConfig.experimental.runtimeServerDeploymentId, + deploymentId: nextConfig.deploymentId, }) // Dev specific diff --git a/packages/next/src/server/evaluate-deployment-id.ts b/packages/next/src/server/evaluate-deployment-id.ts deleted file mode 100644 index a9b053ce82a6cc..00000000000000 --- a/packages/next/src/server/evaluate-deployment-id.ts +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Evaluates a deployment ID from a user-provided function or string. - * Returns the string value, calling the function if needed. - * Returns empty string if undefined (for runtime use where a string is always needed). - * Handles all possible input types at runtime, including broader Function types. - * - * This file is safe to use in edge runtime - it does NOT modify process.env. - */ -export function evaluateDeploymentId( - deploymentId: string | (() => string) | Function | undefined | null | unknown -): string { - // Handle function type (including broader Function type, not just () => string) - if (typeof deploymentId === 'function') { - const result = deploymentId() - if (typeof result !== 'string') { - throw new Error( - 'deploymentId function must return a string. https://nextjs.org/docs/messages/deploymentid-not-a-string' - ) - } - return result - } - - if (typeof deploymentId === 'string') { - return deploymentId - } - - // Handle null, undefined, or any other type - return '' -} diff --git a/packages/next/src/server/route-modules/route-module.ts b/packages/next/src/server/route-modules/route-module.ts index c74e617b1f3244..308f3c3118329c 100644 --- a/packages/next/src/server/route-modules/route-module.ts +++ b/packages/next/src/server/route-modules/route-module.ts @@ -57,7 +57,6 @@ import type { BaseNextRequest } from '../base-http' import type { I18NConfig, NextConfigRuntime } from '../config-shared' import ResponseCache, { type ResponseGenerator } from '../response-cache' import { normalizeAppPath } from '../../shared/lib/router/utils/app-paths' -import { evaluateDeploymentId } from '../evaluate-deployment-id' import { RouterServerContextSymbol, routerServerGlobal, @@ -545,8 +544,7 @@ export abstract class RouteModule< } deploymentId = process.env.NEXT_DEPLOYMENT_ID } else { - // evaluateDeploymentId handles string, function, and undefined cases - deploymentId = evaluateDeploymentId(nextConfig.deploymentId) + deploymentId = nextConfig.deploymentId || '' } return { nextConfig, deploymentId } @@ -948,8 +946,7 @@ export abstract class RouteModule< } deploymentId = process.env.NEXT_DEPLOYMENT_ID } else { - // evaluateDeploymentId handles string, function, and undefined cases - deploymentId = evaluateDeploymentId(nextConfig.deploymentId) + deploymentId = nextConfig.deploymentId || '' } return { diff --git a/packages/next/src/shared/lib/turbopack/manifest-loader.ts b/packages/next/src/shared/lib/turbopack/manifest-loader.ts index 37200a5926564e..5b1a13e4884da6 100644 --- a/packages/next/src/shared/lib/turbopack/manifest-loader.ts +++ b/packages/next/src/shared/lib/turbopack/manifest-loader.ts @@ -204,7 +204,6 @@ export class TurbopackManifestLoader { private readonly buildId: string private readonly deploymentId: string private readonly dev: boolean - private readonly runtimeServerDeploymentId: boolean constructor({ distDir, @@ -212,21 +211,18 @@ export class TurbopackManifestLoader { encryptionKey, dev, deploymentId, - runtimeServerDeploymentId = false, }: { buildId: string distDir: string encryptionKey: string dev: boolean deploymentId: string - runtimeServerDeploymentId?: boolean }) { this.distDir = distDir this.buildId = buildId this.encryptionKey = encryptionKey this.dev = dev this.deploymentId = deploymentId - this.runtimeServerDeploymentId = runtimeServerDeploymentId } delete(key: EntryKey) { @@ -767,14 +763,9 @@ export class TurbopackManifestLoader { const updateFunctionDefinition = ( fun: EdgeFunctionDefinition ): EdgeFunctionDefinition => { - const env = { ...fun.env } - if (process.env.NEXT_DEPLOYMENT_ID) { - env.NEXT_DEPLOYMENT_ID = process.env.NEXT_DEPLOYMENT_ID - } return { ...fun, files: [...(instrumentation?.files ?? []), ...fun.files], - env, } } for (const key of Object.keys(manifest.middleware)) { diff --git a/test/e2e/deployment-id-function/deployment-id-function.test.ts b/test/e2e/deployment-id-function/deployment-id-function.test.ts deleted file mode 100644 index dbf4c1501fd9bf..00000000000000 --- a/test/e2e/deployment-id-function/deployment-id-function.test.ts +++ /dev/null @@ -1,328 +0,0 @@ -import { createNext, isNextDev } from 'e2e-utils' -import { NextInstance } from 'e2e-utils' - -describe('deploymentId function support', () => { - let next: NextInstance | undefined - // Generate unique deployment IDs for each test run to avoid Vercel conflicts - // Use a short unique ID to stay within 32 character limit - const uniqueId = Date.now().toString(36).slice(-6) - - afterEach(async () => { - if (next) { - await next.destroy() - next = undefined - } - }) - - it('should work with deploymentId as a string', async () => { - next = await createNext({ - files: { - 'app/layout.jsx': ` - export default function Layout({ children }) { - return ( - - {children} - - ) - } - `, - 'app/page.jsx': ` - export default function Page() { - return

hello world

- } - `, - 'next.config.js': ` - module.exports = { - deploymentId: 'my-static-deployment-id-${uniqueId}' - } - `, - }, - dependencies: {}, - }) - - const res = await next.fetch('/') - const html = await res.text() - expect(html).toContain('hello world') - }) - - it('should work with deploymentId as a function returning string', async () => { - next = await createNext({ - files: { - 'app/layout.jsx': ` - export default function Layout({ children }) { - return ( - - {children} - - ) - } - `, - 'app/page.jsx': ` - export default function Page() { - return

hello world

- } - `, - 'next.config.js': ` - module.exports = { - deploymentId: () => { - return 'my-function-deployment-id-${uniqueId}' - } - } - `, - }, - dependencies: {}, - }) - - const res = await next.fetch('/') - const html = await res.text() - expect(html).toContain('hello world') - }) - - it('should work with deploymentId function using environment variable', async () => { - next = await createNext({ - files: { - 'app/layout.jsx': ` - export default function Layout({ children }) { - return ( - - {children} - - ) - } - `, - 'app/page.jsx': ` - export default function Page() { - return

hello world

- } - `, - 'next.config.js': ` - module.exports = { - deploymentId: () => { - return process.env.CUSTOM_DEPLOYMENT_ID || 'fallback-id' - } - } - `, - }, - env: { - CUSTOM_DEPLOYMENT_ID: `env-deployment-id-${uniqueId}`, - }, - dependencies: {}, - }) - - const res = await next.fetch('/') - const html = await res.text() - expect(html).toContain('hello world') - }) - - it('should work with useSkewCookie and deploymentId function', async () => { - next = await createNext({ - files: { - 'app/layout.jsx': ` - export default function Layout({ children }) { - return ( - - {children} - - ) - } - `, - 'app/page.jsx': ` - export default function Page() { - return

hello world

- } - `, - 'next.config.js': ` - module.exports = { - experimental: { - useSkewCookie: true - }, - deploymentId: () => { - return 'skew-cookie-deployment-id-${uniqueId}' - } - } - `, - }, - dependencies: {}, - }) - - const res = await next.fetch('/') - const setCookieHeader = res.headers.get('set-cookie') - - // In deploy mode (NEXT_DEPLOYMENT_ID set by Vercel), expect the Vercel deployment ID (starts with dpl_) - // In prebuild mode (NEXT_DEPLOYMENT_ID not set), expect the user-configured ID - if (setCookieHeader?.includes('__vdpl=dpl_')) { - // Deploy mode: expect Vercel's deployment ID (format: dpl_...) - expect(setCookieHeader).toMatch(/__vdpl=dpl_[^;]+/) - } else { - // Prebuild mode: expect user-configured deployment ID - expect(setCookieHeader).toContain( - `__vdpl=skew-cookie-deployment-id-${uniqueId}` - ) - } - }) - - // Note: In dev mode, config validation errors are thrown after the server says "Ready", - // so createNext() resolves before the error is caught. These tests only work in - // start/deploy modes where build-time validation catches the error. - it('should throw error when deploymentId function returns non-string', async () => { - if (isNextDev) { - // Skip in dev mode - validation errors occur after server starts - return - } - - let errorThrown = false - let nextInstance: NextInstance | undefined - try { - nextInstance = await createNext({ - files: { - 'app/layout.jsx': ` - export default function Layout({ children }) { - return ( - - {children} - - ) - } - `, - 'app/page.jsx': ` - export default function Page() { - return

hello world

- } - `, - 'next.config.js': ` - module.exports = { - deploymentId: () => { - return null - } - } - `, - }, - dependencies: {}, - }) - } catch (err: any) { - errorThrown = true - // The error is thrown in the child process, so we just verify that createNext fails - // The actual error message "deploymentId function must return a string" is visible - // in the console output but wrapped differently in different modes: - // - Start mode: "next build failed with code/signal 1" - // - Deploy mode: "Failed to deploy project" - expect(err).toBeDefined() - expect( - err.message.includes('build failed') || - err.message.includes('Failed to deploy') - ).toBe(true) - } finally { - if (nextInstance) { - await nextInstance.destroy() - } - } - // Ensure an error was actually thrown - expect(errorThrown).toBe(true) - }) - - it('should throw error when deploymentId exceeds 32 characters', async () => { - if (isNextDev) { - // Skip in dev mode - validation errors occur after server starts - return - } - - let errorThrown = false - let nextInstance: NextInstance | undefined - try { - nextInstance = await createNext({ - files: { - 'app/layout.jsx': ` - export default function Layout({ children }) { - return ( - - {children} - - ) - } - `, - 'app/page.jsx': ` - export default function Page() { - return

hello world

- } - `, - 'next.config.js': ` - module.exports = { - deploymentId: 'this-is-a-very-long-deployment-id-that-exceeds-32-characters' - } - `, - }, - dependencies: {}, - }) - } catch (err: any) { - errorThrown = true - // The error is thrown in the child process, so we just verify that createNext fails - // The actual error message about exceeding 32 characters is visible in the console output - // but wrapped differently in different modes - expect(err).toBeDefined() - expect( - err.message.includes('build failed') || - err.message.includes('Failed to deploy') - ).toBe(true) - } finally { - if (nextInstance) { - await nextInstance.destroy() - } - } - // Ensure an error was actually thrown - expect(errorThrown).toBe(true) - }) - - it('should throw error when deploymentId function returns string exceeding 32 characters', async () => { - if (isNextDev) { - // Skip in dev mode - validation errors occur after server starts - return - } - - let errorThrown = false - let nextInstance: NextInstance | undefined - try { - nextInstance = await createNext({ - files: { - 'app/layout.jsx': ` - export default function Layout({ children }) { - return ( - - {children} - - ) - } - `, - 'app/page.jsx': ` - export default function Page() { - return

hello world

- } - `, - 'next.config.js': ` - module.exports = { - deploymentId: () => { - return 'this-is-a-very-long-deployment-id-that-exceeds-32-characters' - } - } - `, - }, - dependencies: {}, - }) - } catch (err: any) { - errorThrown = true - // The error is thrown in the child process, so we just verify that createNext fails - // The actual error message about exceeding 32 characters is visible in the console output - // but wrapped differently in different modes - expect(err).toBeDefined() - expect( - err.message.includes('build failed') || - err.message.includes('Failed to deploy') - ).toBe(true) - } finally { - if (nextInstance) { - await nextInstance.destroy() - } - } - // Ensure an error was actually thrown - expect(errorThrown).toBe(true) - }) -}) diff --git a/test/integration/next-dynamic-css-asset-prefix/tsconfig.json b/test/integration/next-dynamic-css-asset-prefix/tsconfig.json index d6bef2a5f92119..ce8701fb2649fa 100644 --- a/test/integration/next-dynamic-css-asset-prefix/tsconfig.json +++ b/test/integration/next-dynamic-css-asset-prefix/tsconfig.json @@ -20,12 +20,6 @@ ], "strictNullChecks": true }, - "include": [ - "next-env.d.ts", - ".next/types/**/*.ts", - "**/*.ts", - "**/*.tsx", - ".next/dev/types/**/*.ts" - ], + "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"], "exclude": ["node_modules", "**/*.test.ts", "**/*.test.tsx"] } diff --git a/test/production/deployment-id-handling/app/next.config.js b/test/production/deployment-id-handling/app/next.config.js index c06352d2623f7c..847dc115c5399b 100644 --- a/test/production/deployment-id-handling/app/next.config.js +++ b/test/production/deployment-id-handling/app/next.config.js @@ -1,13 +1,6 @@ /** @type {import('next').NextConfig} */ module.exports = { - // Read deploymentId from env vars - CUSTOM_DEPLOYMENT_ID takes precedence for explicit tests - // If CUSTOM_DEPLOYMENT_ID is not set, fall back to NEXT_DEPLOYMENT_ID - // This ensures NEXT_DEPLOYMENT_ID is available when next.config.js is loaded, - // which happens before loadEnvConfig might reset process.env - deploymentId: - process.env.CUSTOM_DEPLOYMENT_ID || - process.env.NEXT_DEPLOYMENT_ID || - undefined, + deploymentId: process.env.CUSTOM_DEPLOYMENT_ID, experimental: { useSkewCookie: Boolean(process.env.COOKIE_SKEW), runtimeServerDeploymentId: !!process.env.RUNTIME_SERVER_DEPLOYMENT_ID, diff --git a/test/production/deterministic-build/deployment-id.test.ts b/test/production/deterministic-build/deployment-id.test.ts index c64d7c06ef2b46..721a599b93560e 100644 --- a/test/production/deterministic-build/deployment-id.test.ts +++ b/test/production/deterministic-build/deployment-id.test.ts @@ -27,18 +27,6 @@ async function readFiles(next: NextInstance) { ) } -// TODO static/* browser chunks are content hashed and have the deployment id inlined -const IGNORE_NAME = new RegExp( - [ - 'static/chunks/', - '.*_buildManifest\\.js', - '.*_ssgManifest\\.js', - '.*_clientMiddlewareManifest\\.js', - ] - .map((v) => '(?:\\/|^)' + v + '$') - .join('|') -) - const IGNORE_CONTENT = new RegExp( [ // TODO this contains "env": { "__NEXT_BUILD_ID": "taBOOu8Znzobe4G7wEG_i", @@ -53,10 +41,6 @@ const IGNORE_CONTENT = new RegExp( // These are not critical, as they aren't deployed to the serverless function itself 'client-build-manifest\\.json', 'fallback-build-manifest\\.json', - // TODO These contain manifest file paths that include build IDs - 'build-manifest\\.json', - 'middleware-build-manifest\\.js', - // Contains the deploymentId which is expected to change between deployments 'routes-manifest\\.json', ] .map((v) => '(?:\\/|^)' + v + '$') @@ -102,9 +86,6 @@ const IGNORE_CONTENT = new RegExp( await next.build() let run2 = await readFiles(next) - run1 = run1.filter(([f, _]) => !IGNORE_NAME.test(f)) - run2 = run2.filter(([f, _]) => !IGNORE_NAME.test(f)) - // First, compare file names let run1FileNames = run1.map(([f, _]) => f) let run2FileNames = run2.map(([f, _]) => f) diff --git a/test/unit/generate-deployment-id.test.ts b/test/unit/generate-deployment-id.test.ts deleted file mode 100644 index 13947eaa18260b..00000000000000 --- a/test/unit/generate-deployment-id.test.ts +++ /dev/null @@ -1,263 +0,0 @@ -import { generateDeploymentId } from 'next/dist/build/generate-deployment-id' -import { resolveAndSetDeploymentId } from 'next/dist/build/generate-deployment-id' - -describe('generateDeploymentId', () => { - it('should return undefined when deploymentId is undefined', () => { - expect(generateDeploymentId(undefined)).toBeUndefined() - }) - - it('should return string when deploymentId is a string', () => { - expect(generateDeploymentId('my-deployment-123')).toBe('my-deployment-123') - expect(generateDeploymentId(' my-deployment-123 ')).toBe( - ' my-deployment-123 ' - ) - }) - - it('should call function and return string when deploymentId is a function', () => { - const fn = () => 'my-deployment-123' - expect(generateDeploymentId(fn)).toBe('my-deployment-123') - - const fnWithWhitespace = () => ' my-deployment-123 ' - expect(generateDeploymentId(fnWithWhitespace)).toBe(' my-deployment-123 ') - }) - - it('should throw error when function returns non-string', () => { - const fn = () => 123 as any - expect(() => generateDeploymentId(fn)).toThrow( - 'deploymentId function must return a string' - ) - }) - - it('should handle function that returns empty string', () => { - const fn = () => '' - expect(generateDeploymentId(fn)).toBe('') - }) - - it('should handle empty string deploymentId', () => { - expect(generateDeploymentId('')).toBe('') - expect(generateDeploymentId(' ')).toBe(' ') - }) -}) - -describe('resolveAndSetDeploymentId', () => { - const originalEnv = process.env.NEXT_DEPLOYMENT_ID - - afterEach(() => { - if (originalEnv === undefined) { - delete process.env.NEXT_DEPLOYMENT_ID - } else { - process.env.NEXT_DEPLOYMENT_ID = originalEnv - } - }) - - describe('Precedence: user-configured vs NEXT_DEPLOYMENT_ID', () => { - beforeEach(() => { - delete process.env.NEXT_DEPLOYMENT_ID - }) - - it('should use user-configured deployment ID when both are provided', () => { - const userDeploymentId = 'my-custom-id' - const vercelDeploymentId = 'dpl_abc123xyz' - - process.env.NEXT_DEPLOYMENT_ID = vercelDeploymentId - - const result = resolveAndSetDeploymentId(userDeploymentId, 'user-config') - expect(result).toBe(userDeploymentId) - expect(process.env.NEXT_DEPLOYMENT_ID).toBe(userDeploymentId) - }) - - it('should use NEXT_DEPLOYMENT_ID when user config is not provided', () => { - const vercelDeploymentId = 'dpl_abc123xyz' - process.env.NEXT_DEPLOYMENT_ID = vercelDeploymentId - - const result = resolveAndSetDeploymentId(undefined, 'env-var') - expect(result).toBe(vercelDeploymentId) - expect(process.env.NEXT_DEPLOYMENT_ID).toBe(vercelDeploymentId) - }) - - it('should use user-configured function deployment ID over NEXT_DEPLOYMENT_ID', () => { - const userDeploymentId = 'my-function-id' - const vercelDeploymentId = 'dpl_abc123xyz' - - process.env.NEXT_DEPLOYMENT_ID = vercelDeploymentId - - const fn = () => userDeploymentId - const result = resolveAndSetDeploymentId(fn, 'user-config') - expect(result).toBe(userDeploymentId) - expect(process.env.NEXT_DEPLOYMENT_ID).toBe(userDeploymentId) - }) - - it('should not error when called twice with Vercel deployment ID from env var', () => { - const vercelDeploymentId = 'dpl_abc123xyz' - process.env.NEXT_DEPLOYMENT_ID = vercelDeploymentId - - const firstResult = resolveAndSetDeploymentId(undefined, 'env-var') - expect(firstResult).toBe(vercelDeploymentId) - - const secondResult = resolveAndSetDeploymentId(firstResult, 'env-var') - expect(secondResult).toBe(vercelDeploymentId) - }) - - it('should respect explicit source parameter (env-var)', () => { - const vercelDeploymentId = 'dpl_abc123xyz' - process.env.NEXT_DEPLOYMENT_ID = vercelDeploymentId - - // Explicitly mark as env-var sourced - should not validate - const result = resolveAndSetDeploymentId(vercelDeploymentId, 'env-var') - expect(result).toBe(vercelDeploymentId) - // Should not throw validation error - }) - }) - - describe('Edge cases: undefined, null, and empty values', () => { - beforeEach(() => { - delete process.env.NEXT_DEPLOYMENT_ID - }) - - it('should return empty string when NEXT_DEPLOYMENT_ID is undefined and source is env-var', () => { - delete process.env.NEXT_DEPLOYMENT_ID - const result = resolveAndSetDeploymentId(undefined, 'env-var') - expect(result).toBe('') - }) - - it('should return empty string when NEXT_DEPLOYMENT_ID is empty string and source is env-var', () => { - process.env.NEXT_DEPLOYMENT_ID = '' - const result = resolveAndSetDeploymentId(undefined, 'env-var') - expect(result).toBe('') - }) - - it('should handle empty string user-configured deployment ID (treated as not configured)', () => { - delete process.env.NEXT_DEPLOYMENT_ID - const result = resolveAndSetDeploymentId('', 'user-config') - expect(result).toBe('') - expect(process.env.NEXT_DEPLOYMENT_ID).toBeUndefined() - }) - - it('should handle function returning empty string for user-configured deployment ID (treated as not configured)', () => { - process.env.NEXT_DEPLOYMENT_ID = 'env-var-id' - const fn = () => '' - const result = resolveAndSetDeploymentId(fn, 'user-config') - expect(result).toBe('env-var-id') - expect(process.env.NEXT_DEPLOYMENT_ID).toBe('env-var-id') - }) - - it('should fall back to env var when user-config source but configDeploymentId is undefined', () => { - process.env.NEXT_DEPLOYMENT_ID = 'env-var-id' - const result = resolveAndSetDeploymentId(undefined, 'user-config') - expect(result).toBe('env-var-id') - }) - - it('should return empty string when both user-config and env var are undefined', () => { - delete process.env.NEXT_DEPLOYMENT_ID - const result = resolveAndSetDeploymentId(undefined, 'user-config') - expect(result).toBe('') - }) - - it('should handle user-configured empty string (treated as not configured, falls back to env var)', () => { - process.env.NEXT_DEPLOYMENT_ID = 'env-var-id' - const result = resolveAndSetDeploymentId('', 'user-config') - expect(result).toBe('env-var-id') - expect(process.env.NEXT_DEPLOYMENT_ID).toBe('env-var-id') - }) - - it('should reject whitespace-only user-configured deployment ID (contains invalid characters)', () => { - expect(() => resolveAndSetDeploymentId(' ', 'user-config')).toThrow( - 'contains invalid characters' - ) - }) - }) - - describe('Character validation', () => { - beforeEach(() => { - delete process.env.NEXT_DEPLOYMENT_ID - }) - - it('should reject deploymentId with invalid characters (spaces)', () => { - expect(() => - resolveAndSetDeploymentId('my deployment id', 'user-config') - ).toThrow('contains invalid characters') - }) - - it('should reject deploymentId with invalid characters (question mark)', () => { - expect(() => - resolveAndSetDeploymentId('my-deployment?id=123', 'user-config') - ).toThrow('contains invalid characters') - }) - - it('should reject deploymentId with invalid characters (ampersand)', () => { - expect(() => - resolveAndSetDeploymentId('my-deployment&id=123', 'user-config') - ).toThrow('contains invalid characters') - }) - - it('should reject deploymentId with invalid characters (percent)', () => { - expect(() => - resolveAndSetDeploymentId('my-deployment%20id', 'user-config') - ).toThrow('contains invalid characters') - }) - - it('should reject deploymentId with invalid characters (slash)', () => { - expect(() => - resolveAndSetDeploymentId('my/deployment/id', 'user-config') - ).toThrow('contains invalid characters') - }) - - it('should reject deploymentId with invalid characters (dot)', () => { - expect(() => - resolveAndSetDeploymentId('my.deployment.id', 'user-config') - ).toThrow('contains invalid characters') - }) - - it('should reject deploymentId with control characters', () => { - expect(() => - resolveAndSetDeploymentId('my-deployment\tid', 'user-config') - ).toThrow('contains invalid characters') - }) - - it('should allow deploymentId with valid characters (base62 + hyphen + underscore)', () => { - const result = resolveAndSetDeploymentId( - 'my-deployment_v2-abc123XYZ', - 'user-config' - ) - expect(result).toBe('my-deployment_v2-abc123XYZ') - expect(process.env.NEXT_DEPLOYMENT_ID).toBe('my-deployment_v2-abc123XYZ') - }) - - it('should allow deploymentId with only alphanumeric characters', () => { - const result = resolveAndSetDeploymentId('abc123XYZ789', 'user-config') - expect(result).toBe('abc123XYZ789') - expect(process.env.NEXT_DEPLOYMENT_ID).toBe('abc123XYZ789') - }) - - it('should allow deploymentId with only hyphens', () => { - const result = resolveAndSetDeploymentId('---', 'user-config') - expect(result).toBe('---') - expect(process.env.NEXT_DEPLOYMENT_ID).toBe('---') - }) - - it('should allow deploymentId with only underscores', () => { - const result = resolveAndSetDeploymentId('___', 'user-config') - expect(result).toBe('___') - expect(process.env.NEXT_DEPLOYMENT_ID).toBe('___') - }) - - it('should reject deploymentId from function that returns invalid characters', () => { - const fn = () => 'my deployment id' - expect(() => resolveAndSetDeploymentId(fn, 'user-config')).toThrow( - 'contains invalid characters' - ) - }) - - it('should allow deploymentId from function that returns valid characters', () => { - const fn = () => 'my-deployment_v2-abc123XYZ' - const result = resolveAndSetDeploymentId(fn, 'user-config') - expect(result).toBe('my-deployment_v2-abc123XYZ') - expect(process.env.NEXT_DEPLOYMENT_ID).toBe('my-deployment_v2-abc123XYZ') - }) - - it('should allow empty string (treated as not configured)', () => { - const result = resolveAndSetDeploymentId('', 'user-config') - expect(result).toBe('') - }) - }) -}) From 599fc3f82a40044bb31d64f08dcd7b5a6ded2b8d Mon Sep 17 00:00:00 2001 From: Delba de Oliveira <32464864+delbaoliveira@users.noreply.github.com> Date: Fri, 23 Jan 2026 12:04:13 +0000 Subject: [PATCH 09/10] Docs: Add Next.js Glossary (#88811) Add next.js glossary so we can reference concepts in guides without having to re-explain them. Guides should be succint, like a skill :) Closes: https://linear.app/vercel/issue/DOC-5574/nextjs-glossary --------- Co-authored-by: vercel[bot] <35613825+vercel[bot]@users.noreply.github.com> --- docs/01-app/glossary.mdx | 264 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 264 insertions(+) create mode 100644 docs/01-app/glossary.mdx diff --git a/docs/01-app/glossary.mdx b/docs/01-app/glossary.mdx new file mode 100644 index 00000000000000..aa47c9d5a6cc08 --- /dev/null +++ b/docs/01-app/glossary.mdx @@ -0,0 +1,264 @@ +--- +title: Next.js Glossary +nav_title: Glossary +description: A glossary of common terms used in Next.js. +--- + +# A + +## App Router + +The Next.js router introduced in version 13, built on top of React Server Components. It uses file-system based routing and supports layouts, nested routing, loading states, error handling, and more. Learn more in the [App Router documentation](/docs/app). + +# B + +## Build time + +The stage when your application is being compiled. During build time, Next.js transforms your code into optimized files for production, generates static pages, and prepares assets for deployment. See the [`next build` CLI reference](/docs/app/api-reference/cli/next#next-build-options). + +# C + +## Cache Components + +A feature that enables component and function-level caching using the [`"use cache"` directive](/docs/app/api-reference/directives/use-cache). Cache Components allows you to mix static, cached, and dynamic content within a single route by prerendering a static HTML shell that's served immediately, while dynamic content streams in when ready. Configure cache duration with [`cacheLife()`](/docs/app/api-reference/functions/cacheLife), tag cached data with [`cacheTag()`](/docs/app/api-reference/functions/cacheTag), and invalidate on-demand with [`updateTag()`](/docs/app/api-reference/functions/updateTag). Learn more in the [Cache Components guide](/docs/app/getting-started/cache-components). + +## Catch-all Segments + +Dynamic route segments that can match multiple URL parts using the `[...folder]/page.js` syntax. These segments capture all remaining URL segments and are useful for implementing features like documentation sites or file browsers. Learn more in [Dynamic Route Segments](/docs/app/api-reference/file-conventions/dynamic-routes#catch-all-segments). + +## Client Bundles + +JavaScript bundles sent to the browser. Next.js splits these automatically based on the [module graph](#module-graph) to reduce initial payload size and load only the necessary code for each page. + +## Client Component + +A React component that runs in the browser. In Next.js, Client Components can also be rendered on the server during initial page generation. They can use state, effects, event handlers, and browser APIs, and are marked with the [`"use client"` directive](#use-client-directive) at the top of a file. Learn more in [Server and Client Components](/docs/app/getting-started/server-and-client-components). + +## Client-side navigation + +A navigation technique where the page content updates dynamically without a full page reload. Next.js uses client-side navigation with the [`` component](/docs/app/api-reference/components/link), keeping shared layouts interactive and preserving browser state. Learn more in [Linking and Navigating](/docs/app/getting-started/linking-and-navigating#client-side-transitions). + +## Code Splitting + +The process of dividing your application into smaller JavaScript chunks based on routes. Instead of loading all code upfront, only the code needed for the current route is loaded, reducing initial load time. Next.js automatically performs code splitting based on routes. Learn more in the [Package Bundling guide](/docs/app/guides/package-bundling). + +# D + +## Dynamic APIs + +Functions that access request-specific data, causing a component to opt into [dynamic rendering](#dynamic-rendering). These include: + +- [`cookies()`](/docs/app/api-reference/functions/cookies) - Access request cookies +- [`headers()`](/docs/app/api-reference/functions/headers) - Access request headers +- [`searchParams`](/docs/app/api-reference/file-conventions/page#searchparams-optional) - Access URL query parameters +- [`draftMode()`](/docs/app/api-reference/functions/draft-mode) - Enable or check draft mode + +## Dynamic rendering + +When a component is rendered at [request time](#request-time) rather than [build time](#build-time). A component becomes dynamic when it uses [Dynamic APIs](#dynamic-apis). + +## Dynamic route segments + +[Route segments](#route-segment) that are generated from data at request time. Created by wrapping a folder name in square brackets (e.g., `[slug]`), they allow you to create routes from dynamic data like blog posts or product pages. Learn more in [Dynamic Route Segments](/docs/app/api-reference/file-conventions/dynamic-routes). + +# E + +## Environment Variables + +Configuration values accessible at build time or request time. In Next.js, variables prefixed with `NEXT_PUBLIC_` are exposed to the browser, while others are only available server-side. Learn more in [Environment Variables](/docs/app/guides/environment-variables). + +## Error Boundary + +A React component that catches JavaScript errors in its child component tree and displays a fallback UI. In Next.js, create an [`error.js` file](/docs/app/api-reference/file-conventions/error) to automatically wrap a route segment in an error boundary. Learn more in [Error Handling](/docs/app/getting-started/error-handling). + +# F + +## Font Optimization + +Automatic font optimization using [`next/font`](/docs/app/api-reference/components/font). Next.js self-hosts fonts, eliminates layout shift, and applies best practices for performance. Works with Google Fonts and local font files. Learn more in [Fonts](/docs/app/getting-started/fonts). + +## File-system caching + +A Turbopack feature that stores compiler artifacts on disk between runs, reducing work across `next dev` or `next build` commands for significantly faster compile times. Learn more in [Turbopack FileSystem Caching](/docs/app/api-reference/config/next-config-js/turbopackFileSystemCache). + +# H + +## Hydration + +React's process of attaching event handlers to the DOM to make server-rendered static HTML interactive. During hydration, React reconciles the server-rendered markup with the client-side JavaScript. Learn more in [Server and Client Components](/docs/app/getting-started/server-and-client-components#how-do-server-and-client-components-work-in-nextjs). + +# I + +## Import Aliases + +Custom path mappings that provide shorthand references for frequently used directories. Import aliases reduce the complexity of relative imports and make code more readable and maintainable. Learn more in [Absolute Imports and Module Path Aliases](/docs/app/getting-started/installation#set-up-absolute-imports-and-module-path-aliases). + +## Incremental Static Regeneration (ISR) + +A technique that allows you to update static content without rebuilding the entire site. ISR enables you to use static generation on a per-page basis while revalidating pages in the background as traffic comes in. Learn more in the [ISR guide](/docs/app/guides/incremental-static-regeneration). + +> **Good to know**: In Next.js, ISR is also known as [Revalidation](#revalidation). + +## Intercepting Routes + +A routing pattern that allows loading a route from another part of your application within the current layout. Useful for displaying content (like modals) without the user switching context, while keeping the URL shareable. Learn more in [Intercepting Routes](/docs/app/api-reference/file-conventions/intercepting-routes). + +## Image Optimization + +Automatic image optimization using the [`` component](/docs/app/api-reference/components/image). Next.js optimizes images on-demand, serves them in modern formats like WebP, and automatically handles lazy loading and responsive sizing. Learn more in [Images](/docs/app/getting-started/images). + +# L + +## Layout + +UI that is shared between multiple pages. Layouts preserve state, remain interactive, and do not re-render on navigation. Defined by exporting a React component from a [`layout.js` file](/docs/app/api-reference/file-conventions/layout). Learn more in [Layouts and Pages](/docs/app/getting-started/layouts-and-pages). + +## Loading UI + +Fallback UI shown while a [route segment](#route-segment) is loading. Created by adding a [`loading.js` file](/docs/app/api-reference/file-conventions/loading) to a folder, which automatically wraps the page in a [Suspense boundary](#suspense-boundary). Learn more in [Loading UI](/docs/app/api-reference/file-conventions/loading). + +# M + +## Module Graph + +A graph of file dependencies in your app. Each file (module) is a node, and import/export relationships form the edges. Next.js analyzes this graph to determine optimal bundling and code-splitting strategies. Learn more in [Server and Client Components](/docs/app/getting-started/server-and-client-components#reducing-js-bundle-size). + +## Metadata + +Information about a page used by browsers and search engines, such as title, description, and Open Graph images. In Next.js, define metadata using the [`metadata` export](/docs/app/api-reference/functions/generate-metadata) or [`generateMetadata` function](/docs/app/api-reference/functions/generate-metadata) in layout or page files. Learn more in [Metadata and OG Images](/docs/app/getting-started/metadata-and-og-images). + +## Memoization + +Caching the return value of a function so that calling the same function multiple times during a render pass (request) only executes it once. In Next.js, fetch requests with the same URL and options are automatically memoized. Learn more about [React Cache](https://react.dev/reference/react/cache). + +# N + +## Not Found + +A special component shown when a route doesn't exist or when the [`notFound()` function](/docs/app/api-reference/functions/not-found) is called. Created by adding a [`not-found.js` file](/docs/app/api-reference/file-conventions/not-found) to your app directory. Learn more in [Error Handling](/docs/app/getting-started/error-handling#not-found). + +# P + +## Private Folders + +Folders prefixed with an underscore (e.g., `_components`) that are excluded from the routing system. These folders are used for code organization and shared utilities without creating accessible routes. Learn more in [Private Folders](/docs/app/getting-started/project-structure#private-folders). + +## Page + +UI that is unique to a route. Defined by exporting a React component from a [`page.js` file](/docs/app/api-reference/file-conventions/page) within the `app` directory. Learn more in [Layouts and Pages](/docs/app/getting-started/layouts-and-pages). + +## Parallel Routes + +A pattern that allows simultaneously or conditionally rendering multiple pages within the same layout. Created using named slots with the `@folder` convention, useful for dashboards, modals, and complex layouts. Learn more in [Parallel Routes](/docs/app/api-reference/file-conventions/parallel-routes). + +## Partial Prerendering (PPR) + +A rendering optimization that combines static and dynamic rendering in a single route. The static shell is served immediately while dynamic content streams in when ready, providing the best of both rendering strategies. Learn more in [Cache Components](/docs/app/getting-started/cache-components). + +## Prefetching + +Loading a route in the background before the user navigates to it. Next.js automatically prefetches routes linked with the [`` component](/docs/app/api-reference/components/link) when they enter the viewport, making navigation feel instant. Learn more in the [Prefetching guide](/docs/app/guides/prefetching). + +## Prerendering + +Generating HTML for a page ahead of time, either at build time (static rendering) or in the background (revalidation). The pre-rendered HTML is served immediately, then hydrated on the client. + +## Proxy + +A file ([`proxy.js`](/docs/app/api-reference/file-conventions/proxy)) that runs code on the server before request is completed. Used to implement server-side logic like logging, redirects, and rewrites. Formerly known as Middleware. Learn more in the [Proxy guide](/docs/app/getting-started/proxy). + +# R + +## Redirect + +Sending users from one URL to another. In Next.js, redirects can be configured in [`next.config.js`](/docs/app/api-reference/config/next-config-js/redirects), returned from [Proxy](/docs/app/api-reference/file-conventions/proxy), or triggered programmatically with the [`redirect()` function](/docs/app/api-reference/functions/redirect). Learn more in [Redirecting](/docs/app/guides/redirecting). + +## Request time + +The time when a user makes a request to your application. At request time, dynamic routes are rendered, cookies and headers are accessible, and request-specific data can be used. + +## Revalidation + +The process of updating cached data. Can be time-based (using [`cacheLife()`](/docs/app/api-reference/functions/cacheLife) to set cache duration) or on-demand (using [`cacheTag()`](/docs/app/api-reference/functions/cacheTag) to tag data, then [`updateTag()`](/docs/app/api-reference/functions/updateTag) to invalidate). Learn more in [Caching and Revalidating](/docs/app/getting-started/caching-and-revalidating). + +## Rewrite + +Mapping an incoming request path to a different destination path without changing the URL in the browser. Configured in [`next.config.js`](/docs/app/api-reference/config/next-config-js/rewrites) or returned from [Proxy](/docs/app/api-reference/file-conventions/proxy). Useful for proxying to external services or legacy URLs. + +## Route Groups + +A way to organize routes without affecting the URL structure. Created by wrapping a folder name in parentheses (e.g., `(marketing)`), route groups help organize related routes and enable per-group [layouts](#layout). Learn more in [Route Groups](/docs/app/api-reference/file-conventions/route-groups). + +## Route Handler + +A function that handles HTTP requests for a specific route, defined in a [`route.js` file](/docs/app/api-reference/file-conventions/route). Route Handlers use the Web Request and Response APIs and can handle `GET`, `POST`, `PUT`, `PATCH`, `DELETE`, `HEAD`, and `OPTIONS` methods. Learn more in [Route Handlers](/docs/app/getting-started/route-handlers). + +## Route Segment + +A part of the URL path (between two slashes) defined by a folder in the `app` directory. Each folder represents a segment in the URL structure. Learn more in [Project Structure](/docs/app/getting-started/project-structure). + +## RSC Payload + +The React Server Component Payload—a compact binary representation of the rendered React Server Components tree. Contains the rendered result of Server Components, placeholders for Client Components, and props passed between them. Learn more in [Server and Client Components](/docs/app/getting-started/server-and-client-components#how-do-server-and-client-components-work-in-nextjs). + +# S + +## Server Component + +The default component type in the App Router. Server Components render on the server, can fetch data directly, and don't add to the client JavaScript bundle. They cannot use state or browser APIs. Learn more in [Server and Client Components](/docs/app/getting-started/server-and-client-components). + +## Server Action + +A [Server Function](#server-function) that is passed to a Client Component as a prop or bound to a form action. Server Actions are commonly used for form submissions and data mutations. Learn more in [Server Actions and Mutations](/docs/app/getting-started/updating-data). + +## Server Function + +An asynchronous function that runs on the server, marked with the [`"use server"` directive](/docs/app/api-reference/directives/use-server). Server Functions can be invoked from Client Components. When passed as a prop to a Client Component or bound to a form action, they are called [Server Actions](#server-action). Learn more in [React Server Functions](https://react.dev/reference/rsc/server-functions). + +## Static Export + +A deployment mode that generates a fully static site with HTML, CSS, and JavaScript files. Enabled by setting `output: 'export'` in `next.config.js`. Static exports can be hosted on any static file server without a Node.js server. Learn more in [Static Exports](/docs/app/guides/static-exports). + +## Static rendering + +When a component is rendered at [build time](#build-time) or in the background during [revalidation](#revalidation). The result is cached and can be served from a CDN. Static rendering is the default for components that don't use [Dynamic APIs](#dynamic-apis). + +## Static Assets + +Files such as images, fonts, videos, and other media that are served directly without processing. Static assets are typically stored in the `public` directory and referenced by their relative paths. Learn more in [Static Assets](/docs/app/api-reference/file-conventions/public-folder). + +## Static Shell + +The pre-rendered HTML structure of a page that's served immediately to the browser. With [Partial Prerendering](#partial-prerendering-ppr), the static shell includes all statically renderable content plus [Suspense boundary](#suspense-boundary) fallbacks for dynamic content that streams in later. + +## Streaming + +A technique that allows the server to send parts of a page to the client as they become ready, rather than waiting for the entire page to render. Enabled automatically with [`loading.js`](/docs/app/api-reference/file-conventions/loading) or manual `` boundaries. Learn more in [Linking and Navigating - Streaming](/docs/app/getting-started/linking-and-navigating#streaming). + +## Suspense boundary + +A React [``](https://react.dev/reference/react/Suspense) component that wraps async content and displays fallback UI while it loads. In Next.js, Suspense boundaries define where the [static shell](#static-shell) ends and [streaming](#streaming) begins, enabling [Partial Prerendering](#partial-prerendering-ppr). + +# T + +## Turbopack + +A fast, Rust-based bundler built for Next.js. Turbopack is the default bundler for `next dev` and available for `next build`. It provides significantly faster compilation times compared to Webpack. Learn more in [Turbopack](/docs/app/api-reference/turbopack). + +## Tree Shaking + +The process of removing unused code from your JavaScript bundles during the build process. Next.js automatically tree-shakes your code to reduce bundle sizes. Learn more in the [Package Bundling guide](/docs/app/guides/package-bundling). + +# U + +## `"use cache"` Directive + +A directive that marks a component or function as cacheable. It can be placed at the top of a file to indicate that all exports in the file are cacheable, or inline at the top of a function or component to mark that specific scope as cacheable. Learn more in the [`"use cache"` reference](/docs/app/api-reference/directives/use-cache). + +## `"use client"` Directive + +A special React directive that marks the boundary between server and client code. It must be placed at the top of a file, before any imports or other code. It indicates that React Components, helper functions, variable declarations, and all imported dependencies should be included in the [client bundle](#client-bundles). Learn more in the [`"use client"` reference](/docs/app/api-reference/directives/use-client). + +## `"use server"` Directive + +A directive that marks a function as a [Server Function](#server-function) that can be called from client-side code. It can be placed at the top of a file to indicate that all exports in the file are Server Functions, or inline at the top of a function to mark that specific function. Learn more in the [`"use server"` reference](/docs/app/api-reference/directives/use-server). From 631b2c60c89352e794b7d863db79457a278fe2ac Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Fri, 23 Jan 2026 13:42:30 +0100 Subject: [PATCH 10/10] Add Graphite workflow Cursor command (#88939) ### What? Adds a Cursor command file (`.cursor/commands/gt-workflow.md`) that provides guidance on using Graphite (gt) instead of git for branch and commit operations. ### Why? To make the Graphite workflow easily accessible within Cursor IDE via the `/gt-workflow` command, helping contributors follow the repository's git workflow conventions. ### How? Creates a new Markdown command file that includes: - Quick reference table of Graphite commands - Safety rules for stack operations - Multi-branch fix workflow example - Checklist for verification --- .cursor/commands/gt-workflow.md | 67 +++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .cursor/commands/gt-workflow.md diff --git a/.cursor/commands/gt-workflow.md b/.cursor/commands/gt-workflow.md new file mode 100644 index 00000000000000..f7e8d02330424f --- /dev/null +++ b/.cursor/commands/gt-workflow.md @@ -0,0 +1,67 @@ +# Git Workflow with Graphite + +## Overview + +Use Graphite (`gt`) instead of git for ALL branch and commit operations in this repository. + +## Forbidden Git Commands + +NEVER use these git commands directly: + +- `git push` → use `gt submit --no-edit` +- `git branch` → use `gt create` + +## Graphite Commands + +| Command | Description | +| -------------------------------------- | ------------------------------------------- | +| `gt create -m "message"` | Create a new branch with commit | +| `gt modify -a --no-edit` | Stage all and amend current branch's commit | +| `gt checkout ` | Switch branches | +| `gt sync` | Sync and restack all branches | +| `gt submit --no-edit` | Push and create/update PRs | +| `gt log short` | View stack status | + +## Creating PRs with Descriptions + +All PRs require a description. Use this workflow: + +```bash +gt submit --no-edit +gh pr edit --body "Place description here" +``` + +## Safety Rules + +- Graphite force-pushes everything - old commits only recoverable via reflog +- Never have uncommitted changes when switching branches - they get lost during restack +- Never use `git stash` with Graphite - causes conflicts when `gt modify` restacks +- Never use `git checkout HEAD -- ` after editing - silently restores unfixed version +- Always use `gt checkout` (not `git checkout`) to switch branches +- `gt modify --no-edit` with unstaged/untracked files stages ALL changes +- `gt sync` pulls FROM remote, doesn't push TO remote +- `gt modify` restacks children locally but doesn't push them +- Always verify with `git status -sb` after stack operations + +## Safe Multi-Branch Fix Workflow + +```bash +gt checkout parent-branch +# make edits +gt modify -a --no-edit # Stage all, amend, restack children +git show HEAD -- # VERIFY fix is in commit +gt submit --no-edit # Push immediately + +gt checkout child-branch # Already restacked from gt modify +# make edits +gt modify -a --no-edit +git show HEAD -- # VERIFY +gt submit --no-edit +``` + +## Checklist + +- [ ] Using `gt` commands instead of `git push`/`git branch` +- [ ] No uncommitted changes before switching branches +- [ ] Verified changes with `git status -sb` after stack operations +- [ ] PR description follows `.github/pull_request_template.md` format