Turbopack: Compact only at the end for short sessions (#82224)

### What?

Change the compaction behavior for "short sessions" (like next build). A longer "short session" might commit to the DB multiple times (every 60s currently), which previously resulted in compactions happening after multiple commits to the database. As compactions are expensive and memory hungry, we want to avoid them until the end of the session. At the end of the session we can run it in parallel to node.js SSG.

Author: sokra

crates/napi/src/next_api/project.rs

@@ -253,6 +253,8 @@ pub struct NapiTurboEngineOptions {
     pub dependency_tracking: Option<bool>,
     /// Whether the project is running in a CI environment.
     pub is_ci: Option<bool>,
+    /// Whether the project is running in a short session.
+    pub is_short_session: Option<bool>,
 }
 
 impl From<NapiWatchOptions> for WatchOptions {
@@ -434,12 +436,14 @@ pub fn project_new(
         let persistent_caching = turbo_engine_options.persistent_caching.unwrap_or_default();
         let dependency_tracking = turbo_engine_options.dependency_tracking.unwrap_or(true);
         let is_ci = turbo_engine_options.is_ci.unwrap_or(false);
+        let is_short_session = turbo_engine_options.is_short_session.unwrap_or(false);
         let turbo_tasks = create_turbo_tasks(
             PathBuf::from(&options.dist_dir),
             persistent_caching,
             memory_limit,
             dependency_tracking,
             is_ci,
+            is_short_session,
         )?;
         let turbopack_ctx = NextTurbopackContext::new(turbo_tasks.clone(), napi_callbacks);
 

crates/napi/src/next_api/turbopack_ctx.rs

@@ -185,15 +185,20 @@ pub fn create_turbo_tasks(
     _memory_limit: usize,
     dependency_tracking: bool,
     is_ci: bool,
+    is_short_session: bool,
 ) -> Result<NextTurboTasks> {
     Ok(if persistent_caching {
         let version_info = GitVersionInfo {
             describe: env!("VERGEN_GIT_DESCRIBE"),
             dirty: option_env!("CI").is_none_or(|value| value.is_empty())
                 && env!("VERGEN_GIT_DIRTY") == "true",
         };
-        let (backing_storage, cache_state) =
-            default_backing_storage(&output_path.join("cache/turbopack"), &version_info, is_ci)?;
+        let (backing_storage, cache_state) = default_backing_storage(
+            &output_path.join("cache/turbopack"),
+            &version_info,
+            is_ci,
+            is_short_session,
+        )?;
         let tt = TurboTasks::new(TurboTasksBackend::new(
             BackendOptions {
                 storage_mode: Some(if std::env::var("TURBO_ENGINE_READ_ONLY").is_ok() {

packages/next/src/build/swc/generated-native.d.ts

@@ -212,6 +212,8 @@ export interface NapiTurboEngineOptions {
   dependencyTracking?: boolean
   /** Whether the project is running in a CI environment. */
   isCi?: boolean
+  /** Whether the project is running in a short session. */
+  isShortSession?: boolean
 }
 export declare function projectNew(
   options: NapiProjectOptions,

packages/next/src/build/turbopack-build/impl.ts

@@ -91,6 +91,7 @@ export async function turbopackBuild(): Promise<{
       memoryLimit: config.experimental?.turbopackMemoryLimit,
       dependencyTracking: persistentCaching,
       isCi: isCI,
+      isShortSession: true,
     }
   )
   try {

packages/next/src/server/dev/hot-reloader-turbopack.ts

@@ -254,6 +254,7 @@ export async function createHotReloaderTurbopack(
     {
       persistentCaching: isPersistentCachingEnabled(opts.nextConfig),
       memoryLimit: opts.nextConfig.experimental?.turbopackMemoryLimit,
+      isShortSession: false,
     }
   )
   backgroundLogCompilationEvents(project, {

turbopack/crates/turbo-tasks-backend/src/database/turbo.rs

@@ -30,14 +30,18 @@ const COMPACT_CONFIG: CompactConfig = CompactConfig {
 pub struct TurboKeyValueDatabase {
     db: Arc<TurboPersistence>,
     compact_join_handle: Mutex<Option<JoinHandle<Result<()>>>>,
+    is_ci: bool,
+    is_short_session: bool,
 }
 
 impl TurboKeyValueDatabase {
-    pub fn new(versioned_path: PathBuf) -> Result<Self> {
+    pub fn new(versioned_path: PathBuf, is_ci: bool, is_short_session: bool) -> Result<Self> {
         let db = Arc::new(TurboPersistence::open(versioned_path)?);
         let mut this = Self {
             db: db.clone(),
             compact_join_handle: Mutex::new(None),
+            is_ci,
+            is_short_session,
         };
         // start compaction in background if the database is not empty
         if !db.is_empty() {
@@ -98,8 +102,8 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
         Ok(WriteBatch::concurrent(TurboWriteBatch {
             batch: self.db.write_batch()?,
             db: &self.db,
-            compact_join_handle: &self.compact_join_handle,
-            initial_write: self.db.is_empty(),
+            compact_join_handle: (!self.is_short_session && !self.db.is_empty())
+                .then_some(&self.compact_join_handle),
         }))
     }
 
@@ -110,6 +114,16 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
         if let Some(join_handle) = self.compact_join_handle.lock().take() {
             join_handle.join().unwrap()?;
         }
+        // Compact the database on shutdown
+        self.db.compact(&CompactConfig {
+            max_merge_segment_count: if self.is_ci {
+                // Fully compact in CI to reduce cache size
+                usize::MAX
+            } else {
+                available_parallelism().map_or(4, |c| max(4, c.get()))
+            },
+            ..COMPACT_CONFIG
+        })?;
         // Shutdown the database
         self.db.shutdown()
     }
@@ -118,8 +132,7 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
 pub struct TurboWriteBatch<'a> {
     batch: turbo_persistence::WriteBatch<WriteBuffer<'static>, 5>,
     db: &'a Arc<TurboPersistence>,
-    compact_join_handle: &'a Mutex<Option<JoinHandle<Result<()>>>>,
-    initial_write: bool,
+    compact_join_handle: Option<&'a Mutex<Option<JoinHandle<Result<()>>>>>,
 }
 
 impl<'a> BaseWriteBatch<'a> for TurboWriteBatch<'a> {
@@ -140,7 +153,7 @@ impl<'a> BaseWriteBatch<'a> for TurboWriteBatch<'a> {
         // Commit the write batch
         self.db.commit_write_batch(self.batch)?;
 
-        if !self.initial_write {
+        if let Some(compact_join_handle) = self.compact_join_handle {
             // Start a new compaction in the background
             let db = self.db.clone();
             let handle = spawn(move || {
@@ -150,7 +163,7 @@ impl<'a> BaseWriteBatch<'a> for TurboWriteBatch<'a> {
                     ..COMPACT_CONFIG
                 })
             });
-            self.compact_join_handle.lock().replace(handle);
+            compact_join_handle.lock().replace(handle);
         }
 
         Ok(())

turbopack/crates/turbo-tasks-backend/src/lib.rs

@@ -83,12 +83,13 @@ pub fn turbo_backing_storage(
     base_path: &Path,
     version_info: &GitVersionInfo,
     is_ci: bool,
+    is_short_session: bool,
 ) -> Result<(TurboBackingStorage, StartupCacheState)> {
     KeyValueDatabaseBackingStorage::open_versioned_on_disk(
         base_path.to_owned(),
         version_info,
         is_ci,
-        TurboKeyValueDatabase::new,
+        |path| TurboKeyValueDatabase::new(path, is_ci, is_short_session),
     )
 }
 
@@ -111,13 +112,14 @@ pub fn default_backing_storage(
     path: &Path,
     version_info: &GitVersionInfo,
     is_ci: bool,
+    is_short_session: bool,
 ) -> Result<(DefaultBackingStorage, StartupCacheState)> {
     #[cfg(feature = "lmdb")]
     {
         lmdb_backing_storage(path, version_info, is_ci)
     }
     #[cfg(not(feature = "lmdb"))]
     {
-        turbo_backing_storage(path, version_info, is_ci)
+        turbo_backing_storage(path, version_info, is_ci, is_short_session)
     }
 }

turbopack/crates/turbo-tasks-backend/tests/test_config.trs

@@ -16,7 +16,8 @@
           describe: "test-unversioned",
           dirty: false,
         },
-        false
+        false,
+        true,
       ).unwrap().0
     )
   )

turbopack/crates/turbo-tasks-fetch/tests/test_config.trs

@@ -16,7 +16,8 @@
           describe: "test-unversioned",
           dirty: false,
         },
-        false
+        false,
+        true,
       ).unwrap().0
     )
   )

turbopack/crates/turbopack/tests/node-file-trace.rs

@@ -292,6 +292,7 @@ fn node_file_trace_persistent(#[case] input: CaseInput) {
                     dirty: false,
                 },
                 false,
+                true,
             )
             .unwrap()
             .0,