Turbopack: Compact only at the end for short sessions

Author: sokra

crates/napi/src/next_api/project.rs

@@ -253,6 +253,8 @@ pub struct NapiTurboEngineOptions {
     pub dependency_tracking: Option<bool>,
     /// Whether the project is running in a CI environment.
     pub is_ci: Option<bool>,
+    /// Whether the project is running in a short session.
+    pub is_short_session: Option<bool>,
 }
 
 impl From<NapiWatchOptions> for WatchOptions {
@@ -434,12 +436,14 @@ pub fn project_new(
         let persistent_caching = turbo_engine_options.persistent_caching.unwrap_or_default();
         let dependency_tracking = turbo_engine_options.dependency_tracking.unwrap_or(true);
         let is_ci = turbo_engine_options.is_ci.unwrap_or(false);
+        let is_short_session = turbo_engine_options.is_short_session.unwrap_or(false);
         let turbo_tasks = create_turbo_tasks(
             PathBuf::from(&options.dist_dir),
             persistent_caching,
             memory_limit,
             dependency_tracking,
             is_ci,
+            is_short_session,
         )?;
         let turbopack_ctx = NextTurbopackContext::new(turbo_tasks.clone(), napi_callbacks);
 

crates/napi/src/next_api/turbopack_ctx.rs

@@ -185,15 +185,20 @@ pub fn create_turbo_tasks(
     _memory_limit: usize,
     dependency_tracking: bool,
     is_ci: bool,
+    is_short_session: bool,
 ) -> Result<NextTurboTasks> {
     Ok(if persistent_caching {
         let version_info = GitVersionInfo {
             describe: env!("VERGEN_GIT_DESCRIBE"),
             dirty: option_env!("CI").is_none_or(|value| value.is_empty())
                 && env!("VERGEN_GIT_DIRTY") == "true",
         };
-        let (backing_storage, cache_state) =
-            default_backing_storage(&output_path.join("cache/turbopack"), &version_info, is_ci)?;
+        let (backing_storage, cache_state) = default_backing_storage(
+            &output_path.join("cache/turbopack"),
+            &version_info,
+            is_ci,
+            is_short_session,
+        )?;
         let tt = TurboTasks::new(TurboTasksBackend::new(
             BackendOptions {
                 storage_mode: Some(if std::env::var("TURBO_ENGINE_READ_ONLY").is_ok() {

packages/next/src/build/swc/generated-native.d.ts

@@ -212,6 +212,8 @@ export interface NapiTurboEngineOptions {
   dependencyTracking?: boolean
   /** Whether the project is running in a CI environment. */
   isCi?: boolean
+  /** Whether the project is running in a short session. */
+  isShortSession?: boolean
 }
 export declare function projectNew(
   options: NapiProjectOptions,

packages/next/src/build/turbopack-build/impl.ts

@@ -91,6 +91,7 @@ export async function turbopackBuild(): Promise<{
       memoryLimit: config.experimental?.turbopackMemoryLimit,
       dependencyTracking: persistentCaching,
       isCi: isCI,
+      isShortSession: true,
     }
   )
   try {

packages/next/src/server/dev/hot-reloader-turbopack.ts

@@ -254,6 +254,7 @@ export async function createHotReloaderTurbopack(
     {
       persistentCaching: isPersistentCachingEnabled(opts.nextConfig),
       memoryLimit: opts.nextConfig.experimental?.turbopackMemoryLimit,
+      isShortSession: false,
     }
   )
   backgroundLogCompilationEvents(project, {

turbopack/crates/turbo-tasks-backend/src/database/turbo.rs

@@ -30,14 +30,18 @@ const COMPACT_CONFIG: CompactConfig = CompactConfig {
 pub struct TurboKeyValueDatabase {
     db: Arc<TurboPersistence>,
     compact_join_handle: Mutex<Option<JoinHandle<Result<()>>>>,
+    is_ci: bool,
+    is_short_session: bool,
 }
 
 impl TurboKeyValueDatabase {
-    pub fn new(versioned_path: PathBuf) -> Result<Self> {
+    pub fn new(versioned_path: PathBuf, is_ci: bool, is_short_session: bool) -> Result<Self> {
         let db = Arc::new(TurboPersistence::open(versioned_path)?);
         let mut this = Self {
             db: db.clone(),
             compact_join_handle: Mutex::new(None),
+            is_ci,
+            is_short_session,
         };
         // start compaction in background if the database is not empty
         if !db.is_empty() {
@@ -98,8 +102,8 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
         Ok(WriteBatch::concurrent(TurboWriteBatch {
             batch: self.db.write_batch()?,
             db: &self.db,
-            compact_join_handle: &self.compact_join_handle,
-            initial_write: self.db.is_empty(),
+            compact_join_handle: (!self.is_short_session && !self.db.is_empty())
+                .then_some(&self.compact_join_handle),
         }))
     }
 
@@ -110,6 +114,16 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
         if let Some(join_handle) = self.compact_join_handle.lock().take() {
             join_handle.join().unwrap()?;
         }
+        // Compact the database on shutdown
+        self.db.compact(&CompactConfig {
+            max_merge_segment_count: if self.is_ci {
+                // Fully compact in CI to reduce cache size
+                usize::MAX
+            } else {
+                available_parallelism().map_or(4, |c| max(4, c.get()))
+            },
+            ..COMPACT_CONFIG
+        })?;
         // Shutdown the database
         self.db.shutdown()
     }
@@ -118,8 +132,7 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
 pub struct TurboWriteBatch<'a> {
     batch: turbo_persistence::WriteBatch<WriteBuffer<'static>, 5>,
     db: &'a Arc<TurboPersistence>,
-    compact_join_handle: &'a Mutex<Option<JoinHandle<Result<()>>>>,
-    initial_write: bool,
+    compact_join_handle: Option<&'a Mutex<Option<JoinHandle<Result<()>>>>>,
 }
 
 impl<'a> BaseWriteBatch<'a> for TurboWriteBatch<'a> {
@@ -140,7 +153,7 @@ impl<'a> BaseWriteBatch<'a> for TurboWriteBatch<'a> {
         // Commit the write batch
         self.db.commit_write_batch(self.batch)?;
 
-        if !self.initial_write {
+        if let Some(compact_join_handle) = self.compact_join_handle {
             // Start a new compaction in the background
             let db = self.db.clone();
             let handle = spawn(move || {
@@ -150,7 +163,7 @@ impl<'a> BaseWriteBatch<'a> for TurboWriteBatch<'a> {
                     ..COMPACT_CONFIG
                 })
             });
-            self.compact_join_handle.lock().replace(handle);
+            compact_join_handle.lock().replace(handle);
         }
 
         Ok(())

turbopack/crates/turbo-tasks-backend/src/lib.rs

@@ -83,12 +83,13 @@ pub fn turbo_backing_storage(
     base_path: &Path,
     version_info: &GitVersionInfo,
     is_ci: bool,
+    is_short_session: bool,
 ) -> Result<(TurboBackingStorage, StartupCacheState)> {
     KeyValueDatabaseBackingStorage::open_versioned_on_disk(
         base_path.to_owned(),
         version_info,
         is_ci,
-        TurboKeyValueDatabase::new,
+        |path| TurboKeyValueDatabase::new(path, is_ci, is_short_session),
     )
 }
 
@@ -111,13 +112,14 @@ pub fn default_backing_storage(
     path: &Path,
     version_info: &GitVersionInfo,
     is_ci: bool,
+    is_short_session: bool,
 ) -> Result<(DefaultBackingStorage, StartupCacheState)> {
     #[cfg(feature = "lmdb")]
     {
         lmdb_backing_storage(path, version_info, is_ci)
     }
     #[cfg(not(feature = "lmdb"))]
     {
-        turbo_backing_storage(path, version_info, is_ci)
+        turbo_backing_storage(path, version_info, is_ci, is_short_session)
     }
 }

turbopack/crates/turbo-tasks-backend/tests/test_config.trs

@@ -16,7 +16,8 @@
           describe: "test-unversioned",
           dirty: false,
         },
-        false
+        false,
+        true,
       ).unwrap().0
     )
   )

turbopack/crates/turbo-tasks-fetch/tests/test_config.trs

@@ -16,7 +16,8 @@
           describe: "test-unversioned",
           dirty: false,
         },
-        false
+        false,
+        true,
       ).unwrap().0
     )
   )

turbopack/crates/turbopack/tests/node-file-trace.rs

@@ -292,6 +292,7 @@ fn node_file_trace_persistent(#[case] input: CaseInput) {
                     dirty: false,
                 },
                 false,
+                true,
             )
             .unwrap()
             .0,