Skip to content

Commit d330322

Browse files
authored
Turbopack: ensure max merge segments is respected accros families (#82223)
### What? We try to limit the maximum number of merge jobs with the `max_merge_segment_count` option. But as we run `get_merge_segments` for every key family it only limits the merge jobs per family. This results in much more merge jobs then configured. This change applied the limits across families. Lower number families will be compacted first, which is a new behavior, so maybe we want to reorder the key families to improve the order (see enum KeySpace)
1 parent b8eba4f commit d330322

File tree

2 files changed

+16
-3
lines changed

2 files changed

+16
-3
lines changed

turbopack/crates/turbo-persistence/src/compaction/selector.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ pub fn compute_metrics<T: Compactable>(
104104
}
105105

106106
/// Configuration for the compaction algorithm.
107+
#[derive(Clone)]
107108
pub struct CompactConfig {
108109
/// The minimum number of files to merge at once.
109110
pub min_merge_count: usize,

turbopack/crates/turbo-persistence/src/db.rs

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -787,16 +787,28 @@ impl TurboPersistence {
787787
keys_written: u64,
788788
}
789789

790+
let mut compact_config = compact_config.clone();
791+
let merge_jobs = sst_by_family
792+
.iter()
793+
.map(|ssts_with_ranges| {
794+
if compact_config.max_merge_segment_count == 0 {
795+
return Vec::new();
796+
}
797+
let merge_jobs = get_merge_segments(ssts_with_ranges, &compact_config);
798+
compact_config.max_merge_segment_count -= merge_jobs.len();
799+
merge_jobs
800+
})
801+
.collect::<Vec<_>>();
802+
790803
let result = sst_by_family
791804
.into_par_iter()
805+
.zip(merge_jobs.into_par_iter())
792806
.with_min_len(1)
793807
.enumerate()
794-
.map(|(family, ssts_with_ranges)| {
808+
.map(|(family, (ssts_with_ranges, merge_jobs))| {
795809
let family = family as u32;
796810
let _span = span.clone().entered();
797811

798-
let merge_jobs = get_merge_segments(&ssts_with_ranges, compact_config);
799-
800812
if merge_jobs.is_empty() {
801813
return Ok(PartialResultPerFamily {
802814
new_meta_file: None,

0 commit comments

Comments
 (0)