overlap: use overlapcache

We embed an overlap cache in `FileMetadata` and use it with the
overlap checker.

When we have to open a table, we now try to find a maximal empty
region and the two surrounding data regions so that we can populate
the cache with this information.

The cache will be very useful for optimistic overlap checks which
happen without blocking compactions, and might need to be retried.

Author: RaduBerinde

Diff for: internal/manifest/version.go

@@ -17,6 +17,7 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/internal/overlap/overlapcache"
 	"github.com/cockroachdb/pebble/sstable"
 )
 
@@ -287,6 +288,9 @@ type FileMetadata struct {
 
 	// SyntheticSuffix overrides all suffixes in a table; used for some virtual tables.
 	SyntheticSuffix sstable.SyntheticSuffix
+
+	// OverlapCache is used to speed up overlap checks during ingestion.
+	OverlapCache overlapcache.C
 }
 
 // InternalKeyBounds returns the set of overall table bounds.

Diff for: internal/overlap/checker.go

@@ -8,6 +8,7 @@ package overlap
 
 import (
 	"context"
+	"slices"
 
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/keyspan"
@@ -23,7 +24,8 @@ type WithLSM [manifest.NumLevels]WithLevel
 type WithLevel struct {
 	Result Kind
 	// SplitFile can be set only when result is OnlyBoundary. If it is set, this
-	// file can be split to free up the range of interest.
+	// file can be split to free up the range of interest. SplitFile is not set
+	// for L0 (overlapping tables are allowed in L0).
 	SplitFile *manifest.FileMetadata
 }
 
@@ -89,6 +91,7 @@ func (c *Checker) LSMOverlap(
 		}
 		if res.Result == OnlyBoundary {
 			result[0].Result = OnlyBoundary
+			// We don't set SplitFile for L0 (tables in L0 are allowed to overlap).
 		}
 	}
 	for level := 1; level < manifest.NumLevels; level++ {
@@ -135,11 +138,11 @@ func (c *Checker) LevelOverlap(
 		return WithLevel{Result: Data}, nil
 	}
 	// We have a single file to look at; its boundaries enclose our region.
-	empty, err := c.EmptyRegion(ctx, region, file)
+	overlap, err := c.DataOverlapWithFile(ctx, region, file)
 	if err != nil {
 		return WithLevel{}, err
 	}
-	if !empty {
+	if overlap {
 		return WithLevel{Result: Data}, nil
 	}
 	return WithLevel{
@@ -148,116 +151,183 @@ func (c *Checker) LevelOverlap(
 	}, nil
 }
 
-// EmptyRegion returns true if the given region doesn't overlap with any keys or
-// ranges in the given table.
-func (c *Checker) EmptyRegion(
+// DataOverlapWithFile returns true if the given region overlaps with any keys
+// or spans in the given table.
+func (c *Checker) DataOverlapWithFile(
 	ctx context.Context, region base.UserKeyBounds, m *manifest.FileMetadata,
 ) (bool, error) {
-	empty, err := c.emptyRegionPointsAndRangeDels(ctx, region, m)
-	if err != nil || !empty {
-		return empty, err
+	if overlap, ok := m.OverlapCache.CheckDataOverlap(c.cmp, region); ok {
+		return overlap, nil
 	}
-	return c.emptyRegionRangeKeys(ctx, region, m)
-}
+	// We want to check overlap with file, but we also want to update the cache
+	// with useful information. We try to find two data regions r1 and r2 with a
+	// space-in between; r1 ends before region.Start and r2 ends at or after
+	// region.Start. See overlapcache.C.ReportEmptyRegion().
+	var r1, r2 base.UserKeyBounds
 
-// emptyRegionPointsAndRangeDels returns true if the file doesn't contain any
-// point keys or range del spans that overlap with region.
-func (c *Checker) emptyRegionPointsAndRangeDels(
-	ctx context.Context, region base.UserKeyBounds, m *manifest.FileMetadata,
-) (bool, error) {
-	if !m.HasPointKeys {
-		return true, nil
+	if m.HasPointKeys {
+		lt, ge, err := c.pointKeysAroundKey(ctx, region.Start, m)
+		if err != nil {
+			return false, err
+		}
+		r1 = base.UserKeyBoundsInclusive(lt, lt)
+		r2 = base.UserKeyBoundsInclusive(ge, ge)
+
+		if err := c.extendRegionsWithSpans(ctx, &r1, &r2, region.Start, m, manifest.KeyTypePoint); err != nil {
+			return false, err
+		}
 	}
-	pointBounds := m.UserKeyBoundsByType(manifest.KeyTypePoint)
-	if !pointBounds.Overlaps(c.cmp, &region) {
+	if m.HasRangeKeys {
+		if err := c.extendRegionsWithSpans(ctx, &r1, &r2, region.Start, m, manifest.KeyTypeRange); err != nil {
+			return false, err
+		}
+	}
+	// If the regions now overlap or touch, it's all one big data region.
+	if r1.Start != nil && r2.Start != nil && c.cmp(r1.End.Key, r2.Start) >= 0 {
+		m.OverlapCache.ReportDataRegion(c.cmp, base.UserKeyBounds{
+			Start: r1.Start,
+			End:   r2.End,
+		})
 		return true, nil
 	}
+	m.OverlapCache.ReportEmptyRegion(c.cmp, r1, r2)
+	// There is overlap iff we overlap with r2.
+	overlap := r2.Start != nil && region.End.IsUpperBoundFor(c.cmp, r2.Start)
+	return overlap, nil
+}
+
+// pointKeysAroundKey returns two consecutive point keys: the greatest key that
+// is < key and the smallest key that is >= key. If there is no such key, the
+// corresponding return value is nil. Both lt and ge are nil if the file
+// contains no point keys.
+func (c *Checker) pointKeysAroundKey(
+	ctx context.Context, key []byte, m *manifest.FileMetadata,
+) (lt, ge []byte, _ error) {
+	pointBounds := m.UserKeyBoundsByType(manifest.KeyTypePoint)
+
 	points, err := c.iteratorFactory.Points(ctx, m)
-	if err != nil {
-		return false, err
+	if points == nil || err != nil {
+		return nil, nil, err
 	}
-	if points != nil {
-		defer points.Close()
-		var kv *base.InternalKV
-		if c.cmp(region.Start, pointBounds.Start) <= 0 {
-			kv = points.First()
-		} else {
-			kv = points.SeekGE(region.Start, base.SeekGEFlagsNone)
+	defer points.Close()
+	switch {
+	case c.cmp(key, pointBounds.Start) <= 0:
+		kv := points.First()
+		if kv != nil {
+			ge = slices.Clone(kv.K.UserKey)
 		}
-		if kv == nil && points.Error() != nil {
-			return false, points.Error()
+	case c.cmp(key, pointBounds.End.Key) > 0:
+		kv := points.Last()
+		if kv != nil {
+			lt = slices.Clone(kv.K.UserKey)
 		}
-		if kv != nil && region.End.IsUpperBoundForInternalKey(c.cmp, kv.K) {
-			// Found overlap.
-			return false, nil
+	default:
+		kv := points.SeekLT(key, base.SeekLTFlagsNone)
+		if kv != nil {
+			lt = slices.Clone(kv.K.UserKey)
 		}
-	}
-	rangeDels, err := c.iteratorFactory.RangeDels(ctx, m)
-	if err != nil {
-		return false, err
-	}
-	if rangeDels != nil {
-		defer rangeDels.Close()
-		empty, err := c.emptyFragmentRegion(region, pointBounds.Start, rangeDels)
-		if err != nil || !empty {
-			return empty, err
+		if kv = points.Next(); kv != nil {
+			ge = slices.Clone(kv.K.UserKey)
 		}
 	}
-	// Found no overlap.
-	return true, nil
+	return lt, ge, points.Error()
 }
 
-// emptyRegionRangeKeys returns true if the file doesn't contain any range key
-// spans that overlap with region.
-func (c *Checker) emptyRegionRangeKeys(
-	ctx context.Context, region base.UserKeyBounds, m *manifest.FileMetadata,
-) (bool, error) {
-	if !m.HasRangeKeys {
-		return true, nil
-	}
-	rangeKeyBounds := m.UserKeyBoundsByType(manifest.KeyTypeRange)
-	if !rangeKeyBounds.Overlaps(c.cmp, &region) {
-		return true, nil
+// extendRegionsWithSpans opens a fragment iterator for either range dels or
+// range keys (depending n keyType), finds the last span that ends before key
+// and the following span, and extends/replaces regions r1 and r2.
+func (c *Checker) extendRegionsWithSpans(
+	ctx context.Context,
+	r1, r2 *base.UserKeyBounds,
+	key []byte,
+	m *manifest.FileMetadata,
+	keyType manifest.KeyType,
+) error {
+	var iter keyspan.FragmentIterator
+	var err error
+	if keyType == manifest.KeyTypePoint {
+		iter, err = c.iteratorFactory.RangeDels(ctx, m)
+	} else {
+		iter, err = c.iteratorFactory.RangeKeys(ctx, m)
 	}
-	rangeKeys, err := c.iteratorFactory.RangeKeys(ctx, m)
-	if err != nil {
-		return false, err
+	if iter == nil || err != nil {
+		return err
 	}
-	if rangeKeys != nil {
-		defer rangeKeys.Close()
-		empty, err := c.emptyFragmentRegion(region, rangeKeyBounds.Start, rangeKeys)
-		if err != nil || !empty {
-			return empty, err
+	defer iter.Close()
+
+	fragmentBounds := m.UserKeyBoundsByType(keyType)
+	switch {
+	case c.cmp(key, fragmentBounds.Start) <= 0:
+		span, err := iter.First()
+		if err != nil {
+			return err
+		}
+		c.updateR2(r2, span)
+
+	case !fragmentBounds.End.IsUpperBoundFor(c.cmp, key):
+		span, err := iter.Last()
+		if err != nil {
+			return err
+		}
+		c.updateR1(r1, span)
+
+	default:
+		span, err := iter.SeekGE(key)
+		if err != nil {
+			return err
 		}
+		c.updateR2(r2, span)
+		span, err = iter.Prev()
+		if err != nil {
+			return err
+		}
+		c.updateR1(r1, span)
 	}
-	// Found no overlap.
-	return true, nil
+	return nil
 }
 
-// emptyFragmentRegion returns true if the given iterator doesn't contain any
-// spans that overlap with region. The fragmentLowerBounds is a known lower
-// bound for all the spans.
-func (c *Checker) emptyFragmentRegion(
-	region base.UserKeyBounds, fragmentLowerBound []byte, fragments keyspan.FragmentIterator,
-) (bool, error) {
-	var span *keyspan.Span
-	var err error
-	if c.cmp(region.Start, fragmentLowerBound) <= 0 {
-		// This is an optimization: we know there are no spans before region.Start,
-		// so we can use First.
-		span, err = fragments.First()
-	} else {
-		span, err = fragments.SeekGE(region.Start)
-	}
-	if err != nil {
-		return false, err
-	}
-	if span != nil && span.Empty() {
-		return false, base.AssertionFailedf("fragment iterator produced empty span")
+// updateR1 updates r1, the region of data that ends before a key of interest.
+func (c *Checker) updateR1(r1 *base.UserKeyBounds, s *keyspan.Span) {
+	switch {
+	case s == nil:
+
+	case r1.Start == nil || c.cmp(r1.End.Key, s.Start) < 0:
+		// Region completely to the right of r1.
+		*r1 = base.UserKeyBoundsEndExclusive(slices.Clone(s.Start), slices.Clone(s.End))
+
+	case c.cmp(s.End, r1.Start) < 0:
+		// Region completely to the left of r1, nothing to do.
+
+	default:
+		// Regions are overlapping or touching.
+		if c.cmp(s.Start, r1.Start) < 0 {
+			r1.Start = slices.Clone(s.Start)
+		}
+		if c.cmp(r1.End.Key, s.End) < 0 {
+			r1.End = base.UserKeyExclusive(slices.Clone(s.End))
+		}
 	}
-	if span != nil && region.End.IsUpperBoundFor(c.cmp, span.Start) {
-		// Found overlap.
-		return false, nil
+}
+
+// updateR2 updates r2, the region of data that ends before a key of interest.
+func (c *Checker) updateR2(r2 *base.UserKeyBounds, s *keyspan.Span) {
+	switch {
+	case s == nil:
+
+	case r2.Start == nil || c.cmp(s.End, r2.Start) < 0:
+		// Region completely to the left of r2.
+		*r2 = base.UserKeyBoundsEndExclusive(slices.Clone(s.Start), slices.Clone(s.End))
+
+	case c.cmp(r2.End.Key, s.Start) < 0:
+		// Region completely to the right of r2, nothing to do.
+
+	default:
+		// Regions are overlapping or touching.
+		if c.cmp(s.Start, r2.Start) < 0 {
+			r2.Start = slices.Clone(s.Start)
+		}
+		if c.cmp(r2.End.Key, s.End) < 0 {
+			r2.End = base.UserKeyExclusive(slices.Clone(s.End))
+		}
 	}
-	return true, nil
 }

Diff for: internal/overlap/checker_test.go

@@ -16,14 +16,21 @@ import (
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/manifest"
+	"github.com/cockroachdb/pebble/internal/overlap/overlapcache"
 	"github.com/stretchr/testify/require"
 )
 
 func TestChecker(t *testing.T) {
 	tables := newTestTables()
 	byName := make(map[string]*manifest.FileMetadata)
+	clearCaches := func() {
+		for _, t := range tables.tables {
+			t.meta.OverlapCache = overlapcache.C{}
+		}
+	}
 
 	datadriven.RunTest(t, "testdata/checker", func(t *testing.T, d *datadriven.TestData) string {
+		clearCaches()
 		switch d.Cmd {
 		case "define":
 			tt := testTable{
@@ -99,10 +106,15 @@ func TestChecker(t *testing.T) {
 			tables.tables[tt.meta] = tt
 
 		case "overlap":
+			var withCache bool
 			var metas []*manifest.FileMetadata
 			lines := strings.Split(d.Input, "\n")
 			for _, arg := range d.CmdArgs {
 				name := arg.String()
+				if name == "with-cache" {
+					withCache = true
+					continue
+				}
 				m := byName[name]
 				if m == nil {
 					d.Fatalf(t, "unknown table %q", name)
@@ -113,6 +125,9 @@ func TestChecker(t *testing.T) {
 			c := MakeChecker(bytes.Compare, tables)
 			var buf strings.Builder
 			for _, l := range lines {
+				if !withCache {
+					clearCaches()
+				}
 				bounds := base.ParseUserKeyBounds(l)
 				withLevel, err := c.LevelOverlap(context.Background(), bounds, levelMeta.Slice())
 				require.NoError(t, err)