spanlatch: add cluster setting for slow latch request threshold

This adds a new cluster setting
kv.concurrency.slow_latch_request_duration which configures the
threshold for logging slow latch acquisitions. Previously, logging would
occur after 15 seconds, but now the default is reduced to 5s.

Informs #154271
Release note: None
Epic: None
Part of: CRDB-54843

Author: angeladietz

pkg/kv/kvserver/spanlatch/BUILD.bazel

@@ -14,7 +14,6 @@ go_library(
     importpath = "github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch",
     visibility = ["//visibility:public"],
     deps = [
-        "//pkg/base",
         "//pkg/kv/kvpb",
         "//pkg/kv/kvserver/concurrency/poison",
         "//pkg/kv/kvserver/spanset",

pkg/kv/kvserver/spanlatch/manager.go

@@ -11,7 +11,6 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/cockroachdb/cockroach/pkg/base"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvpb"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/poison"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset"
@@ -585,7 +584,9 @@ func (m *Manager) waitForSignal(
 	}
 	log.Eventf(ctx, "waiting to acquire %s latch %s, held by %s latch %s", waitType, wait, heldType, held)
 	poisonCh := held.g.poison.signalChan()
-	t.Reset(base.SlowRequestThreshold)
+
+	slowThreshold := m.slowLatchRequestThreshold()
+	t.Reset(slowThreshold)
 	for {
 		select {
 		case <-held.g.done.signalChan():
@@ -610,7 +611,7 @@ func (m *Manager) waitForSignal(
 			t.Read = true
 
 			log.Warningf(ctx, "have been waiting %s to acquire %s latch %s, held by %s latch %s",
-				base.SlowRequestThreshold, waitType, wait, heldType, held)
+			slowThreshold, waitType, wait, heldType, held)
 			if m.slowReqs != nil {
 				m.slowReqs.Inc(1)
 				defer m.slowReqs.Dec(1) //nolint:deferloop
@@ -711,6 +712,14 @@ func (m *Manager) longLatchHoldThreshold() time.Duration {
 	return LongLatchHoldThreshold.Get(&m.settings.SV)
 }
 
+// slowLatchRequestThreshold returns the threshold for logging slow latch requests.
+func (m *Manager) slowLatchRequestThreshold() time.Duration {
+	if m.settings == nil {
+		return math.MaxInt64 // disable
+	}
+	return SlowLatchRequestThreshold.Get(&m.settings.SV)
+}
+
 // Metrics holds information about the state of a Manager.
 type Metrics struct {
 	ReadCount  int64

pkg/kv/kvserver/spanlatch/settings.go

@@ -23,3 +23,14 @@ var LongLatchHoldThreshold = settings.RegisterDurationSetting(
 	3*time.Second,
 	settings.NonNegativeDuration,
 )
+
+// SlowLatchRequestThreshold controls when we will log slow latch acquisition
+// attempts. When a latch acquisition has been waiting for this duration, a
+// warning is logged and the slow request metric is incremented.
+var SlowLatchRequestThreshold = settings.RegisterDurationSettingWithExplicitUnit(
+	settings.SystemOnly,
+	"kv.concurrency.slow_latch_request_duration",
+	"the threshold for logging slow latch acquisition attempts",
+	5*time.Second,
+	settings.DurationWithMinimum(10*time.Millisecond),
+)