Add range response KV length as a metric

This adds a metric to allow us to alert on large range responses described in etcd-io#14809. Signed-off-by: Thomas Jungblut <[email protected]>
tjungblu · Nov 7, 2023 · 1dbb35e · 1dbb35e
1 parent 02794b0
commit 1dbb35e
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 0 deletions.
diff --git a/server/etcdserver/metrics.go b/server/etcdserver/metrics.go
@@ -118,6 +118,12 @@ var (
 		Name:      "lease_expired_total",
 		Help:      "The total number of expired leases.",
 	})
+	rangeResponseKvCount = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "etcd",
+		Subsystem: "server",
+		Name:      "range_response_kv_count",
+		Help:      "The number of KVs returned by range calls.",
+	}, []string{"range_begin"})
 
 	currentVersion = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: "etcd",
@@ -168,6 +174,7 @@ func init() {
 	prometheus.MustRegister(slowReadIndex)
 	prometheus.MustRegister(readIndexFailed)
 	prometheus.MustRegister(leaseExpired)
+	prometheus.MustRegister(rangeResponseKvCount)
 	prometheus.MustRegister(currentVersion)
 	prometheus.MustRegister(currentGoVersion)
 	prometheus.MustRegister(serverID)

diff --git a/server/etcdserver/v3_server.go b/server/etcdserver/v3_server.go
@@ -136,6 +136,9 @@ func (s *EtcdServer) Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeRe
 		err = serr
 		return nil, err
 	}
+
+	rangeResponseKvCount.WithLabelValues(string(r.Key)).Observe(float64(len(resp.Kvs)))
+
 	return resp, err
 }