diff --git a/cluster-autoscaler/core/scaledown/nodeevaltracker/max_node_skip_eval_time.go b/cluster-autoscaler/core/scaledown/nodeevaltracker/max_node_skip_eval_time.go index c38c052f7acc..6ec9003e810a 100644 --- a/cluster-autoscaler/core/scaledown/nodeevaltracker/max_node_skip_eval_time.go +++ b/cluster-autoscaler/core/scaledown/nodeevaltracker/max_node_skip_eval_time.go @@ -22,17 +22,23 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/metrics" ) +type metricObserver interface { + ObserveMaxNodeSkipEvalDurationSeconds(duration time.Duration) +} + // MaxNodeSkipEvalTime is a time tracker for the biggest evaluation time of a node during ScaleDown type MaxNodeSkipEvalTime struct { // lastEvalTime is the time of previous currentlyUnneededNodeNames parsing lastEvalTime time.Time // nodeNamesWithTimeStamps is maps of nodeNames with their time of last successful evaluation nodeNamesWithTimeStamps map[string]time.Time + + metrics metricObserver } // NewMaxNodeSkipEvalTime returns LongestNodeScaleDownEvalTime with lastEvalTime set to currentTime func NewMaxNodeSkipEvalTime(currentTime time.Time) *MaxNodeSkipEvalTime { - return &MaxNodeSkipEvalTime{lastEvalTime: currentTime} + return &MaxNodeSkipEvalTime{lastEvalTime: currentTime, metrics: metrics.DefaultMetrics} } // Retrieves the time of the last evaluation of a node. @@ -65,6 +71,6 @@ func (l *MaxNodeSkipEvalTime) Update(nodeNames []string, currentTime time.Time) l.lastEvalTime = currentTime minimumTime := l.getMin() longestDuration := currentTime.Sub(minimumTime) - metrics.ObserveMaxNodeSkipEvalDurationSeconds(longestDuration) + l.metrics.ObserveMaxNodeSkipEvalDurationSeconds(longestDuration) return longestDuration } diff --git a/cluster-autoscaler/core/scaledown/nodeevaltracker/max_node_skip_eval_time_test.go b/cluster-autoscaler/core/scaledown/nodeevaltracker/max_node_skip_eval_time_test.go index 02ea32d96aad..82d7b5fbee86 100644 --- a/cluster-autoscaler/core/scaledown/nodeevaltracker/max_node_skip_eval_time_test.go +++ b/cluster-autoscaler/core/scaledown/nodeevaltracker/max_node_skip_eval_time_test.go @@ -21,8 +21,21 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" ) +type mockMetrics struct { + mock.Mock +} + +func (m *mockMetrics) ObserveMaxNodeSkipEvalDurationSeconds(duration time.Duration) { + m.Called(duration) +} + +func newMaxNodeSkipEvalTime(currentTime time.Time, metrics metricObserver) *MaxNodeSkipEvalTime { + return &MaxNodeSkipEvalTime{lastEvalTime: currentTime, metrics: metrics} +} + func TestMaxNodeSkipEvalTime(t *testing.T) { type testCase struct { name string @@ -66,11 +79,15 @@ func TestMaxNodeSkipEvalTime(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() timestamp := start - maxNodeSkipEvalTime := NewMaxNodeSkipEvalTime(start) + mockMetrics := &mockMetrics{} + maxNodeSkipEvalTime := newMaxNodeSkipEvalTime(start, mockMetrics) + mockMetrics.On("ObserveMaxNodeSkipEvalDurationSeconds", mock.Anything).Return() + for i := 0; i < len(tc.unprocessedNodes); i++ { timestamp = timestamp.Add(1 * time.Second) assert.Equal(t, time.Duration(tc.wantMaxSkipEvalTimeSeconds[i])*time.Second, maxNodeSkipEvalTime.Update(tc.unprocessedNodes[i], timestamp)) assert.Equal(t, len(tc.unprocessedNodes[i]), len(maxNodeSkipEvalTime.nodeNamesWithTimeStamps)) + mockMetrics.AssertCalled(t, "ObserveMaxNodeSkipEvalDurationSeconds", time.Duration(tc.wantMaxSkipEvalTimeSeconds[i])*time.Second) } }) } diff --git a/cluster-autoscaler/metrics/metrics.go b/cluster-autoscaler/metrics/metrics.go index 27a949e0c84b..0513cfef28a6 100644 --- a/cluster-autoscaler/metrics/metrics.go +++ b/cluster-autoscaler/metrics/metrics.go @@ -17,766 +17,243 @@ limitations under the License. package metrics import ( - "fmt" - "strconv" "time" "k8s.io/autoscaler/cluster-autoscaler/simulator" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" - "k8s.io/autoscaler/cluster-autoscaler/utils/gpu" _ "k8s.io/component-base/metrics/prometheus/restclient" // for client-go metrics registration - - k8smetrics "k8s.io/component-base/metrics" - "k8s.io/component-base/metrics/legacyregistry" - klog "k8s.io/klog/v2" -) - -// NodeScaleDownReason describes reason for removing node -type NodeScaleDownReason string - -// FailedScaleUpReason describes reason of failed scale-up -type FailedScaleUpReason string - -// FunctionLabel is a name of Cluster Autoscaler operation for which -// we measure duration -type FunctionLabel string - -// NodeGroupType describes node group relation to CA -type NodeGroupType string - -// PodEvictionResult describes result of the pod eviction attempt -type PodEvictionResult string - -const ( - caNamespace = "cluster_autoscaler" - readyLabel = "ready" - unreadyLabel = "unready" - startingLabel = "notStarted" - unregisteredLabel = "unregistered" - longUnregisteredLabel = "longUnregistered" - - // Underutilized node was removed because of low utilization - Underutilized NodeScaleDownReason = "underutilized" - // Empty node was removed - Empty NodeScaleDownReason = "empty" - // Unready node was removed - Unready NodeScaleDownReason = "unready" - - // CloudProviderError caused scale-up to fail - CloudProviderError FailedScaleUpReason = "cloudProviderError" - // APIError caused scale-up to fail - APIError FailedScaleUpReason = "apiCallError" - // Timeout was encountered when trying to scale-up - Timeout FailedScaleUpReason = "timeout" - - // DirectionScaleDown is the direction of skipped scaling event when scaling in (shrinking) - DirectionScaleDown string = "down" - // DirectionScaleUp is the direction of skipped scaling event when scaling out (growing) - DirectionScaleUp string = "up" - - // CpuResourceLimit minimum or maximum reached, check the direction label to determine min or max - CpuResourceLimit string = "CpuResourceLimit" - // MemoryResourceLimit minimum or maximum reached, check the direction label to determine min or max - MemoryResourceLimit string = "MemoryResourceLimit" - - // autoscaledGroup is managed by CA - autoscaledGroup NodeGroupType = "autoscaled" - // autoprovisionedGroup have been created by CA (Node Autoprovisioning), - // is currently autoscaled and can be removed by CA if it's no longer needed - autoprovisionedGroup NodeGroupType = "autoprovisioned" - - // LogLongDurationThreshold defines the duration after which long function - // duration will be logged (in addition to being counted in metric). - // This is meant to help find unexpectedly long function execution times for - // debugging purposes. - LogLongDurationThreshold = 5 * time.Second - // PodEvictionSucceed means creation of the pod eviction object succeed - PodEvictionSucceed PodEvictionResult = "succeeded" - // PodEvictionFailed means creation of the pod eviction object failed - PodEvictionFailed PodEvictionResult = "failed" -) - -// Names of Cluster Autoscaler operations -const ( - ScaleDown FunctionLabel = "scaleDown" - ScaleDownNodeDeletion FunctionLabel = "scaleDown:nodeDeletion" - ScaleDownSoftTaintUnneeded FunctionLabel = "scaleDown:softTaintUnneeded" - ScaleUp FunctionLabel = "scaleUp" - BuildPodEquivalenceGroups FunctionLabel = "scaleUp:buildPodEquivalenceGroups" - Estimate FunctionLabel = "scaleUp:estimate" - FindUnneeded FunctionLabel = "findUnneeded" - UpdateState FunctionLabel = "updateClusterState" - FilterOutSchedulable FunctionLabel = "filterOutSchedulable" - CloudProviderRefresh FunctionLabel = "cloudProviderRefresh" - Main FunctionLabel = "main" - Poll FunctionLabel = "poll" - Reconfigure FunctionLabel = "reconfigure" - Autoscaling FunctionLabel = "autoscaling" - LoopWait FunctionLabel = "loopWait" - BulkListAllGceInstances FunctionLabel = "bulkListInstances:listAllInstances" - BulkListMigInstances FunctionLabel = "bulkListInstances:listMigInstances" -) - -var ( - /**** Metrics related to cluster state ****/ - clusterSafeToAutoscale = k8smetrics.NewGauge( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "cluster_safe_to_autoscale", - Help: "Whether or not cluster is healthy enough for autoscaling. 1 if it is, 0 otherwise.", - }, - ) - - nodesCount = k8smetrics.NewGaugeVec( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "nodes_count", - Help: "Number of nodes in cluster.", - }, []string{"state"}, - ) - - nodeGroupsCount = k8smetrics.NewGaugeVec( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "node_groups_count", - Help: "Number of node groups managed by CA.", - }, []string{"node_group_type"}, - ) - - // Unschedulable pod count can be from scheduler-marked-unschedulable pods or not-yet-processed pods (unknown) - unschedulablePodsCount = k8smetrics.NewGaugeVec( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "unschedulable_pods_count", - Help: "Number of unschedulable pods in the cluster.", - }, []string{"type"}, - ) - - maxNodesCount = k8smetrics.NewGauge( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "max_nodes_count", - Help: "Maximum number of nodes in all node groups", - }, - ) - - cpuCurrentCores = k8smetrics.NewGauge( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "cluster_cpu_current_cores", - Help: "Current number of cores in the cluster, minus deleting nodes.", - }, - ) - - cpuLimitsCores = k8smetrics.NewGaugeVec( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "cpu_limits_cores", - Help: "Minimum and maximum number of cores in the cluster.", - }, []string{"direction"}, - ) - - memoryCurrentBytes = k8smetrics.NewGauge( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "cluster_memory_current_bytes", - Help: "Current number of bytes of memory in the cluster, minus deleting nodes.", - }, - ) - - memoryLimitsBytes = k8smetrics.NewGaugeVec( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "memory_limits_bytes", - Help: "Minimum and maximum number of bytes of memory in cluster.", - }, []string{"direction"}, - ) - - nodesGroupMinNodes = k8smetrics.NewGaugeVec( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "node_group_min_count", - Help: "Minimum number of nodes in the node group", - }, []string{"node_group"}, - ) - - nodesGroupMaxNodes = k8smetrics.NewGaugeVec( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "node_group_max_count", - Help: "Maximum number of nodes in the node group", - }, []string{"node_group"}, - ) - - nodesGroupTargetSize = k8smetrics.NewGaugeVec( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "node_group_target_count", - Help: "Target number of nodes in the node group by CA.", - }, []string{"node_group"}, - ) - - nodesGroupHealthiness = k8smetrics.NewGaugeVec( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "node_group_healthiness", - Help: "Whether or not node group is healthy enough for autoscaling. 1 if it is, 0 otherwise.", - }, []string{"node_group"}, - ) - - nodeGroupBackOffStatus = k8smetrics.NewGaugeVec( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "node_group_backoff_status", - Help: "Whether or not node group is backoff for not autoscaling. 1 if it is, 0 otherwise.", - }, []string{"node_group", "reason"}, - ) - - /**** Metrics related to autoscaler execution ****/ - lastActivity = k8smetrics.NewGaugeVec( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "last_activity", - Help: "Last time certain part of CA logic executed.", - }, []string{"activity"}, - ) - - functionDuration = k8smetrics.NewHistogramVec( - &k8smetrics.HistogramOpts{ - Namespace: caNamespace, - Name: "function_duration_seconds", - Help: "Time taken by various parts of CA main loop.", - Buckets: k8smetrics.ExponentialBuckets(0.01, 1.5, 30), // 0.01, 0.015, 0.0225, ..., 852.2269299239293, 1278.3403948858938 - }, []string{"function"}, - ) - - functionDurationSummary = k8smetrics.NewSummaryVec( - &k8smetrics.SummaryOpts{ - Namespace: caNamespace, - Name: "function_duration_quantile_seconds", - Help: "Quantiles of time taken by various parts of CA main loop.", - MaxAge: time.Hour, - }, []string{"function"}, - ) - - pendingNodeDeletions = k8smetrics.NewGauge( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "pending_node_deletions", - Help: "Number of nodes that haven't been removed or aborted after finished scale-down phase.", - }, - ) - - /**** Metrics related to autoscaler operations ****/ - errorsCount = k8smetrics.NewCounterVec( - &k8smetrics.CounterOpts{ - Namespace: caNamespace, - Name: "errors_total", - Help: "The number of CA loops failed due to an error.", - }, []string{"type"}, - ) - - scaleUpCount = k8smetrics.NewCounter( - &k8smetrics.CounterOpts{ - Namespace: caNamespace, - Name: "scaled_up_nodes_total", - Help: "Number of nodes added by CA.", - }, - ) - - gpuScaleUpCount = k8smetrics.NewCounterVec( - &k8smetrics.CounterOpts{ - Namespace: caNamespace, - Name: "scaled_up_gpu_nodes_total", - Help: "Number of GPU nodes added by CA, by GPU name.", - }, []string{"gpu_resource_name", "gpu_name"}, - ) - - failedScaleUpCount = k8smetrics.NewCounterVec( - &k8smetrics.CounterOpts{ - Namespace: caNamespace, - Name: "failed_scale_ups_total", - Help: "Number of times scale-up operation has failed.", - }, []string{"reason"}, - ) - - failedGPUScaleUpCount = k8smetrics.NewCounterVec( - &k8smetrics.CounterOpts{ - Namespace: caNamespace, - Name: "failed_gpu_scale_ups_total", - Help: "Number of times scale-up operation has failed.", - }, []string{"reason", "gpu_resource_name", "gpu_name"}, - ) - - scaleDownCount = k8smetrics.NewCounterVec( - &k8smetrics.CounterOpts{ - Namespace: caNamespace, - Name: "scaled_down_nodes_total", - Help: "Number of nodes removed by CA.", - }, []string{"reason"}, - ) - - gpuScaleDownCount = k8smetrics.NewCounterVec( - &k8smetrics.CounterOpts{ - Namespace: caNamespace, - Name: "scaled_down_gpu_nodes_total", - Help: "Number of GPU nodes removed by CA, by reason and GPU name.", - }, []string{"reason", "gpu_resource_name", "gpu_name"}, - ) - - evictionsCount = k8smetrics.NewCounterVec( - &k8smetrics.CounterOpts{ - Namespace: caNamespace, - Name: "evicted_pods_total", - Help: "Number of pods evicted by CA", - }, []string{"eviction_result"}, - ) - - unneededNodesCount = k8smetrics.NewGauge( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "unneeded_nodes_count", - Help: "Number of nodes currently considered unneeded by CA.", - }, - ) - - unremovableNodesCount = k8smetrics.NewGaugeVec( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "unremovable_nodes_count", - Help: "Number of nodes currently considered unremovable by CA.", - }, - []string{"reason"}, - ) - - scaleDownInCooldown = k8smetrics.NewGauge( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "scale_down_in_cooldown", - Help: "Whether or not the scale down is in cooldown. 1 if its, 0 otherwise.", - }, - ) - - oldUnregisteredNodesRemovedCount = k8smetrics.NewCounter( - &k8smetrics.CounterOpts{ - Namespace: caNamespace, - Name: "old_unregistered_nodes_removed_count", - Help: "Number of unregistered nodes removed by CA.", - }, - ) - - overflowingControllersCount = k8smetrics.NewGauge( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "overflowing_controllers_count", - Help: "Number of controllers that own a large set of heterogenous pods, preventing CA from treating these pods as equivalent.", - }, - ) - - skippedScaleEventsCount = k8smetrics.NewCounterVec( - &k8smetrics.CounterOpts{ - Namespace: caNamespace, - Name: "skipped_scale_events_count", - Help: "Count of scaling events that the CA has chosen to skip.", - }, - []string{"direction", "reason"}, - ) - - nodeGroupCreationCount = k8smetrics.NewCounterVec( - &k8smetrics.CounterOpts{ - Namespace: caNamespace, - Name: "created_node_groups_total", - Help: "Number of node groups created by Node Autoprovisioning.", - }, - []string{"group_type"}, - ) - - nodeGroupDeletionCount = k8smetrics.NewCounterVec( - &k8smetrics.CounterOpts{ - Namespace: caNamespace, - Name: "deleted_node_groups_total", - Help: "Number of node groups deleted by Node Autoprovisioning.", - }, - []string{"group_type"}, - ) - - nodeTaintsCount = k8smetrics.NewGaugeVec( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "node_taints_count", - Help: "Number of taints per type used in the cluster.", - }, - []string{"type"}, - ) - - inconsistentInstancesMigsCount = k8smetrics.NewGauge( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "inconsistent_instances_migs_count", - Help: "Number of migs where instance count according to InstanceGroupManagers.List() differs from the results of Instances.List(). This can happen when some instances are abandoned or a user edits instance 'created-by' metadata.", - }, - ) - - binpackingHeterogeneity = k8smetrics.NewHistogramVec( - &k8smetrics.HistogramOpts{ - Namespace: caNamespace, - Name: "binpacking_heterogeneity", - Help: "Number of groups of equivalent pods being processed as a part of the same binpacking simulation.", - Buckets: k8smetrics.ExponentialBuckets(1, 2, 6), // 1, 2, 4, ..., 32 - }, []string{"instance_type", "cpu_count", "namespace_count"}, - ) - - maxNodeSkipEvalDurationSeconds = k8smetrics.NewGauge( - &k8smetrics.GaugeOpts{ - Namespace: caNamespace, - Name: "max_node_skip_eval_duration_seconds", - Help: "Maximum evaluation time of a node being skipped during ScaleDown.", - }, - ) - - scaleDownNodeRemovalLatency = k8smetrics.NewHistogramVec( - &k8smetrics.HistogramOpts{ - Namespace: caNamespace, - Name: "node_removal_latency_seconds", - Help: "Latency from when an unneeded node is eligible for scale down until it is removed (deleted=true) or it became needed again (deleted=false).", - Buckets: k8smetrics.ExponentialBuckets(1, 1.5, 19), // ~1s → ~24min - }, []string{"deleted"}, - ) ) -// RegisterAll registers all metrics. -func RegisterAll(emitPerNodeGroupMetrics bool) { - legacyregistry.MustRegister(clusterSafeToAutoscale) - legacyregistry.MustRegister(nodesCount) - legacyregistry.MustRegister(nodeGroupsCount) - legacyregistry.MustRegister(unschedulablePodsCount) - legacyregistry.MustRegister(maxNodesCount) - legacyregistry.MustRegister(cpuCurrentCores) - legacyregistry.MustRegister(cpuLimitsCores) - legacyregistry.MustRegister(memoryCurrentBytes) - legacyregistry.MustRegister(memoryLimitsBytes) - legacyregistry.MustRegister(lastActivity) - legacyregistry.MustRegister(functionDuration) - legacyregistry.MustRegister(functionDurationSummary) - legacyregistry.MustRegister(errorsCount) - legacyregistry.MustRegister(scaleUpCount) - legacyregistry.MustRegister(gpuScaleUpCount) - legacyregistry.MustRegister(failedScaleUpCount) - legacyregistry.MustRegister(failedGPUScaleUpCount) - legacyregistry.MustRegister(scaleDownCount) - legacyregistry.MustRegister(gpuScaleDownCount) - legacyregistry.MustRegister(evictionsCount) - legacyregistry.MustRegister(unneededNodesCount) - legacyregistry.MustRegister(unremovableNodesCount) - legacyregistry.MustRegister(scaleDownInCooldown) - legacyregistry.MustRegister(oldUnregisteredNodesRemovedCount) - legacyregistry.MustRegister(overflowingControllersCount) - legacyregistry.MustRegister(skippedScaleEventsCount) - legacyregistry.MustRegister(nodeGroupCreationCount) - legacyregistry.MustRegister(nodeGroupDeletionCount) - legacyregistry.MustRegister(pendingNodeDeletions) - legacyregistry.MustRegister(nodeTaintsCount) - legacyregistry.MustRegister(inconsistentInstancesMigsCount) - legacyregistry.MustRegister(binpackingHeterogeneity) - legacyregistry.MustRegister(maxNodeSkipEvalDurationSeconds) - legacyregistry.MustRegister(scaleDownNodeRemovalLatency) - - if emitPerNodeGroupMetrics { - legacyregistry.MustRegister(nodesGroupMinNodes) - legacyregistry.MustRegister(nodesGroupMaxNodes) - legacyregistry.MustRegister(nodesGroupTargetSize) - legacyregistry.MustRegister(nodesGroupHealthiness) - legacyregistry.MustRegister(nodeGroupBackOffStatus) - } -} +// DefaultMetrics is a default implementation using global legacyregistry +var DefaultMetrics = newCaMetricsImpl() // InitMetrics initializes all metrics func InitMetrics() { - for _, errorType := range []errors.AutoscalerErrorType{errors.CloudProviderError, errors.ApiCallError, errors.InternalError, errors.TransientError, errors.ConfigurationError, errors.NodeGroupDoesNotExistError, errors.UnexpectedScaleDownStateError} { - errorsCount.WithLabelValues(string(errorType)).Add(0) - } - - for _, reason := range []FailedScaleUpReason{CloudProviderError, APIError, Timeout} { - scaleDownCount.WithLabelValues(string(reason)).Add(0) - failedScaleUpCount.WithLabelValues(string(reason)).Add(0) - } - - for _, result := range []PodEvictionResult{PodEvictionSucceed, PodEvictionFailed} { - evictionsCount.WithLabelValues(string(result)).Add(0) - } - - skippedScaleEventsCount.WithLabelValues(DirectionScaleDown, CpuResourceLimit).Add(0) - skippedScaleEventsCount.WithLabelValues(DirectionScaleDown, MemoryResourceLimit).Add(0) - skippedScaleEventsCount.WithLabelValues(DirectionScaleUp, CpuResourceLimit).Add(0) - skippedScaleEventsCount.WithLabelValues(DirectionScaleUp, MemoryResourceLimit).Add(0) + DefaultMetrics.InitMetrics() +} +// RegisterAll registers all metrics +func RegisterAll(emitPerNodeGroupMetrics bool) { + DefaultMetrics.RegisterAll(emitPerNodeGroupMetrics) } // UpdateDurationFromStart records the duration of the step identified by the // label using start time func UpdateDurationFromStart(label FunctionLabel, start time.Time) { - duration := time.Now().Sub(start) - UpdateDuration(label, duration) + DefaultMetrics.UpdateDurationFromStart(label, start) } // UpdateDuration records the duration of the step identified by the label func UpdateDuration(label FunctionLabel, duration time.Duration) { - if duration > LogLongDurationThreshold { - klog.V(4).Infof("Function %s took %v to complete", label, duration) - } - functionDuration.WithLabelValues(string(label)).Observe(duration.Seconds()) - functionDurationSummary.WithLabelValues(string(label)).Observe(duration.Seconds()) + DefaultMetrics.UpdateDuration(label, duration) } // UpdateLastTime records the time the step identified by the label was started func UpdateLastTime(label FunctionLabel, now time.Time) { - lastActivity.WithLabelValues(string(label)).Set(float64(now.Unix())) + DefaultMetrics.UpdateLastTime(label, now) } // UpdateClusterSafeToAutoscale records if cluster is safe to autoscale func UpdateClusterSafeToAutoscale(safe bool) { - if safe { - clusterSafeToAutoscale.Set(1) - } else { - clusterSafeToAutoscale.Set(0) - } + DefaultMetrics.UpdateClusterSafeToAutoscale(safe) } // UpdateNodesCount records the number of nodes in cluster func UpdateNodesCount(ready, unready, starting, longUnregistered, unregistered int) { - nodesCount.WithLabelValues(readyLabel).Set(float64(ready)) - nodesCount.WithLabelValues(unreadyLabel).Set(float64(unready)) - nodesCount.WithLabelValues(startingLabel).Set(float64(starting)) - nodesCount.WithLabelValues(longUnregisteredLabel).Set(float64(longUnregistered)) - nodesCount.WithLabelValues(unregisteredLabel).Set(float64(unregistered)) + DefaultMetrics.UpdateNodesCount(ready, unready, starting, longUnregistered, unregistered) } // UpdateNodeGroupsCount records the number of node groups managed by CA func UpdateNodeGroupsCount(autoscaled, autoprovisioned int) { - nodeGroupsCount.WithLabelValues(string(autoscaledGroup)).Set(float64(autoscaled)) - nodeGroupsCount.WithLabelValues(string(autoprovisionedGroup)).Set(float64(autoprovisioned)) + DefaultMetrics.UpdateNodeGroupsCount(autoscaled, autoprovisioned) } // UpdateUnschedulablePodsCount records number of currently unschedulable pods func UpdateUnschedulablePodsCount(uschedulablePodsCount, schedulerUnprocessedCount int) { - UpdateUnschedulablePodsCountWithLabel(uschedulablePodsCount, "unschedulable") - UpdateUnschedulablePodsCountWithLabel(schedulerUnprocessedCount, "scheduler_unprocessed") + DefaultMetrics.UpdateUnschedulablePodsCount(uschedulablePodsCount, schedulerUnprocessedCount) } // UpdateUnschedulablePodsCountWithLabel records number of currently unschedulable pods wil label "type" value "label" func UpdateUnschedulablePodsCountWithLabel(uschedulablePodsCount int, label string) { - unschedulablePodsCount.WithLabelValues(label).Set(float64(uschedulablePodsCount)) + DefaultMetrics.UpdateUnschedulablePodsCountWithLabel(uschedulablePodsCount, label) } // UpdateMaxNodesCount records the current maximum number of nodes being set for all node groups func UpdateMaxNodesCount(nodesCount int) { - maxNodesCount.Set(float64(nodesCount)) + DefaultMetrics.UpdateMaxNodesCount(nodesCount) } // UpdateClusterCPUCurrentCores records the number of cores in the cluster, minus deleting nodes func UpdateClusterCPUCurrentCores(coresCount int64) { - cpuCurrentCores.Set(float64(coresCount)) + DefaultMetrics.UpdateClusterCPUCurrentCores(coresCount) } // UpdateCPULimitsCores records the minimum and maximum number of cores in the cluster func UpdateCPULimitsCores(minCoresCount int64, maxCoresCount int64) { - cpuLimitsCores.WithLabelValues("minimum").Set(float64(minCoresCount)) - cpuLimitsCores.WithLabelValues("maximum").Set(float64(maxCoresCount)) + DefaultMetrics.UpdateCPULimitsCores(minCoresCount, maxCoresCount) } // UpdateClusterMemoryCurrentBytes records the number of bytes of memory in the cluster, minus deleting nodes func UpdateClusterMemoryCurrentBytes(memoryCount int64) { - memoryCurrentBytes.Set(float64(memoryCount)) + DefaultMetrics.UpdateClusterMemoryCurrentBytes(memoryCount) } // UpdateMemoryLimitsBytes records the minimum and maximum bytes of memory in the cluster func UpdateMemoryLimitsBytes(minMemoryCount int64, maxMemoryCount int64) { - memoryLimitsBytes.WithLabelValues("minimum").Set(float64(minMemoryCount)) - memoryLimitsBytes.WithLabelValues("maximum").Set(float64(maxMemoryCount)) + DefaultMetrics.UpdateMemoryLimitsBytes(minMemoryCount, maxMemoryCount) } // UpdateNodeGroupMin records the node group minimum allowed number of nodes func UpdateNodeGroupMin(nodeGroup string, minNodes int) { - nodesGroupMinNodes.WithLabelValues(nodeGroup).Set(float64(minNodes)) + DefaultMetrics.UpdateNodeGroupMin(nodeGroup, minNodes) } // UpdateNodeGroupMax records the node group maximum allowed number of nodes func UpdateNodeGroupMax(nodeGroup string, maxNodes int) { - nodesGroupMaxNodes.WithLabelValues(nodeGroup).Set(float64(maxNodes)) + DefaultMetrics.UpdateNodeGroupMax(nodeGroup, maxNodes) } // UpdateNodeGroupTargetSize records the node group target size func UpdateNodeGroupTargetSize(targetSizes map[string]int) { - for nodeGroup, targetSize := range targetSizes { - nodesGroupTargetSize.WithLabelValues(nodeGroup).Set(float64(targetSize)) - } + DefaultMetrics.UpdateNodeGroupTargetSize(targetSizes) } // UpdateNodeGroupHealthStatus records if node group is healthy to autoscaling func UpdateNodeGroupHealthStatus(nodeGroup string, healthy bool) { - if healthy { - nodesGroupHealthiness.WithLabelValues(nodeGroup).Set(1) - } else { - nodesGroupHealthiness.WithLabelValues(nodeGroup).Set(0) - } + DefaultMetrics.UpdateNodeGroupHealthStatus(nodeGroup, healthy) } // UpdateNodeGroupBackOffStatus records if node group is backoff for not autoscaling func UpdateNodeGroupBackOffStatus(nodeGroup string, backoffReasonStatus map[string]bool) { - if len(backoffReasonStatus) == 0 { - nodeGroupBackOffStatus.WithLabelValues(nodeGroup, "").Set(0) - } else { - for reason, backoff := range backoffReasonStatus { - if backoff { - nodeGroupBackOffStatus.WithLabelValues(nodeGroup, reason).Set(1) - } else { - nodeGroupBackOffStatus.WithLabelValues(nodeGroup, reason).Set(0) - } - } - } + DefaultMetrics.UpdateNodeGroupBackOffStatus(nodeGroup, backoffReasonStatus) } // RegisterError records any errors preventing Cluster Autoscaler from working. // No more than one error should be recorded per loop. func RegisterError(err errors.AutoscalerError) { - errorsCount.WithLabelValues(string(err.Type())).Add(1.0) + DefaultMetrics.RegisterError(err) } // RegisterScaleUp records number of nodes added by scale up func RegisterScaleUp(nodesCount int, gpuResourceName, gpuType string) { - scaleUpCount.Add(float64(nodesCount)) - if gpuType != gpu.MetricsNoGPU { - gpuScaleUpCount.WithLabelValues(gpuResourceName, gpuType).Add(float64(nodesCount)) - } + DefaultMetrics.RegisterScaleUp(nodesCount, gpuResourceName, gpuType) } // RegisterFailedScaleUp records a failed scale-up operation func RegisterFailedScaleUp(reason FailedScaleUpReason, gpuResourceName, gpuType string) { - failedScaleUpCount.WithLabelValues(string(reason)).Inc() - if gpuType != gpu.MetricsNoGPU { - failedGPUScaleUpCount.WithLabelValues(string(reason), gpuResourceName, gpuType).Inc() - } + DefaultMetrics.RegisterFailedScaleUp(reason, gpuResourceName, gpuType) } // RegisterScaleDown records number of nodes removed by scale down func RegisterScaleDown(nodesCount int, gpuResourceName, gpuType string, reason NodeScaleDownReason) { - scaleDownCount.WithLabelValues(string(reason)).Add(float64(nodesCount)) - if gpuType != gpu.MetricsNoGPU { - gpuScaleDownCount.WithLabelValues(string(reason), gpuResourceName, gpuType).Add(float64(nodesCount)) - } + DefaultMetrics.RegisterScaleDown(nodesCount, gpuResourceName, gpuType, reason) } // RegisterEvictions records number of evicted pods succeed or failed func RegisterEvictions(podsCount int, result PodEvictionResult) { - evictionsCount.WithLabelValues(string(result)).Add(float64(podsCount)) + DefaultMetrics.RegisterEvictions(podsCount, result) } // UpdateUnneededNodesCount records number of currently unneeded nodes func UpdateUnneededNodesCount(nodesCount int) { - unneededNodesCount.Set(float64(nodesCount)) + DefaultMetrics.UpdateUnneededNodesCount(nodesCount) } // UpdateUnremovableNodesCount records number of currently unremovable nodes func UpdateUnremovableNodesCount(unremovableReasonCounts map[simulator.UnremovableReason]int) { - for reason, count := range unremovableReasonCounts { - unremovableNodesCount.WithLabelValues(fmt.Sprintf("%v", reason)).Set(float64(count)) - } + DefaultMetrics.UpdateUnremovableNodesCount(unremovableReasonCounts) } // RegisterNodeGroupCreation registers node group creation func RegisterNodeGroupCreation() { - RegisterNodeGroupCreationWithLabelValues("") + DefaultMetrics.RegisterNodeGroupCreation() } // RegisterNodeGroupCreationWithLabelValues registers node group creation with the provided labels func RegisterNodeGroupCreationWithLabelValues(groupType string) { - nodeGroupCreationCount.WithLabelValues(groupType).Add(1.0) + DefaultMetrics.RegisterNodeGroupCreationWithLabelValues(groupType) } // RegisterNodeGroupDeletion registers node group deletion func RegisterNodeGroupDeletion() { - RegisterNodeGroupDeletionWithLabelValues("") + DefaultMetrics.RegisterNodeGroupDeletion() } // RegisterNodeGroupDeletionWithLabelValues registers node group deletion with the provided labels func RegisterNodeGroupDeletionWithLabelValues(groupType string) { - nodeGroupDeletionCount.WithLabelValues(groupType).Add(1.0) + DefaultMetrics.RegisterNodeGroupDeletionWithLabelValues(groupType) } // UpdateScaleDownInCooldown registers if the cluster autoscaler // scaledown is in cooldown func UpdateScaleDownInCooldown(inCooldown bool) { - if inCooldown { - scaleDownInCooldown.Set(1.0) - } else { - scaleDownInCooldown.Set(0.0) - } + DefaultMetrics.UpdateScaleDownInCooldown(inCooldown) } // RegisterOldUnregisteredNodesRemoved records number of old unregistered // nodes that have been removed by the cluster autoscaler func RegisterOldUnregisteredNodesRemoved(nodesCount int) { - oldUnregisteredNodesRemovedCount.Add(float64(nodesCount)) + DefaultMetrics.RegisterOldUnregisteredNodesRemoved(nodesCount) } // UpdateOverflowingControllers sets the number of controllers that could not // have their pods cached. func UpdateOverflowingControllers(count int) { - overflowingControllersCount.Set(float64(count)) + DefaultMetrics.UpdateOverflowingControllers(count) } // RegisterSkippedScaleDownCPU increases the count of skipped scale outs because of CPU resource limits func RegisterSkippedScaleDownCPU() { - skippedScaleEventsCount.WithLabelValues(DirectionScaleDown, CpuResourceLimit).Add(1.0) + DefaultMetrics.RegisterSkippedScaleDownCPU() } // RegisterSkippedScaleDownMemory increases the count of skipped scale outs because of Memory resource limits func RegisterSkippedScaleDownMemory() { - skippedScaleEventsCount.WithLabelValues(DirectionScaleDown, MemoryResourceLimit).Add(1.0) + DefaultMetrics.RegisterSkippedScaleDownMemory() } // RegisterSkippedScaleUpCPU increases the count of skipped scale outs because of CPU resource limits func RegisterSkippedScaleUpCPU() { - skippedScaleEventsCount.WithLabelValues(DirectionScaleUp, CpuResourceLimit).Add(1.0) + DefaultMetrics.RegisterSkippedScaleUpCPU() } // RegisterSkippedScaleUpMemory increases the count of skipped scale outs because of Memory resource limits func RegisterSkippedScaleUpMemory() { - skippedScaleEventsCount.WithLabelValues(DirectionScaleUp, MemoryResourceLimit).Add(1.0) + DefaultMetrics.RegisterSkippedScaleUpMemory() } // ObservePendingNodeDeletions records the current value of nodes_pending_deletion metric func ObservePendingNodeDeletions(value int) { - pendingNodeDeletions.Set(float64(value)) + DefaultMetrics.ObservePendingNodeDeletions(value) } // ObserveNodeTaintsCount records the node taints count of given type. func ObserveNodeTaintsCount(taintType string, count float64) { - nodeTaintsCount.WithLabelValues(taintType).Set(count) + DefaultMetrics.ObserveNodeTaintsCount(taintType, count) } // UpdateInconsistentInstancesMigsCount records the observed number of migs where instance count // according to InstanceGroupManagers.List() differs from the results of Instances.List(). // This can happen when some instances are abandoned or a user edits instance 'created-by' metadata. func UpdateInconsistentInstancesMigsCount(migCount int) { - inconsistentInstancesMigsCount.Set(float64(migCount)) + DefaultMetrics.UpdateInconsistentInstancesMigsCount(migCount) } // ObserveBinpackingHeterogeneity records the number of pod equivalence groups // considered in a single binpacking estimation. func ObserveBinpackingHeterogeneity(instanceType, cpuCount, namespaceCount string, pegCount int) { - binpackingHeterogeneity.WithLabelValues(instanceType, cpuCount, namespaceCount).Observe(float64(pegCount)) + DefaultMetrics.ObserveBinpackingHeterogeneity(instanceType, cpuCount, namespaceCount, pegCount) } // UpdateScaleDownNodeRemovalLatency records the time after which node was deleted/needed // again after being marked unneded func UpdateScaleDownNodeRemovalLatency(deleted bool, duration time.Duration) { - scaleDownNodeRemovalLatency.WithLabelValues(strconv.FormatBool(deleted)).Observe(duration.Seconds()) + DefaultMetrics.UpdateScaleDownNodeRemovalLatency(deleted, duration) } // ObserveMaxNodeSkipEvalDurationSeconds records the longest time during which node was skipped during ScaleDown. // If a node is skipped multiple times consecutively, we store only the earliest timestamp. func ObserveMaxNodeSkipEvalDurationSeconds(duration time.Duration) { - maxNodeSkipEvalDurationSeconds.Set(duration.Seconds()) + DefaultMetrics.ObserveMaxNodeSkipEvalDurationSeconds(duration) } diff --git a/cluster-autoscaler/metrics/metrics_impl.go b/cluster-autoscaler/metrics/metrics_impl.go new file mode 100644 index 000000000000..4aa7d6644cd7 --- /dev/null +++ b/cluster-autoscaler/metrics/metrics_impl.go @@ -0,0 +1,847 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "fmt" + "strconv" + "time" + + "k8s.io/autoscaler/cluster-autoscaler/simulator" + + "k8s.io/autoscaler/cluster-autoscaler/utils/errors" + "k8s.io/autoscaler/cluster-autoscaler/utils/gpu" + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" + _ "k8s.io/component-base/metrics/prometheus/restclient" // for client-go metrics registration + + k8smetrics "k8s.io/component-base/metrics" + klog "k8s.io/klog/v2" +) + +// NodeScaleDownReason describes reason for removing node +type NodeScaleDownReason string + +// FailedScaleUpReason describes reason of failed scale-up +type FailedScaleUpReason string + +// FunctionLabel is a name of Cluster Autoscaler operation for which +// we measure duration +type FunctionLabel string + +// NodeGroupType describes node group relation to CA +type NodeGroupType string + +// PodEvictionResult describes result of the pod eviction attempt +type PodEvictionResult string + +const ( + caNamespace = "cluster_autoscaler" + readyLabel = "ready" + unreadyLabel = "unready" + startingLabel = "notStarted" + unregisteredLabel = "unregistered" + longUnregisteredLabel = "longUnregistered" + + // Underutilized node was removed because of low utilization + Underutilized NodeScaleDownReason = "underutilized" + // Empty node was removed + Empty NodeScaleDownReason = "empty" + // Unready node was removed + Unready NodeScaleDownReason = "unready" + + // CloudProviderError caused scale-up to fail + CloudProviderError FailedScaleUpReason = "cloudProviderError" + // APIError caused scale-up to fail + APIError FailedScaleUpReason = "apiCallError" + // Timeout was encountered when trying to scale-up + Timeout FailedScaleUpReason = "timeout" + + // DirectionScaleDown is the direction of skipped scaling event when scaling in (shrinking) + DirectionScaleDown string = "down" + // DirectionScaleUp is the direction of skipped scaling event when scaling out (growing) + DirectionScaleUp string = "up" + + // CpuResourceLimit minimum or maximum reached, check the direction label to determine min or max + CpuResourceLimit string = "CpuResourceLimit" + // MemoryResourceLimit minimum or maximum reached, check the direction label to determine min or max + MemoryResourceLimit string = "MemoryResourceLimit" + + // autoscaledGroup is managed by CA + autoscaledGroup NodeGroupType = "autoscaled" + // autoprovisionedGroup have been created by CA (Node Autoprovisioning), + // is currently autoscaled and can be removed by CA if it's no longer needed + autoprovisionedGroup NodeGroupType = "autoprovisioned" + + // LogLongDurationThreshold defines the duration after which long function + // duration will be logged (in addition to being counted in metric). + // This is meant to help find unexpectedly long function execution times for + // debugging purposes. + LogLongDurationThreshold = 5 * time.Second + // PodEvictionSucceed means creation of the pod eviction object succeed + PodEvictionSucceed PodEvictionResult = "succeeded" + // PodEvictionFailed means creation of the pod eviction object failed + PodEvictionFailed PodEvictionResult = "failed" +) + +// Names of Cluster Autoscaler operations +const ( + ScaleDown FunctionLabel = "scaleDown" + ScaleDownNodeDeletion FunctionLabel = "scaleDown:nodeDeletion" + ScaleDownSoftTaintUnneeded FunctionLabel = "scaleDown:softTaintUnneeded" + ScaleUp FunctionLabel = "scaleUp" + BuildPodEquivalenceGroups FunctionLabel = "scaleUp:buildPodEquivalenceGroups" + Estimate FunctionLabel = "scaleUp:estimate" + FindUnneeded FunctionLabel = "findUnneeded" + UpdateState FunctionLabel = "updateClusterState" + FilterOutSchedulable FunctionLabel = "filterOutSchedulable" + CloudProviderRefresh FunctionLabel = "cloudProviderRefresh" + Main FunctionLabel = "main" + Poll FunctionLabel = "poll" + Reconfigure FunctionLabel = "reconfigure" + Autoscaling FunctionLabel = "autoscaling" + LoopWait FunctionLabel = "loopWait" + BulkListAllGceInstances FunctionLabel = "bulkListInstances:listAllInstances" + BulkListMigInstances FunctionLabel = "bulkListInstances:listMigInstances" +) + +type caMetricsImpl struct { + registry metrics.KubeRegistry + + clusterSafeToAutoscale *k8smetrics.Gauge + nodesCount *k8smetrics.GaugeVec + nodeGroupsCount *k8smetrics.GaugeVec + unschedulablePodsCount *k8smetrics.GaugeVec + maxNodesCount *k8smetrics.Gauge + cpuCurrentCores *k8smetrics.Gauge + cpuLimitsCores *k8smetrics.GaugeVec + memoryCurrentBytes *k8smetrics.Gauge + memoryLimitsBytes *k8smetrics.GaugeVec + nodesGroupMinNodes *k8smetrics.GaugeVec + nodesGroupMaxNodes *k8smetrics.GaugeVec + nodesGroupTargetSize *k8smetrics.GaugeVec + nodesGroupHealthiness *k8smetrics.GaugeVec + nodeGroupBackOffStatus *k8smetrics.GaugeVec + + // Metrics related to autoscaler execution + lastActivity *k8smetrics.GaugeVec + functionDuration *k8smetrics.HistogramVec + functionDurationSummary *k8smetrics.SummaryVec + pendingNodeDeletions *k8smetrics.Gauge + + // Metrics related to autoscaler operations + errorsCount *k8smetrics.CounterVec + scaleUpCount *k8smetrics.Counter + gpuScaleUpCount *k8smetrics.CounterVec + failedScaleUpCount *k8smetrics.CounterVec + failedGPUScaleUpCount *k8smetrics.CounterVec + scaleDownCount *k8smetrics.CounterVec + gpuScaleDownCount *k8smetrics.CounterVec + evictionsCount *k8smetrics.CounterVec + unneededNodesCount *k8smetrics.Gauge + unremovableNodesCount *k8smetrics.GaugeVec + scaleDownInCooldown *k8smetrics.Gauge + oldUnregisteredNodesRemovedCount *k8smetrics.Counter + overflowingControllersCount *k8smetrics.Gauge + skippedScaleEventsCount *k8smetrics.CounterVec + nodeGroupCreationCount *k8smetrics.CounterVec + nodeGroupDeletionCount *k8smetrics.CounterVec + nodeTaintsCount *k8smetrics.GaugeVec + inconsistentInstancesMigsCount *k8smetrics.Gauge + binpackingHeterogeneity *k8smetrics.HistogramVec + maxNodeSkipEvalDurationSeconds *k8smetrics.Gauge + scaleDownNodeRemovalLatency *k8smetrics.HistogramVec +} + +func newCaMetricsImplWithRegistry(registry metrics.KubeRegistry) *caMetricsImpl { + reg := newCaMetricsImpl() + reg.registry = registry + return reg +} + +func newCaMetricsImpl() *caMetricsImpl { + return &caMetricsImpl{ + /**** Metrics related to cluster state ****/ + clusterSafeToAutoscale: k8smetrics.NewGauge( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "cluster_safe_to_autoscale", + Help: "Whether or not cluster is healthy enough for autoscaling. 1 if it is, 0 otherwise.", + }, + ), + + nodesCount: k8smetrics.NewGaugeVec( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "nodes_count", + Help: "Number of nodes in cluster.", + }, []string{"state"}, + ), + + nodeGroupsCount: k8smetrics.NewGaugeVec( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "node_groups_count", + Help: "Number of node groups managed by CA.", + }, []string{"node_group_type"}, + ), + + // Unschedulable pod count can be from scheduler-marked-unschedulable pods or not-yet-processed pods (unknown) + unschedulablePodsCount: k8smetrics.NewGaugeVec( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "unschedulable_pods_count", + Help: "Number of unschedulable pods in the cluster.", + }, []string{"type"}, + ), + + maxNodesCount: k8smetrics.NewGauge( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "max_nodes_count", + Help: "Maximum number of nodes in all node groups", + }, + ), + + cpuCurrentCores: k8smetrics.NewGauge( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "cluster_cpu_current_cores", + Help: "Current number of cores in the cluster, minus deleting nodes.", + }, + ), + + cpuLimitsCores: k8smetrics.NewGaugeVec( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "cpu_limits_cores", + Help: "Minimum and maximum number of cores in the cluster.", + }, []string{"direction"}, + ), + + memoryCurrentBytes: k8smetrics.NewGauge( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "cluster_memory_current_bytes", + Help: "Current number of bytes of memory in the cluster, minus deleting nodes.", + }, + ), + + memoryLimitsBytes: k8smetrics.NewGaugeVec( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "memory_limits_bytes", + Help: "Minimum and maximum number of bytes of memory in cluster.", + }, []string{"direction"}, + ), + + nodesGroupMinNodes: k8smetrics.NewGaugeVec( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "node_group_min_count", + Help: "Minimum number of nodes in the node group", + }, []string{"node_group"}, + ), + + nodesGroupMaxNodes: k8smetrics.NewGaugeVec( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "node_group_max_count", + Help: "Maximum number of nodes in the node group", + }, []string{"node_group"}, + ), + + nodesGroupTargetSize: k8smetrics.NewGaugeVec( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "node_group_target_count", + Help: "Target number of nodes in the node group by CA.", + }, []string{"node_group"}, + ), + + nodesGroupHealthiness: k8smetrics.NewGaugeVec( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "node_group_healthiness", + Help: "Whether or not node group is healthy enough for autoscaling. 1 if it is, 0 otherwise.", + }, []string{"node_group"}, + ), + + nodeGroupBackOffStatus: k8smetrics.NewGaugeVec( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "node_group_backoff_status", + Help: "Whether or not node group is backoff for not autoscaling. 1 if it is, 0 otherwise.", + }, []string{"node_group", "reason"}, + ), + + /**** Metrics related to autoscaler execution ****/ + lastActivity: k8smetrics.NewGaugeVec( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "last_activity", + Help: "Last time certain part of CA logic executed.", + }, []string{"activity"}, + ), + + functionDuration: k8smetrics.NewHistogramVec( + &k8smetrics.HistogramOpts{ + Namespace: caNamespace, + Name: "function_duration_seconds", + Help: "Time taken by various parts of CA main loop.", + Buckets: k8smetrics.ExponentialBuckets(0.01, 1.5, 30), // 0.01, 0.015, 0.0225, ..., 852.2269299239293, 1278.3403948858938 + }, []string{"function"}, + ), + + functionDurationSummary: k8smetrics.NewSummaryVec( + &k8smetrics.SummaryOpts{ + Namespace: caNamespace, + Name: "function_duration_quantile_seconds", + Help: "Quantiles of time taken by various parts of CA main loop.", + MaxAge: time.Hour, + }, []string{"function"}, + ), + + pendingNodeDeletions: k8smetrics.NewGauge( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "pending_node_deletions", + Help: "Number of nodes that haven't been removed or aborted after finished scale-down phase.", + }, + ), + + /**** Metrics related to autoscaler operations ****/ + errorsCount: k8smetrics.NewCounterVec( + &k8smetrics.CounterOpts{ + Namespace: caNamespace, + Name: "errors_total", + Help: "The number of CA loops failed due to an error.", + }, []string{"type"}, + ), + + scaleUpCount: k8smetrics.NewCounter( + &k8smetrics.CounterOpts{ + Namespace: caNamespace, + Name: "scaled_up_nodes_total", + Help: "Number of nodes added by CA.", + }, + ), + + gpuScaleUpCount: k8smetrics.NewCounterVec( + &k8smetrics.CounterOpts{ + Namespace: caNamespace, + Name: "scaled_up_gpu_nodes_total", + Help: "Number of GPU nodes added by CA, by GPU name.", + }, []string{"gpu_resource_name", "gpu_name"}, + ), + + failedScaleUpCount: k8smetrics.NewCounterVec( + &k8smetrics.CounterOpts{ + Namespace: caNamespace, + Name: "failed_scale_ups_total", + Help: "Number of times scale-up operation has failed.", + }, []string{"reason"}, + ), + + failedGPUScaleUpCount: k8smetrics.NewCounterVec( + &k8smetrics.CounterOpts{ + Namespace: caNamespace, + Name: "failed_gpu_scale_ups_total", + Help: "Number of times scale-up operation has failed.", + }, []string{"reason", "gpu_resource_name", "gpu_name"}, + ), + + scaleDownCount: k8smetrics.NewCounterVec( + &k8smetrics.CounterOpts{ + Namespace: caNamespace, + Name: "scaled_down_nodes_total", + Help: "Number of nodes removed by CA.", + }, []string{"reason"}, + ), + + gpuScaleDownCount: k8smetrics.NewCounterVec( + &k8smetrics.CounterOpts{ + Namespace: caNamespace, + Name: "scaled_down_gpu_nodes_total", + Help: "Number of GPU nodes removed by CA, by reason and GPU name.", + }, []string{"reason", "gpu_resource_name", "gpu_name"}, + ), + + evictionsCount: k8smetrics.NewCounterVec( + &k8smetrics.CounterOpts{ + Namespace: caNamespace, + Name: "evicted_pods_total", + Help: "Number of pods evicted by CA", + }, []string{"eviction_result"}, + ), + + unneededNodesCount: k8smetrics.NewGauge( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "unneeded_nodes_count", + Help: "Number of nodes currently considered unneeded by CA.", + }, + ), + + unremovableNodesCount: k8smetrics.NewGaugeVec( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "unremovable_nodes_count", + Help: "Number of nodes currently considered unremovable by CA.", + }, + []string{"reason"}, + ), + + scaleDownInCooldown: k8smetrics.NewGauge( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "scale_down_in_cooldown", + Help: "Whether or not the scale down is in cooldown. 1 if its, 0 otherwise.", + }, + ), + + oldUnregisteredNodesRemovedCount: k8smetrics.NewCounter( + &k8smetrics.CounterOpts{ + Namespace: caNamespace, + Name: "old_unregistered_nodes_removed_count", + Help: "Number of unregistered nodes removed by CA.", + }, + ), + + overflowingControllersCount: k8smetrics.NewGauge( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "overflowing_controllers_count", + Help: "Number of controllers that own a large set of heterogenous pods, preventing CA from treating these pods as equivalent.", + }, + ), + + skippedScaleEventsCount: k8smetrics.NewCounterVec( + &k8smetrics.CounterOpts{ + Namespace: caNamespace, + Name: "skipped_scale_events_count", + Help: "Count of scaling events that the CA has chosen to skip.", + }, + []string{"direction", "reason"}, + ), + + nodeGroupCreationCount: k8smetrics.NewCounterVec( + &k8smetrics.CounterOpts{ + Namespace: caNamespace, + Name: "created_node_groups_total", + Help: "Number of node groups created by Node Autoprovisioning.", + }, + []string{"group_type"}, + ), + + nodeGroupDeletionCount: k8smetrics.NewCounterVec( + &k8smetrics.CounterOpts{ + Namespace: caNamespace, + Name: "deleted_node_groups_total", + Help: "Number of node groups deleted by Node Autoprovisioning.", + }, + []string{"group_type"}, + ), + + nodeTaintsCount: k8smetrics.NewGaugeVec( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "node_taints_count", + Help: "Number of taints per type used in the cluster.", + }, + []string{"type"}, + ), + + inconsistentInstancesMigsCount: k8smetrics.NewGauge( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "inconsistent_instances_migs_count", + Help: "Number of migs where instance count according to InstanceGroupManagers.List() differs from the results of Instances.List(). This can happen when some instances are abandoned or a user edits instance 'created-by' metadata.", + }, + ), + + binpackingHeterogeneity: k8smetrics.NewHistogramVec( + &k8smetrics.HistogramOpts{ + Namespace: caNamespace, + Name: "binpacking_heterogeneity", + Help: "Number of groups of equivalent pods being processed as a part of the same binpacking simulation.", + Buckets: k8smetrics.ExponentialBuckets(1, 2, 6), // 1, 2, 4, ..., 32 + }, []string{"instance_type", "cpu_count", "namespace_count"}, + ), + + maxNodeSkipEvalDurationSeconds: k8smetrics.NewGauge( + &k8smetrics.GaugeOpts{ + Namespace: caNamespace, + Name: "max_node_skip_eval_duration_seconds", + Help: "Maximum evaluation time of a node being skipped during ScaleDown.", + }, + ), + + scaleDownNodeRemovalLatency: k8smetrics.NewHistogramVec( + &k8smetrics.HistogramOpts{ + Namespace: caNamespace, + Name: "node_removal_latency_seconds", + Help: "Latency from when an unneeded node is eligible for scale down until it is removed (deleted=true) or it became needed again (deleted=false).", + Buckets: k8smetrics.ExponentialBuckets(1, 1.5, 19), // ~1s → ~24min + }, []string{"deleted"}, + ), + } +} + +func (m *caMetricsImpl) mustRegister(cs ...k8smetrics.Registerable) { + if m.registry != nil { + m.registry.MustRegister(cs...) + return + } + legacyregistry.MustRegister(cs...) +} + +// RegisterAll registers all metrics. +func (m *caMetricsImpl) RegisterAll(emitPerNodeGroupMetrics bool) { + m.mustRegister(m.clusterSafeToAutoscale) + m.mustRegister(m.nodesCount) + m.mustRegister(m.nodeGroupsCount) + m.mustRegister(m.unschedulablePodsCount) + m.mustRegister(m.maxNodesCount) + m.mustRegister(m.cpuCurrentCores) + m.mustRegister(m.cpuLimitsCores) + m.mustRegister(m.memoryCurrentBytes) + m.mustRegister(m.memoryLimitsBytes) + m.mustRegister(m.lastActivity) + m.mustRegister(m.functionDuration) + m.mustRegister(m.functionDurationSummary) + m.mustRegister(m.errorsCount) + m.mustRegister(m.scaleUpCount) + m.mustRegister(m.gpuScaleUpCount) + m.mustRegister(m.failedScaleUpCount) + m.mustRegister(m.failedGPUScaleUpCount) + m.mustRegister(m.scaleDownCount) + m.mustRegister(m.gpuScaleDownCount) + m.mustRegister(m.evictionsCount) + m.mustRegister(m.unneededNodesCount) + m.mustRegister(m.unremovableNodesCount) + m.mustRegister(m.scaleDownInCooldown) + m.mustRegister(m.oldUnregisteredNodesRemovedCount) + m.mustRegister(m.overflowingControllersCount) + m.mustRegister(m.skippedScaleEventsCount) + m.mustRegister(m.nodeGroupCreationCount) + m.mustRegister(m.nodeGroupDeletionCount) + m.mustRegister(m.pendingNodeDeletions) + m.mustRegister(m.nodeTaintsCount) + m.mustRegister(m.inconsistentInstancesMigsCount) + m.mustRegister(m.binpackingHeterogeneity) + m.mustRegister(m.maxNodeSkipEvalDurationSeconds) + m.mustRegister(m.scaleDownNodeRemovalLatency) + + if emitPerNodeGroupMetrics { + m.mustRegister(m.nodesGroupMinNodes) + m.mustRegister(m.nodesGroupMaxNodes) + m.mustRegister(m.nodesGroupTargetSize) + m.mustRegister(m.nodesGroupHealthiness) + m.mustRegister(m.nodeGroupBackOffStatus) + } +} + +// InitMetrics initializes all metrics +func (m *caMetricsImpl) InitMetrics() { + for _, errorType := range []errors.AutoscalerErrorType{errors.CloudProviderError, errors.ApiCallError, errors.InternalError, errors.TransientError, errors.ConfigurationError, errors.NodeGroupDoesNotExistError, errors.UnexpectedScaleDownStateError} { + m.errorsCount.WithLabelValues(string(errorType)).Add(0) + } + + for _, reason := range []FailedScaleUpReason{CloudProviderError, APIError, Timeout} { + m.scaleDownCount.WithLabelValues(string(reason)).Add(0) + m.failedScaleUpCount.WithLabelValues(string(reason)).Add(0) + } + + for _, result := range []PodEvictionResult{PodEvictionSucceed, PodEvictionFailed} { + m.evictionsCount.WithLabelValues(string(result)).Add(0) + } + + m.skippedScaleEventsCount.WithLabelValues(DirectionScaleDown, CpuResourceLimit).Add(0) + m.skippedScaleEventsCount.WithLabelValues(DirectionScaleDown, MemoryResourceLimit).Add(0) + m.skippedScaleEventsCount.WithLabelValues(DirectionScaleUp, CpuResourceLimit).Add(0) + m.skippedScaleEventsCount.WithLabelValues(DirectionScaleUp, MemoryResourceLimit).Add(0) + +} + +// UpdateDurationFromStart records the duration of the step identified by the +// label using start time +func (m *caMetricsImpl) UpdateDurationFromStart(label FunctionLabel, start time.Time) { + duration := time.Now().Sub(start) + m.UpdateDuration(label, duration) +} + +// UpdateDuration records the duration of the step identified by the label +func (m *caMetricsImpl) UpdateDuration(label FunctionLabel, duration time.Duration) { + if duration > LogLongDurationThreshold { + klog.V(4).Infof("Function %s took %v to complete", label, duration) + } + m.functionDuration.WithLabelValues(string(label)).Observe(duration.Seconds()) + m.functionDurationSummary.WithLabelValues(string(label)).Observe(duration.Seconds()) +} + +// UpdateLastTime records the time the step identified by the label was started +func (m *caMetricsImpl) UpdateLastTime(label FunctionLabel, now time.Time) { + m.lastActivity.WithLabelValues(string(label)).Set(float64(now.Unix())) +} + +// UpdateClusterSafeToAutoscale records if cluster is safe to autoscale +func (m *caMetricsImpl) UpdateClusterSafeToAutoscale(safe bool) { + if safe { + m.clusterSafeToAutoscale.Set(1) + } else { + m.clusterSafeToAutoscale.Set(0) + } +} + +// UpdateNodesCount records the number of nodes in cluster +func (m *caMetricsImpl) UpdateNodesCount(ready, unready, starting, longUnregistered, unregistered int) { + m.nodesCount.WithLabelValues(readyLabel).Set(float64(ready)) + m.nodesCount.WithLabelValues(unreadyLabel).Set(float64(unready)) + m.nodesCount.WithLabelValues(startingLabel).Set(float64(starting)) + m.nodesCount.WithLabelValues(longUnregisteredLabel).Set(float64(longUnregistered)) + m.nodesCount.WithLabelValues(unregisteredLabel).Set(float64(unregistered)) +} + +// UpdateNodeGroupsCount records the number of node groups managed by CA +func (m *caMetricsImpl) UpdateNodeGroupsCount(autoscaled, autoprovisioned int) { + m.nodeGroupsCount.WithLabelValues(string(autoscaledGroup)).Set(float64(autoscaled)) + m.nodeGroupsCount.WithLabelValues(string(autoprovisionedGroup)).Set(float64(autoprovisioned)) +} + +// UpdateUnschedulablePodsCount records number of currently unschedulable pods +func (m *caMetricsImpl) UpdateUnschedulablePodsCount(uschedulablePodsCount, schedulerUnprocessedCount int) { + m.UpdateUnschedulablePodsCountWithLabel(uschedulablePodsCount, "unschedulable") + m.UpdateUnschedulablePodsCountWithLabel(schedulerUnprocessedCount, "scheduler_unprocessed") +} + +// UpdateUnschedulablePodsCountWithLabel records number of currently unschedulable pods wil label "type" value "label" +func (m *caMetricsImpl) UpdateUnschedulablePodsCountWithLabel(uschedulablePodsCount int, label string) { + m.unschedulablePodsCount.WithLabelValues(label).Set(float64(uschedulablePodsCount)) +} + +// UpdateMaxNodesCount records the current maximum number of nodes being set for all node groups +func (m *caMetricsImpl) UpdateMaxNodesCount(nodesCount int) { + m.maxNodesCount.Set(float64(nodesCount)) +} + +// UpdateClusterCPUCurrentCores records the number of cores in the cluster, minus deleting nodes +func (m *caMetricsImpl) UpdateClusterCPUCurrentCores(coresCount int64) { + m.cpuCurrentCores.Set(float64(coresCount)) +} + +// UpdateCPULimitsCores records the minimum and maximum number of cores in the cluster +func (m *caMetricsImpl) UpdateCPULimitsCores(minCoresCount int64, maxCoresCount int64) { + m.cpuLimitsCores.WithLabelValues("minimum").Set(float64(minCoresCount)) + m.cpuLimitsCores.WithLabelValues("maximum").Set(float64(maxCoresCount)) +} + +// UpdateClusterMemoryCurrentBytes records the number of bytes of memory in the cluster, minus deleting nodes +func (m *caMetricsImpl) UpdateClusterMemoryCurrentBytes(memoryCount int64) { + m.memoryCurrentBytes.Set(float64(memoryCount)) +} + +// UpdateMemoryLimitsBytes records the minimum and maximum bytes of memory in the cluster +func (m *caMetricsImpl) UpdateMemoryLimitsBytes(minMemoryCount int64, maxMemoryCount int64) { + m.memoryLimitsBytes.WithLabelValues("minimum").Set(float64(minMemoryCount)) + m.memoryLimitsBytes.WithLabelValues("maximum").Set(float64(maxMemoryCount)) +} + +// UpdateNodeGroupMin records the node group minimum allowed number of nodes +func (m *caMetricsImpl) UpdateNodeGroupMin(nodeGroup string, minNodes int) { + m.nodesGroupMinNodes.WithLabelValues(nodeGroup).Set(float64(minNodes)) +} + +// UpdateNodeGroupMax records the node group maximum allowed number of nodes +func (m *caMetricsImpl) UpdateNodeGroupMax(nodeGroup string, maxNodes int) { + m.nodesGroupMaxNodes.WithLabelValues(nodeGroup).Set(float64(maxNodes)) +} + +// UpdateNodeGroupTargetSize records the node group target size +func (m *caMetricsImpl) UpdateNodeGroupTargetSize(targetSizes map[string]int) { + for nodeGroup, targetSize := range targetSizes { + m.nodesGroupTargetSize.WithLabelValues(nodeGroup).Set(float64(targetSize)) + } +} + +// UpdateNodeGroupHealthStatus records if node group is healthy to autoscaling +func (m *caMetricsImpl) UpdateNodeGroupHealthStatus(nodeGroup string, healthy bool) { + if healthy { + m.nodesGroupHealthiness.WithLabelValues(nodeGroup).Set(1) + } else { + m.nodesGroupHealthiness.WithLabelValues(nodeGroup).Set(0) + } +} + +// UpdateNodeGroupBackOffStatus records if node group is backoff for not autoscaling +func (m *caMetricsImpl) UpdateNodeGroupBackOffStatus(nodeGroup string, backoffReasonStatus map[string]bool) { + if len(backoffReasonStatus) == 0 { + m.nodeGroupBackOffStatus.WithLabelValues(nodeGroup, "").Set(0) + } else { + for reason, backoff := range backoffReasonStatus { + if backoff { + m.nodeGroupBackOffStatus.WithLabelValues(nodeGroup, reason).Set(1) + } else { + m.nodeGroupBackOffStatus.WithLabelValues(nodeGroup, reason).Set(0) + } + } + } +} + +// RegisterError records any errors preventing Cluster Autoscaler from working. +// No more than one error should be recorded per loop. +func (m *caMetricsImpl) RegisterError(err errors.AutoscalerError) { + m.errorsCount.WithLabelValues(string(err.Type())).Add(1.0) +} + +// RegisterScaleUp records number of nodes added by scale up +func (m *caMetricsImpl) RegisterScaleUp(nodesCount int, gpuResourceName, gpuType string) { + m.scaleUpCount.Add(float64(nodesCount)) + if gpuType != gpu.MetricsNoGPU { + m.gpuScaleUpCount.WithLabelValues(gpuResourceName, gpuType).Add(float64(nodesCount)) + } +} + +// RegisterFailedScaleUp records a failed scale-up operation +func (m *caMetricsImpl) RegisterFailedScaleUp(reason FailedScaleUpReason, gpuResourceName, gpuType string) { + m.failedScaleUpCount.WithLabelValues(string(reason)).Inc() + if gpuType != gpu.MetricsNoGPU { + m.failedGPUScaleUpCount.WithLabelValues(string(reason), gpuResourceName, gpuType).Inc() + } +} + +// RegisterScaleDown records number of nodes removed by scale down +func (m *caMetricsImpl) RegisterScaleDown(nodesCount int, gpuResourceName, gpuType string, reason NodeScaleDownReason) { + m.scaleDownCount.WithLabelValues(string(reason)).Add(float64(nodesCount)) + if gpuType != gpu.MetricsNoGPU { + m.gpuScaleDownCount.WithLabelValues(string(reason), gpuResourceName, gpuType).Add(float64(nodesCount)) + } +} + +// RegisterEvictions records number of evicted pods succeed or failed +func (m *caMetricsImpl) RegisterEvictions(podsCount int, result PodEvictionResult) { + m.evictionsCount.WithLabelValues(string(result)).Add(float64(podsCount)) +} + +// UpdateUnneededNodesCount records number of currently unneeded nodes +func (m *caMetricsImpl) UpdateUnneededNodesCount(nodesCount int) { + m.unneededNodesCount.Set(float64(nodesCount)) +} + +// UpdateUnremovableNodesCount records number of currently unremovable nodes +func (m *caMetricsImpl) UpdateUnremovableNodesCount(unremovableReasonCounts map[simulator.UnremovableReason]int) { + for reason, count := range unremovableReasonCounts { + m.unremovableNodesCount.WithLabelValues(fmt.Sprintf("%v", reason)).Set(float64(count)) + } +} + +// RegisterNodeGroupCreation registers node group creation +func (m *caMetricsImpl) RegisterNodeGroupCreation() { + m.RegisterNodeGroupCreationWithLabelValues("") +} + +// RegisterNodeGroupCreationWithLabelValues registers node group creation with the provided labels +func (m *caMetricsImpl) RegisterNodeGroupCreationWithLabelValues(groupType string) { + m.nodeGroupCreationCount.WithLabelValues(groupType).Add(1.0) +} + +// RegisterNodeGroupDeletion registers node group deletion +func (m *caMetricsImpl) RegisterNodeGroupDeletion() { + m.RegisterNodeGroupDeletionWithLabelValues("") +} + +// RegisterNodeGroupDeletionWithLabelValues registers node group deletion with the provided labels +func (m *caMetricsImpl) RegisterNodeGroupDeletionWithLabelValues(groupType string) { + m.nodeGroupDeletionCount.WithLabelValues(groupType).Add(1.0) +} + +// UpdateScaleDownInCooldown registers if the cluster autoscaler +// scaledown is in cooldown +func (m *caMetricsImpl) UpdateScaleDownInCooldown(inCooldown bool) { + if inCooldown { + m.scaleDownInCooldown.Set(1.0) + } else { + m.scaleDownInCooldown.Set(0.0) + } +} + +// RegisterOldUnregisteredNodesRemoved records number of old unregistered +// nodes that have been removed by the cluster autoscaler +func (m *caMetricsImpl) RegisterOldUnregisteredNodesRemoved(nodesCount int) { + m.oldUnregisteredNodesRemovedCount.Add(float64(nodesCount)) +} + +// UpdateOverflowingControllers sets the number of controllers that could not +// have their pods cached. +func (m *caMetricsImpl) UpdateOverflowingControllers(count int) { + m.overflowingControllersCount.Set(float64(count)) +} + +// RegisterSkippedScaleDownCPU increases the count of skipped scale outs because of CPU resource limits +func (m *caMetricsImpl) RegisterSkippedScaleDownCPU() { + m.skippedScaleEventsCount.WithLabelValues(DirectionScaleDown, CpuResourceLimit).Add(1.0) +} + +// RegisterSkippedScaleDownMemory increases the count of skipped scale outs because of Memory resource limits +func (m *caMetricsImpl) RegisterSkippedScaleDownMemory() { + m.skippedScaleEventsCount.WithLabelValues(DirectionScaleDown, MemoryResourceLimit).Add(1.0) +} + +// RegisterSkippedScaleUpCPU increases the count of skipped scale outs because of CPU resource limits +func (m *caMetricsImpl) RegisterSkippedScaleUpCPU() { + m.skippedScaleEventsCount.WithLabelValues(DirectionScaleUp, CpuResourceLimit).Add(1.0) +} + +// RegisterSkippedScaleUpMemory increases the count of skipped scale outs because of Memory resource limits +func (m *caMetricsImpl) RegisterSkippedScaleUpMemory() { + m.skippedScaleEventsCount.WithLabelValues(DirectionScaleUp, MemoryResourceLimit).Add(1.0) +} + +// ObservePendingNodeDeletions records the current value of nodes_pending_deletion metric +func (m *caMetricsImpl) ObservePendingNodeDeletions(value int) { + m.pendingNodeDeletions.Set(float64(value)) +} + +// ObserveNodeTaintsCount records the node taints count of given type. +func (m *caMetricsImpl) ObserveNodeTaintsCount(taintType string, count float64) { + m.nodeTaintsCount.WithLabelValues(taintType).Set(count) +} + +// UpdateInconsistentInstancesMigsCount records the observed number of migs where instance count +// according to InstanceGroupManagers.List() differs from the results of Instances.List(). +// This can happen when some instances are abandoned or a user edits instance 'created-by' metadata. +func (m *caMetricsImpl) UpdateInconsistentInstancesMigsCount(migCount int) { + m.inconsistentInstancesMigsCount.Set(float64(migCount)) +} + +// ObserveBinpackingHeterogeneity records the number of pod equivalence groups +// considered in a single binpacking estimation. +func (m *caMetricsImpl) ObserveBinpackingHeterogeneity(instanceType, cpuCount, namespaceCount string, pegCount int) { + m.binpackingHeterogeneity.WithLabelValues(instanceType, cpuCount, namespaceCount).Observe(float64(pegCount)) +} + +// UpdateScaleDownNodeRemovalLatency records the time after which node was deleted/needed +// again after being marked unneded +func (m *caMetricsImpl) UpdateScaleDownNodeRemovalLatency(deleted bool, duration time.Duration) { + m.scaleDownNodeRemovalLatency.WithLabelValues(strconv.FormatBool(deleted)).Observe(duration.Seconds()) +} + +// ObserveMaxNodeSkipEvalDurationSeconds records the longest time during which node was skipped during ScaleDown. +// If a node is skipped multiple times consecutively, we store only the earliest timestamp. +func (m *caMetricsImpl) ObserveMaxNodeSkipEvalDurationSeconds(duration time.Duration) { + m.maxNodeSkipEvalDurationSeconds.Set(duration.Seconds()) +} diff --git a/cluster-autoscaler/metrics/metrics_test.go b/cluster-autoscaler/metrics/metrics_test.go index 4bbe87b526f6..89211d4b9a11 100644 --- a/cluster-autoscaler/metrics/metrics_test.go +++ b/cluster-autoscaler/metrics/metrics_test.go @@ -21,24 +21,30 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" + "k8s.io/component-base/metrics" ) func TestDisabledPerNodeGroupMetrics(t *testing.T) { - t.Skip("Registering metrics multiple times causes panic. Skipping until the test is fixed to not impact other tests.") - RegisterAll(false) - assert.False(t, nodesGroupMinNodes.IsCreated()) - assert.False(t, nodesGroupMaxNodes.IsCreated()) + // Use a custom registry for isolation to avoid panics from re-registering metrics. + reg := metrics.NewKubeRegistry() + assert.NotNil(t, reg) + m := newCaMetricsImplWithRegistry(reg) + m.RegisterAll(false) + assert.False(t, m.nodesGroupMinNodes.IsCreated()) + assert.False(t, m.nodesGroupMaxNodes.IsCreated()) } func TestEnabledPerNodeGroupMetrics(t *testing.T) { - t.Skip("Registering metrics multiple times causes panic. Skipping until the test is fixed to not impact other tests.") - RegisterAll(true) - assert.True(t, nodesGroupMinNodes.IsCreated()) - assert.True(t, nodesGroupMaxNodes.IsCreated()) - - UpdateNodeGroupMin("foo", 2) - UpdateNodeGroupMax("foo", 100) - - assert.Equal(t, 2, int(testutil.ToFloat64(nodesGroupMinNodes.GaugeVec.WithLabelValues("foo")))) - assert.Equal(t, 100, int(testutil.ToFloat64(nodesGroupMaxNodes.GaugeVec.WithLabelValues("foo")))) + // Use a custom registry for isolation + reg := metrics.NewKubeRegistry() + m := newCaMetricsImplWithRegistry(reg) + m.RegisterAll(true) + assert.True(t, m.nodesGroupMinNodes.IsCreated()) + assert.True(t, m.nodesGroupMaxNodes.IsCreated()) + + m.UpdateNodeGroupMin("foo", 2) + m.UpdateNodeGroupMax("foo", 100) + + assert.Equal(t, 2, int(testutil.ToFloat64(m.nodesGroupMinNodes.GaugeVec.WithLabelValues("foo")))) + assert.Equal(t, 100, int(testutil.ToFloat64(m.nodesGroupMaxNodes.GaugeVec.WithLabelValues("foo")))) }