Skip to content

Expose DPA mode and resource metrics #55475

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gh-worker-dd-mergequeue-cf854d merged 5 commits into from
Aug 31, 2026
+380 −21
Merged

Expose DPA mode and resource metrics #55475

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
98de2f7
Expose DPA mode and resources
datadog-bits Aug 26, 2026
6f83b01
Use shared metric tag constants
datadog-bits Aug 26, 2026
035fa5a
Handle DPA metric review feedback
datadog-bits Aug 26, 2026
0935a31
Add missing metrics Bazel dep
datadog-bits Aug 27, 2026
303a68e
Remove redundant controlled resources tag
datadog-bits Aug 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pkg/clusteragent/autoscaling/workload/metrics/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ go_library(
importpath = "github.com/DataDog/datadog-agent/pkg/clusteragent/autoscaling/workload/metrics",
visibility = ["//visibility:public"],
deps = [
"//comp/core/tagger/tags",
"//pkg/clusteragent/autoscaling/workload/model",
"//pkg/clusteragent/metricsstore",
"//pkg/util/kubernetes",
Expand Down
117 changes: 97 additions & 20 deletions pkg/clusteragent/autoscaling/workload/metrics/generator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@ import (
"strconv"
"strings"

taggerTags "github.com/DataDog/datadog-agent/comp/core/tagger/tags"
corev1 "k8s.io/api/core/v1"

datadoghqcommon "github.com/DataDog/datadog-operator/api/datadoghq/common"
datadoghq "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha2"

"github.com/DataDog/datadog-agent/pkg/clusteragent/autoscaling/workload/model"
"github.com/DataDog/datadog-agent/pkg/clusteragent/metricsstore"
Expand All @@ -24,6 +26,15 @@ import (

const (
metricPrefix = "datadog.cluster_agent.autoscaling.workload"

allContainersTagValue = "all"

dpaModeTagKey = "dpa_mode"
dpaDimensionTagKey = "dpa_dimension"
resourceNameTagKey = "resource_name"

dpaDimensionHorizontal = "horizontal"
dpaDimensionVertical = "vertical"
)

// Tag generation helper functions
Expand Down Expand Up @@ -57,8 +68,8 @@ func baseAutoscalerTags(internal *model.PodAutoscalerInternal) []string {

func resourceTags(containerName, resourceName string) []string {
return []string{
"resource_name:" + resourceName,
"kube_container_name:" + containerName,
resourceNameTagKey + ":" + resourceName,
taggerTags.KubeContainerName + ":" + containerName,
}
}

Expand All @@ -67,6 +78,53 @@ func conditionTags(baseTags []string, conditionType string) []string {
return append(baseTags, "type:"+conditionType)
}

func applyModeTags(baseTags []string, applyMode, dimension string) []string {
return append(baseTags, dpaModeTagKey+":"+applyMode, dpaDimensionTagKey+":"+dimension)
}

func controlledResourceTags(baseTags []string, containerName string, resource corev1.ResourceName) []string {
if containerName == "*" {
containerName = allContainersTagValue
}
return append(baseTags, taggerTags.KubeContainerName+":"+containerName, resourceNameTagKey+":"+string(resource), dpaDimensionTagKey+":"+dpaDimensionVertical)
}

func applyModeTagValue(spec *datadoghq.DatadogPodAutoscalerSpec) string {
mode := datadoghq.DatadogPodAutoscalerApplyModeApply
if spec != nil && spec.ApplyPolicy != nil && spec.ApplyPolicy.Mode != "" {
mode = spec.ApplyPolicy.Mode
}
return strings.ToLower(string(mode))
}

func controlledResourcesForMetrics(resources []corev1.ResourceName) []corev1.ResourceName {
if resources == nil {
return []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory}
}
return resources
}

func appendApplyModeMetrics(metrics metricsstore.StructuredMetrics, internal *model.PodAutoscalerInternal, baseTags []string) metricsstore.StructuredMetrics {
applyMode := applyModeTagValue(internal.Spec())
if internal.IsHorizontalScalingEnabled() {
metrics = append(metrics, metricsstore.StructuredMetric{
Name: metricPrefix + ".apply_mode",
Type: metricsstore.MetricTypeGauge,
Value: 1.0,
Tags: applyModeTags(baseTags, applyMode, dpaDimensionHorizontal),
})
}
if internal.IsVerticalScalingEnabled() {
metrics = append(metrics, metricsstore.StructuredMetric{
Name: metricPrefix + ".apply_mode",
Type: metricsstore.MetricTypeGauge,
Value: 1.0,
Tags: applyModeTags(baseTags, applyMode, dpaDimensionVertical),
})
}
return metrics
}

// objectiveTags generates tags for autoscaler objective (target) metrics.
// index is the 0-based position of the objective in spec.objectives[]; it guarantees a unique
// tag-set per objective (so multiple objectives never collapse into the same timeseries), which
Expand All @@ -77,10 +135,10 @@ func conditionTags(baseTags []string, conditionType string) []string {
func objectiveTags(baseTags []string, objectiveType, valueType, resourceName, containerName string, index int) []string {
tags := append(baseTags, "objective_type:"+objectiveType, "value_type:"+valueType, "objective_index:"+strconv.Itoa(index))
if resourceName != "" {
tags = append(tags, "resource_name:"+resourceName)
tags = append(tags, resourceNameTagKey+":"+resourceName)
}
if containerName != "" {
tags = append(tags, "kube_container_name:"+containerName)
tags = append(tags, taggerTags.KubeContainerName+":"+containerName)
}
return tags
}
Expand Down Expand Up @@ -171,7 +229,10 @@ func GeneratePodAutoscalerMetrics(internal *model.PodAutoscalerInternal) metrics
Tags: baseTags,
})

// 3. Horizontal scaling received replicas
// 3. DPA apply mode
metrics = appendApplyModeMetrics(metrics, internal, baseTags)

// 4. Horizontal scaling received replicas
if scalingValues.Horizontal != nil {
metrics = append(metrics, metricsstore.StructuredMetric{
Name: metricPrefix + ".horizontal_scaling_received_replicas",
Expand All @@ -181,7 +242,7 @@ func GeneratePodAutoscalerMetrics(internal *model.PodAutoscalerInternal) metrics
})
}

// 4. Vertical scaling received requests and limits
// 5. Vertical scaling received requests and limits
if scalingValues.Vertical != nil {
for _, containerResources := range scalingValues.Vertical.ContainerResources {
// Requests
Expand All @@ -206,7 +267,7 @@ func GeneratePodAutoscalerMetrics(internal *model.PodAutoscalerInternal) metrics
}
}

// 5. Horizontal scaling last action metrics
// 6. Horizontal scaling last action metrics
lastHorizontalActions := internal.HorizontalLastActions()
actionSource := ""
if sv := internal.ScalingValues(); sv.Horizontal != nil {
Expand Down Expand Up @@ -240,7 +301,7 @@ func GeneratePodAutoscalerMetrics(internal *model.PodAutoscalerInternal) metrics
Tags: append(horizontalTags, "status:ok"),
})

// 6. Vertical scaling last action metrics
// 7. Vertical scaling last action metrics
metrics = append(metrics, metricsstore.StructuredMetric{
Name: metricPrefix + ".vertical_rollout_triggered",
Type: metricsstore.MetricTypeMonotonicCount,
Expand All @@ -255,7 +316,7 @@ func GeneratePodAutoscalerMetrics(internal *model.PodAutoscalerInternal) metrics
Tags: append(baseWithVerticalSourceTags, "status:ok"),
})

// 7. In-place vertical scaling action metrics
// 8. In-place vertical scaling action metrics
metrics = append(metrics, metricsstore.StructuredMetric{
Name: metricPrefix + ".vertical_inplace.patch",
Type: metricsstore.MetricTypeMonotonicCount,
Expand Down Expand Up @@ -310,7 +371,7 @@ func GeneratePodAutoscalerMetrics(internal *model.PodAutoscalerInternal) metrics
Tags: baseWithVerticalSourceTags,
})

// 8. Vertical scaled/evicted replica gauges
// 9. Vertical scaled/evicted replica gauges
if scaledReplicas := internal.ScaledReplicas(); scaledReplicas != nil {
metrics = append(metrics, metricsstore.StructuredMetric{
Name: metricPrefix + ".status.vertical.scaled_replicas",
Expand All @@ -328,7 +389,7 @@ func GeneratePodAutoscalerMetrics(internal *model.PodAutoscalerInternal) metrics
})
}

// 9. Local recommender horizontal metrics
// 10. Local recommender horizontal metrics
if fallbackHorizontal := internal.FallbackScalingValues().Horizontal; fallbackHorizontal != nil {
localSourceTags := append(baseTags, "source:"+string(fallbackHorizontal.Source))
metrics = append(metrics, metricsstore.StructuredMetric{
Expand All @@ -347,7 +408,7 @@ func GeneratePodAutoscalerMetrics(internal *model.PodAutoscalerInternal) metrics
}
}

// 10. Horizontal scaling constraints
// 11. Horizontal scaling constraints
if spec := internal.Spec(); spec != nil && spec.Constraints != nil {
if spec.Constraints.MaxReplicas != nil {
metrics = append(metrics, metricsstore.StructuredMetric{
Expand All @@ -366,11 +427,27 @@ func GeneratePodAutoscalerMetrics(internal *model.PodAutoscalerInternal) metrics
})
}

// 11. Vertical scaling container constraints (per container, CPU in millicores, memory in bytes)
// 12. Vertical scaling container constraints (per container, CPU in millicores, memory in bytes)
// Mirror the resolveMinMaxBounds fallback from controller_vertical_helpers.go:
// prefer top-level MinAllowed/MaxAllowed; fall back to deprecated Requests field.
for _, container := range spec.Constraints.Containers {
containerTags := append(baseTags, "kube_container_name:"+container.Name)
containerTags := append(baseTags, taggerTags.KubeContainerName+":"+container.Name)

if container.Enabled == nil || *container.Enabled {
seenResources := make(map[corev1.ResourceName]struct{})
for _, resource := range controlledResourcesForMetrics(container.ControlledResources) {
Comment thread
clamoriniere marked this conversation as resolved.
Outdated
if _, seen := seenResources[resource]; seen {
continue
}
seenResources[resource] = struct{}{}
metrics = append(metrics, metricsstore.StructuredMetric{
Name: metricPrefix + ".vertical_scaling.controlled_resources",
Type: metricsstore.MetricTypeGauge,
Value: 1.0,
Tags: controlledResourceTags(baseTags, container.Name, resource),
})
}
}

effectiveMin := container.MinAllowed
if len(effectiveMin) == 0 && container.Requests != nil {
Expand Down Expand Up @@ -416,7 +493,7 @@ func GeneratePodAutoscalerMetrics(internal *model.PodAutoscalerInternal) metrics
}
}

// 12. Autoscaling objectives (target values from spec)
// 13. Autoscaling objectives (target values from spec)
if spec := internal.Spec(); spec != nil {
for idx, objective := range spec.Objectives {
switch objective.Type {
Expand Down Expand Up @@ -460,9 +537,9 @@ func GeneratePodAutoscalerMetrics(internal *model.PodAutoscalerInternal) metrics
}
}

// 13. Status metrics and autoscaler conditions (from upstream CR)
// 14. Status metrics and autoscaler conditions (from upstream CR)
if podAutoscaler := internal.UpstreamCR(); podAutoscaler != nil {
// 13a. Horizontal desired replicas from status
// 14a. Horizontal desired replicas from status
if horizontal := podAutoscaler.Status.Horizontal; horizontal != nil && horizontal.Target != nil {
metrics = append(metrics, metricsstore.StructuredMetric{
Name: metricPrefix + ".status.desired.replicas",
Expand All @@ -472,10 +549,10 @@ func GeneratePodAutoscalerMetrics(internal *model.PodAutoscalerInternal) metrics
})
}

// 13b. Vertical desired resources from status (per container, CPU in millicores, memory in bytes)
// 14b. Vertical desired resources from status (per container, CPU in millicores, memory in bytes)
if vertical := podAutoscaler.Status.Vertical; vertical != nil && vertical.Target != nil {
for _, container := range vertical.Target.DesiredResources {
containerTags := append(baseTags, "kube_container_name:"+container.Name)
containerTags := append(baseTags, taggerTags.KubeContainerName+":"+container.Name)
if cpuReq, ok := container.Requests[corev1.ResourceCPU]; ok {
metrics = append(metrics, metricsstore.StructuredMetric{
Name: metricPrefix + ".status.vertical.desired.container.cpu.request",
Expand Down Expand Up @@ -511,7 +588,7 @@ func GeneratePodAutoscalerMetrics(internal *model.PodAutoscalerInternal) metrics
}
}

// 13c. Autoscaler conditions
// 14c. Autoscaler conditions
for _, condition := range podAutoscaler.Status.Conditions {
value := 0.0
if condition.Status == corev1.ConditionTrue {
Expand Down
Loading
Loading