fix: count GetMetricData billed metrics per AWS pricing, not per query

pkg/clients/cloudwatch/client.go

@@ -15,6 +15,9 @@ package cloudwatch
 import (
 	"context"
 	"log/slog"
+	"math"
+	"sort"
+	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -25,6 +28,11 @@ import (
 	"github.com/prometheus-community/yet-another-cloudwatch-exporter/pkg/promutil"
 )
 
+// maxStatisticsPerBilledMetric is the number of statistics AWS lets you
+// request for a single metric before billing it as an additional metric
+// requested. See https://aws.amazon.com/cloudwatch/pricing/.
+const maxStatisticsPerBilledMetric = 5
+
 type Client interface {
 	// ListMetrics returns the list of metrics and dimensions for a given namespace
 	// and metric name. Results pagination is handled automatically; the caller
@@ -190,7 +198,7 @@ func (c client) GetMetricData(ctx context.Context, getMetricData []*model.Cloudw
 		ScanBy:            "TimestampDescending",
 	}
 	var resp aws_cloudwatch.GetMetricDataOutput
-	c.scrapeMetrics.CloudwatchGetMetricDataAPIMetricsCounter.Add(float64(len(input.MetricDataQueries)))
+	c.scrapeMetrics.CloudwatchGetMetricDataAPIMetricsCounter.Add(billedGetMetricDataMetricsCount(namespace, getMetricData))
 	c.logger.Debug("GetMetricData", "input", input)
 
 	paginator := aws_cloudwatch.NewGetMetricDataPaginator(c.cloudwatchAPI, input, func(options *aws_cloudwatch.GetMetricDataPaginatorOptions) {
@@ -214,6 +222,46 @@ func (c client) GetMetricData(ctx context.Context, getMetricData []*model.Cloudw
 	return toMetricDataResult(resp, exportAllDataPoints)
 }
 
+// billedGetMetricDataMetricsCount returns the number of metrics this GetMetricData call is
+// billed for. AWS bills GetMetricData per metric (namespace + name + dimensions) requested,
+// and lets up to maxStatisticsPerBilledMetric statistics of the same metric be requested for
+// the price of one. YACE requests each statistic of a metric as its own MetricDataQuery, so
+// counting queries directly overcounts whenever a metric has more than one statistic.
+// See https://aws.amazon.com/cloudwatch/pricing/.
+func billedGetMetricDataMetricsCount(namespace string, getMetricData []*model.CloudwatchData) float64 {
+	queriesByMetric := make(map[metricIdentity]int, len(getMetricData))
+	for _, data := range getMetricData {
+		queriesByMetric[metricIdentityOf(namespace, data)]++
+	}
+
+	var billed float64
+	for _, queries := range queriesByMetric {
+		billed += math.Ceil(float64(queries) / maxStatisticsPerBilledMetric)
+	}
+	return billed
+}
+
+// metricIdentity uniquely identifies the CloudWatch metric (namespace + name + dimensions) that
+// data was requested for, regardless of which statistic was requested.
+type metricIdentity struct {
+	namespace  string
+	metricName string
+	dimensions string
+}
+
+func metricIdentityOf(namespace string, data *model.CloudwatchData) metricIdentity {
+	dimensions := make([]string, 0, len(data.Dimensions))
+	for _, dim := range data.Dimensions {
+		dimensions = append(dimensions, dim.Name+"="+dim.Value)
+	}
+	sort.Strings(dimensions)
+	return metricIdentity{
+		namespace:  namespace,
+		metricName: data.MetricName,
+		dimensions: strings.Join(dimensions, ","),
+	}
+}
+
 func toMetricDataResult(resp aws_cloudwatch.GetMetricDataOutput, exportAllDataPoints bool) []MetricDataResult {
 	output := make([]MetricDataResult, 0, len(resp.MetricDataResults))
 	for _, metricDataResult := range resp.MetricDataResults {

pkg/clients/cloudwatch/client_test.go

@@ -21,6 +21,8 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus-community/yet-another-cloudwatch-exporter/pkg/model"
 )
 
 func Test_toMetricDataResult(t *testing.T) {
@@ -134,3 +136,81 @@ func Test_toMetricDataResult(t *testing.T) {
 		})
 	}
 }
+
+func Test_billedGetMetricDataMetricsCount(t *testing.T) {
+	dataWithStat := func(metricName string, dimensions []model.Dimension, statistic string) *model.CloudwatchData {
+		return &model.CloudwatchData{
+			MetricName: metricName,
+			Dimensions: dimensions,
+			GetMetricDataProcessingParams: &model.GetMetricDataProcessingParams{
+				Statistic: statistic,
+			},
+		}
+	}
+
+	volumeDimensions := []model.Dimension{{Name: "VolumeId", Value: "vol-1"}}
+
+	testCases := []struct {
+		name          string
+		getMetricData []*model.CloudwatchData
+		expected      float64
+	}{
+		{
+			name:          "no data",
+			getMetricData: []*model.CloudwatchData{},
+			expected:      0,
+		},
+		{
+			name: "single metric single statistic bills as one metric",
+			getMetricData: []*model.CloudwatchData{
+				dataWithStat("VolumeReadBytes", volumeDimensions, "Average"),
+			},
+			expected: 1,
+		},
+		{
+			name: "single metric with up to 5 statistics bills as one metric",
+			getMetricData: []*model.CloudwatchData{
+				dataWithStat("VolumeReadBytes", volumeDimensions, "Minimum"),
+				dataWithStat("VolumeReadBytes", volumeDimensions, "Maximum"),
+				dataWithStat("VolumeReadBytes", volumeDimensions, "Average"),
+				dataWithStat("VolumeReadBytes", volumeDimensions, "Sum"),
+				dataWithStat("VolumeReadBytes", volumeDimensions, "SampleCount"),
+			},
+			expected: 1,
+		},
+		{
+			name: "single metric with 6 statistics bills as two metrics",
+			getMetricData: []*model.CloudwatchData{
+				dataWithStat("VolumeReadBytes", volumeDimensions, "Minimum"),
+				dataWithStat("VolumeReadBytes", volumeDimensions, "Maximum"),
+				dataWithStat("VolumeReadBytes", volumeDimensions, "Average"),
+				dataWithStat("VolumeReadBytes", volumeDimensions, "Sum"),
+				dataWithStat("VolumeReadBytes", volumeDimensions, "SampleCount"),
+				dataWithStat("VolumeReadBytes", volumeDimensions, "p99"),
+			},
+			expected: 2,
+		},
+		{
+			name: "distinct metrics with the same dimensions bill separately",
+			getMetricData: []*model.CloudwatchData{
+				dataWithStat("VolumeReadBytes", volumeDimensions, "Average"),
+				dataWithStat("VolumeWriteBytes", volumeDimensions, "Average"),
+			},
+			expected: 2,
+		},
+		{
+			name: "same metric name on different resources bills separately",
+			getMetricData: []*model.CloudwatchData{
+				dataWithStat("VolumeReadBytes", []model.Dimension{{Name: "VolumeId", Value: "vol-1"}}, "Average"),
+				dataWithStat("VolumeReadBytes", []model.Dimension{{Name: "VolumeId", Value: "vol-2"}}, "Average"),
+			},
+			expected: 2,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			require.Equal(t, tc.expected, billedGetMetricDataMetricsCount("AWS/EBS", tc.getMetricData))
+		})
+	}
+}