From 864447784738d64a1494ce499a3a4d9818eaee77 Mon Sep 17 00:00:00 2001 From: Manas Sivakumar Date: Sun, 13 Jul 2025 23:26:44 +0530 Subject: [PATCH 1/8] [feat] Network Costs --- .../prometheus/network_costs_test.go | 195 ++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 test/integration/prometheus/network_costs_test.go diff --git a/test/integration/prometheus/network_costs_test.go b/test/integration/prometheus/network_costs_test.go new file mode 100644 index 0000000..d1446fa --- /dev/null +++ b/test/integration/prometheus/network_costs_test.go @@ -0,0 +1,195 @@ +package prometheus + +// Description - Compares Network Costs from Prometheus and Allocation + +import ( + // "fmt" + "github.com/opencost/opencost-integration-tests/pkg/api" + "github.com/opencost/opencost-integration-tests/pkg/prometheus" + "github.com/opencost/opencost-integration-tests/pkg/utils" + "slices" + "testing" + "time" +) + +const tolerance = 0.05 + +func TestNetworkCosts(t *testing.T) { + apiObj := api.NewAPI() + + testCases := []struct { + name string + window string + aggregate string + accumulate string + }{ + { + name: "Yesterday", + window: "24h", + aggregate: "namespace", + accumulate: "false", + }, + } + + t.Logf("testCases: %v", testCases) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + + // Any data that is in a "raw allocation only" is not valid in any + // sort of cumulative Allocation (like one that is added). + + type NetworkCostsAggregate struct { + PromNetworkTransferBytes float64 + PromNetworkReceiveBytes float64 + Pods []string + AllocNetworkTransferBytes float64 + AllocNetworkReceiveBytes float64 + } + networkCostsNamespaceMap := make(map[string]*NetworkCostsAggregate) + + queryEnd := time.Now().UTC().Truncate(time.Hour).Add(time.Hour) + endTime := queryEnd.Unix() + // Collect Namespace results from Prometheus + client := prometheus.NewClient() + + //////////////////////////////////////////////////////////////////////////// + // Network Receive Bytes + + // sum(increase(container_network_receive_bytes_total{pod!=""}[24h:5m])) by (pod, namespace)` + //////////////////////////////////////////////////////////////////////////// + + promNetworkReceiveInput := prometheus.PrometheusInput{ + Metric: "container_network_receive_bytes_total", + } + promNetworkReceiveInput.IgnoreFilters = map[string][]string{ + "pod": {""}, + } + promNetworkReceiveInput.Function = []string{"increase", "sum"} + promNetworkReceiveInput.QueryWindow = tc.window + promNetworkReceiveInput.QueryResolution = "5m" + promNetworkReceiveInput.AggregateBy = []string{"pod", "namespace"} + promNetworkReceiveInput.Time = &endTime + + promNetworkReceiveResponse, err := client.RunPromQLQuery(promNetworkReceiveInput) + if err != nil { + t.Fatalf("Error while calling Prometheus API %v", err) + } + + //////////////////////////////////////////////////////////////////////////// + // Network Transfer Bytes + + // sum(increase(container_network_transmit_bytes_total{pod!="", %s}[%s:%dm])) by (pod_name, pod, namespace, %s)` + //////////////////////////////////////////////////////////////////////////// + + promNetworkTransferInput := prometheus.PrometheusInput{ + Metric: "container_network_transmit_bytes_total", + } + promNetworkTransferInput.IgnoreFilters = map[string][]string{ + "pod": {""}, + } + promNetworkTransferInput.Function = []string{"increase", "sum"} + promNetworkTransferInput.QueryWindow = tc.window + promNetworkTransferInput.QueryResolution = "5m" + promNetworkTransferInput.AggregateBy = []string{"pod", "namespace"} + promNetworkTransferInput.Time = &endTime + + promNetworkTransferResponse, err := client.RunPromQLQuery(promNetworkTransferInput) + if err != nil { + t.Fatalf("Error while calling Prometheus API %v", err) + } + + // Network Receive Bytes + for _, promNetworkReceiveResponse := range promNetworkReceiveResponse.Data.Result { + namespace := promNetworkReceiveResponse.Metric.Namespace + pod := promNetworkReceiveResponse.Metric.Pod + networkReceiveBytesPod := promNetworkReceiveResponse.Value.Value + networkCostsNamespace, ok := networkCostsNamespaceMap[namespace] + if !ok { + networkCostsNamespaceMap[namespace] = &NetworkCostsAggregate{ + PromNetworkReceiveBytes: networkReceiveBytesPod, + PromNetworkTransferBytes: 0.0, + AllocNetworkReceiveBytes: 0.0, + AllocNetworkTransferBytes: 0.0, + Pods: []string{pod}, + } + continue + } + + networkCostsNamespace.Pods = append(networkCostsNamespace.Pods, pod) + networkCostsNamespace.PromNetworkReceiveBytes += networkReceiveBytesPod + } + + // Network Transfer Bytes + for _, promNetworkTransferResponse := range promNetworkTransferResponse.Data.Result { + namespace := promNetworkTransferResponse.Metric.Namespace + pod := promNetworkTransferResponse.Metric.Pod + networkTransferBytesPod := promNetworkTransferResponse.Value.Value + networkCostsNamespace, ok := networkCostsNamespaceMap[namespace] + if !ok { + networkCostsNamespaceMap[namespace] = &NetworkCostsAggregate{ + PromNetworkReceiveBytes: networkTransferBytesPod, + PromNetworkTransferBytes: networkTransferBytesPod, + AllocNetworkReceiveBytes: 0.0, + AllocNetworkTransferBytes: 0.0, + Pods: []string{pod}, + } + continue + } + if !slices.Contains(networkCostsNamespace.Pods, pod) { + networkCostsNamespace.Pods = append(networkCostsNamespace.Pods, pod) + } + networkCostsNamespace.PromNetworkTransferBytes += networkTransferBytesPod + } + + + ///////////////////////////////////////////// + // API Client + ///////////////////////////////////////////// + + // Why doesn't allocation work on Namespace aggregate? + apiResponse, err := apiObj.GetAllocation(api.AllocationRequest{ + Window: tc.window, + Aggregate: tc.aggregate, + Accumulate: tc.accumulate, + }) + + if err != nil { + t.Fatalf("Error while calling Allocation API %v", err) + } + if apiResponse.Code != 200 { + t.Errorf("API returned non-200 code") + } + + for namespace, allocationResponseItem := range apiResponse.Data[0] { + networkCostsNamespace, ok := networkCostsNamespaceMap[namespace] + if !ok { + networkCostsNamespaceMap[namespace] = &NetworkCostsAggregate{ + PromNetworkReceiveBytes: 0.0, + PromNetworkTransferBytes: 0.0, + AllocNetworkReceiveBytes: allocationResponseItem.NetworkReceiveBytes, + AllocNetworkTransferBytes: allocationResponseItem.NetworkTransferBytes, + } + continue + } + networkCostsNamespace.AllocNetworkReceiveBytes = allocationResponseItem.NetworkReceiveBytes + networkCostsNamespace.AllocNetworkTransferBytes = allocationResponseItem.NetworkTransferBytes + } + + for namespace, networkCostValues := range networkCostsNamespaceMap { + t.Logf("Namespace %s", namespace) + withinRange, diff_percent := utils.AreWithinPercentage(networkCostValues.AllocNetworkTransferBytes, networkCostValues.PromNetworkTransferBytes, tolerance) + if !withinRange { + t.Errorf(" - NetworkTransferBytes[Fail]: DifferencePercent: %0.2f, Prometheus: %0.2f, /allocation: %0.2f", diff_percent, networkCostValues.PromNetworkTransferBytes, networkCostValues.AllocNetworkTransferBytes) + } else { + t.Logf(" - NetworkTransferBytes[Pass]: ~ %0.2f", networkCostValues.PromNetworkTransferBytes) + } + if !withinRange { + t.Errorf(" - NetworkReceiveBytes[Fail]: DifferencePercent: %0.2f, Prometheus: %0.2f, /allocation: %0.2f", diff_percent, networkCostValues.PromNetworkReceiveBytes, networkCostValues.AllocNetworkReceiveBytes) + } else { + t.Logf(" - NetworkReceiveBytes[Pass]: ~ %0.2f", networkCostValues.PromNetworkReceiveBytes) + } + } + }) + } +} From 1a8cd565fca7e44c2977bc470388bc21cb782721 Mon Sep 17 00:00:00 2001 From: Manas Sivakumar Date: Sun, 13 Jul 2025 23:28:26 +0530 Subject: [PATCH 2/8] remove unclosed tilde --- test/integration/prometheus/network_costs_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/prometheus/network_costs_test.go b/test/integration/prometheus/network_costs_test.go index d1446fa..1c53cf1 100644 --- a/test/integration/prometheus/network_costs_test.go +++ b/test/integration/prometheus/network_costs_test.go @@ -56,7 +56,7 @@ func TestNetworkCosts(t *testing.T) { //////////////////////////////////////////////////////////////////////////// // Network Receive Bytes - // sum(increase(container_network_receive_bytes_total{pod!=""}[24h:5m])) by (pod, namespace)` + // sum(increase(container_network_receive_bytes_total{pod!=""}[24h:5m])) by (pod, namespace) //////////////////////////////////////////////////////////////////////////// promNetworkReceiveInput := prometheus.PrometheusInput{ @@ -79,7 +79,7 @@ func TestNetworkCosts(t *testing.T) { //////////////////////////////////////////////////////////////////////////// // Network Transfer Bytes - // sum(increase(container_network_transmit_bytes_total{pod!="", %s}[%s:%dm])) by (pod_name, pod, namespace, %s)` + // sum(increase(container_network_transmit_bytes_total{pod!="", %s}[%s:%dm])) by (pod_name, pod, namespace, %s) //////////////////////////////////////////////////////////////////////////// promNetworkTransferInput := prometheus.PrometheusInput{ From bdcb0849cc94b6de51a609e3c3e80561bbc31886 Mon Sep 17 00:00:00 2001 From: Manas Sivakumar Date: Mon, 14 Jul 2025 21:35:00 +0530 Subject: [PATCH 3/8] run test in integration pipeline using bats --- README.md | 2 +- test/integration/prometheus/test.bats | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7efdceb..1cfd71f 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ WIP TESTS | [ ] | Ground Truth: storage byte hours|kube_persistentvolume_capacity_bytes| /allocations | For each aggregate, query prometheus to get the PVC capacity bytes. in the corresponding allocation ensure bytes match | Low | | [ ] | Ground Truth: PV costs | pv_hourly_cost | /allocations | Query cloud provider pricing to get expected costs for a given PV. ensure that the prom results match expected costs. ensure allocations API call returns PVs that match the costs. fail the tests if no PVs are in the returned allocations data | Medium | | [ ] | Ground Truth: PV info | kubecost_pv_info | /allocation | For each aggregate, query prometheus to get the PV info for each PV. Then, query allocations and confirm each PV is present in every aggregation and that the info matches. | Low | -| [ ] | Ground Truth: Network Bytes | container_network_receive_bytes_total/container_network_transmit_bytes_total | /allocation | For each aggregate, query prometheus to get thenetwork transmit/receive byte for every container. THen, query allocations and confirm each allocation matches the pod entry, and that for other aggregations, that the sums match. | Low | +| ✅ | Ground Truth: Network Bytes | container_network_receive_bytes_total/container_network_transmit_bytes_total | /allocation | For each aggregate, query prometheus to get thenetwork transmit/receive byte for every container. THen, query allocations and confirm each allocation matches the pod entry, and that for other aggregations, that the sums match. | Low | | [ ] | Ground Truth: Node Labels | kube_node_labels | /asset | For each aggregate, query prometheus to get the labels for each node. then, query the assets API and confirm the labels match for all the different aggregations | Low | | [ ] | Ground Truth: Node annotations | kube_node_annotations | /asset | For each aggregate, query prometheus to get the annotations for each node. then, query the assets API and confirm the annotations match for all the different aggregations | Low | | [ ] | Ground Truth: Labels | kube_pod_labels/kube_namespace_labels | /allocation | For each aggregate, query prometheus to get the labels for each pod and namespace. then, query the allocation API and for each result, confirm the labels in each aggregate contain the expected labels | Medium | diff --git a/test/integration/prometheus/test.bats b/test/integration/prometheus/test.bats index 209040c..e4790ca 100644 --- a/test/integration/prometheus/test.bats +++ b/test/integration/prometheus/test.bats @@ -61,4 +61,13 @@ teardown() { go test ./test/integration/prometheus/gpu_average_usage_test.go } +# ------------------------------------------------------ + + +# ------------------------------------------------------ +# Network Costs +@test "prometheus: Average GPU Usage Costs" { + go test ./test/integration/prometheus/network_costs_test.go +} + # ------------------------------------------------------ \ No newline at end of file From 974807b780f8f6f0ce3994a3e5a47f9689887b9c Mon Sep 17 00:00:00 2001 From: Manas Sivakumar Date: Mon, 14 Jul 2025 21:37:52 +0530 Subject: [PATCH 4/8] [Error] Fix Name of Test in bats --- test/integration/prometheus/test.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/prometheus/test.bats b/test/integration/prometheus/test.bats index e4790ca..0ff05a1 100644 --- a/test/integration/prometheus/test.bats +++ b/test/integration/prometheus/test.bats @@ -66,7 +66,7 @@ teardown() { # ------------------------------------------------------ # Network Costs -@test "prometheus: Average GPU Usage Costs" { +@test "prometheus: Network Cost" { go test ./test/integration/prometheus/network_costs_test.go } From b23344406beb8acda788fda4407c979cd70ce94e Mon Sep 17 00:00:00 2001 From: Manas Sivakumar Date: Thu, 24 Jul 2025 19:23:50 +0530 Subject: [PATCH 5/8] [feat] Network Zone Costs --- pkg/prometheus/client.go | 1 + test/integration/prometheus/client_test.go | 2 +- .../prometheus/network_internet_costs_test.go | 170 ++++++++++++++++++ test/integration/prometheus/test.bats | 6 +- 4 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 test/integration/prometheus/network_internet_costs_test.go diff --git a/pkg/prometheus/client.go b/pkg/prometheus/client.go index 4349c80..58fbfea 100644 --- a/pkg/prometheus/client.go +++ b/pkg/prometheus/client.go @@ -53,6 +53,7 @@ type PrometheusResponse struct { Result []struct { Metric struct { Pod string `json:"pod"` + PodName string `json:"pod_name"` Namespace string `json:"namespace"` Container string `json:"container"` } `json:"metric"` diff --git a/test/integration/prometheus/client_test.go b/test/integration/prometheus/client_test.go index e199d89..c6b7ae4 100644 --- a/test/integration/prometheus/client_test.go +++ b/test/integration/prometheus/client_test.go @@ -1,4 +1,4 @@ -package main +package prometheus import ( "fmt" diff --git a/test/integration/prometheus/network_internet_costs_test.go b/test/integration/prometheus/network_internet_costs_test.go new file mode 100644 index 0000000..2b02c39 --- /dev/null +++ b/test/integration/prometheus/network_internet_costs_test.go @@ -0,0 +1,170 @@ +package prometheus + +// Description - Compares Network Zone Internet from Prometheus and Allocation + +import ( + // "fmt" + "github.com/opencost/opencost-integration-tests/pkg/api" + "github.com/opencost/opencost-integration-tests/pkg/prometheus" + "github.com/opencost/opencost-integration-tests/pkg/utils" + "testing" + "time" +) + +const tolerance = 0.05 + +func TestNetworkInternetCosts(t *testing.T) { + apiObj := api.NewAPI() + + testCases := []struct { + name string + window string + aggregate string + accumulate string + }{ + { + name: "Yesterday", + window: "24h", + aggregate: "pod", + accumulate: "false", + }, + } + + t.Logf("testCases: %v", testCases) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + + // Any data that is in a "raw allocation only" is not valid in any + // sort of cumulative Allocation (like one that is added). + + type NetworkCostsAggregate struct { + PromNetworkInternetGiB float64 + AllocNetworkInternetGiB float64 + } + + networkCostsPodMap := make(map[string]*NetworkCostsAggregate) + + queryEnd := time.Now().UTC().Truncate(time.Hour).Add(time.Hour) + endTime := queryEnd.Unix() + // Collect Namespace results from Prometheus + client := prometheus.NewClient() + + //////////////////////////////////////////////////////////////////////////// + // Network Internet GiB + + // sum(increase(kubecost_pod_network_egress_bytes_total{internet="true"}[24h:5m])) by (pod_name, namespace) / 1024 / 1024 / 1024 + // Apply Division by 1024^3 when you are looping over the response + //////////////////////////////////////////////////////////////////////////// + + promNetworkInternetInput := prometheus.PrometheusInput{ + Metric: "kubecost_pod_network_egress_bytes_total", + } + promNetworkInternetInput.Filters = map[string]string{ + "internet": "true", + } + promNetworkInternetInput.Function = []string{"increase", "sum"} + promNetworkInternetInput.QueryWindow = tc.window + promNetworkInternetInput.QueryResolution = "5m" + promNetworkInternetInput.AggregateBy = []string{"pod_name", "namespace"} + promNetworkInternetInput.Time = &endTime + + promNetworkInternetResponse, err := client.RunPromQLQuery(promNetworkInternetInput) + if err != nil { + t.Fatalf("Error while calling Prometheus API %v", err) + } + + //////////////////////////////////////////////////////////////////////////// + // Network Internet price per GiB + + // avg(avg_over_time(kubecost_network_internet_egress_cost{%s}[%s])) by (%s) + //////////////////////////////////////////////////////////////////////////// + + promNetworkInternetCostInput := prometheus.PrometheusInput{ + Metric: "kubecost_network_internet_egress_cost", + } + promNetworkInternetCostInput.Function = []string{"avg_over_time", "avg"} + promNetworkInternetCostInput.QueryWindow = tc.window + promNetworkInternetCostInput.Time = &endTime + + promNetworkInternetCostResponse, err := client.RunPromQLQuery(promNetworkInternetCostInput) + if err != nil { + t.Fatalf("Error while calling Prometheus API %v", err) + } + + // -------------------------------- + // Network Internet Cost for all Pods + // -------------------------------- + + networkInternetCost := promNetworkInternetCostResponse.Data.Result[0].Value.Value + + // -------------------------------- + // Assign Network Costs to Pods and Cumulate based on Namespace + // -------------------------------- + + // Form a key based on namespace and pod name + + for _, promNetworkInternetItem := range promNetworkInternetResponse.Data.Result { + // namespace := promNetworkInternetItem.Metric.Namespace + pod := promNetworkInternetItem.Metric.PodName + gib := promNetworkInternetItem.Value.Value + + + networkCostsPodMap[pod] = &NetworkCostsAggregate{ + PromNetworkInternetGiB: (gib / 1024 / 1024 / 1024) * networkInternetCost, + AllocNetworkInternetGiB: 0.0, + } + + // networkCostsNamespace, ok := networkCostsPodMap[namespace] + // if !ok { + // networkCostsPodMap[pod] = &NetworkCostsAggregate{ + // PromNetworkInternetGiB: (gib / 1024 / 1024 / 1024) * networkInternetCost, + // AllocNetworkInternetGiB: 0.0, + // } + // continue + // } + } + + + ///////////////////////////////////////////// + // API Client + ///////////////////////////////////////////// + + // Why doesn't allocation work on Namespace aggregate? + apiResponse, err := apiObj.GetAllocation(api.AllocationRequest{ + Window: tc.window, + Aggregate: tc.aggregate, + Accumulate: tc.accumulate, + }) + + if err != nil { + t.Fatalf("Error while calling Allocation API %v", err) + } + if apiResponse.Code != 200 { + t.Errorf("API returned non-200 code") + } + + for pod, allocationResponseItem := range apiResponse.Data[0] { + networkCostsPod, ok := networkCostsPodMap[pod] + if !ok { + networkCostsPodMap[pod] = &NetworkCostsAggregate{ + PromNetworkInternetGiB: 0.0, + AllocNetworkInternetGiB: allocationResponseItem.NetworkInternetCost, + } + continue + } + networkCostsPod.AllocNetworkInternetGiB = allocationResponseItem.NetworkInternetCost + } + + for pod, networkCostValues := range networkCostsPodMap { + t.Logf("Pod %s", pod) + withinRange, diff_percent := utils.AreWithinPercentage(networkCostValues.AllocNetworkInternetGiB, networkCostValues.PromNetworkInternetGiB, tolerance) + if !withinRange { + t.Errorf(" - NetworkInternetCost[Fail]: DifferencePercent: %0.2f, Prometheus: %0.9f, /allocation: %0.9f", diff_percent, networkCostValues.PromNetworkInternetGiB, networkCostValues.AllocNetworkInternetGiB) + } else { + t.Logf(" - NetworkInternetCost[Pass]: ~ %0.5f", networkCostValues.PromNetworkInternetGiB) + } + } + }) + } +} diff --git a/test/integration/prometheus/test.bats b/test/integration/prometheus/test.bats index 0ff05a1..ed7f941 100644 --- a/test/integration/prometheus/test.bats +++ b/test/integration/prometheus/test.bats @@ -66,8 +66,12 @@ teardown() { # ------------------------------------------------------ # Network Costs -@test "prometheus: Network Cost" { +@test "prometheus: Network Transfer and Receive Bytes" { go test ./test/integration/prometheus/network_costs_test.go } +@test "prometheus: Network Internet Cost" { + go test ./test/integration/prometheus/network_internet_costs_test.go +} + # ------------------------------------------------------ \ No newline at end of file From 39c1dc9c97d728be782e1a82d98efc4db4b5b375 Mon Sep 17 00:00:00 2001 From: Manas Sivakumar Date: Sat, 26 Jul 2025 21:59:29 +0530 Subject: [PATCH 6/8] Edge case where all costs are below 1 cent --- .../prometheus/network_internet_costs_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/integration/prometheus/network_internet_costs_test.go b/test/integration/prometheus/network_internet_costs_test.go index 2b02c39..741e18e 100644 --- a/test/integration/prometheus/network_internet_costs_test.go +++ b/test/integration/prometheus/network_internet_costs_test.go @@ -156,7 +156,14 @@ func TestNetworkInternetCosts(t *testing.T) { networkCostsPod.AllocNetworkInternetGiB = allocationResponseItem.NetworkInternetCost } + noNegligibleCosts := false + negligilbleCost := 0.01 // 1 Cent of a Dollar for pod, networkCostValues := range networkCostsPodMap { + if networkCostValues.AllocNetworkInternetGiB < negligilbleCost { + continue + } else { + noNegligibleCosts = true + } t.Logf("Pod %s", pod) withinRange, diff_percent := utils.AreWithinPercentage(networkCostValues.AllocNetworkInternetGiB, networkCostValues.PromNetworkInternetGiB, tolerance) if !withinRange { @@ -165,6 +172,9 @@ func TestNetworkInternetCosts(t *testing.T) { t.Logf(" - NetworkInternetCost[Pass]: ~ %0.5f", networkCostValues.PromNetworkInternetGiB) } } + if !noNegligibleCosts { + t.Errorf("NetWork Internet Costs for all Pods are below 1 cent and hence cannot be considered as costs from resource usage and validated.") + } }) } } From 08b4800be96fd583d80a3fc83cd3429ba8e70aef Mon Sep 17 00:00:00 2001 From: Manas Sivakumar Date: Mon, 28 Jul 2025 22:50:04 +0530 Subject: [PATCH 7/8] Changed Variable name to a one that makes sense --- test/integration/prometheus/network_internet_costs_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/prometheus/network_internet_costs_test.go b/test/integration/prometheus/network_internet_costs_test.go index 741e18e..9c4240a 100644 --- a/test/integration/prometheus/network_internet_costs_test.go +++ b/test/integration/prometheus/network_internet_costs_test.go @@ -156,13 +156,13 @@ func TestNetworkInternetCosts(t *testing.T) { networkCostsPod.AllocNetworkInternetGiB = allocationResponseItem.NetworkInternetCost } - noNegligibleCosts := false + validCostsSeen := false negligilbleCost := 0.01 // 1 Cent of a Dollar for pod, networkCostValues := range networkCostsPodMap { if networkCostValues.AllocNetworkInternetGiB < negligilbleCost { continue } else { - noNegligibleCosts = true + validCostsSeen = true } t.Logf("Pod %s", pod) withinRange, diff_percent := utils.AreWithinPercentage(networkCostValues.AllocNetworkInternetGiB, networkCostValues.PromNetworkInternetGiB, tolerance) @@ -172,7 +172,7 @@ func TestNetworkInternetCosts(t *testing.T) { t.Logf(" - NetworkInternetCost[Pass]: ~ %0.5f", networkCostValues.PromNetworkInternetGiB) } } - if !noNegligibleCosts { + if !validCostsSeen { t.Errorf("NetWork Internet Costs for all Pods are below 1 cent and hence cannot be considered as costs from resource usage and validated.") } }) From 290c12efccef6e0cd65d41749ffac657b87d68ea Mon Sep 17 00:00:00 2001 From: Manas Sivakumar Date: Mon, 28 Jul 2025 22:59:06 +0530 Subject: [PATCH 8/8] Two Similar Tests for Zone and Region Network Costs --- .../prometheus/network_internet_costs_test.go | 2 +- .../prometheus/network_region_costs_test.go | 182 ++++++++++++++++++ .../prometheus/network_zone_costs_test.go | 182 ++++++++++++++++++ test/integration/prometheus/test.bats | 8 + 4 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 test/integration/prometheus/network_region_costs_test.go create mode 100644 test/integration/prometheus/network_zone_costs_test.go diff --git a/test/integration/prometheus/network_internet_costs_test.go b/test/integration/prometheus/network_internet_costs_test.go index 9c4240a..73a828a 100644 --- a/test/integration/prometheus/network_internet_costs_test.go +++ b/test/integration/prometheus/network_internet_costs_test.go @@ -1,6 +1,6 @@ package prometheus -// Description - Compares Network Zone Internet from Prometheus and Allocation +// Description - Compares Network Internet Costs from Prometheus and Allocation import ( // "fmt" diff --git a/test/integration/prometheus/network_region_costs_test.go b/test/integration/prometheus/network_region_costs_test.go new file mode 100644 index 0000000..6d0f6f3 --- /dev/null +++ b/test/integration/prometheus/network_region_costs_test.go @@ -0,0 +1,182 @@ +package prometheus + +// Description - Compares Network Region Costs from Prometheus and Allocation + +import ( + // "fmt" + "github.com/opencost/opencost-integration-tests/pkg/api" + "github.com/opencost/opencost-integration-tests/pkg/prometheus" + "github.com/opencost/opencost-integration-tests/pkg/utils" + "testing" + "time" +) + +const tolerance = 0.05 + +func TestNetworkRegionCosts(t *testing.T) { + apiObj := api.NewAPI() + + testCases := []struct { + name string + window string + aggregate string + accumulate string + }{ + { + name: "Yesterday", + window: "24h", + aggregate: "pod", + accumulate: "false", + }, + } + + t.Logf("testCases: %v", testCases) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + + // Any data that is in a "raw allocation only" is not valid in any + // sort of cumulative Allocation (like one that is added). + + type NetworkCostsAggregate struct { + PromNetworkRegionGiB float64 + AllocNetworkRegionGiB float64 + } + + networkCostsPodMap := make(map[string]*NetworkCostsAggregate) + + queryEnd := time.Now().UTC().Truncate(time.Hour).Add(time.Hour) + endTime := queryEnd.Unix() + // Collect Namespace results from Prometheus + client := prometheus.NewClient() + + //////////////////////////////////////////////////////////////////////////// + // Network Region GiB + + // sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", same_zone="false", same_region="false", %s}[%s:%dm])) by (pod_name, namespace, %s) / 1024 / 1024 / 1024 + // Apply Division by 1024^3 when you are looping over the response + //////////////////////////////////////////////////////////////////////////// + + promNetworkRegionInput := prometheus.PrometheusInput{ + Metric: "kubecost_pod_network_egress_bytes_total", + } + promNetworkRegionInput.Filters = map[string]string{ + "internet": "false", + "same_Region": "false", + "same_region": "false", + } + promNetworkRegionInput.Function = []string{"increase", "sum"} + promNetworkRegionInput.QueryWindow = tc.window + promNetworkRegionInput.QueryResolution = "5m" + promNetworkRegionInput.AggregateBy = []string{"pod_name", "namespace"} + promNetworkRegionInput.Time = &endTime + + promNetworkRegionResponse, err := client.RunPromQLQuery(promNetworkRegionInput) + if err != nil { + t.Fatalf("Error while calling Prometheus API %v", err) + } + + //////////////////////////////////////////////////////////////////////////// + // Network Region price per GiB + + // avg(avg_over_time(kubecost_network_region_egress_cost{%s}[%s])) by (%s) + //////////////////////////////////////////////////////////////////////////// + + promNetworkRegionCostInput := prometheus.PrometheusInput{ + Metric: "kubecost_network_region_egress_cost", + } + promNetworkRegionCostInput.Function = []string{"avg_over_time", "avg"} + promNetworkRegionCostInput.QueryWindow = tc.window + promNetworkRegionCostInput.Time = &endTime + + promNetworkRegionCostResponse, err := client.RunPromQLQuery(promNetworkRegionCostInput) + if err != nil { + t.Fatalf("Error while calling Prometheus API %v", err) + } + + // -------------------------------- + // Network Region Cost for all Pods + // -------------------------------- + + networkRegionCost := promNetworkRegionCostResponse.Data.Result[0].Value.Value + + // -------------------------------- + // Assign Network Costs to Pods and Cumulate based on Namespace + // -------------------------------- + + // Form a key based on namespace and pod name + + for _, promNetworkRegionItem := range promNetworkRegionResponse.Data.Result { + // namespace := promNetworkRegionItem.Metric.Namespace + pod := promNetworkRegionItem.Metric.PodName + gib := promNetworkRegionItem.Value.Value + + + networkCostsPodMap[pod] = &NetworkCostsAggregate{ + PromNetworkRegionGiB: (gib / 1024 / 1024 / 1024) * networkRegionCost, + AllocNetworkRegionGiB: 0.0, + } + + // networkCostsNamespace, ok := networkCostsPodMap[namespace] + // if !ok { + // networkCostsPodMap[pod] = &NetworkCostsAggregate{ + // PromNetworkRegionGiB: (gib / 1024 / 1024 / 1024) * networkRegionCost, + // AllocNetworkRegionGiB: 0.0, + // } + // continue + // } + } + + + ///////////////////////////////////////////// + // API Client + ///////////////////////////////////////////// + + // Why doesn't allocation work on Namespace aggregate? + apiResponse, err := apiObj.GetAllocation(api.AllocationRequest{ + Window: tc.window, + Aggregate: tc.aggregate, + Accumulate: tc.accumulate, + }) + + if err != nil { + t.Fatalf("Error while calling Allocation API %v", err) + } + if apiResponse.Code != 200 { + t.Errorf("API returned non-200 code") + } + + for pod, allocationResponseItem := range apiResponse.Data[0] { + networkCostsPod, ok := networkCostsPodMap[pod] + if !ok { + networkCostsPodMap[pod] = &NetworkCostsAggregate{ + PromNetworkRegionGiB: 0.0, + AllocNetworkRegionGiB: allocationResponseItem.NetworkCrossRegionCost, + } + continue + } + networkCostsPod.AllocNetworkRegionGiB = allocationResponseItem.NetworkCrossRegionCost + } + + validCostsSeen := false + negligilbleCost := 0.01 // 1 Cent of a Dollar + for pod, networkCostValues := range networkCostsPodMap { + if networkCostValues.AllocNetworkRegionGiB < negligilbleCost { + continue + } else { + validCostsSeen = true + } + t.Logf("Pod %s", pod) + withinRange, diff_percent := utils.AreWithinPercentage(networkCostValues.AllocNetworkRegionGiB, networkCostValues.PromNetworkRegionGiB, tolerance) + if !withinRange { + t.Errorf(" - NetworkRegionCost[Fail]: DifferencePercent: %0.2f, Prometheus: %0.9f, /allocation: %0.9f", diff_percent, networkCostValues.PromNetworkRegionGiB, networkCostValues.AllocNetworkRegionGiB) + } else { + t.Logf(" - NetworkRegionCost[Pass]: ~ %0.5f", networkCostValues.PromNetworkRegionGiB) + } + } + if !validCostsSeen { + t.Errorf("NetWork Region Costs for all Pods are below 1 cent and hence cannot be considered as costs from resource usage and validated.") + } + }) + } +} diff --git a/test/integration/prometheus/network_zone_costs_test.go b/test/integration/prometheus/network_zone_costs_test.go new file mode 100644 index 0000000..841bd81 --- /dev/null +++ b/test/integration/prometheus/network_zone_costs_test.go @@ -0,0 +1,182 @@ +package prometheus + +// Description - Compares Network Zone Costs from Prometheus and Allocation + +import ( + // "fmt" + "github.com/opencost/opencost-integration-tests/pkg/api" + "github.com/opencost/opencost-integration-tests/pkg/prometheus" + "github.com/opencost/opencost-integration-tests/pkg/utils" + "testing" + "time" +) + +const tolerance = 0.05 + +func TestNetworkZoneCosts(t *testing.T) { + apiObj := api.NewAPI() + + testCases := []struct { + name string + window string + aggregate string + accumulate string + }{ + { + name: "Yesterday", + window: "24h", + aggregate: "pod", + accumulate: "false", + }, + } + + t.Logf("testCases: %v", testCases) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + + // Any data that is in a "raw allocation only" is not valid in any + // sort of cumulative Allocation (like one that is added). + + type NetworkCostsAggregate struct { + PromNetworkZoneGiB float64 + AllocNetworkZoneGiB float64 + } + + networkCostsPodMap := make(map[string]*NetworkCostsAggregate) + + queryEnd := time.Now().UTC().Truncate(time.Hour).Add(time.Hour) + endTime := queryEnd.Unix() + // Collect Namespace results from Prometheus + client := prometheus.NewClient() + + //////////////////////////////////////////////////////////////////////////// + // Network Zone GiB + + // sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", same_zone="false", same_region="true", %s}[%s:%dm])) by (pod_name, namespace, %s) / 1024 / 1024 / 1024 + // Apply Division by 1024^3 when you are looping over the response + //////////////////////////////////////////////////////////////////////////// + + promNetworkZoneInput := prometheus.PrometheusInput{ + Metric: "kubecost_pod_network_egress_bytes_total", + } + promNetworkZoneInput.Filters = map[string]string{ + "internet": "false", + "same_zone": "false", + "same_region": "true", + } + promNetworkZoneInput.Function = []string{"increase", "sum"} + promNetworkZoneInput.QueryWindow = tc.window + promNetworkZoneInput.QueryResolution = "5m" + promNetworkZoneInput.AggregateBy = []string{"pod_name", "namespace"} + promNetworkZoneInput.Time = &endTime + + promNetworkZoneResponse, err := client.RunPromQLQuery(promNetworkZoneInput) + if err != nil { + t.Fatalf("Error while calling Prometheus API %v", err) + } + + //////////////////////////////////////////////////////////////////////////// + // Network Zone price per GiB + + // avg(avg_over_time(kubecost_network_zone_egress_cost{%s}[%s])) by (%s) + //////////////////////////////////////////////////////////////////////////// + + promNetworkZoneCostInput := prometheus.PrometheusInput{ + Metric: "kubecost_network_zone_egress_cost", + } + promNetworkZoneCostInput.Function = []string{"avg_over_time", "avg"} + promNetworkZoneCostInput.QueryWindow = tc.window + promNetworkZoneCostInput.Time = &endTime + + promNetworkZoneCostResponse, err := client.RunPromQLQuery(promNetworkZoneCostInput) + if err != nil { + t.Fatalf("Error while calling Prometheus API %v", err) + } + + // -------------------------------- + // Network Zone Cost for all Pods + // -------------------------------- + + networkZoneCost := promNetworkZoneCostResponse.Data.Result[0].Value.Value + + // -------------------------------- + // Assign Network Costs to Pods and Cumulate based on Namespace + // -------------------------------- + + // Form a key based on namespace and pod name + + for _, promNetworkZoneItem := range promNetworkZoneResponse.Data.Result { + // namespace := promNetworkZoneItem.Metric.Namespace + pod := promNetworkZoneItem.Metric.PodName + gib := promNetworkZoneItem.Value.Value + + + networkCostsPodMap[pod] = &NetworkCostsAggregate{ + PromNetworkZoneGiB: (gib / 1024 / 1024 / 1024) * networkZoneCost, + AllocNetworkZoneGiB: 0.0, + } + + // networkCostsNamespace, ok := networkCostsPodMap[namespace] + // if !ok { + // networkCostsPodMap[pod] = &NetworkCostsAggregate{ + // PromNetworkZoneGiB: (gib / 1024 / 1024 / 1024) * networkZoneCost, + // AllocNetworkZoneGiB: 0.0, + // } + // continue + // } + } + + + ///////////////////////////////////////////// + // API Client + ///////////////////////////////////////////// + + // Why doesn't allocation work on Namespace aggregate? + apiResponse, err := apiObj.GetAllocation(api.AllocationRequest{ + Window: tc.window, + Aggregate: tc.aggregate, + Accumulate: tc.accumulate, + }) + + if err != nil { + t.Fatalf("Error while calling Allocation API %v", err) + } + if apiResponse.Code != 200 { + t.Errorf("API returned non-200 code") + } + + for pod, allocationResponseItem := range apiResponse.Data[0] { + networkCostsPod, ok := networkCostsPodMap[pod] + if !ok { + networkCostsPodMap[pod] = &NetworkCostsAggregate{ + PromNetworkZoneGiB: 0.0, + AllocNetworkZoneGiB: allocationResponseItem.NetworkCrossZoneCost, + } + continue + } + networkCostsPod.AllocNetworkZoneGiB = allocationResponseItem.NetworkCrossZoneCost + } + + validCostsSeen := false + negligilbleCost := 0.01 // 1 Cent of a Dollar + for pod, networkCostValues := range networkCostsPodMap { + if networkCostValues.AllocNetworkZoneGiB < negligilbleCost { + continue + } else { + validCostsSeen = true + } + t.Logf("Pod %s", pod) + withinRange, diff_percent := utils.AreWithinPercentage(networkCostValues.AllocNetworkZoneGiB, networkCostValues.PromNetworkZoneGiB, tolerance) + if !withinRange { + t.Errorf(" - NetworkZoneCost[Fail]: DifferencePercent: %0.2f, Prometheus: %0.9f, /allocation: %0.9f", diff_percent, networkCostValues.PromNetworkZoneGiB, networkCostValues.AllocNetworkZoneGiB) + } else { + t.Logf(" - NetworkZoneCost[Pass]: ~ %0.5f", networkCostValues.PromNetworkZoneGiB) + } + } + if !validCostsSeen { + t.Errorf("NetWork Zone Costs for all Pods are below 1 cent and hence cannot be considered as costs from resource usage and validated.") + } + }) + } +} diff --git a/test/integration/prometheus/test.bats b/test/integration/prometheus/test.bats index ed7f941..75746a3 100644 --- a/test/integration/prometheus/test.bats +++ b/test/integration/prometheus/test.bats @@ -74,4 +74,12 @@ teardown() { go test ./test/integration/prometheus/network_internet_costs_test.go } +@test "prometheus: Network Zone Cost" { + go test ./test/integration/prometheus/network_zone_costs_test.go +} + +@test "prometheus: Network Region Cost" { + go test ./test/integration/prometheus/network_region_costs_test.go +} + # ------------------------------------------------------ \ No newline at end of file