From fe69e1fe43bf58224f6c8fb675c1490abb2c6820 Mon Sep 17 00:00:00 2001 From: Spencer Ugbo Date: Tue, 12 Aug 2025 09:35:50 +0100 Subject: [PATCH 1/5] add metric tests --- test/helpers/test_containers_utils.go | 187 ++++++++++++++++++ test/integration/metrics/metrics_test.go | 179 +++++++++++++++++ .../integration/utils/mock_collector_utils.go | 50 +++++ test/mock/collector/nginx-agent.conf | 17 +- 4 files changed, 429 insertions(+), 4 deletions(-) create mode 100644 test/integration/metrics/metrics_test.go create mode 100644 test/integration/utils/mock_collector_utils.go diff --git a/test/helpers/test_containers_utils.go b/test/helpers/test_containers_utils.go index baee47d40..9be20294a 100644 --- a/test/helpers/test_containers_utils.go +++ b/test/helpers/test_containers_utils.go @@ -25,6 +25,12 @@ type Parameters struct { LogMessage string } +type MockCollectorContainers struct { + AgentOSS testcontainers.Container + Otel testcontainers.Container + Prometheus testcontainers.Container +} + func StartContainer( ctx context.Context, tb testing.TB, @@ -296,6 +302,156 @@ func StartAuxiliaryMockManagementPlaneGrpcContainer(ctx context.Context, tb test return container } +func StartMockCollectorStack(ctx context.Context, tb testing.TB, + containerNetwork *testcontainers.DockerNetwork, agentConfig string, +) *MockCollectorContainers { + tb.Helper() + + packageName := Env(tb, "PACKAGE_NAME") + packageRepo := Env(tb, "PACKAGES_REPO") + baseImage := Env(tb, "BASE_IMAGE") + buildTarget := Env(tb, "BUILD_TARGET") + dockerfilePath := Env(tb, "DOCKERFILE_PATH") + + // agentPlus, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + // ContainerRequest: testcontainers.ContainerRequest{ + // FromDockerfile: testcontainers.FromDockerfile{ + // Context: "../../../", + // Dockerfile: "./test/docker/nginx-plus/deb/Dockerfile", + // KeepImage: false, + // PrintBuildLog: true, + // BuildArgs: map[string]*string{ + // "PACKAGE_NAME": ToPtr(packageName), + // "PACKAGES_REPO": ToPtr(packageRepo), + // "BASE_IMAGE": ToPtr(baseImage), + // "OS_RELEASE": ToPtr(osRelease), + // "OS_VERSION": ToPtr(osVersion), + // "ENTRY_POINT": ToPtr("./test/docker/entrypoint.sh"), + // "CONTAINER_NGINX_IMAGE_REGISTRY": ToPtr(containerRegistry), + // "IMAGE_PATH": ToPtr(imagePath), + // "TAG": ToPtr(tag), + // }, + // BuildOptionsModifier: func(buildOptions *types.ImageBuildOptions) { + // buildOptions.Target = buildTarget + // }, + // }, + // Name: "agent-with-nginx-plus", + // Networks: []string{containerNetwork.Name}, + // Files: []testcontainers.ContainerFile{ + // { + // HostFilePath: "../../mock/collector/nginx-agent.conf", + // ContainerFilePath: "/etc/nginx-agent/nginx-agent.conf", + // FileMode: configFilePermissions, + // }, + // { + // HostFilePath: "../../mock/collector/nginx-plus/nginx.conf", + // ContainerFilePath: "/etc/nginx/nginx.conf", + // FileMode: configFilePermissions, + // }, + // { + // HostFilePath: "../../mock/collector/nginx-plus/conf.d/default.conf", + // ContainerFilePath: "/etc/nginx/conf.d/default.conf", + // FileMode: configFilePermissions, + // }, + // }, + // }, + // Started: true, + // }) + // require.NoError(tb, err) + + agentOSS, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + FromDockerfile: testcontainers.FromDockerfile{ + Context: "../../../", + Dockerfile: dockerfilePath, + KeepImage: false, + PrintBuildLog: true, + BuildArgs: map[string]*string{ + "PACKAGE_NAME": ToPtr(packageName), + "PACKAGES_REPO": ToPtr(packageRepo), + "BASE_IMAGE": ToPtr(baseImage), + "ENTRY_POINT": ToPtr("./test/docker/entrypoint.sh"), + }, + BuildOptionsModifier: func(buildOptions *types.ImageBuildOptions) { + buildOptions.Target = buildTarget + }, + }, + Name: "agent-with-nginx-oss", + Networks: []string{containerNetwork.Name}, + Files: []testcontainers.ContainerFile{ + { + HostFilePath: agentConfig, + ContainerFilePath: "/etc/nginx-agent/nginx-agent.conf", + FileMode: configFilePermissions, + }, + { + HostFilePath: "../../mock/collector/nginx-oss/nginx.conf", + ContainerFilePath: "/etc/nginx/nginx.conf", + FileMode: configFilePermissions, + }, + { + HostFilePath: "../../mock/collector/nginx-oss/conf.d/default.conf", + ContainerFilePath: "/etc/nginx/conf.d/default.conf", + FileMode: configFilePermissions, + }, + }, + }, + Started: true, + }) + require.NoError(tb, err) + + otel, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + FromDockerfile: testcontainers.FromDockerfile{ + Context: "../../../", + Dockerfile: "./test/mock/collector/mock-collector/Dockerfile", + KeepImage: false, + PrintBuildLog: true, + }, + Name: "otel-collector", + ExposedPorts: []string{"4317/tcp", "9090/tcp", "9775/tcp"}, + Networks: []string{containerNetwork.Name}, + Files: []testcontainers.ContainerFile{ + { + HostFilePath: "../../mock/collector/otel-collector.yaml", + ContainerFilePath: "/etc/otel-collector.yaml", + FileMode: configFilePermissions, + }, + }, + WaitingFor: wait.ForLog("Everything is ready. Begin running and processing data."), + }, + Started: true, + }) + require.NoError(tb, err) + + prometheus, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: "prom/prometheus:latest", + Name: "prometheus", + ExposedPorts: []string{"9090/tcp"}, + Networks: []string{containerNetwork.Name}, + Files: []testcontainers.ContainerFile{ + { + HostFilePath: "../../mock/collector/prometheus.yaml", + ContainerFilePath: "/etc/prometheus/prometheus.yaml", + FileMode: configFilePermissions, + }, + }, + Cmd: []string{"--config.file=/etc/prometheus/prometheus.yml"}, + WaitingFor: wait.ForLog("Server is ready to receive web requests."), + }, + Started: true, + }) + require.NoError(tb, err) + + return &MockCollectorContainers{ + // AgentPlus: agentPlus, + AgentOSS: agentOSS, + Otel: otel, + Prometheus: prometheus, + } +} + func ToPtr[T any](value T) *T { return &value } @@ -359,3 +515,34 @@ func LogAndTerminateContainers( require.NoError(tb, err) } } + +func LogAndTerminateStack(ctx context.Context, tb testing.TB, + containers *MockCollectorContainers, +) { + tb.Helper() + + logAndTerminate := func(name string, container testcontainers.Container) { + if container == nil { + tb.Logf("Skipping log collection for %s: container is nil", name) + return + } + + tb.Logf("======================== Logging %s Container Logs ========================", name) + logReader, err := container.Logs(ctx) + require.NoError(tb, err) + + buf, err := io.ReadAll(logReader) + require.NoError(tb, err) + logs := string(buf) + + tb.Log(logs) + + err = container.Terminate(ctx) + require.NoError(tb, err) + } + + // logAndTerminate("agent-plus", containers.AgentPlus) + logAndTerminate("agent-oss", containers.AgentOSS) + logAndTerminate("otel", containers.Otel) + logAndTerminate("prometheus", containers.Prometheus) +} diff --git a/test/integration/metrics/metrics_test.go b/test/integration/metrics/metrics_test.go new file mode 100644 index 000000000..e8fa32b0b --- /dev/null +++ b/test/integration/metrics/metrics_test.go @@ -0,0 +1,179 @@ +package metrics + +import ( + "context" + "fmt" + "net/http" + "testing" + "time" + + "github.com/nginx/agent/v3/test/integration/utils" + + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + "github.com/stretchr/testify/suite" + "github.com/testcontainers/testcontainers-go" +) + +type MetricsTestSuite struct { + suite.Suite + ctx context.Context + teardownTest func(testing.TB) + metricFamilies map[string]*dto.MetricFamily +} + +func (s *MetricsTestSuite) SetupSuite() { + s.ctx = context.Background() + s.teardownTest = utils.SetupMetricsTest(s.T()) + time.Sleep(30 * time.Second) + s.metricFamilies = scrapeCollectorMetricFamilies(s.T(), s.ctx, utils.MockCollectorStack.Otel) +} + +func (s *MetricsTestSuite) TearDownSuite() { + s.teardownTest(s.T()) +} + +func (s *MetricsTestSuite) TestNginxOSS_Test1_TestRequestCount() { + family := s.metricFamilies["nginx_http_request_count"] + s.Require().NotNil(family, "nginx_http_requests_count metric family should not be nil") + + baselineMetric := sumMetricFamily(family) + s.T().Logf("nginx http requests count observed total: %v", baselineMetric) + + for i := 0; i < 5; i++ { + utils.MockCollectorStack.AgentOSS.Exec(s.ctx, []string{ + "curl", "-s", "http://127.0.0.1/", + }) + } + + time.Sleep(30 * time.Second) + + s.metricFamilies = scrapeCollectorMetricFamilies(s.T(), s.ctx, utils.MockCollectorStack.Otel) + + family = s.metricFamilies["nginx_http_request_count"] + s.Require().NotNil(family, "nginx_http_requests_count metric family should not be nil") + + got := sumMetricFamily(family) + + s.T().Logf("nginx http requests observed total: %f", got) + + // expected request total should be 'want' + the 5 curl requests above + 1 health check request + s.Require().GreaterOrEqual(got, baselineMetric+5, "nginx http requests count should increase by at least 5") +} + +func (s *MetricsTestSuite) TestNginxOSS_Test2_TestResponseCode() { + family := s.metricFamilies["nginx_http_response_count"] + s.T().Logf("nginx_http_response_count family: %v", family) + s.Require().NotNil(family, "nginx_http_response_count metric family should not be nil") + + for i := 1; i < 5; i++ { + code := fmt.Sprintf("%dxx", i) + s.T().Logf("nginx http response code %s total: %v", code, sumMetricFamilyLabel(family, "nginx_status_range", code)) + } + + s.Require().Greater(sumMetricFamily(family), 0.0, "expected some nginx http response codes") +} + +func (s *MetricsTestSuite) TestHostMetrics_Test1_TestSystemCPUUtilization() { + family := s.metricFamilies["system_cpu_utilization"] + s.T().Logf("system cpu utilization metric family: %v", family) + s.Require().NotNil(family, "system_cpu_utilization metric family should not be nil") + + cpuUtilization := sumMetricFamily(family) + + s.T().Logf("system cpu utilization: %v", cpuUtilization) + s.Require().Greater(cpuUtilization, 0.0, "expected some system cpu utilization") +} + +func (s *MetricsTestSuite) TestHostMetrics_Test2_TestSystemMemoryUsage() { + family := s.metricFamilies["system_memory_usage"] + s.T().Logf("system memory usage metric family: %v", family) + s.Require().NotNil(family, "system_memory_usage metric family should not be nil") + + memoryUsage := sumMetricFamily(family) + + s.T().Logf("system memory usage: %v", memoryUsage) + s.Require().Greater(memoryUsage, 0.0, "expected some system memory usage") +} + +func TestMetricsTestSuite(t *testing.T) { + suite.Run(t, new(MetricsTestSuite)) +} + +func scrapeCollectorMetricFamilies(t *testing.T, ctx context.Context, otelContainer testcontainers.Container) map[string]*dto.MetricFamily { + t.Helper() + + host, _ := otelContainer.Host(ctx) + port, _ := otelContainer.MappedPort(ctx, "9775") + + resp, err := http.Get(fmt.Sprintf("http://%s:%s/metrics", host, port.Port())) + if err != nil { + t.Fatalf("failed to get response from Otel Collector: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("Unexpected status code: %d", resp.StatusCode) + } + + parser := expfmt.TextParser{} + metricFamilies, err := parser.TextToMetricFamilies(resp.Body) + if err != nil { + t.Fatalf("failed to parse metrics: %v", err) + } + + return metricFamilies +} + +func sumMetricFamily(metricFamily *dto.MetricFamily) float64 { + var total float64 + for _, metric := range metricFamily.Metric { + if value, ok := metricValue(metricFamily, metric); ok { + total += value + } + } + return total +} + +func sumMetricFamilyLabel(metricFamily *dto.MetricFamily, key, val string) float64 { + var total float64 + for _, metric := range metricFamily.Metric { + labels := map[string]string{} + for _, labelPair := range metric.Label { + labels[labelPair.GetName()] = labelPair.GetValue() + } + if labels[key] != val { + continue + } + if value, ok := metricValue(metricFamily, metric); ok { + total += value + } + } + return total +} + +func metricValue(metricFamily *dto.MetricFamily, metric *dto.Metric) (float64, bool) { + switch metricFamily.GetType() { + case dto.MetricType_COUNTER: + if counter := metric.GetCounter(); counter != nil { + return counter.GetValue(), true + } + case dto.MetricType_GAUGE: + if gauge := metric.GetGauge(); gauge != nil { + return gauge.GetValue(), true + } + } + return 0, false +} + +//func matchLabels(labels map[string]string, filter string) bool { +// if filter == "" { +// return true +// } +// if i := strings.IndexByte(filter, '='); i >= 0 { +// key := filter[:i] +// val := filter[i+1:] +// return labels[key] == val +// } +// _, ok := labels[filter] +// return ok +//} diff --git a/test/integration/utils/mock_collector_utils.go b/test/integration/utils/mock_collector_utils.go new file mode 100644 index 000000000..53234a321 --- /dev/null +++ b/test/integration/utils/mock_collector_utils.go @@ -0,0 +1,50 @@ +package utils + +import ( + "context" + "github.com/testcontainers/testcontainers-go" + "os" + "testing" + + "github.com/nginx/agent/v3/test/helpers" +) + +var MockCollectorStack *helpers.MockCollectorContainers + +// SetupMetricsTest similar to SetupConnectionTest() +func SetupMetricsTest(tb testing.TB) func(testing.TB) { + tb.Helper() + ctx := context.Background() + + if os.Getenv("TEST_ENV") == "Container" { + setupStackEnvironment(ctx, tb) + } + return func(tb testing.TB) { + tb.Helper() + + if os.Getenv("TEST_ENV") == "Container" { + helpers.LogAndTerminateStack( + ctx, + tb, + MockCollectorStack, + ) + } + } +} + +func setupStackEnvironment(ctx context.Context, tb testing.TB) { + tb.Helper() + tb.Log("Running tests in a container environment") + + containerNetwork := createContainerNetwork(ctx, tb) + setupMockCollectorStack(ctx, tb, containerNetwork) +} + +func setupMockCollectorStack(ctx context.Context, tb testing.TB, containerNetwork *testcontainers.DockerNetwork) { + tb.Helper() + + tb.Log("Starting mock collector stack") + + agentConfig := "../../mock/collector/nginx-agent.conf" + MockCollectorStack = helpers.StartMockCollectorStack(ctx, tb, containerNetwork, agentConfig) +} diff --git a/test/mock/collector/nginx-agent.conf b/test/mock/collector/nginx-agent.conf index dd870c390..3ef1c5e6c 100644 --- a/test/mock/collector/nginx-agent.conf +++ b/test/mock/collector/nginx-agent.conf @@ -49,7 +49,8 @@ collector: network: {} filesystem: {} otlp: - - server: + "default": + server: host: "127.0.0.1" port: 4317 auth: @@ -61,11 +62,13 @@ collector: key: /tmp/key.pem generate_self_signed_cert: true processors: - batch: {} + batch: + "default": {} exporters: - debug: + debug: {} otlp: - - server: + "default": + server: host: "otel-collector" port: 4317 authenticator: headers_setter @@ -79,3 +82,9 @@ collector: - action: insert key: "authorization" value: "fake-authorization" + pipelines: + metrics: + "default": + receivers: ["otlp/default", "host_metrics", "nginx_metrics"] + processors: ["batch/default"] + exporters: ["otlp/default", "debug"] From 01bca5bed98b3635f307150d523398e820514662 Mon Sep 17 00:00:00 2001 From: Spencer Ugbo Date: Tue, 12 Aug 2025 09:42:02 +0100 Subject: [PATCH 2/5] update otel collector readme --- test/mock/collector/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/mock/collector/README.md b/test/mock/collector/README.md index 938fab5f7..c06cfdd43 100644 --- a/test/mock/collector/README.md +++ b/test/mock/collector/README.md @@ -24,6 +24,11 @@ To start run everything run the following make run-mock-management-otel-collector ``` +To start everything except the NGINX Plus & NGINX App Protect run the following +``` +make run-mock-otel-collector-without-nap +``` + Once everything is started there should be 7 containers running ``` CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES @@ -43,3 +48,8 @@ To stop everything run the following ``` make stop-mock-management-otel-collector ``` + +Or run the following if you started everything except the NGINX Plus & NGINX App Protect +``` +make stop-mock-otel-collector-without-nap +``` From 8823d6a48cc1039e44852b487e8ea64f7c73d5fd Mon Sep 17 00:00:00 2001 From: Spencer Ugbo Date: Tue, 12 Aug 2025 15:49:12 +0100 Subject: [PATCH 3/5] fix small issue with test --- test/helpers/test_containers_utils.go | 100 ++++++----- test/integration/metrics/metrics_test.go | 169 ++++++------------ .../integration/utils/mock_collector_utils.go | 144 ++++++++++++++- 3 files changed, 244 insertions(+), 169 deletions(-) diff --git a/test/helpers/test_containers_utils.go b/test/helpers/test_containers_utils.go index 9be20294a..bf8ae6890 100644 --- a/test/helpers/test_containers_utils.go +++ b/test/helpers/test_containers_utils.go @@ -26,6 +26,7 @@ type Parameters struct { } type MockCollectorContainers struct { + // AgentPlus testcontainers.Container AgentOSS testcontainers.Container Otel testcontainers.Container Prometheus testcontainers.Container @@ -310,52 +311,57 @@ func StartMockCollectorStack(ctx context.Context, tb testing.TB, packageName := Env(tb, "PACKAGE_NAME") packageRepo := Env(tb, "PACKAGES_REPO") baseImage := Env(tb, "BASE_IMAGE") + // osRelease := Env(tb, "OS_RELEASE") + // osVersion := Env(tb, "OS_VERSION") buildTarget := Env(tb, "BUILD_TARGET") dockerfilePath := Env(tb, "DOCKERFILE_PATH") + // containerRegistry := Env(tb, "CONTAINER_NGINX_IMAGE_REGISTRY") + // tag := Env(tb, "TAG") + // imagePath := Env(tb, "IMAGE_PATH") // agentPlus, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - // ContainerRequest: testcontainers.ContainerRequest{ - // FromDockerfile: testcontainers.FromDockerfile{ - // Context: "../../../", - // Dockerfile: "./test/docker/nginx-plus/deb/Dockerfile", - // KeepImage: false, - // PrintBuildLog: true, - // BuildArgs: map[string]*string{ - // "PACKAGE_NAME": ToPtr(packageName), - // "PACKAGES_REPO": ToPtr(packageRepo), - // "BASE_IMAGE": ToPtr(baseImage), - // "OS_RELEASE": ToPtr(osRelease), - // "OS_VERSION": ToPtr(osVersion), - // "ENTRY_POINT": ToPtr("./test/docker/entrypoint.sh"), - // "CONTAINER_NGINX_IMAGE_REGISTRY": ToPtr(containerRegistry), - // "IMAGE_PATH": ToPtr(imagePath), - // "TAG": ToPtr(tag), - // }, - // BuildOptionsModifier: func(buildOptions *types.ImageBuildOptions) { - // buildOptions.Target = buildTarget - // }, - // }, - // Name: "agent-with-nginx-plus", - // Networks: []string{containerNetwork.Name}, - // Files: []testcontainers.ContainerFile{ - // { - // HostFilePath: "../../mock/collector/nginx-agent.conf", - // ContainerFilePath: "/etc/nginx-agent/nginx-agent.conf", - // FileMode: configFilePermissions, - // }, - // { - // HostFilePath: "../../mock/collector/nginx-plus/nginx.conf", - // ContainerFilePath: "/etc/nginx/nginx.conf", - // FileMode: configFilePermissions, - // }, - // { - // HostFilePath: "../../mock/collector/nginx-plus/conf.d/default.conf", - // ContainerFilePath: "/etc/nginx/conf.d/default.conf", - // FileMode: configFilePermissions, - // }, - // }, - // }, - // Started: true, + // ContainerRequest: testcontainers.ContainerRequest{ + // FromDockerfile: testcontainers.FromDockerfile{ + // Context: "../../../", + // Dockerfile: "./test/docker/nginx-plus/deb/Dockerfile", + // KeepImage: false, + // PrintBuildLog: true, + // BuildArgs: map[string]*string{ + // "PACKAGE_NAME": ToPtr(packageName), + // "PACKAGES_REPO": ToPtr(packageRepo), + // "BASE_IMAGE": ToPtr(baseImage), + // "OS_RELEASE": ToPtr(osRelease), + // "OS_VERSION": ToPtr(osVersion), + // "ENTRY_POINT": ToPtr("./test/docker/entrypoint.sh"), + // "CONTAINER_NGINX_IMAGE_REGISTRY": ToPtr(containerRegistry), + // "IMAGE_PATH": ToPtr(imagePath), + // "TAG": ToPtr(tag), + // }, + // BuildOptionsModifier: func(buildOptions *types.ImageBuildOptions) { + // buildOptions.Target = "install-nginx" + // }, + // }, + // Name: "agent-with-nginx-plus", + // Networks: []string{containerNetwork.Name}, + // Files: []testcontainers.ContainerFile{ + // { + // HostFilePath: agentConfig, + // ContainerFilePath: "/etc/nginx-agent/nginx-agent.conf", + // FileMode: configFilePermissions, + // }, + // { + // HostFilePath: "../../mock/collector/nginx-plus/nginx.conf", + // ContainerFilePath: "/etc/nginx/nginx.conf", + // FileMode: configFilePermissions, + // }, + // { + // HostFilePath: "../../mock/collector/nginx-plus/conf.d/default.conf", + // ContainerFilePath: "/etc/nginx/conf.d/default.conf", + // FileMode: configFilePermissions, + // }, + // }, + // }, + // Started: true, // }) // require.NoError(tb, err) @@ -367,10 +373,10 @@ func StartMockCollectorStack(ctx context.Context, tb testing.TB, KeepImage: false, PrintBuildLog: true, BuildArgs: map[string]*string{ - "PACKAGE_NAME": ToPtr(packageName), - "PACKAGES_REPO": ToPtr(packageRepo), - "BASE_IMAGE": ToPtr(baseImage), - "ENTRY_POINT": ToPtr("./test/docker/entrypoint.sh"), + "PACKAGE_NAME": ToPtr(packageName), + "PACKAGES_REPO": ToPtr(packageRepo), + "BASE_IMAGE": ToPtr(baseImage), + "ENTRY_POINT": ToPtr("./test/docker/entrypoint.sh"), }, BuildOptionsModifier: func(buildOptions *types.ImageBuildOptions) { buildOptions.Target = buildTarget @@ -443,7 +449,7 @@ func StartMockCollectorStack(ctx context.Context, tb testing.TB, Started: true, }) require.NoError(tb, err) - + return &MockCollectorContainers{ // AgentPlus: agentPlus, AgentOSS: agentOSS, diff --git a/test/integration/metrics/metrics_test.go b/test/integration/metrics/metrics_test.go index e8fa32b0b..16136e6be 100644 --- a/test/integration/metrics/metrics_test.go +++ b/test/integration/metrics/metrics_test.go @@ -1,18 +1,19 @@ +// Copyright (c) F5, Inc. +// +// This source code is licensed under the Apache License, Version 2.0 license found in the +// LICENSE file in the root directory of this source tree. + package metrics import ( "context" - "fmt" - "net/http" "testing" "time" "github.com/nginx/agent/v3/test/integration/utils" dto "github.com/prometheus/client_model/go" - "github.com/prometheus/common/expfmt" "github.com/stretchr/testify/suite" - "github.com/testcontainers/testcontainers-go" ) type MetricsTestSuite struct { @@ -26,7 +27,7 @@ func (s *MetricsTestSuite) SetupSuite() { s.ctx = context.Background() s.teardownTest = utils.SetupMetricsTest(s.T()) time.Sleep(30 * time.Second) - s.metricFamilies = scrapeCollectorMetricFamilies(s.T(), s.ctx, utils.MockCollectorStack.Otel) + s.metricFamilies = utils.ScrapeCollectorMetricFamilies(s.T(), s.ctx, utils.MockCollectorStack.Otel) } func (s *MetricsTestSuite) TearDownSuite() { @@ -35,145 +36,77 @@ func (s *MetricsTestSuite) TearDownSuite() { func (s *MetricsTestSuite) TestNginxOSS_Test1_TestRequestCount() { family := s.metricFamilies["nginx_http_request_count"] - s.Require().NotNil(family, "nginx_http_requests_count metric family should not be nil") - - baselineMetric := sumMetricFamily(family) - s.T().Logf("nginx http requests count observed total: %v", baselineMetric) - - for i := 0; i < 5; i++ { - utils.MockCollectorStack.AgentOSS.Exec(s.ctx, []string{ - "curl", "-s", "http://127.0.0.1/", - }) + s.T().Logf("nginx_http_request_count metric family: %v", family) + s.Require().NotNil(family) + + baselineMetric := utils.SumMetricFamily(family) + s.T().Logf("NGINX HTTP request count total: %v", baselineMetric) + + requestCount := 5 + for range requestCount { + url := "http://127.0.0.1/" + _, _, err := utils.MockCollectorStack.AgentOSS.Exec( + s.ctx, + []string{"curl", "-s", url}, + ) + s.Require().NoError(err) } - time.Sleep(30 * time.Second) - - s.metricFamilies = scrapeCollectorMetricFamilies(s.T(), s.ctx, utils.MockCollectorStack.Otel) + time.Sleep(65 * time.Second) + s.metricFamilies = utils.ScrapeCollectorMetricFamilies(s.T(), s.ctx, utils.MockCollectorStack.Otel) family = s.metricFamilies["nginx_http_request_count"] - s.Require().NotNil(family, "nginx_http_requests_count metric family should not be nil") - - got := sumMetricFamily(family) + s.T().Logf("nginx_http_request_count metric family: %v", family) + s.Require().NotNil(family) - s.T().Logf("nginx http requests observed total: %f", got) + got := utils.SumMetricFamily(family) - // expected request total should be 'want' + the 5 curl requests above + 1 health check request - s.Require().GreaterOrEqual(got, baselineMetric+5, "nginx http requests count should increase by at least 5") + s.T().Logf("NGINX HTTP request count total: %v", got) + s.Require().GreaterOrEqual(got, baselineMetric+float64(requestCount)) } func (s *MetricsTestSuite) TestNginxOSS_Test2_TestResponseCode() { family := s.metricFamilies["nginx_http_response_count"] s.T().Logf("nginx_http_response_count family: %v", family) - s.Require().NotNil(family, "nginx_http_response_count metric family should not be nil") - - for i := 1; i < 5; i++ { - code := fmt.Sprintf("%dxx", i) - s.T().Logf("nginx http response code %s total: %v", code, sumMetricFamilyLabel(family, "nginx_status_range", code)) + s.Require().NotNil(family) + + responseCodes := []string{"1xx", "2xx", "3xx", "4xx"} + codeRes := make([]float64, 0, len(responseCodes)) + for code := range responseCodes { + codeRes = append(codeRes, utils.SumMetricFamilyLabel(family, "nginx_status_range", responseCodes[code])) + s.T().Logf("NGINX HTTP response code %s total: %v", responseCodes[code], codeRes[code]) + s.Require().NotNil(codeRes[code]) } - - s.Require().Greater(sumMetricFamily(family), 0.0, "expected some nginx http response codes") } func (s *MetricsTestSuite) TestHostMetrics_Test1_TestSystemCPUUtilization() { family := s.metricFamilies["system_cpu_utilization"] - s.T().Logf("system cpu utilization metric family: %v", family) - s.Require().NotNil(family, "system_cpu_utilization metric family should not be nil") + s.T().Logf("system_cpu_utilization metric family: %v", family) + s.Require().NotNil(family) - cpuUtilization := sumMetricFamily(family) + cpuUtilizationSystem := utils.SumMetricFamilyLabel(family, "state", "system") + cpuUtilizationUser := utils.SumMetricFamilyLabel(family, "state", "user") - s.T().Logf("system cpu utilization: %v", cpuUtilization) - s.Require().Greater(cpuUtilization, 0.0, "expected some system cpu utilization") + s.T().Logf("System cpu utilization: %v", cpuUtilizationSystem) + s.T().Logf("System cpu utilization: %v", cpuUtilizationUser) + s.Require().NotNil(cpuUtilizationSystem) + s.Require().NotNil(cpuUtilizationUser) } func (s *MetricsTestSuite) TestHostMetrics_Test2_TestSystemMemoryUsage() { family := s.metricFamilies["system_memory_usage"] - s.T().Logf("system memory usage metric family: %v", family) - s.Require().NotNil(family, "system_memory_usage metric family should not be nil") + s.T().Logf("system_memory_usage metric family: %v", family) + s.Require().NotNil(family) - memoryUsage := sumMetricFamily(family) + memoryUsageFree := utils.SumMetricFamilyLabel(family, "state", "free") + memoryUsageUsed := utils.SumMetricFamilyLabel(family, "state", "used") - s.T().Logf("system memory usage: %v", memoryUsage) - s.Require().Greater(memoryUsage, 0.0, "expected some system memory usage") + s.T().Logf("System memory usage: %v", memoryUsageFree) + s.T().Logf("System memory usage: %v", memoryUsageUsed) + s.Require().NotNil(memoryUsageFree) + s.Require().NotNil(memoryUsageUsed) } func TestMetricsTestSuite(t *testing.T) { suite.Run(t, new(MetricsTestSuite)) } - -func scrapeCollectorMetricFamilies(t *testing.T, ctx context.Context, otelContainer testcontainers.Container) map[string]*dto.MetricFamily { - t.Helper() - - host, _ := otelContainer.Host(ctx) - port, _ := otelContainer.MappedPort(ctx, "9775") - - resp, err := http.Get(fmt.Sprintf("http://%s:%s/metrics", host, port.Port())) - if err != nil { - t.Fatalf("failed to get response from Otel Collector: %v", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("Unexpected status code: %d", resp.StatusCode) - } - - parser := expfmt.TextParser{} - metricFamilies, err := parser.TextToMetricFamilies(resp.Body) - if err != nil { - t.Fatalf("failed to parse metrics: %v", err) - } - - return metricFamilies -} - -func sumMetricFamily(metricFamily *dto.MetricFamily) float64 { - var total float64 - for _, metric := range metricFamily.Metric { - if value, ok := metricValue(metricFamily, metric); ok { - total += value - } - } - return total -} - -func sumMetricFamilyLabel(metricFamily *dto.MetricFamily, key, val string) float64 { - var total float64 - for _, metric := range metricFamily.Metric { - labels := map[string]string{} - for _, labelPair := range metric.Label { - labels[labelPair.GetName()] = labelPair.GetValue() - } - if labels[key] != val { - continue - } - if value, ok := metricValue(metricFamily, metric); ok { - total += value - } - } - return total -} - -func metricValue(metricFamily *dto.MetricFamily, metric *dto.Metric) (float64, bool) { - switch metricFamily.GetType() { - case dto.MetricType_COUNTER: - if counter := metric.GetCounter(); counter != nil { - return counter.GetValue(), true - } - case dto.MetricType_GAUGE: - if gauge := metric.GetGauge(); gauge != nil { - return gauge.GetValue(), true - } - } - return 0, false -} - -//func matchLabels(labels map[string]string, filter string) bool { -// if filter == "" { -// return true -// } -// if i := strings.IndexByte(filter, '='); i >= 0 { -// key := filter[:i] -// val := filter[i+1:] -// return labels[key] == val -// } -// _, ok := labels[filter] -// return ok -//} diff --git a/test/integration/utils/mock_collector_utils.go b/test/integration/utils/mock_collector_utils.go index 53234a321..9e1694d11 100644 --- a/test/integration/utils/mock_collector_utils.go +++ b/test/integration/utils/mock_collector_utils.go @@ -1,28 +1,44 @@ +// Copyright (c) F5, Inc. +// +// This source code is licensed under the Apache License, Version 2.0 license found in the +// LICENSE file in the root directory of this source tree. + package utils import ( + "bytes" "context" - "github.com/testcontainers/testcontainers-go" + "fmt" + "net" + "net/http" "os" "testing" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + + "github.com/go-resty/resty/v2" + "github.com/testcontainers/testcontainers-go" + "github.com/nginx/agent/v3/test/helpers" ) var MockCollectorStack *helpers.MockCollectorContainers -// SetupMetricsTest similar to SetupConnectionTest() +const envContainer = "Container" + func SetupMetricsTest(tb testing.TB) func(testing.TB) { tb.Helper() ctx := context.Background() - if os.Getenv("TEST_ENV") == "Container" { + if os.Getenv("TEST_ENV") == envContainer { setupStackEnvironment(ctx, tb) } + return func(tb testing.TB) { tb.Helper() - if os.Getenv("TEST_ENV") == "Container" { + if os.Getenv("TEST_ENV") == envContainer { helpers.LogAndTerminateStack( ctx, tb, @@ -48,3 +64,123 @@ func setupMockCollectorStack(ctx context.Context, tb testing.TB, containerNetwor agentConfig := "../../mock/collector/nginx-agent.conf" MockCollectorStack = helpers.StartMockCollectorStack(ctx, tb, containerNetwork, agentConfig) } + +func ScrapeCollectorMetricFamilies(t *testing.T, ctx context.Context, + otelContainer testcontainers.Container, +) map[string]*dto.MetricFamily { + t.Helper() + + host, _ := otelContainer.Host(ctx) + port, _ := otelContainer.MappedPort(ctx, "9775") + + address := net.JoinHostPort(host, port.Port()) + url := fmt.Sprintf("http://%s/metrics", address) + + client := resty.New() + resp, err := client.R().EnableTrace().Get(url) + if err != nil { + t.Fatalf("failed to get response from Otel Collector: %v", err) + } + if resp.StatusCode() != http.StatusOK { + t.Fatalf("Unexpected status code: %d", resp.StatusCode()) + } + + parser := expfmt.TextParser{} + metricFamilies, err := parser.TextToMetricFamilies(bytes.NewReader(resp.Body())) + if err != nil { + t.Fatalf("failed to parse metrics: %v", err) + } + + return metricFamilies +} + +func SumMetricFamily(metricFamily *dto.MetricFamily) float64 { + var total float64 + for _, metric := range metricFamily.GetMetric() { + if value := metricValue(metricFamily, metric); value != nil { + total += *value + } + } + + return total +} + +func SumMetricFamilyLabel(metricFamily *dto.MetricFamily, key, val string) float64 { + var total float64 + for _, metric := range metricFamily.GetMetric() { + labels := make(map[string]string) + for _, labelPair := range metric.GetLabel() { + labels[labelPair.GetName()] = labelPair.GetValue() + } + if labels[key] != val { + continue + } + if value := metricValue(metricFamily, metric); value != nil { + total += *value + } + } + + return total +} + +func metricValue(metricFamily *dto.MetricFamily, metric *dto.Metric) *float64 { + switch metricFamily.GetType() { + case dto.MetricType_COUNTER: + return getCounterValue(metric) + case dto.MetricType_GAUGE: + return getGaugeValue(metric) + case dto.MetricType_SUMMARY: + return getSummaryValue(metric) + case dto.MetricType_UNTYPED: + return getUntypedValue(metric) + case dto.MetricType_HISTOGRAM, dto.MetricType_GAUGE_HISTOGRAM: + return getHistogramValue(metric) + } + + return nil +} + +func getCounterValue(metric *dto.Metric) *float64 { + if counter := metric.GetCounter(); counter != nil { + val := counter.GetValue() + return &val + } + + return nil +} + +func getGaugeValue(metric *dto.Metric) *float64 { + if gauge := metric.GetGauge(); gauge != nil { + val := gauge.GetValue() + return &val + } + + return nil +} + +func getSummaryValue(metric *dto.Metric) *float64 { + if summary := metric.GetSummary(); summary != nil { + val := summary.GetSampleSum() + return &val + } + + return nil +} + +func getUntypedValue(metric *dto.Metric) *float64 { + if untyped := metric.GetUntyped(); untyped != nil { + val := untyped.GetValue() + return &val + } + + return nil +} + +func getHistogramValue(metric *dto.Metric) *float64 { + if histogram := metric.GetHistogram(); histogram != nil { + val := histogram.GetSampleSum() + return &val + } + + return nil +} From 6f0ee808b8034b124b9f4a3863200953482bdd0a Mon Sep 17 00:00:00 2001 From: Spencer Ugbo Date: Tue, 12 Aug 2025 17:20:18 +0100 Subject: [PATCH 4/5] update depedencies versions --- api/grpc/mpi/v1/command.pb.go | 2 +- api/grpc/mpi/v1/common.pb.go | 2 +- api/grpc/mpi/v1/files.pb.go | 2 +- go.mod | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/grpc/mpi/v1/command.pb.go b/api/grpc/mpi/v1/command.pb.go index 7c137c5e9..1c4fa9b0c 100644 --- a/api/grpc/mpi/v1/command.pb.go +++ b/api/grpc/mpi/v1/command.pb.go @@ -8,7 +8,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.6 +// protoc-gen-go v1.36.7 // protoc (unknown) // source: mpi/v1/command.proto diff --git a/api/grpc/mpi/v1/common.pb.go b/api/grpc/mpi/v1/common.pb.go index 9ba42f536..26b95bf14 100644 --- a/api/grpc/mpi/v1/common.pb.go +++ b/api/grpc/mpi/v1/common.pb.go @@ -5,7 +5,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.6 +// protoc-gen-go v1.36.7 // protoc (unknown) // source: mpi/v1/common.proto diff --git a/api/grpc/mpi/v1/files.pb.go b/api/grpc/mpi/v1/files.pb.go index 9ebb60f91..c68585426 100644 --- a/api/grpc/mpi/v1/files.pb.go +++ b/api/grpc/mpi/v1/files.pb.go @@ -5,7 +5,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.6 +// protoc-gen-go v1.36.7 // protoc (unknown) // source: mpi/v1/files.proto diff --git a/go.mod b/go.mod index 1c1886634..3111055dc 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,8 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/tcplogreceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/testbed v0.124.1 + github.com/prometheus/client_model v0.6.1 + github.com/prometheus/common v0.62.0 github.com/shirou/gopsutil/v4 v4.25.3 github.com/spf13/pflag v1.0.6 github.com/stretchr/testify v1.10.0 @@ -202,8 +204,6 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.16.0 // indirect github.com/rs/cors v1.11.1 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect From ee3ce964f72f58e541c967e6b50a66af426479e6 Mon Sep 17 00:00:00 2001 From: Spencer Ugbo Date: Fri, 22 Aug 2025 16:07:21 +0100 Subject: [PATCH 5/5] added metric generation and polling --- Makefile | 4 +- test/helpers/test_containers_utils.go | 108 +++++++-------- test/integration/metrics/metrics_test.go | 131 +++++++++++------- .../integration/utils/mock_collector_utils.go | 116 +++++++++++++++- test/mock/collector/nginx-oss/nginx.conf | 10 +- 5 files changed, 256 insertions(+), 113 deletions(-) diff --git a/Makefile b/Makefile index 19467116d..dff4e1627 100644 --- a/Makefile +++ b/Makefile @@ -159,11 +159,11 @@ build-mock-management-otel-collector: mkdir -p $(BUILD_DIR)/mock-management-otel-collector @CGO_ENABLED=0 GOARCH=$(OSARCH) GOOS=linux $(GOBUILD) -o $(BUILD_DIR)/mock-management-otel-collector/collector test/mock/collector/mock-collector/main.go -integration-test: $(SELECTED_PACKAGE) build-mock-management-plane-grpc +integration-test: $(SELECTED_PACKAGE) build-mock-management-plane-grpc build-mock-management-otel-collector TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-local" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \ PACKAGES_REPO=$(OSS_PACKAGES_REPO) PACKAGE_NAME=$(PACKAGE_NAME) BASE_IMAGE=$(BASE_IMAGE) DOCKERFILE_PATH=$(DOCKERFILE_PATH) IMAGE_PATH=$(IMAGE_PATH) TAG=${IMAGE_TAG} \ OS_VERSION=$(OS_VERSION) OS_RELEASE=$(OS_RELEASE) \ - go test -v ./test/integration/installuninstall ./test/integration/managementplane ./test/integration/auxiliarycommandserver ./test/integration/nginxless + go test -v ./test/integration/installuninstall ./test/integration/managementplane ./test/integration/auxiliarycommandserver ./test/integration/nginxless ./test/integration/metrics official-image-integration-test: $(SELECTED_PACKAGE) build-mock-management-plane-grpc TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} BUILD_TARGET="install" \ diff --git a/test/helpers/test_containers_utils.go b/test/helpers/test_containers_utils.go index bf8ae6890..2b64773e5 100644 --- a/test/helpers/test_containers_utils.go +++ b/test/helpers/test_containers_utils.go @@ -26,7 +26,7 @@ type Parameters struct { } type MockCollectorContainers struct { - // AgentPlus testcontainers.Container + AgentPlus testcontainers.Container AgentOSS testcontainers.Container Otel testcontainers.Container Prometheus testcontainers.Container @@ -311,59 +311,59 @@ func StartMockCollectorStack(ctx context.Context, tb testing.TB, packageName := Env(tb, "PACKAGE_NAME") packageRepo := Env(tb, "PACKAGES_REPO") baseImage := Env(tb, "BASE_IMAGE") - // osRelease := Env(tb, "OS_RELEASE") - // osVersion := Env(tb, "OS_VERSION") + osRelease := Env(tb, "OS_RELEASE") + osVersion := Env(tb, "OS_VERSION") buildTarget := Env(tb, "BUILD_TARGET") dockerfilePath := Env(tb, "DOCKERFILE_PATH") - // containerRegistry := Env(tb, "CONTAINER_NGINX_IMAGE_REGISTRY") - // tag := Env(tb, "TAG") - // imagePath := Env(tb, "IMAGE_PATH") - - // agentPlus, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - // ContainerRequest: testcontainers.ContainerRequest{ - // FromDockerfile: testcontainers.FromDockerfile{ - // Context: "../../../", - // Dockerfile: "./test/docker/nginx-plus/deb/Dockerfile", - // KeepImage: false, - // PrintBuildLog: true, - // BuildArgs: map[string]*string{ - // "PACKAGE_NAME": ToPtr(packageName), - // "PACKAGES_REPO": ToPtr(packageRepo), - // "BASE_IMAGE": ToPtr(baseImage), - // "OS_RELEASE": ToPtr(osRelease), - // "OS_VERSION": ToPtr(osVersion), - // "ENTRY_POINT": ToPtr("./test/docker/entrypoint.sh"), - // "CONTAINER_NGINX_IMAGE_REGISTRY": ToPtr(containerRegistry), - // "IMAGE_PATH": ToPtr(imagePath), - // "TAG": ToPtr(tag), - // }, - // BuildOptionsModifier: func(buildOptions *types.ImageBuildOptions) { - // buildOptions.Target = "install-nginx" - // }, - // }, - // Name: "agent-with-nginx-plus", - // Networks: []string{containerNetwork.Name}, - // Files: []testcontainers.ContainerFile{ - // { - // HostFilePath: agentConfig, - // ContainerFilePath: "/etc/nginx-agent/nginx-agent.conf", - // FileMode: configFilePermissions, - // }, - // { - // HostFilePath: "../../mock/collector/nginx-plus/nginx.conf", - // ContainerFilePath: "/etc/nginx/nginx.conf", - // FileMode: configFilePermissions, - // }, - // { - // HostFilePath: "../../mock/collector/nginx-plus/conf.d/default.conf", - // ContainerFilePath: "/etc/nginx/conf.d/default.conf", - // FileMode: configFilePermissions, - // }, - // }, - // }, - // Started: true, - // }) - // require.NoError(tb, err) + containerRegistry := Env(tb, "CONTAINER_NGINX_IMAGE_REGISTRY") + tag := Env(tb, "TAG") + imagePath := Env(tb, "IMAGE_PATH") + + agentPlus, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + FromDockerfile: testcontainers.FromDockerfile{ + Context: "../../../", + Dockerfile: "./test/docker/nginx-plus/deb/Dockerfile", + KeepImage: false, + PrintBuildLog: true, + BuildArgs: map[string]*string{ + "PACKAGE_NAME": ToPtr(packageName), + "PACKAGES_REPO": ToPtr(packageRepo), + "BASE_IMAGE": ToPtr(baseImage), + "OS_RELEASE": ToPtr(osRelease), + "OS_VERSION": ToPtr(osVersion), + "ENTRY_POINT": ToPtr("./test/docker/entrypoint.sh"), + "CONTAINER_NGINX_IMAGE_REGISTRY": ToPtr(containerRegistry), + "IMAGE_PATH": ToPtr(imagePath), + "TAG": ToPtr(tag), + }, + BuildOptionsModifier: func(buildOptions *types.ImageBuildOptions) { + buildOptions.Target = "install-nginx" + }, + }, + Name: "agent-with-nginx-plus", + Networks: []string{containerNetwork.Name}, + Files: []testcontainers.ContainerFile{ + { + HostFilePath: agentConfig, + ContainerFilePath: "/etc/nginx-agent/nginx-agent.conf", + FileMode: configFilePermissions, + }, + { + HostFilePath: "../../mock/collector/nginx-plus/nginx.conf", + ContainerFilePath: "/etc/nginx/nginx.conf", + FileMode: configFilePermissions, + }, + { + HostFilePath: "../../mock/collector/nginx-plus/conf.d/default.conf", + ContainerFilePath: "/etc/nginx/conf.d/default.conf", + FileMode: configFilePermissions, + }, + }, + }, + Started: true, + }) + require.NoError(tb, err) agentOSS, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: testcontainers.ContainerRequest{ @@ -451,7 +451,7 @@ func StartMockCollectorStack(ctx context.Context, tb testing.TB, require.NoError(tb, err) return &MockCollectorContainers{ - // AgentPlus: agentPlus, + AgentPlus: agentPlus, AgentOSS: agentOSS, Otel: otel, Prometheus: prometheus, @@ -547,7 +547,7 @@ func LogAndTerminateStack(ctx context.Context, tb testing.TB, require.NoError(tb, err) } - // logAndTerminate("agent-plus", containers.AgentPlus) + logAndTerminate("agent-plus", containers.AgentPlus) logAndTerminate("agent-oss", containers.AgentOSS) logAndTerminate("otel", containers.Otel) logAndTerminate("prometheus", containers.Prometheus) diff --git a/test/integration/metrics/metrics_test.go b/test/integration/metrics/metrics_test.go index 16136e6be..ebb86423d 100644 --- a/test/integration/metrics/metrics_test.go +++ b/test/integration/metrics/metrics_test.go @@ -7,13 +7,10 @@ package metrics import ( "context" - "testing" - "time" - "github.com/nginx/agent/v3/test/integration/utils" - dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/suite" + "testing" ) type MetricsTestSuite struct { @@ -26,85 +23,117 @@ type MetricsTestSuite struct { func (s *MetricsTestSuite) SetupSuite() { s.ctx = context.Background() s.teardownTest = utils.SetupMetricsTest(s.T()) - time.Sleep(30 * time.Second) + utils.WaitUntilNextScrapeCycle(s.T(), s.ctx) +} + +func (s *MetricsTestSuite) SetupTest() { s.metricFamilies = utils.ScrapeCollectorMetricFamilies(s.T(), s.ctx, utils.MockCollectorStack.Otel) } +func (s *MetricsTestSuite) TearDownTest() { + utils.WaitUntilNextScrapeCycle(s.T(), s.ctx) +} + func (s *MetricsTestSuite) TearDownSuite() { s.teardownTest(s.T()) } -func (s *MetricsTestSuite) TestNginxOSS_Test1_TestRequestCount() { - family := s.metricFamilies["nginx_http_request_count"] - s.T().Logf("nginx_http_request_count metric family: %v", family) +// Check that the NGINX OSS request count metric increases after generating some requests +func (s *MetricsTestSuite) TestNginxOSS_TestRequestCount() { + metricName := "nginx_http_request_count" + family := s.metricFamilies[metricName] + s.T().Logf("%s metric family: %v", metricName, family) s.Require().NotNil(family) - baselineMetric := utils.SumMetricFamily(family) - s.T().Logf("NGINX HTTP request count total: %v", baselineMetric) - - requestCount := 5 - for range requestCount { - url := "http://127.0.0.1/" - _, _, err := utils.MockCollectorStack.AgentOSS.Exec( - s.ctx, - []string{"curl", "-s", url}, - ) - s.Require().NoError(err) - } + var baselineMetric []float64 + baselineMetric = append(baselineMetric, utils.SumMetricFamily(family)) + s.T().Logf("NGINX HTTP request count total: %v", baselineMetric[0]) - time.Sleep(65 * time.Second) + requestCount := 10 + utils.GenerateMetrics(s.ctx, s.T(), utils.MockCollectorStack.AgentOSS, requestCount, "2xx") - s.metricFamilies = utils.ScrapeCollectorMetricFamilies(s.T(), s.ctx, utils.MockCollectorStack.Otel) - family = s.metricFamilies["nginx_http_request_count"] - s.T().Logf("nginx_http_request_count metric family: %v", family) - s.Require().NotNil(family) + var emptyList []string + got := utils.PollingForMetrics(s.T(), s.ctx, s.metricFamilies, metricName, "", emptyList, baselineMetric) - got := utils.SumMetricFamily(family) - - s.T().Logf("NGINX HTTP request count total: %v", got) - s.Require().GreaterOrEqual(got, baselineMetric+float64(requestCount)) + s.T().Logf("NGINX HTTP request count total: %v", got[0]) + s.Require().Greater(got[0], baselineMetric[0]) } -func (s *MetricsTestSuite) TestNginxOSS_Test2_TestResponseCode() { - family := s.metricFamilies["nginx_http_response_count"] - s.T().Logf("nginx_http_response_count family: %v", family) +// Check that the NGINX OSS response count metric increases after generating some requests for each response code +func (s *MetricsTestSuite) TestNginxOSS_TestResponseCode() { + metricName := "nginx_http_response_count" + family := s.metricFamilies[metricName] + s.T().Logf("%s metric family: %v", metricName, family) s.Require().NotNil(family) - responseCodes := []string{"1xx", "2xx", "3xx", "4xx"} - codeRes := make([]float64, 0, len(responseCodes)) + responseCodes := []string{"1xx", "2xx", "3xx", "4xx", "5xx"} + respBaseline := make([]float64, len(responseCodes)) for code := range responseCodes { - codeRes = append(codeRes, utils.SumMetricFamilyLabel(family, "nginx_status_range", responseCodes[code])) - s.T().Logf("NGINX HTTP response code %s total: %v", responseCodes[code], codeRes[code]) - s.Require().NotNil(codeRes[code]) + respBaseline[code] = utils.SumMetricFamilyLabel(family, "nginx_status_range", responseCodes[code]) + s.T().Logf("NGINX HTTP response code %s total: %v", responseCodes[code], respBaseline[code]) + s.Require().NotNil(respBaseline[code]) + } + + requestCount := 10 + for code := range responseCodes { + utils.GenerateMetrics(s.ctx, s.T(), utils.MockCollectorStack.AgentOSS, requestCount, responseCodes[code]) + } + + got := utils.PollingForMetrics(s.T(), s.ctx, s.metricFamilies, metricName, "nginx_status_range", responseCodes, respBaseline) + for code := range responseCodes { + s.T().Logf("NGINX HTTP response code %s total: %v", responseCodes[code], got[code]) + s.Require().Greater(got[code], respBaseline[code]) } } -func (s *MetricsTestSuite) TestHostMetrics_Test1_TestSystemCPUUtilization() { +// Check that the system CPU utilization metric increases after generating some requests +func (s *MetricsTestSuite) TestHostMetrics_TestSystemCPUUtilization() { family := s.metricFamilies["system_cpu_utilization"] s.T().Logf("system_cpu_utilization metric family: %v", family) s.Require().NotNil(family) - cpuUtilizationSystem := utils.SumMetricFamilyLabel(family, "state", "system") - cpuUtilizationUser := utils.SumMetricFamilyLabel(family, "state", "user") + states := []string{"system", "user"} + respBaseline := make([]float64, len(states)) + for state := range states { + respBaseline[state] = utils.SumMetricFamilyLabel(family, "state", states[state]) + s.T().Logf("CPU utilization for %s: %v", states[state], respBaseline[state]) + s.Require().NotNil(respBaseline[state]) + } + + utils.GenerateMetrics(s.ctx, s.T(), utils.MockCollectorStack.AgentOSS, 20, "2xx") + + got := utils.PollingForMetrics(s.T(), s.ctx, s.metricFamilies, "system_cpu_utilization", "state", states, respBaseline) - s.T().Logf("System cpu utilization: %v", cpuUtilizationSystem) - s.T().Logf("System cpu utilization: %v", cpuUtilizationUser) - s.Require().NotNil(cpuUtilizationSystem) - s.Require().NotNil(cpuUtilizationUser) + for state := range states { + s.T().Logf("CPU utilization for %s: %v", states[state], got[state]) + s.Require().Greater(got[state], respBaseline[state]) + } } -func (s *MetricsTestSuite) TestHostMetrics_Test2_TestSystemMemoryUsage() { +// Check that the system memory usage metric changes after generating some requests +func (s *MetricsTestSuite) TestHostMetrics_TestSystemMemoryUsage() { family := s.metricFamilies["system_memory_usage"] s.T().Logf("system_memory_usage metric family: %v", family) s.Require().NotNil(family) - memoryUsageFree := utils.SumMetricFamilyLabel(family, "state", "free") - memoryUsageUsed := utils.SumMetricFamilyLabel(family, "state", "used") + states := []string{"free", "used"} + respBaseline := make([]float64, len(states)) + for state := range states { + respBaseline[state] = utils.SumMetricFamilyLabel(family, "state", states[state]) + s.T().Logf("Memory %s: %v", states[state], respBaseline[state]) + s.Require().NotNil(respBaseline[state]) + } + + utils.GenerateMetrics(s.ctx, s.T(), utils.MockCollectorStack.AgentOSS, 20, "2xx") + + got := utils.PollingForMetrics(s.T(), s.ctx, s.metricFamilies, "system_memory_usage", "state", states, respBaseline) + + for state := range states { + s.T().Logf("Memory %s: %v", states[state], got[state]) + } + s.Require().Less(got[0], respBaseline[0]) + s.Require().Greater(got[1], respBaseline[1]) - s.T().Logf("System memory usage: %v", memoryUsageFree) - s.T().Logf("System memory usage: %v", memoryUsageUsed) - s.Require().NotNil(memoryUsageFree) - s.Require().NotNil(memoryUsageUsed) } func TestMetricsTestSuite(t *testing.T) { diff --git a/test/integration/utils/mock_collector_utils.go b/test/integration/utils/mock_collector_utils.go index 9e1694d11..141efcd00 100644 --- a/test/integration/utils/mock_collector_utils.go +++ b/test/integration/utils/mock_collector_utils.go @@ -9,13 +9,13 @@ import ( "bytes" "context" "fmt" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" "net" "net/http" "os" "testing" - - dto "github.com/prometheus/client_model/go" - "github.com/prometheus/common/expfmt" + "time" "github.com/go-resty/resty/v2" "github.com/testcontainers/testcontainers-go" @@ -94,6 +94,116 @@ func ScrapeCollectorMetricFamilies(t *testing.T, ctx context.Context, return metricFamilies } +func GenerateMetrics(ctx context.Context, t *testing.T, container testcontainers.Container, requestCount int, expectedCode string) { + t.Helper() + + t.Logf("Generating %d requests with expected response code %s", requestCount, expectedCode) + + var url string + switch expectedCode { + case "1xx": + url = "http://127.0.0.1:9091/" + case "2xx": + url = "http://127.0.0.1:9092/" + case "3xx": + url = "http://127.0.0.1:9093/" + case "4xx": + url = "http://127.0.0.1:9094/" + case "5xx": + url = "http://127.0.0.1:9095/" + default: + url = "http://127.0.0.1/" + } + + for range requestCount { + _, _, err := container.Exec( + ctx, + []string{"curl", "-s", url}, + ) + if err != nil { + t.Fatalf("failed to curl nginx: %s", err) + } + } +} + +func PollingForMetrics(t *testing.T, ctx context.Context, metricFamilies map[string]*dto.MetricFamily, metricName string, labelKey string, labelValues []string, baselineValue []float64, +) []float64 { + t.Helper() + + pollCtx, cancel := context.WithTimeout(ctx, 200*time.Second) + defer cancel() + + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + family := metricFamilies[metricName] + + var res = make([]float64, len(baselineValue)) + for { + select { + case <-pollCtx.Done(): + t.Fatalf("timed out waiting for metric %s to be greater than %v", metricName, baselineValue) + return res + case <-ticker.C: + metricFamilies = ScrapeCollectorMetricFamilies(t, ctx, MockCollectorStack.Otel) + family = metricFamilies[metricName] + if family == nil { + t.Logf("Metric %s not found, retrying...", metricName) + continue + } + + if len(family.GetMetric()) == 1 { + metric := SumMetricFamily(family) + if metric != baselineValue[0] { + return []float64{metric} + } + } + if len(family.GetMetric()) > 1 { + foundAllMetrics := true + for val := range labelValues { + metric := SumMetricFamilyLabel(family, labelKey, labelValues[val]) + if metric != baselineValue[val] { + res[val] = metric + } else { + foundAllMetrics = false + } + } + + if foundAllMetrics && len(res) > 0 { + + return res + } + } + } + } +} + +func WaitUntilNextScrapeCycle(t *testing.T, ctx context.Context) { + t.Helper() + + waitCtx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + for { + select { + case <-waitCtx.Done(): + t.Fatalf("Timed out waiting for next scrape") + return + case <-ticker.C: + freshMetrics := ScrapeCollectorMetricFamilies(t, ctx, MockCollectorStack.Otel) + got := PollingForMetrics(t, ctx, freshMetrics, "nginx_http_request_count", "", []string{}, []float64{0}) + + if len(got) > 0 && got[0] == 1 { + t.Logf("Successfully detected new scrape cycle, request count: %f", got[0]) + return + } + } + } +} + func SumMetricFamily(metricFamily *dto.MetricFamily) float64 { var total float64 for _, metric := range metricFamily.GetMetric() { diff --git a/test/mock/collector/nginx-oss/nginx.conf b/test/mock/collector/nginx-oss/nginx.conf index 21f458f52..ef4620a48 100644 --- a/test/mock/collector/nginx-oss/nginx.conf +++ b/test/mock/collector/nginx-oss/nginx.conf @@ -27,7 +27,7 @@ http { server { listen 9091; - return 200 "hello from http workload 1 \n"; + return 100 "hello from http workload 1 \n"; } server { @@ -36,11 +36,15 @@ http { } server { listen 9093; - return 200 "hello from stream workload 1 \n"; + return 300 "hello from stream workload 1 \n"; } server { listen 9094; - return 200 "hello from stream workload 2 \n"; + return 400 "hello from stream workload 2 \n"; + } + server { + listen 9095; + return 500 "hello from stream workload 2 \n"; } upstream nginx1 { server 127.0.0.1:9091;