diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18845ce4c..7eedb2d81 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -321,6 +321,127 @@ jobs: if: always() run: bash ./scripts/dashboard/format_results.sh ${{job.status}} ${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}} ${{github.workspace}} + oss-metrics-tests: + name: Metrics Integration Tests - Official OSS Images + needs: build-unsigned-snapshot + if: ${{ !github.event.pull_request.head.repo.fork && !startsWith(github.head_ref, 'dependabot-') }} + runs-on: ubuntu-22.04 + strategy: + matrix: + container: + - image: "bookworm" + version: "stable" + release: "debian" + - image: "bookworm" + version: "mainline" + release: "debian" + - image: "alpine" + version: "stable" + release: "alpine" + - image: "alpine" + version: "mainline" + release: "alpine" + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version-file: 'go.mod' + cache: false + - name: Download Packages + uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2 + with: + name: nginx-agent-unsigned-snapshots + path: build + + - name: Create Results Directory + run: mkdir -p ${{ github.workspace }}/test/dashboard/logs/${{ github.job }}/${{matrix.container.image}}-${{matrix.container.version}} + + - name: Start Promtail + uses: ./.github/actions/start-promtail + with: + loki_url: ${{ secrets.LOKI_DASHBOARD_URL }} + + - name: Run Integration Tests + run: | + go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }} + CONTAINER_NGINX_IMAGE_REGISTRY="docker-registry.nginx.com" \ + TAG="${{ matrix.container.version }}-${{ matrix.container.image }}" \ + OS_RELEASE="${{ matrix.container.release }}" OS_VERSION="${{ matrix.container.version }}" \ + make metrics-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}}/raw_logs.log + exit "${PIPESTATUS[0]}" + + - name: Format Results + if: always() + run: bash ./scripts/dashboard/format_results.sh ${{job.status}} ${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}} ${{github.workspace}} + + plus-metrics-tests: + name: Metrics Integration Tests - Official Plus Images + needs: build-unsigned-snapshot + if: ${{ !github.event.pull_request.head.repo.fork && !startsWith(github.head_ref, 'dependabot-') }} + runs-on: ubuntu-22.04 + strategy: + matrix: + container: + - image: "alpine" + version: "3.22" + plus: "r35" + release: "alpine" + path: "/nginx-plus/agent" + - image: "alpine" + version: "3.21" + plus: "r34" + release: "alpine" + path: "/nginx-plus/agent" + - image: "debian" + version: "bookworm" + plus: "r35" + release: "debian" + path: "/nginx-plus/agent" + - image: "debian" + version: "bookworm" + plus: "r34" + release: "debian" + path: "/nginx-plus/agent" + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version-file: 'go.mod' + cache: false + - name: Download Packages + uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2 + with: + name: nginx-agent-unsigned-snapshots + path: build + - name: Login to Docker Registry + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ${{ secrets.TEST_REGISTRY_URL }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + + - name: Create Results Directory + run: mkdir -p ${{ github.workspace }}/test/dashboard/logs/${{ github.job }}/${{matrix.container.image}}-${{matrix.container.version}} + + - name: Start Promtail + uses: ./.github/actions/start-promtail + with: + loki_url: ${{ secrets.LOKI_DASHBOARD_URL }} + + - name: Run Integration Tests + run: | + go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }} + CONTAINER_NGINX_IMAGE_REGISTRY="${{ secrets.TEST_REGISTRY_URL }}" \ + TAG="${{ matrix.container.plus }}-${{ matrix.container.image }}-${{ matrix.container.version }}" \ + OS_RELEASE="${{ matrix.container.release }}" OS_VERSION="${{ matrix.container.version }}" IMAGE_PATH="${{ matrix.container.path }}" \ + NGINX_LICENSE_JWT="${{ secrets.TEST_JWT }}" \ + make metrics-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}}/raw_logs.log + exit "${PIPESTATUS[0]}" + + - name: Format Results + if: always() + run: bash ./scripts/dashboard/format_results.sh ${{job.status}} ${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}} ${{github.workspace}} + performance-tests: name: Performance Tests runs-on: ubuntu-22.04 diff --git a/Makefile b/Makefile index 7aef0323f..f9d9b0152 100644 --- a/Makefile +++ b/Makefile @@ -162,7 +162,7 @@ build-mock-management-otel-collector: mkdir -p $(BUILD_DIR)/mock-management-otel-collector @CGO_ENABLED=0 GOARCH=$(OSARCH) GOOS=linux $(GOBUILD) -o $(BUILD_DIR)/mock-management-otel-collector/collector test/mock/collector/mock-collector/main.go -integration-test: $(SELECTED_PACKAGE) build-mock-management-plane-grpc +integration-test: $(SELECTED_PACKAGE) build-mock-management-plane-grpc TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-local" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \ PACKAGES_REPO=$(OSS_PACKAGES_REPO) PACKAGE_NAME=$(PACKAGE_NAME) BASE_IMAGE=$(BASE_IMAGE) DOCKERFILE_PATH=$(DOCKERFILE_PATH) IMAGE_PATH=$(IMAGE_PATH) TAG=${IMAGE_TAG} \ OS_VERSION=$(OS_VERSION) OS_RELEASE=$(OS_RELEASE) \ @@ -180,6 +180,13 @@ official-image-integration-test: $(SELECTED_PACKAGE) build-mock-management-plane OS_VERSION=$(OS_VERSION) OS_RELEASE=$(OS_RELEASE) IMAGE_PATH=$(IMAGE_PATH) \ NGINX_LICENSE_JWT=$(NGINX_LICENSE_JWT) \ go test -v ./test/integration/managementplane ./test/integration/auxiliarycommandserver + +metrics-test: $(SELECTED_PACKAGE) build-mock-management-otel-collector + TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} BUILD_TARGET="install" \ + PACKAGES_REPO=$(OSS_PACKAGES_REPO) TAG=${TAG} PACKAGE_NAME=$(PACKAGE_NAME) BASE_IMAGE=$(BASE_IMAGE) DOCKERFILE_PATH=$(OFFICIAL_IMAGE_DOCKERFILE_PATH) \ + OS_VERSION=$(OS_VERSION) OS_RELEASE=$(OS_RELEASE) IMAGE_PATH=$(IMAGE_PATH) \ + NGINX_LICENSE_JWT=$(NGINX_LICENSE_JWT) \ + go test -v ./test/integration/metrics performance-test: mkdir -p $(TEST_BUILD_DIR) diff --git a/api/grpc/mpi/v1/command.pb.go b/api/grpc/mpi/v1/command.pb.go index 39ef93489..b5f0346f4 100644 --- a/api/grpc/mpi/v1/command.pb.go +++ b/api/grpc/mpi/v1/command.pb.go @@ -8,7 +8,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.8 +// protoc-gen-go v1.36.9 // protoc (unknown) // source: mpi/v1/command.proto diff --git a/api/grpc/mpi/v1/common.pb.go b/api/grpc/mpi/v1/common.pb.go index 890c2b073..e6d06cf37 100644 --- a/api/grpc/mpi/v1/common.pb.go +++ b/api/grpc/mpi/v1/common.pb.go @@ -5,7 +5,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.8 +// protoc-gen-go v1.36.9 // protoc (unknown) // source: mpi/v1/common.proto diff --git a/api/grpc/mpi/v1/files.pb.go b/api/grpc/mpi/v1/files.pb.go index 1873a19e2..d379ed70a 100644 --- a/api/grpc/mpi/v1/files.pb.go +++ b/api/grpc/mpi/v1/files.pb.go @@ -5,7 +5,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.8 +// protoc-gen-go v1.36.9 // protoc (unknown) // source: mpi/v1/files.proto diff --git a/go.mod b/go.mod index 525362278..bd8efef74 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,8 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/tcplogreceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/testbed v0.124.1 + github.com/prometheus/client_model v0.6.1 + github.com/prometheus/common v0.62.0 github.com/shirou/gopsutil/v4 v4.25.7 github.com/spf13/pflag v1.0.6 github.com/stretchr/testify v1.10.0 @@ -201,8 +203,6 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.16.0 // indirect github.com/rs/cors v1.11.1 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect diff --git a/test/config/nginx/nginx-for-metric-testing.conf b/test/config/nginx/nginx-for-metric-testing.conf new file mode 100644 index 000000000..134adc88d --- /dev/null +++ b/test/config/nginx/nginx-for-metric-testing.conf @@ -0,0 +1,62 @@ +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"' + '"$upstream_cache_status"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + server { + listen 80 default_server; + server_name localhost; + + location /1xx { + return 100 "Success 1xx response code \n"; + } + location /2xx { + return 200 "Success 2xx response code \n"; + } + + location /3xx { + return 300 "Success 3xx response code \n"; + } + + location /4xx { + return 400 "Success 4xx response code \n"; + } + + location /5xx { + return 500 "Success 5xx response code \n"; + } + + location /api { + stub_status; + allow 127.0.0.1; + deny all; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + } +} diff --git a/test/config/nginx/nginx-plus-for-metric-testing.conf b/test/config/nginx/nginx-plus-for-metric-testing.conf new file mode 100644 index 000000000..a65d6978f --- /dev/null +++ b/test/config/nginx/nginx-plus-for-metric-testing.conf @@ -0,0 +1,62 @@ +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"' + '"$upstream_cache_status"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + server { + listen 80 default_server; + server_name localhost; + + location /1xx { + return 100 "Success 1xx response code \n"; + } + location /2xx { + return 200 "Success 2xx response code \n"; + } + + location /3xx { + return 300 "Success 3xx response code \n"; + } + + location /4xx { + return 400 "Success 4xx response code \n"; + } + + location /5xx { + return 500 "Success 5xx response code \n"; + } + + location /api/ { + api write=on; + allow 127.0.0.1; + deny all; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + } +} diff --git a/test/helpers/test_containers_utils.go b/test/helpers/test_containers_utils.go index a7227a2b8..f5cf3de5b 100644 --- a/test/helpers/test_containers_utils.go +++ b/test/helpers/test_containers_utils.go @@ -26,6 +26,12 @@ type Parameters struct { LogMessage string } +type MockCollectorContainers struct { + Agent testcontainers.Container + Otel testcontainers.Container + Prometheus testcontainers.Container +} + func StartContainer( ctx context.Context, tb testing.TB, @@ -311,6 +317,69 @@ func StartAuxiliaryMockManagementPlaneGrpcContainer(ctx context.Context, tb test return container } +func StartMockCollectorStack(ctx context.Context, tb testing.TB, + containerNetwork *testcontainers.DockerNetwork, +) *MockCollectorContainers { + tb.Helper() + + otel, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + FromDockerfile: testcontainers.FromDockerfile{ + Context: "../../../", + Dockerfile: "./test/mock/collector/mock-collector/Dockerfile", + KeepImage: false, + PrintBuildLog: true, + }, + ExposedPorts: []string{"4317/tcp", "9090/tcp", "9775/tcp"}, + Networks: []string{containerNetwork.Name}, + NetworkAliases: map[string][]string{ + containerNetwork.Name: { + "otel-collector", + }, + }, + Files: []testcontainers.ContainerFile{ + { + HostFilePath: "../../mock/collector/otel-collector.yaml", + ContainerFilePath: "/etc/otel-collector.yaml", + FileMode: configFilePermissions, + }, + }, + WaitingFor: wait.ForLog("Everything is ready. Begin running and processing data."), + }, + Started: true, + }) + require.NoError(tb, err) + + prometheus, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: "prom/prometheus:latest", + ExposedPorts: []string{"9090/tcp"}, + Networks: []string{containerNetwork.Name}, + NetworkAliases: map[string][]string{ + containerNetwork.Name: { + "prometheus", + }, + }, + Files: []testcontainers.ContainerFile{ + { + HostFilePath: "../../mock/collector/prometheus.yaml", + ContainerFilePath: "/etc/prometheus/prometheus.yaml", + FileMode: configFilePermissions, + }, + }, + Cmd: []string{"--config.file=/etc/prometheus/prometheus.yml"}, + WaitingFor: wait.ForLog("Server is ready to receive web requests."), + }, + Started: true, + }) + require.NoError(tb, err) + + return &MockCollectorContainers{ + Otel: otel, + Prometheus: prometheus, + } +} + func ToPtr[T any](value T) *T { return &value } @@ -374,3 +443,33 @@ func LogAndTerminateContainers( require.NoError(tb, err) } } + +func LogAndTerminateStack(ctx context.Context, tb testing.TB, + containers *MockCollectorContainers, +) { + tb.Helper() + + logAndTerminate := func(name string, container testcontainers.Container) { + if container == nil { + tb.Logf("Skipping log collection for %s: container is nil", name) + return + } + + tb.Logf("======================== Logging %s Container Logs ========================", name) + logReader, err := container.Logs(ctx) + require.NoError(tb, err) + + buf, err := io.ReadAll(logReader) + require.NoError(tb, err) + logs := string(buf) + + tb.Log(logs) + + err = container.Terminate(ctx) + require.NoError(tb, err) + } + + logAndTerminate("Agent", containers.Agent) + logAndTerminate("Otel Collector", containers.Otel) + logAndTerminate("Prometheus", containers.Prometheus) +} diff --git a/test/integration/metrics/metrics_test.go b/test/integration/metrics/metrics_test.go new file mode 100644 index 000000000..9e329bf00 --- /dev/null +++ b/test/integration/metrics/metrics_test.go @@ -0,0 +1,156 @@ +// Copyright (c) F5, Inc. +// +// This source code is licensed under the Apache License, Version 2.0 license found in the +// LICENSE file in the root directory of this source tree. + +package metrics + +import ( + "context" + "log/slog" + "os" + "testing" + + "github.com/nginx/agent/v3/test/integration/utils" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/suite" +) + +type MetricsTestSuite struct { + suite.Suite + ctx context.Context + teardownTest func(testing.TB) + metricFamilies map[string]*dto.MetricFamily +} + +func (s *MetricsTestSuite) SetupSuite() { + slog.Info("starting metric tests") + s.ctx = context.Background() + s.teardownTest = utils.SetupMetricsTest(s.T()) + utils.WaitForMetricsToExist(s.T(), s.ctx) +} + +func (s *MetricsTestSuite) SetupTest() { + s.metricFamilies = utils.ScrapeCollectorMetricFamilies(s.T(), s.ctx, utils.MockCollectorStack.Otel) +} + +func (s *MetricsTestSuite) TearDownTest() { + if s.T().Skipped() { + return + } + utils.WaitUntilNextScrapeCycle(s.T(), s.ctx) +} + +func (s *MetricsTestSuite) TearDownSuite() { + slog.Info("finished metric tests") + s.teardownTest(s.T()) +} + +// Check that the NGINX request count metric increases after generating requests +func (s *MetricsTestSuite) TestNginxMetrics_TestRequestCount() { + slog.Info("starting nginx request count metric test") + metricName := "nginx_http_request_count" + family := s.metricFamilies[metricName] + s.Require().NotNil(family) + + var baselineMetric []float64 + baselineMetric = append(baselineMetric, utils.SumMetricFamily(family)) + s.T().Logf("NGINX HTTP request count total: %v", baselineMetric[0]) + + requestCount := 50 + utils.GenerateMetrics(s.ctx, s.T(), utils.MockCollectorStack.Agent, requestCount, "2xx") + + got := utils.PollingForMetrics(s.T(), s.ctx, metricName, utils.LabelFilter{ + Key: "", + Values: []string{}, + }, baselineMetric) + + s.T().Logf("NGINX HTTP request count total: %v", got[0]) + s.Require().Greater(got[0], baselineMetric[0]) + slog.Info("finished nginx request count metric test") +} + +// Check that the NGINX response count metric increases after generating requests for each response code +func (s *MetricsTestSuite) TestNginxMetrics_TestResponseCode() { + if os.Getenv("IMAGE_PATH") != "/nginx/agent" { + s.T().Skip("Skipping test for NGINX OSS specific metric") + } + slog.Info("starting nginx response count metric test") + metricName := "nginx_http_response_count" + family := s.metricFamilies[metricName] + s.Require().NotNil(family) + + responseCodes := []string{"1xx", "2xx", "3xx", "4xx", "5xx"} + respBaseline := make([]float64, len(responseCodes)) + for code := range responseCodes { + respBaseline[code] = utils.SumMetricFamilyLabel(family, "nginx_status_range", responseCodes[code]) + s.T().Logf("NGINX HTTP response code %s total: %v", responseCodes[code], respBaseline[code]) + s.Require().NotNil(respBaseline[code]) + } + + requestCount := 20 + for code := range responseCodes { + utils.GenerateMetrics(s.ctx, s.T(), utils.MockCollectorStack.Agent, requestCount, responseCodes[code]) + } + + got := utils.PollingForMetrics(s.T(), s.ctx, metricName, utils.LabelFilter{ + Key: "nginx_status_range", + Values: responseCodes, + }, respBaseline) + for code := range responseCodes { + s.T().Logf("NGINX HTTP response code %s total: %v", responseCodes[code], got[code]) + s.Require().Greater(got[code], respBaseline[code]) + } + slog.Info("finished nginx response count metric test") +} + +// Check that the system CPU utilization metric increases after generating requests +func (s *MetricsTestSuite) TestHostMetrics_TestSystemCPUUtilization() { + slog.Info("starting host cpu utilization metric test") + family := s.metricFamilies["system_cpu_utilization"] + s.Require().NotNil(family) + + states := []string{"system", "user"} + respBaseline := make([]float64, len(states)) + for state := range states { + respBaseline[state] = utils.SumMetricFamilyLabel(family, "state", states[state]) + s.T().Logf("CPU utilization for %s: %v", states[state], respBaseline[state]) + s.Require().NotNil(respBaseline[state]) + } + + utils.GenerateMetrics(s.ctx, s.T(), utils.MockCollectorStack.Agent, 20, "2xx") + + got := utils.PollingForMetrics(s.T(), s.ctx, + "system_cpu_utilization", utils.LabelFilter{ + Key: "state", + Values: states, + }, respBaseline) + + for state := range states { + s.T().Logf("CPU utilization for %s: %v", states[state], got[state]) + s.Require().Greater(got[state], respBaseline[state]) + } + + slog.Info("finished host cpu utilization metric test") +} + +// Verify that the system memory usage metric is being collected +func (s *MetricsTestSuite) TestHostMetrics_TestSystemMemoryUsage() { + slog.Info("starting host memory usage metric test") + family := s.metricFamilies["system_memory_usage"] + s.Require().NotNil(family) + + states := []string{"free", "used"} + respBaseline := make([]float64, len(states)) + for state := range states { + respBaseline[state] = utils.SumMetricFamilyLabel(family, "state", states[state]) + s.T().Logf("Memory %s: %v", states[state], respBaseline[state]) + s.Require().NotNil(respBaseline[state]) + } + + slog.Info("finished host memory usage metric test") +} + +func TestMetricsTestSuite(t *testing.T) { + suite.Run(t, new(MetricsTestSuite)) +} diff --git a/test/integration/utils/metrics_utils.go b/test/integration/utils/metrics_utils.go new file mode 100644 index 000000000..221147cff --- /dev/null +++ b/test/integration/utils/metrics_utils.go @@ -0,0 +1,132 @@ +// Copyright (c) F5, Inc. +// +// This source code is licensed under the Apache License, Version 2.0 license found in the +// LICENSE file in the root directory of this source tree. + +package utils + +import ( + "testing" + + dto "github.com/prometheus/client_model/go" +) + +// SumMetricFamily retrieves the sum of the metric values in a MetricFamily +func SumMetricFamily(metricFamily *dto.MetricFamily) float64 { + var total float64 + for _, metric := range metricFamily.GetMetric() { + if value := metricValue(metricFamily, metric); value != nil { + total += *value + } + } + + return total +} + +// SumMetricFamilyLabel retrieves the sum of the metric values for a specific label in a MetricFamily +func SumMetricFamilyLabel(metricFamily *dto.MetricFamily, key, val string) float64 { + var total float64 + for _, metric := range metricFamily.GetMetric() { + labels := make(map[string]string) + for _, labelPair := range metric.GetLabel() { + labels[labelPair.GetName()] = labelPair.GetValue() + } + if labels[key] != val { + continue + } + if value := metricValue(metricFamily, metric); value != nil { + total += *value + } + } + + return total +} + +// checkSingleMetricValue compares a metric's current value against a baseline value +func checkSingleMetricValue(t *testing.T, family *dto.MetricFamily, baselineValue float64) (float64, bool) { + t.Helper() + metric := SumMetricFamily(family) + + return metric, metric != baselineValue +} + +// checkLabeledMetricValue compares labeled metrics' current values against a set of baseline values +func checkLabeledMetricValue(t *testing.T, family *dto.MetricFamily, labelKey string, + labelValues []string, baselineValues []float64, +) ([]float64, bool) { + t.Helper() + results := make([]float64, len(baselineValues)) + allDifferent := true + + for val := range labelValues { + metric := SumMetricFamilyLabel(family, labelKey, labelValues[val]) + results[val] = metric + if metric == baselineValues[val] { + allDifferent = false + } + } + + return results, allDifferent && len(results) > 0 +} + +func metricValue(metricFamily *dto.MetricFamily, metric *dto.Metric) *float64 { + switch metricFamily.GetType() { + case dto.MetricType_COUNTER: + return counterValue(metric) + case dto.MetricType_GAUGE: + return gaugeValue(metric) + case dto.MetricType_SUMMARY: + return summaryValue(metric) + case dto.MetricType_UNTYPED: + return untypedValue(metric) + case dto.MetricType_HISTOGRAM, dto.MetricType_GAUGE_HISTOGRAM: + return histogramValue(metric) + } + + return nil +} + +func counterValue(metric *dto.Metric) *float64 { + if counter := metric.GetCounter(); counter != nil { + val := counter.GetValue() + return &val + } + + return nil +} + +func gaugeValue(metric *dto.Metric) *float64 { + if gauge := metric.GetGauge(); gauge != nil { + val := gauge.GetValue() + return &val + } + + return nil +} + +func summaryValue(metric *dto.Metric) *float64 { + if summary := metric.GetSummary(); summary != nil { + val := summary.GetSampleSum() + return &val + } + + return nil +} + +func untypedValue(metric *dto.Metric) *float64 { + if untyped := metric.GetUntyped(); untyped != nil { + val := untyped.GetValue() + return &val + } + + return nil +} + +func histogramValue(metric *dto.Metric) *float64 { + if histogram := metric.GetHistogram(); histogram != nil { + val := histogram.GetSampleSum() + return &val + } + + return nil +} diff --git a/test/integration/utils/mock_collector_utils.go b/test/integration/utils/mock_collector_utils.go new file mode 100644 index 000000000..73e1e8619 --- /dev/null +++ b/test/integration/utils/mock_collector_utils.go @@ -0,0 +1,281 @@ +// Copyright (c) F5, Inc. +// +// This source code is licensed under the Apache License, Version 2.0 license found in the +// LICENSE file in the root directory of this source tree. + +package utils + +import ( + "bytes" + "context" + "fmt" + "net" + "net/http" + "os" + "testing" + "time" + + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + + "github.com/go-resty/resty/v2" + "github.com/testcontainers/testcontainers-go" + + "github.com/nginx/agent/v3/test/helpers" +) + +var MockCollectorStack *helpers.MockCollectorContainers + +const ( + envContainer = "Container" + tickerTime = 1 * time.Second + timeoutTime = 100 * time.Second + plusImagePath = "/nginx-plus/agent" +) + +type LabelFilter struct { + Key string + Values []string +} + +func SetupMetricsTest(tb testing.TB) func(testing.TB) { + tb.Helper() + ctx := context.Background() + + if os.Getenv("TEST_ENV") == envContainer { + setupStackEnvironment(ctx, tb) + } + + return func(tb testing.TB) { + tb.Helper() + + if os.Getenv("TEST_ENV") == envContainer { + helpers.LogAndTerminateStack( + ctx, + tb, + MockCollectorStack, + ) + } + } +} + +func setupStackEnvironment(ctx context.Context, tb testing.TB) { + tb.Helper() + tb.Log("Running tests in a container environment") + + containerNetwork := CreateContainerNetwork(ctx, tb) + setupMockCollectorStack(ctx, tb, containerNetwork) +} + +func setupMockCollectorStack(ctx context.Context, tb testing.TB, containerNetwork *testcontainers.DockerNetwork) { + tb.Helper() + + tb.Log("Starting mock collector stack") + + nginxConfPath := "../../config/nginx/nginx-for-metric-testing.conf" + if os.Getenv("IMAGE_PATH") == plusImagePath { + nginxConfPath = "../../config/nginx/nginx-plus-for-metric-testing.conf" + } + agentConfig := "../../mock/collector/nginx-agent.conf" + + params := &helpers.Parameters{ + NginxConfigPath: nginxConfPath, + NginxAgentConfigPath: agentConfig, + LogMessage: "Starting NGINX Agent", + } + + MockCollectorStack = helpers.StartMockCollectorStack(ctx, tb, containerNetwork) + MockCollectorStack.Agent = helpers.StartContainer(ctx, tb, containerNetwork, params) +} + +func ScrapeCollectorMetricFamilies(t *testing.T, ctx context.Context, + otelContainer testcontainers.Container, +) map[string]*dto.MetricFamily { + t.Helper() + + host, _ := otelContainer.Host(ctx) + port, _ := otelContainer.MappedPort(ctx, "9775") + + address := net.JoinHostPort(host, port.Port()) + url := fmt.Sprintf("http://%s/metrics", address) + + client := resty.New() + resp, err := client.R().EnableTrace().Get(url) + if err != nil { + t.Fatalf("failed to get response from Otel Collector: %v", err) + } + if resp.StatusCode() != http.StatusOK { + t.Fatalf("Unexpected status code: %d", resp.StatusCode()) + } + + parser := expfmt.TextParser{} + metricFamilies, err := parser.TextToMetricFamilies(bytes.NewReader(resp.Body())) + if err != nil { + t.Fatalf("failed to parse metrics: %v", err) + } + + return metricFamilies +} + +func GenerateMetrics(ctx context.Context, t *testing.T, container testcontainers.Container, + requestCount int, expectedCode string, +) { + t.Helper() + + t.Logf("Generating %d requests with expected response code %s", requestCount, expectedCode) + + var url string + switch expectedCode { + case "1xx": + url = "http://127.0.0.1:80/1xx" + case "2xx": + url = "http://127.0.0.1:80/2xx" + case "3xx": + url = "http://127.0.0.1:80/3xx" + case "4xx": + url = "http://127.0.0.1:80/4xx" + case "5xx": + url = "http://127.0.0.1:80/5xx" + + default: + url = "http://127.0.0.1/" + } + + for range requestCount { + _, _, err := container.Exec( + ctx, + []string{"curl", "-s", url}, + ) + if err != nil { + t.Fatalf("failed to curl nginx: %s", err) + } + } +} + +func PollingForMetrics(t *testing.T, ctx context.Context, metricName string, + labelFilter LabelFilter, baselineValues []float64, +) []float64 { + t.Helper() + + pollCtx, cancel := context.WithTimeout(ctx, timeoutTime) + defer cancel() + + ticker := time.NewTicker(tickerTime) + defer ticker.Stop() + + for { + select { + case <-pollCtx.Done(): + t.Fatalf("timed out waiting for metric %s to be greater than %v", metricName, baselineValues) + return nil + case <-ticker.C: + family := ScrapeCollectorMetricFamilies(t, ctx, MockCollectorStack.Otel)[metricName] + if family == nil { + t.Logf("Metric %s not found, retrying...", metricName) + continue + } + + values, changed := metricValueChangeCheck(t, family, labelFilter.Key, labelFilter.Values, baselineValues) + if !changed { + continue + } + + return values + } + } +} + +func WaitUntilNextScrapeCycle(t *testing.T, ctx context.Context) { + t.Helper() + + waitCtx, cancel := context.WithTimeout(ctx, timeoutTime) + defer cancel() + + ticker := time.NewTicker(tickerTime) + defer ticker.Stop() + + prevScrapeValue, err := requestCountMetric(t, ctx) + if err != nil { + t.Fatalf("Failed to get initial scrape value: %v", err) + return + } + + for { + select { + case <-waitCtx.Done(): + t.Fatalf("Timed out waiting for new scrape cycle") + return + case <-ticker.C: + currentMetric, requestErr := requestCountMetric(t, ctx) + if requestErr != nil { + continue + } + + if currentMetric != prevScrapeValue { + t.Log("Successfully detected new scrape cycle") + + return + } + } + } +} + +func WaitForMetricsToExist(t *testing.T, ctx context.Context) { + t.Helper() + + waitCtx, cancel := context.WithTimeout(ctx, timeoutTime) + defer cancel() + + ticker := time.NewTicker(tickerTime) + defer ticker.Stop() + + for { + select { + case <-waitCtx.Done(): + t.Fatal("Timed out waiting for NGINX metrics to exist") + return + case <-ticker.C: + family := ScrapeCollectorMetricFamilies(t, ctx, MockCollectorStack.Otel)["nginx_http_request_count"] + if family != nil { + t.Log("NGINX metrics found") + return + } + t.Log("NGINX metrics not found, retrying...") + } + } +} + +func requestCountMetric(t *testing.T, ctx context.Context) (float64, error) { + t.Helper() + + family := ScrapeCollectorMetricFamilies(t, ctx, MockCollectorStack.Otel)["nginx_http_request_count"] + if family == nil { + return 0, fmt.Errorf("metric nginx_http_request_count not found: %v", family) + } + + return SumMetricFamily(family), nil +} + +// metricValueChangeCheck checks if the metric values in a MetricFamily have changed +func metricValueChangeCheck(t *testing.T, family *dto.MetricFamily, labelKey string, + labelValues []string, baselineValues []float64) ( + []float64, bool, +) { + t.Helper() + + if len(family.GetMetric()) == 1 { + value, changed := checkSingleMetricValue(t, family, baselineValues[0]) + if changed { + return []float64{value}, true + } + } + + if len(family.GetMetric()) > 1 { + values, allChanged := checkLabeledMetricValue(t, family, labelKey, labelValues, baselineValues) + if allChanged { + return values, true + } + } + + return []float64{0}, false +} diff --git a/test/mock/collector/README.md b/test/mock/collector/README.md index 938fab5f7..c06cfdd43 100644 --- a/test/mock/collector/README.md +++ b/test/mock/collector/README.md @@ -24,6 +24,11 @@ To start run everything run the following make run-mock-management-otel-collector ``` +To start everything except the NGINX Plus & NGINX App Protect run the following +``` +make run-mock-otel-collector-without-nap +``` + Once everything is started there should be 7 containers running ``` CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES @@ -43,3 +48,8 @@ To stop everything run the following ``` make stop-mock-management-otel-collector ``` + +Or run the following if you started everything except the NGINX Plus & NGINX App Protect +``` +make stop-mock-otel-collector-without-nap +``` diff --git a/test/mock/collector/nginx-agent.conf b/test/mock/collector/nginx-agent.conf index 3ef1c5e6c..ee759c854 100644 --- a/test/mock/collector/nginx-agent.conf +++ b/test/mock/collector/nginx-agent.conf @@ -40,9 +40,9 @@ collector: level: DEBUG receivers: container_metrics: - collection_interval: 1m0s + collection_interval: 15s host_metrics: - collection_interval: 1m0s + collection_interval: 15s initial_delay: 1s scrapers: disk: {} diff --git a/test/mock/collector/nginx-oss/nginx.conf b/test/mock/collector/nginx-oss/nginx.conf index 21f458f52..ef4620a48 100644 --- a/test/mock/collector/nginx-oss/nginx.conf +++ b/test/mock/collector/nginx-oss/nginx.conf @@ -27,7 +27,7 @@ http { server { listen 9091; - return 200 "hello from http workload 1 \n"; + return 100 "hello from http workload 1 \n"; } server { @@ -36,11 +36,15 @@ http { } server { listen 9093; - return 200 "hello from stream workload 1 \n"; + return 300 "hello from stream workload 1 \n"; } server { listen 9094; - return 200 "hello from stream workload 2 \n"; + return 400 "hello from stream workload 2 \n"; + } + server { + listen 9095; + return 500 "hello from stream workload 2 \n"; } upstream nginx1 { server 127.0.0.1:9091; diff --git a/test/mock/collector/nginx-plus/nginx.conf b/test/mock/collector/nginx-plus/nginx.conf index e83b9bec9..bfc898d70 100644 --- a/test/mock/collector/nginx-plus/nginx.conf +++ b/test/mock/collector/nginx-plus/nginx.conf @@ -34,7 +34,7 @@ http { } server { listen 9091; - return 200 "hello from http workload 1 \n"; + return 100 "hello from http workload 1 \n"; } server { @@ -43,11 +43,15 @@ http { } server { listen 9093; - return 200 "hello from stream workload 1 \n"; + return 300 "hello from stream workload 1 \n"; } server { listen 9094; - return 200 "hello from stream workload 2 \n"; + return 400 "hello from stream workload 2 \n"; + } + server { + listen 9095; + return 500 "hello from stream workload 2 \n"; } upstream nginx1 { server 127.0.0.1:9091;