wip

Author: cataphract

.gitlab-ci.yml

@@ -6,7 +6,7 @@ macrobenchmarks:
   needs: []
   trigger:
     include: ".gitlab/benchmarks.yml"
-  rules:
-    - if: $NIGHTLY_BENCHMARKS || $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME =~ /^release\/v/
-      when: always
-    - when: manual
+  # rules:
+  #   - if: $NIGHTLY_BENCHMARKS || $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME =~ /^release\/v/
+  #     when: always
+  #   - when: manual

.gitlab/benchmarks.yml

@@ -9,25 +9,25 @@ include:
  file: 'images/templates/gitlab/notify-slo-breaches.template.yml'
 
 variables:
-  MACROBENCHMARKS_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:cpp-nginx
+  MACROBENCHMARKS_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:glopes-cpp-nginx
   BUILD_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/images/mirror/b1o7r7e0/nginx_musl_toolchain
-  NGINX_VERSION: 1.26.0
+  NGINX_VERSION: 1.28.0
 
 build-nginx-module:
   stage: build
   tags: ["arch:amd64"]
   timeout: 20min
   image: $BUILD_IMAGE
   variables:
-    WAF: OFF
+    WAF: ON
     COVERAGE: OFF
     RUM: OFF
-    BUILD_TYPE: Release
+    BUILD_TYPE: RelWithDebInfo
     ARCH: x86_64
   script:
     - git submodule sync && git submodule update --init --recursive
     - export NGINX_SRC_DIR="$PWD/nginx"
-    - make build-musl-aux
+    - MAKE_JOB_COUNT=8 make build-musl-aux
   artifacts:
     name: "artifacts"
     when: always
@@ -44,14 +44,18 @@ build-nginx-module:
   image: $MACROBENCHMARKS_CI_IMAGE
   script:
     - cp .musl-build/ngx_http_datadog_module.so /usr/lib/nginx/modules/ngx_http_datadog_module.so
-    - git clone --branch cpp/nginx https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform && cd platform
+    - git clone --branch glopes/cpp/nginx https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform && cd platform
     - bp-runner bp-runner.yml --debug
+    - mkdir -p artifacts
+    - cp -v platform/artifacts/*.converted.json artifacts/
+    - cp -v platform/artifacts/*.so artifacts/
+    - cp -v platform/artifacts/lscpu.txt artifacts/
   artifacts:
-    name: "artifacts"
+    name: artifacts
     when: always
     paths:
-      - platform/artifacts/
-    expire_in: 3 months
+      - artifacts
+    expire_in: 1 month
   variables:
     # Benchmark's env variables. Modify to tweak benchmark parameters.
     DD_TRACE_DEBUG: "false"
@@ -71,20 +75,80 @@ build-nginx-module:
     K6_OPTIONS_HIGH_LOAD_PRE_ALLOCATED_VUS: 4
     K6_OPTIONS_HIGH_LOAD_MAX_VUS: 4
 
+    K6_OPTIONS_POST_NORMAL_OPERATION_PRE_ALLOCATED_VUS: 1500
+    K6_OPTIONS_POST_NORMAL_OPERATION_MAX_VUS: 2000
+    K6_OPTIONS_POST_HIGH_LOAD_PRE_ALLOCATED_VUS: 7500
+    K6_OPTIONS_POST_HIGH_LOAD_MAX_VUS: 10000
+
   # Workaround: Currently we're not running the benchmarks on every PR, but GitHub still shows them as pending.
   # By marking the benchmarks as allow_failure, the Github checks are not displayed.
   allow_failure: true
 
-baseline:
+.candidate-benchmarks:
   extends: .benchmarks
   variables:
-    DD_BENCHMARKS_CONFIGURATION: baseline
+    ARTIFACT_ROLE: candidate
+    TEST_ARTIFACT: $CI_PROJECT_DIR/.musl-build/ngx_http_datadog_module.so
 
-only-tracing:
+.baseline-benchmarks:
   extends: .benchmarks
+  before_script:
+    - |
+      echo "Installing vault"
+      cd /tmp
+      curl -o vault.zip https://releases.hashicorp.com/vault/1.20.0/vault_1.20.0_linux_amd64.zip
+      unzip vault.zip
+      cd -
+    - CIRCLECI_TOKEN=$(/tmp/vault kv get --format=json kv/k8s/gitlab-runner/nginx-datadog/circleci | jq -r .data.data.token)
+    # this is a token only giving read access to public data
+    - GITHUB_TOKEN=$(/tmp/vault kv get --format=json kv/k8s/gitlab-runner/nginx-datadog/github_pub | jq -r .data.data.token)
+  variables:
+    ARTIFACT_ROLE: baseline
+  rules:
+    - if: '$CI_COMMIT_BRANCH != "master"'
+
+# Candidate benchmarks
+candidate-baselineconf:
+  extends: .candidate-benchmarks
+  variables:
+    DD_BENCHMARKS_CONFIGURATION: baseline
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master"'
+
+candidate-only-tracing:
+  extends: .candidate-benchmarks
+  variables:
+    DD_BENCHMARKS_CONFIGURATION: only-tracing
+
+candidate-appsec:
+  extends: .candidate-benchmarks
+  variables:
+    DD_BENCHMARKS_CONFIGURATION: appsec
+
+# Baseline benchmarks (only non-master)
+baseline-only-tracing:
+  extends: .baseline-benchmarks
   variables:
     DD_BENCHMARKS_CONFIGURATION: only-tracing
 
+baseline-appsec:
+  extends: .baseline-benchmarks
+  variables:
+    DD_BENCHMARKS_CONFIGURATION: appsec
+
+# Comparison between candidate and baseline
+compare-candidate-baseline:
+  stage: run-benchmarks
+  tags: ["arch:amd64"]
+  image: $MACROBENCHMARKS_CI_IMAGE
+  needs: ["baseline-only-tracing", "candidate-only-tracing", "baseline-appsec", "candidate-appsec"]
+  script:
+    - git clone --branch glopes/cpp/nginx https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform && cd platform
+    - cp -v artifacts/* platform/artifacts
+    - cd platform
+    - steps/analyze-results.sh
+    - steps/post-pr-comment.sh
+
 # This repository is using pre-release performance quality gates.
 
 # On release/v{major}.{minor}.{patch} branch, verify that the latest CI pipeline passed the check-slo-breaches job.