benchmarks.yml

# Insert more benchmark logic here

.dsm-kafka-benchmarks:
  stage: benchmarks
  rules:
    - when: never # testing startup benchmarks only for now
    - if: '$POPULATE_CACHE'
      when: never
    - if: $CI_PIPELINE_SOURCE != "schedule"
      changes:
        paths:
          - dd-java-agent/instrumentation/kafka*/**/*
        compare_to: "master"
      when: on_success
    - when: manual
      allow_failure: true
  tags: ["runner:apm-k8s-tweaked-metal"]
  interruptible: true
  timeout: 1h
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:java-dsm-kafka
  needs: [ "build", "publish-artifacts-to-s3"]
  script:
    - git clone --branch java/kafka-dsm-overhead https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform.git platform && cd platform
    - ./steps/run-benchmarks.sh
  artifacts:
    name: "artifacts"
    when: always
    paths:
      - platform/artifacts/
    expire_in: 3 months
  variables:
    UPSTREAM_PROJECT_ID: $CI_PROJECT_ID # The ID of the current project. This ID is unique across all projects on the GitLab instance.
    UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME # "dd-trace-java"
    UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME # The branch or tag name for which project is built.
    UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA # The commit revision the project is built for.
    FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY: "true"

dsm-kafka-producer-benchmark:
  extends: .dsm-kafka-benchmarks
  variables:
    BP_KAFKA_SCENARIO_DIR: producer-benchmark

dsm-kafka-consumer-benchmark:
  extends: .dsm-kafka-benchmarks
  variables:
    BP_KAFKA_SCENARIO_DIR: consumer-benchmark

debugger-benchmarks:
  stage: benchmarks
  rules:
    - when: never # testing startup benchmarks only for now
    - if: '$POPULATE_CACHE'
      when: never
    - if: $CI_PIPELINE_SOURCE != "schedule"
      changes:
        paths:
          - dd-java-agent/agent-debugger/**/*
        compare_to: "master"
      when: on_success
    - when: manual
      allow_failure: true
  tags: ["runner:apm-k8s-tweaked-metal"]
  interruptible: true
  timeout: 1h
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:java-debugger
  needs: ["build", "publish-artifacts-to-s3"]
  script:
    - export ARTIFACTS_DIR="$(pwd)/reports" && mkdir -p "${ARTIFACTS_DIR}"
    - git clone --branch java/debugger-benchmarks https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform.git /platform && cd /platform
    - numactl --cpunodebind=1 --membind=1 bp-runner bp-runner.yml --debug
    - "./steps/create-report.sh || :"
    - "./steps/post-pr-comment.sh || :"
  artifacts:
    name: "artifacts"
    when: always
    paths:
      - /go/src/github.com/DataDog/apm-reliability/dd-trace-java/reports/
    expire_in: 3 months
  variables:
    UPSTREAM_PROJECT_ID: $CI_PROJECT_ID # The ID of the current project. This ID is unique across all projects on the GitLab instance.
    UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME # "dd-trace-java"
    UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME # The branch or tag name for which project is built.
    UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA # The commit revision the project is built for.
    FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY: "true"

.benchmark-compare-template:
  stage: benchmark-comparison
  image: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:latest
  tags: ["arch:amd64"]
  interruptible: true
  rules:
    - if: '$POPULATE_CACHE'
      when: never
    - when: on_success
  script:
    - |
      if [ "$(uname -m)" = "x86_64" ]; then AAA="amd64"; else AAA="arm64"; fi
      wget -O /tmp/authanywhere "https://binaries.ddbuild.io/dd-source/authanywhere/LATEST/authanywhere-linux-${AAA}"
      chmod +x /tmp/authanywhere
      mv /tmp/authanywhere /usr/local/bin/authanywhere
    - mkdir -p "$(pwd)/reports/${BENCHMARK_TYPE}"
    - .gitlab/scripts/get-baseline-commit-info.sh "$(pwd)/reports/${BENCHMARK_TYPE}"
    - .gitlab/scripts/benchmark-compare.sh "${BENCHMARK_TYPE}"
  artifacts:
    name: "benchmark-compare-${BENCHMARK_TYPE}"
    when: always
    paths:
      - reports/${BENCHMARK_TYPE}/
    expire_in: 3 months
  variables:
    TARGET_BRANCH: "master"

benchmarks-compare-startup:
  extends: .benchmark-compare-template
  needs:
    - job: linux-java-spring-petclinic-microbenchmark-startup-tracing
      artifacts: false
    - job: linux-java-spring-petclinic-microbenchmark-startup-profiling
      artifacts: false
    - job: linux-java-spring-petclinic-microbenchmark-startup-appsec
      artifacts: false
    - job: linux-java-spring-petclinic-microbenchmark-startup-iast
      artifacts: false
    - job: linux-java-insecure-bank-microbenchmark-startup-tracing
      artifacts: false
    - job: linux-java-insecure-bank-microbenchmark-startup-iast
      artifacts: false
  variables:
    BENCHMARK_TYPE: startup

benchmarks-compare-load:
  extends: .benchmark-compare-template
  rules:
    - when: never # testing startup benchmarks only for now
  variables:
    BENCHMARK_TYPE: load

benchmarks-compare-dacapo:
  extends: .benchmark-compare-template
  rules:
    - when: never # TEMP: testing startup benchmarks only for now
  variables:
    BENCHMARK_TYPE: dacapo

benchmarks-post-pr-comment:
  stage: benchmark-comparison
  image: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:latest
  tags: ["arch:amd64"]
  interruptible: true
  needs:
    - job: benchmarks-compare-startup
      artifacts: true
    - job: benchmarks-compare-load
      artifacts: true
    - job: benchmarks-compare-dacapo
      artifacts: true
  rules:
    - when: never # testing startup benchmarks only for now
    - if: '$POPULATE_CACHE'
      when: never
    - when: always
  script:
    - .gitlab/scripts/post-benchmark-pr-comment.sh
  variables:
    UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME
    UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME
    BP_HEADER: "Benchmarks"
    BP_ON_DUPLICATE: "replace"