Test all benchmarks now

Author: sarahchen6

.gitlab-ci.yml

@@ -7,18 +7,12 @@ include:
   - project: 'DataDog/apm-reliability/apm-sdks-benchmarks'
     file: '.gitlab/ci-java-spring-petclinic.yml'
     ref: 'main'
-    rules:
-      - when: never # disable while testing merge base logic
   - project: 'DataDog/apm-reliability/apm-sdks-benchmarks'
     file: '.gitlab/ci-java-insecure-bank.yml'
     ref: 'main'
-    rules:
-      - when: never # disable while testing merge base logic
   - project: 'DataDog/apm-reliability/apm-sdks-benchmarks'
     file: '.gitlab/ci-java-dacapo.yml'
     ref: 'main'
-    rules:
-      - when: never # disable while testing merge base logic
 
 stages:
   - build

.gitlab/benchmarks.yml

@@ -3,7 +3,6 @@
 .dsm-kafka-benchmarks:
   stage: benchmarks
   rules:
-    - when: never # disable while testing merge base logic
     - if: '$POPULATE_CACHE'
       when: never
     - if: $CI_PIPELINE_SOURCE != "schedule"
@@ -48,7 +47,6 @@ dsm-kafka-consumer-benchmark:
 debugger-benchmarks:
   stage: benchmarks
   rules:
-    - when: never # disable while testing merge base logic
     - if: '$POPULATE_CACHE'
       when: never
     - if: $CI_PIPELINE_SOURCE != "schedule"
@@ -89,7 +87,6 @@ debugger-benchmarks:
   tags: ["arch:amd64"]
   interruptible: true
   rules:
-    - when: never # disable while testing merge base logic
     - if: '$POPULATE_CACHE'
       when: never
     - when: on_success
@@ -139,7 +136,6 @@ benchmarks-post-pr-comment:
     - job: benchmarks-compare-dacapo
       artifacts: true
   rules:
-    - when: never # disable while testing merge base logic
     - if: '$POPULATE_CACHE'
       when: never
     - when: always
@@ -150,29 +146,3 @@ benchmarks-post-pr-comment:
     UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME
     BP_HEADER: "Benchmarks"
     BP_ON_DUPLICATE: "replace"
-
-# DEBUG MERGE BASE LOGIC IMMEDIATELY instead of waiting on all benchmarks to complete
-benchmarks-debug-baseline:
-  stage: benchmark-comparison
-  image: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:6845f3c7
-  tags: ["arch:amd64"]
-  interruptible: true
-  needs: []
-  rules:
-    - when: on_success
-  script:
-    - |
-      if [ "$(uname -m)" = "x86_64" ]; then AAA="amd64"; else AAA="arm64"; fi
-      wget -O /tmp/authanywhere "https://binaries.ddbuild.io/dd-source/authanywhere/LATEST/authanywhere-linux-${AAA}"
-      chmod +x /tmp/authanywhere
-      mv /tmp/authanywhere /usr/local/bin/authanywhere
-    - mkdir -p reports/debug
-    - .gitlab/scripts/get-baseline-commit-info.sh "reports/debug"
-    - echo "Resolved baseline:"
-    - cat reports/debug/baseline-info.env
-    - if [[ -f "reports/debug/fallback_to_master.txt" ]]; then cat "reports/debug/fallback_to_master.txt"; fi
-  artifacts:
-    when: always
-    paths:
-      - reports/debug/
-    expire_in: 1 week

.gitlab/scripts/benchmark-compare.sh

@@ -70,7 +70,7 @@ list_matching_jobs() {
   curl --silent --show-error --fail \
     --header "JOB-TOKEN: ${CI_JOB_TOKEN}" \
     "${JOBS_API_URL}/pipelines/${pipeline_id}/jobs?scope[]=success&per_page=100" \
-    | python3 - "${pattern}" <<'PY'
+    | python3 -c '
 import json
 import re
 import sys
@@ -82,8 +82,8 @@ for job in jobs:
         continue
     if not job.get("artifacts_file") or not job["artifacts_file"].get("filename"):
         continue
-    print(f'{job["id"]}|{job["name"]}')
-PY
+    print("{}|{}".format(job["id"], job["name"]))
+' "${pattern}"
 }
 
 download_job_artifacts() {
@@ -249,6 +249,11 @@ convert_dacapo_job() {
   local source_dir="$1"
   local side="$2"
   local variant="$3"
+  local dacapo_root="${source_dir}/dacapo"
+
+  if [[ ! -d "${dacapo_root}" ]]; then
+    return
+  fi
 
   while IFS= read -r benchmark_dir; do
     local benchmark_name
@@ -264,7 +269,7 @@ convert_dacapo_job() {
       "${benchmark_dir}"
 
     post_process_dacapo "${out_file}" "${benchmark_name}" "${variant}" "${side}"
-  done < <(find "${source_dir}" -type d -path '*/dacapo/*/*')
+  done < <(find "${dacapo_root}" -mindepth 2 -maxdepth 2 -type d)
 }
 
 convert_load_job() {
@@ -287,7 +292,7 @@ convert_load_job() {
   mkdir -p "${target_dir}"
 
   benchmark_analyzer merge \
-    --mergeby="['scenario', 'application', 'variant', 'baseline_or_candidate', 'git_branch', 'git_commit_sha', 'git_commit_date', 'cpu_model', 'kernel_version', 'ci_job_date', 'ci_job_id', 'ci_pipeline_id']" \
+    --mergeby="['scenario', 'application', 'variant', 'baseline_or_candidate', 'git_branch', 'git_commit_sha', 'git_commit_date', 'cpu_model', 'kernel_version']" \
     --outpath="${out_file}" \
     "${k6_files[@]}"
 

.gitlab/scripts/get-baseline-commit-info.sh

@@ -19,15 +19,6 @@ fi
 readonly PROJECT_API_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}"
 readonly BTI_TOKEN_URL="https://bti-ci-api.us1.ddbuild.io/internal/ci/gitlab/token?owner=DataDog&repository=apm-reliability/dd-trace-java"
 
-log_debug() {
-  echo "[baseline-debug] $*" >&2
-}
-
-response_snippet() {
-  local response="$1"
-  echo "${response}" | tr '\n' ' ' | sed 's/[[:space:]]\+/ /g' | cut -c1-220
-}
-
 get_private_token() {
   local auth_header bti_response http_status response_body private_token
 
@@ -45,7 +36,7 @@ get_private_token() {
   response_body="$(echo "${bti_response}" | sed '/HTTP_STATUS:/d')"
 
   if [[ "${http_status}" != "200" ]]; then
-    echo "BTI token request failed: status=${http_status}, body='$(response_snippet "${response_body}")'" >&2
+    echo "BTI token request failed with status ${http_status}." >&2
     return 1
   fi
 
@@ -63,17 +54,14 @@ get_pipeline_id_for_commit() {
   local api_url="${PROJECT_API_URL}/repository/commits/${commit_sha}"
   local response pipeline_id private_token
 
-  log_debug "query commit endpoint: ${api_url}"
   private_token="$(get_private_token)" || return 1
   response="$(curl --request GET --silent --show-error --header "PRIVATE-TOKEN: ${private_token}" "${api_url}" || true)"
   pipeline_id="$(echo "${response}" | grep -o '"last_pipeline"[^}]*"id":[0-9]*' | grep -o '[0-9]*$' | head -1 || true)"
   if [[ -n "${pipeline_id}" && "${pipeline_id}" != "null" ]]; then
-    log_debug "found pipeline_id=${pipeline_id} for commit_sha=${commit_sha}"
-    echo "${pipeline_id}"
+    echo "Pipeline ID for commit ${commit_sha}: ${pipeline_id}"
     return 0
   fi
 
-  log_debug "no last_pipeline.id for commit_sha=${commit_sha}; response='$(response_snippet "${response}")'"
   return 1
 }
 
@@ -82,16 +70,10 @@ get_latest_pipeline_id_for_branch() {
   local api_url="${PROJECT_API_URL}/pipelines?ref=${branch}&order_by=id&sort=desc&per_page=1"
   local response pipeline_id private_token
 
-  log_debug "query pipelines endpoint: ${api_url}"
   private_token="$(get_private_token)" || return 1
   response="$(curl --request GET --silent --show-error --header "PRIVATE-TOKEN: ${private_token}" "${api_url}" || true)"
   pipeline_id="$(echo "${response}" | grep -o '"id":[0-9]*' | head -1 | grep -o '[0-9]*' || true)"
-  if [[ -n "${pipeline_id}" ]]; then
-    log_debug "found latest pipeline_id=${pipeline_id} for branch=${branch}"
-  else
-    log_debug "no pipeline found for branch=${branch}; response='$(response_snippet "${response}")'"
-  fi
-  echo "${pipeline_id}"
+  echo "Pipeline ID for branch ${branch}: ${pipeline_id}"
 }
 
 resolve_merge_base_sha() {
@@ -110,19 +92,15 @@ BASELINE_PIPELINE_ID=""
 BASELINE_SOURCE="merge_base"
 FALLBACK_TO_MASTER="false"
 
-log_debug "resolved merge_base_sha='${MERGE_BASE_SHA:-<empty>}' target_branch='${TARGET_BRANCH}'"
-
 if [[ -n "${MERGE_BASE_SHA}" ]]; then
   BASELINE_PIPELINE_ID="$(get_pipeline_id_for_commit "${MERGE_BASE_SHA}" || true)"
 fi
 
 if [[ -z "${BASELINE_PIPELINE_ID}" ]]; then
   FALLBACK_TO_MASTER="true"
   BASELINE_SOURCE="${TARGET_BRANCH}"
-  log_debug "merge-base pipeline not found; falling back to branch='${TARGET_BRANCH}'"
 
   BASELINE_SHA="$(git rev-parse "origin/${TARGET_BRANCH}" 2>/dev/null || true)"
-  log_debug "resolved branch sha from local git: '${BASELINE_SHA:-<empty>}'"
   if [[ -n "${BASELINE_SHA}" ]]; then
     BASELINE_PIPELINE_ID="$(get_pipeline_id_for_commit "${BASELINE_SHA}" || true)"
   fi