fix(ci): migrate benchmarks to benchmarking-platform trigger (#607)

Author: jbachorik

.github/chainguard/async-profiler-build.ci.sts.yaml

@@ -1,4 +1,4 @@
-# Allow java-profiler GitLab CI to publish reports and manage issues
+# Allow java-profiler GitLab CI to post reports
 issuer: https://gitlab.ddbuild.io
 
 subject_pattern: "project_path:DataDog/java-profiler:ref_type:branch:ref:.*"

.gitlab-ci.yml

@@ -31,6 +31,7 @@ stages:
   - integration-test
   - reliability
   - benchmarks
+  - post-benchmarks
   - fuzz
   - notify
 

.gitlab/benchmarks/.gitlab-ci.yml

@@ -1,20 +1,22 @@
 variables:
-  PREPARE_IMAGE: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:latest
   DD_OCTO_STS_IMAGE: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1
 
-.benchmark_job:
-  extends: .deploy-sa
+# Bridge job: triggers the BP pipeline and blocks until it completes.
+# Bridge jobs cannot appear in other jobs' needs: — downstream jobs use
+# stage ordering (post-benchmarks stage runs after benchmarks stage).
+benchmarks-trigger:
   stage: benchmarks
-  timeout: 6h
-  variables:
-    ITERATIONS: "${BENCHMARK_ITERATIONS:-1}"
-    MODES: "${BENCHMARK_MODES:-cpu,wall,alloc,memleak}"
+  # Bridge jobs cannot have before_script, so CANCELLED is checked via rules.
+  # interruptible: false prevents orphaning the BP downstream pipeline on push.
+  interruptible: false
   needs:
     - job: get-versions
       artifacts: true
     - job: deploy-artifact
       artifacts: false
   rules:
+    - if: '$CANCELLED == "true"'
+      when: never
     - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null'
       when: never
     - if: '$CI_PIPELINE_SOURCE == "schedule"'
@@ -24,113 +26,51 @@ variables:
     - if: '$CI_PIPELINE_SOURCE == "web"'
       when: manual
       allow_failure: true
-    # Run automatically and non-blocking on any other source (push/trigger/api/
-    # etc.) — mirrors the integration-test rules. The before_script CANCELLED
-    # gate skips branches with no open PR.
+    # Run automatically and non-blocking on any other source (push/trigger/api/etc.)
     - when: on_success
       allow_failure: true
-  script: |
-    # setup the env
-    export ARTIFACTS_DIR="$(pwd)/reports" && (mkdir "${ARTIFACTS_DIR}" || :)
-    export CANDIDATE_VERSION=${CURRENT_VERSION}
-    export BASELINE_VERSION=${PREVIOUS_VERSION}
-    export PLATFORM_DIR=".benchmarks/platform"
-
-    # check for missing candidate version
-    if [ -z "${CANDIDATE_VERSION}" ]; then echo "Missing candidate version. Skipping."; exit 0; fi
-
-    # fetch the common platform scripts
-    git -c url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf="https://github.com/DataDog/" \
-      clone --branch dd-trace-go https://github.com/DataDog/benchmarking-platform ${PLATFORM_DIR}
-
-    # apply the specific step scripts
-    cp -r .gitlab/benchmarks/steps/* ${PLATFORM_DIR}/steps/
-    chmod a+x ${PLATFORM_DIR}/steps/*
-
-    # check for mode validity
-    ${PLATFORM_DIR}/steps/check_modes.sh
-    if [ "$(cat .job_status)" == "SKIP" ]; then exit 0; fi
-
-    # run benchmarks
-    ${PLATFORM_DIR}/steps/capture-hardware-software-info.sh
-    ${PLATFORM_DIR}/steps/run-benchmarks.sh
-    ${PLATFORM_DIR}/steps/analyze-results.sh
-    ${PLATFORM_DIR}/steps/upload-results-to-s3.sh
-  parallel:
-    matrix:
-      - RUN_MODE: ["cpu", "wall", "alloc", "memleak", "cpu,wall", "memleak,alloc", "cpu,wall,alloc,memleak"]
-  artifacts:
-    when: always
-    name: "reports"
-    paths:
-      - reports/
-    expire_in: 3 months
-
-benchmarks-candidate-amd64:
-  extends: .benchmark_job
-  tags: ["arch:amd64"]
-  image: $BENCHMARK_IMAGE_AMD64
-
-benchmarks-candidate-aarch64:
-  extends: .benchmark_job
-  tags: ["arch:arm64"]
-  image: $BENCHMARK_IMAGE_ARM64
   variables:
-    KUBERNETES_MEMORY_REQUEST: 200Gi
-    KUBERNETES_MEMORY_LIMIT: 200Gi
+    CANDIDATE_VERSION: "${CURRENT_VERSION}"
+    BASELINE_VERSION: "${PREVIOUS_VERSION}"
+    BENCHMARK_ITERATIONS: "${BENCHMARK_ITERATIONS:-5}"
+    BENCHMARK_MODES: "${BENCHMARK_MODES:-cpu,wall,alloc,memleak}"
+    DDPROF_COMMIT_SHA: "${CI_COMMIT_SHA}"
+    DDPROF_COMMIT_BRANCH: "${CI_COMMIT_BRANCH}"
+    UPSTREAM_PROJECT_NAME: "java-profiler"
+    UPSTREAM_BRANCH: "${CI_COMMIT_BRANCH}"
+    UPSTREAM_PIPELINE_ID: "${CI_PIPELINE_ID}"
+  trigger:
+    project: DataDog/apm-reliability/benchmarking-platform
+    branch: java-profiler
+    strategy: depend
 
-post-benchmarks-pr-comment:
-  extends: .retry-config
-  stage: benchmarks
-  tags: ["arch:arm64"]
-  image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1
-  id_tokens:
-    DDOCTOSTS_ID_TOKEN:
-      aud: dd-octo-sts
-  needs:
-    - job: prepare:start
-      artifacts: true
-    - job: benchmarks-candidate-amd64
-      artifacts: true
-    - job: benchmarks-candidate-aarch64
-      artifacts: true
-  rules:
-    - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null'
-      when: never
-    - if: '$CI_PIPELINE_SOURCE == "schedule"'
-      when: never
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-      when: never
-    # Always run when the candidate jobs ran, regardless of source, so results
-    # are posted back to the PR.
-    - when: always
-  timeout: 5m
-  script:
-    - .gitlab/benchmarks/post-pr-comment.sh reports
-  allow_failure: true
 
 publish-benchmark-gh-pages:
-  stage: benchmarks
+  stage: post-benchmarks
   tags: ["arch:arm64"]
-  image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1
+  image: $DD_OCTO_STS_IMAGE
   id_tokens:
     DDOCTOSTS_ID_TOKEN:
       aud: dd-octo-sts
-  needs:
-    - job: benchmarks-candidate-amd64
-      artifacts: true
-    - job: benchmarks-candidate-aarch64
-      artifacts: true
+  # Serialize concurrent GH Pages pushes. publish-gh-pages.sh uses
+  # 'git push --force'; two concurrent pushes race and the slower one
+  # silently discards the faster one's history update.
+  resource_group: gh-pages-publish
   rules:
     - if: '$CI_PIPELINE_SOURCE == "schedule"'
       when: never
+    - if: '$CANCELLED == "true"'
+      when: never
+    - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null'
+      when: never
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "main"'
       when: always
-  timeout: 10m
+  timeout: 15m
   script:
-    - ./.gitlab/benchmarks/publish-gh-pages.sh
+    - mkdir -p reports
+    - .gitlab/benchmarks/download-bp-reports.sh reports
+    - ./.gitlab/benchmarks/publish-gh-pages.sh reports
   allow_failure: true
 
 include:
   - local: .gitlab/common.yml
-  - local: .gitlab/benchmarks/images.yml

.gitlab/benchmarks/download-bp-reports.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+# Downloads result JSONs from the BP downstream pipeline via the GitLab CI API.
+#
+# Requires only curl and python3 (stdlib) — no aws CLI, pip, or boto3 needed.
+# BP jobs already store artifacts in GitLab; this fetches them directly from
+# the downstream pipeline triggered by benchmarks-trigger.
+set -uo pipefail   # intentionally no -e: we handle errors explicitly
+
+DEST="${1:-reports}"
+mkdir -p "${DEST}"
+
+TMPDIR_LOCAL=$(mktemp -d)
+trap 'rm -rf "${TMPDIR_LOCAL}"' EXIT
+
+# ── helper: curl with explicit HTTP status checking ──────────────────────────
+# Usage: api_get <url> <output_file>
+# Returns 0 on 2xx, prints diagnostics and returns 1 otherwise.
+api_get() {
+  local url="$1" out="$2"
+  local http_code
+  http_code=$(curl -s -o "${out}" -w "%{http_code}" \
+    --header "JOB-TOKEN: ${CI_JOB_TOKEN}" "${url}")
+  if [[ "${http_code}" != 2* ]]; then
+    echo "  API ${url##*/}: HTTP ${http_code} — $(python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('message','?'))" < "${out}" 2>/dev/null || echo 'see above')"
+    return 1
+  fi
+  return 0
+}
+
+# ── 1. find the benchmarks-trigger bridge ────────────────────────────────────
+BRIDGES_FILE="${TMPDIR_LOCAL}/bridges.json"
+echo "Querying bridges for pipeline ${CI_PIPELINE_ID}…"
+if ! api_get \
+  "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/bridges" \
+  "${BRIDGES_FILE}"; then
+  echo "Cannot read pipeline bridges (job token may lack Reporter access) — skipping download"
+  exit 0
+fi
+
+read -r BP_PROJECT_ID DOWNSTREAM_PIPELINE_ID < <(python3 - "${BRIDGES_FILE}" <<'PYEOF'
+import json, sys
+with open(sys.argv[1]) as f:
+    bridges = json.load(f)
+for b in bridges:
+    if b.get("name") == "benchmarks-trigger":
+        dp = b.get("downstream_pipeline") or {}
+        if dp.get("id") and dp.get("project_id") and dp.get("status") == "success":
+            print(dp["project_id"], dp["id"])
+            sys.exit(0)
+print("", "")
+PYEOF
+)
+
+if [ -z "${DOWNSTREAM_PIPELINE_ID:-}" ]; then
+  echo "benchmarks-trigger bridge not found or did not run — skipping download"
+  exit 0
+fi
+echo "BP downstream pipeline: project=${BP_PROJECT_ID}  pipeline=${DOWNSTREAM_PIPELINE_ID}"
+
+# ── 2. list jobs in the downstream pipeline (paginated) ──────────────────────
+echo "Listing BP pipeline jobs…"
+JOB_IDS=""
+PAGE=1
+while true; do
+  JOBS_FILE="${TMPDIR_LOCAL}/jobs_page${PAGE}.json"
+  if ! api_get \
+    "${CI_API_V4_URL}/projects/${BP_PROJECT_ID}/pipelines/${DOWNSTREAM_PIPELINE_ID}/jobs?per_page=100&page=${PAGE}" \
+    "${JOBS_FILE}"; then
+    echo "Cannot list BP pipeline jobs — skipping download"
+    exit 0
+  fi
+  PAGE_IDS=$(python3 -c "
+import json, sys
+jobs = json.load(open(sys.argv[1]))
+print(' '.join(str(j['id']) for j in jobs))
+print(len(jobs), file=sys.stderr)
+" "${JOBS_FILE}" 2>"${TMPDIR_LOCAL}/page_count.txt")
+  JOB_IDS="${JOB_IDS} ${PAGE_IDS}"
+  PAGE_COUNT=$(cat "${TMPDIR_LOCAL}/page_count.txt")
+  if [ "${PAGE_COUNT}" -lt 100 ]; then
+    break
+  fi
+  PAGE=$((PAGE + 1))
+done
+
+# ── 3. download result_*.json from each job's artifact zip ───────────────────
+DOWNLOADED=0
+for JOB_ID in ${JOB_IDS}; do
+  ART_ZIP="${TMPDIR_LOCAL}/art_${JOB_ID}.zip"
+  ART_STATUS=$(curl -s -o "${ART_ZIP}" -w "%{http_code}" \
+    --header "JOB-TOKEN: ${CI_JOB_TOKEN}" \
+    "${CI_API_V4_URL}/projects/${BP_PROJECT_ID}/jobs/${JOB_ID}/artifacts" 2>/dev/null)
+  if [[ "${ART_STATUS}" == 2* ]]; then
+    # Contract with BP (DataDog/apm-reliability/benchmarking-platform#190):
+    # artifacts are stored under the "artifacts/" prefix with names matching
+    # "result_*.json". If BP renames either, this silently extracts nothing.
+    # -j: junk paths (strip artifacts/ prefix), -q: quiet, -o: overwrite
+    if unzip -q -j "${ART_ZIP}" "artifacts/result_*.json" -d "${DEST}/" 2>/dev/null; then
+      DOWNLOADED=$((DOWNLOADED + 1))
+    fi
+  fi
+done
+
+RESULT_COUNT=$(find "${DEST}" -name "result_*.json" | wc -l)
+echo "result_*.json files: ${RESULT_COUNT} (from ${DOWNLOADED} BP job(s))"
+
+if [ "${RESULT_COUNT}" -eq 0 ]; then
+  echo "WARNING: no result JSONs found — BP jobs may not have run or produced artifacts yet"
+  exit 1
+fi

.gitlab/benchmarks/post-pr-comment.sh

@@ -5,7 +5,7 @@
 # Usage: publish-gh-pages.sh [reports-dir]
 #
 # Updates benchmark history and regenerates GitHub Pages site.
-# Reports are available at: https://datadog.github.io/async-profiler-build/benchmarks/
+# Reports are available at: https://datadog.github.io/java-profiler/benchmarks/
 #
 # In CI: Uses Octo-STS for secure, short-lived GitHub tokens (no secrets needed)
 # Locally: Use 'devflow gitlab auth' for Octo-STS, or set GITHUB_TOKEN env var