Merge branch 'master' into smola/telemetry-logs-tags

Author: smola

.circleci/upload_ciapp.sh

@@ -20,6 +20,7 @@ java_prop() {
 # Upload test results to CI Visibility
 junit_upload() {
     # based on tracer implementation: https://github.com/DataDog/dd-trace-java/blob/master/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/TestDecorator.java#L55-L77
+    # Overwriting the tag with the GitHub repo URL instead of the GitLab one. Otherwise, some Test Optimization features won't work.
     DD_API_KEY=$1 \
         datadog-ci junit upload --service $SERVICE_NAME \
         --logs \
@@ -30,6 +31,7 @@ junit_upload() {
         --tags "os.architecture:$(java_prop os.arch)" \
         --tags "os.platform:$(java_prop os.name)" \
         --tags "os.version:$(java_prop os.version)" \
+        --tags "git.repository_url:https://github.com/DataDog/dd-trace-java" \
         ./results
 }
 

.github/workflows/README.md

@@ -115,16 +115,6 @@ _Action:_
 
 _Notes:_ Results are sent on both production and staging environments.
 
-### check-ci-pipelines [🔗](check-ci-pipelines.yaml)
-
-_Trigger:_ When opening or updating a PR.
-
-_Action:_ This action will check all other continuous integration jobs (Github action, Gitlab, CircleCi), and will fail if any of them fails.
-The purpose of this job is to be required for PR merges, achieving Green CI Policy.
-It got an `ignored` parameters to exclude some jobs if they are temprorary failing.
-
-_Recovery:_ Manually trigger the action on the desired branch.
-
 ### comment-on-submodule-update [🔗](comment-on-submodule-update.yaml)
 
 _Trigger:_ When creating a PR commits to `master` or a `release/*` branch with a Git Submodule update.

.github/workflows/analyze-changes.yaml

@@ -40,7 +40,7 @@ jobs:
             ${{ runner.os }}-gradle-
         
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18
+        uses: github/codeql-action/init@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0
         with:
           languages: 'java'
           build-mode: 'manual'
@@ -57,7 +57,7 @@ jobs:
             --build-cache --parallel --stacktrace --no-daemon --max-workers=4
 
       - name: Perform CodeQL Analysis and upload results to GitHub Security tab
-        uses: github/codeql-action/analyze@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18
+        uses: github/codeql-action/analyze@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0
 
   trivy:
     name: Analyze changes with Trivy
@@ -109,7 +109,7 @@ jobs:
           ls -laR "./workspace/.trivy"
 
       - name: Run Trivy security scanner
-        uses: aquasecurity/trivy-action@6c175e9c4083a92bbca2f9724c8a5e33bc2d97a5 # v0.30.0
+        uses: aquasecurity/trivy-action@76071ef0d7ec797419534a183b498b4d6366cf37 # v0.31.0
         with:
           scan-type: rootfs
           scan-ref: './workspace/.trivy/'
@@ -122,7 +122,7 @@ jobs:
           TRIVY_JAVA_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-java-db,public.ecr.aws/aquasecurity/trivy-java-db
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18
+        uses: github/codeql-action/upload-sarif@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0
         if: always()
         with:
           sarif_file: 'trivy-results.sarif'

.github/workflows/check-ci-pipelines.yml

@@ -62,3 +62,12 @@ jobs:
       scenarios_groups: tracer-release
       excluded_scenarios: CROSSED_TRACING_LIBRARIES,INTEGRATIONS_AWS,APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,PROFILING  # require AWS and datadog credentials 
       skip_empty_scenarios: true
+
+  # Ensure the main job is run to completion
+  check:
+    name: Check system tests success
+    runs-on: ubuntu-latest
+    needs:
+    - main
+    steps:
+    - run: exit 0

.github/workflows/run-system-tests.yaml

@@ -52,7 +52,7 @@ jobs:
           echo "::notice::Using Docker build image tag: ${TAG}"
       - name: Update the Docker build image in GitLab CI config
         run: |
-          sed -i 's|BUILDER_IMAGE_VERSION_PREFIX:.*|BUILDER_IMAGE_VERSION_PREFIX:"${{ steps.define-tag.outputs.tag }}-"|' .gitlab-ci.yml
+          sed -i '' -E 's|(BUILDER_IMAGE_VERSION_PREFIX:)[^#]*([#].*)|\1 "${{ steps.define-tag.outputs.tag }}-" \2|' .gitlab-ci.yml
       - name: Commit and push changes
         env:
           GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.github/workflows/update-docker-build-image.yaml

@@ -1,7 +1,4 @@
 include:
-  - project: DataDog/apm-reliability/libdatadog-build
-    ref: 0f677257308e1c379af490b754febfb40fa2c06d
-    file: templates/ci_authenticated_job.yml
   - local: ".gitlab/one-pipeline.locked.yml"
   - local: ".gitlab/benchmarks.yml"
   - local: ".gitlab/macrobenchmarks.yml"
@@ -20,14 +17,18 @@ stages:
   - generate-signing-key
 
 variables:
+  # Gitlab runner features; see https://docs.gitlab.com/runner/configuration/feature-flags.html
+  # Fold and time all script sections
+  FF_SCRIPT_SECTIONS: 1
+
   REGISTRY: 486234852809.dkr.ecr.us-east-1.amazonaws.com
   BUILD_JOB_NAME: "build"
   DEPENDENCY_CACHE_POLICY: pull
   BUILD_CACHE_POLICY: pull
   GRADLE_VERSION: "8.5" # must match gradle-wrapper.properties
   MAVEN_REPOSITORY_PROXY: "http://artifactual.artifactual.all-clusters.local-dc.fabric.dog:8081/repository/maven-central/"
   GRADLE_PLUGIN_PROXY: "http://artifactual.artifactual.all-clusters.local-dc.fabric.dog:8081/repository/gradle-plugin-portal-proxy/"
-  BUILDER_IMAGE_VERSION_PREFIX: "v25.05-"
+  BUILDER_IMAGE_VERSION_PREFIX: "v25.06-" # use either an empty string (e.g. "") for latest images or a version followed by a hyphen (e.g. "v25.05-")
   REPO_NOTIFICATION_CHANNEL: "#apm-java-escalations"
   DEFAULT_TEST_JVMS: /^(8|11|17|21)$/
   PROFILE_TESTS:
@@ -48,7 +49,6 @@ variables:
       - "21"
       - "semeru11"
       - "oracle8"
-      - "ubuntu17"
       - "zulu8"
       - "semeru8"
       - "ibm8"
@@ -96,6 +96,12 @@ default:
   - ONE_INDEXED_NODE_INDEX=${CI_NODE_INDEX:-1}; export NORMALIZED_NODE_INDEX=$((ONE_INDEXED_NODE_INDEX - 1))
   - echo "NORMALIZED_NODE_TOTAL=${NORMALIZED_NODE_TOTAL}, NORMALIZED_NODE_INDEX=$NORMALIZED_NODE_INDEX"
 
+.cgroup_info: &cgroup_info
+  - source .gitlab/gitlab-utils.sh
+  - gitlab_section_start "cgroup-info" "cgroup info"
+  - .gitlab/cgroup-info.sh
+  - gitlab_section_end "cgroup-info"
+
 .gradle_build: &gradle_build
   image: ghcr.io/datadog/dd-trace-java-docker-build:${BUILDER_IMAGE_VERSION_PREFIX}base
   stage: build
@@ -127,20 +133,31 @@ default:
       policy: $BUILD_CACHE_POLICY
   before_script:
     - source .gitlab/gitlab-utils.sh
-    - export GRADLE_USER_HOME=`pwd`/.gradle
+    - mkdir -p .gradle
+    - export GRADLE_USER_HOME=$(pwd)/.gradle
+    - |
+      # Don't put jvm args here as it will be picked up by child gradle processes used in tests
+      cat << EOF > $GRADLE_USER_HOME/gradle.properties
+      mavenRepositoryProxy=$MAVEN_REPOSITORY_PROXY
+      gradlePluginProxy=$GRADLE_PLUGIN_PROXY
+      EOF
+    - |
+      # replace maven central part by MAVEN_REPOSITORY_PROXY in .mvn/wrapper/maven-wrapper.properties
+      sed -i "s|https://repo.maven.apache.org/maven2/|$MAVEN_REPOSITORY_PROXY|g" .mvn/wrapper/maven-wrapper.properties
     - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx$GRADLE_MEM -Xms$GRADLE_MEM -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
-    - export GRADLE_ARGS=" --build-cache --stacktrace --no-daemon --parallel --max-workers=$GRADLE_WORKERS -PmavenRepositoryProxy=$MAVEN_REPOSITORY_PROXY -PgradlePluginProxy=$GRADLE_PLUGIN_PROXY"
+    - export GRADLE_ARGS=" --build-cache --stacktrace --no-daemon --parallel --max-workers=$GRADLE_WORKERS"
     - *normalize_node_index
     # for weird reasons, gradle will always "chmod 700" the .gradle folder
     # with Gitlab caching, .gradle is always owned by root and thus gradle's chmod invocation fails
     # This dance is a hack to have .gradle owned by the Gitlab runner user
     - gitlab_section_start "gradle-dance" "Fix .gradle directory permissions"
-    - mkdir -p .gradle
     - cp -r .gradle .gradle-copy
     - rm -rf .gradle
     - mv .gradle-copy .gradle
     - ls -la
     - gitlab_section_end "gradle-dance"
+  after_script:
+    - *cgroup_info
 
 build:
   extends: .gradle_build
@@ -218,6 +235,38 @@ populate_dep_cache:
 #      - GRADLE_TARGET: ":smokeTest"
 #        CACHE_TYPE: "smoke"
 
+publish-artifacts-to-s3:
+  image: registry.ddbuild.io/images/mirror/amazon/aws-cli:2.4.29
+  stage: publish
+  needs: [ build ]
+  script:
+    - source upstream.env
+    - export VERSION="${UPSTREAM_TRACER_VERSION%~*}" # remove ~githash from the end of version
+    - aws s3 cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-java-agent.jar
+    - aws s3 cp workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-trace-api.jar
+    - aws s3 cp workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-trace-ot.jar
+    - aws s3 cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-java-agent.jar
+    - aws s3 cp workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-trace-api.jar
+    - aws s3 cp workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-trace-ot.jar
+    - |
+      cat << EOF > links.json 
+        {
+          "S3 Links": [
+            {
+              "external_link": {
+                "label": "Public Link to dd-java-agent.jar",
+                "url": "https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${CI_PIPELINE_ID}/dd-java-agent.jar"
+              }
+            }
+          ]
+        }
+      EOF
+  artifacts:
+    reports:
+      annotations:
+        - links.json
+
+
 spotless:
   extends: .gradle_build
   stage: tests
@@ -244,6 +293,7 @@ test_published_artifacts:
     - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1G -Xms1G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
     - ./gradlew check --info $GRADLE_ARGS
   after_script:
+    - *cgroup_info
     - source .gitlab/gitlab-utils.sh
     - gitlab_section_start "collect-reports" "Collecting reports"
     - .circleci/collect_reports.sh
@@ -262,6 +312,7 @@ test_published_artifacts:
   script:
     - ./gradlew $GRADLE_TARGET -PskipTests -PrunBuildSrcTests -PskipSpotless -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS
   after_script:
+    - *cgroup_info
     - source .gitlab/gitlab-utils.sh
     - gitlab_section_start "collect-reports" "Collecting reports"
     - .circleci/collect_reports.sh --destination ./check_reports --move
@@ -320,8 +371,9 @@ muzzle:
     - ./gradlew writeMuzzleTasksToFile $GRADLE_ARGS
     - sort workspace/build/muzzleTasks > sortedMuzzleTasks
     - split --number=l/$NORMALIZED_NODE_TOTAL --suffix-length=1 --numeric-suffixes sortedMuzzleTasks muzzleSplit
-    - ./gradlew `cat muzzleSplit${NORMALIZED_NODE_INDEX} | xargs` $GRADLE_ARGS
+    - ./gradlew $(cat muzzleSplit${NORMALIZED_NODE_INDEX} | xargs) $GRADLE_ARGS
   after_script:
+    - *cgroup_info
     - source .gitlab/gitlab-utils.sh
     - gitlab_section_start "collect-reports" "Collecting reports"
     - .circleci/collect_reports.sh
@@ -342,6 +394,7 @@ muzzle-dep-report:
     - export SKIP_BUILDSCAN="true"
     - ./gradlew generateMuzzleReport muzzleInstrumentationReport $GRADLE_ARGS
   after_script:
+    - *cgroup_info
     - .circleci/collect_muzzle_deps.sh
   artifacts:
     when: always
@@ -401,6 +454,7 @@ muzzle-dep-report:
   after_script:
     - *restore_pretest_env
     - *set_datadog_api_keys
+    - *cgroup_info
     - source .gitlab/gitlab-utils.sh
     - gitlab_section_start "collect-reports" "Collecting reports"
     - .circleci/collect_reports.sh
@@ -553,7 +607,7 @@ test_smoke:
     GRADLE_PARAMS: "-PskipFlakyTests"
     CACHE_TYPE: "smoke"
   parallel:
-    matrix: *test_matrix_2
+    matrix: *test_matrix_4
 
 test_ssi_smoke:
   extends: .test_job
@@ -564,7 +618,7 @@ test_ssi_smoke:
     DD_INJECT_FORCE: "true"
     DD_INJECTION_ENABLED: "tracer"
   parallel:
-    matrix: *test_matrix_2
+    matrix: *test_matrix_4
 
 test_smoke_graalvm:
   extends: .test_job
@@ -742,37 +796,3 @@ create_key:
     expire_in: 13 mos
     paths:
       - pubkeys
-
-tracer-base-image-release:
-  extends: .ci_authenticated_job
-  stage: publish
-  needs: [ build ]
-  rules:
-    - if: '$POPULATE_CACHE'
-      when: never
-    - if: '$CI_COMMIT_TAG =~ /^v1\..*/'
-      when: on_success
-  dependencies:
-    - build
-  script:
-    - echo $GH_TOKEN|docker login ghcr.io/datadog -u uploader --password-stdin
-    - mkdir -p ./tooling/ci/binaries/ && cp workspace/dd-java-agent/build/libs/*.jar ./tooling/ci/binaries/dd-java-agent.jar
-    - docker buildx build -t ghcr.io/datadog/dd-trace-java/dd-trace-java:latest -f ./tooling/ci/Dockerfile .
-    - docker push ghcr.io/datadog/dd-trace-java/dd-trace-java:latest
-
-tracer-base-image-snapshot:
-  extends: .ci_authenticated_job
-  stage: publish
-  needs: [ build ]
-  rules:
-    - if: '$POPULATE_CACHE'
-      when: never
-    - if: '$CI_COMMIT_BRANCH == "master"'
-      when: on_success
-  dependencies:
-    - build
-  script:
-    - echo $GH_TOKEN|docker login ghcr.io/datadog -u uploader --password-stdin
-    - mkdir -p ./tooling/ci/binaries/ && cp workspace/dd-java-agent/build/libs/*.jar ./tooling/ci/binaries/dd-java-agent.jar
-    - docker buildx build -t ghcr.io/datadog/dd-trace-java/dd-trace-java:latest_snapshot -f ./tooling/ci/Dockerfile .
-    - docker push ghcr.io/datadog/dd-trace-java/dd-trace-java:latest_snapshot

.gitlab-ci.yml

@@ -4,7 +4,7 @@
   timeout: 1h
   tags: ["runner:apm-k8s-tweaked-metal"]
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-java-benchmarks
-  needs: [ "build" ]
+  needs: [ "build", "publish-artifacts-to-s3" ]
   rules:
     - if: '$POPULATE_CACHE'
       when: never
@@ -15,7 +15,7 @@
   script:
     - export ARTIFACTS_DIR="$(pwd)/reports" && mkdir -p "${ARTIFACTS_DIR}"
     - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
-    - git clone --branch dd-trace-java/tracer-benchmarks https://github.com/DataDog/benchmarking-platform.git /platform && cd /platform
+    - git clone --branch dd-trace-java/tracer-benchmarks-parallel https://github.com/DataDog/benchmarking-platform.git /platform && cd /platform
   artifacts:
     name: "reports"
     paths:
@@ -85,7 +85,7 @@ benchmarks-post-results:
   interruptible: true
   timeout: 1h
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:java-dsm-kafka
-  needs: [ "build" ]
+  needs: [ "build", "publish-artifacts-to-s3"]
   script:
     - git clone --branch java/kafka-dsm-overhead https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform.git platform && cd platform
     - ./steps/run-benchmarks.sh
@@ -129,7 +129,7 @@ debugger-benchmarks:
   interruptible: true
   timeout: 1h
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:java-debugger
-  needs: ["build"]
+  needs: ["build", "publish-artifacts-to-s3"]
   script:
     - export ARTIFACTS_DIR="$(pwd)/reports" && mkdir -p "${ARTIFACTS_DIR}"
     - git clone --branch java/debugger-benchmarks https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform.git /platform && cd /platform