Merge branch 'master' into dougqh/avoid-metric-boxing

Author: mcculls

.github/chainguard/self.pin-system-tests.create-pr.sts.yaml

@@ -1,11 +1,11 @@
 issuer: https://token.actions.githubusercontent.com
 
-subject_pattern: repo:DataDog/dd-trace-java:ref:refs/heads/(master|release/v.+)
+subject_pattern: repo:DataDog/dd-trace-java:ref:refs/(heads/master|tags/v\d+\.\d+\.0)
 
 claim_pattern:
-  event_name: (create|workflow_dispatch)
-  ref: refs/heads/(master|release/v.+)
-  job_workflow_ref: DataDog/dd-trace-java/\.github/workflows/pin-system-tests\.yaml@refs/heads/(master|release/v.+)
+  event_name: (workflow_dispatch|push)
+  ref: refs/(heads/master|tags/v\d+\.\d+\.0)
+  job_workflow_ref: DataDog/dd-trace-java/\.github/workflows/create-release-branch\.yaml@refs/heads/master
 
 permissions:
   contents: write

.github/workflows/README.md

@@ -53,9 +53,9 @@ This redirects GitHub's "Merge when ready" button to the Datadog merge queue sys
 
 _Trigger:_ When a git tag matching the pattern "vM.N.0" is pushed (e.g. for a minor release).
 
-_Action:_ Create a release branch that corresponds to the pushed tag (e.g. "release/vM.N.x").
+_Action:_ Create a release branch that corresponds to the pushed tag (e.g. "release/vM.N.x"). Then open a PR against the release branch that pins system tests.
 
-_Recovery:_ Manually create the branch from the "vM.N.0" git tag.
+_Recovery:_ Manually create the release branch from the "vM.N.0" git tag, and pin system tests to this branch by running the `./tooling/update_system_test_reference.sh` script.
 
 ### draft-release-notes-on-tag [🔗](draft-release-notes-on-tag.yaml)
 

.github/workflows/create-release-branch.yaml

@@ -16,6 +16,8 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write # Allow pushing the release branch
+    outputs:
+      release-branch-name: ${{ steps.define-release-branch.outputs.branch }}
     steps:
       - name: Determine tag
         id: determine-tag
@@ -31,21 +33,21 @@ jobs:
           fi
           echo "tag=${TAG}" >> "$GITHUB_OUTPUT"
 
-      - name: Define branch name from tag
-        id: define-branch
+      - name: Define release branch name from tag
+        id: define-release-branch
         run: |
           TAG=${{ steps.determine-tag.outputs.tag }}
-          echo "branch=release/${TAG%.0}.x" >> "$GITHUB_OUTPUT"
+          echo "branch=test/${TAG%.0}.x" >> "$GITHUB_OUTPUT" # TODO: change back to release/ branch after testing
 
       - name: Checkout dd-trace-java at tag
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2
         with:
           ref: ${{ steps.determine-tag.outputs.tag }}
 
       - name: Check if branch already exists
-        id: check-branch
+        id: check-release-branch
         run: |
-          BRANCH=${{ steps.define-branch.outputs.branch }}
+          BRANCH=${{ steps.define-release-branch.outputs.branch }}
           if git ls-remote --heads origin "$BRANCH" | grep -q "$BRANCH"; then
             echo "creating_new_branch=false" >> "$GITHUB_OUTPUT"
             echo "Branch $BRANCH already exists - skipping creation"
@@ -55,7 +57,88 @@ jobs:
           fi
 
       - name: Create and push release branch
-        if: steps.check-branch.outputs.creating_new_branch == 'true'
+        if: steps.check-release-branch.outputs.creating_new_branch == 'true'
         run: |
-          git checkout -b "${{ steps.define-branch.outputs.branch }}"
-          git push -u origin "${{ steps.define-branch.outputs.branch }}"
+          git checkout -b "${{ steps.define-release-branch.outputs.branch }}"
+          git push -u origin "${{ steps.define-release-branch.outputs.branch }}"
+
+  pin-system-tests:
+    needs: create-release-branch
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      id-token: write # required for OIDC token federation
+    steps:
+      - uses: DataDog/dd-octo-sts-action@acaa02eee7e3bb0839e4272dacb37b8f3b58ba80 # v1.0.3
+        id: octo-sts
+        with:
+          scope: DataDog/dd-trace-java
+          policy: self.pin-system-tests.create-pr
+
+      - name: Checkout dd-trace-java at release branch
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2
+        with:
+          ref: ${{ needs.create-release-branch.outputs.release-branch-name }}
+      
+      - name: Get latest commit SHA of base release branch
+        id: get-latest-commit-sha
+        run: |
+          echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Define pin-system-tests branch name
+        id: define-pin-branch
+        run: echo "branch=ci/pin-system-tests-$(date +'%Y%m%d')" >> $GITHUB_OUTPUT
+
+      - name: Check if pin-system-tests branch already exists
+        id: check-pin-branch
+        run: |
+          BRANCH=${{ steps.define-pin-branch.outputs.branch }}
+          if git ls-remote --heads origin "$BRANCH" | grep -q "$BRANCH"; then
+            echo "ERROR: Branch $BRANCH already exists - please delete it and re-run the workflow."
+            exit 1
+          else
+            echo "Branch $BRANCH does not exist - creating it now."
+          fi
+      
+      - name: Update system-tests references to latest commit SHA of system-tests main
+        run: ./tooling/update_system_test_reference.sh
+      
+      - name: Check if changes should be committed
+        id: check-changes
+        run: |
+          if [[ -z "$(git status -s)" ]]; then
+            echo "ERROR: No changes to commit - the system-tests reference was not updated."
+            exit 1
+          else
+            echo "Changes to commit:"
+            git status -s
+          fi
+      
+      - name: Commit changes
+        id: create-commit
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git commit -m "chore: Pin system-tests for release branch" .github/workflows/run-system-tests.yaml
+          echo "commit=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+      
+      - name: Push changes
+        uses: DataDog/commit-headless@05d7b7ee023e2c7d01c47832d420c2503cd416f3 # action/v2.0.3
+        with:
+          token: "${{ steps.octo-sts.outputs.token }}"
+          branch: "${{ steps.define-pin-branch.outputs.branch }}"
+          head-sha: "${{ steps.get-latest-commit-sha.outputs.sha }}"
+          create-branch: true
+          command: push
+          commits: "${{ steps.create-commit.outputs.commit }}"
+
+      - name: Create pull request
+        env:
+          GH_TOKEN: ${{ steps.octo-sts.outputs.token }}
+        run: |
+          gh pr create --title "Pin system tests for release branch" \
+            --base ${{ needs.create-release-branch.outputs.release-branch-name }} \
+            --head ${{ steps.define-pin-branch.outputs.branch }} \
+            --label "tag: dependencies" \
+            --label "tag: no release notes" \
+            --body "This PR pins the system-tests reference for the release branch."

.gitlab-ci.yml

@@ -146,11 +146,12 @@ default:
   stage: build
   variables:
     MAVEN_OPTS: "-Xms256M -Xmx1024M"
-    GRADLE_WORKERS: 2
-    GRADLE_MEM: 3G
-    KUBERNETES_CPU_REQUEST: 8
-    KUBERNETES_MEMORY_REQUEST: 10Gi
-    KUBERNETES_MEMORY_LIMIT: 10Gi
+    GRADLE_WORKERS: 6
+    GRADLE_MEMORY_MIN: 1G
+    GRADLE_MEMORY_MAX: 4G
+    KUBERNETES_CPU_REQUEST: 10
+    KUBERNETES_MEMORY_REQUEST: 20Gi
+    KUBERNETES_MEMORY_LIMIT: 20Gi
     CACHE_TYPE: "lib" #default
     FF_USE_FASTZIP: "true"
     CACHE_COMPRESSION_LEVEL: "slowest"
@@ -197,11 +198,11 @@ default:
     # replace maven central part by MAVEN_REPOSITORY_PROXY in .mvn/wrapper/maven-wrapper.properties
     - sed -i "s|https://repo.maven.apache.org/maven2/|$MAVEN_REPOSITORY_PROXY|g" .mvn/wrapper/maven-wrapper.properties
     - mkdir -p .mvn/caches
-    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms$GRADLE_MEM -Xmx$GRADLE_MEM -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
+    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms$GRADLE_MEMORY_MIN -Xmx$GRADLE_MEMORY_MAX -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
     - export GRADLE_ARGS=" --build-cache --stacktrace --no-daemon --parallel --max-workers=$GRADLE_WORKERS"
     - *normalize_node_index
-    # for weird reasons, gradle will always "chmod 700" the .gradle folder
-    # with Gitlab caching, .gradle is always owned by root and thus gradle's chmod invocation fails
+    # for weird reasons, Gradle will always "chmod 700" the .gradle folder
+    # with Gitlab caching, .gradle is always owned by root and thus Gradle's chmod invocation fails
     # This dance is a hack to have .gradle owned by the Gitlab runner user
     - gitlab_section_start "gradle-dance" "Fix .gradle directory permissions"
     - cp -r .gradle .gradle-copy
@@ -295,10 +296,6 @@ build_tests:
   variables:
     BUILD_CACHE_POLICY: push
     DEPENDENCY_CACHE_POLICY: pull
-    GRADLE_MEM: 4G
-    GRADLE_WORKERS: 3
-    KUBERNETES_MEMORY_REQUEST: 18Gi
-    KUBERNETES_MEMORY_LIMIT: 18Gi
   parallel:
     matrix:
       - GRADLE_TARGET: ":baseTest"
@@ -379,11 +376,7 @@ spotless:
   stage: tests
   needs: []
   variables:
-    # TODO: Latest version of spotless is failing with OOM on CI only.
-    # Setting 8G memory solving this issue, but we need to solve it eventually.
-    GRADLE_MEM: 8G
-    KUBERNETES_MEMORY_REQUEST: 18Gi
-    KUBERNETES_MEMORY_LIMIT: 18Gi
+    GRADLE_MEMORY_MAX: 6G
   script:
     - ./gradlew --version
     - ./gradlew spotlessCheck $GRADLE_ARGS
@@ -577,11 +570,6 @@ muzzle-dep-report:
   needs: [ build_tests ]
   stage: tests
   variables:
-    KUBERNETES_MEMORY_REQUEST: 20Gi
-    KUBERNETES_MEMORY_LIMIT: 20Gi
-    KUBERNETES_CPU_REQUEST: 10
-    GRADLE_WORKERS: 4
-    GRADLE_MEM: 3G
     GRADLE_PARAMS: "-PskipFlakyTests"
     CONTINUE_ON_FAILURE: "false"
     TESTCONTAINERS_CHECKS_DISABLE: "true"
@@ -607,7 +595,7 @@ muzzle-dep-report:
       export PROFILER_COMMAND="-XX:StartFlightRecording=settings=profile,filename=/tmp/${CI_JOB_NAME_SLUG}.jfr,dumponexit=true";
       fi
     - *prepare_test_env
-    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms$GRADLE_MEM -Xmx$GRADLE_MEM $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp -Djava.util.prefs.userRoot=/tmp/.java/.userPrefs-${CI_JOB_ID}' -Ddatadog.forkedMaxHeapSize=1024M -Ddatadog.forkedMinHeapSize=128M"
+    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms$GRADLE_MEMORY_MIN -Xmx$GRADLE_MEMORY_MAX $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp -Djava.util.prefs.userRoot=/tmp/.java/.userPrefs-${CI_JOB_ID}' -Ddatadog.forkedMinHeapSize=128M -Ddatadog.forkedMaxHeapSize=1024M"
     - ./gradlew --version
     - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PtestJvm=$testJvm -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS --continue || $CONTINUE_ON_FAILURE
   after_script:
@@ -698,7 +686,7 @@ test_inst:
     GRADLE_TARGET: ":instrumentationTest"
     CACHE_TYPE: "inst"
   parallel:
-    matrix: *test_matrix_6
+    matrix: *test_matrix_8
 
 test_inst_latest:
   extends: .test_job_with_test_agent

AGENTS.md

@@ -7,26 +7,20 @@ It ships ~120 integrations (~200 instrumentations) for tracing, profiling, AppSe
 
 ## Project layout
 
+See [ARCHITECTURE.md](ARCHITECTURE.md) for detailed module descriptions.
+
 ```
-dd-java-agent/            Main agent
-  instrumentation/        All auto-instrumentations (one dir per framework)
-  agent-bootstrap/        Bootstrap classloader classes
-  agent-builder/          Agent build & bytecode weaving
-  agent-tooling/          Shared tooling for instrumentations
-  agent-{product}/        Product-specific modules (ci-visibility, iast, profiling, debugger, llmobs, aiguard, ...)
-  appsec/                 Application Security (WAF, threat detection)
+dd-java-agent/            Main agent (shadow jar, instrumentations, product modules)
 dd-trace-api/             Public API & configuration constants
 dd-trace-core/            Core tracing engine (spans, propagation, writer)
-dd-trace-ot/              OpenTracing compatibility layer
+dd-trace-ot/              Legacy OpenTracing compatibility library
 internal-api/             Internal shared API across modules
+components/               Shared low-level components (context, environment, json)
 products/                 Sub-products (feature flagging, metrics)
 communication/            HTTP transport to Datadog Agent
-components/               Shared low-level components
 remote-config/            Remote configuration support
 telemetry/                Agent telemetry
 utils/                    Shared utility modules (config, time, socket, test, etc.)
-metadata/                 Supported configurations metadata & requirements
-benchmark/                Performance benchmarks
 dd-smoke-tests/           Smoke tests (real apps + agent)
 docs/                     Developer documentation (see below)
 ```
@@ -35,6 +29,7 @@ docs/                     Developer documentation (see below)
 
 | Topic | File |
 |---|---|
+| Architecture & design | [ARCHITECTURE.md](ARCHITECTURE.md) |
 | Building from source | [BUILDING.md](BUILDING.md) |
 | Contributing & PR guidelines | [CONTRIBUTING.md](CONTRIBUTING.md) |
 | How instrumentations work | [docs/how_instrumentations_work.md](docs/how_instrumentations_work.md) |