diff --git a/.github/workflows/create-test-mirror-pr.yml b/.github/workflows/create-test-mirror-pr.yml new file mode 100644 index 0000000..dc1ead8 --- /dev/null +++ b/.github/workflows/create-test-mirror-pr.yml @@ -0,0 +1,116 @@ +name: Create test image mirror PR + +on: + workflow_dispatch: + inputs: + pr_number: + description: "PR number in dd-trace-java-docker-build (e.g. 123)" + required: true + +jobs: + create-test-mirror-pr: + runs-on: ubuntu-latest + permissions: + id-token: write # Required for OIDC token federation + contents: read + pull-requests: write + steps: + - uses: DataDog/dd-octo-sts-action@acaa02eee7e3bb0839e4272dacb37b8f3b58ba80 # v1.0.3 + id: octo-sts + with: + scope: DataDog/images + policy: dd-trace-java-docker-build.update-mirror + + - name: Checkout DataDog/dd-trace-java-docker-build + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: dd-trace-java-docker-build + + - name: Checkout DataDog/images + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: DataDog/images + token: ${{ steps.octo-sts.outputs.token }} + path: images + + - name: Capture images HEAD SHA + id: images-head + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + working-directory: images + + - name: Install crane + run: | + CRANE_VERSION="0.20.2" + curl -fsSL "https://github.com/google/go-containerregistry/releases/download/v${CRANE_VERSION}/go-containerregistry_Linux_x86_64.tar.gz" -o crane.tar.gz + tar -xzf crane.tar.gz crane + sudo mv crane /usr/local/bin/crane + rm crane.tar.gz + + - name: Resolve digests and add new or update existing digests in mirror files + id: update-mirror + env: + PR_NUMBER: ${{ github.event.inputs.pr_number }} + run: bash "${GITHUB_WORKSPACE}/dd-trace-java-docker-build/scripts/create-test-mirror-entries.sh" + working-directory: images + + - name: Define branch name + id: define-branch + run: echo "branch=ci/add-dd-trace-java-docker-build-test-images-pr${{ github.event.inputs.pr_number }}" >> "$GITHUB_OUTPUT" + + - name: Commit changes + id: create-commit + env: + PR_NUMBER: ${{ github.event.inputs.pr_number }} + MODE: ${{ steps.update-mirror.outputs.mode }} + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add mirror.yaml mirror.lock.yaml + if git diff --cached --quiet; then + echo "No changes detected in mirror files; skipping commit." + echo "has_changes=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + git commit -m "chore: Update dd-trace-java-docker-build test image digests for PR #${PR_NUMBER}" + echo "has_changes=true" >> "$GITHUB_OUTPUT" + echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + working-directory: images + + - name: Push changes + if: ${{ steps.create-commit.outputs.has_changes == 'true' }} + uses: DataDog/commit-headless@05d7b7ee023e2c7d01c47832d420c2503cd416f3 # action/v2.0.3 + with: + token: "${{ steps.octo-sts.outputs.token }}" + branch: "${{ steps.define-branch.outputs.branch }}" + head-sha: "${{ steps.images-head.outputs.sha }}" + create-branch: true + command: push + commits: "${{ steps.create-commit.outputs.commit }}" + working-directory: images + + - name: Create pull request + id: images-pr + if: ${{ steps.create-commit.outputs.has_changes == 'true' }} + env: + GH_TOKEN: ${{ steps.octo-sts.outputs.token }} + PR_NUMBER: ${{ github.event.inputs.pr_number }} + run: | + PR_URL=$(gh pr create \ + --repo DataDog/images \ + --draft \ + --title "Update dd-trace-java-docker-build test images for PR #${PR_NUMBER}" \ + --base master \ + --head "${{ steps.define-branch.outputs.branch }}" \ + --body "Adds/updates mirror entries for \`${PR_NUMBER}_merge-*\` test images from DataDog/dd-trace-java-docker-build#${PR_NUMBER}. These images should be removed after testing.") + echo "pr_url=${PR_URL}" >> "$GITHUB_OUTPUT" + + - name: Comment on source PR with mirror cleanup reminder + if: ${{ steps.update-mirror.outputs.mode == 'add' && steps.create-commit.outputs.has_changes == 'true' }} + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.inputs.pr_number }} + IMAGES_PR_URL: ${{ steps.images-pr.outputs.pr_url }} + run: | + gh pr comment "${PR_NUMBER}" \ + --repo DataDog/dd-trace-java-docker-build \ + --body "Mirrored test images for \`${PR_NUMBER}_merge-*\` were added in ${IMAGES_PR_URL}. When you've finished validating the image, please remove the mirrored test images." diff --git a/.github/workflows/docker-tag.yml b/.github/workflows/docker-tag.yml index e891e73..0645cc5 100644 --- a/.github/workflows/docker-tag.yml +++ b/.github/workflows/docker-tag.yml @@ -1,4 +1,4 @@ -name: Tag new images version +name: Tag new images version # triggers update-mirror-digests workflow on: schedule: # Quarterly schedule, roughly aligned with JDK CPU diff --git a/.github/workflows/update-mirror-digests.yml b/.github/workflows/update-mirror-digests.yml new file mode 100644 index 0000000..5b5ce8a --- /dev/null +++ b/.github/workflows/update-mirror-digests.yml @@ -0,0 +1,119 @@ +name: Update mirror digests for ci-* images + +on: + workflow_run: + workflows: ["Tag new images version"] + types: [completed] + workflow_dispatch: + +jobs: + update-mirror-digests: + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + permissions: + id-token: write # Required for OIDC token federation + contents: read + steps: + - uses: DataDog/dd-octo-sts-action@acaa02eee7e3bb0839e4272dacb37b8f3b58ba80 # v1.0.3 + id: octo-sts + with: + scope: DataDog/images + policy: dd-trace-java-docker-build.update-mirror + + - name: Checkout DataDog/dd-trace-java-docker-build + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: dd-trace-java-docker-build + + - name: Checkout DataDog/images + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: DataDog/images + token: ${{ steps.octo-sts.outputs.token }} + path: images + + - name: Capture images HEAD SHA + id: images-head + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + working-directory: images + + - name: Install crane + run: | + CRANE_VERSION="0.20.2" + curl -fsSL "https://github.com/google/go-containerregistry/releases/download/v${CRANE_VERSION}/go-containerregistry_Linux_x86_64.tar.gz" -o crane.tar.gz + tar -xzf crane.tar.gz crane + sudo mv crane /usr/local/bin/crane + rm crane.tar.gz + + - name: Get baseline digest for ci-base image # base variant used to check freshness + id: baseline + run: | + BASELINE=$(awk '/source:.*dd-trace-java-docker-build:ci-base/{found=1; next} found && /digest:/{print $2; exit}' images/mirror.lock.yaml || true) + echo "digest=${BASELINE}" >> "$GITHUB_OUTPUT" + echo "Baseline ci-base digest: ${BASELINE:-}" + + - name: Wait for new ci-base image to be published + run: | + BASELINE="${{ steps.baseline.outputs.digest }}" + DEADLINE=$((SECONDS + 1800)) + echo "Waiting for ci-base digest to differ from: ${BASELINE:-}" + while [[ $SECONDS -lt $DEADLINE ]]; do + CURRENT=$(crane digest ghcr.io/datadog/dd-trace-java-docker-build:ci-base 2>/dev/null || true) + if [[ -n "$CURRENT" && "$CURRENT" != "$BASELINE" ]]; then + echo "New ci-base digest detected: $CURRENT" + exit 0 + fi + echo "No change yet (current: ${CURRENT:-unavailable}), retrying in 60s..." + sleep 60 + done + echo "::error::Timeout after 30 minutes: ci-base digest did not change from existing mirror" + exit 1 + + - name: Resolve digests and update mirror.lock.yaml files + run: bash "${GITHUB_WORKSPACE}/dd-trace-java-docker-build/scripts/update-ci-image-digests.sh" + working-directory: images + + - name: Define branch name + id: define-branch + run: echo "branch=ci/update-dd-trace-java-docker-build-ci-digests-$(date +'%Y%m%d')" >> "$GITHUB_OUTPUT" + + - name: Commit changes + id: create-commit + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add mirror.lock.yaml + if git diff --cached --quiet; then + echo "No changes detected in mirror files; skipping commit." + echo "has_changes=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + git commit -m "chore: Update dd-trace-java-docker-build ci-* image digests" + echo "has_changes=true" >> "$GITHUB_OUTPUT" + echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + working-directory: images + + - name: Push changes + if: ${{ steps.create-commit.outputs.has_changes == 'true' }} + uses: DataDog/commit-headless@05d7b7ee023e2c7d01c47832d420c2503cd416f3 # action/v2.0.3 + with: + token: "${{ steps.octo-sts.outputs.token }}" + branch: "${{ steps.define-branch.outputs.branch }}" + head-sha: "${{ steps.images-head.outputs.sha }}" + create-branch: true + command: push + commits: "${{ steps.create-commit.outputs.commit }}" + working-directory: images + + - name: Create pull request + if: ${{ steps.create-commit.outputs.has_changes == 'true' }} + env: + GH_TOKEN: ${{ steps.octo-sts.outputs.token }} + run: | + gh pr create \ + --repo DataDog/images \ + --draft \ + --title "Update dd-trace-java-docker-build ci-* image digests" \ + --base master \ + --head "${{ steps.define-branch.outputs.branch }}" \ + --body "Automated digest update for \`dd-trace-java-docker-build\` \`ci-*\` images after tagging." diff --git a/README.md b/README.md index 0c97229..1f4336a 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,17 @@ # dd-trace-java-docker-build -This repository holds Docker images for continuous integration jobs at [dd-trace-java](https://github.com/datadog/dd-trace-java). +This repository holds the original Docker images for continuous integration jobs in [dd-trace-java](https://github.com/datadog/dd-trace-java). The images built here are mirrored into `registry.ddbuild.io` before use in `dd-trace-java` CI. This is to ensure that all CI images are properly signed. See [DataDog/images image-mirroring](https://github.com/DataDog/images#image-mirroring) for more details. ## Usage -Pre-built images are available in [GitHub Container Registry](https://github.com/DataDog/dd-trace-java-docker-build/pkgs/container/dd-trace-java-docker-build). +Pre-built images are available in the [GitHub Container Registry](https://github.com/DataDog/dd-trace-java-docker-build/pkgs/container/dd-trace-java-docker-build). Image variants are available on a per JDK basis: -- The `base` variant and its aliases, `8`, `11`, `17`, `21`, `25`, and `tip`, contain the base Eclipse Temurin JDK 8, 11, 17, 21, 25, and tip JDK version releases, -- The `zulu8`, `zulu11`, `oracle8`, `ibm8`, `semeru8`, `semeru11`, `semeru17`, `graalvm17`, `graalvm21`, and `graalvm25` variants all contain the base JDKs in addition to the specific JDK from their name, -- The `latest` variant contains the base JDKs and all the above specific JDKs. +- The `base` variant and its aliases — `8`, `11`, `17`, `21`, `25`, and `tip` — contain the base Eclipse Temurin JDK 8, 11, 17, 21, 25, and tip JDK version releases. +- The `zulu8`, `zulu11`, `oracle8`, `ibm8`, `semeru8`, `semeru11`, `semeru17`, `graalvm17`, `graalvm21`, and `graalvm25` variants all contain the base JDKs in addition to the specific JDK from their name. +- The `latest` variant contains the base JDKs and all of the specific JDKs above. -Images are tagged via the [Tag new images version](https://github.com/DataDog/dd-trace-java-docker-build/actions/workflows/docker-tag.yml) workflow. This workflow tags the latest images built from the specified branch with a `ci-` prefix. It runs quarterly on `master` but can also be triggered manually as needed. +Images are tagged via the [Tag new images version](https://github.com/DataDog/dd-trace-java-docker-build/actions/workflows/docker-tag.yml) workflow. This workflow tags the latest images built from the specified branch with a `ci-` prefix and triggers the [Update mirror digests for ci-* images](https://github.com/DataDog/dd-trace-java-docker-build/actions/workflows/update-mirror-digests.yml) workflow which automatically creates a PR that updates the pinned `ci-*` image digests in `DataDog/images`. It runs quarterly on `master` but can also be triggered manually as needed. ## Development @@ -26,3 +26,16 @@ And then check the built images: ```bash ./build --test ``` + +## Testing + +Images are built per PR for ease in testing. These test images are prefixed with `N_merge-`, where N is the PR number. See the [GitHub Container Registry](https://github.com/DataDog/dd-trace-java-docker-build/pkgs/container/dd-trace-java-docker-build) for examples. + +To test these images in `dd-trace-java` CI: + +1. Open a PR in [DataDog/dd-trace-java-docker-build](https://github.com/DataDog/dd-trace-java-docker-build) with the changes you want to test. Let's say these changes are made in PR #123 ([example](https://github.com/DataDog/dd-trace-java-docker-build/pull/123)). +2. Run the [Create test image mirror PR](https://github.com/DataDog/dd-trace-java-docker-build/actions/workflows/create-test-mirror-pr.yml) workflow with `PR_NUMBER=123`. This automatically opens a PR in [DataDog/images](https://github.com/DataDog/images) that adds mirror entries for the `123_merge-*` test images. Merge the PR if not done automatically by the `dd-prapprover` bot. +3. Open a PR in [DataDog/dd-trace-java](https://github.com/DataDog/dd-trace-java) that sets `BUILDER_IMAGE_VERSION_PREFIX: "123_merge-"` in `.gitlab-ci.yml`. Here, you can check your test images with `DataDog/dd-trace-java` CI. +4. Every time you want to test changes made in PR #123, ensure the test image SHAs in `DataDog/images` are updated. This is done by running the [Create test image mirror PR](https://github.com/DataDog/dd-trace-java-docker-build/actions/workflows/create-test-mirror-pr.yml) workflow each time with `PR_NUMBER=123`. +5. When the test images look good and `DataDog/dd-trace-java` CI is green, merge your `DataDog/dd-trace-java-docker-build` PR #123, close the test `DataDog/dd-trace-java` PR, and **remove the test images from the `DataDog/images` repo**. +6. Finally, run the [Tag new images version](https://github.com/DataDog/dd-trace-java-docker-build/actions/workflows/docker-tag.yml) workflow. The [Update mirror digests for ci-* images](https://github.com/DataDog/dd-trace-java-docker-build/actions/workflows/update-mirror-digests.yml) workflow will automatically open a PR in `DataDog/images`, updating the pinned `ci-*` digests. `dd-trace-java` CI should automatically pick up these updated images a few minutes after the PR is merged. diff --git a/scripts/create-test-mirror-entries.sh b/scripts/create-test-mirror-entries.sh new file mode 100755 index 0000000..6993a69 --- /dev/null +++ b/scripts/create-test-mirror-entries.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# create-test-mirror-entries.sh — add or update {PR_NUMBER}_merge-* test image entries +# in mirror.yaml and mirror.lock.yaml files in the DataDog/images repo. +# +# This script is called in the create-test-mirror-pr Github workflow. +# It must be run from the root of DataDog/images and requires crane to be installed. +# +# Required env var: +# PR_NUMBER — pull request number in dd-trace-java-docker-build (numeric) +# +# Outputs (when GITHUB_OUTPUT is set): +# mode=add|update, indicating whether the test images were added or their digests were updated + +set -euo pipefail + +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if ! [[ "${PR_NUMBER}" =~ ^[0-9]+$ ]]; then + echo "::error::PR_NUMBER must be numeric (got: '${PR_NUMBER}')" >&2 + exit 1 +fi + +readonly PREFIX="${PR_NUMBER}_merge-" + +# Check if entries already exist in mirror.yaml (use base variant as tester) +if grep -qF "ghcr.io/datadog/dd-trace-java-docker-build:${PREFIX}base" mirror.yaml; then + MODE="update" + echo "Entries for '${PREFIX}' already exist — updating digests only" +else + MODE="add" + echo "No entries found for '${PREFIX}' — adding new entries" +fi + +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + echo "mode=${MODE}" >> "${GITHUB_OUTPUT}" +fi + +# shellcheck source=scripts/get-image-digests.sh +source "${SCRIPT_DIR}/get-image-digests.sh" + +if [[ "$MODE" == "add" ]]; then + for variant in "${CI_VARIANTS[@]}"; do + tag="${PREFIX}${variant}" + printf ' - source: "%s:%s"\n dest:\n repo: "dd-trace-java-docker-build"\n tag: "%s"\n replication_target: ""\n' \ + "ghcr.io/datadog/dd-trace-java-docker-build" "${tag}" "${tag}" >> mirror.yaml + done + echo "Appended ${#CI_VARIANTS[@]} entries to mirror.yaml" + + for variant in "${CI_VARIANTS[@]}"; do + tag="${PREFIX}${variant}" + printf ' - source: %s:%s\n digest: %s\n' \ + "ghcr.io/datadog/dd-trace-java-docker-build" "${tag}" "${DIGESTS[$variant]}" >> mirror.lock.yaml + done + echo "Appended ${#CI_VARIANTS[@]} entries to mirror.lock.yaml" +else + for variant in "${CI_VARIANTS[@]}"; do + tag="${PREFIX}${variant}" + update_digest "${tag}" "${DIGESTS[$variant]}" mirror.lock.yaml + echo "Updated mirror.lock.yaml: ${tag}" + done +fi diff --git a/scripts/get-image-digests.sh b/scripts/get-image-digests.sh new file mode 100755 index 0000000..474d0ad --- /dev/null +++ b/scripts/get-image-digests.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# get-image-digests.sh — source this script to populate CI_VARIANTS and DIGESTS. +# +# Required env vars: +# PREFIX — tag prefix (e.g. "ci-" or "138_merge-") +# +# After sourcing, callers have access to: +# CI_VARIANTS — indexed array of variant names +# DIGESTS — associative array mapping image variant to its latest digest (sha256:...) + +set -euo pipefail + +readonly CI_VARIANTS=(base 7 8 11 17 21 25 tip zulu8 zulu11 oracle8 ibm8 semeru8 semeru11 semeru17 graalvm17 graalvm21 graalvm25) + +# update_digest TAG DIGEST FILE +# Finds the line "source: ...:TAG" and updates the digest on the following line. +update_digest() { + local tag="$1" digest="$2" file="$3" + awk -v tag="${tag}" -v digest="${digest}" ' + $0 ~ ("source:.*:" tag "$") { found=1 } + found && /digest:/ { sub(/digest: sha256:[a-f0-9]*/, "digest: " digest); found=0 } + { print } + ' "${file}" > "${file}.tmp" && mv "${file}.tmp" "${file}" +} + +echo "Resolving digests for ${#CI_VARIANTS[@]} variants (prefix: '${PREFIX}')..." >&2 +declare -A DIGESTS +for variant in "${CI_VARIANTS[@]}"; do + tag="${PREFIX}${variant}" + echo -n " ${tag} ... " >&2 + DIGESTS[$variant]="$(crane digest "ghcr.io/datadog/dd-trace-java-docker-build:${tag}")" + echo "${DIGESTS[$variant]}" >&2 +done diff --git a/scripts/update-ci-image-digests.sh b/scripts/update-ci-image-digests.sh new file mode 100755 index 0000000..e64d779 --- /dev/null +++ b/scripts/update-ci-image-digests.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# update-ci-image-digests.sh — update pinned ci-* image digests in mirror.lock.yaml in the DataDog/images repo. +# +# This script is called in the update-mirror-digests Github workflow. +# It must be run from the root of DataDog/images and requires crane to be installed. + +set -euo pipefail + +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Verify all ci-* entries exist before updating +PREFIX="ci-" +# shellcheck source=scripts/get-image-digests.sh +source "${SCRIPT_DIR}/get-image-digests.sh" + +for variant in "${CI_VARIANTS[@]}"; do + if ! grep -qF "ghcr.io/datadog/dd-trace-java-docker-build:ci-${variant}" mirror.lock.yaml; then + echo "::error::Missing from mirror.lock.yaml: ci-${variant}" >&2 + echo "Bootstrap ci-* entries manually before running this workflow." >&2 + exit 1 + fi +done + +# Update existing digest entries in mirror.lock.yaml in-place +for variant in "${CI_VARIANTS[@]}"; do + tag="ci-${variant}" + update_digest "${tag}" "${DIGESTS[$variant]}" mirror.lock.yaml + echo "Updated mirror.lock.yaml: ${tag}" +done