Skip to content

Commit eccf277

Browse files
sarahchen6sarahchen6
authored
Merge pull request #139 from DataDog/sarahchen6/mirror-images
Add image mirroring support
2 parents 76b21cf + 8e3e105 commit eccf277

File tree

7 files changed

+378
-7
lines changed

7 files changed

+378
-7
lines changed

‎.github/workflows/create-test-mirror-pr.yml‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
name: Create test image mirror PR
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
pr_number:
7+
description: "PR number in dd-trace-java-docker-build (e.g. 123)"
8+
required: true
9+
10+
jobs:
11+
create-test-mirror-pr:
12+
runs-on: ubuntu-latest
13+
permissions:
14+
id-token: write # Required for OIDC token federation
15+
contents: read
16+
pull-requests: write
17+
steps:
18+
- uses: DataDog/dd-octo-sts-action@acaa02eee7e3bb0839e4272dacb37b8f3b58ba80 # v1.0.3
19+
id: octo-sts
20+
with:
21+
scope: DataDog/images
22+
policy: dd-trace-java-docker-build.update-mirror
23+
24+
- name: Checkout DataDog/dd-trace-java-docker-build
25+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
26+
with:
27+
path: dd-trace-java-docker-build
28+
29+
- name: Checkout DataDog/images
30+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
31+
with:
32+
repository: DataDog/images
33+
token: ${{ steps.octo-sts.outputs.token }}
34+
path: images
35+
36+
- name: Capture images HEAD SHA
37+
id: images-head
38+
run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
39+
working-directory: images
40+
41+
- name: Install crane
42+
run: |
43+
CRANE_VERSION="0.20.2"
44+
curl -fsSL "https://github.com/google/go-containerregistry/releases/download/v${CRANE_VERSION}/go-containerregistry_Linux_x86_64.tar.gz" -o crane.tar.gz
45+
tar -xzf crane.tar.gz crane
46+
sudo mv crane /usr/local/bin/crane
47+
rm crane.tar.gz
48+
49+
- name: Resolve digests and add new or update existing digests in mirror files
50+
id: update-mirror
51+
env:
52+
PR_NUMBER: ${{ github.event.inputs.pr_number }}
53+
run: bash "${GITHUB_WORKSPACE}/dd-trace-java-docker-build/scripts/create-test-mirror-entries.sh"
54+
working-directory: images
55+
56+
- name: Define branch name
57+
id: define-branch
58+
run: echo "branch=ci/add-dd-trace-java-docker-build-test-images-pr${{ github.event.inputs.pr_number }}" >> "$GITHUB_OUTPUT"
59+
60+
- name: Commit changes
61+
id: create-commit
62+
env:
63+
PR_NUMBER: ${{ github.event.inputs.pr_number }}
64+
MODE: ${{ steps.update-mirror.outputs.mode }}
65+
run: |
66+
git config user.name "github-actions[bot]"
67+
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
68+
git add mirror.yaml mirror.lock.yaml
69+
if git diff --cached --quiet; then
70+
echo "No changes detected in mirror files; skipping commit."
71+
echo "has_changes=false" >> "$GITHUB_OUTPUT"
72+
exit 0
73+
fi
74+
git commit -m "chore: Update dd-trace-java-docker-build test image digests for PR #${PR_NUMBER}"
75+
echo "has_changes=true" >> "$GITHUB_OUTPUT"
76+
echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
77+
working-directory: images
78+
79+
- name: Push changes
80+
if: ${{ steps.create-commit.outputs.has_changes == 'true' }}
81+
uses: DataDog/commit-headless@05d7b7ee023e2c7d01c47832d420c2503cd416f3 # action/v2.0.3
82+
with:
83+
token: "${{ steps.octo-sts.outputs.token }}"
84+
branch: "${{ steps.define-branch.outputs.branch }}"
85+
head-sha: "${{ steps.images-head.outputs.sha }}"
86+
create-branch: true
87+
command: push
88+
commits: "${{ steps.create-commit.outputs.commit }}"
89+
working-directory: images
90+
91+
- name: Create pull request
92+
id: images-pr
93+
if: ${{ steps.create-commit.outputs.has_changes == 'true' }}
94+
env:
95+
GH_TOKEN: ${{ steps.octo-sts.outputs.token }}
96+
PR_NUMBER: ${{ github.event.inputs.pr_number }}
97+
run: |
98+
PR_URL=$(gh pr create \
99+
--repo DataDog/images \
100+
--draft \
101+
--title "Update dd-trace-java-docker-build test images for PR #${PR_NUMBER}" \
102+
--base master \
103+
--head "${{ steps.define-branch.outputs.branch }}" \
104+
--body "Adds/updates mirror entries for \`${PR_NUMBER}_merge-*\` test images from DataDog/dd-trace-java-docker-build#${PR_NUMBER}. These images should be removed after testing.")
105+
echo "pr_url=${PR_URL}" >> "$GITHUB_OUTPUT"
106+
107+
- name: Comment on source PR with mirror cleanup reminder
108+
if: ${{ steps.update-mirror.outputs.mode == 'add' && steps.create-commit.outputs.has_changes == 'true' }}
109+
env:
110+
GH_TOKEN: ${{ github.token }}
111+
PR_NUMBER: ${{ github.event.inputs.pr_number }}
112+
IMAGES_PR_URL: ${{ steps.images-pr.outputs.pr_url }}
113+
run: |
114+
gh pr comment "${PR_NUMBER}" \
115+
--repo DataDog/dd-trace-java-docker-build \
116+
--body "Mirrored test images for \`${PR_NUMBER}_merge-*\` were added in ${IMAGES_PR_URL}. When you've finished validating the image, please remove the mirrored test images."

‎.github/workflows/docker-tag.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Tag new images version
1+
name: Tag new images version # triggers update-mirror-digests workflow
22
on:
33
schedule:
44
# Quarterly schedule, roughly aligned with JDK CPU

‎.github/workflows/update-mirror-digests.yml‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
name: Update mirror digests for ci-* images
2+
3+
on:
4+
workflow_run:
5+
workflows: ["Tag new images version"]
6+
types: [completed]
7+
workflow_dispatch:
8+
9+
jobs:
10+
update-mirror-digests:
11+
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
12+
runs-on: ubuntu-latest
13+
permissions:
14+
id-token: write # Required for OIDC token federation
15+
contents: read
16+
steps:
17+
- uses: DataDog/dd-octo-sts-action@acaa02eee7e3bb0839e4272dacb37b8f3b58ba80 # v1.0.3
18+
id: octo-sts
19+
with:
20+
scope: DataDog/images
21+
policy: dd-trace-java-docker-build.update-mirror
22+
23+
- name: Checkout DataDog/dd-trace-java-docker-build
24+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
25+
with:
26+
path: dd-trace-java-docker-build
27+
28+
- name: Checkout DataDog/images
29+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
30+
with:
31+
repository: DataDog/images
32+
token: ${{ steps.octo-sts.outputs.token }}
33+
path: images
34+
35+
- name: Capture images HEAD SHA
36+
id: images-head
37+
run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
38+
working-directory: images
39+
40+
- name: Install crane
41+
run: |
42+
CRANE_VERSION="0.20.2"
43+
curl -fsSL "https://github.com/google/go-containerregistry/releases/download/v${CRANE_VERSION}/go-containerregistry_Linux_x86_64.tar.gz" -o crane.tar.gz
44+
tar -xzf crane.tar.gz crane
45+
sudo mv crane /usr/local/bin/crane
46+
rm crane.tar.gz
47+
48+
- name: Get baseline digest for ci-base image # base variant used to check freshness
49+
id: baseline
50+
run: |
51+
BASELINE=$(awk '/source:.*dd-trace-java-docker-build:ci-base/{found=1; next} found && /digest:/{print $2; exit}' images/mirror.lock.yaml || true)
52+
echo "digest=${BASELINE}" >> "$GITHUB_OUTPUT"
53+
echo "Baseline ci-base digest: ${BASELINE:-<none found>}"
54+
55+
- name: Wait for new ci-base image to be published
56+
run: |
57+
BASELINE="${{ steps.baseline.outputs.digest }}"
58+
DEADLINE=$((SECONDS + 1800))
59+
echo "Waiting for ci-base digest to differ from: ${BASELINE:-<none>}"
60+
while [[ $SECONDS -lt $DEADLINE ]]; do
61+
CURRENT=$(crane digest ghcr.io/datadog/dd-trace-java-docker-build:ci-base 2>/dev/null || true)
62+
if [[ -n "$CURRENT" && "$CURRENT" != "$BASELINE" ]]; then
63+
echo "New ci-base digest detected: $CURRENT"
64+
exit 0
65+
fi
66+
echo "No change yet (current: ${CURRENT:-unavailable}), retrying in 60s..."
67+
sleep 60
68+
done
69+
echo "::error::Timeout after 30 minutes: ci-base digest did not change from existing mirror"
70+
exit 1
71+
72+
- name: Resolve digests and update mirror.lock.yaml files
73+
run: bash "${GITHUB_WORKSPACE}/dd-trace-java-docker-build/scripts/update-ci-image-digests.sh"
74+
working-directory: images
75+
76+
- name: Define branch name
77+
id: define-branch
78+
run: echo "branch=ci/update-dd-trace-java-docker-build-ci-digests-$(date +'%Y%m%d')" >> "$GITHUB_OUTPUT"
79+
80+
- name: Commit changes
81+
id: create-commit
82+
run: |
83+
git config user.name "github-actions[bot]"
84+
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
85+
git add mirror.lock.yaml
86+
if git diff --cached --quiet; then
87+
echo "No changes detected in mirror files; skipping commit."
88+
echo "has_changes=false" >> "$GITHUB_OUTPUT"
89+
exit 0
90+
fi
91+
git commit -m "chore: Update dd-trace-java-docker-build ci-* image digests"
92+
echo "has_changes=true" >> "$GITHUB_OUTPUT"
93+
echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
94+
working-directory: images
95+
96+
- name: Push changes
97+
if: ${{ steps.create-commit.outputs.has_changes == 'true' }}
98+
uses: DataDog/commit-headless@05d7b7ee023e2c7d01c47832d420c2503cd416f3 # action/v2.0.3
99+
with:
100+
token: "${{ steps.octo-sts.outputs.token }}"
101+
branch: "${{ steps.define-branch.outputs.branch }}"
102+
head-sha: "${{ steps.images-head.outputs.sha }}"
103+
create-branch: true
104+
command: push
105+
commits: "${{ steps.create-commit.outputs.commit }}"
106+
working-directory: images
107+
108+
- name: Create pull request
109+
if: ${{ steps.create-commit.outputs.has_changes == 'true' }}
110+
env:
111+
GH_TOKEN: ${{ steps.octo-sts.outputs.token }}
112+
run: |
113+
gh pr create \
114+
--repo DataDog/images \
115+
--draft \
116+
--title "Update dd-trace-java-docker-build ci-* image digests" \
117+
--base master \
118+
--head "${{ steps.define-branch.outputs.branch }}" \
119+
--body "Automated digest update for \`dd-trace-java-docker-build\` \`ci-*\` images after tagging."

‎README.md‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
11
# dd-trace-java-docker-build
22

3-
This repository holds Docker images for continuous integration jobs at [dd-trace-java](https://github.com/datadog/dd-trace-java).
3+
This repository holds the original Docker images for continuous integration jobs in [dd-trace-java](https://github.com/datadog/dd-trace-java). The images built here are mirrored into `registry.ddbuild.io` before use in `dd-trace-java` CI. This is to ensure that all CI images are properly signed. See [DataDog/images image-mirroring](https://github.com/DataDog/images#image-mirroring) for more details.
44

55
## Usage
66

7-
Pre-built images are available in [GitHub Container Registry](https://github.com/DataDog/dd-trace-java-docker-build/pkgs/container/dd-trace-java-docker-build).
7+
Pre-built images are available in the [GitHub Container Registry](https://github.com/DataDog/dd-trace-java-docker-build/pkgs/container/dd-trace-java-docker-build).
88

99
Image variants are available on a per JDK basis:
10-
- The `base` variant and its aliases, `8`, `11`, `17`, `21`, `25`, and `tip`, contain the base Eclipse Temurin JDK 8, 11, 17, 21, 25, and tip JDK version releases,
11-
- The `zulu8`, `zulu11`, `oracle8`, `ibm8`, `semeru8`, `semeru11`, `semeru17`, `graalvm17`, `graalvm21`, and `graalvm25` variants all contain the base JDKs in addition to the specific JDK from their name,
12-
- The `latest` variant contains the base JDKs and all the above specific JDKs.
10+
- The `base` variant and its aliases — `8`, `11`, `17`, `21`, `25`, and `tip` — contain the base Eclipse Temurin JDK 8, 11, 17, 21, 25, and tip JDK version releases.
11+
- The `zulu8`, `zulu11`, `oracle8`, `ibm8`, `semeru8`, `semeru11`, `semeru17`, `graalvm17`, `graalvm21`, and `graalvm25` variants all contain the base JDKs in addition to the specific JDK from their name.
12+
- The `latest` variant contains the base JDKs and all of the specific JDKs above.
1313

14-
Images are tagged via the [Tag new images version](https://github.com/DataDog/dd-trace-java-docker-build/actions/workflows/docker-tag.yml) workflow. This workflow tags the latest images built from the specified branch with a `ci-` prefix. It runs quarterly on `master` but can also be triggered manually as needed.
14+
Images are tagged via the [Tag new images version](https://github.com/DataDog/dd-trace-java-docker-build/actions/workflows/docker-tag.yml) workflow. This workflow tags the latest images built from the specified branch with a `ci-` prefix and triggers the [Update mirror digests for ci-* images](https://github.com/DataDog/dd-trace-java-docker-build/actions/workflows/update-mirror-digests.yml) workflow which automatically creates a PR that updates the pinned `ci-*` image digests in `DataDog/images`. It runs quarterly on `master` but can also be triggered manually as needed.
1515

1616
## Development
1717

@@ -26,3 +26,16 @@ And then check the built images:
2626
```bash
2727
./build --test
2828
```
29+
30+
## Testing
31+
32+
Images are built per PR for ease in testing. These test images are prefixed with `N_merge-`, where N is the PR number. See the [GitHub Container Registry](https://github.com/DataDog/dd-trace-java-docker-build/pkgs/container/dd-trace-java-docker-build) for examples.
33+
34+
To test these images in `dd-trace-java` CI:
35+
36+
1. Open a PR in [DataDog/dd-trace-java-docker-build](https://github.com/DataDog/dd-trace-java-docker-build) with the changes you want to test. Let's say these changes are made in PR #123 ([example](https://github.com/DataDog/dd-trace-java-docker-build/pull/123)).
37+
2. Run the [Create test image mirror PR](https://github.com/DataDog/dd-trace-java-docker-build/actions/workflows/create-test-mirror-pr.yml) workflow with `PR_NUMBER=123`. This automatically opens a PR in [DataDog/images](https://github.com/DataDog/images) that adds mirror entries for the `123_merge-*` test images. Merge the PR if not done automatically by the `dd-prapprover` bot.
38+
3. Open a PR in [DataDog/dd-trace-java](https://github.com/DataDog/dd-trace-java) that sets `BUILDER_IMAGE_VERSION_PREFIX: "123_merge-"` in `.gitlab-ci.yml`. Here, you can check your test images with `DataDog/dd-trace-java` CI.
39+
4. Every time you want to test changes made in PR #123, ensure the test image SHAs in `DataDog/images` are updated. This is done by running the [Create test image mirror PR](https://github.com/DataDog/dd-trace-java-docker-build/actions/workflows/create-test-mirror-pr.yml) workflow each time with `PR_NUMBER=123`.
40+
5. When the test images look good and `DataDog/dd-trace-java` CI is green, merge your `DataDog/dd-trace-java-docker-build` PR #123, close the test `DataDog/dd-trace-java` PR, and **remove the test images from the `DataDog/images` repo**.
41+
6. Finally, run the [Tag new images version](https://github.com/DataDog/dd-trace-java-docker-build/actions/workflows/docker-tag.yml) workflow. The [Update mirror digests for ci-* images](https://github.com/DataDog/dd-trace-java-docker-build/actions/workflows/update-mirror-digests.yml) workflow will automatically open a PR in `DataDog/images`, updating the pinned `ci-*` digests. `dd-trace-java` CI should automatically pick up these updated images a few minutes after the PR is merged.

‎scripts/create-test-mirror-entries.sh‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#!/usr/bin/env bash
2+
# create-test-mirror-entries.sh — add or update {PR_NUMBER}_merge-* test image entries
3+
# in mirror.yaml and mirror.lock.yaml files in the DataDog/images repo.
4+
#
5+
# This script is called in the create-test-mirror-pr Github workflow.
6+
# It must be run from the root of DataDog/images and requires crane to be installed.
7+
#
8+
# Required env var:
9+
# PR_NUMBER — pull request number in dd-trace-java-docker-build (numeric)
10+
#
11+
# Outputs (when GITHUB_OUTPUT is set):
12+
# mode=add|update, indicating whether the test images were added or their digests were updated
13+
14+
set -euo pipefail
15+
16+
readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
17+
18+
if ! [[ "${PR_NUMBER}" =~ ^[0-9]+$ ]]; then
19+
echo "::error::PR_NUMBER must be numeric (got: '${PR_NUMBER}')" >&2
20+
exit 1
21+
fi
22+
23+
readonly PREFIX="${PR_NUMBER}_merge-"
24+
25+
# Check if entries already exist in mirror.yaml (use base variant as tester)
26+
if grep -qF "ghcr.io/datadog/dd-trace-java-docker-build:${PREFIX}base" mirror.yaml; then
27+
MODE="update"
28+
echo "Entries for '${PREFIX}' already exist — updating digests only"
29+
else
30+
MODE="add"
31+
echo "No entries found for '${PREFIX}' — adding new entries"
32+
fi
33+
34+
if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
35+
echo "mode=${MODE}" >> "${GITHUB_OUTPUT}"
36+
fi
37+
38+
# shellcheck source=scripts/get-image-digests.sh
39+
source "${SCRIPT_DIR}/get-image-digests.sh"
40+
41+
if [[ "$MODE" == "add" ]]; then
42+
for variant in "${CI_VARIANTS[@]}"; do
43+
tag="${PREFIX}${variant}"
44+
printf ' - source: "%s:%s"\n dest:\n repo: "dd-trace-java-docker-build"\n tag: "%s"\n replication_target: ""\n' \
45+
"ghcr.io/datadog/dd-trace-java-docker-build" "${tag}" "${tag}" >> mirror.yaml
46+
done
47+
echo "Appended ${#CI_VARIANTS[@]} entries to mirror.yaml"
48+
49+
for variant in "${CI_VARIANTS[@]}"; do
50+
tag="${PREFIX}${variant}"
51+
printf ' - source: %s:%s\n digest: %s\n' \
52+
"ghcr.io/datadog/dd-trace-java-docker-build" "${tag}" "${DIGESTS[$variant]}" >> mirror.lock.yaml
53+
done
54+
echo "Appended ${#CI_VARIANTS[@]} entries to mirror.lock.yaml"
55+
else
56+
for variant in "${CI_VARIANTS[@]}"; do
57+
tag="${PREFIX}${variant}"
58+
update_digest "${tag}" "${DIGESTS[$variant]}" mirror.lock.yaml
59+
echo "Updated mirror.lock.yaml: ${tag}"
60+
done
61+
fi

‎scripts/get-image-digests.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/usr/bin/env bash
2+
# get-image-digests.sh — source this script to populate CI_VARIANTS and DIGESTS.
3+
#
4+
# Required env vars:
5+
# PREFIX — tag prefix (e.g. "ci-" or "138_merge-")
6+
#
7+
# After sourcing, callers have access to:
8+
# CI_VARIANTS — indexed array of variant names
9+
# DIGESTS — associative array mapping image variant to its latest digest (sha256:...)
10+
11+
set -euo pipefail
12+
13+
readonly CI_VARIANTS=(base 7 8 11 17 21 25 tip zulu8 zulu11 oracle8 ibm8 semeru8 semeru11 semeru17 graalvm17 graalvm21 graalvm25)
14+
15+
# update_digest TAG DIGEST FILE
16+
# Finds the line "source: ...:TAG" and updates the digest on the following line.
17+
update_digest() {
18+
local tag="$1" digest="$2" file="$3"
19+
awk -v tag="${tag}" -v digest="${digest}" '
20+
$0 ~ ("source:.*:" tag "$") { found=1 }
21+
found && /digest:/ { sub(/digest: sha256:[a-f0-9]*/, "digest: " digest); found=0 }
22+
{ print }
23+
' "${file}" > "${file}.tmp" && mv "${file}.tmp" "${file}"
24+
}
25+
26+
echo "Resolving digests for ${#CI_VARIANTS[@]} variants (prefix: '${PREFIX}')..." >&2
27+
declare -A DIGESTS
28+
for variant in "${CI_VARIANTS[@]}"; do
29+
tag="${PREFIX}${variant}"
30+
echo -n " ${tag} ... " >&2
31+
DIGESTS[$variant]="$(crane digest "ghcr.io/datadog/dd-trace-java-docker-build:${tag}")"
32+
echo "${DIGESTS[$variant]}" >&2
33+
done

0 commit comments

Comments
 (0)