Skip to content

Commit cccc22c

Browse files
apiarian-datadogapiarian-datadog
committed
chore: more fixes for serverless-init builds
1 parent 23eebef commit cccc22c

File tree

2 files changed

+98
-12
lines changed

2 files changed

+98
-12
lines changed

‎.github/workflows/release-serverless-init.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,15 @@ jobs:
5353
ref: ${{ github.event.inputs.agentBranch }}
5454
path: datadog-agent
5555

56+
# Pin QEMU to a known-good version. The default (binfmt:latest) has broken
57+
# arm64 emulation multiple times due to QEMU segfaults in libc-bin triggers:
58+
# - Feb 2025: qemu-v9.2.0 (https://github.com/tonistiigi/binfmt/issues/240)
59+
# - Mar 2026: qemu-v10.2.1
60+
# Use test-qemu-versions.yml to sweep versions before bumping this pin.
5661
- name: Set up QEMU
5762
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 #v3.7.0
63+
with:
64+
image: tonistiigi/binfmt:qemu-v10.1.3
5865

5966
- name: Set up Docker Buildx
6067
uses: docker/setup-buildx-action@v3

‎.github/workflows/test-qemu-versions.yml‎

Lines changed: 91 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,41 @@
1+
# Test QEMU/binfmt versions for arm64 cross-compilation on x86_64 runners.
2+
#
3+
# The serverless-init release build uses QEMU (via docker/setup-qemu-action) to
4+
# emulate arm64 during multi-platform Docker builds. The QEMU version provided by
5+
# tonistiigi/binfmt:latest has broken arm64 emulation multiple times, causing
6+
# segfaults during "Processing triggers for libc-bin" in the ubuntu:22.04
7+
# compresser stage of our Dockerfiles.
8+
#
9+
# Known broken versions:
10+
# - qemu-v9.2.0: Feb 2025, https://github.com/tonistiigi/binfmt/issues/240
11+
# - qemu-v8.1.5: also segfaults in simple docker run tests
12+
# - qemu-v8.1.4: also segfaults in simple docker run tests
13+
# - qemu-v10.2.1: Mar 2026, passes simple tests but segfaults under real build
14+
# load (parallel Go compilation + compresser stage under QEMU)
15+
#
16+
# Known working versions (as of Mar 2026):
17+
# - qemu-v10.1.3: released Feb 17 2026, passes all tests
18+
# - qemu-v10.0.4: passes all tests
19+
# - qemu-v9.2.2: has QEMU-side fix for segfault (actions/runner-images#11662)
20+
# - qemu-v8.0.4: passes all tests
21+
# - qemu-v7.0.0-28: long-standing known-good baseline
22+
#
23+
# Relevant issues:
24+
# - https://github.com/tonistiigi/binfmt/issues/215
25+
# - https://github.com/tonistiigi/binfmt/issues/240
26+
# - https://github.com/tonistiigi/binfmt/issues/245
27+
# - https://github.com/actions/runner-images/issues/11662
28+
#
29+
# The pinned version in release-serverless-init.yml should be updated to the
30+
# newest version that passes here AND succeeds in a real build. Note that this
31+
# test workflow may not catch all failures — the real build's parallel multi-stage
32+
# Dockerfile (Go compilation + compresser running concurrently under QEMU) can
33+
# trigger segfaults that simpler tests miss (as seen with v10.2.1).
34+
#
35+
# Usage: dispatch this workflow from the Actions tab, then check which versions
36+
# pass/fail. Use the newest passing version as the pin in release-serverless-init.yml,
37+
# and verify with an actual release build before merging.
38+
139
name: Test QEMU versions for arm64 emulation
240

341
on:
@@ -10,21 +48,23 @@ jobs:
1048
fail-fast: false
1149
matrix:
1250
qemu_image:
13-
# v10.x - current era
14-
- "tonistiigi/binfmt:qemu-v10.2.1" # current latest, known broken
15-
- "tonistiigi/binfmt:qemu-v10.1.3" # released Feb 17 2026, day of last good build
16-
- "tonistiigi/binfmt:qemu-v10.0.4" # Jan 2026
51+
# v10.x
52+
- "tonistiigi/binfmt:qemu-v10.2.1"
53+
- "tonistiigi/binfmt:qemu-v10.1.3"
54+
- "tonistiigi/binfmt:qemu-v10.0.4"
1755
# v9.x
18-
- "tonistiigi/binfmt:qemu-v9.2.2" # reportedly has fix for segfault issue
19-
- "tonistiigi/binfmt:qemu-v9.2.0" # known broken (Feb 2025 incident)
56+
- "tonistiigi/binfmt:qemu-v9.2.2"
57+
- "tonistiigi/binfmt:qemu-v9.2.0"
2058
# v8.x
21-
- "tonistiigi/binfmt:qemu-v8.1.5" # reported working in issue #245
59+
- "tonistiigi/binfmt:qemu-v8.1.5"
2260
- "tonistiigi/binfmt:qemu-v8.1.4"
2361
- "tonistiigi/binfmt:qemu-v8.0.4"
24-
# v7.x - known good baseline
62+
# v7.x
2563
- "tonistiigi/binfmt:qemu-v7.0.0-28"
2664
name: "QEMU ${{ matrix.qemu_image }}"
2765
steps:
66+
- uses: actions/checkout@v6.0.2
67+
2868
- name: Set up QEMU
2969
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 #v3.7.0
3070
with:
@@ -36,12 +76,51 @@ jobs:
3676
- name: Set up Docker Buildx
3777
uses: docker/setup-buildx-action@v3
3878

39-
- name: "Test: ubuntu:22.04 apt-get install (matching compresser stage)"
79+
# Simple docker run tests — catches deterministically broken versions
80+
# (e.g. v9.2.0, v8.1.5, v8.1.4 all fail here).
81+
- name: "Test: docker run - ubuntu:22.04 apt-get (simple)"
4082
run: |
4183
docker run --rm --platform linux/arm64 ubuntu:22.04 \
42-
bash -c "apt-get update && apt-get install -y zip binutils && echo 'SUCCESS: apt-get completed'"
84+
bash -c "apt-get update && apt-get install -y zip binutils && echo 'SUCCESS: docker run apt-get completed'"
4385
44-
- name: "Test: alpine:3.16 apk add (matching builder stage)"
86+
- name: "Test: docker run - alpine:3.16 apk add (simple)"
4587
run: |
4688
docker run --rm --platform linux/arm64 alpine:3.16 \
47-
sh -c "apk add --no-cache git make musl-dev gcc && echo 'SUCCESS: apk completed'"
89+
sh -c "apk add --no-cache git make musl-dev gcc && echo 'SUCCESS: docker run apk completed'"
90+
91+
# Buildx tests — closer to the real build, using docker buildx build with
92+
# Dockerfiles that match the stages in Dockerfile.build and Dockerfile.alpine.build.
93+
# Note: even these may not catch load-dependent failures; see header comment.
94+
- name: "Test: buildx - compresser stage (matches real Dockerfile.build)"
95+
run: |
96+
cat > /tmp/Dockerfile.test-compresser <<'DOCKERFILE'
97+
FROM ubuntu:22.04 as compresser
98+
RUN apt-get update && apt-get install -y zip binutils
99+
RUN mkdir /extensions
100+
WORKDIR /extensions
101+
RUN echo "SUCCESS: buildx compresser stage completed"
102+
DOCKERFILE
103+
docker buildx build --platform linux/arm64 -f /tmp/Dockerfile.test-compresser /tmp
104+
105+
- name: "Test: buildx - alpine builder stage (matches real Dockerfile.alpine.build)"
106+
run: |
107+
cat > /tmp/Dockerfile.test-alpine <<'DOCKERFILE'
108+
FROM alpine:3.16 as builder
109+
RUN apk add --no-cache git make musl-dev gcc
110+
COPY --from=golang:1.24.6-alpine /usr/local/go/ /usr/lib/go
111+
ENV GOROOT /usr/lib/go
112+
ENV GOPATH /go
113+
ENV PATH /go/bin:$PATH
114+
RUN mkdir -p ${GOPATH}/src ${GOPATH}/bin
115+
RUN echo "SUCCESS: buildx alpine builder stage completed"
116+
DOCKERFILE
117+
docker buildx build --platform linux/arm64 -f /tmp/Dockerfile.test-alpine /tmp
118+
119+
- name: "Test: buildx - multi-platform (both arches, matches real build)"
120+
run: |
121+
cat > /tmp/Dockerfile.test-multi <<'DOCKERFILE'
122+
FROM ubuntu:22.04
123+
RUN apt-get update && apt-get install -y zip binutils
124+
RUN echo "SUCCESS: buildx multi-platform completed"
125+
DOCKERFILE
126+
docker buildx build --platform linux/amd64,linux/arm64 -f /tmp/Dockerfile.test-multi /tmp

0 commit comments

Comments
 (0)