Skip to content

Commit 0886f98

Browse files
kvinwangkvinwang
committed
refactor: use BuildKit --build-context to eliminate script copies
Instead of copying pin-packages.sh and config-qemu.sh into each service's shared/ directory, use BuildKit's named build contexts (--build-context build-shared=build/shared) so Dockerfiles can directly COPY --from=build-shared. This means: - Scripts exist only in build/shared/ (single source of truth) - No copies to track, no sync checks needed - docker build works standalone with: --build-context build-shared=build/shared - build-image.sh passes it automatically via build-lib.sh - Each service's shared/ dir contains only pinned-packages data files
1 parent dd64202 commit 0886f98

16 files changed

Lines changed: 33 additions & 234 deletions

File tree

‎.github/workflows/docker-build-check.yml‎

Lines changed: 10 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -11,41 +11,8 @@ on:
1111
branches: [ master, next, dev-* ]
1212

1313
jobs:
14-
check-shared-scripts:
15-
runs-on: ubuntu-latest
16-
steps:
17-
- uses: actions/checkout@v4
18-
19-
- name: Verify shared build scripts are in sync
20-
run: |
21-
failed=false
22-
for dir in gateway/dstack-app/builder/shared kms/dstack-app/builder/shared verifier/builder/shared; do
23-
if ! diff -q build/shared/pin-packages.sh "$dir/pin-packages.sh" >/dev/null 2>&1; then
24-
echo "ERROR: $dir/pin-packages.sh is out of sync with build/shared/pin-packages.sh"
25-
failed=true
26-
fi
27-
done
28-
for dir in kms/dstack-app/builder/shared verifier/builder/shared; do
29-
if ! diff -q build/shared/config-qemu.sh "$dir/config-qemu.sh" >/dev/null 2>&1; then
30-
echo "ERROR: $dir/config-qemu.sh is out of sync with build/shared/config-qemu.sh"
31-
failed=true
32-
fi
33-
done
34-
if [ "$failed" = "true" ]; then
35-
echo ""
36-
echo "Run the following to fix:"
37-
echo " cp build/shared/pin-packages.sh gateway/dstack-app/builder/shared/"
38-
echo " cp build/shared/pin-packages.sh kms/dstack-app/builder/shared/"
39-
echo " cp build/shared/pin-packages.sh verifier/builder/shared/"
40-
echo " cp build/shared/config-qemu.sh kms/dstack-app/builder/shared/"
41-
echo " cp build/shared/config-qemu.sh verifier/builder/shared/"
42-
exit 1
43-
fi
44-
echo "All shared build scripts are in sync."
45-
4614
gateway:
4715
runs-on: ubuntu-latest
48-
needs: check-shared-scripts
4916
steps:
5017
- uses: actions/checkout@v4
5118

@@ -60,6 +27,8 @@ jobs:
6027
load: true
6128
tags: dstack-gateway-check:latest
6229
provenance: false
30+
build-contexts: |
31+
build-shared=build/shared
6332
build-args: |
6433
DSTACK_REV=${{ github.event.pull_request.head.sha || github.sha }}
6534
DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}
@@ -76,6 +45,7 @@ jobs:
7645
--target gateway-builder \
7746
--tag gateway-builder-check:latest \
7847
--provenance=false \
48+
--build-context build-shared=build/shared \
7949
--build-arg "DSTACK_REV=${{ github.event.pull_request.head.sha || github.sha }}" \
8050
--build-arg "DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}" \
8151
gateway/dstack-app/builder
@@ -87,7 +57,6 @@ jobs:
8757
8858
kms:
8959
runs-on: ubuntu-latest
90-
needs: check-shared-scripts
9160
steps:
9261
- uses: actions/checkout@v4
9362

@@ -102,6 +71,8 @@ jobs:
10271
load: true
10372
tags: dstack-kms-check:latest
10473
provenance: false
74+
build-contexts: |
75+
build-shared=build/shared
10576
build-args: |
10677
DSTACK_REV=${{ github.event.pull_request.head.sha || github.sha }}
10778
DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}
@@ -118,6 +89,7 @@ jobs:
11889
--target kms-builder \
11990
--tag kms-builder-check:latest \
12091
--provenance=false \
92+
--build-context build-shared=build/shared \
12193
--build-arg "DSTACK_REV=${{ github.event.pull_request.head.sha || github.sha }}" \
12294
--build-arg "DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}" \
12395
kms/dstack-app/builder
@@ -135,7 +107,6 @@ jobs:
135107
136108
verifier:
137109
runs-on: ubuntu-latest
138-
needs: check-shared-scripts
139110
steps:
140111
- uses: actions/checkout@v4
141112

@@ -151,6 +122,8 @@ jobs:
151122
load: true
152123
tags: dstack-verifier-check:latest
153124
provenance: false
125+
build-contexts: |
126+
build-shared=build/shared
154127
build-args: |
155128
DSTACK_REV=${{ github.event.pull_request.head.sha || github.sha }}
156129
DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}
@@ -168,6 +141,7 @@ jobs:
168141
--tag verifier-builder-check:latest \
169142
--provenance=false \
170143
--file verifier/builder/Dockerfile \
144+
--build-context build-shared=build/shared \
171145
--build-arg "DSTACK_REV=${{ github.event.pull_request.head.sha || github.sha }}" \
172146
--build-arg "DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}" \
173147
verifier
@@ -185,6 +159,7 @@ jobs:
185159
--tag verifier-acpi-check:latest \
186160
--provenance=false \
187161
--file verifier/builder/Dockerfile \
162+
--build-context build-shared=build/shared \
188163
--build-arg "DSTACK_REV=${{ github.event.pull_request.head.sha || github.sha }}" \
189164
--build-arg "DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}" \
190165
verifier

‎.github/workflows/gateway-release.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ jobs:
5454
tags: ${{ vars.DOCKERHUB_ORG }}/dstack-gateway:${{ env.VERSION }}
5555
platforms: linux/amd64
5656
provenance: false
57+
build-contexts: |
58+
build-shared=build/shared
5759
build-args: |
5860
DSTACK_REV=${{ env.GIT_REV }}
5961
SOURCE_DATE_EPOCH=${{ env.TIMESTAMP }}

‎.github/workflows/kms-release.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ jobs:
5454
tags: ${{ vars.DOCKERHUB_ORG }}/dstack-kms:${{ env.VERSION }}
5555
platforms: linux/amd64
5656
provenance: false
57+
build-contexts: |
58+
build-shared=build/shared
5759
build-args: |
5860
DSTACK_REV=${{ env.GIT_REV }}
5961
DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}

‎.github/workflows/verifier-release.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ jobs:
5454
tags: ${{ vars.DOCKERHUB_ORG }}/dstack-verifier:${{ env.VERSION }}
5555
platforms: linux/amd64
5656
provenance: false
57+
build-contexts: |
58+
build-shared=build/shared
5759
build-args: |
5860
DSTACK_REV=${{ env.GIT_REV }}
5961
DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}.git

‎build/shared/build-lib.sh‎

Lines changed: 2 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ set -euo pipefail
1717

1818
BUILDKIT_VERSION="v0.20.2"
1919
BUILDKIT_BUILDER="buildkit_20"
20+
BUILD_SHARED_DIR="$REPO_ROOT/build/shared"
2021

2122
ensure_buildkit() {
2223
if ! docker buildx inspect "$BUILDKIT_BUILDER" &>/dev/null; then
@@ -47,6 +48,7 @@ docker_build() {
4748
--builder "$BUILDKIT_BUILDER"
4849
--progress=plain
4950
--output "type=docker,name=$image_name,rewrite-timestamp=true"
51+
--build-context "build-shared=$BUILD_SHARED_DIR"
5052
--build-arg "SOURCE_DATE_EPOCH=$commit_timestamp"
5153
--build-arg "DSTACK_REV=$GIT_REV"
5254
--build-arg "DSTACK_SRC_URL=$DSTACK_SRC_URL"
@@ -67,30 +69,6 @@ docker_build() {
6769
extract_packages "$image_name" "$pkg_list_file"
6870
}
6971

70-
# Verify that local copies of shared scripts match the canonical versions in build/shared/.
71-
check_shared_scripts() {
72-
local dest_dir=$1
73-
local need_qemu=${2:-false}
74-
local canonical="$REPO_ROOT/build/shared"
75-
local failed=false
76-
77-
if ! diff -q "$canonical/pin-packages.sh" "$dest_dir/pin-packages.sh" &>/dev/null; then
78-
echo "ERROR: $dest_dir/pin-packages.sh is out of sync with build/shared/pin-packages.sh" >&2
79-
echo " Run: cp build/shared/pin-packages.sh $dest_dir/" >&2
80-
failed=true
81-
fi
82-
if [ "$need_qemu" = "true" ]; then
83-
if ! diff -q "$canonical/config-qemu.sh" "$dest_dir/config-qemu.sh" &>/dev/null; then
84-
echo "ERROR: $dest_dir/config-qemu.sh is out of sync with build/shared/config-qemu.sh" >&2
85-
echo " Run: cp build/shared/config-qemu.sh $dest_dir/" >&2
86-
failed=true
87-
fi
88-
fi
89-
if [ "$failed" = "true" ]; then
90-
exit 1
91-
fi
92-
}
93-
9472
# Verify that pinned-packages files haven't changed (idempotency check).
9573
check_clean_tree() {
9674
local check_path=$1

‎gateway/dstack-app/builder/Dockerfile‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
# SPDX-License-Identifier: Apache-2.0
44

55
FROM rust:1.92.0@sha256:48851a839d6a67370c9dbe0e709bedc138e3e404b161c5233aedcf2b717366e4 AS gateway-builder
6-
COPY ./shared /build
6+
COPY --from=build-shared pin-packages.sh /build/
7+
COPY ./shared/*-pinned-packages.txt /build/
78
ARG DSTACK_REV
89
ARG DSTACK_SRC_URL=https://github.com/Dstack-TEE/dstack.git
910
WORKDIR /build
@@ -26,7 +27,8 @@ RUN cd dstack && cargo build --release -p dstack-gateway --target x86_64-unknown
2627
RUN echo "${DSTACK_REV}" > /build/.GIT_REV
2728

2829
FROM debian:bookworm@sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe
29-
COPY ./shared /build
30+
COPY --from=build-shared pin-packages.sh /build/
31+
COPY ./shared/pinned-packages.txt /build/
3032
WORKDIR /build
3133
RUN ./pin-packages.sh ./pinned-packages.txt && \
3234
apt-get update && \

‎gateway/dstack-app/builder/build-image.sh‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ GIT_REV=$(git -C "$REPO_ROOT" rev-parse "$GIT_REV")
2626
DSTACK_SRC_URL=${DSTACK_SRC_URL:-https://github.com/Dstack-TEE/dstack.git}
2727

2828
ensure_buildkit
29-
check_shared_scripts "$SHARED_DIR"
3029

3130
touch "$SHARED_DIR/builder-pinned-packages.txt"
3231
touch "$SHARED_DIR/pinned-packages.txt"

‎gateway/dstack-app/builder/shared/pin-packages.sh‎

Lines changed: 0 additions & 36 deletions
This file was deleted.

‎kms/dstack-app/builder/Dockerfile‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
# SPDX-License-Identifier: Apache-2.0
44

55
FROM rust:1.92.0@sha256:48851a839d6a67370c9dbe0e709bedc138e3e404b161c5233aedcf2b717366e4 AS kms-builder
6-
COPY ./shared /build
6+
COPY --from=build-shared pin-packages.sh /build/
7+
COPY ./shared/*-pinned-packages.txt /build/
78
ARG DSTACK_REV
89
ARG DSTACK_SRC_URL=https://github.com/Dstack-TEE/dstack.git
910
WORKDIR /build
@@ -26,7 +27,8 @@ RUN cd dstack && cargo build --release -p dstack-kms --target x86_64-unknown-lin
2627
RUN echo "${DSTACK_REV}" > /build/.GIT_REV
2728

2829
FROM debian:bookworm@sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe
29-
COPY ./shared /build
30+
COPY --from=build-shared pin-packages.sh config-qemu.sh /build/
31+
COPY ./shared/qemu-pinned-packages.txt /build/
3032
WORKDIR /build
3133
ARG QEMU_REV=dbcec07c0854bf873d346a09e87e4c993ccf2633
3234
RUN ./pin-packages.sh ./qemu-pinned-packages.txt && \

‎kms/dstack-app/builder/build-image.sh‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ GIT_REV=$(git -C "$REPO_ROOT" rev-parse "$GIT_REV")
2626
DSTACK_SRC_URL=${DSTACK_SRC_URL:-https://github.com/Dstack-TEE/dstack.git}
2727

2828
ensure_buildkit
29-
check_shared_scripts "$SHARED_DIR" true
3029

3130
touch "$SHARED_DIR/builder-pinned-packages.txt"
3231
touch "$SHARED_DIR/qemu-pinned-packages.txt"

0 commit comments

Comments
 (0)