Skip to content

Commit bb30ec6

Browse files
github-actions[bot]potiuk
github-actions[bot]
and
potiuk
authored
[v3-2-test] Install rustup and cargo in CI and prod build images (#64725) (#64729)
Verify rustup-init binary with SHA256 checksum instead of curl-pipe-sh Download the rustup-init binary directly and verify its SHA256 checksum before execution, instead of piping the shell installer script through sh. Pin rustup-init to version 1.29.0 with hardcoded SHA256 checksums for amd64 and arm64, matching the existing cosign verification pattern. This prevents a compromised server from serving a tampered binary with a matching checksum. (cherry picked from commit 1b28933) Co-authored-by: Jarek Potiuk <jarek@potiuk.com>
1 parent e17bf20 commit bb30ec6

File tree

4 files changed

+104
-3
lines changed

4 files changed

+104
-3
lines changed

‎.github/workflows/additional-ci-image-checks.yml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,8 +116,10 @@ jobs:
116116
(github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
117117
118118
# Check that after earlier cache push, breeze command will build quickly
119+
# This build is a bit slow from in-the scratch builds, so we should run it only in
120+
# regular PRs
119121
check-that-image-builds-quickly:
120-
timeout-minutes: 17
122+
timeout-minutes: 25
121123
name: Check that image builds quickly
122124
runs-on: ${{ fromJSON(inputs.runners) }}
123125
env:
@@ -141,4 +143,6 @@ jobs:
141143
- name: "Install Breeze"
142144
uses: ./.github/actions/breeze
143145
- name: "Check that image builds quickly"
144-
run: breeze shell --max-time 900 --platform "${PLATFORM}"
146+
# Synchronize to be a little bit shorter than above timeout-minutes to make sure that
147+
# if the build takes too long the job will fail with logs. 22 minutes * 60 s = 1320 seconds
148+
run: breeze shell --max-time 1320 --platform "${PLATFORM}"

‎Dockerfile‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,8 @@ fi
122122
AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.10.18}
123123
PYTHON_LTO=${PYTHON_LTO:-true}
124124
GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4}
125+
RUSTUP_DEFAULT_TOOLCHAIN=${RUSTUP_DEFAULT_TOOLCHAIN:-stable}
126+
RUSTUP_VERSION=${RUSTUP_VERSION:-1.29.0}
125127
COSIGN_VERSION=${COSIGN_VERSION:-3.0.5}
126128

127129
if [[ "${1}" == "runtime" ]]; then
@@ -493,6 +495,33 @@ function install_golang() {
493495
rm -rf /usr/local/go && tar -C /usr/local -xzf go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz
494496
}
495497

498+
function install_rustup() {
499+
local arch
500+
arch="$(dpkg --print-architecture)"
501+
declare -A rustup_targets=(
502+
[amd64]="x86_64-unknown-linux-gnu"
503+
[arm64]="aarch64-unknown-linux-gnu"
504+
)
505+
declare -A rustup_sha256s=(
506+
# https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/{target}/rustup-init.sha256
507+
[amd64]="4acc9acc76d5079515b46346a485974457b5a79893cfb01112423c89aeb5aa10"
508+
[arm64]="9732d6c5e2a098d3521fca8145d826ae0aaa067ef2385ead08e6feac88fa5792"
509+
)
510+
local target="${rustup_targets[${arch}]}"
511+
local rustup_sha256="${rustup_sha256s[${arch}]}"
512+
if [[ -z "${target}" ]]; then
513+
echo "Unsupported architecture for rustup: ${arch}"
514+
exit 1
515+
fi
516+
curl --proto '=https' --tlsv1.2 -sSf \
517+
"https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/${target}/rustup-init" \
518+
-o /tmp/rustup-init
519+
echo "${rustup_sha256} /tmp/rustup-init" | sha256sum --check
520+
chmod +x /tmp/rustup-init
521+
/tmp/rustup-init -y --default-toolchain "${RUSTUP_DEFAULT_TOOLCHAIN}"
522+
rm -f /tmp/rustup-init
523+
}
524+
496525
function apt_clean() {
497526
apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false
498527
rm -rf /var/lib/apt/lists/* /var/log/*
@@ -508,6 +537,7 @@ else
508537
install_debian_dev_dependencies
509538
install_python
510539
install_additional_dev_dependencies
540+
install_rustup
511541
if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then
512542
install_golang
513543
fi
@@ -1843,6 +1873,10 @@ ENV DEV_APT_DEPS=${DEV_APT_DEPS} \
18431873

18441874
ARG PYTHON_LTO
18451875

1876+
ENV RUSTUP_HOME="/usr/local/rustup"
1877+
ENV CARGO_HOME="/usr/local/cargo"
1878+
ENV PATH="${CARGO_HOME}/bin:${PATH}"
1879+
18461880
COPY --from=scripts install_os_dependencies.sh /scripts/docker/
18471881
RUN PYTHON_LTO=${PYTHON_LTO} bash /scripts/docker/install_os_dependencies.sh dev
18481882

‎Dockerfile.ci‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@ fi
6262
AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.10.18}
6363
PYTHON_LTO=${PYTHON_LTO:-true}
6464
GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4}
65+
RUSTUP_DEFAULT_TOOLCHAIN=${RUSTUP_DEFAULT_TOOLCHAIN:-stable}
66+
RUSTUP_VERSION=${RUSTUP_VERSION:-1.29.0}
6567
COSIGN_VERSION=${COSIGN_VERSION:-3.0.5}
6668

6769
if [[ "${1}" == "runtime" ]]; then
@@ -433,6 +435,33 @@ function install_golang() {
433435
rm -rf /usr/local/go && tar -C /usr/local -xzf go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz
434436
}
435437

438+
function install_rustup() {
439+
local arch
440+
arch="$(dpkg --print-architecture)"
441+
declare -A rustup_targets=(
442+
[amd64]="x86_64-unknown-linux-gnu"
443+
[arm64]="aarch64-unknown-linux-gnu"
444+
)
445+
declare -A rustup_sha256s=(
446+
# https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/{target}/rustup-init.sha256
447+
[amd64]="4acc9acc76d5079515b46346a485974457b5a79893cfb01112423c89aeb5aa10"
448+
[arm64]="9732d6c5e2a098d3521fca8145d826ae0aaa067ef2385ead08e6feac88fa5792"
449+
)
450+
local target="${rustup_targets[${arch}]}"
451+
local rustup_sha256="${rustup_sha256s[${arch}]}"
452+
if [[ -z "${target}" ]]; then
453+
echo "Unsupported architecture for rustup: ${arch}"
454+
exit 1
455+
fi
456+
curl --proto '=https' --tlsv1.2 -sSf \
457+
"https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/${target}/rustup-init" \
458+
-o /tmp/rustup-init
459+
echo "${rustup_sha256} /tmp/rustup-init" | sha256sum --check
460+
chmod +x /tmp/rustup-init
461+
/tmp/rustup-init -y --default-toolchain "${RUSTUP_DEFAULT_TOOLCHAIN}"
462+
rm -f /tmp/rustup-init
463+
}
464+
436465
function apt_clean() {
437466
apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false
438467
rm -rf /var/lib/apt/lists/* /var/log/*
@@ -448,6 +477,7 @@ else
448477
install_debian_dev_dependencies
449478
install_python
450479
install_additional_dev_dependencies
480+
install_rustup
451481
if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then
452482
install_golang
453483
fi
@@ -1646,6 +1676,9 @@ ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \
16461676
ARG AIRFLOW_PYTHON_VERSION="3.12.13"
16471677
ENV AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION}
16481678
ENV GOLANG_MAJOR_MINOR_VERSION="1.26.1"
1679+
ENV RUSTUP_HOME="/usr/local/rustup"
1680+
ENV CARGO_HOME="/usr/local/cargo"
1681+
ENV PATH="${CARGO_HOME}/bin:${PATH}"
16491682

16501683
ARG PYTHON_LTO
16511684

@@ -1805,7 +1838,7 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
18051838
AIRFLOW_PREK_VERSION=${AIRFLOW_PREK_VERSION}
18061839

18071840
# The PATH is needed for python to find installed and cargo to build the wheels
1808-
ENV PATH="/usr/python/bin:/root/.local/bin:/root/.cargo/bin:${PATH}"
1841+
ENV PATH="/usr/python/bin:/root/.local/bin:${PATH}"
18091842
# Useful for creating a cache id based on the underlying architecture, preventing the use of cached python packages from
18101843
# an incorrect architecture.
18111844
ARG TARGETARCH

‎scripts/docker/install_os_dependencies.sh‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ fi
2828
AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.10.18}
2929
PYTHON_LTO=${PYTHON_LTO:-true}
3030
GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4}
31+
RUSTUP_DEFAULT_TOOLCHAIN=${RUSTUP_DEFAULT_TOOLCHAIN:-stable}
32+
RUSTUP_VERSION=${RUSTUP_VERSION:-1.29.0}
3133
COSIGN_VERSION=${COSIGN_VERSION:-3.0.5}
3234

3335
if [[ "${1}" == "runtime" ]]; then
@@ -399,6 +401,33 @@ function install_golang() {
399401
rm -rf /usr/local/go && tar -C /usr/local -xzf go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz
400402
}
401403

404+
function install_rustup() {
405+
local arch
406+
arch="$(dpkg --print-architecture)"
407+
declare -A rustup_targets=(
408+
[amd64]="x86_64-unknown-linux-gnu"
409+
[arm64]="aarch64-unknown-linux-gnu"
410+
)
411+
declare -A rustup_sha256s=(
412+
# https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/{target}/rustup-init.sha256
413+
[amd64]="4acc9acc76d5079515b46346a485974457b5a79893cfb01112423c89aeb5aa10"
414+
[arm64]="9732d6c5e2a098d3521fca8145d826ae0aaa067ef2385ead08e6feac88fa5792"
415+
)
416+
local target="${rustup_targets[${arch}]}"
417+
local rustup_sha256="${rustup_sha256s[${arch}]}"
418+
if [[ -z "${target}" ]]; then
419+
echo "Unsupported architecture for rustup: ${arch}"
420+
exit 1
421+
fi
422+
curl --proto '=https' --tlsv1.2 -sSf \
423+
"https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/${target}/rustup-init" \
424+
-o /tmp/rustup-init
425+
echo "${rustup_sha256} /tmp/rustup-init" | sha256sum --check
426+
chmod +x /tmp/rustup-init
427+
/tmp/rustup-init -y --default-toolchain "${RUSTUP_DEFAULT_TOOLCHAIN}"
428+
rm -f /tmp/rustup-init
429+
}
430+
402431
function apt_clean() {
403432
apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false
404433
rm -rf /var/lib/apt/lists/* /var/log/*
@@ -414,6 +443,7 @@ else
414443
install_debian_dev_dependencies
415444
install_python
416445
install_additional_dev_dependencies
446+
install_rustup
417447
if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then
418448
install_golang
419449
fi

0 commit comments

Comments
 (0)