ci-builder

#!/usr/bin/env bash

# Copyright Materialize, Inc. and contributors. All rights reserved.
#
# Use of this software is governed by the Business Source License
# included in the LICENSE file at the root of this repository.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0.
#
# ci-builder — builds and releases CI builder image.

# NOTE(benesch): this script is reaching the breaking point in Bash. We should
# rewrite it in Python before adding much more logic to it.

set -euo pipefail

NIGHTLY_RUST_DATE=2026-04-02

workdir=$(pwd)
cd "$(dirname "$0")/.."

. misc/shlib/shlib.bash

if [[ $# -lt 2 ]]
then
    echo "usage: $0 <command> <stable|nightly|min|console> [<args>...]

Manages the ci-builder Docker image, which contains the dependencies required
to build, test, and deploy the code in this repository.

Commands:
    run         run a command in the ci-builder image
    build       build the ci-builder image locally
    exists      reports via the exit code whether the ci-builder image exists
    tag         reports the tag for the ci-builder image
    root-shell  open a root shell to the most recently started ci-builder container

For details, consult ci/builder/README.md."
    exit 1
fi

cmd=$1 && shift
flavor=$1 && shift

rust_date=
case "$flavor" in
    min)
        docker_target=ci-builder-min
        rust_version=$(sed -n 's/rust-version = "\(.*\)"/\1/p' Cargo.toml)
        ;;
    stable)
        docker_target=ci-builder-full
        rust_version=$(sed -n 's/rust-version = "\(.*\)"/\1/p' Cargo.toml)
        ;;
    nightly)
        docker_target=ci-builder-full
        rust_version=nightly
        rust_date=/$NIGHTLY_RUST_DATE
        ;;
    console)
        docker_target=ci-builder-console
        rust_version=$(sed -n 's/rust-version = "\(.*\)"/\1/p' Cargo.toml)
        ;;
    *)
        printf "unknown CI builder flavor %q\n" "$flavor"
        exit 1
        ;;
esac

arch_gcc=${MZ_DEV_CI_BUILDER_ARCH:-$(arch_gcc)}
arch_go=$(arch_go "$arch_gcc")

cid_file=ci/builder/.${flavor%%-*}.cidfile

rust_components=rustc,cargo,rust-std-$arch_gcc-unknown-linux-gnu,llvm-tools-preview
if [[ $rust_version = nightly ]]; then
    rust_components+=,miri-preview,rustfmt-preview
else
    rust_components+=,clippy-preview,rustfmt-preview
fi

# Target a specific CPU to get better performance, i.e. use more modern instructions.
#
# All of the x86-64 and ARM hardware we run in production support the v3 and neoverse-n1 micro
# architectures, respectively.
#
# Sync: This target-cpu should be kept in sync with the one in ci-builder and .cargo/config.
x86_64_target_cpu="x86-64-v3"
aarch64_target_cpu="neoverse-n1"

rust_cpu_target=""
case "$arch_gcc" in
    x86_64) rust_cpu_target="$x86_64_target_cpu" ;;
    aarch64) rust_cpu_target="$aarch64_target_cpu" ;;
    *) die "unknown host architecture \"$arch\"" ;;
esac

# Enable specific CPU features that we know our cloud infra supports.
#
# Note: specifying a target_cpu enables many features, these are specific ones that target_cpu
# might miss.
#
# Sync: This list of features should be kept in sync with the one in xcompile and .cargo/config.
x86_64_target_features="+aes,+pclmulqdq"
aaarch64_target_features="+aes,+sha2"

rust_target_features=""
case "$arch_gcc" in
    x86_64) rust_target_features="$x86_64_target_features" ;;
    aarch64) rust_target_features="$aaarch64_target_features" ;;
    *) die "unknown host architecture \"$arch\"" ;;
esac

docker_build_args="\
--build-arg ARCH_GCC=$arch_gcc \
--build-arg ARCH_GO=$arch_go \
--build-arg RUST_VERSION=$rust_version \
--build-arg RUST_DATE=$rust_date \
--build-arg RUST_COMPONENTS=$rust_components \
--build-arg RUST_CPU_TARGET=$rust_cpu_target \
--build-arg RUST_TARGET_FEATURES=$rust_target_features"

uid=$(id -u)
gid=$(id -g)
[[ "$uid" -lt 500 ]] && uid=501
[[ "$gid" -lt 500 ]] && gid=$uid

build() {
    # shellcheck disable=SC2086 # intentional splitting of build args string
    docker buildx build --pull \
        --cache-from=materialize/ci-builder:"$cache_tag" \
        --cache-to=type=inline,mode=max \
        $docker_build_args \
        --tag materialize/ci-builder:"$tag" \
        --tag ghcr.io/materializeinc/materialize/ci-builder:"$tag" \
        --tag materialize/ci-builder:"$cache_tag" \
        --target $docker_target \
        "$@" ci/builder
}

shasum=sha1sum
if ! command_exists "$shasum"; then
    shasum=shasum
fi
if ! command_exists "$shasum"; then
    die "error: ci-builder: unable to find suitable SHA-1 tool; need either sha1sum or shasum"
fi

# The tag is the base32 encoded hash of the ci/builder directory. This logic is
# similar to what mzbuild uses. Unfortunately we can't use mzbuild itself due to
# a chicken-and-egg problem: mzbuild depends on the Python packages that are
# *inside* this image. See materialize.git.expand_globs in the Python code for
# details on this computation.
tag=$({
    cat \
        <(git diff --name-only -z 4b825dc642cb6eb9a060e54bf8d69288fbee4904 ci/builder bin/ci-builder) \
        <(git ls-files --others --exclude-standard -z ci/builder) \
        | LC_ALL=C sort -z \
        | xargs -0 "$shasum"
    printf '%s\n' "flavor:$flavor" "build-args:$docker_build_args"
} | python3 -c '
import base64
import hashlib
import sys

input = sys.stdin.buffer.read()
hash = base64.b32encode(hashlib.sha1(input).digest())
print(hash.decode())
')
cache_tag=cache-$flavor-$rust_version-$arch_go

image_registry="materialize"
if is_truthy "${CI:-0}" || is_truthy "${MZ_GHCR:-0}"; then
    image_registry="ghcr.io/materializeinc/materialize"
fi

case "$cmd" in
    build)
        build "$@"
        ;;
    exists)
        docker manifest inspect "$image_registry"/ci-builder:"$tag" &> /dev/null
        ;;
    tag)
        echo "$tag"
        ;;
    push)
        docker login ghcr.io -u materialize-bot --password "$GITHUB_GHCR_TOKEN"
        build --push "$@"
        ;;
    run)
        docker_command=()

        detach_container=false
        interactive=false
        tty=false
        secrets=true
        container_name_param=""

        while [[ $# -gt 0 ]]; do
            case $1 in
                --detach)
                    detach_container=true
                    shift # past argument
                    ;;
                --name)
                    container_name_param="$2"
                    shift # past argument
                    shift # past value
                    ;;
                --interactive)
                    interactive=true
                    shift # past argument
                    ;;
                --tty)
                    tty=true
                    shift # past argument
                    ;;
                --no-secrets)
                    secrets=false
                    shift # past argument
                    ;;
                *)
                    docker_command+=("$1")
                    shift # past argument
                    ;;
            esac
        done

        claude_dir=$HOME/.claude
        codex_dir=$HOME/.codex
        npm_dir=$HOME/.npm-builder
        mkdir -p "$(pwd)/target-xcompile" ~/.kube "$claude_dir" "$codex_dir" "$npm_dir"

        args=(
            --cidfile "$cid_file"
            --rm --interactive
            --init
            --volume "$(pwd)/target-xcompile:/mnt/build"
            --volume "$claude_dir:/tmp/.claude"
            --volume "$HOME/.claude.json:/tmp/.claude.json"
            --volume "$codex_dir:/tmp/.codex"
            --volume "$npm_dir:/tmp/.npm"
            --volume "$workdir:$workdir"
            --workdir "$workdir"
            --env XDG_CACHE_HOME=/mnt/build/cache
            --env MZ_SOFT_ASSERTIONS
            --env NO_COLOR
            --env RUST_MIN_STACK
            --env MZ_DEV_BUILD_SHA
            --env MZ_GHCR
            --env BUILDKITE_BRANCH
            --env BUILDKITE_ORGANIZATION_SLUG
            --env BUILDKITE_PULL_REQUEST
        )

        if [[ $secrets == "true" ]]; then
            args+=(
                --env AWS_ACCESS_KEY_ID
                --env AWS_DEFAULT_REGION
                --env AWS_SECRET_ACCESS_KEY
                --env AWS_SESSION_TOKEN
                --env CANARY_LOADTEST_APP_PASSWORD
                --env CANARY_LOADTEST_PASSWORD
                --env CLOUDTEST_CLUSTER_DEFINITION_FILE
                --env COMMON_ANCESTOR_OVERRIDE
                --env CONFLUENT_CLOUD_QA_CANARY_CSR_PASSWORD
                --env CONFLUENT_CLOUD_QA_CANARY_CSR_USERNAME
                --env CONFLUENT_CLOUD_QA_CANARY_KAFKA_PASSWORD
                --env CONFLUENT_CLOUD_QA_CANARY_KAFKA_USERNAME
                --env AZURE_SERVICE_ACCOUNT_USERNAME
                --env AZURE_SERVICE_ACCOUNT_PASSWORD
                --env AZURE_SERVICE_ACCOUNT_TENANT
                --env GCP_SERVICE_ACCOUNT_JSON
                --env GITHUB_TOKEN
                --env GITHUB_GHCR_TOKEN
                --env GPG_KEY
                --env LAUNCHDARKLY_API_TOKEN
                --env LAUNCHDARKLY_SDK_KEY
                --env NIGHTLY_CANARY_APP_PASSWORD
                --env MZ_CI_LICENSE_KEY
                --env MZ_CLI_APP_PASSWORD
                --env NPM_TOKEN
                --env POLAR_SIGNALS_API_TOKEN
                --env PRODUCTION_ANALYTICS_USERNAME
                --env PRODUCTION_ANALYTICS_APP_PASSWORD
                --env PYPI_TOKEN
                --env QA_CLUSTER_SPEC_SHEET_POSTGRES_HOSTNAME
                --env QA_CLUSTER_SPEC_SHEET_POSTGRES_PASSWORD
                --env QA_CLUSTER_SPEC_SHEET_MYSQL_HOSTNAME
                --env QA_CLUSTER_SPEC_SHEET_MYSQL_PASSWORD
                # For Miri with nightly Rust
                --env ZOOKEEPER_ADDR
                --env KAFKA_ADDRS
                --env SCHEMA_REGISTRY_URL
                --env STEP_START_TIMESTAMP_WITH_TZ
                --env POSTGRES_URL
                --env METADATA_BACKEND_URL
                # For ci-closed-issues-detect
                --env GITHUB_CI_ISSUE_REFERENCE_CHECKER_TOKEN
                --env LINEAR_READ_ONLY_TOKEN
                # For auto_cut_release
                --env GIT_AUTHOR_EMAIL
                --env GIT_AUTHOR_NAME
                --env GIT_COMMITTER_EMAIL
                --env GIT_COMMITTER_NAME
                # For cloud canary
                --env REDPANDA_CLOUD_CLIENT_ID
                --env REDPANDA_CLOUD_CLIENT_SECRET
                --env QA_BENCHMARKING_APP_PASSWORD
                # For self managed docs
                --env DOCKERHUB_USERNAME
                --env DOCKERHUB_ACCESS_TOKEN
                # For configuring the metadata store
                --env EXTERNAL_METADATA_STORE
                # For console
                --env CONSOLE_E2E_TEST_STAGING_PASSWORD
                --env CONSOLE_E2E_TEST_PRODUCTION_PASSWORD
                --env E2E_TEST_STAGING_FRONTEGG_CLIENT_ID
                --env E2E_TEST_STAGING_ORB_API_KEY
                --env E2E_TEST_LICENSE_KEY
                --env CLOUD_DEPLOY_KEY
                --env VERCEL_TOKEN
                --env VERCEL_ORG_ID
                --env VERCEL_PROJECT_ID
            )
            for env in $(printenv | grep -E '^(BUILDKITE|MZCOMPOSE|CI)' | sed 's/=.*//'); do
                args+=(--env "$env")
            done
            # Forward the host's SSH agent, if available.
            if [[ "${SSH_AUTH_SOCK:-}" ]]; then
                args+=(
                    --volume "$SSH_AUTH_SOCK:/tmp/ssh-agent.sock"
                    --env "SSH_AUTH_SOCK=/tmp/ssh-agent.sock"
                )
            fi
        fi

        if [[ $detach_container == "true" ]]; then
            args+=("--detach")
        fi

        if [[ $interactive == "true" ]]; then
            args+=("--interactive")
        fi

        if [[ $tty == "true" ]]; then
            args+=("--tty")
        fi

        if [[ -n "$container_name_param" ]]; then
            args+=("--name=$container_name_param")
        fi

        if [[ -t 1 ]]; then
            args+=(--tty)
        fi
        # Forward the host's Kubernetes config.
        args+=(
            # Need to forward the entire directory to allow creation and
            # deletion of config.lock.
            --volume "$HOME/.kube:/kube"
            --env "KUBECONFIG=/kube/config"
        )
        # Forward the GitHub output file, if available.
        if [[ "${GITHUB_OUTPUT:-}" ]]; then
            args+=(
                --volume "$GITHUB_OUTPUT:/tmp/github-output"
                --env "GITHUB_OUTPUT=/tmp/github-output"
            )
        fi
        if [[ "$(uname -s)" = Linux ]]; then
            args+=(
                --user "$(id -u):$(stat -c %g /var/run/docker.sock)"
            )

            if [[ $secrets == "true" ]]; then
                # Allow Docker-in-Docker by mounting the Docker socket in the
                # container. Host networking allows us to see ports created by
                # containers that we launch.
                args+=(
                    --volume "/var/run/docker.sock:/var/run/docker.sock"
                    --network host
                    --env "DOCKER_TLS_VERIFY=${DOCKER_TLS_VERIFY-}"
                    --env "DOCKER_HOST=${DOCKER_HOST-}"
                )

                # Forward Docker configuration too, if available.
                docker_dir=${DOCKER_CONFIG:-$HOME/.docker}
                if [[ -d "$docker_dir" ]]; then
                    args+=(
                        --volume "$docker_dir:/docker"
                        --env "DOCKER_CONFIG=/docker"
                    )
                fi
            fi

            # Override the Docker daemon we use to run the builder itself, if
            # requested.
            export DOCKER_HOST=${MZ_DEV_CI_BUILDER_DOCKER_HOST-${DOCKER_HOST-}}
            export DOCKER_TLS_VERIFY=${MZ_DEV_CI_BUILDER_DOCKER_TLS_VERIFY-${DOCKER_TLS_VERIFY-}}
            export DOCKER_CERT_PATH=${MZ_DEV_CI_BUILDER_DOCKER_CERT_PATH-${DOCKER_CERT_PATH-}}

            # Forward the host's buildkite-agent binary, if available.
            if command -v buildkite-agent > /dev/null 2>&1; then
                args+=(--volume "$(command -v buildkite-agent)":/usr/local/bin/buildkite-agent)
            fi

            # Install a persistent volume to hold Cargo metadata. We can't
            # forward the host's `~/.cargo` directly to the container, since
            # that can forward binaries in `~/.cargo/bin` that override the
            # version of Cargo installed in the container (!).
            args+=(--volume "mz-ci-builder-cargo:/cargo")
        else
            args+=(--user "$(id -u):1001")
        fi
        if [[ "${CI_BUILDER_SCCACHE:-}" ]]; then
            args+=(
                --env "RUSTC_WRAPPER=sccache"
                --env SCCACHE_BUCKET
            )
        fi
        # For git-worktrees add the original repository at its original path
        if [[ "$(git rev-parse --git-dir)" != "$(git rev-parse --git-common-dir)" ]]; then
            GIT_ROOT_DIR="$(git rev-parse --git-dir | sed -e "s#/.git/worktrees/.*#/#")"
            args+=(--volume "$GIT_ROOT_DIR:$GIT_ROOT_DIR")
        fi
        rm -f "$cid_file"
        image="$image_registry/ci-builder:$tag"
        # Try downloading the image a few times in case of registry flakiness
        if [[ "${CI:-}" ]]; then
            if ! docker inspect "$image" > /dev/null 2>&1; then
                docker pull "$image" || (sleep 3 && docker pull "$image") || (sleep 3 && docker pull "$image") || sleep 3
            fi
        fi
        docker run "${args[@]}" "$image" eatmydata "${docker_command[@]}"
        ;;
    root-shell)
        docker exec --interactive --tty --user 0:0 "$(<"$cid_file")" eatmydata ci/builder/root-shell.sh
        ;;
    *)
        printf "unknown command %q\n" "$cmd"
        exit 1
        ;;
esac