diff --git a/.github/actions/docker-setup/action.yaml b/.github/actions/docker-setup/action.yaml index b9ba3a5d18..3d9422f97d 100644 --- a/.github/actions/docker-setup/action.yaml +++ b/.github/actions/docker-setup/action.yaml @@ -1,5 +1,5 @@ name: 'Docker Setup' -description: 'Set up Docker Buildx and log in to Docker Hub' +description: 'Set up Docker Buildx and log in to Docker Hub and GHCR' inputs: docker_username: description: 'Docker Hub username' @@ -22,6 +22,13 @@ runs: username: ${{ inputs.docker_username }} password: ${{ inputs.docker_password }} + - name: Log in to ghcr.io + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ inputs.github_token }} + # This will be used as a secret to authenticate with Git repo pulls - name: Create .netrc file run: | @@ -29,4 +36,3 @@ runs: echo "login x-access-token" >> ${{ runner.temp }}/netrc echo "password ${{ inputs.github_token }}" >> ${{ runner.temp }}/netrc shell: bash - diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 008228a8d7..121baffcdc 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -236,11 +236,43 @@ jobs: path: artifacts/${{ matrix.artifact }} if-no-files-found: error + # --------------------------------------------------------------------------- + # engine-base-images — publish engine-specific GHCR bases for this commit SHA + # --------------------------------------------------------------------------- + engine-base-images: + needs: [context] + name: "Engine Base ${{ matrix.base }}" + if: needs.context.outputs.is_fork != 'true' + strategy: + fail-fast: false + matrix: + include: + - base: engine-builder + - base: engine-runtime-full + - base: engine-runtime-slim + runs-on: ubuntu-24.04 + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - name: Log in to ghcr.io + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build & Push Engine Base + run: | + TAG_OVERRIDE=${{ needs.context.outputs.sha }} \ + ./scripts/docker-builder-base/build-push.sh ${{ matrix.base }} --push + # --------------------------------------------------------------------------- # docker-images — per-arch runtime images pushed to Docker Hub # --------------------------------------------------------------------------- docker-images: - needs: [context] + needs: [context, engine-base-images] name: "Docker ${{ matrix.arch_suffix }}" if: needs.context.outputs.is_fork != 'true' strategy: @@ -283,6 +315,7 @@ jobs: target: engine-full platforms: ${{ matrix.platform }} build-args: | + ENGINE_BASE_TAG=${{ needs.context.outputs.sha }} BUILD_FRONTEND=${{ steps.mode.outputs.build_frontend }} CARGO_BUILD_MODE=${{ steps.mode.outputs.cargo_build_mode }} - name: Build & Push (rivetdev/engine:slim) @@ -295,6 +328,7 @@ jobs: target: engine-slim platforms: ${{ matrix.platform }} build-args: | + ENGINE_BASE_TAG=${{ needs.context.outputs.sha }} BUILD_FRONTEND=${{ steps.mode.outputs.build_frontend }} CARGO_BUILD_MODE=${{ steps.mode.outputs.cargo_build_mode }} @@ -401,7 +435,7 @@ jobs: # ---- build TypeScript packages (turbo dep graph picks up native) ---- - name: Build TypeScript packages - run: pnpm build -F rivetkit -F '@rivetkit/*' -F '!@rivetkit/shared-data' -F '!@rivetkit/engine-frontend' -F '!@rivetkit/mcp-hub' -F '!@rivetkit/sqlite-native' -F '!@rivetkit/rivetkit-native' + run: pnpm build -F rivetkit -F '@rivetkit/*' -F '!@rivetkit/shared-data' -F '!@rivetkit/engine-frontend' -F '!@rivetkit/mcp-hub' -F '!@rivetkit/sqlite-native' -F '!@rivetkit/sqlite-wasm' -F '!@rivetkit/rivetkit-native' - name: Pack inspector run: npx turbo build:pack-inspector -F rivetkit diff --git a/CLAUDE.md b/CLAUDE.md index 9d860174ee..eab2ca7f00 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -76,6 +76,8 @@ cd self-host/compose/dev docker-compose up -d ``` +- Rebuild publish base images with `scripts/docker-builder-base/build-push.sh --push`; update `BASE_TAG` when rebuilding shared builder bases, while engine bases are published per commit in `publish.yaml`. + ### Git Commands ```bash # Use conventional commits with a single-line commit message, no co-author diff --git a/docker/builder-base/engine-builder.Dockerfile b/docker/builder-base/engine-builder.Dockerfile new file mode 100644 index 0000000000..cdea807f0a --- /dev/null +++ b/docker/builder-base/engine-builder.Dockerfile @@ -0,0 +1,42 @@ +# syntax=docker/dockerfile:1.10.0 +# Base image for Linux engine container builds. +# Pre-bakes Rust, Node.js 22, corepack, build dependencies, and the +# FoundationDB client library for each target architecture. +# +# Build & push: scripts/docker-builder-base/build-push.sh engine-builder --push +FROM mcr.microsoft.com/devcontainers/rust:1-1-bookworm + +ARG TARGETARCH + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + cmake \ + curl \ + g++ \ + git \ + gpg \ + libclang-dev \ + libpq-dev \ + libssl-dev \ + make \ + openssl \ + pkg-config \ + wget && \ + rustup toolchain install 1.91.0 && \ + rustup default 1.91.0 && \ + curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && \ + apt-get install -y --no-install-recommends nodejs && \ + corepack enable && \ + rm -rf /var/lib/apt/lists/* && \ + if [ "$TARGETARCH" = "arm64" ]; then \ + curl -Lf -o /lib/libfdb_c.so "https://github.com/apple/foundationdb/releases/download/7.3.68/libfdb_c.aarch64.so"; \ + else \ + curl -Lf -o /lib/libfdb_c.so "https://github.com/apple/foundationdb/releases/download/7.3.68/libfdb_c.x86_64.so"; \ + fi + +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true \ + COREPACK_ENABLE_DOWNLOAD_PROMPT=0 + +WORKDIR /app diff --git a/docker/builder-base/engine-runtime-full.Dockerfile b/docker/builder-base/engine-runtime-full.Dockerfile new file mode 100644 index 0000000000..54475b163c --- /dev/null +++ b/docker/builder-base/engine-runtime-full.Dockerfile @@ -0,0 +1,23 @@ +# syntax=docker/dockerfile:1.10.0 +# Base image for the full Linux engine runtime image. +# +# Build & push: scripts/docker-builder-base/build-push.sh engine-runtime-full --push +FROM mcr.microsoft.com/devcontainers/base:debian + +ARG TARGETARCH + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + dirmngr \ + gpg \ + openssl && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + if [ "$TARGETARCH" = "arm64" ]; then \ + curl -Lf -o /lib/libfdb_c.so "https://github.com/apple/foundationdb/releases/download/7.3.68/libfdb_c.aarch64.so"; \ + else \ + curl -Lf -o /lib/libfdb_c.so "https://github.com/apple/foundationdb/releases/download/7.3.68/libfdb_c.x86_64.so"; \ + fi diff --git a/docker/builder-base/engine-runtime-slim.Dockerfile b/docker/builder-base/engine-runtime-slim.Dockerfile new file mode 100644 index 0000000000..ccdc21b1ec --- /dev/null +++ b/docker/builder-base/engine-runtime-slim.Dockerfile @@ -0,0 +1,21 @@ +# syntax=docker/dockerfile:1.10.0 +# Base image for the slim Linux engine runtime image. +# +# Build & push: scripts/docker-builder-base/build-push.sh engine-runtime-slim --push +FROM mcr.microsoft.com/devcontainers/base:debian + +ARG TARGETARCH + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + openssl && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + if [ "$TARGETARCH" = "arm64" ]; then \ + curl -Lf -o /lib/libfdb_c.so "https://github.com/apple/foundationdb/releases/download/7.3.68/libfdb_c.aarch64.so"; \ + else \ + curl -Lf -o /lib/libfdb_c.so "https://github.com/apple/foundationdb/releases/download/7.3.68/libfdb_c.x86_64.so"; \ + fi diff --git a/docker/engine/Dockerfile b/docker/engine/Dockerfile index 21585f48f4..84752088e9 100644 --- a/docker/engine/Dockerfile +++ b/docker/engine/Dockerfile @@ -1,8 +1,10 @@ # syntax=docker/dockerfile:1.10.0 +ARG ENGINE_BASE_TAG=latest + # MARK: Builder # TODO(RVT-4168): Compile libfdb from scratch for ARM -FROM rust:1.91.0-trixie AS builder +FROM ghcr.io/rivet-dev/rivet/engine-base-builder:${ENGINE_BASE_TAG} AS builder # Docker automatically provides TARGETARCH ARG TARGETARCH @@ -13,37 +15,6 @@ ARG VITE_APP_API_URL=__SAME__ ARG VITE_APP_TURNSTILE_SITE_KEY= ARG OVERRIDE_GIT_SHA -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update -y && \ - apt-get install -y \ - curl \ - g++ \ - git \ - libclang-dev \ - libpq-dev \ - libssl-dev \ - pkg-config \ - ca-certificates \ - gpg \ - openssl \ - wget \ - cmake \ - make && \ - curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && \ - apt-get install -y nodejs && \ - corepack enable && \ - if [ "$TARGETARCH" = "arm64" ]; then \ - curl -Lf -o /lib/libfdb_c.so "https://github.com/apple/foundationdb/releases/download/7.3.68/libfdb_c.aarch64.so"; \ - else \ - curl -Lf -o /lib/libfdb_c.so "https://github.com/apple/foundationdb/releases/download/7.3.68/libfdb_c.x86_64.so"; \ - fi - -# Disable interactive prompt -ENV COREPACK_ENABLE_DOWNLOAD_PROMPT=0 - -# Pull via Git CLI to improve reliability in CI -ENV CARGO_NET_GIT_FETCH_WITH_CLI=true - WORKDIR /app COPY . . @@ -80,27 +51,7 @@ RUN \ cp target/$CARGO_BUILD_MODE/rivet-engine /app/dist/ # MARK: Engine (full, base) -FROM debian:13.1-slim AS engine-full-base - -# Docker automatically provides TARGETARCH -ARG TARGETARCH - -ENV DEBIAN_FRONTEND=noninteractive -# - Install curl for health checks -RUN apt-get update -y && \ - apt-get install -y \ - ca-certificates \ - openssl \ - curl \ - gpg \ - dirmngr && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - if [ "$TARGETARCH" = "arm64" ]; then \ - curl -Lf -o /lib/libfdb_c.so "https://github.com/apple/foundationdb/releases/download/7.3.68/libfdb_c.aarch64.so"; \ - else \ - curl -Lf -o /lib/libfdb_c.so "https://github.com/apple/foundationdb/releases/download/7.3.68/libfdb_c.x86_64.so"; \ - fi +FROM ghcr.io/rivet-dev/rivet/engine-base-runtime-full:${ENGINE_BASE_TAG} AS engine-full-base # MARK: Engine (Full) FROM engine-full-base AS engine-full @@ -113,26 +64,11 @@ ENTRYPOINT ["/usr/bin/rivet-engine"] CMD ["start"] # MARK: Engine (Slim) -FROM debian:13.1-slim AS engine-slim +FROM ghcr.io/rivet-dev/rivet/engine-base-runtime-slim:${ENGINE_BASE_TAG} AS engine-slim LABEL org.opencontainers.image.source=https://github.com/rivet-dev/rivet -# Docker automatically provides TARGETARCH -ARG TARGETARCH - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update -y && \ - apt-get install -y ca-certificates openssl curl && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - if [ "$TARGETARCH" = "arm64" ]; then \ - curl -Lf -o /lib/libfdb_c.so "https://github.com/apple/foundationdb/releases/download/7.3.68/libfdb_c.aarch64.so"; \ - else \ - curl -Lf -o /lib/libfdb_c.so "https://github.com/apple/foundationdb/releases/download/7.3.68/libfdb_c.x86_64.so"; \ - fi - COPY --from=builder /app/dist/rivet-engine /usr/bin/rivet-engine ENTRYPOINT ["/usr/bin/rivet-engine"] CMD ["start"] - diff --git a/scripts/docker-builder-base/build-push.sh b/scripts/docker-builder-base/build-push.sh index 73d23a206a..20d880b575 100755 --- a/scripts/docker-builder-base/build-push.sh +++ b/scripts/docker-builder-base/build-push.sh @@ -8,27 +8,71 @@ set -e # ./build-push.sh osxcross --push # Build and push osxcross base # ./build-push.sh linux-musl # Build only (no push) # ./build-push.sh all --push # Build and push all bases (parallel) +# TAG_OVERRIDE=engine-base-v1 ./build-push.sh engine-builder --push # -# Available bases: osxcross, linux-musl, linux-gnu, windows-mingw +# Available bases: osxcross, linux-musl, linux-gnu, windows-mingw, +# engine-builder, engine-runtime-full, engine-runtime-slim # -# Images are tagged with the git commit SHA that built them: +# Images default to the current git commit SHA unless TAG_OVERRIDE is set: # ghcr.io/rivet-dev/rivet/builder-base-osxcross: +# ghcr.io/rivet-dev/rivet/engine-base-builder: # -# After pushing, update BASE_TAG in .github/workflows/publish.yaml -# to reference the new tag. +# After pushing shared builder bases, update BASE_TAG in +# .github/workflows/publish.yaml to reference the new tag. SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" DOCKERFILE_DIR="$REPO_ROOT/docker/builder-base" REGISTRY="ghcr.io/rivet-dev/rivet" -TAG="$(git -C "$REPO_ROOT" rev-parse --short HEAD)" +TAG="${TAG_OVERRIDE:-$(git -C "$REPO_ROOT" rev-parse --short HEAD)}" -BASES="osxcross linux-musl linux-gnu windows-mingw" +BASES="osxcross linux-musl linux-gnu windows-mingw engine-builder engine-runtime-full engine-runtime-slim" +MULTIARCH_BUILDER="rivet-multiarch" + +image_for_base() { + local name="$1" + case "$name" in + engine-builder) + echo "$REGISTRY/engine-base-builder:$TAG" + ;; + engine-runtime-full) + echo "$REGISTRY/engine-base-runtime-full:$TAG" + ;; + engine-runtime-slim) + echo "$REGISTRY/engine-base-runtime-slim:$TAG" + ;; + *) + echo "$REGISTRY/builder-base-$name:$TAG" + ;; + esac +} + +is_multiarch_base() { + local name="$1" + case "$name" in + engine-builder|engine-runtime-full|engine-runtime-slim) + return 0 + ;; + *) + return 1 + ;; + esac +} + +ensure_multiarch_builder() { + if docker buildx inspect "$MULTIARCH_BUILDER" >/dev/null 2>&1; then + docker buildx use "$MULTIARCH_BUILDER" >/dev/null + else + docker buildx create --name "$MULTIARCH_BUILDER" --driver docker-container --use >/dev/null + fi + docker buildx inspect "$MULTIARCH_BUILDER" --bootstrap >/dev/null +} build_one() { local name="$1" local dockerfile="$DOCKERFILE_DIR/${name}.Dockerfile" - local image="$REGISTRY/builder-base-${name}:${TAG}" + local image + image="$(image_for_base "$name")" if [ ! -f "$dockerfile" ]; then echo "ERROR: Dockerfile not found: $dockerfile" @@ -36,15 +80,27 @@ build_one() { fi echo "==> Building $image" - DOCKER_BUILDKIT=1 docker build -f "$dockerfile" -t "$image" "$REPO_ROOT" + if is_multiarch_base "$name"; then + ensure_multiarch_builder + DOCKER_BUILDKIT=1 docker buildx build --builder "$MULTIARCH_BUILDER" --platform linux/amd64 -f "$dockerfile" -t "$image" --load "$REPO_ROOT" + else + DOCKER_BUILDKIT=1 docker build -f "$dockerfile" -t "$image" "$REPO_ROOT" + fi echo "==> Built: $image" } push_one() { local name="$1" - local image="$REGISTRY/builder-base-${name}:${TAG}" + local dockerfile="$DOCKERFILE_DIR/${name}.Dockerfile" + local image + image="$(image_for_base "$name")" echo "==> Pushing $image" - docker push "$image" + if is_multiarch_base "$name"; then + ensure_multiarch_builder + DOCKER_BUILDKIT=1 docker buildx build --builder "$MULTIARCH_BUILDER" --platform linux/amd64,linux/arm64 -f "$dockerfile" -t "$image" --push "$REPO_ROOT" + else + docker push "$image" + fi echo "==> Pushed: $image" } @@ -113,4 +169,4 @@ fi echo "" echo "Done. Tag: $TAG" echo "" -echo "Update FROM lines in Dockerfiles to use tag: $TAG" +echo "Update BASE_TAG to use tag: $TAG if you rebuilt shared builder bases" diff --git a/scripts/publish/src/ci/bin.ts b/scripts/publish/src/ci/bin.ts index 26f4d3e680..4f80a9c1df 100644 --- a/scripts/publish/src/ci/bin.ts +++ b/scripts/publish/src/ci/bin.ts @@ -337,7 +337,6 @@ program `npm install rivetkit@${tag}`, `npm install @rivetkit/react@${tag}`, `npm install @rivetkit/rivetkit-native@${tag}`, - `npm install @rivetkit/sqlite-wasm@${tag}`, `npm install @rivetkit/workflow-engine@${tag}`, "```", "", diff --git a/scripts/publish/src/lib/packages.ts b/scripts/publish/src/lib/packages.ts index 1d8af80c98..f0eb9a1e26 100644 --- a/scripts/publish/src/lib/packages.ts +++ b/scripts/publish/src/lib/packages.ts @@ -8,12 +8,13 @@ * those optionals at install time, so they must exist on the registry before * anyone installs the meta. * - * NOTE: `@rivetkit/sqlite-native` is deliberately NOT discovered here. The - * sqlite-native Rust crate is now statically linked into - * `@rivetkit/rivetkit-native` via `libsqlite3-sys` + the `rivetkit-sqlite-native` - * workspace dep, so the standalone npm package is redundant. The committed - * `rivetkit-typescript/packages/sqlite-native/*` tree is deprecated and stays - * on the registry at its last published version. + * NOTE: `@rivetkit/sqlite-native` and `@rivetkit/sqlite-wasm` are deliberately + * NOT discovered here. The sqlite-native Rust crate is now statically linked + * into `@rivetkit/rivetkit-native` via `libsqlite3-sys` + + * the `rivetkit-sqlite-native` workspace dep, so the standalone npm package is + * redundant. The old sqlite-wasm package was removed from the workspace but + * its package.json remains for compatibility. Both stay on the registry at + * their last published versions. */ import { execSync } from "node:child_process"; import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; @@ -41,6 +42,7 @@ export const EXCLUDED = new Set([ "@rivetkit/engine-frontend", "@rivetkit/mcp-hub", "@rivetkit/sqlite-native", + "@rivetkit/sqlite-wasm", "example-agent-os", "example-agent-os-e2e", ]); diff --git a/scripts/publish/src/local/cut-release.ts b/scripts/publish/src/local/cut-release.ts index 3a2534d28c..7e6f620030 100644 --- a/scripts/publish/src/local/cut-release.ts +++ b/scripts/publish/src/local/cut-release.ts @@ -154,7 +154,7 @@ async function main() { await $({ stdio: "inherit", cwd: repoRoot, - })`pnpm build -F rivetkit -F @rivetkit/* -F !@rivetkit/shared-data -F !@rivetkit/engine-frontend -F !@rivetkit/mcp-hub -F !@rivetkit/sqlite-native -F !@rivetkit/rivetkit-native`; + })`pnpm build -F rivetkit -F @rivetkit/* -F !@rivetkit/shared-data -F !@rivetkit/engine-frontend -F !@rivetkit/mcp-hub -F !@rivetkit/sqlite-native -F !@rivetkit/sqlite-wasm -F !@rivetkit/rivetkit-native`; await $({ stdio: "inherit", cwd: repoRoot,