diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0012ed4854..5ec02468aa 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -70,8 +70,11 @@ jobs: npmAuthToken: "${{ secrets.NPM_TOKEN }}" EOF + # TODO: CI yarn.lock differs from local for some reason & causes errors # Install dependencies - yarn install --immutable + # yarn install --immutable + yarn config set -H enableImmutableInstalls false + yarn install if [ "${{ inputs.latest }}" = "true" ]; then ./scripts/release/main.ts --version "${{ github.event.inputs.version }}" --setupCi @@ -86,142 +89,68 @@ jobs: strategy: matrix: include: - - platform: x86_64-unknown-linux-gnu + - platform: linux runner: [self-hosted, Linux, X64] - suffix: x86-linux - # - platform: x86_64-pc-windows-gnu + target: x86_64-unknown-linux-musl + # - platform: windows # runner: [self-hosted, Linux, X64] - # suffix: x86-windows - - platform: x86_64-apple-darwin + # target: x86_64-pc-windows-musl + - platform: macos runner: [self-hosted, Linux, X64] - suffix: x86-mac - - platform: aarch64-apple-darwin + target: x86_64-apple-darwin + - platform: macos runner: [self-hosted, Linux, ARM64] - suffix: aarch64-mac + target: aarch64-apple-darwin + arch: arm64 runs-on: ${{ matrix.runner }} - container: - image: rust:1.82.0 steps: - uses: actions/checkout@v4 - - # Required for running in Docker - - name: Git LFS Pull - run: | - apt-get update - apt-get install git-lfs - git lfs pull - - - uses: actions/setup-node@v4 with: - node-version: 20 - - - run: corepack enable + lfs: 'true' - - name: Build - env: - # Expose correct target since we're building cross-platform - OVERRIDE_TARGET: ${{ matrix.platform }} - - # Expose env - CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER: x86_64-linux-gnu-gcc - CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER: x86_64-apple-darwin20.4-clang - CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER: aarch64-apple-darwin20.4-clang - CC_x86_64_apple_darwin: x86_64-apple-darwin20.4-clang - CXX_x86_64_apple_darwin: x86_64-apple-darwin20.4-clang++ - CC_aarch64_apple_darwin: aarch64-apple-darwin20.4-clang - CXX_aarch64_apple_darwin: aarch64-apple-darwin20.4-clang++ - MACOSX_DEPLOYMENT_TARGET: "10.7" - # Use RUSTFLAGS to skip building aws-lc-rs when targeting macOS - # CARGO_FEATURE_RUSTLS_NATIVE_CERTS: 0 - # CARGO_RUSTLS_NATIVE_CERTS: 0 - # RUSTFLAGS: "--cfg tokio_unstable ${{ contains(matrix.platform, 'apple-darwin') && '--cfg tokio_unstable --cfg rustls_native_certs --cfg aws_lc_rs' || '' }}" - - # Expose token for pulling GH artifacts - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - # Pull via Git CLI to improve reliability in CI - ENV CARGO_NET_GIT_FETCH_WITH_CLI: "true" + - name: Build using toolchain Dockerfile run: | - # Install required components for each platform - apt-get update - if [[ "${{ matrix.platform }}" == "x86_64-pc-windows-gnu" ]]; then - echo "Installing Windows components" - - apt-get install -y gcc-mingw-w64-x86-64 - elif [[ "${{ matrix.platform }}" == "x86_64-unknown-linux-gnu" ]]; then - echo "Installing Linux components" - - apt-get install -y gcc-x86-64-linux-gnu libc6-dev-amd64-cross - elif [[ "${{ matrix.platform }}" == "x86_64-apple-darwin" || "${{ matrix.platform }}" == "aarch64-apple-darwin" ]]; then - echo "Installing macOS components" - - apt-get install -y clang cmake patch libxml2-dev wget xz-utils curl - - # Install osxcross - git config --global --add safe.directory '*' - git clone https://github.com/tpoechtrager/osxcross /root/osxcross - ( - cd /root/osxcross - wget -nc https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.3.sdk.tar.xz - mv MacOSX11.3.sdk.tar.xz tarballs/ - UNATTENDED=yes OSX_VERSION_MIN=10.7 ./build.sh - ) - - # Add osxcross to PATH permanently - echo 'export PATH="/root/osxcross/target/bin:$PATH"' >> ~/.bashrc - export PATH="/root/osxcross/target/bin:$PATH" - - # Verify compiler installation - echo 'clang path:' - which x86_64-apple-darwin20.4-clang - echo 'clang version:' - x86_64-apple-darwin20.4-clang --version - fi - - # Install targets - rustup target add ${{ matrix.platform }} - - mkdir -p /root/.cargo && \ - echo '\ - [target.x86_64-unknown-linux-gnu]\n\ - linker = "x86_64-linux-gnu-gcc"\n\ - \n\ - [target.x86_64-pc-windows-gnu]\n\ - linker = "x86_64-w64-mingw32-gcc"\n\ - \n\ - [target.x86_64-apple-darwin]\n\ - linker = "x86_64-apple-darwin20.4-clang"\n\ - ar = "x86_64-apple-darwin20.4-ar"\n\ - \n\ - [target.aarch64-apple-darwin]\n\ - linker = "aarch64-apple-darwin20.4-clang"\n\ - ar = "aarch64-apple-darwin20.4-ar"\n\ - ' > /root/.cargo/config.toml - - # Build - cargo build --bin rivet --release --target ${{ matrix.platform }} + # Use Docker BuildKit + export DOCKER_BUILDKIT=1 + + # Build the binary using our Dockerfile + ARCH_ARG="${{ matrix.arch }}" + docker/toolchain/build.sh ${{ matrix.target }} + + # Make sure dist directory exists and binary is there + ls -la dist/ - name: Upload artifacts env: AWS_ACCESS_KEY_ID: ${{ secrets.R2_RELEASES_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_RELEASES_SECRET_ACCESS_KEY }} run: | + # Install dependencies for AWS CLI apt-get update - apt-get install -y awscli + apt-get install -y unzip curl - COMMIT_SHA_SHORT="${GITHUB_SHA::7}" - BINARY_PATH="target/${{ matrix.platform }}/release/rivet" - BINARY_NAME="rivet-${{ matrix.suffix }}" - if [[ "${{ matrix.platform }}" == "x86_64-pc-windows-gnu" ]]; then - BINARY_PATH="${BINARY_PATH}.exe" - BINARY_NAME="${BINARY_NAME}.exe" + # Install AWS CLI - use ARM version if running on ARM architecture + if [ "${{ matrix.arch }}" = "arm64" ]; then + curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" + else + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" fi + unzip awscliv2.zip + ./aws/install --update + + COMMIT_SHA_SHORT="${GITHUB_SHA::7}" + BINARY_PATH="dist/rivet-${{ matrix.target }}" + # Must specify --checksum-algorithm for compatibility with R2 aws s3 cp \ "${BINARY_PATH}" \ - "s3://rivet/${COMMIT_SHA_SHORT}/${BINARY_NAME}" \ + "s3://rivet/${COMMIT_SHA_SHORT}/rivet-${{ matrix.target }}" \ --region auto \ - --endpoint-url https://2a94c6a0ced8d35ea63cddc86c2681e7.r2.cloudflarestorage.com/rivet-releases + --endpoint-url https://2a94c6a0ced8d35ea63cddc86c2681e7.r2.cloudflarestorage.com/rivet-releases \ + --checksum-algorithm CRC32 docker: name: "Build & Push Docker Images" @@ -253,12 +182,6 @@ jobs: apt-get install git-lfs git lfs pull - - uses: actions/setup-node@v4 - with: - node-version: 20 - - - run: corepack enable - - name: Set outputs id: vars run: echo "sha_short=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT @@ -394,4 +317,3 @@ jobs: else ./scripts/release/main.ts --version "${{ github.event.inputs.version }}" --no-latest --completeCi fi - diff --git a/docker/toolchain/build.sh b/docker/toolchain/build.sh new file mode 100755 index 0000000000..611238d8e1 --- /dev/null +++ b/docker/toolchain/build.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -e + +# Default to x86_64-unknown-linux-gnu if no target specified +TARGET=${1:-x86_64-unknown-linux-musl} + +case $TARGET in + x86_64-unknown-linux-musl) + echo "Building for Linux x86_64 platform" + DOCKERFILE="linux.Dockerfile" + BINARY="rivet-$TARGET" + ;; + aarch64-apple-darwin) + echo "Building for macOS ARM64 platform" + DOCKERFILE="macos.Dockerfile" + BINARY="rivet-$TARGET" + ;; + x86_64-apple-darwin) + echo "Building for macOS x86_64 platform" + DOCKERFILE="macos.Dockerfile" + BINARY="rivet-$TARGET" + ;; + x86_64-pc-windows-gnu) + echo "Building for Windows platform" + DOCKERFILE="windows.Dockerfile" + BINARY="rivet-$TARGET.exe" + ;; + *) + echo "Unsupported target: $TARGET" + exit 1 + ;; +esac + +# Build docker image +DOCKER_BUILDKIT=1 docker build -f docker/toolchain/$DOCKERFILE -t rivet-cli-builder-$TARGET . + +# Extract binary +CONTAINER_ID=$(docker create rivet-cli-builder-$TARGET) +mkdir -p dist +docker cp "$CONTAINER_ID:/artifacts/$BINARY" dist/ +docker rm "$CONTAINER_ID" + +# Make binary executable (skip for Windows .exe files) +if [[ ! "$BINARY" == *.exe ]]; then + chmod +x dist/$BINARY +fi + +echo "Binary saved to: dist/$BINARY" diff --git a/docker/toolchain/linux.Dockerfile b/docker/toolchain/linux.Dockerfile new file mode 100644 index 0000000000..29b47b7941 --- /dev/null +++ b/docker/toolchain/linux.Dockerfile @@ -0,0 +1,55 @@ +# syntax=docker/dockerfile:1.4 +FROM rust:1.82.0 + +# Install dependencies +RUN apt-get update && apt-get install -y \ + musl-tools \ + libssl-dev \ + pkg-config \ + protobuf-compiler \ + ca-certificates \ + git-lfs \ + musl-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install musl target +RUN rustup target add x86_64-unknown-linux-musl + +# Set up OpenSSL for musl target +ENV SSL_VER=1.1.1w +RUN wget https://www.openssl.org/source/openssl-$SSL_VER.tar.gz \ + && tar -xzf openssl-$SSL_VER.tar.gz \ + && cd openssl-$SSL_VER \ + && ./Configure no-shared no-async --prefix=/musl --openssldir=/musl/ssl linux-x86_64 \ + && make -j$(nproc) \ + && make install_sw \ + && cd .. \ + && rm -rf openssl-$SSL_VER* + +# Configure OpenSSL env vars for the build +ENV OPENSSL_DIR=/musl \ + OPENSSL_INCLUDE_DIR=/musl/include \ + OPENSSL_LIB_DIR=/musl/lib \ + PKG_CONFIG_ALLOW_CROSS=1 + +# Set environment variables +ENV CARGO_INCREMENTAL=0 \ + RUSTFLAGS="--cfg tokio_unstable" \ + CARGO_NET_GIT_FETCH_WITH_CLI=true + +# Set working directory +WORKDIR /build + +# Build CLI instructions +COPY . . + +# Build for Linux with musl (static binary) +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + --mount=type=cache,target=/build/target \ + cargo build --bin rivet --release --target x86_64-unknown-linux-musl -v && \ + mkdir -p /artifacts && \ + cp target/x86_64-unknown-linux-musl/release/rivet /artifacts/rivet-x86_64-unknown-linux-musl + +# Default command to show help +CMD ["ls", "-la", "/artifacts"] diff --git a/docker/toolchain/macos.Dockerfile b/docker/toolchain/macos.Dockerfile new file mode 100644 index 0000000000..644dd92f19 --- /dev/null +++ b/docker/toolchain/macos.Dockerfile @@ -0,0 +1,80 @@ +# syntax=docker/dockerfile:1.4 +FROM rust:1.82.0 + +# Install dependencies +RUN apt-get update && apt-get install -y \ + git-lfs \ + protobuf-compiler \ + clang \ + cmake \ + patch \ + libxml2-dev \ + wget \ + xz-utils \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Install osxcross +RUN git config --global --add safe.directory '*' && \ + git clone https://github.com/tpoechtrager/osxcross /root/osxcross && \ + cd /root/osxcross && \ + wget -nc https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.3.sdk.tar.xz && \ + mv MacOSX11.3.sdk.tar.xz tarballs/ && \ + UNATTENDED=yes OSX_VERSION_MIN=10.7 ./build.sh + +# Add osxcross to PATH +ENV PATH="/root/osxcross/target/bin:$PATH" + +# Install macOS target +RUN rustup target add x86_64-apple-darwin aarch64-apple-darwin + +# Configure Cargo for cross-compilation +RUN mkdir -p /root/.cargo && \ + echo '\ +[target.x86_64-apple-darwin]\n\ +linker = "x86_64-apple-darwin20.4-clang"\n\ +ar = "x86_64-apple-darwin20.4-ar"\n\ +\n\ +[target.aarch64-apple-darwin]\n\ +linker = "aarch64-apple-darwin20.4-clang"\n\ +ar = "aarch64-apple-darwin20.4-ar"\n\ +' > /root/.cargo/config.toml + +# Set environment variables for cross-compilation +ENV CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=x86_64-apple-darwin20.4-clang \ + CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER=aarch64-apple-darwin20.4-clang \ + CC_x86_64_apple_darwin=x86_64-apple-darwin20.4-clang \ + CXX_x86_64_apple_darwin=x86_64-apple-darwin20.4-clang++ \ + CC_aarch64_apple_darwin=aarch64-apple-darwin20.4-clang \ + CXX_aarch64_apple_darwin=aarch64-apple-darwin20.4-clang++ \ + MACOSX_DEPLOYMENT_TARGET=10.7 \ + # Skip aws-lc-rs with rustls certs config when building for macOS + RUSTFLAGS="--cfg tokio_unstable --cfg rustls_native_certs --cfg aws_lc_rs" \ + CARGO_FEATURE_RUSTLS_NATIVE_CERTS=0 \ + CARGO_RUSTLS_NATIVE_CERTS=0 \ + CARGO_INCREMENTAL=0 \ + CARGO_NET_GIT_FETCH_WITH_CLI=true + +# Set working directory +WORKDIR /build + +# Copy the source code +COPY . . + +# Build for x86_64 macOS +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + --mount=type=cache,target=/build/target \ + cargo build --bin rivet --release --target x86_64-apple-darwin && \ + mkdir -p /artifacts && \ + cp target/x86_64-apple-darwin/release/rivet /artifacts/rivet-x86_64-apple-darwin + +# Build for ARM64 macOS +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + --mount=type=cache,target=/build/target \ + cargo build --bin rivet --release --target aarch64-apple-darwin && \ + cp target/aarch64-apple-darwin/release/rivet /artifacts/rivet-aarch64-apple-darwin + +# Default command to show help +CMD ["ls", "-la", "/artifacts"] diff --git a/docker/toolchain/windows.Dockerfile b/docker/toolchain/windows.Dockerfile new file mode 100644 index 0000000000..13b6c244e8 --- /dev/null +++ b/docker/toolchain/windows.Dockerfile @@ -0,0 +1,43 @@ +# syntax=docker/dockerfile:1.4 +FROM rust:1.82.0 + +# Install dependencies +RUN apt-get update && apt-get install -y \ + git-lfs \ + protobuf-compiler \ + gcc-mingw-w64-x86-64 \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Install target +RUN rustup target add x86_64-pc-windows-gnu + +# Configure Cargo for Windows cross-compilation +RUN mkdir -p /root/.cargo && \ + echo '\ +[target.x86_64-pc-windows-gnu]\n\ +linker = "x86_64-w64-mingw32-gcc"\n\ +' > /root/.cargo/config.toml + +# Set environment variables for cross-compilation +ENV CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER=x86_64-w64-mingw32-gcc \ + RUSTFLAGS="--cfg tokio_unstable" \ + CARGO_INCREMENTAL=0 \ + CARGO_NET_GIT_FETCH_WITH_CLI=true + +# Set working directory +WORKDIR /build + +# Copy the source code +COPY . . + +# Build for Windows +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + --mount=type=cache,target=/build/target \ + cargo build --bin rivet --release --target x86_64-pc-windows-gnu && \ + mkdir -p /artifacts && \ + cp target/x86_64-pc-windows-gnu/release/rivet.exe /artifacts/rivet-x86_64-pc-windows-gnu.exe + +# Default command to show help +CMD ["ls", "-la", "/artifacts"] diff --git a/docker/universal/Dockerfile b/docker/universal/Dockerfile index 8f16fc50e4..a525da1143 100644 --- a/docker/universal/Dockerfile +++ b/docker/universal/Dockerfile @@ -2,7 +2,7 @@ # MARK: Builder # TODO(RVT-4168): Compile libfdb from scratch for ARM -FROM --platform=linux/amd64 rust:1.82.0-bookworm AS builder +FROM --platform=linux/amd64 rust:1.85.0-bookworm AS builder ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update -y && \ @@ -90,7 +90,7 @@ FROM debian:12.9-slim AS server-slim ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update -y && \ - apt-get install -y ca-certificates openssl && \ + apt-get install -y ca-certificates openssl curl && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ curl -Lf -o /lib/libfdb_c.so "https://github.com/apple/foundationdb/releases/download/7.1.60/libfdb_c.x86_64.so" @@ -117,7 +117,7 @@ CMD ["/usr/bin/rivet-guard"] FROM --platform=linux/amd64 debian:12.9-slim AS base-runner ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update -y && \ - apt-get install -y ca-certificates openssl curl tini && \ + apt-get install -y ca-certificates openssl curl tini curl && \ curl -Lf -o /lib/libfdb_c.so "https://github.com/apple/foundationdb/releases/download/7.1.60/libfdb_c.x86_64.so" # MARK: Runner (Full) diff --git a/site/src/content/docs/self-hosting/server-spec.json b/site/src/content/docs/self-hosting/server-spec.json index 06bdf46368..4f872f87cb 100644 --- a/site/src/content/docs/self-hosting/server-spec.json +++ b/site/src/content/docs/self-hosting/server-spec.json @@ -145,6 +145,7 @@ "rivet": { "default": { "namespace": "rivet", + "instance_id": null, "default_cluster_id": null, "clusters": null, "provision": null, @@ -1643,6 +1644,11 @@ } ] }, + "instance_id": { + "description": "If specified, will use this as the default cluster ID.\n\nThis will have no effect if applied after the cluster has first ran.", + "type": "string", + "format": "uuid" + }, "job_run": { "default": null, "allOf": [