feat: pre-build Cargo dependencies to speed up serverless function compilation (#99)

* feat: pre-build Cargo dependencies to speed up serverless function compilation

* chore: revert bullshit

* feat: pre-build crates to same target

* fix: getrandom_backend custom

* fix: path

* feat: build only satellite and sputnik

* fix: inlcudes

* fix: pin

* feat: a pity

* docs: todo

* feat: try to improve rebuild

* docs: update comment

Author: peterpeterparker

Dockerfile.full

@@ -8,6 +8,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt update && apt install -y \
     bash \
     curl \
     build-essential \
+    jq \
     && rm -rf /var/lib/apt/lists/*
 
 # Create a working user
@@ -44,6 +45,11 @@ COPY --chown=apprunner:apprunner ./kit/setup/bootstrap ./kit/setup/bootstrap
 # Install core tools required for building WASM (same as in the main Juno repository)
 RUN ./kit/setup/bootstrap
 
+# Pre-build Cargo dependencies to speed up serverless function compilation when used in the actions
+COPY --chown=apprunner:apprunner ./kit/deps /home/apprunner/deps
+COPY --chown=apprunner:apprunner ./kit/setup/prebuild ./kit/setup/prebuild
+RUN ./kit/setup/prebuild
+
 # --- Runtime Stage ---
 FROM node:24-slim@sha256:76d0ed0ed93bed4f4376211e9d8fddac4d8b3fbdb54cc45955696001a3c91152
 
@@ -100,6 +106,12 @@ COPY --from=builder --chown=apprunner:apprunner /juno/target/juno-main/Cargo.loc
 COPY --from=builder --chown=apprunner:apprunner /juno/target/juno-main/Cargo.toml /juno/Cargo.toml
 COPY --from=builder --chown=apprunner:apprunner /juno/target/juno-main/rust-toolchain.toml /juno/rust-toolchain.toml
 
+# Copy pre-built Cargo dependencies to avoid recompiling them on each CI run
+# Particularly useful for Sputnik since its version is always the one bundled
+# in the image
+COPY --from=builder --chown=apprunner:apprunner /juno/target /juno/target
+COPY --from=builder --chown=apprunner:apprunner /home/apprunner/.cargo/registry /home/apprunner/.cargo/registry
+
 # Resolves cargo build error:
 # network failure seems to have happened
 # if a proxy or similar is necessary `net.git-fetch-with-cli` may help here

kit/deps/Cargo.toml

@@ -0,0 +1,22 @@
+# Dummy crate to pre-download and build the latest published junobuild-* dependencies into the
+# Cargo target cache, so developers don't have to download them on each CI run.
+[package]
+name = "deps"
+version = "0.0.1"
+edition = "2021"
+
+[lib]
+
+[dependencies]
+candid = "0.10.20"
+ic-cdk = "0.19.0"
+ic-cdk-macros = "0.19.0"
+serde = "1.0.225"
+junobuild-satellite = { version = "0.6.0", default-features = false }
+junobuild-macros = "0.4.0"
+junobuild-utils = "0.4.0"
+junobuild-shared = "0.8.0"
+junobuild-collections = "0.5.0"
+junobuild-storage = "0.7.0"
+junobuild-cdn = "0.7.0"
+junobuild-auth = "0.4.0"

kit/deps/src/lib.rs

@@ -0,0 +1,3 @@
+use junobuild_satellite::include_satellite;
+
+include_satellite!();

kit/setup/prebuild

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+JUNO_TARGET_DIR=/juno/target
+
+# Pre-build all cargo dependencies for Sputnik build
+cd "$JUNO_MAIN_DIR" \
+    && mkdir -p src/satellite/src && touch src/satellite/src/lib.rs \
+    && mkdir -p src/sputnik/src && touch src/sputnik/src/lib.rs \
+    && mkdir -p src/libs/macros/src && touch src/libs/macros/src/lib.rs \
+    && mkdir -p src/libs/satellite/src && touch src/libs/satellite/src/lib.rs \
+    && mkdir -p src/libs/shared/src && touch src/libs/shared/src/lib.rs \
+    && mkdir -p src/libs/utils/src && touch src/libs/utils/src/lib.rs \
+    && mkdir -p src/libs/collections/src && touch src/libs/collections/src/lib.rs \
+    && mkdir -p src/libs/storage/src && touch src/libs/storage/src/lib.rs \
+    && mkdir -p src/libs/cdn/src && touch src/libs/cdn/src/lib.rs \
+    && mkdir -p src/libs/auth/src && touch src/libs/auth/src/lib.rs \
+    && CARGO_TARGET_DIR="$JUNO_TARGET_DIR" ./docker/build --only-dependencies --sputnik
+
+# Pre-download latest published junobuild-* dependencies into the Cargo registry cache.
+# Note: even though we compile them here, the artifacts won't be reused at runtime because
+# Cargo fingerprints are tied to the workspace - the developer's project is a different
+# workspace, so everything will recompile. Only the download step is saved.
+# TODO: with a third party tool like sscache we might be able to cache the artifacts
+# https://web.mit.edu/rust-lang_v1.25/arch/amd64_ubuntu1404/share/doc/rust/html/cargo/guide/build-cache.html
+# https://github.com/mozilla/sccache
+cd /home/apprunner/deps
+RUSTFLAGS='--cfg getrandom_backend="custom" -A deprecated' cargo build --target wasm32-unknown-unknown --target-dir "$JUNO_TARGET_DIR" --release