[#121] Improve CI speed

Author: markharding

.gitlab/ci/build_app.yml

@@ -58,7 +58,7 @@ app-web-image:
     - !reference [.release-workflow, rules]
   image:
     name: moby/buildkit:rootless
-    entrypoint: ['']
+    entrypoint: [""]
   before_script:
     - mkdir -p ~/.docker
     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > ~/.docker/config.json
@@ -71,8 +71,9 @@ app-web-image:
         --local dockerfile=./apps/polycentric \
         --opt build-arg:EXPO_PUBLIC_POLYCENTRIC_SEED_SERVERS=$EXPO_PUBLIC_POLYCENTRIC_SEED_SERVERS \
         --opt build-arg:SKIP_WORKSPACE_BUILD=1 \
-        --import-cache type=registry,ref=$CI_REGISTRY_IMAGE/app:$CI_COMMIT_SHA-cache \
-        --export-cache type=registry,ref=$CI_REGISTRY_IMAGE/app:$CI_COMMIT_SHA-cache,mode=max \
+        --import-cache type=registry,ref=$CI_REGISTRY_IMAGE/app:cache-$CI_DEFAULT_BRANCH \
+        --import-cache type=registry,ref=$CI_REGISTRY_IMAGE/app:cache-$CI_COMMIT_REF_SLUG \
+        --export-cache type=registry,ref=$CI_REGISTRY_IMAGE/app:cache-$CI_COMMIT_REF_SLUG,mode=max \
         --output type=image,name=$CI_REGISTRY_IMAGE/app:$CI_COMMIT_SHA,push=true
 
 .app-build-expo:

.gitlab/ci/build_js_sdks.yml

@@ -18,8 +18,17 @@
 
 .js-sdks-base:
   extends: .js-sdk-workflow
+  variables:
+    PNPM_STORE_DIR: ${CI_PROJECT_DIR}/.pnpm-store
+  cache:
+    key: js-sdks-pnpm-${CI_COMMIT_REF_SLUG}
+    fallback_keys:
+      - js-sdks-pnpm-${CI_DEFAULT_BRANCH}
+    paths:
+      - .pnpm-store
   before_script:
     - npm install -g pnpm@10
+    - pnpm config set store-dir ${PNPM_STORE_DIR}
     - pnpm config set @polycentric:registry https://${CI_SERVER_HOST}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/
     - pnpm install
 

.gitlab/ci/build_rn_sdk.yml

@@ -25,19 +25,16 @@ rn-android-build:
     - !reference [.app-workflow, rules]
     - !reference [.release-workflow, rules]
   cache:
-    - key:
-        files:
-          - packages/rs-core/Cargo.lock
-        prefix: rs-core-cargo-deps
+    - key: rs-core-cargo-deps-${CI_COMMIT_REF_SLUG}
+      fallback_keys:
+        - rs-core-cargo-deps-${CI_DEFAULT_BRANCH}
       paths:
         - .cargo/registry/index
         - .cargo/registry/cache
         - .cargo/git/db
-    - key:
-        files:
-          - packages/rs-core/Cargo.lock
-          - pnpm-lock.yaml
-        prefix: rs-core-target-${CI_JOB_NAME}
+    - key: rs-core-target-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}
+      fallback_keys:
+        - rs-core-target-${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}
       paths:
         - target
   script:

.gitlab/ci/build_rs_core.yml

@@ -20,18 +20,16 @@
   variables:
     RUST_CORE_PATH: packages/rs-core
   cache:
-    - key:
-        files:
-          - packages/rs-core/Cargo.lock
-        prefix: rs-core-cargo-deps
+    - key: rs-core-cargo-deps-${CI_COMMIT_REF_SLUG}
+      fallback_keys:
+        - rs-core-cargo-deps-${CI_DEFAULT_BRANCH}
       paths:
         - .cargo/registry/index
         - .cargo/registry/cache
         - .cargo/git/db
-    - key:
-        files:
-          - packages/rs-core/Cargo.lock
-        prefix: rs-core-target-${CI_JOB_NAME}
+    - key: rs-core-target-${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}
+      fallback_keys:
+        - rs-core-target-${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}
       paths:
         - target
 
@@ -92,22 +90,6 @@ rs-core-uniffi-web-build:
     - !reference [.rn-sdk-workflow, rules]
     - !reference [.app-workflow, rules]
     - !reference [.release-workflow, rules]
-  cache:
-    - key:
-        files:
-          - packages/rs-core/Cargo.lock
-        prefix: rs-core-cargo-deps
-      paths:
-        - .cargo/registry/index
-        - .cargo/registry/cache
-        - .cargo/git/db
-    - key:
-        files:
-          - packages/rs-core/Cargo.lock
-          - pnpm-lock.yaml
-        prefix: rs-core-target-${CI_JOB_NAME}
-      paths:
-        - target
   script:
     - cd packages/rs-core-uniffi-web
     - |

.gitlab/ci/build_services.yml

@@ -1,7 +1,16 @@
-
 .service-rust-base:
   image: ${RUST_IMAGE}
   stage: check
+  cache:
+    key: ${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}
+    fallback_keys:
+      - ${CI_JOB_NAME}-${CI_DEFAULT_BRANCH}
+      - ${CI_JOB_NAME}
+    paths:
+      - .cargo/registry/index/
+      - .cargo/registry/cache/
+      - .cargo/git/db/
+      - target/
 
 .service-format-check:
   extends: .service-rust-base
@@ -20,20 +29,10 @@
   script:
     - cargo test --package ${SERVICE_NAME}
 
-.service-build:
-  extends: .service-rust-base
-  stage: build
-  script:
-    - cargo build --release --package ${SERVICE_NAME}
-  artifacts:
-    paths:
-      - target/release/${SERVICE_NAME}
-    expire_in: 1 week
-
 .service-image:
   image:
     name: moby/buildkit:rootless
-    entrypoint: ['']
+    entrypoint: [""]
   before_script:
     - mkdir -p ~/.docker
     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > ~/.docker/config.json
@@ -44,8 +43,9 @@
         --frontend dockerfile.v0 \
         --local context=. \
         --local dockerfile=./services/${SERVICE_DIR} \
-        --import-cache type=registry,ref=$CI_REGISTRY_IMAGE/${SERVICE_NAME}:$CI_COMMIT_SHA-cache \
-        --export-cache type=registry,ref=$CI_REGISTRY_IMAGE/${SERVICE_NAME}:$CI_COMMIT_SHA-cache,mode=max \
+        --import-cache type=registry,ref=$CI_REGISTRY_IMAGE/${SERVICE_NAME}:cache-$CI_DEFAULT_BRANCH \
+        --import-cache type=registry,ref=$CI_REGISTRY_IMAGE/${SERVICE_NAME}:cache-$CI_COMMIT_REF_SLUG \
+        --export-cache type=registry,ref=$CI_REGISTRY_IMAGE/${SERVICE_NAME}:cache-$CI_COMMIT_REF_SLUG,mode=max \
         --output type=image,name=$CI_REGISTRY_IMAGE/${SERVICE_NAME}:$CI_COMMIT_SHA,push=true
 
 ########################################################################
@@ -90,11 +90,6 @@ server-clippy:
     - .server-workflow
     - .service-clippy
 
-server-build:
-  extends:
-    - .server-workflow
-    - .service-build
-
 server-image:
   extends:
     - .server-workflow
@@ -145,11 +140,6 @@ moderation-clippy:
     - .moderation-workflow
     - .service-clippy
 
-moderation-build:
-  extends:
-    - .moderation-workflow
-    - .service-build
-
 moderation-image:
   extends:
     - .moderation-workflow
@@ -203,11 +193,6 @@ push-notifications-test:
     - .push-notifications-workflow
     - .service-test
 
-push-notifications-build:
-  extends:
-    - .push-notifications-workflow
-    - .service-build
-
 push-notifications-image:
   extends:
     - .push-notifications-workflow

.gitlab/ci/scan_dependencies.yml

@@ -18,6 +18,9 @@ trivy:
     # Console output
     - trivy filesystem --scanners misconfig,vuln --exit-code 1 --severity CRITICAL .
   cache:
+    key: trivy-${CI_COMMIT_REF_SLUG}
+    fallback_keys:
+      - trivy-${CI_DEFAULT_BRANCH}
     paths:
       - .trivycache/
   artifacts:

apps/polycentric/Dockerfile

@@ -26,20 +26,17 @@ COPY packages packages
 COPY tools tools
 RUN pnpm install --frozen-lockfile
 RUN if [ -z "$SKIP_WORKSPACE_BUILD" ]; then pnpm run build; fi
-RUN cd apps/polycentric && npx expo export --platform web
 # Non symlinked packages and node_modules
 RUN pnpm --filter polycentric-app --prod --legacy deploy /deploy
+RUN cd /deploy && npx expo export --platform web
 
 # Stage 3: Run
 FROM base AS runner
 WORKDIR /app
 ENV NODE_ENV=production
 ENV PORT=8080
 
-COPY --from=build /deploy/node_modules ./node_modules
-COPY --from=build /deploy/package.json ./package.json
-COPY --from=build /app/apps/polycentric/dist ./dist
-COPY --from=build /app/apps/polycentric/server.js ./server.js
+COPY --from=build /deploy .
 
 EXPOSE 8080
 CMD ["node", "server.js"]

services/moderation/Dockerfile

@@ -1,23 +1,32 @@
-FROM rust:1.96-slim AS build
-
+FROM rust:1.96-slim AS chef
 WORKDIR /app
+COPY services/common/build-deps.sh build-deps.sh
+RUN sh build-deps.sh
+RUN cargo install cargo-chef --locked
 
-COPY Cargo.toml Cargo.toml
-COPY Cargo.lock Cargo.lock
-COPY protos protos
+FROM chef AS planner
+COPY Cargo.toml Cargo.lock ./
 COPY packages packages
 COPY services services
 COPY tools tools
+RUN cargo chef prepare --recipe-path recipe.json
 
-RUN services/common/build-deps.sh
-
-RUN cargo build --release --package moderation-service
-RUN cargo build --release --package moderation-migration
+FROM chef AS builder
+COPY --from=planner /app/recipe.json recipe.json
+COPY protos protos
+RUN cargo chef cook --release --recipe-path recipe.json
+COPY Cargo.toml Cargo.lock ./
+COPY packages packages
+COPY services services
+COPY tools tools
+RUN cargo build --release \
+    --package moderation-service \
+    --package moderation-migration
 
 FROM rust:1.96-slim
 WORKDIR /app
 COPY services/common/runtime-deps.sh runtime-deps.sh
 RUN sh runtime-deps.sh
-COPY --from=build /app/target/release/moderation-service /app/moderation-service
-COPY --from=build /app/target/release/moderation-migration /app/moderation-migration
-CMD ["/app/moderation-service"]
+COPY --from=builder /app/target/release/moderation-service /app/moderation-service
+COPY --from=builder /app/target/release/moderation-migration /app/moderation-migration
+CMD ["/app/moderation-service"]

services/push-notifications/Dockerfile

@@ -1,23 +1,32 @@
-FROM rust:1.96-slim AS build
-
+FROM rust:1.96-slim AS chef
 WORKDIR /app
+COPY services/common/build-deps.sh build-deps.sh
+RUN sh build-deps.sh
+RUN cargo install cargo-chef --locked
 
-COPY Cargo.toml Cargo.toml
-COPY Cargo.lock Cargo.lock
-COPY protos protos
+FROM chef AS planner
+COPY Cargo.toml Cargo.lock ./
 COPY packages packages
 COPY services services
 COPY tools tools
+RUN cargo chef prepare --recipe-path recipe.json
 
-RUN services/common/build-deps.sh
-
-RUN cargo build --release --package push-notifications-service
-RUN cargo build --release --package push-notifications-migration
+FROM chef AS builder
+COPY --from=planner /app/recipe.json recipe.json
+COPY protos protos
+RUN cargo chef cook --release --recipe-path recipe.json
+COPY Cargo.toml Cargo.lock ./
+COPY packages packages
+COPY services services
+COPY tools tools
+RUN cargo build --release \
+    --package push-notifications-service \
+    --package push-notifications-migration
 
 FROM rust:1.96-slim
 WORKDIR /app
 COPY services/common/runtime-deps.sh runtime-deps.sh
 RUN sh runtime-deps.sh
-COPY --from=build /app/target/release/push-notifications-service /app/push-notifications-service
-COPY --from=build /app/target/release/push-notifications-migration /app/push-notifications-migration
+COPY --from=builder /app/target/release/push-notifications-service /app/push-notifications-service
+COPY --from=builder /app/target/release/push-notifications-migration /app/push-notifications-migration
 CMD ["/app/push-notifications-service"]

services/server/Dockerfile

@@ -1,23 +1,32 @@
-FROM rust:1.96-slim AS build
-
+FROM rust:1.96-slim AS chef
 WORKDIR /app
+COPY services/common/build-deps.sh build-deps.sh
+RUN sh build-deps.sh
+RUN cargo install cargo-chef --locked
 
-COPY Cargo.toml Cargo.toml
-COPY Cargo.lock Cargo.lock
-COPY protos protos
+FROM chef AS planner
+COPY Cargo.toml Cargo.lock ./
 COPY packages packages
 COPY services services
 COPY tools tools
+RUN cargo chef prepare --recipe-path recipe.json
 
-RUN services/common/build-deps.sh
-
-RUN cargo build --release --package server
-RUN cargo build --release --package migration
+FROM chef AS builder
+COPY --from=planner /app/recipe.json recipe.json
+COPY protos protos
+RUN cargo chef cook --release --recipe-path recipe.json
+COPY Cargo.toml Cargo.lock ./
+COPY packages packages
+COPY services services
+COPY tools tools
+RUN cargo build --release \
+    --package server \
+    --package migration
 
 FROM rust:1.96-slim
 WORKDIR /app
 COPY services/common/runtime-deps.sh runtime-deps.sh
 RUN sh runtime-deps.sh
-COPY --from=build /app/target/release/server /app/server
-COPY --from=build /app/target/release/migration /app/migration
+COPY --from=builder /app/target/release/server /app/server
+COPY --from=builder /app/target/release/migration /app/migration
 CMD ["/app/server"]