Desktop: Improve Nix build times with more caching (#4007)

Author: timon-schelling

.github/workflows/build.yml

@@ -643,7 +643,7 @@ jobs:
         run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache
 
       - name: 📦 Build Nix package
-        run: nix build .#graphite${{ inputs.debug && '-dev' || '' }} --no-link --print-out-paths
+        run: nix build .#graphite${{ inputs.debug && '-dev' || '' }} --no-link --print-out-paths --print-build-logs
 
       - name: 📤 Push to Nix cache
         env:
@@ -653,8 +653,16 @@ jobs:
           nix run nixpkgs#cachix -- authtoken $NIX_CACHE_AUTH_TOKEN
           nix build .#graphite${{ inputs.debug && '-dev' || '' }} --no-link --print-out-paths | nix run nixpkgs#cachix -- push $NIX_CACHE_NAME
 
+      - name: 📤 Push Dependencies to dev Nix cache
+        env:
+          NIX_CACHE_AUTH_TOKEN: ${{ secrets.NIX_CACHE_AUTH_TOKEN_DEV }}
+          NIX_CACHE_NAME: graphite-dev
+        run: |
+          nix run nixpkgs#cachix -- authtoken $NIX_CACHE_AUTH_TOKEN
+          nix build .#graphite${{ inputs.debug && '-dev' || '' }}.deps --no-link --print-out-paths | nix run nixpkgs#cachix -- push $NIX_CACHE_NAME
+
       - name: 🏗 Build Linux bundle
-        run: nix build .#graphite${{ inputs.debug && '-dev' || '' }}-bundle.tar.xz && cp ./result ./graphite-linux-bundle.tar.xz
+        run: nix build .#graphite-bundle${{ inputs.debug && '-dev' || '' }}.tar.xz && cp ./result ./graphite-linux-bundle.tar.xz
 
       - name: 📦 Upload Linux bundle
         uses: actions/upload-artifact@v6
@@ -692,7 +700,7 @@ jobs:
 
       - name: 🏗 Build Flatpak
         run: |
-          nix build .#graphite${{ inputs.debug && '-dev' || '' }}-flatpak-manifest
+          nix build .#graphite-flatpak-manifest${{ inputs.debug && '-dev' || '' }}
 
           rm -rf .flatpak
           mkdir -p .flatpak

.github/workflows/provide-shaders.yml

@@ -35,3 +35,11 @@ jobs:
             raster_nodes_shaders_entrypoint.wgsl \
             "${{ github.sha }} raster_nodes_shaders_entrypoint.wgsl" \
             ${{ secrets.ARTIFACTS_REPO_TOKEN }}
+
+      - name: 📤 Push shaders to dev Nix cache
+        env:
+          NIX_CACHE_AUTH_TOKEN: ${{ secrets.NIX_CACHE_AUTH_TOKEN_DEV }}
+          NIX_CACHE_NAME: graphite-dev
+        run: |
+          nix run nixpkgs#cachix -- authtoken $NIX_CACHE_AUTH_TOKEN
+          nix build .#graphite-raster-nodes-shaders --no-link --print-out-paths | nix run nixpkgs#cachix -- push $NIX_CACHE_NAME

.nix/default.nix

@@ -60,9 +60,9 @@ in
       graphite-raster-nodes-shaders = lib.call ./pkgs/graphite-raster-nodes-shaders.nix;
       graphite-branding = lib.call ./pkgs/graphite-branding.nix;
       graphite-bundle = (lib.call ./pkgs/graphite-bundle.nix) { };
-      graphite-dev-bundle = (lib.call ./pkgs/graphite-bundle.nix) { graphite = graphite-dev; };
+      graphite-bundle-dev = (lib.call ./pkgs/graphite-bundle.nix) { graphite = graphite-dev; };
       graphite-flatpak-manifest = (lib.call ./pkgs/graphite-flatpak-manifest.nix) { };
-      graphite-dev-flatpak-manifest = (lib.call ./pkgs/graphite-flatpak-manifest.nix) { graphite-bundle = graphite-dev-bundle; };
+      graphite-flatpak-manifest-dev = (lib.call ./pkgs/graphite-flatpak-manifest.nix) { graphite-bundle = graphite-bundle-dev; };
 
       # TODO: graphene-cli = lib.call ./pkgs/graphene-cli.nix;
 

.nix/deps/crane.nix

@@ -1,5 +1,28 @@
-{ pkgs, inputs, ... }:
+{
+  pkgs,
+  inputs,
+  ...
+}:
 
 {
-  lib = inputs.crane.mkLib pkgs;
+  lib = (inputs.crane.mkLib pkgs) // {
+    vendorCargoDepsFlatten =
+      src:
+      pkgs.stdenvNoCC.mkDerivation (finalAttrs: {
+        name = "graphite-cargo-vendored";
+        inherit src;
+
+        installPhase = ''
+          cp -rL --no-preserve=mode "$src" "$out"
+          chmod -R u+w "$out"
+          find "$out" -type f -print0 | xargs -r -0 sed -i "s|$src|$out|g"
+        '';
+
+        disallowedReferences = [ finalAttrs.src ];
+
+        dontUnpack = true;
+        dontConfigure = true;
+        dontBuild = true;
+      });
+  };
 }

.nix/pkgs/graphite-raster-nodes-shaders.nix

@@ -1,16 +1,16 @@
-{ info, deps, ... }:
+{
+  info,
+  deps,
+  self,
+  system,
+  ...
+}:
 
 (deps.crane.lib.overrideToolchain (_: deps.rustGPU.toolchain)).buildPackage {
-  pname = "raster-nodes-shaders";
+  pname = "graphite-raster-nodes-shaders";
   inherit (info) version src;
 
-  cargoVendorDir = deps.crane.lib.vendorMultipleCargoDeps {
-    inherit (deps.crane.lib.findCargoFiles (deps.crane.lib.cleanCargoSource info.src)) cargoConfigs;
-    cargoLockList = [
-      "${info.src}/Cargo.lock"
-      "${deps.rustGPU.toolchain.availableComponents.rust-src}/lib/rustlib/src/rust/library/Cargo.lock"
-    ];
-  };
+  inherit (self.packages.${system}.graphite) cargoVendorDir cargoArtifacts;
 
   strictDeps = true;
 

.nix/pkgs/graphite.nix

@@ -14,125 +14,107 @@
 
 let
   branding = self.packages.${system}.graphite-branding;
-  cargoVendorDir = deps.crane.lib.vendorCargoDeps { inherit (info) src; };
-  resourcesCommon = {
-    pname = "${info.pname}-resources";
-    inherit (info) version src;
-    inherit cargoVendorDir;
-    strictDeps = true;
-    nativeBuildInputs = [
-      pkgs.pkg-config
-      pkgs.lld
-      pkgs.nodejs
-      pkgs.nodePackages.npm
-      pkgs.binaryen
-      pkgs.wasm-bindgen-cli_0_2_100
-      pkgs.wasm-pack
-      pkgs.cargo-about
-    ];
-    buildInputs = [ pkgs.openssl ];
-    env.CARGO_PROFILE = if dev then "dev" else "release";
-    cargoExtraArgs = "--target wasm32-unknown-unknown -p graphite-wasm-wrapper --no-default-features --features native";
-    doCheck = false;
-  };
-  resources = deps.crane.lib.buildPackage (
-    resourcesCommon
-    // {
-      cargoArtifacts = deps.crane.lib.buildDepsOnly resourcesCommon;
-
-      npmDeps = pkgs.importNpmLock {
-        npmRoot = "${info.src}/frontend";
-      };
-
-      npmRoot = "frontend";
-      npmConfigScript = "setup";
-      makeCacheWritable = true;
-
-      nativeBuildInputs = [
-        pkgs.importNpmLock.npmConfigHook
-        pkgs.removeReferencesTo
-      ]
-      ++ resourcesCommon.nativeBuildInputs;
-
-      prePatch = ''
-        mkdir branding
-        cp -r ${branding}/* branding
-        cp ${info.src}/.branding branding/.branding
-      '';
-
-      buildPhase = ''
-        export HOME="$TMPDIR"
-
-        pushd frontend
-        npm run native:build-${if dev then "dev" else "production"}
-        popd
-      '';
-
-      installPhase = ''
-        mkdir -p $out
-        cp -r frontend/dist/* $out/
-      '';
-
-      postFixup = ''
-        find "$out" -type f -exec remove-references-to -t "${cargoVendorDir}" '{}' +
-      '';
-    }
-  );
   libs = [
     pkgs.wayland
     pkgs.vulkan-loader
     pkgs.libGL
     pkgs.openssl
     pkgs.libraw
-
     # X11 Support
     pkgs.libxkbcommon
     pkgs.libXcursor
     pkgs.libxcb
     pkgs.libX11
   ];
+
   common = {
     inherit (info) pname version src;
-    inherit cargoVendorDir;
-    strictDeps = true;
+    cargoVendorDir = deps.crane.lib.vendorCargoDepsFlatten (
+      deps.crane.lib.vendorMultipleCargoDeps {
+        inherit (deps.crane.lib.findCargoFiles (deps.crane.lib.cleanCargoSource info.src)) cargoConfigs;
+        cargoLockList = [
+          "${info.src}/Cargo.lock"
+          "${deps.rustGPU.toolchain.availableComponents.rust-src}/lib/rustlib/src/rust/library/Cargo.lock"
+        ];
+      }
+    );
     buildInputs = libs;
-    nativeBuildInputs = [
-      pkgs.pkg-config
-      pkgs.cargo-about
-      pkgs.removeReferencesTo
-    ];
-    env = deps.cef.env // {
-      CARGO_PROFILE = if dev then "dev" else "release";
-    };
-    cargoExtraArgs = "-p graphite-desktop";
+    strictDeps = true;
     doCheck = false;
   };
+
+  cargoArtifacts = deps.crane.lib.buildDepsOnly (
+    common
+    // {
+      nativeBuildInputs = [
+        pkgs.pkg-config
+        pkgs.lld
+      ];
+      env = deps.cef.env;
+      buildPhase =
+        let
+          profile = if dev then "dev" else "release";
+        in
+        ''
+          cargo check --profile ${profile} --locked -p graphite-desktop-platform-linux
+          cargo build --profile ${profile} --locked -p graphite-desktop-platform-linux
+
+          cargo check --profile ${profile} --target wasm32-unknown-unknown --locked -p graphite-wasm-wrapper --no-default-features --features native
+          cargo build --profile ${profile} --target wasm32-unknown-unknown --locked -p graphite-wasm-wrapper --no-default-features --features native
+
+          cargo check --locked -p third-party-licenses --features desktop
+          cargo build --locked -p third-party-licenses --features desktop
+
+          cargo check --profile ${profile} --locked -p graphite-desktop-bundle
+          cargo build --profile ${profile} --locked -p graphite-desktop-bundle
+        '';
+    }
+  );
 in
 
 deps.crane.lib.buildPackage (
   common
   // {
-    cargoArtifacts = deps.crane.lib.buildDepsOnly common;
+    inherit cargoArtifacts;
 
-    env = common.env // {
-      RASTER_NODES_SHADER_PATH = self.packages.${system}.graphite-raster-nodes-shaders;
-      EMBEDDED_RESOURCES = resources;
-      GRAPHITE_GIT_COMMIT_HASH = self.rev or "unknown";
-      GRAPHITE_GIT_COMMIT_DATE = self.lastModified or "unknown";
-    };
+    buildInputs = libs;
+    nativeBuildInputs = [
+      pkgs.pkg-config
+      pkgs.lld
+      pkgs.nodejs
+      pkgs.nodePackages.npm
+      pkgs.binaryen
+      pkgs.wasm-bindgen-cli_0_2_100
+      pkgs.wasm-pack
+      pkgs.cargo-about
+      pkgs.removeReferencesTo
+      pkgs.importNpmLock.npmConfigHook
+    ];
 
     npmDeps = pkgs.importNpmLock {
       npmRoot = "${info.src}/frontend";
     };
     npmRoot = "frontend";
-    nativeBuildInputs = [
-      pkgs.importNpmLock.npmConfigHook
-      pkgs.nodePackages.npm
-    ]
-    ++ common.nativeBuildInputs;
+    npmConfigScript = "setup";
+    makeCacheWritable = true;
+
+    env = deps.cef.env // {
+      RASTER_NODES_SHADER_PATH = self.packages.${system}.graphite-raster-nodes-shaders;
+      GRAPHITE_GIT_COMMIT_HASH = self.rev or "unknown";
+      GRAPHITE_GIT_COMMIT_DATE = self.lastModified or "unknown";
+    };
+
+    postPatch = ''
+      mkdir branding
+      cp -r ${branding}/* branding
+      cp ${info.src}/.branding branding/.branding
+    '';
 
     preBuild = ''
-      ${lib.getExe self.packages.${system}.tools.third-party-licenses}
+      # Prevent `package-installer.js` from trying to update npm dependencies
+      touch -r frontend/package-lock.json -d '+1 year' frontend/node_modules/.install-timestamp
+
+      export HOME="$TMPDIR"
     ''
     + (
       if self ? rev then
@@ -143,6 +125,10 @@ deps.crane.lib.buildPackage (
         ""
     );
 
+    buildPhaseCargoCommand = "cargo run build desktop${if dev then " debug" else ""}";
+
+    doNotPostBuildInstallCargoBinaries = true;
+
     installPhase = ''
       mkdir -p $out/bin
       cp target/${if dev then "debug" else "release"}/graphite $out/bin/graphite
@@ -161,12 +147,14 @@ deps.crane.lib.buildPackage (
     '';
 
     postFixup = ''
-      remove-references-to -t "${cargoVendorDir}" $out/bin/graphite
+      remove-references-to -t "${common.cargoVendorDir}" $out/bin/graphite
 
       patchelf \
         --set-rpath "${pkgs.lib.makeLibraryPath libs}:${deps.cef.env.CEF_PATH}" \
         --add-needed libGL.so \
         $out/bin/graphite
     '';
+
+    passthru.deps = cargoArtifacts;
   }
 )