Desktop: Improve Nix build times with more caching

Author: timon-schelling

.github/workflows/build.yml

@@ -643,7 +643,7 @@ jobs:
         run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache
 
       - name: 📦 Build Nix package
-        run: nix build .#graphite${{ inputs.debug && '-dev' || '' }} --no-link --print-out-paths
+        run: nix build .#graphite${{ inputs.debug && '-dev' || '' }} --no-link --print-out-paths --print-build-logs
 
       - name: 📤 Push to Nix cache
         env:
@@ -653,8 +653,16 @@ jobs:
           nix run nixpkgs#cachix -- authtoken $NIX_CACHE_AUTH_TOKEN
           nix build .#graphite${{ inputs.debug && '-dev' || '' }} --no-link --print-out-paths | nix run nixpkgs#cachix -- push $NIX_CACHE_NAME
 
+      - name: 📤 Push Dependencies to dev Nix cache
+        env:
+          NIX_CACHE_AUTH_TOKEN: ${{ secrets.NIX_CACHE_AUTH_TOKEN_DEV }}
+          NIX_CACHE_NAME: graphite-dev
+        run: |
+          nix run nixpkgs#cachix -- authtoken $NIX_CACHE_AUTH_TOKEN
+          nix build .#graphite${{ inputs.debug && '-dev' || '' }}.deps --no-link --print-out-paths | nix run nixpkgs#cachix -- push $NIX_CACHE_NAME
+
       - name: 🏗 Build Linux bundle
-        run: nix build .#graphite${{ inputs.debug && '-dev' || '' }}-bundle.tar.xz && cp ./result ./graphite-linux-bundle.tar.xz
+        run: nix build .#graphite-bundle${{ inputs.debug && '-dev' || '' }}.tar.xz && cp ./result ./graphite-linux-bundle.tar.xz
 
       - name: 📦 Upload Linux bundle
         uses: actions/upload-artifact@v6
@@ -692,7 +700,7 @@ jobs:
 
       - name: 🏗 Build Flatpak
         run: |
-          nix build .#graphite${{ inputs.debug && '-dev' || '' }}-flatpak-manifest
+          nix build .#graphite-flatpak-manifest${{ inputs.debug && '-dev' || '' }}
 
           rm -rf .flatpak
           mkdir -p .flatpak

.nix/default.nix

@@ -60,9 +60,9 @@ in
       graphite-raster-nodes-shaders = lib.call ./pkgs/graphite-raster-nodes-shaders.nix;
       graphite-branding = lib.call ./pkgs/graphite-branding.nix;
       graphite-bundle = (lib.call ./pkgs/graphite-bundle.nix) { };
-      graphite-dev-bundle = (lib.call ./pkgs/graphite-bundle.nix) { graphite = graphite-dev; };
+      graphite-bundle-dev = (lib.call ./pkgs/graphite-bundle.nix) { graphite = graphite-dev; };
       graphite-flatpak-manifest = (lib.call ./pkgs/graphite-flatpak-manifest.nix) { };
-      graphite-dev-flatpak-manifest = (lib.call ./pkgs/graphite-flatpak-manifest.nix) { graphite-bundle = graphite-dev-bundle; };
+      graphite-flatpak-manifest-dev = (lib.call ./pkgs/graphite-flatpak-manifest.nix) { graphite-bundle = graphite-bundle-dev; };
 
       # TODO: graphene-cli = lib.call ./pkgs/graphene-cli.nix;
 

.nix/deps/crane.nix

@@ -1,5 +1,17 @@
 { pkgs, inputs, ... }:
 
 {
-  lib = inputs.crane.mkLib pkgs;
+  lib = (inputs.crane.mkLib pkgs) // {
+    combineArtifacts =
+      name: artifacts:
+      pkgs.runCommand name
+        {
+          nativeBuildInputs = [ pkgs.zstd ];
+        }
+        ''
+          mkdir -p $out target
+          ${pkgs.lib.concatMapStringsSep "\n" (a: "tar -xf ${a}/target.tar.zst -C target") artifacts}
+          tar -cf $out/target.tar.zst -C target --use-compress-program "zstd -T0" .
+        '';
+  };
 }

.nix/pkgs/graphite.nix

@@ -14,125 +14,106 @@
 
 let
   branding = self.packages.${system}.graphite-branding;
-  cargoVendorDir = deps.crane.lib.vendorCargoDeps { inherit (info) src; };
-  resourcesCommon = {
-    pname = "${info.pname}-resources";
-    inherit (info) version src;
-    inherit cargoVendorDir;
-    strictDeps = true;
-    nativeBuildInputs = [
-      pkgs.pkg-config
-      pkgs.lld
-      pkgs.nodejs
-      pkgs.nodePackages.npm
-      pkgs.binaryen
-      pkgs.wasm-bindgen-cli_0_2_100
-      pkgs.wasm-pack
-      pkgs.cargo-about
-    ];
-    buildInputs = [ pkgs.openssl ];
-    env.CARGO_PROFILE = if dev then "dev" else "release";
-    cargoExtraArgs = "--target wasm32-unknown-unknown -p graphite-wasm-wrapper --no-default-features --features native";
-    doCheck = false;
-  };
-  resources = deps.crane.lib.buildPackage (
-    resourcesCommon
-    // {
-      cargoArtifacts = deps.crane.lib.buildDepsOnly resourcesCommon;
-
-      npmDeps = pkgs.importNpmLock {
-        npmRoot = "${info.src}/frontend";
-      };
-
-      npmRoot = "frontend";
-      npmConfigScript = "setup";
-      makeCacheWritable = true;
-
-      nativeBuildInputs = [
-        pkgs.importNpmLock.npmConfigHook
-        pkgs.removeReferencesTo
-      ]
-      ++ resourcesCommon.nativeBuildInputs;
-
-      prePatch = ''
-        mkdir branding
-        cp -r ${branding}/* branding
-        cp ${info.src}/.branding branding/.branding
-      '';
-
-      buildPhase = ''
-        export HOME="$TMPDIR"
-
-        pushd frontend
-        npm run native:build-${if dev then "dev" else "production"}
-        popd
-      '';
-
-      installPhase = ''
-        mkdir -p $out
-        cp -r frontend/dist/* $out/
-      '';
-
-      postFixup = ''
-        find "$out" -type f -exec remove-references-to -t "${cargoVendorDir}" '{}' +
-      '';
-    }
-  );
+  profile = if dev then "dev" else "release";
   libs = [
     pkgs.wayland
     pkgs.vulkan-loader
     pkgs.libGL
     pkgs.openssl
     pkgs.libraw
-
     # X11 Support
     pkgs.libxkbcommon
     pkgs.libXcursor
     pkgs.libxcb
     pkgs.libX11
   ];
+
   common = {
-    inherit (info) pname version src;
-    inherit cargoVendorDir;
-    strictDeps = true;
+    inherit (info) version src;
+    cargoVendorDir = deps.crane.lib.vendorCargoDeps { inherit (info) src; };
     buildInputs = libs;
-    nativeBuildInputs = [
-      pkgs.pkg-config
-      pkgs.cargo-about
-      pkgs.removeReferencesTo
-    ];
-    env = deps.cef.env // {
-      CARGO_PROFILE = if dev then "dev" else "release";
-    };
-    cargoExtraArgs = "-p graphite-desktop";
+    strictDeps = true;
     doCheck = false;
   };
+
+  wasmDeps = deps.crane.lib.buildDepsOnly (
+    common
+    // {
+      pname = "${info.pname}-wasm";
+      nativeBuildInputs = [
+        pkgs.pkg-config
+        pkgs.lld
+      ];
+      buildInputs = [ pkgs.openssl ];
+      env.CARGO_PROFILE = profile;
+      cargoExtraArgs = "--target wasm32-unknown-unknown -p graphite-wasm-wrapper --no-default-features --features native";
+    }
+  );
+
+  nativeDeps = deps.crane.lib.buildDepsOnly (
+    common
+    // {
+      pname = "${info.pname}-native";
+      nativeBuildInputs = [ pkgs.pkg-config ];
+      env = deps.cef.env // {
+        CARGO_PROFILE = profile;
+      };
+      cargoExtraArgs = "-p graphite-desktop-platform-linux";
+    }
+  );
+
+  depsCombined = deps.crane.lib.combineArtifacts "${info.pname}-deps-${info.version}" [
+    wasmDeps
+    nativeDeps
+  ];
 in
 
 deps.crane.lib.buildPackage (
   common
   // {
-    cargoArtifacts = deps.crane.lib.buildDepsOnly common;
+    inherit (info) pname;
 
-    env = common.env // {
-      RASTER_NODES_SHADER_PATH = self.packages.${system}.graphite-raster-nodes-shaders;
-      EMBEDDED_RESOURCES = resources;
-      GRAPHITE_GIT_COMMIT_HASH = self.rev or "unknown";
-      GRAPHITE_GIT_COMMIT_DATE = self.lastModified or "unknown";
-    };
+    cargoArtifacts = depsCombined;
+
+    buildInputs = libs;
+    nativeBuildInputs = [
+      pkgs.pkg-config
+      pkgs.lld
+      pkgs.nodejs
+      pkgs.nodePackages.npm
+      pkgs.binaryen
+      pkgs.wasm-bindgen-cli_0_2_100
+      pkgs.wasm-pack
+      pkgs.cargo-about
+      pkgs.removeReferencesTo
+      pkgs.importNpmLock.npmConfigHook
+    ];
 
     npmDeps = pkgs.importNpmLock {
       npmRoot = "${info.src}/frontend";
     };
     npmRoot = "frontend";
-    nativeBuildInputs = [
-      pkgs.importNpmLock.npmConfigHook
-      pkgs.nodePackages.npm
-    ]
-    ++ common.nativeBuildInputs;
+    npmConfigScript = "setup";
+    makeCacheWritable = true;
+
+    env = deps.cef.env // {
+      CARGO_PROFILE = profile;
+      RASTER_NODES_SHADER_PATH = self.packages.${system}.graphite-raster-nodes-shaders;
+      GRAPHITE_GIT_COMMIT_HASH = self.rev or "unknown";
+      GRAPHITE_GIT_COMMIT_DATE = self.lastModified or "unknown";
+    };
+
+    postPatch = ''
+      mkdir branding
+      cp -r ${branding}/* branding
+      cp ${info.src}/.branding branding/.branding
+    '';
 
     preBuild = ''
-      ${lib.getExe self.packages.${system}.tools.third-party-licenses}
+      # Prevent `package-installer.js` from trying to update npm dependencies
+      touch -r frontend/package-lock.json -d '+1 year' frontend/node_modules/.install-timestamp
+
+      export HOME="$TMPDIR"
     ''
     + (
       if self ? rev then
@@ -143,6 +124,10 @@ deps.crane.lib.buildPackage (
         ""
     );
 
+    buildPhaseCargoCommand = "cargo run build desktop${if dev then " debug" else ""}";
+
+    doNotPostBuildInstallCargoBinaries = true;
+
     installPhase = ''
       mkdir -p $out/bin
       cp target/${if dev then "debug" else "release"}/graphite $out/bin/graphite
@@ -161,12 +146,14 @@ deps.crane.lib.buildPackage (
     '';
 
     postFixup = ''
-      remove-references-to -t "${cargoVendorDir}" $out/bin/graphite
+      remove-references-to -t "${common.cargoVendorDir}" $out/bin/graphite
 
       patchelf \
         --set-rpath "${pkgs.lib.makeLibraryPath libs}:${deps.cef.env.CEF_PATH}" \
         --add-needed libGL.so \
         $out/bin/graphite
     '';
+
+    passthru.deps = depsCombined;
   }
 )