add disk script

Why
===
* The nix modules disk builds too slowly.
* Moving the squashfs and tarball steps out of nix should make it
faster
* We can also speed up things and make the disk smaller with different
compression choices

What changed
===
* move squashfs and tarball steps into one script called disk-script
that you can run with `nix run -L .#disk-script` or `nix run -L .#disk-script-dev`
* run xargs cp in parallel
* use lz4 compression for squashfs and 1M block size
* use --best --recursive for pigz compression on tarball
* add output for the time each stage takes

Test plan
===
* ran it locally

```
xargs copy took 306 seconds
mksquashfs took 148 seconds
tar took 140 seconds
```
~10 minutes

```
$ du -h /tmp/tmp.Lj42PAdmKN/disk.sqsh
21G     /tmp/tmp.Lj42PAdmKN/disk.sqsh

$ du -h /tmp/tmp.Lj42PAdmKN/disk.sqsh.tar.gz
17G     /tmp/tmp.Lj42PAdmKN/disk.sqsh.tar.gz
```

Author: ryantm

pkgs/default.nix

@@ -7,18 +7,45 @@ let
   revstring_long = self.rev or "dirty";
   revstring = builtins.substring 0 7 revstring_long;
 
-  dev-module-ids = [ "python-3.10" "python-3.11" "nodejs-18" "nodejs-20" "nodejs-22" "go-1.21" "docker" "replit" "replit-rtld-loader" "ruby" "ruby-3.2" "postgresql-16" ];
-
-  mkPhonyOCI = pkgs.callPackage ./mk-phony-oci { ztoc-rs = self.inputs.ztoc-rs.packages.x86_64-linux.default; };
+  dev-module-ids = [
+    "python-3.10"
+    "python-3.11"
+    "nodejs-18"
+    "nodejs-20"
+    "nodejs-22"
+    "go-1.21"
+    "docker"
+    "replit"
+    "replit-rtld-loader"
+    "ruby"
+    "ruby-3.2"
+    "postgresql-16"
+  ];
+
+  mkPhonyOCI = pkgs.callPackage ./mk-phony-oci {
+    ztoc-rs = self.inputs.ztoc-rs.packages.x86_64-linux.default;
+  };
 
   bundle-fn = pkgs.callPackage ./bundle { inherit self; };
 
-  bundle-squashfs-fn = { moduleIds ? null, diskName ? "disk.raw" }:
+  bundle-squashfs-fn =
+    { moduleIds ? null
+    , diskName ? "disk.raw"
+    ,
+    }:
     pkgs.callPackage ./bundle-image {
       bundle = bundle-fn { inherit moduleIds; };
       inherit revstring diskName;
     };
 
+  disk-script-fn =
+    { moduleIds ? null
+    ,
+    }:
+    pkgs.callPackage ./disk-script {
+      bundle = bundle-fn { inherit moduleIds; };
+    };
+
 in
 rec {
   default = moduleit;
@@ -39,6 +66,12 @@ rec {
   # For prod use: builds the Nixmodules disk image
   bundle-image-tarball = pkgs.callPackage ./bundle-image-tarball { inherit bundle-image revstring; };
 
+  disk-script-dev = disk-script-fn {
+    moduleIds = dev-module-ids;
+  };
+
+  disk-script = disk-script-fn { };
+
   # For dev use: builds the shared Nixmodules disk
   bundle-squashfs = bundle-squashfs-fn {
     moduleIds = dev-module-ids;
@@ -52,13 +85,16 @@ rec {
   };
 
   phony-oci-bundles = mapAttrs
-    (moduleId: _:
-      mkPhonyOCI {
-        inherit moduleId;
-        module = self.deploymentModules.${moduleId};
-      })
+    (
+      moduleId: _:
+        mkPhonyOCI {
+          inherit moduleId;
+          module = self.deploymentModules.${moduleId};
+        }
+    )
     modules;
 
   deploymentModules = self.deploymentModules;
 
-} // modules
+}
+  // modules

pkgs/disk-script/default.nix

@@ -0,0 +1,57 @@
+{
+  writeShellApplication,
+  bundle,
+  squashfsTools,
+  gnutar,
+  pigz,
+  coreutils,
+  findutils,
+  closureInfo,
+}:
+
+let
+  diskClosureInfo = closureInfo { rootPaths = [ bundle ]; };
+in
+writeShellApplication {
+  name = "disk-script";
+  runtimeInputs = [
+    coreutils
+    findutils
+    gnutar
+    squashfsTools
+    pigz
+  ];
+  text = ''
+    set -x
+    TMP_DIR=$(mktemp -d)
+
+    cd "$TMP_DIR"
+
+    root="$TMP_DIR/root"
+    diskImage="$TMP_DIR/disk.sqsh"
+    tarball="$TMP_DIR/disk.sqsh.tar.gz"
+
+    (
+        mkdir -p "$root/nix/store" "$root/etc/nixmodules"
+
+        cp --archive --reflink=auto "${bundle}/etc/nixmodules/"* "$root/etc/nixmodules"
+
+        SECONDS=0
+        xargs -P "$(nproc)" cp -a --reflink=auto -t "$root/nix/store/" < "${diskClosureInfo}/store-paths"
+        echo "xargs copy took $SECONDS seconds" >&2
+
+        echo "making squashfs..."
+        SECONDS=0
+        mksquashfs "$root" "$diskImage" -force-uid 11000 -force-gid 11000 -comp lz4 -b 1M
+        echo "mksquashfs took $SECONDS seconds" >&2
+
+        SECONDS=0
+        tar --use-compress-program="pigz --best --recursive | pv" -Scf "$tarball" disk.sqsh 
+        echo "tar took $SECONDS seconds" >&2
+
+        echo Tarball created at "$tarball" >&2
+    ) 1>&2
+
+    echo "$tarball"
+  '';
+}