Update nixpkgs and other flake dependencies (#425)

* nix-builder: fix cache variant name lookups

* {rocm,xpu}Packages: let `makeScope` define `callPackage`

Before this change we erroneously inherited `callPackage` from nixpkgs.
This `callPackage` does not have access to the new scope (the ROCm/XPU
package set). However, this error was masked by `makeScope` overriding
`callPackage`, so `final` had the correct version. The latest nixpkgs
supports overriding `callPackage`, which surfaced this bug.

* pythonWheelDepsCheckHook: do not use a conflicting name for disabling

* nvidia-cutlass-dsl: remove unused stdenv argument

* torch-bin: passing stdenv in the body is deprecated

* torch-bin: disable nixpkgs runtime deps check hook

We use our own hook.

* libffi_3_3: skip check (fails upstream)

* jupyter-server: disable pytest check

This check takes a long time, sometimes times out, and is already
checked upstream.

* flake: bump up nixpkgs

* rocmPackages.openmp-extras-runtime: libffi_3_2 is not needed anymore

* nix fmt

* XPU: fix `dlopen()` issue with loading Torch extension on recent glibc

`dlopen` by the Python interpreter failed with:

cannot enable executable stack as shared object requires: Invalid argument

Pass the `-z noexecstack` to the linker to mark the library as not
requiring an executable stack.

Author: danieldk

flake.lock

@@ -178,7 +178,7 @@ elseif(GPU_LANG STREQUAL "SYCL")
   endif()
 
 
-  set(sycl_link_flags "-fsycl;--offload-compress;-fsycl-targets=spir64_gen,spir64;-Xs;-device pvc,xe-lpg,ats-m150 -options ' -cl-intel-enable-auto-large-GRF-mode -cl-poison-unsupported-fp64-kernels -cl-intel-greater-than-4GB-buffer-required';")
+  set(sycl_link_flags "-Wl,-z,noexecstack;-fsycl;--offload-compress;-fsycl-targets=spir64_gen,spir64;-Xs;-device pvc,xe-lpg,ats-m150 -options ' -cl-intel-enable-auto-large-GRF-mode -cl-poison-unsupported-fp64-kernels -cl-intel-greater-than-4GB-buffer-required';")
   set(sycl_flags "-fsycl;-fhonor-nans;-fhonor-infinities;-fno-associative-math;-fno-approx-func;-fno-sycl-instrument-device-code;--offload-compress;-fsycl-targets=spir64_gen,spir64;")
   set(GPU_FLAGS "${sycl_flags}")
   set(GPU_ARCHES "")

kernel-builder/src/pyproject/templates/torch/preamble.cmake

@@ -2,13 +2,13 @@
   lib,
   pkgs,
 
-# Every `buildSets` argument is a list of build sets. Each build set is
-# a attrset of the form
-#
-#     { pkgs = <nixpkgs>, torch = <torch drv> }
-#
-# The Torch derivation is built as-is. So e.g. the ABI version should
-# already be set.
+  # Every `buildSets` argument is a list of build sets. Each build set is
+  # a attrset of the form
+  #
+  #     { pkgs = <nixpkgs>, torch = <torch drv> }
+  #
+  # The Torch derivation is built as-is. So e.g. the ABI version should
+  # already be set.
 }:
 
 let

nix-builder/lib/build.nix

@@ -30,11 +30,11 @@
             allOutputs python3.pkgs.nvidia-cutlass-dsl
           )
         );
-      buildSetLinkFarm = buildSet: pkgs.linkFarm buildSet.torch.variant (buildSetOutputs buildSet);
+      buildSetLinkFarm = buildSet: pkgs.linkFarm buildSet.variants.torch.arch (buildSetOutputs buildSet);
     in
     pkgs.linkFarm "packages-for-cache" (
       map (buildSet: {
-        name = buildSet.torch.variant;
+        name = buildSet.variants.torch.arch;
         path = buildSetLinkFarm buildSet;
       }) buildSets
     );

nix-builder/lib/cache.nix

@@ -18,17 +18,6 @@ in
 
   kernel-layout-check = prev.callPackage ./pkgs/kernel-layout-check { };
 
-  # Used by ROCm.
-  libffi_3_2 = final.libffi_3_3.overrideAttrs (
-    finalAttrs: _: {
-      version = "3.2.1";
-      src = final.fetchurl {
-        url = with finalAttrs; "https://gcc.gnu.org/pub/${pname}/${pname}-${version}.tar.gz";
-        hash = "sha256-0G67jh2aItGeONY/24OVQlPzm+3F1GIyoFZFaFciyjc=";
-      };
-    }
-  );
-
   nvtx = final.callPackage ./pkgs/nvtx { };
 
   metal-cpp = final.callPackage ./pkgs/metal-cpp { };
@@ -128,6 +117,13 @@ in
           };
         });
 
+        jupyter-server = python-super.jupyter-server.overrideAttrs (
+          _: prevAttrs: {
+            # Gets stuck sometimes, already tested in nixpkgs.
+            dontUsePytestCheck = true;
+          }
+        );
+
         nvidia-cutlass-dsl = python-self.callPackage ./pkgs/python-modules/nvidia-cutlass-dsl { };
 
         nvidia-cutlass-dsl-libs = python-self.callPackage ./pkgs/python-modules/nvidia-cutlass-dsl-libs { };

nix-builder/overlay.nix

@@ -15,7 +15,7 @@ pythonWheelDepsCheckHook() {
     echo "Finished executing pythonWheelDepsCheck"
 }
 
-if [ -z "${dontCheckRuntimeDeps-}" ]; then
+if [ -z "${dontCheckWheelDeps-}" ]; then
     echo "Using pythonWheelDepsCheckHook"
     # Ideally, this would be post-install, but we have to guarantee
     # that the relax hook runs before this, so we move it a phase

nix-builder/pkgs/python-modules/hooks/python-wheel-deps-check-hook.sh

@@ -1,6 +1,5 @@
 {
   lib,
-  stdenv,
   fetchPypi,
   python,
 

nix-builder/pkgs/python-modules/nvidia-cutlass-dsl/default.nix

@@ -125,13 +125,11 @@ let
   };
 
 in
-buildPythonPackage {
+buildPythonPackage.override { stdenv = effectiveStdenv; } {
   pname = "torch";
   inherit version;
   format = "wheel";
 
-  stdenv = effectiveStdenv;
-
   outputs = [
     "out" # output standard python package
     "cxxdev" # propagated deps for the cmake consumers of torch
@@ -143,6 +141,9 @@ buildPythonPackage {
     inherit url hash;
   };
 
+  # We use our own pythonWheelDepsHook.
+  dontCheckRuntimeDeps = true;
+
   nativeBuildInputs = [
     pythonRelaxWheelDepsHook
     pythonWheelDepsCheckHook

nix-builder/pkgs/python-modules/torch/binary/generic.nix

@@ -11,7 +11,6 @@
   hip-runtime-amd,
   hsa-rocr,
   perl,
-  rocm,
   rocm-core,
   rocm-device-libs,
   rocm-opencl,
@@ -34,7 +33,7 @@ let
 in
 stdenv.mkDerivation {
   pname = "rocm-clr";
-  version = rocm.version;
+  version = rocm-core.version;
 
   nativeBuildInputs = [
     markForRocmRootHook

nix-builder/pkgs/rocm-packages/clr.nix

@@ -3,7 +3,7 @@ final: prev:
 # Create a package for all components in the ROCm runfile metadata.
 prev.lib.mapAttrs (
   pname: metadata:
-  prev.callPackage ./generic.nix {
+  final.callPackage ./generic.nix {
     inherit pname;
     inherit (metadata) components deps version;
     rocmPackages = final;