electronic-structure · simonpintarelli · Mar 24, 2026 · Mar 24, 2026 · Mar 25, 2026 · Mar 25, 2026
diff --git a/ci/build-rocm.sh b/ci/build-rocm.sh
@@ -2,20 +2,22 @@
 
 set -xeuo pipefail
 
-export SPACK_SYSTEM_CONFIG_PATH=/user-environment/config
-
 # make sure we keep the stage direcorty
 spack config --scope=spack add config:build_stage:/dev/shm/spack-stage
 # we might need to install dependencies too, e.g. nlcglib in case of API changes
 spack config --scope=spack add config:install_tree:root:/dev/shm/spack-stage
 
 spack env create -d ./spack-env
 # add local repository with current sirius recipe
-spack -e ./spack-env repo add $REPO
+# spack -e ./spack-env repo add $REPO
+spack -e ./spack-env config add 'include:[/user-environment/config]'
 
 spack -e ./spack-env config add "packages:all:variants:[amdgpu_target=${ROCM_ARCH},amdgpu_target_sram_ecc=${ROCM_ARCH},+rocm]"
+# TODO: update once this is has changed upstream
+# spack -e ./spack-env config add "packages:all:providers:mpi:[cray-mpich@9.1.0+rocm]"
 
 spack -e ./spack-env add $SPEC
+spack -e ./spack-env add cray-mpich@9.1.0+rocm
 
 # build sirius from source
 spack -e ./spack-env develop -p $PWD sirius@develop

diff --git a/ci/cscs-beverin.yml b/ci/cscs-beverin.yml
@@ -32,7 +32,7 @@ build sirius mi300:
   variables:
     SPEC: sirius@develop +rocm +apps +tests +scalapack ~python +dftd4 +dftd3 +vdwxc build_type=RelWithDebInfo
     ROCM_ARCH: gfx942
-    REPO: ./ci/sirius-uenv-recipe-mi300/spack_repo/sirius_ci
+    REPO: ./ci/spack_repo/sirius_ci
   script:
     - |
       #
@@ -100,15 +100,14 @@ beverin test gpu band parallel:
     USE_MPI: 'YES'
     TEST_COMMAND: ctest -L gpu_band_parallel --timeout 3600
 
-# skip this, it wants to use cusolver
-# beverin test gpu k-point parallel:
-#   extends: [.run_tests_beverin]
-#   variables:
-#     SLURM_NTASKS: 12
-#     SLURM_GPUS_PER_TASK: 1
-#     SLURM_CPU_BIND: 'verbose'
-#     USE_MPI: 'YES'
-#     TEST_COMMAND: ctest -L gpu_k_point_parallel
+beverin test gpu k-point parallel:
+  extends: [.run_tests_beverin]
+  variables:
+    SLURM_NTASKS: 12
+    SLURM_GPUS_PER_TASK: 1
+    SLURM_CPU_BIND: 'verbose'
+    USE_MPI: 'YES'
+    TEST_COMMAND: ctest -L gpu_k_point_parallel
 
 # eiger test cpu serial:
 #   extends: [.run_tests_eiger]

diff --git a/ci/sirius-uenv-recipe-mi300/config.yaml b/ci/sirius-uenv-recipe-mi300/config.yaml
@@ -5,6 +5,6 @@ spack:
   repo: https://github.com/spack/spack.git
   packages:
     repo: https://github.com/spack/spack-packages.git
-    commit: 5f20b9190596e0b875141e8cee03f0d3847ad65c
+    commit: 876d656ad17366f92e2233d843ba285d655150dd # March 9
 store: /user-environment
 description: SIRIUS dependencies for SIRIUS CI
diff --git a/ci/sirius-uenv-recipe-mi300/environments.yaml b/ci/sirius-uenv-recipe-mi300/environments.yaml
@@ -1,16 +1,15 @@
 sirius-env:
   compiler: [gcc]
   network:
-      mpi: cray-mpich@8.1.32+rocm
+      mpi: cray-mpich@9.1.0+rocm
       specs: [ 'libfabric@2.3 +rocm' ]
-  unify: when_possible
+  unify: true
   specs:
-  - sirius@develop ~python +vdwxc +apps +tests +dlaf +scalapack +rocm +dftd4 +dftd3
+  - sirius@develop ~python +vdwxc +apps +tests +dlaf +scalapack +rocm +dftd4 +dftd3 ~elpa
   - kokkos@4.7.02+serial
-  - sirius@develop +elpa
   - nlcglib@1.3.0
   - eigen@3
-  - hip@6.3.3 ^mesa@23.3.6
+  - hip@7.1.0 ^mesa
   variants:
   - amdgpu_target=gfx942
   - amdgpu_target_sram_ecc=gfx942

diff --git a/ci/sirius-uenv-recipe-mi300/repo/packages b/ci/sirius-uenv-recipe-mi300/repo/packages
@@ -0,0 +1 @@
+../../spack_repo/sirius_ci/packages
diff --git a/ci/sirius-uenv-recipe-mi300/spack_repo/sirius_ci/packages b/ci/sirius-uenv-recipe-mi300/spack_repo/sirius_ci/packages
diff --git a/ci/spack_repo/sirius_ci/packages/nlcglib/package.py b/ci/spack_repo/sirius_ci/packages/nlcglib/package.py
@@ -0,0 +1,155 @@
+# Copyright Spack Project Developers. See COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack_repo.builtin.build_systems.cmake import CMakePackage
+from spack_repo.builtin.build_systems.cuda import CudaPackage
+from spack_repo.builtin.build_systems.rocm import ROCmPackage
+
+from spack.package import *
+
+
+class Nlcglib(CMakePackage, CudaPackage, ROCmPackage):
+    """Nonlinear CG methods for wave-function optimization in DFT."""
+
+    homepage = "https://github.com/simonpintarelli/nlcglib"
+    git = "https://github.com/simonpintarelli/nlcglib.git"
+    url = "https://github.com/simonpintarelli/nlcglib/archive/v0.9.tar.gz"
+
+    maintainers("simonpintarelli")
+
+    license("BSD-3-Clause")
+
+    version("develop", branch="develop")
+    version("1.3.0", sha256="d6adfe97407be2ecf41eda6f530e9724052f5298c5f884113969a9204530a745")
+    version("1.2.0", sha256="bb3676472cf7cc9effe06e416ecf4ef38e6c58c3e423dc70b8b6b32890bc89f2")
+    version("1.1.0", sha256="9e7c2eea84a5ce191bd9af08f6c890717f7b6e88be7bd15cfe774eb0e0dabd8a")
+    version("1.0b", sha256="086c46f06a117f267cbdf1df4ad42a8512689a9610885763f463469fb15e82dc")
+    version("0.9", sha256="8d5bc6b85ee714fb3d6480f767e7f43e5e7d569116cf60e48f533a7f50a37a08")
+
+    variant("openmp", default=True, description="Use OpenMP")
+    variant("tests", default=False, description="Build tests")
+    variant(
+        "build_type",
+        default="Release",
+        description="CMake build type",
+        values=("Debug", "Release", "RelWithDebInfo"),
+    )
+
+    variant(
+        "gpu_direct",
+        default=False,
+        description="Enable GPU direct. Required to support distributed wave-functions.",
+    )
+
+
+    depends_on("cxx", type="build")  # generated
+
+    depends_on("cmake@3.23:", type="build")
+    depends_on("mpi")
+    depends_on("lapack")
+    depends_on("fmt", when="@1.3:")
+
+    requires(
+        "^[virtuals=lapack] openblas",
+        "^[virtuals=lapack] intel-oneapi-mkl",
+        policy="one_of",
+        msg="Only mkl or openblas are supported for blas/lapack with ldak",
+    )
+
+    depends_on("kokkos~cuda~rocm", when="~cuda~rocm")
+    depends_on("kokkos+openmp", when="+openmp")
+
+    depends_on("googletest", type="build", when="+tests")
+    depends_on("nlohmann-json")
+    depends_on("kokkos+serial@4:", when="@1.1:")
+
+    # MKLConfig.cmake introduced in 2021.3
+    conflicts("intel-oneapi-mkl@:2021.2", when="^intel-oneapi-mkl")
+    conflicts("+gpu_direct", when="~rocm~cuda")
+
+    with when("@:0.9"):
+        conflicts("+rocm")
+        conflicts("^kokkos@4:")
+
+    with when("+rocm"):
+        variant("magma", default=True, description="Use magma eigenvalue solver (AMDGPU)")
+        depends_on("magma+rocm", when="+magma")
+        depends_on("kokkos+rocm")
+        depends_on("rocblas")
+        depends_on("rocsolver")
+
+    with when("+cuda"):
+        depends_on("kokkos+cuda_lambda+wrapper", when="%gcc")
+        depends_on("kokkos+cuda")
+        for arch in CudaPackage.cuda_arch_values:
+            depends_on(f"kokkos cuda_arch={arch}", when=f"cuda_arch={arch}")
+
+    def cmake_args(self):
+        options = [
+            self.define_from_variant("USE_OPENMP", "openmp"),
+            self.define_from_variant("BUILD_TESTS", "tests"),
+            self.define_from_variant("USE_ROCM", "rocm"),
+            self.define_from_variant("USE_GPU_DIRECT", "gpu_direct"),
+            self.define_from_variant("USE_MAGMA", "magma"),
+            self.define_from_variant("USE_CUDA", "cuda"),
+        ]
+
+        if self.spec.satisfies("^[virtuals=lapack] intel-oneapi-mkl"):
+            mkl_mapper = {
+                "threading": {"none": "sequential", "openmp": "gnu_thread", "tbb": "tbb_thread"},
+                "mpi": {"intel-oneapi-mpi": "intelmpi", "mpich": "mpich", "openmpi": "openmpi"},
+            }
+
+            mkl_threads = mkl_mapper["threading"][
+                self.spec["intel-oneapi-mkl"].variants["threads"].value
+            ]
+
+            mpi_provider = self.spec["mpi"].name
+            if mpi_provider in ["mpich", "cray-mpich", "mvapich", "mvapich2"]:
+                mkl_mpi = mkl_mapper["mpi"]["mpich"]
+            else:
+                mkl_mpi = mkl_mapper["mpi"][mpi_provider]
+
+            options.extend(
+                [
+                    self.define("LAPACK_VENDOR", "MKLONEAPI"),
+                    self.define("MKL_INTERFACE", "lp64"),
+                    self.define("MKL_THREADING", mkl_threads),
+                    self.define("MKL_MPI", mkl_mpi),
+                ]
+            )
+        else:
+            options.append(self.define("LAPACK_VENDOR", "OpenBLAS"))
+
+        if "+cuda%gcc" in self.spec:
+            options.append(
+                self.define("CMAKE_CXX_COMPILER", self["kokkos-nvcc-wrapper"].kokkos_cxx)
+            )
+
+        if "+cuda" in self.spec:
+            cuda_archs = self.spec.variants["cuda_arch"].value
+            if "@:0.9" in self.spec:
+                cuda_flags = " ".join(
+                    ["-gencode arch=compute_{0},code=sm_{0}".format(x) for x in cuda_archs]
+                )
+                options.append(self.define("CMAKE_CUDA_FLAGS", cuda_flags))
+            else:
+                options.append(self.define("CMAKE_CUDA_ARCHITECTURES", cuda_archs))
+
+        if "^cuda+allow-unsupported-compilers" in self.spec:
+            options.append(self.define("CMAKE_CUDA_FLAGS", "--allow-unsupported-compiler"))
+
+        if "+rocm" in self.spec:
+            archs = ",".join(self.spec.variants["amdgpu_target"].value)
+            options.extend(
+                [
+                    self.define("CMAKE_CXX_COMPILER", self.spec["hip"].hipcc),
+                    self.define("HIP_HCC_FLAGS", f"--amdgpu-target={archs}"),
+                    self.define(
+                        "CMAKE_CXX_FLAGS", f"--amdgpu-target={archs} --offload-arch={archs}"
+                    ),
+                ]
+            )
+
+        return options
diff --git a/ci/spack_repo/sirius_ci/packages/q_e_sirius/package.py b/ci/spack_repo/sirius_ci/packages/q_e_sirius/package.py
@@ -0,0 +1,100 @@
+# Copyright Spack Project Developers. See COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+# adapted from official quantum espresso package
+
+
+from spack_repo.builtin.build_systems.cmake import CMakePackage
+
+from spack.package import *
+
+
+class QESirius(CMakePackage):
+    """SIRIUS enabled fork of QuantumESPRESSO."""
+
+    homepage = "https://github.com/electronic-structure/q-e-sirius/"
+    url = "https://github.com/electronic-structure/q-e-sirius/archive/refs/tags/q-e-sirius/1.0.0.tar.gz"
+    git = "https://github.com/electronic-structure/q-e-sirius.git"
+
+    maintainers("simonpintarelli")
+
+    license("GPL-2.0-or-later")
+
+    version("develop-ristretto", branch="ristretto", submodules=True)
+    version("1.0.2", sha256="6885979d6c23b81b49d4a96c7f73f2eac618adccb0181bfb96ae2318664d9828")
+    version("1.0.1", sha256="512f982aa60fe9fd1cc588fa270e74427c66b62cb2d02ac1cb6cd07dcbe72204")
+    version("1.0.0", sha256="d85485db8e9252a0bcd67a6a348b2a74626030183199b0edeb97f14c33bca15b")
+
+    variant("openmp", default=True, description="Enables OpenMP support")
+    variant("libxc", default=False, description="Support functionals through libxc")
+    variant("sirius_apps", default=False, description="Build SIRIUS standalone binaries")
+    # Support for HDF5 has been added starting in QE 6.1.0 and is
+    # still experimental
+    variant(
+        "hdf5",
+        default="none",
+        description="Orbital and density data I/O with HDF5",
+        values=("parallel", "serial", "none"),
+        multi=False,
+    )
+
+    depends_on("fortran", type="build")
+    depends_on("c", type="build")
+    depends_on("cxx", type="build")
+
+    depends_on("sirius +fortran")
+    depends_on("sirius +apps", when="+sirius_apps")
+    depends_on("sirius ~apps", when="~sirius_apps")
+    depends_on("sirius +openmp", when="+openmp")
+    depends_on("sirius@develop", when="@develop-ristretto")
+
+    depends_on("mpi")
+    depends_on("elpa", when="+elpa")
+    depends_on("libxc", when="+libxc")
+    depends_on("fftw-api@3")
+    depends_on("blas")
+    depends_on("lapack")
+    depends_on("git", type="build")
+    depends_on("pkgconfig", type="build")
+
+    variant("scalapack", default=True, description="Enables scalapack support")
+
+    with when("+scalapack"):
+        depends_on("scalapack")
+        variant("elpa", default=False, description="Uses elpa as an eigenvalue solver")
+
+    # Versions of HDF5 prior to 1.8.16 lead to QE runtime errors
+    depends_on("hdf5@1.8.16:+fortran+hl+mpi", when="hdf5=parallel")
+    depends_on("hdf5@1.8.16:+fortran+hl~mpi", when="hdf5=serial")
+
+    with when("+openmp"):
+        requires("^fftw+openmp", when="^[virtuals=fftw-api] fftw")
+        requires("^openblas threads=openmp", when="^[virtuals=blas] openblas")
+        requires("^intel-oneapi-mkl threads=openmp", when="^[virtuals=blas] intel-oneapi-mkl")
+
+    def cmake_args(self):
+        args = [
+            "-DQE_ENABLE_SIRIUS=ON",
+            "-DQE_ENABLE_CUDA=OFF",
+            "-DQE_LAPACK_INTERNAL=OFF",
+            "-DQE_ENABLE_DOC=OFF",
+            "-DQE_ENABLE_MPI=ON",
+            self.define_from_variant("QE_ENABLE_OPENMP", "openmp"),
+            self.define_from_variant("QE_ENABLE_ELPA", "elpa"),
+            self.define_from_variant("QE_ENABLE_LIBXC", "libxc"),
+            self.define_from_variant("QE_ENABLE_SCALAPACK", "scalapack"),
+        ]
+
+        if not self.spec.satisfies("hdf5=none"):
+            args.append(self.define("QE_ENABLE_HDF5", True))
+
+        # Work around spack issue #19970 where spack sets
+        # rpaths for MKL just during make, but cmake removes
+        # them during make install.
+        if self.spec.satisfies("^[virtuals=lapack] intel-oneapi-mkl"):
+            args.append("-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON")
+        spec = self.spec
+        args.append(self.define("BLAS_LIBRARIES", spec["blas"].libs.joined(";")))
+        args.append(self.define("LAPACK_LIBRARIES", spec["lapack"].libs.joined(";")))
+
+        return args
diff --git a/ci/spack_repo/sirius_ci/packages/rocblas/0001-remove-blas-override-7.2.patch b/ci/spack_repo/sirius_ci/packages/rocblas/0001-remove-blas-override-7.2.patch
@@ -0,0 +1,19 @@
+diff --git a/projects/rocblas/clients/CMakeLists.txt b/projects/rocblas/clients/CMakeLists.txt
+index 0f32675..65e458d 100644
+--- a/projects/rocblas/clients/CMakeLists.txt
++++ b/projects/rocblas/clients/CMakeLists.txt
+@@ -163,14 +163,6 @@ if( BUILD_CLIENTS_BENCHMARKS OR BUILD_CLIENTS_TESTS)
+         find_package(PkgConfig)
+         if(NOT PKG_CONFIG_FOUND)
+           message( FATAL_ERROR "Could not find libblis and pkgconfig is not available" )
+-        else()
+-          pkg_search_module(PKGBLAS cblas)
+-          if(PKGBLAS_FOUND)
+-            set( BLAS_LIBRARY ${PKGBLAS_LIBRARIES} )
+-            set( BLAS_INCLUDE_DIR ${PKGBLAS_INCLUDE_DIRS} )
+-          else()
+-            message( FATAL_ERROR "Could not find libblis and pkgconfig can not find any other implementation of cblas" )
+-          endif()
+         endif()
+       endif()
+     else()
diff --git a/ci/spack_repo/sirius_ci/packages/rocblas/0001-remove-blas-override.patch b/ci/spack_repo/sirius_ci/packages/rocblas/0001-remove-blas-override.patch
@@ -0,0 +1,19 @@
+diff --git a/clients/CMakeLists.txt b/clients/CMakeLists.txt
+index 0f32675..65e458d 100644
+--- a/clients/CMakeLists.txt
++++ b/clients/CMakeLists.txt
+@@ -163,14 +163,6 @@ if( BUILD_CLIENTS_BENCHMARKS OR BUILD_CLIENTS_TESTS)
+         find_package(PkgConfig)
+         if(NOT PKG_CONFIG_FOUND)
+           message( FATAL_ERROR "Could not find libblis and pkgconfig is not available" )
+-        else()
+-          pkg_search_module(PKGBLAS cblas)
+-          if(PKGBLAS_FOUND)
+-            set( BLAS_LIBRARY ${PKGBLAS_LIBRARIES} )
+-            set( BLAS_INCLUDE_DIR ${PKGBLAS_INCLUDE_DIRS} )
+-          else()
+-            message( FATAL_ERROR "Could not find libblis and pkgconfig can not find any other implementation of cblas" )
+-          endif()
+         endif()
+       endif()
+     else()