From 90e9bf7b12b59c3aaca877ce3f39e196de6d8f4d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 6 Feb 2025 12:09:36 -0800 Subject: [PATCH 1/8] RTD: Fix GA Integration (#1718) GA was dropped from RTD in early Oct, 2024. This adds it again. --- docs/requirements.txt | 1 + docs/source/conf.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index 9f3c2ba702..a23405e002 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -10,5 +10,6 @@ sphinx>=5.3,<7.2 sphinx-copybutton sphinx-design sphinx_rtd_theme>=1.1.1 +sphinxcontrib-googleanalytics sphinxcontrib-svg2pdfconverter sphinxcontrib.programoutput diff --git a/docs/source/conf.py b/docs/source/conf.py index f93e0a2763..b3daa58296 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -41,6 +41,7 @@ 'breathe', 'sphinx_copybutton', 'sphinx_design', + 'sphinxcontrib.googleanalytics', 'sphinxcontrib.programoutput', 'sphinxcontrib.rsvgconverter', 'matplotlib.sphinxext.plot_directive'] @@ -48,6 +49,10 @@ if not on_rtd: extensions.append('sphinx.ext.githubpages') +# Google Analytics +googleanalytics_id = "G-3P1ZT3SQQ5" +googleanalytics_enabled = True + # breathe config breathe_projects = {'openPMD-api': '../xml'} breathe_default_project = 'openPMD-api' From b800efd84b98bef3ea72ea28bee47a49bf468cc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 21 Mar 2025 23:42:17 +0100 Subject: [PATCH 2/8] CI: Move Ubuntu 20.04 workflows to 22.04 (#1731) --- .clang-tidy | 2 +- .../spack.yaml | 10 +- .../spack.yaml | 20 +-- .../spack.yaml | 18 +-- .../spack.yaml | 8 +- .../clangtidy_nopy_ompi_h5_ad2/spack.yaml | 20 +-- .../spack.yaml | 18 +-- .../spack.yaml | 30 +++-- .../{install_nvcc11.sh => install_nvcc12.sh} | 13 +- ...all_nvhpc21-11.sh => install_nvhpc25-1.sh} | 2 +- .github/workflows/intel.yml | 29 +--- .github/workflows/linux.yml | 124 +++++++++--------- .github/workflows/nvidia.yml | 14 +- .github/workflows/tooling.yml | 20 +-- examples/3a_write_thetaMode_serial.cpp | 4 +- examples/5_write_parallel.cpp | 2 +- examples/8_benchmark_parallel.cpp | 2 +- examples/8b_benchmark_read_parallel.cpp | 2 +- src/IO/HDF5/HDF5IOHandler.cpp | 2 +- test/ParallelIOTest.cpp | 16 +-- test/SerialIOTest.cpp | 19 +-- 21 files changed, 176 insertions(+), 199 deletions(-) rename .github/ci/spack-envs/{clang7_nopy_nompi_h5_libcpp => clang11_nopy_nompi_h5_libcpp}/spack.yaml (80%) rename .github/ci/spack-envs/{clang7_nopy_ompi_h5_ad2 => clang11_nopy_ompi_h5_ad2}/spack.yaml (78%) rename .github/ci/spack-envs/{clang7_nopy_ompi_h5_ad2_libcpp => clang11_nopy_ompi_h5_ad2_libcpp}/spack.yaml (80%) rename .github/ci/spack-envs/{clang14_py311_nompi_h5_ad2 => clang15_py311_nompi_h5_ad2}/spack.yaml (89%) rename .github/ci/spack-envs/{gcc7_py36_ompi_h5_ad2 => gcc12_py36_ompi_h5_ad2}/spack.yaml (79%) rename .github/ci/spack-envs/{clang8_py38_mpich_h5_ad2 => gcc13_py312_mpich_h5_ad2}/spack.yaml (66%) rename .github/workflows/dependencies/{install_nvcc11.sh => install_nvcc12.sh} (70%) rename .github/workflows/dependencies/{install_nvhpc21-11.sh => install_nvhpc25-1.sh} (91%) diff --git a/.clang-tidy b/.clang-tidy index 034a863660..3d5d0cd25c 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -2,5 +2,5 @@ # FIXME: all performance-* reports # FIXME: all cert-* reports # FIXME: all bugprone-* reports -Checks: -*,bugprone-*,-bugprone-unhandled-self-assignment,-bugprone-parent-virtual-call,-bugprone-narrowing-conversions,-bugprone-exception-escape,-bugprone-string-literal-with-embedded-nul,cppcoreguidelines-slicing,mpi-*,readability-non-const-parameter,performance-*,modernize-*,-modernize-use-trailing-return-type,-modernize-use-bool-literals,-modernize-avoid-c-arrays,-modernize-use-auto,-modernize-return-braced-init-list +Checks: -*,bugprone-*,-bugprone-unhandled-self-assignment,-bugprone-parent-virtual-call,-bugprone-narrowing-conversions,-bugprone-exception-escape,-bugprone-string-literal-with-embedded-nul,cppcoreguidelines-slicing,mpi-*,readability-non-const-parameter,performance-*,modernize-*,-modernize-use-trailing-return-type,-modernize-use-bool-literals,-modernize-avoid-c-arrays,-modernize-use-auto,-modernize-return-braced-init-list,-bugprone-easily-swappable-parameters HeaderFilterRegex: '((^(?!\/share\/openPMD\/).*)*include\/openPMD\/.+\.hpp|src\/^(?!binding).+\.cpp$)' diff --git a/.github/ci/spack-envs/clang7_nopy_nompi_h5_libcpp/spack.yaml b/.github/ci/spack-envs/clang11_nopy_nompi_h5_libcpp/spack.yaml similarity index 80% rename from .github/ci/spack-envs/clang7_nopy_nompi_h5_libcpp/spack.yaml rename to .github/ci/spack-envs/clang11_nopy_nompi_h5_libcpp/spack.yaml index 9e22b6a7f4..97be9833b2 100644 --- a/.github/ci/spack-envs/clang7_nopy_nompi_h5_libcpp/spack.yaml +++ b/.github/ci/spack-envs/clang11_nopy_nompi_h5_libcpp/spack.yaml @@ -12,7 +12,7 @@ spack: all: target: [x86_64] variants: ~mpi ~fortran - compiler: [clang@7.0.1] + compiler: [clang@11.1.0] compilers: - compiler: @@ -22,13 +22,13 @@ spack: cxxflags: -stdlib=libc++ ldflags: -stdlib=libc++ modules: [] - operating_system: ubuntu20.04 + operating_system: ubuntu22.04 paths: - cc: /usr/bin/clang-7 - cxx: /usr/bin/clang++-7 + cc: /usr/bin/clang-11 + cxx: /usr/bin/clang++-11 f77: null fc: null - spec: clang@7.0.1 + spec: clang@11.1.0 target: x86_64 config: diff --git a/.github/ci/spack-envs/clang7_nopy_ompi_h5_ad2/spack.yaml b/.github/ci/spack-envs/clang11_nopy_ompi_h5_ad2/spack.yaml similarity index 78% rename from .github/ci/spack-envs/clang7_nopy_ompi_h5_ad2/spack.yaml rename to .github/ci/spack-envs/clang11_nopy_ompi_h5_ad2/spack.yaml index fa2e588608..f9a821a720 100644 --- a/.github/ci/spack-envs/clang7_nopy_ompi_h5_ad2/spack.yaml +++ b/.github/ci/spack-envs/clang11_nopy_ompi_h5_ad2/spack.yaml @@ -12,25 +12,25 @@ spack: packages: adios2: - variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 + variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 ~blosc2 ~mgard cmake: externals: - - spec: cmake@3.23.0 + - spec: cmake@3.31.5 prefix: /usr buildable: False openmpi: externals: - - spec: openmpi@2.1.1 + - spec: openmpi@4.1.2 prefix: /usr buildable: False perl: externals: - - spec: perl@5.26.1 + - spec: perl@5.34.0 prefix: /usr buildable: False python: externals: - - spec: python@3.8.0 + - spec: python@3.10.12 prefix: /usr buildable: False all: @@ -38,7 +38,7 @@ spack: variants: ~fortran providers: mpi: [openmpi] - compiler: [clang@7.0.0] + compiler: [clang@11.1.0] compilers: - compiler: @@ -46,13 +46,13 @@ spack: extra_rpaths: [] flags: {} modules: [] - operating_system: ubuntu20.04 + operating_system: ubuntu22.04 paths: - cc: /usr/lib/llvm-7/bin/clang - cxx: /usr/lib/llvm-7/bin/clang++ + cc: /usr/bin/clang-11 + cxx: /usr/bin/clang++-11 f77: /usr/bin/gfortran fc: /usr/bin/gfortran - spec: clang@7.0.0 + spec: clang@11.1.0 target: x86_64 config: diff --git a/.github/ci/spack-envs/clang7_nopy_ompi_h5_ad2_libcpp/spack.yaml b/.github/ci/spack-envs/clang11_nopy_ompi_h5_ad2_libcpp/spack.yaml similarity index 80% rename from .github/ci/spack-envs/clang7_nopy_ompi_h5_ad2_libcpp/spack.yaml rename to .github/ci/spack-envs/clang11_nopy_ompi_h5_ad2_libcpp/spack.yaml index 8a8dd00024..2dd6b004d1 100644 --- a/.github/ci/spack-envs/clang7_nopy_ompi_h5_ad2_libcpp/spack.yaml +++ b/.github/ci/spack-envs/clang11_nopy_ompi_h5_ad2_libcpp/spack.yaml @@ -15,22 +15,22 @@ spack: variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 cmake: externals: - - spec: cmake@3.23.0 + - spec: cmake@3.31.5 prefix: /usr buildable: False openmpi: externals: - - spec: openmpi@2.1.1 + - spec: openmpi@4.1.2 prefix: /usr buildable: False perl: externals: - - spec: perl@5.26.1 + - spec: perl@5.34.0 prefix: /usr buildable: False python: externals: - - spec: python@3.8.0 + - spec: python@3.10.12 prefix: /usr buildable: False all: @@ -38,7 +38,7 @@ spack: variants: ~fortran providers: mpi: [openmpi] - compiler: [clang@7.0.1] + compiler: [clang@11.1.0] compilers: - compiler: @@ -48,13 +48,13 @@ spack: cxxflags: -stdlib=libc++ ldflags: -stdlib=libc++ modules: [] - operating_system: ubuntu20.04 + operating_system: ubuntu22.04 paths: - cc: /usr/bin/clang-7 - cxx: /usr/bin/clang++-7 + cc: /usr/bin/clang-11 + cxx: /usr/bin/clang++-11 f77: /usr/bin/gfortran fc: /usr/bin/gfortran - spec: clang@7.0.1 + spec: clang@11.1.0 target: x86_64 config: diff --git a/.github/ci/spack-envs/clang14_py311_nompi_h5_ad2/spack.yaml b/.github/ci/spack-envs/clang15_py311_nompi_h5_ad2/spack.yaml similarity index 89% rename from .github/ci/spack-envs/clang14_py311_nompi_h5_ad2/spack.yaml rename to .github/ci/spack-envs/clang15_py311_nompi_h5_ad2/spack.yaml index 06e9d1ac40..18b2179b3e 100644 --- a/.github/ci/spack-envs/clang14_py311_nompi_h5_ad2/spack.yaml +++ b/.github/ci/spack-envs/clang15_py311_nompi_h5_ad2/spack.yaml @@ -34,7 +34,7 @@ spack: variants: ~fortran providers: mpi: [openmpi] - compiler: [clang@14.0.0] + compiler: [clang@15.0.7] compilers: - compiler: @@ -44,11 +44,11 @@ spack: modules: [] operating_system: ubuntu22.04 paths: - cc: /usr/lib/llvm-14/bin/clang - cxx: /usr/lib/llvm-14/bin/clang++ + cc: /usr/bin/clang-15 + cxx: /usr/bin/clang++-15 f77: /usr/bin/gfortran fc: /usr/bin/gfortran - spec: clang@14.0.0 + spec: clang@15.0.7 target: x86_64 config: diff --git a/.github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/spack.yaml b/.github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/spack.yaml index 1543ec794c..26fe755392 100644 --- a/.github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/spack.yaml +++ b/.github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/spack.yaml @@ -12,25 +12,25 @@ spack: packages: adios2: - variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 + variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 ~mgard cmake: externals: - - spec: cmake@3.23.0 + - spec: cmake@3.31.5 prefix: /usr buildable: False openmpi: externals: - - spec: openmpi@2.1.1 + - spec: openmpi@4.1.2 prefix: /usr buildable: False perl: externals: - - spec: perl@5.26.1 + - spec: perl@5.34.0 prefix: /usr buildable: False python: externals: - - spec: python@3.8.0 + - spec: python@3.10.12 prefix: /usr buildable: False all: @@ -38,7 +38,7 @@ spack: variants: ~fortran providers: mpi: [openmpi] - compiler: [clang@10.0.0] + compiler: [clang@14.0.0] compilers: - compiler: @@ -46,13 +46,13 @@ spack: extra_rpaths: [] flags: {} modules: [] - operating_system: ubuntu20.04 + operating_system: ubuntu22.04 paths: - cc: /usr/lib/llvm-10/bin/clang - cxx: /usr/lib/llvm-10/bin/clang++ + cc: /usr/bin/clang-14 + cxx: /usr/bin/clang++-14 f77: /usr/bin/gfortran fc: /usr/bin/gfortran - spec: clang@10.0.0 + spec: clang@14.0.0 target: x86_64 config: diff --git a/.github/ci/spack-envs/gcc7_py36_ompi_h5_ad2/spack.yaml b/.github/ci/spack-envs/gcc12_py36_ompi_h5_ad2/spack.yaml similarity index 79% rename from .github/ci/spack-envs/gcc7_py36_ompi_h5_ad2/spack.yaml rename to .github/ci/spack-envs/gcc12_py36_ompi_h5_ad2/spack.yaml index 8785650cf2..fbad0397d0 100644 --- a/.github/ci/spack-envs/gcc7_py36_ompi_h5_ad2/spack.yaml +++ b/.github/ci/spack-envs/gcc12_py36_ompi_h5_ad2/spack.yaml @@ -15,28 +15,28 @@ spack: variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 cmake: externals: - - spec: cmake@3.23.0 + - spec: cmake@3.31.5 prefix: /usr buildable: False openmpi: externals: - - spec: openmpi@2.1.1 + - spec: openmpi@4.1.2 prefix: /usr buildable: False perl: externals: - - spec: perl@5.26.1 + - spec: perl@5.34.0 prefix: /usr buildable: False python: externals: - - spec: python@3.6.3 + - spec: python@3.10.12 prefix: /usr buildable: False all: target: [x86_64] variants: ~fortran - compiler: [gcc@7.0.0] + compiler: [gcc@12.3.0] compilers: - compiler: @@ -44,13 +44,13 @@ spack: extra_rpaths: [] flags: {} modules: [] - operating_system: ubuntu20.04 + operating_system: ubuntu22.04 paths: - cc: /usr/bin/gcc-7 - cxx: /usr/bin/g++-7 + cc: /usr/bin/gcc-12 + cxx: /usr/bin/g++-12 f77: /usr/bin/gfortran fc: /usr/bin/gfortran - spec: gcc@7.0.0 + spec: gcc@12.3.0 target: x86_64 config: diff --git a/.github/ci/spack-envs/clang8_py38_mpich_h5_ad2/spack.yaml b/.github/ci/spack-envs/gcc13_py312_mpich_h5_ad2/spack.yaml similarity index 66% rename from .github/ci/spack-envs/clang8_py38_mpich_h5_ad2/spack.yaml rename to .github/ci/spack-envs/gcc13_py312_mpich_h5_ad2/spack.yaml index a6d568611f..fc1235b7f5 100644 --- a/.github/ci/spack-envs/clang8_py38_mpich_h5_ad2/spack.yaml +++ b/.github/ci/spack-envs/gcc13_py312_mpich_h5_ad2/spack.yaml @@ -6,31 +6,33 @@ # spack: specs: - - adios2@2.7.1 - - hdf5 + - adios2@2.10 +mpi + - hdf5 +mpi - mpich packages: adios2: - variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 + variants: +mpi ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 ~blosc2 ~mgard cmake: externals: - - spec: cmake@3.23.0 - prefix: /usr + - spec: cmake@3.31.6 + prefix: /usr/local buildable: False + hdf5: + variants: +mpi mpich: externals: - - spec: mpich@3.3 + - spec: mpich@4.2.0 prefix: /usr buildable: False perl: externals: - - spec: perl@5.26.1 + - spec: perl@5.38.2 prefix: /usr buildable: False python: externals: - - spec: python@3.8.0 + - spec: python@3.12.3 prefix: /usr buildable: False all: @@ -38,7 +40,7 @@ spack: variants: ~fortran providers: mpi: [mpich] - compiler: [clang@8.0.0] + compiler: [gcc@13.3.0] compilers: - compiler: @@ -46,17 +48,17 @@ spack: extra_rpaths: [] flags: {} modules: [] - operating_system: ubuntu20.04 + operating_system: ubuntu24.04 paths: - cc: /usr/lib/llvm-8/bin/clang - cxx: /usr/lib/llvm-8/bin/clang++ + cc: /usr/bin/gcc + cxx: /usr/bin/g++ f77: /usr/bin/gfortran fc: /usr/bin/gfortran - spec: clang@8.0.0 + spec: gcc@13.3.0 target: x86_64 config: - build_jobs: 2 + build_jobs: 4 mirrors: E4S: https://cache.e4s.io diff --git a/.github/workflows/dependencies/install_nvcc11.sh b/.github/workflows/dependencies/install_nvcc12.sh similarity index 70% rename from .github/workflows/dependencies/install_nvcc11.sh rename to .github/workflows/dependencies/install_nvcc12.sh index 9b0948926b..e1817fce79 100755 --- a/.github/workflows/dependencies/install_nvcc11.sh +++ b/.github/workflows/dependencies/install_nvcc12.sh @@ -15,14 +15,15 @@ sudo apt-get install -y \ pkg-config \ wget -sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub -echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 /" \ +sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub +echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 /" \ | sudo tee /etc/apt/sources.list.d/cuda.list sudo apt-get update +apt search cuda-compiler sudo apt-get install -y \ - cuda-command-line-tools-11-2 \ - cuda-compiler-11-2 \ - cuda-cupti-dev-11-2 \ - cuda-minimal-build-11-2 + cuda-command-line-tools-12-8 \ + cuda-compiler-12-8 \ + cuda-cupti-dev-12-8 \ + cuda-minimal-build-12-8 sudo ln -s cuda-11.2 /usr/local/cuda diff --git a/.github/workflows/dependencies/install_nvhpc21-11.sh b/.github/workflows/dependencies/install_nvhpc25-1.sh similarity index 91% rename from .github/workflows/dependencies/install_nvhpc21-11.sh rename to .github/workflows/dependencies/install_nvhpc25-1.sh index f333d7121b..4772629847 100755 --- a/.github/workflows/dependencies/install_nvhpc21-11.sh +++ b/.github/workflows/dependencies/install_nvhpc25-1.sh @@ -17,7 +17,7 @@ sudo apt-get install -y \ echo 'deb [trusted=yes] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | \ sudo tee /etc/apt/sources.list.d/nvhpc.list sudo apt-get update -y -sudo apt-get install -y --no-install-recommends nvhpc-21-11 +sudo apt-get install -y --no-install-recommends nvhpc-25-1 # things should reside in /opt/nvidia/hpc_sdk now diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index d115f9e3e2..a70891af7d 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -7,36 +7,9 @@ concurrency: cancel-in-progress: true jobs: - icc_cxxonly: - name: ICC C++ only - runs-on: ubuntu-20.04 - if: github.event.pull_request.draft == false - steps: - - uses: actions/checkout@v4 - - name: Install - run: | - sudo .github/workflows/dependencies/install_icc - - name: Build - # Due to compiler bugs in Intel compiler, we need to disable warning 1011 - # (missing return value), otherwise `if constexpr` functions - # don't compile. - # See https://community.intel.com/t5/Intel-C-Compiler/quot-if-constexpr-quot-and-quot-missing-return-statement-quot-in/td-p/1154551 - # Using a local pragma does not work due to the reasons stated there. - env: {CXXFLAGS: -Werror -wd1011} - run: | - set +e; source /opt/intel/oneapi/setvars.sh; set -e - share/openPMD/download_samples.sh build - cmake -S . -B build \ - -DCMAKE_C_COMPILER=$(which icc) \ - -DCMAKE_CXX_COMPILER=$(which icpc) \ - -DopenPMD_USE_PYTHON=OFF \ - -DopenPMD_USE_MPI=OFF - cmake --build build --parallel 2 - ctest --test-dir build --output-on-failure - icx_cxxonly: name: ICX C++ only - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index faad7cd9e0..78a26dc15b 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -7,26 +7,26 @@ concurrency: cancel-in-progress: true jobs: - clang7_nopy_nompi_h5_libcpp: - runs-on: ubuntu-20.04 + clang11_nopy_nompi_h5_libcpp: + runs-on: ubuntu-22.04 if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 - name: Spack Cache uses: actions/cache@v3 - with: {path: /opt/spack, key: clang7_nopy_nompi_h5_libcpp_v2} + with: {path: /opt/spack, key: clang11_nopy_nompi_h5_libcpp_v2} - name: Install run: | sudo apt-get update - sudo apt-get install clang-7 libc++-7-dev libc++abi-7-dev + sudo apt-get install clang-11 libc++-11-dev libc++abi-11-dev sudo .github/workflows/dependencies/install_spack - name: Build - env: {CC: clang-7, CXX: clang++-7, CXXFLAGS: -stdlib=libc++ -Werror -Wno-ignored-attributes -Wno-unused-const-variable} + env: {CC: clang-11, CXX: clang++-11, CXXFLAGS: -stdlib=libc++ -Werror -Wno-ignored-attributes -Wno-unused-const-variable} # -Wno-ignored-attributes -Wno-unused-const-variable: clang-6 has a # false positive on src/auxiliary/Filesystem.cpp # [[maybe_unused]] MPI_Datatype const MPI_Types< unsigned >::value = MPI_UNSIGNED; run: | - eval $(spack env activate --sh .github/ci/spack-envs/clang7_nopy_nompi_h5_libcpp/) + eval $(spack env activate --sh .github/ci/spack-envs/clang11_nopy_nompi_h5_libcpp/) spack install share/openPMD/download_samples.sh build @@ -36,25 +36,25 @@ jobs: -DopenPMD_USE_HDF5=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON - cmake --build build --parallel 2 + cmake --build build --parallel 4 cd build ctest --output-on-failure - clang7_nopy_ompi_h5_ad2_libcpp: - runs-on: ubuntu-20.04 + clang11_nopy_ompi_h5_ad2_libcpp: + runs-on: ubuntu-22.04 if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 - name: Spack Cache uses: actions/cache@v3 - with: {path: /opt/spack, key: clang7_nopy_ompi_h5_ad2_libcpp_v2} + with: {path: /opt/spack, key: clang11_nopy_ompi_h5_ad2_libcpp_v2} - name: Install run: | sudo apt-get update - sudo apt-get install clang-7 libc++-7-dev libc++abi-7-dev gfortran libopenmpi-dev python3 + sudo apt-get install clang-11 libc++-11-dev libc++abi-11-dev gfortran libopenmpi-dev python3 sudo .github/workflows/dependencies/install_spack - name: Build - env: {CC: clang-7, CXX: clang++-7, CXXFLAGS: -stdlib=libc++ -Werror -Wno-ignored-attributes -Wno-unused-const-variable} + env: {CC: clang-11, CXX: clang++-11, CXXFLAGS: -stdlib=libc++ -Werror -Wno-ignored-attributes -Wno-unused-const-variable} # -Wno-ignored-attributes -Wno-unused-const-variable: clang-6 has a # false positive on src/auxiliary/Filesystem.cpp # [[maybe_unused]] MPI_Datatype const MPI_Types< unsigned >::value = MPI_UNSIGNED; @@ -63,7 +63,8 @@ jobs: mpiexec --version perl --version python --version - eval $(spack env activate --sh .github/ci/spack-envs/clang7_nopy_ompi_h5_ad2_libcpp/) + sudo ln -s "$(which cmake)" /usr/bin/cmake + eval $(spack env activate --sh .github/ci/spack-envs/clang11_nopy_ompi_h5_ad2_libcpp/) spack install share/openPMD/download_samples.sh build @@ -74,7 +75,7 @@ jobs: -DopenPMD_USE_ADIOS2=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON - cmake --build build --parallel 2 + cmake --build build --parallel 4 cd build ctest --output-on-failure @@ -83,23 +84,24 @@ jobs: find . -name *.bp.dir | xargs -n1 -P1 -I {} rm -rf {} ctest --output-on-failure - clang7_nopy_ompi_h5_ad2: - runs-on: ubuntu-20.04 + clang11_nopy_ompi_h5_ad2: + runs-on: ubuntu-22.04 if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 - name: Spack Cache uses: actions/cache@v3 - with: {path: /opt/spack, key: clang7_nopy_ompi_h5_ad2_v2 } + with: {path: /opt/spack, key: clang11_nopy_ompi_h5_ad2_v2 } - name: Install run: | sudo apt-get update - sudo apt-get install clang-7 gfortran libopenmpi-dev python3 + sudo apt-get install clang-11 gfortran libopenmpi-dev python3 sudo .github/workflows/dependencies/install_spack - name: Build - env: {CC: clang-7, CXX: clang++-7, CXXFLAGS: -Werror} + env: {CC: clang-11, CXX: clang++-11, CXXFLAGS: -Werror} run: | - eval $(spack env activate --sh .github/ci/spack-envs/clang7_nopy_ompi_h5_ad2/) + sudo ln -s "$(which cmake)" /usr/bin/cmake + eval $(spack env activate --sh .github/ci/spack-envs/clang11_nopy_ompi_h5_ad2/) spack install share/openPMD/download_samples.sh build @@ -110,25 +112,25 @@ jobs: -DopenPMD_USE_ADIOS2=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON - cmake --build build --parallel 2 + cmake --build build --parallel 4 cd build ctest --output-on-failure # TODO # clang7_py36_nompi_h5_ad2_libstdc++ - clang14_py311_nompi_h5_ad2: + clang15_py311_nompi_h5_ad2: runs-on: ubuntu-22.04 if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 - name: Spack Cache uses: actions/cache@v3 - with: {path: /opt/spack, key: clang14_py311_nompi_h5_ad2_v2 } + with: {path: /opt/spack, key: clang15_py311_nompi_h5_ad2_v2 } - name: Install run: | sudo apt update - sudo apt install clang-14 cmake gfortran libhdf5-dev python3.11 python3.11-dev wget python3.11-venv + sudo apt install clang-15 cmake gfortran libhdf5-dev python3.11 python3.11-dev wget sudo .github/workflows/dependencies/install_spack python3.11 -m venv /opt/python_venv . /opt/python_venv/bin/activate @@ -138,14 +140,11 @@ jobs: -DCMAKE_INSTALL_PREFIX=toml11_install \ -DCMAKE_CXX_STANDARD_REQUIRED=OFF \ -DCMAKE_CXX_STANDARD=11 - cmake --build build_toml11 -j 2 --target install + cmake --build build_toml11 -j 4 --target install - name: Build - env: {CC: clang-14, CXX: clang++-14, CXXFLAGS: -Werror} + env: {CC: clang-15, CXX: clang++-15, CXXFLAGS: -Werror} run: | - # Build Spack packages against system Python - # and activate the virtual environment just for the openPMD build. - # Spack does not play nice with venv. - eval $(spack env activate --sh .github/ci/spack-envs/clang14_py311_nompi_h5_ad2/) + eval $(spack env activate --sh .github/ci/spack-envs/clang15_py311_nompi_h5_ad2/) spack install . /opt/python_venv/bin/activate @@ -159,32 +158,36 @@ jobs: -DopenPMD_USE_INVASIVE_TESTS=ON \ -DopenPMD_USE_INTERNAL_TOML11=OFF \ -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DPython_EXECUTABLE="$(which python)" - cmake --build build --parallel 2 + -DPython_EXECUTABLE="$(which python3.11)" + cmake --build build --parallel 4 ctest --test-dir build --output-on-failure - # ADIOS2 v2.7.1 - clang8_py38_mpich_h5_ad2: - runs-on: ubuntu-20.04 - if: github.event.pull_request.draft == false + # ADIOS2 v2.10 + gcc13_py312_mpich_h5_ad2: + runs-on: ubuntu-24.04 + # disabled because of unclear runtime issues + if: 0 + #if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 - name: Spack Cache uses: actions/cache@v3 - with: {path: /opt/spack, key: clang8_py38_mpich_h5_ad2_newLayout_v2 } + with: {path: /opt/spack, key: gcc13_py312_mpich_h5_ad2 } - name: Install run: | sudo apt-get update - sudo apt-get install clang-8 gfortran libmpich-dev python3 + sudo apt-get remove openmpi* libopenmpi* *hdf5* || true + sudo apt-get install g++ gfortran libmpich-dev mpich python3 sudo .github/workflows/dependencies/install_spack - name: Build - env: {CC: clang-8, CXX: clang++-8, CXXFLAGS: -Werror} + env: {CC: gcc, CXX: g++, MPICH_CC: gcc, MPICH_CXX: g++, CXXFLAGS: -Werror} run: | cmake --version mpiexec --version + mpicxx --version perl --version python --version - eval $(spack env activate --sh .github/ci/spack-envs/clang8_py38_mpich_h5_ad2/) + eval $(spack env activate --sh .github/ci/spack-envs/gcc13_py312_mpich_h5_ad2/) spack install share/openPMD/download_samples.sh build @@ -194,7 +197,7 @@ jobs: -DopenPMD_USE_HDF5=ON \ -DopenPMD_USE_ADIOS2=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON - cmake --build build --parallel 2 + cmake --build build --parallel 4 cd build ctest --output-on-failure @@ -213,18 +216,18 @@ jobs: # gcc-4.8.5_py35_nompi_h5 # gcc-7.4.0_py_ompi_h5_ad2_coveralls - gcc7_py36_pd_dd_ompi_h5_ad2: - runs-on: ubuntu-20.04 + gcc12_py36_pd_dd_ompi_h5_ad2: + runs-on: ubuntu-22.04 if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 - name: Spack Cache uses: actions/cache@v3 - with: {path: /opt/spack, key: gcc7_py36_ompi_h5_ad2_v2 } + with: {path: /opt/spack, key: gcc12_py36_ompi_h5_ad2_v2 } - name: Install run: | sudo apt-get update - sudo apt-get install g++-7 gfortran libopenmpi-dev python3 python3-setuptools + sudo apt-get install g++-12 gfortran libopenmpi-dev python3 python3-setuptools sudo .github/workflows/dependencies/install_spack python3 -m pip install -U pip python3 -m pip install -U numpy @@ -233,9 +236,10 @@ jobs: python3 -m pip install -U dask python3 -m pip install -U pyarrow - name: Build - env: {CC: gcc-7, CXX: g++-7, CXXFLAGS: -Werror} + env: {CC: gcc-12, CXX: g++-12, CXXFLAGS: -Werror} run: | - eval $(spack env activate --sh .github/ci/spack-envs/gcc7_py36_ompi_h5_ad2/) + sudo ln -s "$(which cmake)" /usr/bin/cmake + eval $(spack env activate --sh .github/ci/spack-envs/gcc12_py36_ompi_h5_ad2/) spack install share/openPMD/download_samples.sh build @@ -245,12 +249,11 @@ jobs: -DopenPMD_USE_HDF5=ON \ -DopenPMD_USE_ADIOS2=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON - cmake --build build --parallel 2 - cd build - ctest --output-on-failure + cmake --build build --parallel 4 + ctest --test-dir build --output-on-failure gcc9_py38_pd_nompi_h5_ad2_libcpp: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 @@ -268,12 +271,11 @@ jobs: -DopenPMD_USE_MPI=ON \ -DopenPMD_USE_HDF5=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON - cmake --build build --parallel 2 - cd build - ctest --output-on-failure + cmake --build build --parallel 4 + ctest --test-dir build --output-on-failure musllinux_py10: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 if: github.event.pull_request.draft == false container: image: quay.io/pypa/musllinux_1_1_x86_64 @@ -298,10 +300,9 @@ jobs: -DopenPMD_USE_MPI=OFF \ -DopenPMD_USE_HDF5=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON \ - -DPython_EXECUTABLE=$(which python) - cmake --build build --parallel 2 - cd build - ctest --output-on-failure + -DPython_EXECUTABLE=$(which python3.10) + cmake --build build --parallel 4 + ctest --test-dir build --output-on-failure conda_ompi_all: runs-on: ubuntu-22.04 @@ -329,6 +330,5 @@ jobs: -DopenPMD_USE_HDF5=ON \ -DopenPMD_USE_ADIOS2=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON - cmake --build build --parallel 2 - cd build - ctest --output-on-failure + cmake --build build --parallel 4 + ctest --test-dir build --output-on-failure diff --git a/.github/workflows/nvidia.yml b/.github/workflows/nvidia.yml index bc18238997..cb182e4ef6 100644 --- a/.github/workflows/nvidia.yml +++ b/.github/workflows/nvidia.yml @@ -9,13 +9,13 @@ concurrency: jobs: tests-cuda11: name: CTK@11.2 - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 if: github.event.pull_request.draft == false env: {CXX: nvcc, CXXFLAGS: "--forward-unknown-to-host-compiler -Xcompiler -Werror"} steps: - uses: actions/checkout@v4 - name: Dependencies - run: .github/workflows/dependencies/install_nvcc11.sh + run: .github/workflows/dependencies/install_nvcc12.sh - name: Build & Install run: | export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} @@ -31,9 +31,9 @@ jobs: cmake --build build --parallel 2 ctest --test-dir build --output-on-failure - tests-nvhpc21-11-nvcc: - name: NVHPC@21.11 - runs-on: ubuntu-20.04 + tests-nvhpc25-1-nvcc: + name: NVHPC@25.1 + runs-on: ubuntu-22.04 if: github.event.pull_request.draft == false # Catch warnings: # line 4314: error: variable "::autoRegistrar73" was declared but never referenced @@ -41,11 +41,11 @@ jobs: steps: - uses: actions/checkout@v4 - name: Dependencies - run: .github/workflows/dependencies/install_nvhpc21-11.sh + run: .github/workflows/dependencies/install_nvhpc25-1.sh - name: Build & Install run: | source /etc/profile.d/modules.sh - module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/21.11 + module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/25.1 which nvcc || echo "nvcc not in PATH!" which nvc++ || echo "nvc++ not in PATH!" diff --git a/.github/workflows/tooling.yml b/.github/workflows/tooling.yml index 03d5070d86..e12dde1410 100644 --- a/.github/workflows/tooling.yml +++ b/.github/workflows/tooling.yml @@ -7,19 +7,19 @@ concurrency: cancel-in-progress: true jobs: - clangtidy10_nopy_ompi_h5_ad2: + clangtidy14_nopy_ompi_h5_ad2: name: clang-tidy w/o py - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 - name: Spack Cache uses: actions/cache@v3 - with: {path: /opt/spack, key: clangtidy10_nopy_ompi_h5_ad2 } + with: {path: /opt/spack, key: clangtidy14_nopy_ompi_h5_ad2 } - name: Install run: | sudo apt-get update - sudo apt-get install clang clang-tidy gfortran libopenmpi-dev python + sudo apt-get install clang clang-tidy gfortran libopenmpi-dev python-is-python3 sudo .github/workflows/dependencies/install_spack - name: Build env: {CC: clang, CXX: clang++} @@ -35,24 +35,24 @@ jobs: cat build/clang-tidy.log if [[ $(wc -m E_r_data(num_fields * N_r * N_z); - std::vector E_t_data(num_fields * N_r * N_z); + std::vector E_r_data(size_t(num_fields) * N_r * N_z); + std::vector E_t_data(size_t(num_fields) * N_r * N_z); std::iota(E_r_data.begin(), E_r_data.end(), 0.0); std::iota(E_t_data.begin(), E_t_data.end(), 0.f); diff --git a/examples/5_write_parallel.cpp b/examples/5_write_parallel.cpp index 3cf0f01883..92c985a2c8 100644 --- a/examples/5_write_parallel.cpp +++ b/examples/5_write_parallel.cpp @@ -43,7 +43,7 @@ int main(int argc, char *argv[]) // global data set to write: [MPI_Size * 10, 300] // each rank writes a 10x300 slice with its MPI rank as values auto const value = float(mpi_size); - std::vector local_data(10 * 300, value); + std::vector local_data(size_t(10) * 300, value); if (0 == mpi_rank) cout << "Set up a 2D array with 10x300 elements per MPI rank (" << mpi_size << "x) that will be written to disk\n"; diff --git a/examples/8_benchmark_parallel.cpp b/examples/8_benchmark_parallel.cpp index 2dc63632ff..1e098cbf12 100644 --- a/examples/8_benchmark_parallel.cpp +++ b/examples/8_benchmark_parallel.cpp @@ -92,7 +92,7 @@ int main(int argc, char *argv[]) // Total (in this case 3D) dataset across all MPI ranks. // Will be the same for all configured benchmarks. - openPMD::Extent total{100 * scale_up, 100, 1000}; + openPMD::Extent total{size_t(100) * scale_up, 100, 1000}; // The blockslicer assigns to each rank its part of the dataset. The rank // will write to and read from that part. OneDimensionalBlockSlicer is a diff --git a/examples/8b_benchmark_read_parallel.cpp b/examples/8b_benchmark_read_parallel.cpp index bfea6331e9..71950420d5 100644 --- a/examples/8b_benchmark_read_parallel.cpp +++ b/examples/8b_benchmark_read_parallel.cpp @@ -754,7 +754,7 @@ class TestInput Extent pExtent = idVal.getExtent(); - auto blob = pExtent[0] / (10 * m_MPISize); + auto blob = pExtent[0] / (size_t(10) * m_MPISize); if (0 == blob) return; diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index 874768adbe..48e24fd89b 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -86,7 +86,7 @@ HDF5IOHandlerImpl::HDF5IOHandlerImpl( , m_H5T_CDOUBLE{H5Tcreate(H5T_COMPOUND, sizeof(double) * 2)} , m_H5T_CLONG_DOUBLE{H5Tcreate(H5T_COMPOUND, sizeof(long double) * 2)} , m_H5T_LONG_DOUBLE_80_LE{H5Tcopy(H5T_IEEE_F64BE)} - , m_H5T_CLONG_DOUBLE_80_LE{H5Tcreate(H5T_COMPOUND, 16 * 2)} + , m_H5T_CLONG_DOUBLE_80_LE{H5Tcreate(H5T_COMPOUND, size_t(16) * 2)} { // create a h5py compatible bool type VERIFY( diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index a6c8e68375..eb56f284ce 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -708,7 +708,7 @@ void write_4D_test(std::string const &file_ending) // every rank out of mpi_size MPI ranks contributes two writes: // - sliced in first dimension (partioned by rank) // - last dimension: every rank has two chunks to contribute - std::vector data(2 * 10 * 6 * 4, mpi_rank); + std::vector data(2. * 10 * 6 * 4, mpi_rank); E_x.resetDataset({Datatype::DOUBLE, {mpi_size * 2, 10, 6, 8}}); E_x.storeChunk(data, {mpi_rank * 2, 0, 0, 0}, {2, 10, 6, 4}); @@ -788,7 +788,7 @@ void close_iteration_test(std::string const &file_ending) auto chunk = E_x_read.loadChunk({0, 0}, {mpi_size, 4}); it0.close(/* flush = */ false); read.flush(); - for (size_t i = 0; i < 4 * mpi_size; ++i) + for (size_t i = 0; i < size_t(4) * mpi_size; ++i) { REQUIRE(data[i % 4] == chunk.get()[i]); } @@ -812,7 +812,7 @@ void close_iteration_test(std::string const &file_ending) auto E_x_read = it1.meshes["E"]["x"]; auto chunk = E_x_read.loadChunk({0, 0}, {mpi_size, 4}); it1.close(/* flush = */ true); - for (size_t i = 0; i < 4 * mpi_size; ++i) + for (size_t i = 0; i < size_t(4) * mpi_size; ++i) { REQUIRE(data[i % 4] == chunk.get()[i]); } @@ -886,7 +886,7 @@ void file_based_write_read(std::string const &file_ending) unsigned const global_Nz = local_Nz * mpi_size; unsigned const global_Nx = 300u; using precision = double; - std::vector E_x_data(global_Nx * local_Nz); + std::vector E_x_data(size_t(global_Nx) * local_Nz); // filling some values: 0, 1, ... std::iota(E_x_data.begin(), E_x_data.end(), local_Nz * mpi_rank); std::transform( @@ -942,7 +942,7 @@ void file_based_write_read(std::string const &file_ending) std::to_string(local_Nz) + "]"); E_x.resetDataset(dataset); - Offset chunk_offset = {0, local_Nz * mpi_rank}; + Offset chunk_offset = {0, size_t(local_Nz) * mpi_rank}; Extent chunk_extent = {global_Nx, local_Nz}; E_x.storeChunk(E_x_data, chunk_offset, chunk_extent); series.flush(); @@ -1015,7 +1015,7 @@ void hipace_like_write(std::string const &file_ending) unsigned const global_Nz = local_Nz * mpi_size; unsigned const global_Nx = 300u; using precision = double; - std::vector E_x_data(global_Nx * local_Nz); + std::vector E_x_data(size_t(global_Nx) * local_Nz); // filling some values: 0, 1, ... std::iota(E_x_data.begin(), E_x_data.end(), local_Nz * mpi_rank); std::transform( @@ -1129,7 +1129,7 @@ void hipace_like_write(std::string const &file_ending) io::determineDatatype(), {global_Nx, global_Nz}); E_x.resetDataset(dataset); - Offset chunk_offset = {0, local_Nz * mpi_rank}; + Offset chunk_offset = {0, size_t(local_Nz) * mpi_rank}; Extent chunk_extent = {global_Nx, local_Nz}; auto const copyToShared = [](std::vector const &data) { auto d = std::shared_ptr( @@ -1647,7 +1647,7 @@ void append_mode( auto chunk = it.meshes["E"]["x"].template loadChunk( {0, 0}, {unsigned(mpi_size), 10}); it.seriesFlush(); - for (size_t i = 0; i < unsigned(mpi_size) * 10; ++i) + for (size_t i = 0; i < size_t(mpi_size) * 10; ++i) { REQUIRE(chunk.get()[i] == 999); } diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 834b35ca02..44b6aa2624 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -2253,20 +2253,21 @@ inline void sample_write_thetaMode(std::string const &file_ending) unsigned int const N_z = 64; std::shared_ptr E_r_data( - new float[num_fields * N_r * N_z], [](float const *p) { delete[] p; }); + new float[size_t(num_fields) * N_r * N_z], + [](float const *p) { delete[] p; }); std::shared_ptr E_t_data( - new double[num_fields * N_r * N_z], + new double[size_t(num_fields) * N_r * N_z], [](double const *p) { delete[] p; }); float e_r{0}; std::generate( - E_r_data.get(), E_r_data.get() + num_fields * N_r * N_z, [&e_r] { - return e_r += 1.0f; - }); + E_r_data.get(), + E_r_data.get() + size_t(num_fields * N_r * N_z), + [&e_r] { return e_r += 1.0f; }); double e_t{100}; std::generate( - E_t_data.get(), E_t_data.get() + num_fields * N_r * N_z, [&e_t] { - return e_t += 2.0; - }); + E_t_data.get(), + E_t_data.get() + size_t(num_fields * N_r * N_z), + [&e_t] { return e_t += 2.0; }); std::stringstream geos; geos << "m=" << num_modes << ";imag=+"; @@ -4156,7 +4157,7 @@ enum class FlushDuringStep void adios2_bp5_flush(std::string const &cfg, FlushDuringStep flushDuringStep) { - constexpr size_t size = 1024 * 1024; + constexpr size_t size = size_t(1024) * 1024; auto getsize = []() { off_t res = 0; From 6a174d6e9eb832f17cd939be37a170c1223cf313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 25 Mar 2025 13:15:08 +0100 Subject: [PATCH 3/8] Reactivate MPICH runner (gcc13_py312_mpich_h5_ad2) (#1739) * Revert "CI: Disable MPICH for now" This reverts commit de9e7ad02944e91206bea3a603b4d0ea10771d21. * We shall try installin MPICH from Spack for now Ubuntu package broken https://bugs.launchpad.net/ubuntu/+source/mpich/+bug/2072338 * Take out version checks * Revert "Take out version checks" This reverts commit b22e4b32970975992ec888dd65f01a9ba09b13c6. * Revert "We shall try installin MPICH from Spack for now" This reverts commit 33b847fe7a5f6e81db76eb4fe4c2ec5c156c4001. * Alternative approach: self-compiled MPICH * Move MPICH installation to separate file --- .../gcc13_py312_mpich_h5_ad2/spack.yaml | 4 ++-- .github/workflows/dependencies/install_mpich | 16 ++++++++++++++++ .github/workflows/linux.yml | 11 +++++++---- 3 files changed, 25 insertions(+), 6 deletions(-) create mode 100755 .github/workflows/dependencies/install_mpich diff --git a/.github/ci/spack-envs/gcc13_py312_mpich_h5_ad2/spack.yaml b/.github/ci/spack-envs/gcc13_py312_mpich_h5_ad2/spack.yaml index fc1235b7f5..6b3851858f 100644 --- a/.github/ci/spack-envs/gcc13_py312_mpich_h5_ad2/spack.yaml +++ b/.github/ci/spack-envs/gcc13_py312_mpich_h5_ad2/spack.yaml @@ -22,8 +22,8 @@ spack: variants: +mpi mpich: externals: - - spec: mpich@4.2.0 - prefix: /usr + - spec: mpich@4.2.1 + prefix: /usr/local buildable: False perl: externals: diff --git a/.github/workflows/dependencies/install_mpich b/.github/workflows/dependencies/install_mpich new file mode 100755 index 0000000000..ba36bf03fc --- /dev/null +++ b/.github/workflows/dependencies/install_mpich @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# + +set -eu -o pipefail + +mpich_ver="4.3.0" + +cd /opt +wget https://www.mpich.org/static/downloads/${mpich_ver}/mpich-${mpich_ver}.tar.gz +tar -xzf mpich-${mpich_ver}.tar.gz +cd mpich-${mpich_ver} +./configure --prefix=/usr/local +make -j 4 +make -j 4 install +cd .. +rm -rf mpich-${mpich_ver} mpich-${mpich_ver}.tar.gz diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 78a26dc15b..15fd5d922f 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -165,9 +165,7 @@ jobs: # ADIOS2 v2.10 gcc13_py312_mpich_h5_ad2: runs-on: ubuntu-24.04 - # disabled because of unclear runtime issues - if: 0 - #if: github.event.pull_request.draft == false + if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 - name: Spack Cache @@ -177,8 +175,13 @@ jobs: run: | sudo apt-get update sudo apt-get remove openmpi* libopenmpi* *hdf5* || true - sudo apt-get install g++ gfortran libmpich-dev mpich python3 + sudo apt-get install g++ gfortran python3 sudo .github/workflows/dependencies/install_spack + + # Need to build this manually due to broken MPICH package in Ubuntu 24.04 + # https://bugs.launchpad.net/ubuntu/+source/mpich/+bug/2072338 + sudo .github/workflows/dependencies/install_mpich + - name: Build env: {CC: gcc, CXX: g++, MPICH_CC: gcc, MPICH_CXX: g++, CXXFLAGS: -Werror} run: | From b081e1a2813ddb21a70e8fc8fdac7f734f950062 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 2 Apr 2025 18:41:54 +0200 Subject: [PATCH 4/8] Fix broken CI: Support for old CMake versions (#1742) * Check for broken CI * Deactivate dependencies that are not yet compatible with CMake 4.0 --- .../ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/spack.yaml | 4 ++++ .github/ci/spack-envs/gcc12_py36_ompi_h5_ad2/spack.yaml | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/spack.yaml b/.github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/spack.yaml index 26fe755392..55e5c16c97 100644 --- a/.github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/spack.yaml +++ b/.github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/spack.yaml @@ -13,6 +13,10 @@ spack: packages: adios2: variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 ~mgard + c-blosc2: + # snappy broken on CMake 4.0 + # fixed snappy not yet deployed to a Spack release + variants: ~snappy cmake: externals: - spec: cmake@3.31.5 diff --git a/.github/ci/spack-envs/gcc12_py36_ompi_h5_ad2/spack.yaml b/.github/ci/spack-envs/gcc12_py36_ompi_h5_ad2/spack.yaml index fbad0397d0..750bf288a5 100644 --- a/.github/ci/spack-envs/gcc12_py36_ompi_h5_ad2/spack.yaml +++ b/.github/ci/spack-envs/gcc12_py36_ompi_h5_ad2/spack.yaml @@ -12,7 +12,14 @@ spack: packages: adios2: - variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 + # mgard not supported on CMake 4.0 due to unresolved compatibility + # issue in dependency yaml-cpp + # https://github.com/jbeder/yaml-cpp/issues/1352 + variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 ~mgard + c-blosc2: + # snappy broken on CMake 4.0 + # fixed snappy not yet deployed to a Spack release + variants: ~snappy cmake: externals: - spec: cmake@3.31.5 From 1e0e6303bcd3e1d7cbf3325ff5db25beab64e42c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 3 Jun 2025 11:35:55 +0200 Subject: [PATCH 5/8] Fixes --- .github/workflows/linux.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 15fd5d922f..9b21230ce3 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -132,9 +132,7 @@ jobs: sudo apt update sudo apt install clang-15 cmake gfortran libhdf5-dev python3.11 python3.11-dev wget sudo .github/workflows/dependencies/install_spack - python3.11 -m venv /opt/python_venv - . /opt/python_venv/bin/activate - python -m pip install numpy pandas + python3.11 -m pip install numpy pandas git clone -b v4.0.3 https://github.com/ToruNiina/toml11 cmake -S toml11 -B build_toml11 \ -DCMAKE_INSTALL_PREFIX=toml11_install \ @@ -146,7 +144,6 @@ jobs: run: | eval $(spack env activate --sh .github/ci/spack-envs/clang15_py311_nompi_h5_ad2/) spack install - . /opt/python_venv/bin/activate share/openPMD/download_samples.sh build export CMAKE_PREFIX_PATH="$(realpath toml11_install):$CMAKE_PREFIX_PATH" @@ -288,15 +285,10 @@ jobs: run: | apk update apk add hdf5-dev - # Use a virtual environment in order to avoid compatibility issues - # between the various Python installations in this image. - python3.10 -m venv /opt/python_env - . /opt/python_env/bin/activate - python -m pip install numpy + python3.10 -m pip install numpy - name: Build env: {CXXFLAGS: -Werror} run: | - . /opt/python_env/bin/activate share/openPMD/download_samples.sh build cmake -S . -B build \ -DopenPMD_USE_PYTHON=ON \ From 5bb4ecf874a876608f2d0d2b90f2c20c4013eb4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 3 Jun 2025 14:04:41 +0200 Subject: [PATCH 6/8] Upgrade Spack --- .github/workflows/dependencies/install_spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependencies/install_spack b/.github/workflows/dependencies/install_spack index 30d7d06a92..7a45620503 100755 --- a/.github/workflows/dependencies/install_spack +++ b/.github/workflows/dependencies/install_spack @@ -3,7 +3,7 @@ set -eu -o pipefail -spack_ver="0.17.1" +spack_ver="0.22.3" cd /opt if [[ -d spack && ! -f spack_${spack_ver} ]] From b391b13d83f0c527932ee7295a55c5b8cfa51bf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 3 Jun 2025 14:05:16 +0200 Subject: [PATCH 7/8] Fix warning --- test/ParallelIOTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index eb56f284ce..104db037dd 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -2213,7 +2213,7 @@ auto bug_1655_bp5_writer_hangup(std::string const &ext) -> void MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); auto const value = float(mpi_size * 100 + mpi_rank); - std::vector local_data(10 * 300, value); + std::vector local_data(size_t(10) * 300, value); std::string filename = "../samples/ptl_%T." + ext; From ae5a3f94b696bd6ecfd9c3465fb6f63e8e9faee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 3 Jun 2025 14:32:20 +0200 Subject: [PATCH 8/8] Fixes after upgrading spack --- .../ci/spack-envs/clang11_nopy_ompi_h5_ad2_libcpp/spack.yaml | 2 +- .github/ci/spack-envs/clang15_py311_nompi_h5_ad2/spack.yaml | 2 +- .github/workflows/tooling.yml | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/ci/spack-envs/clang11_nopy_ompi_h5_ad2_libcpp/spack.yaml b/.github/ci/spack-envs/clang11_nopy_ompi_h5_ad2_libcpp/spack.yaml index 2dd6b004d1..492841adde 100644 --- a/.github/ci/spack-envs/clang11_nopy_ompi_h5_ad2_libcpp/spack.yaml +++ b/.github/ci/spack-envs/clang11_nopy_ompi_h5_ad2_libcpp/spack.yaml @@ -12,7 +12,7 @@ spack: packages: adios2: - variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 + variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 ~mgard ~blosc2 cmake: externals: - spec: cmake@3.31.5 diff --git a/.github/ci/spack-envs/clang15_py311_nompi_h5_ad2/spack.yaml b/.github/ci/spack-envs/clang15_py311_nompi_h5_ad2/spack.yaml index 18b2179b3e..13a986c4a9 100644 --- a/.github/ci/spack-envs/clang15_py311_nompi_h5_ad2/spack.yaml +++ b/.github/ci/spack-envs/clang15_py311_nompi_h5_ad2/spack.yaml @@ -13,7 +13,7 @@ spack: hdf5: variants: ~mpi adios2: - variants: ~mpi ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 + variants: ~mpi ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 ~mgard cmake: externals: - spec: cmake@3.22.1 diff --git a/.github/workflows/tooling.yml b/.github/workflows/tooling.yml index e12dde1410..bf3869c63d 100644 --- a/.github/workflows/tooling.yml +++ b/.github/workflows/tooling.yml @@ -24,6 +24,7 @@ jobs: - name: Build env: {CC: clang, CXX: clang++} run: | + sudo ln -s "$(which cmake)" /usr/bin/cmake eval $(spack env activate --sh .github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/) spack install @@ -54,6 +55,7 @@ jobs: - name: Build env: {CC: mpicc, CXX: mpic++, OMPI_CC: clang-14, OMPI_CXX: clang++-14, CXXFLAGS: -Werror, OPENPMD_HDF5_CHUNKS: none, OPENPMD_TEST_NFILES_MAX: 100} run: | + sudo ln -s "$(which cmake)" /usr/bin/cmake eval $(spack env activate --sh .github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/) spack install SOURCEPATH="$(pwd)"