Python 3.11 (openPMD#1323)

* CI: Python 3.11 Build w/ Clang 14

* Docs: Python 3.7 - 3.11

* HDF5: Remove Unused Variable `total_data_size`

Author: ax3l

…clang12_py38_nompi_h5_ad1_ad2/spack.yaml …lang14_py311_nompi_h5_ad1_ad2/spack.yaml.github/ci/spack-envs/clang12_py38_nompi_h5_ad1_ad2/spack.yaml renamed to .github/ci/spack-envs/clang14_py311_nompi_h5_ad1_ad2/spack.yaml .github/ci/spack-envs/clang12_py38_nompi_h5_ad1_ad2/spack.yaml renamed to .github/ci/spack-envs/clang14_py311_nompi_h5_ad1_ad2/spack.yaml

@@ -19,39 +19,39 @@ spack:
       variants: ~mpi ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2
     cmake:
       externals:
-      - spec: cmake@3.23.0
+      - spec: cmake@3.22.1
         prefix: /usr
       buildable: False
     perl:
       externals:
-      - spec: perl@5.26.1
+      - spec: perl@5.34.0
         prefix: /usr
       buildable: False
     python:
       externals:
-      - spec: python@3.8.0
+      - spec: python@3.11.0
         prefix: /usr
       buildable: False
     all:
       target: [x86_64]
       variants: ~fortran
       providers:
         mpi: [openmpi]
-      compiler: [clang@12.0.0]
+      compiler: [clang@14.0.0]
 
   compilers:
   - compiler:
       environment: {}
       extra_rpaths: []
       flags: {}
       modules: []
-      operating_system: ubuntu20.04
+      operating_system: ubuntu22.04
       paths:
-        cc: /usr/lib/llvm-12/bin/clang
-        cxx: /usr/lib/llvm-12/bin/clang++
+        cc: /usr/lib/llvm-14/bin/clang
+        cxx: /usr/lib/llvm-14/bin/clang++
         f77: /usr/bin/gfortran
         fc: /usr/bin/gfortran
-      spec: clang@12.0.0
+      spec: clang@14.0.0
       target: x86_64
 
   config:

.github/workflows/linux.yml

@@ -116,23 +116,26 @@ jobs:
 # TODO
 #  clang7_py36_nompi_h5_ad1_ad2_libstdc++
 
-  clang12_py38_nompi_h5_ad1_ad2:
-    runs-on: ubuntu-20.04
+  clang14_py311_nompi_h5_ad1_ad2:
+    runs-on: ubuntu-22.04
     if: github.event.pull_request.draft == false
     steps:
     - uses: actions/checkout@v2
     - name: Spack Cache
       uses: actions/cache@v2
-      with: {path: /opt/spack, key: clang12_py38_nompi_h5_ad1_ad2_v2 }
+      with: {path: /opt/spack, key: clang14_py311_nompi_h5_ad1_ad2_v2 }
     - name: Install
       run: |
-        sudo apt-get update
-        sudo apt-get install clang-12 libhdf5-dev python3 python3-pip python3-numpy python3-pandas
+        sudo apt update
+        sudo apt install clang-14 cmake gfortran libhdf5-dev python3.11 python3.11-dev wget
+        wget https://bootstrap.pypa.io/get-pip.py
+        python3.11 get-pip.py
+        python3.11 -m pip install numpy pandas
         sudo .github/workflows/dependencies/install_spack
     - name: Build
-      env: {CC: clang-12, CXX: clang++-12, CXXFLAGS: -Werror}
+      env: {CC: clang-14, CXX: clang++-14, CXXFLAGS: -Werror}
       run: |
-        eval $(spack env activate --sh .github/ci/spack-envs/clang12_py38_nompi_h5_ad1_ad2/)
+        eval $(spack env activate --sh .github/ci/spack-envs/clang14_py311_nompi_h5_ad1_ad2/)
         spack install
 
         share/openPMD/download_samples.sh build
@@ -143,7 +146,8 @@ jobs:
           -DopenPMD_USE_ADIOS1=ON \
           -DopenPMD_USE_ADIOS2=ON \
           -DopenPMD_USE_INVASIVE_TESTS=ON \
-          -DCMAKE_VERBOSE_MAKEFILE=ON
+          -DCMAKE_VERBOSE_MAKEFILE=ON     \
+          -DPython_EXECUTABLE=$(which python3.11)
         cmake --build build --parallel 2
         ctest --test-dir build --output-on-failure
 

CHANGELOG.rst

@@ -17,6 +17,7 @@ Changes to "0.14.0"
 Features
 """"""""
 
+- Python: support of 3.10 and 3.11, removal of 3.6 #1323
 - include internally shipped toml11 v3.7.1 #1148 #1227
 - pybind11: require version 2.10.1+ #1220 #1322
 

CMakeLists.txt

@@ -396,7 +396,7 @@ if(CMAKE_VERSION VERSION_LESS 3.18.0)
     set(_PY_DEV_MODULE Development)
 endif()
 if(openPMD_USE_PYTHON STREQUAL AUTO)
-    find_package(Python 3.6.0 COMPONENTS Interpreter ${_PY_DEV_MODULE})
+    find_package(Python 3.7.0 COMPONENTS Interpreter ${_PY_DEV_MODULE})
     if(Python_FOUND)
         if(openPMD_USE_INTERNAL_PYBIND11)
             add_subdirectory("${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/pybind11")

Dockerfile

@@ -5,8 +5,8 @@ FROM       quay.io/pypa/manylinux2010_x86_64 as build-env
 # FROM       quay.io/pypa/manylinux1_x86_64 as build-env
 ENV        DEBIAN_FRONTEND noninteractive
 
-# Python 3.6-3.10 via "36m 37m 38 39 310"
-ARG        PY_VERSIONS="36m 37m 38 39 310"
+# Python 3.7-3.11 via "37m 38 39 311"
+ARG        PY_VERSIONS="37m 38 39 310 311"
 
 # static libs need relocatable symbols for linking to shared python lib
 ENV        CFLAGS="-fPIC ${CFLAGS}"
@@ -159,20 +159,6 @@ RUN        python3.8 -c "import openpmd_api as io; print(io.__version__); print(
 RUN        python3.8 -m openpmd_api.ls --help
 RUN        openpmd-ls --help
 
-# test in fresh env: Ubuntu:18.04 + Python 3.6
-FROM       ubuntu:18.04
-ENV        DEBIAN_FRONTEND noninteractive
-COPY --from=build-env /wheelhouse/openPMD_api-*-cp36-cp36m-manylinux2010_x86_64.whl .
-RUN        apt-get update \
-           && apt-get install -y --no-install-recommends python3 python3-pip \
-           && rm -rf /var/lib/apt/lists/*
-RUN        python3 --version \
-           && python3 -m pip install -U pip \
-           && python3 -m pip install openPMD_api-*-cp36-cp36m-manylinux2010_x86_64.whl
-RUN        python3 -c "import openpmd_api as io; print(io.__version__); print(io.variants)"
-RUN        python3 -m openpmd_api.ls --help
-RUN        openpmd-ls --help
-
 # test in fresh env: Debian:Bullseye + Python 3.9
 FROM       debian:bullseye
 ENV        DEBIAN_FRONTEND noninteractive
@@ -203,6 +189,20 @@ RUN        python3.10 -c "import openpmd_api as io; print(io.__version__); print
 RUN        python3.10 -m openpmd_api.ls --help
 RUN        openpmd-ls --help
 
+# test in fresh env: Debian:Bullseye + Python 3.11
+FROM       debian:bullseye
+ENV        DEBIAN_FRONTEND noninteractive
+COPY --from=build-env /wheelhouse/openPMD_api-*-cp311-cp311-manylinux2010_x86_64.whl .
+RUN        apt-get update \
+           && apt-get install -y --no-install-recommends python3.10 python3-distutils ca-certificates curl \
+           && rm -rf /var/lib/apt/lists/*
+RUN        python3.11 --version \
+           && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \
+           && python3.11 get-pip.py \
+           && python3.11 -m pip install openPMD_api-*-cp311-cp311-manylinux2010_x86_64.whl
+RUN        python3.11 -c "import openpmd_api as io; print(io.__version__); print(io.variants)"
+RUN        python3.11 -m openpmd_api.ls --help
+RUN        openpmd-ls --help
 
 # copy binary artifacts (wheels)
 FROM       quay.io/pypa/manylinux2010_x86_64

NEWS.rst

@@ -9,7 +9,7 @@ Upgrade Guide
 Building openPMD-api now requires a compiler that supports C++17 or newer.
 ``MPark.Variant`` is not a dependency anymore (kudos and thanks for the great time!).
 
-Python 3.10 is now supported.
+Python 3.10 & 3.11 are now supported, Python 3.6 is removed.
 openPMD-api now depends on `toml11 <https://github.com/ToruNiina/toml11>`__ 3.7.1+.
 pybind11 2.10.1 is now the minimally supported version for Python support.
 Catch2 2.13.9 is now the minimally supported version for tests.

README.md

@@ -119,7 +119,7 @@ while those can be built either with or without:
 
 Optional language bindings:
 * Python:
-  * Python 3.6 - 3.10
+  * Python 3.7 - 3.11
   * pybind11 2.10.1+
   * numpy 1.15+
   * mpi4py 2.1+ (optional, for MPI)

conda.yml

@@ -36,7 +36,7 @@ dependencies:
   - pre-commit
   - pyarrow  # for dask
 #  - pybind11  # shipped internally
-  - python>=3.6
+  - python>=3.7
 
 # just a note for later hackery, we could install pip packages inside the env, too:
 #  - pip:

docs/source/dev/dependencies.rst

@@ -39,7 +39,7 @@ Optional: language bindings
 
 * Python:
 
-  * Python 3.6 - 3.10
+  * Python 3.7 - 3.11
   * pybind11 2.10.1+
   * numpy 1.15+
   * mpi4py 2.1+ (optional, for MPI)

setup.py

@@ -190,7 +190,7 @@ def build_extension(self, ext):
     cmdclass=dict(build_ext=CMakeBuild),
     # scripts=['openpmd-ls'],
     zip_safe=False,
-    python_requires='>=3.6, <3.11',
+    python_requires='>=3.7',
     # tests_require=['pytest'],
     install_requires=install_requires,
     # see: src/bindings/python/cli
@@ -219,11 +219,11 @@ def build_extension(self, ext):
         'Topic :: Database :: Front-Ends',
         'Programming Language :: C++',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
         ('License :: OSI Approved :: '
          'GNU Lesser General Public License v3 or later (LGPLv3+)'),
     ],