diff --git a/.github/actions/environment/unit-mpi/action.yml b/.github/actions/environment/unit-mpi/action.yml deleted file mode 100644 index a0263bfbe..000000000 --- a/.github/actions/environment/unit-mpi/action.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Set environment variables for mpi unit tests - -description: - -inputs: - n-procs: - default: 2 - -runs: - using: composite - steps: - - name: set and export variables - shell: bash - run: | - TESTMON_KEY="testmon-unit-mpi" - RUN_MAXWELL="mpirun -n 1 pytest -m single --testmon-forceselect -v --with-mpi --model-name Maxwell" - PYTEST_CMD="mpirun -n ${{ inputs.n-procs }} pytest -v --testmon --with-mpi" - echo "TESTMON_KEY=${TESTMON_KEY}" >> $GITHUB_ENV - echo "RUN_MAXWELL=${RUN_MAXWELL}" >> $GITHUB_ENV - echo "PYTEST_CMD=${PYTEST_CMD}" >> $GITHUB_ENV \ No newline at end of file diff --git a/.github/actions/environment/unit-serial/action.yml b/.github/actions/environment/unit-serial/action.yml deleted file mode 100644 index 07d01cb9d..000000000 --- a/.github/actions/environment/unit-serial/action.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Set environment variables for serial unit tests - -description: - -runs: - using: composite - steps: - - name: set and export variables - shell: bash - run: | - TESTMON_KEY="testmon-unit" - UNINSTALL_MPI="pip uninstall -y mpi4py" - PYTEST_CMD="pytest -v --testmon" - echo "TESTMON_KEY=${TESTMON_KEY}" >> $GITHUB_ENV - echo "UNINSTALL_MPI=${UNINSTALL_MPI}" >> $GITHUB_ENV - echo "PYTEST_CMD=${PYTEST_CMD}" >> $GITHUB_ENV \ No newline at end of file diff --git a/.github/actions/tests/unit/action.yml b/.github/actions/tests/unit/action.yml deleted file mode 100644 index 1e1ce476a..000000000 --- a/.github/actions/tests/unit/action.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: "Run unit tests" - -description: - -inputs: - struphy-path: - required: true - env-name: - default: "" - -runs: - using: composite - steps: - - name: Run unit tests (serial) in container - shell: bash - env: - TESTMON_DATAFILE: ${{ github.workspace }}/.testmondata-unit - TEST_FILES: "bsplines/tests/ \ - console/tests/ \ - feec/tests/ \ - fields_background/tests/ \ - geometry/tests/ \ - initial/tests/ \ - kinetic_background/tests/ \ - linear_algebra/tests/ \ - ode/tests/ \ - pic/tests/ \ - polar/tests/ \ - post_processing/tests/ \ - propagators/tests/" - run: | - set +e - source ${{ inputs.env-name }}/bin/activate || true - set -e - pip show struphy - struphy compile --status - cd ${{ inputs.struphy-path }} - pwd - echo "Running $RUN_MAXWELL" - $RUN_MAXWELL - echo "Running $UNINSTALL_MPI" - $UNINSTALL_MPI - git status || true - echo "Running $PYTEST_CMD $TEST_FILES" - $PYTEST_CMD $TEST_FILES diff --git a/.github/workflows/reusable-scheduled.yml b/.github/workflows/reusable-scheduled.yml index 8baef906c..fa12f281a 100644 --- a/.github/workflows/reusable-scheduled.yml +++ b/.github/workflows/reusable-scheduled.yml @@ -9,6 +9,7 @@ on: jobs: test: + name: ${{ matrix.test-type }} (${{ matrix.compile-language }}, ${{ matrix.shard }}) runs-on: ${{ inputs.os }} env: OMPI_MCA_rmaps_base_oversubscribe: 1 # Linux @@ -19,6 +20,20 @@ jobs: python-version: ["3.12"] compile-language: ["fortran", "c"] test-type: ["unit", "model", "verification"] #"quickstart", "tutorials"] + shard: ["shard-1", "shard-2", "shard-3", "shard-4"] + exclude: + - test-type: model + shard: shard-2 + - test-type: model + shard: shard-3 + - test-type: model + shard: shard-4 + - test-type: verification + shard: shard-2 + - test-type: verification + shard: shard-3 + - test-type: verification + shard: shard-4 steps: # Checkout the repository @@ -74,22 +89,60 @@ jobs: language: ${{ matrix.compile-language }} - name: Set environment for mpi run - uses: ./.github/actions/environment/unit-mpi + run: | + { + echo 'TESTMON_KEY=testmon-unit-mpi' + echo 'RUN_MAXWELL=mpirun -n 1 pytest -m single --testmon-forceselect -v --with-mpi --model-name Maxwell' + echo 'UNINSTALL_MPI=' + echo 'PYTEST_CMD=mpirun -n 2 pytest -v --testmon --with-mpi' + } >> "$GITHUB_ENV" - name: Run unit tests with MPI if: matrix.test-type == 'unit' - uses: ./.github/actions/tests/unit - with: - struphy-path: ${{ env.STRUPHY_PATH }} + env: + TESTMON_DATAFILE: ${{ github.workspace }}/.testmondata-unit-${{ matrix.shard }} + run: | + case "${{ matrix.shard }}" in + shard-1) + TEST_FILES="bsplines/tests/ console/tests/ fields_background/tests/ geometry/tests/ initial/tests/ kinetic_background/tests/ linear_algebra/tests/" + ;; + shard-2) + TEST_FILES="propagators/tests/ ode/tests/ polar/tests/ post_processing/tests/" + ;; + shard-3) + TEST_FILES="feec/tests/" + ;; + shard-4) + TEST_FILES="pic/tests/" + ;; + *) + echo "Unknown test shard: ${{ matrix.shard }}" >&2 + exit 1 + ;; + esac + set +e + source /bin/activate || true + set -e + pip show struphy + struphy compile --status + cd ${{ env.STRUPHY_PATH }} + pwd + echo "Running $RUN_MAXWELL" + $RUN_MAXWELL + echo "Running $UNINSTALL_MPI" + $UNINSTALL_MPI + git status || true + echo "Running $PYTEST_CMD $TEST_FILES" + $PYTEST_CMD $TEST_FILES - name: Run model tests with MPI - if: matrix.test-type == 'model' + if: matrix.test-type == 'model' && matrix.shard == 'shard-1' shell: bash run: | mpirun -oversubscribe -n 2 pytest -x -m models --with-mpi $STRUPHY_PATH/models/tests/default_params/ - name: Run verification tests with MPI - if: matrix.test-type == 'verification' + if: matrix.test-type == 'verification' && matrix.shard == 'shard-1' shell: bash run: | mpirun --oversubscribe -n 2 pytest -x --with-mpi $STRUPHY_PATH/models/tests/verification/ diff --git a/.github/workflows/reusable-unit-testing.yml b/.github/workflows/reusable-unit-testing.yml index d2dca306d..6fbf21f07 100644 --- a/.github/workflows/reusable-unit-testing.yml +++ b/.github/workflows/reusable-unit-testing.yml @@ -21,8 +21,29 @@ permissions: contents: read jobs: + preflight: + runs-on: ${{ inputs.os }} + steps: + - name: Check memory + run: | + free -h + grep -E 'MemTotal|MemAvailable|SwapTotal|SwapFree' /proc/meminfo + unit-tests-in-container-with-struphy: + name: Unit tests (${{ matrix.shard }}) runs-on: ${{ inputs.os }} + strategy: + fail-fast: false + matrix: + include: + - shard: shard-1 + test_files: bsplines/tests/ console/tests/ fields_background/tests/ geometry/tests/ initial/tests/ kinetic_background/tests/ linear_algebra/tests/ + - shard: shard-2 + test_files: propagators/tests/ ode/tests/ polar/tests/ post_processing/tests/ + - shard: shard-3 + test_files: feec/tests/ + - shard: shard-4 + test_files: pic/tests/ container: image: ghcr.io/struphy-hub/struphy/ubuntu-with-struphy:latest @@ -34,6 +55,11 @@ jobs: - name: Check for dockerenv file run: (ls /.dockerenv && echo Found dockerenv) || (echo No dockerenv) + - name: Memory inside container + run: | + free -h + grep -E 'MemTotal|MemAvailable|SwapTotal|SwapFree' /proc/meminfo + - name: Checkout repo uses: actions/checkout@v4 with: @@ -42,11 +68,23 @@ jobs: - name: Set environment for serial run if: inputs.n-procs == 1 - uses: ./.github/actions/environment/unit-serial + run: | + { + echo 'TESTMON_KEY=testmon-unit' + echo 'RUN_MAXWELL=' + echo 'UNINSTALL_MPI=pip uninstall -y mpi4py' + echo 'PYTEST_CMD=pytest -v --testmon' + } >> "$GITHUB_ENV" - name: Set environment for mpi run if: inputs.n-procs > 1 - uses: ./.github/actions/environment/unit-mpi + run: | + { + echo 'TESTMON_KEY=testmon-unit-mpi' + echo 'RUN_MAXWELL=mpirun -n 1 pytest -m single --testmon-forceselect -v --with-mpi --model-name Maxwell' + echo 'UNINSTALL_MPI=' + echo 'PYTEST_CMD=mpirun -n ${{ inputs.n-procs }} pytest -v --testmon --with-mpi' + } >> "$GITHUB_ENV" - name: Check .testmondata 1 run: | @@ -56,12 +94,12 @@ jobs: uses: actions/cache@v4 with: path: | - .testmondata-unit - key: ${{ env.TESTMON_KEY }}-${{ github.ref_name }}-${{ github.event.number }}-${{ github.run_number }} + .testmondata-unit-${{ matrix.shard }} + key: ${{ env.TESTMON_KEY }}-${{ matrix.shard }}-${{ github.ref_name }}-${{ github.event.number }}-${{ github.run_number }} restore-keys: | - ${{ env.TESTMON_KEY }}-${{ github.ref_name }}-${{ github.event.number }}- - ${{ env.TESTMON_KEY }}-${{ github.ref_name }} - ${{ env.TESTMON_KEY }}-devel + ${{ env.TESTMON_KEY }}-${{ matrix.shard }}-${{ github.ref_name }}-${{ github.event.number }}- + ${{ env.TESTMON_KEY }}-${{ matrix.shard }}-${{ github.ref_name }} + ${{ env.TESTMON_KEY }}-${{ matrix.shard }}-devel ${{ env.TESTMON_KEY }}- - name: Check .testmondata 2 @@ -88,7 +126,21 @@ jobs: env-name: /struphy_fortran_/env_fortran_ - name: Run unit tests - uses: ./.github/actions/tests/unit - with: - struphy-path: ${{ env.STRUPHY_PATH }} - env-name: /struphy_fortran_/env_fortran_ \ No newline at end of file + env: + TESTMON_DATAFILE: ${{ github.workspace }}/.testmondata-unit-${{ matrix.shard }} + TEST_FILES: ${{ matrix.test_files }} + run: | + set +e + source /struphy_fortran_/env_fortran_/bin/activate || true + set -e + pip show struphy + struphy compile --status + cd ${{ env.STRUPHY_PATH }} + pwd + echo "Running $RUN_MAXWELL" + $RUN_MAXWELL + echo "Running $UNINSTALL_MPI" + $UNINSTALL_MPI + git status || true + echo "Running $PYTEST_CMD $TEST_FILES" + $PYTEST_CMD $TEST_FILES \ No newline at end of file diff --git a/docker/ubuntu-latest-with-struphy.dockerfile b/docker/ubuntu-latest-with-struphy.dockerfile index 6a95d3a9c..9b3381414 100644 --- a/docker/ubuntu-latest-with-struphy.dockerfile +++ b/docker/ubuntu-latest-with-struphy.dockerfile @@ -12,6 +12,7 @@ FROM ubuntu:latest ARG DEBIAN_FRONTEND=noninteractive +ARG PYTHON_VERSION=3.13 # install linux packages RUN apt update -y && apt clean \ @@ -19,10 +20,11 @@ RUN apt update -y && apt clean \ && add-apt-repository -y ppa:deadsnakes/ppa \ && apt update -y -RUN apt install -y python3 \ - && apt install -y python3-dev \ - && apt install -y python3-pip \ - && apt install -y python3-venv +RUN apt install -y python${PYTHON_VERSION} \ + && apt install -y python${PYTHON_VERSION}-dev \ + && apt install -y python${PYTHON_VERSION}-venv \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \ + && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} RUN apt install -y gfortran gcc \ && apt install -y liblapack-dev libblas-dev @@ -44,32 +46,32 @@ RUN apt install -y g++ liblapack3 cmake cmake-curses-gui zlib1g-dev libnetcdf-de && export CXX=`which g++` # install three versions of struphy -RUN git clone https://github.com/struphy-hub/struphy.git struphy_c_ \ - && cd struphy_c_ \ - && python3 -m venv env_c_ \ - && . env_c_/bin/activate \ - && pip install -U pip \ - && pip install -e .[phys,mpi,doc] --no-cache-dir \ - && struphy compile \ - && deactivate +# RUN git clone https://github.com/struphy-hub/struphy.git struphy_c_ \ +# && cd struphy_c_ \ +# && python${PYTHON_VERSION} -m venv env_c_ \ +# && . env_c_/bin/activate \ +# && pip install -U pip \ +# && pip install -e .[phys,mpi,doc] --no-cache-dir \ +# && struphy compile \ +# && deactivate RUN git clone https://github.com/struphy-hub/struphy.git struphy_fortran_\ && cd struphy_fortran_ \ - && python3 -m venv env_fortran_ \ + && python${PYTHON_VERSION} -m venv env_fortran_ \ && . env_fortran_/bin/activate \ && pip install -U pip \ && pip install -e .[phys,mpi,doc] --no-cache-dir \ && struphy compile --language fortran -y \ && deactivate -RUN git clone https://github.com/struphy-hub/struphy.git struphy_fortran_--omp-pic\ - && cd struphy_fortran_--omp-pic \ - && python3 -m venv env_fortran_--omp-pic \ - && . env_fortran_--omp-pic/bin/activate \ - && pip install -U pip \ - && pip install -e .[phys,mpi,doc] --no-cache-dir \ - && struphy compile --language fortran --omp-pic -y \ - && deactivate +# RUN git clone https://github.com/struphy-hub/struphy.git struphy_fortran_--omp-pic\ +# && cd struphy_fortran_--omp-pic \ +# && python${PYTHON_VERSION} -m venv env_fortran_--omp-pic \ +# && . env_fortran_--omp-pic/bin/activate \ +# && pip install -U pip \ +# && pip install -e .[phys,mpi,doc] --no-cache-dir \ +# && struphy compile --language fortran --omp-pic -y \ +# && deactivate # allow mpirun as root ENV OMPI_ALLOW_RUN_AS_ROOT=1 diff --git a/pyproject.toml b/pyproject.toml index 3a600a09b..ca2288feb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "ipywidgets<=8.1.8", "plotly<=6.7.0", "pyvista<=0.48.0", - "trame<=3.12.0", + "trame<=3.13.0", "trame-vtk<=2.11.8", "trame-vuetify<=3.2.2", "nest_asyncio2<=1.7.2", diff --git a/src/struphy/pic/tests/test_sph.py b/src/struphy/pic/tests/test_sph.py index bdfdd093c..4a1ea8cbd 100644 --- a/src/struphy/pic/tests/test_sph.py +++ b/src/struphy/pic/tests/test_sph.py @@ -60,6 +60,7 @@ def test_sph_evaluation_1d( if derivative == 0: ppb = 1000 else: + # return # skip non-tesselation test for derivative in 1D ppb = 20000 loading_params = LoadingParameters(ppb=ppb, seed=223) @@ -543,6 +544,7 @@ def test_evaluation_SPH_h_convergence_1d(boxes_per_dim, bc_x, eval_pts, tesselat ppb = 160 loading_params = LoadingParameters(ppb=ppb, loading="tesselation") else: + # return # skip non-tesselation test for h-convergence in 1D Np = 160000 ppb = None loading_params = LoadingParameters(Np=Np, ppb=ppb, seed=1607) @@ -660,6 +662,7 @@ def test_evaluation_mc_Np_and_h_convergence_1d(boxes_per_dim, bc_x, eval_pts, te ppbs = [4, 8, 16, 32, 64] Nps = [None] * len(ppbs) else: + # return # skip non-tesselation test for convergence in 1D Nps = [(2**k) * 10**3 for k in range(-2, 9)] ppbs = [None] * len(Nps) @@ -1968,6 +1971,7 @@ def u_xyz(x, y, z): # direction="x", # show_plot=True, # ) - test_sph_velocity_evaluation_2d( - (12, 12, 1), "gaussian_2d", 1, "periodic", "periodic", 11, tesselation=False, show_plot=True - ) + # test_sph_velocity_evaluation_2d( + # (12, 12, 1), "gaussian_2d", 1, "periodic", "periodic", 11, tesselation=False, show_plot=True + # ) + test_sph_evaluation_1d((24, 1, 1), "trigonometric_1d", 0, "periodic", 11, tesselation=False, show_plot=True)