From 9ad54879266c5dabc554965a62b1d69624952ff7 Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Mon, 11 May 2026 14:04:54 +0200 Subject: [PATCH 01/21] do not use eager update in unit tests --- .github/actions/install/struphy_in_container/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install/struphy_in_container/action.yml b/.github/actions/install/struphy_in_container/action.yml index fb591491c..cb020b5bb 100644 --- a/.github/actions/install/struphy_in_container/action.yml +++ b/.github/actions/install/struphy_in_container/action.yml @@ -26,7 +26,7 @@ runs: git status source env_fortran_/bin/activate which python3 - pip install -U --upgrade-strategy eager -e ".[phys,mpi]" + pip install -e ".[phys,mpi]" pip install -e ".[doc]" PYTHON=$(which python) STRUPHY_PATH=$($PYTHON -c "import struphy; print(struphy.__path__[0])") From 9d0872554aa370e001f48bf459451b664f9134a7 Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Mon, 11 May 2026 15:44:14 +0200 Subject: [PATCH 02/21] skip derivative tests for 1d monte carlo sph eval --- src/struphy/pic/tests/test_sph.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/struphy/pic/tests/test_sph.py b/src/struphy/pic/tests/test_sph.py index bdfdd093c..27399d5d9 100644 --- a/src/struphy/pic/tests/test_sph.py +++ b/src/struphy/pic/tests/test_sph.py @@ -60,6 +60,7 @@ def test_sph_evaluation_1d( if derivative == 0: ppb = 1000 else: + return # skip non-tesselation test for derivative in 1D ppb = 20000 loading_params = LoadingParameters(ppb=ppb, seed=223) @@ -1968,6 +1969,7 @@ def u_xyz(x, y, z): # direction="x", # show_plot=True, # ) - test_sph_velocity_evaluation_2d( - (12, 12, 1), "gaussian_2d", 1, "periodic", "periodic", 11, tesselation=False, show_plot=True - ) + # test_sph_velocity_evaluation_2d( + # (12, 12, 1), "gaussian_2d", 1, "periodic", "periodic", 11, tesselation=False, show_plot=True + # ) + test_sph_evaluation_1d((24, 1, 1), "trigonometric_1d", 0, "periodic", 11, tesselation=False, show_plot=True) From ae50599998d3f75fe280f37a0447c3be027a3da0 Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Mon, 11 May 2026 15:55:52 +0200 Subject: [PATCH 03/21] update dependency bounds --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3a600a09b..ca2288feb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "ipywidgets<=8.1.8", "plotly<=6.7.0", "pyvista<=0.48.0", - "trame<=3.12.0", + "trame<=3.13.0", "trame-vtk<=2.11.8", "trame-vuetify<=3.2.2", "nest_asyncio2<=1.7.2", From de6e6fdecbc765bdd5fdde3509a6aefaa495d1e4 Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Mon, 11 May 2026 15:57:14 +0200 Subject: [PATCH 04/21] re-add eager strategy for unit tests --- .github/actions/install/struphy_in_container/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install/struphy_in_container/action.yml b/.github/actions/install/struphy_in_container/action.yml index cb020b5bb..fb591491c 100644 --- a/.github/actions/install/struphy_in_container/action.yml +++ b/.github/actions/install/struphy_in_container/action.yml @@ -26,7 +26,7 @@ runs: git status source env_fortran_/bin/activate which python3 - pip install -e ".[phys,mpi]" + pip install -U --upgrade-strategy eager -e ".[phys,mpi]" pip install -e ".[doc]" PYTHON=$(which python) STRUPHY_PATH=$($PYTHON -c "import struphy; print(struphy.__path__[0])") From 8409da24d47a5ed3fab980a8749792ae90dd45cc Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Tue, 12 May 2026 06:54:35 +0200 Subject: [PATCH 05/21] check available memory in and outside the container --- .github/workflows/reusable-unit-testing.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/reusable-unit-testing.yml b/.github/workflows/reusable-unit-testing.yml index d2dca306d..cc3ed3b64 100644 --- a/.github/workflows/reusable-unit-testing.yml +++ b/.github/workflows/reusable-unit-testing.yml @@ -21,6 +21,14 @@ permissions: contents: read jobs: + preflight: + runs-on: ${{ inputs.os }} + steps: + - name: Check memory + run: | + free -h + grep -E 'MemTotal|MemAvailable|SwapTotal|SwapFree' /proc/meminfo + unit-tests-in-container-with-struphy: runs-on: ${{ inputs.os }} @@ -34,6 +42,11 @@ jobs: - name: Check for dockerenv file run: (ls /.dockerenv && echo Found dockerenv) || (echo No dockerenv) + - name: Memory inside container + run: | + free -h + grep -E 'MemTotal|MemAvailable|SwapTotal|SwapFree' /proc/meminfo + - name: Checkout repo uses: actions/checkout@v4 with: From ee776888b822a45de3e77ac041ea14c2f87d0387 Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Tue, 12 May 2026 07:39:37 +0200 Subject: [PATCH 06/21] skip two more memory-heavy mc sph tests --- src/struphy/pic/tests/test_sph.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/struphy/pic/tests/test_sph.py b/src/struphy/pic/tests/test_sph.py index 27399d5d9..2d525f1e3 100644 --- a/src/struphy/pic/tests/test_sph.py +++ b/src/struphy/pic/tests/test_sph.py @@ -544,6 +544,7 @@ def test_evaluation_SPH_h_convergence_1d(boxes_per_dim, bc_x, eval_pts, tesselat ppb = 160 loading_params = LoadingParameters(ppb=ppb, loading="tesselation") else: + return # skip non-tesselation test for h-convergence in 1D Np = 160000 ppb = None loading_params = LoadingParameters(Np=Np, ppb=ppb, seed=1607) @@ -661,6 +662,7 @@ def test_evaluation_mc_Np_and_h_convergence_1d(boxes_per_dim, bc_x, eval_pts, te ppbs = [4, 8, 16, 32, 64] Nps = [None] * len(ppbs) else: + return # skip non-tesselation test for convergence in 1D Nps = [(2**k) * 10**3 for k in range(-2, 9)] ppbs = [None] * len(Nps) From acd1f53488c0204f141b3bafbd7b70a704c09800 Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Tue, 12 May 2026 08:14:37 +0200 Subject: [PATCH 07/21] try sharding the unit tests --- .../actions/environment/unit-mpi/action.yml | 20 ------ .../environment/unit-serial/action.yml | 16 ----- .github/actions/tests/unit/action.yml | 45 ------------- .github/workflows/reusable-scheduled.yml | 65 +++++++++++++++++-- .github/workflows/reusable-unit-testing.yml | 61 +++++++++++++---- 5 files changed, 109 insertions(+), 98 deletions(-) delete mode 100644 .github/actions/environment/unit-mpi/action.yml delete mode 100644 .github/actions/environment/unit-serial/action.yml delete mode 100644 .github/actions/tests/unit/action.yml diff --git a/.github/actions/environment/unit-mpi/action.yml b/.github/actions/environment/unit-mpi/action.yml deleted file mode 100644 index a0263bfbe..000000000 --- a/.github/actions/environment/unit-mpi/action.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Set environment variables for mpi unit tests - -description: - -inputs: - n-procs: - default: 2 - -runs: - using: composite - steps: - - name: set and export variables - shell: bash - run: | - TESTMON_KEY="testmon-unit-mpi" - RUN_MAXWELL="mpirun -n 1 pytest -m single --testmon-forceselect -v --with-mpi --model-name Maxwell" - PYTEST_CMD="mpirun -n ${{ inputs.n-procs }} pytest -v --testmon --with-mpi" - echo "TESTMON_KEY=${TESTMON_KEY}" >> $GITHUB_ENV - echo "RUN_MAXWELL=${RUN_MAXWELL}" >> $GITHUB_ENV - echo "PYTEST_CMD=${PYTEST_CMD}" >> $GITHUB_ENV \ No newline at end of file diff --git a/.github/actions/environment/unit-serial/action.yml b/.github/actions/environment/unit-serial/action.yml deleted file mode 100644 index 07d01cb9d..000000000 --- a/.github/actions/environment/unit-serial/action.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Set environment variables for serial unit tests - -description: - -runs: - using: composite - steps: - - name: set and export variables - shell: bash - run: | - TESTMON_KEY="testmon-unit" - UNINSTALL_MPI="pip uninstall -y mpi4py" - PYTEST_CMD="pytest -v --testmon" - echo "TESTMON_KEY=${TESTMON_KEY}" >> $GITHUB_ENV - echo "UNINSTALL_MPI=${UNINSTALL_MPI}" >> $GITHUB_ENV - echo "PYTEST_CMD=${PYTEST_CMD}" >> $GITHUB_ENV \ No newline at end of file diff --git a/.github/actions/tests/unit/action.yml b/.github/actions/tests/unit/action.yml deleted file mode 100644 index 1e1ce476a..000000000 --- a/.github/actions/tests/unit/action.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: "Run unit tests" - -description: - -inputs: - struphy-path: - required: true - env-name: - default: "" - -runs: - using: composite - steps: - - name: Run unit tests (serial) in container - shell: bash - env: - TESTMON_DATAFILE: ${{ github.workspace }}/.testmondata-unit - TEST_FILES: "bsplines/tests/ \ - console/tests/ \ - feec/tests/ \ - fields_background/tests/ \ - geometry/tests/ \ - initial/tests/ \ - kinetic_background/tests/ \ - linear_algebra/tests/ \ - ode/tests/ \ - pic/tests/ \ - polar/tests/ \ - post_processing/tests/ \ - propagators/tests/" - run: | - set +e - source ${{ inputs.env-name }}/bin/activate || true - set -e - pip show struphy - struphy compile --status - cd ${{ inputs.struphy-path }} - pwd - echo "Running $RUN_MAXWELL" - $RUN_MAXWELL - echo "Running $UNINSTALL_MPI" - $UNINSTALL_MPI - git status || true - echo "Running $PYTEST_CMD $TEST_FILES" - $PYTEST_CMD $TEST_FILES diff --git a/.github/workflows/reusable-scheduled.yml b/.github/workflows/reusable-scheduled.yml index 8baef906c..f8c286ea1 100644 --- a/.github/workflows/reusable-scheduled.yml +++ b/.github/workflows/reusable-scheduled.yml @@ -9,6 +9,7 @@ on: jobs: test: + name: ${{ matrix.test-type }} (${{ matrix.compile-language }}, ${{ matrix.shard }}) runs-on: ${{ inputs.os }} env: OMPI_MCA_rmaps_base_oversubscribe: 1 # Linux @@ -19,6 +20,20 @@ jobs: python-version: ["3.12"] compile-language: ["fortran", "c"] test-type: ["unit", "model", "verification"] #"quickstart", "tutorials"] + shard: ["shard-1", "shard-2", "shard-3", "shard-4"] + exclude: + - test-type: model + shard: shard-2 + - test-type: model + shard: shard-3 + - test-type: model + shard: shard-4 + - test-type: verification + shard: shard-2 + - test-type: verification + shard: shard-3 + - test-type: verification + shard: shard-4 steps: # Checkout the repository @@ -74,22 +89,60 @@ jobs: language: ${{ matrix.compile-language }} - name: Set environment for mpi run - uses: ./.github/actions/environment/unit-mpi + run: | + { + echo 'TESTMON_KEY=testmon-unit-mpi' + echo 'RUN_MAXWELL=mpirun -n 1 pytest -m single --testmon-forceselect -v --with-mpi --model-name Maxwell' + echo 'UNINSTALL_MPI=' + echo 'PYTEST_CMD=mpirun -n 2 pytest -v --testmon --with-mpi' + } >> "$GITHUB_ENV" - name: Run unit tests with MPI if: matrix.test-type == 'unit' - uses: ./.github/actions/tests/unit - with: - struphy-path: ${{ env.STRUPHY_PATH }} + env: + TESTMON_DATAFILE: ${{ github.workspace }}/.testmondata-unit-${{ matrix.shard }} + run: | + case "${{ matrix.shard }}" in + shard-1) + TEST_FILES="bsplines/tests/ console/tests/ feec/tests/" + ;; + shard-2) + TEST_FILES="fields_background/tests/ geometry/tests/ initial/tests/" + ;; + shard-3) + TEST_FILES="kinetic_background/tests/ linear_algebra/tests/ ode/tests/" + ;; + shard-4) + TEST_FILES="pic/tests/ polar/tests/ post_processing/tests/ propagators/tests/" + ;; + *) + echo "Unknown test shard: ${{ matrix.shard }}" >&2 + exit 1 + ;; + esac + set +e + source /bin/activate || true + set -e + pip show struphy + struphy compile --status + cd ${{ env.STRUPHY_PATH }} + pwd + echo "Running $RUN_MAXWELL" + $RUN_MAXWELL + echo "Running $UNINSTALL_MPI" + $UNINSTALL_MPI + git status || true + echo "Running $PYTEST_CMD $TEST_FILES" + $PYTEST_CMD $TEST_FILES - name: Run model tests with MPI - if: matrix.test-type == 'model' + if: matrix.test-type == 'model' && matrix.shard == 'shard-1' shell: bash run: | mpirun -oversubscribe -n 2 pytest -x -m models --with-mpi $STRUPHY_PATH/models/tests/default_params/ - name: Run verification tests with MPI - if: matrix.test-type == 'verification' + if: matrix.test-type == 'verification' && matrix.shard == 'shard-1' shell: bash run: | mpirun --oversubscribe -n 2 pytest -x --with-mpi $STRUPHY_PATH/models/tests/verification/ diff --git a/.github/workflows/reusable-unit-testing.yml b/.github/workflows/reusable-unit-testing.yml index cc3ed3b64..491cedbbf 100644 --- a/.github/workflows/reusable-unit-testing.yml +++ b/.github/workflows/reusable-unit-testing.yml @@ -30,7 +30,20 @@ jobs: grep -E 'MemTotal|MemAvailable|SwapTotal|SwapFree' /proc/meminfo unit-tests-in-container-with-struphy: + name: Unit tests (${{ matrix.shard }}) runs-on: ${{ inputs.os }} + strategy: + fail-fast: false + matrix: + include: + - shard: shard-1 + test_files: bsplines/tests/ console/tests/ feec/tests/ + - shard: shard-2 + test_files: fields_background/tests/ geometry/tests/ initial/tests/ + - shard: shard-3 + test_files: kinetic_background/tests/ linear_algebra/tests/ ode/tests/ + - shard: shard-4 + test_files: pic/tests/ polar/tests/ post_processing/tests/ propagators/tests/ container: image: ghcr.io/struphy-hub/struphy/ubuntu-with-struphy:latest @@ -55,11 +68,23 @@ jobs: - name: Set environment for serial run if: inputs.n-procs == 1 - uses: ./.github/actions/environment/unit-serial + run: | + { + echo 'TESTMON_KEY=testmon-unit' + echo 'RUN_MAXWELL=' + echo 'UNINSTALL_MPI=pip uninstall -y mpi4py' + echo 'PYTEST_CMD=pytest -v --testmon' + } >> "$GITHUB_ENV" - name: Set environment for mpi run if: inputs.n-procs > 1 - uses: ./.github/actions/environment/unit-mpi + run: | + { + echo 'TESTMON_KEY=testmon-unit-mpi' + echo 'RUN_MAXWELL=mpirun -n 1 pytest -m single --testmon-forceselect -v --with-mpi --model-name Maxwell' + echo 'UNINSTALL_MPI=' + echo 'PYTEST_CMD=mpirun -n ${{ inputs.n-procs }} pytest -v --testmon --with-mpi' + } >> "$GITHUB_ENV" - name: Check .testmondata 1 run: | @@ -69,12 +94,12 @@ jobs: uses: actions/cache@v4 with: path: | - .testmondata-unit - key: ${{ env.TESTMON_KEY }}-${{ github.ref_name }}-${{ github.event.number }}-${{ github.run_number }} + .testmondata-unit-${{ matrix.shard }} + key: ${{ env.TESTMON_KEY }}-${{ matrix.shard }}-${{ github.ref_name }}-${{ github.event.number }}-${{ github.run_number }} restore-keys: | - ${{ env.TESTMON_KEY }}-${{ github.ref_name }}-${{ github.event.number }}- - ${{ env.TESTMON_KEY }}-${{ github.ref_name }} - ${{ env.TESTMON_KEY }}-devel + ${{ env.TESTMON_KEY }}-${{ matrix.shard }}-${{ github.ref_name }}-${{ github.event.number }}- + ${{ env.TESTMON_KEY }}-${{ matrix.shard }}-${{ github.ref_name }} + ${{ env.TESTMON_KEY }}-${{ matrix.shard }}-devel ${{ env.TESTMON_KEY }}- - name: Check .testmondata 2 @@ -101,7 +126,21 @@ jobs: env-name: /struphy_fortran_/env_fortran_ - name: Run unit tests - uses: ./.github/actions/tests/unit - with: - struphy-path: ${{ env.STRUPHY_PATH }} - env-name: /struphy_fortran_/env_fortran_ \ No newline at end of file + env: + TESTMON_DATAFILE: ${{ github.workspace }}/.testmondata-unit-${{ matrix.shard }} + TEST_FILES: ${{ matrix.test_files }} + run: | + set +e + source /struphy_fortran_/env_fortran_/bin/activate || true + set -e + pip show struphy + struphy compile --status + cd ${{ env.STRUPHY_PATH }} + pwd + echo "Running $RUN_MAXWELL" + $RUN_MAXWELL + echo "Running $UNINSTALL_MPI" + $UNINSTALL_MPI + git status || true + echo "Running $PYTEST_CMD $TEST_FILES" + $PYTEST_CMD $TEST_FILES \ No newline at end of file From a322af1407aef5c74767afe5c2c93927a5bc4e39 Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Tue, 12 May 2026 08:41:39 +0200 Subject: [PATCH 08/21] remove pyvista from image and push to ghcr --- .github/workflows/ghcr.yml | 3 +++ docker/ubuntu-latest-with-struphy.dockerfile | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 17a0706bc..b0847024b 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -6,6 +6,9 @@ on: push: branches: - devel + pull_request: + branches: + - devel # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: diff --git a/docker/ubuntu-latest-with-struphy.dockerfile b/docker/ubuntu-latest-with-struphy.dockerfile index 6a95d3a9c..6170c594d 100644 --- a/docker/ubuntu-latest-with-struphy.dockerfile +++ b/docker/ubuntu-latest-with-struphy.dockerfile @@ -34,9 +34,6 @@ RUN apt install -y git \ && apt install -y pandoc graphviz sqlite3 \ && bash -c "source ~/.bashrc" -# for .show_3d methods (pyvista) -RUN apt install -y libosmesa6 libosmesa6-dev libegl-mesa0 - # for gvec RUN apt install -y g++ liblapack3 cmake cmake-curses-gui zlib1g-dev libnetcdf-dev libnetcdff-dev \ && export FC=`which gfortran` \ From 1790f82dc853e2499c5f6e8e862d97975ce1aafb Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Tue, 12 May 2026 08:47:50 +0200 Subject: [PATCH 09/21] change sharding of unit tests --- .github/workflows/reusable-scheduled.yml | 8 ++++---- .github/workflows/reusable-unit-testing.yml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/reusable-scheduled.yml b/.github/workflows/reusable-scheduled.yml index f8c286ea1..4f678ca95 100644 --- a/.github/workflows/reusable-scheduled.yml +++ b/.github/workflows/reusable-scheduled.yml @@ -104,16 +104,16 @@ jobs: run: | case "${{ matrix.shard }}" in shard-1) - TEST_FILES="bsplines/tests/ console/tests/ feec/tests/" + TEST_FILES="bsplines/tests/ console/tests/ fields_background/tests/ geometry/tests/ initial/tests/ kinetic_background/tests/ linear_algebra/tests/ ode/tests/ polar/tests/ post_processing/tests/" ;; shard-2) - TEST_FILES="fields_background/tests/ geometry/tests/ initial/tests/" + TEST_FILES="propagators/tests/" ;; shard-3) - TEST_FILES="kinetic_background/tests/ linear_algebra/tests/ ode/tests/" + TEST_FILES="feec/tests/" ;; shard-4) - TEST_FILES="pic/tests/ polar/tests/ post_processing/tests/ propagators/tests/" + TEST_FILES="pic/tests/" ;; *) echo "Unknown test shard: ${{ matrix.shard }}" >&2 diff --git a/.github/workflows/reusable-unit-testing.yml b/.github/workflows/reusable-unit-testing.yml index 491cedbbf..b1c90e247 100644 --- a/.github/workflows/reusable-unit-testing.yml +++ b/.github/workflows/reusable-unit-testing.yml @@ -37,13 +37,13 @@ jobs: matrix: include: - shard: shard-1 - test_files: bsplines/tests/ console/tests/ feec/tests/ + test_files: bsplines/tests/ console/tests/ fields_background/tests/ geometry/tests/ initial/tests/ kinetic_background/tests/ linear_algebra/tests/ ode/tests/ polar/tests/ post_processing/tests/ - shard: shard-2 - test_files: fields_background/tests/ geometry/tests/ initial/tests/ + test_files: propagators/tests/ - shard: shard-3 - test_files: kinetic_background/tests/ linear_algebra/tests/ ode/tests/ + test_files: feec/tests/ - shard: shard-4 - test_files: pic/tests/ polar/tests/ post_processing/tests/ propagators/tests/ + test_files: pic/tests/ container: image: ghcr.io/struphy-hub/struphy/ubuntu-with-struphy:latest From c9f7e687426ddfdb0aac0d5c25176b1af84747da Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Tue, 12 May 2026 09:14:00 +0200 Subject: [PATCH 10/21] install feectools always; do not push images on PR --- .github/workflows/ghcr.yml | 3 --- .github/workflows/reusable-unit-testing.yml | 10 +++++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index b0847024b..17a0706bc 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -6,9 +6,6 @@ on: push: branches: - devel - pull_request: - branches: - - devel # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: diff --git a/.github/workflows/reusable-unit-testing.yml b/.github/workflows/reusable-unit-testing.yml index b1c90e247..868e54f2b 100644 --- a/.github/workflows/reusable-unit-testing.yml +++ b/.github/workflows/reusable-unit-testing.yml @@ -109,13 +109,13 @@ jobs: - name: Install Struphy in Container uses: ./.github/actions/install/struphy_in_container - - name: Get submodule diff - uses: ./.github/actions/submodule-diff - with: - start-dir: /struphy_fortran_ + # - name: Get submodule diff + # uses: ./.github/actions/submodule-diff + # with: + # start-dir: /struphy_fortran_ - name: Reinstall feectools from submodule - if: env.SUBMOD_CHANGED == 'true' + # if: env.SUBMOD_CHANGED == 'true' uses: ./.github/actions/install/feectools-submodule with: env-name: /struphy_fortran_/env_fortran_ From a3ec0b0aa320aefb31afd951c02c3182ebf60dbd Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Tue, 12 May 2026 09:54:33 +0200 Subject: [PATCH 11/21] do just one install in container --- .github/actions/install/struphy_in_container/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/install/struphy_in_container/action.yml b/.github/actions/install/struphy_in_container/action.yml index fb591491c..7d371974f 100644 --- a/.github/actions/install/struphy_in_container/action.yml +++ b/.github/actions/install/struphy_in_container/action.yml @@ -26,8 +26,7 @@ runs: git status source env_fortran_/bin/activate which python3 - pip install -U --upgrade-strategy eager -e ".[phys,mpi]" - pip install -e ".[doc]" + pip install -U --upgrade-strategy eager -e ".[phys,mpi,doc]" PYTHON=$(which python) STRUPHY_PATH=$($PYTHON -c "import struphy; print(struphy.__path__[0])") echo "Struphy is installed at: $STRUPHY_PATH" From 85a53c09d749ba631e5c6910bafd62f9006d5a29 Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Tue, 12 May 2026 10:21:56 +0200 Subject: [PATCH 12/21] monitor memory --- .github/workflows/reusable-unit-testing.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/reusable-unit-testing.yml b/.github/workflows/reusable-unit-testing.yml index 868e54f2b..ad672d81a 100644 --- a/.github/workflows/reusable-unit-testing.yml +++ b/.github/workflows/reusable-unit-testing.yml @@ -125,6 +125,18 @@ jobs: with: env-name: /struphy_fortran_/env_fortran_ + - name: Monitor memory + run: | + while true; do + date + free -h + echo "cgroup memory:" + cat /sys/fs/cgroup/memory.current 2>/dev/null || true + cat /sys/fs/cgroup/memory.max 2>/dev/null || true + echo + sleep 10 + done & + - name: Run unit tests env: TESTMON_DATAFILE: ${{ github.workspace }}/.testmondata-unit-${{ matrix.shard }} From 049f7c72691bb4319269b506b98bd09b24bf9108 Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Tue, 12 May 2026 10:28:29 +0200 Subject: [PATCH 13/21] check mmemory before unit tests --- .github/workflows/reusable-unit-testing.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/reusable-unit-testing.yml b/.github/workflows/reusable-unit-testing.yml index ad672d81a..e5cb398c2 100644 --- a/.github/workflows/reusable-unit-testing.yml +++ b/.github/workflows/reusable-unit-testing.yml @@ -125,18 +125,6 @@ jobs: with: env-name: /struphy_fortran_/env_fortran_ - - name: Monitor memory - run: | - while true; do - date - free -h - echo "cgroup memory:" - cat /sys/fs/cgroup/memory.current 2>/dev/null || true - cat /sys/fs/cgroup/memory.max 2>/dev/null || true - echo - sleep 10 - done & - - name: Run unit tests env: TESTMON_DATAFILE: ${{ github.workspace }}/.testmondata-unit-${{ matrix.shard }} @@ -154,5 +142,7 @@ jobs: echo "Running $UNINSTALL_MPI" $UNINSTALL_MPI git status || true + free -h + grep -E 'MemTotal|MemAvailable|SwapTotal|SwapFree' /proc/meminfo echo "Running $PYTEST_CMD $TEST_FILES" $PYTEST_CMD $TEST_FILES \ No newline at end of file From e337da77748a10cacacf2e3337e470ec567caa75 Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Tue, 12 May 2026 11:41:55 +0200 Subject: [PATCH 14/21] build new image --- .github/workflows/ghcr.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 17a0706bc..b0847024b 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -6,6 +6,9 @@ on: push: branches: - devel + pull_request: + branches: + - devel # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: From f2311d2bd095fa437a4510c90776e9e3302c9354 Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Tue, 12 May 2026 12:26:28 +0200 Subject: [PATCH 15/21] change sharding a bit --- .github/workflows/reusable-unit-testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable-unit-testing.yml b/.github/workflows/reusable-unit-testing.yml index e5cb398c2..9a2215b7f 100644 --- a/.github/workflows/reusable-unit-testing.yml +++ b/.github/workflows/reusable-unit-testing.yml @@ -37,9 +37,9 @@ jobs: matrix: include: - shard: shard-1 - test_files: bsplines/tests/ console/tests/ fields_background/tests/ geometry/tests/ initial/tests/ kinetic_background/tests/ linear_algebra/tests/ ode/tests/ polar/tests/ post_processing/tests/ + test_files: bsplines/tests/ console/tests/ fields_background/tests/ geometry/tests/ initial/tests/ kinetic_background/tests/ linear_algebra/tests/ - shard: shard-2 - test_files: propagators/tests/ + test_files: propagators/tests/ ode/tests/ polar/tests/ post_processing/tests/ - shard: shard-3 test_files: feec/tests/ - shard: shard-4 From bfdc66c940e3cee1a149792a52f79e2d5ce9a4c7 Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Tue, 12 May 2026 12:27:21 +0200 Subject: [PATCH 16/21] do not publish to ghcr --- .github/workflows/ghcr.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index b0847024b..17a0706bc 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -6,9 +6,6 @@ on: push: branches: - devel - pull_request: - branches: - - devel # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: From d185f4f2f1330f39ec56c2144830193ac98a743a Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Tue, 12 May 2026 14:10:12 +0200 Subject: [PATCH 17/21] install osmesa again --- .github/workflows/ghcr.yml | 3 +++ docker/ubuntu-latest-with-struphy.dockerfile | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 17a0706bc..b0847024b 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -6,6 +6,9 @@ on: push: branches: - devel + pull_request: + branches: + - devel # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: diff --git a/docker/ubuntu-latest-with-struphy.dockerfile b/docker/ubuntu-latest-with-struphy.dockerfile index 6170c594d..6a95d3a9c 100644 --- a/docker/ubuntu-latest-with-struphy.dockerfile +++ b/docker/ubuntu-latest-with-struphy.dockerfile @@ -34,6 +34,9 @@ RUN apt install -y git \ && apt install -y pandoc graphviz sqlite3 \ && bash -c "source ~/.bashrc" +# for .show_3d methods (pyvista) +RUN apt install -y libosmesa6 libosmesa6-dev libegl-mesa0 + # for gvec RUN apt install -y g++ liblapack3 cmake cmake-curses-gui zlib1g-dev libnetcdf-dev libnetcdff-dev \ && export FC=`which gfortran` \ From a841ad1350caf96f3b9bd908754d919884185136 Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Tue, 12 May 2026 15:20:09 +0200 Subject: [PATCH 18/21] pin python version to 3.13 in image with struphy --- docker/ubuntu-latest-with-struphy.dockerfile | 44 ++++++++++---------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/docker/ubuntu-latest-with-struphy.dockerfile b/docker/ubuntu-latest-with-struphy.dockerfile index 6a95d3a9c..9b3381414 100644 --- a/docker/ubuntu-latest-with-struphy.dockerfile +++ b/docker/ubuntu-latest-with-struphy.dockerfile @@ -12,6 +12,7 @@ FROM ubuntu:latest ARG DEBIAN_FRONTEND=noninteractive +ARG PYTHON_VERSION=3.13 # install linux packages RUN apt update -y && apt clean \ @@ -19,10 +20,11 @@ RUN apt update -y && apt clean \ && add-apt-repository -y ppa:deadsnakes/ppa \ && apt update -y -RUN apt install -y python3 \ - && apt install -y python3-dev \ - && apt install -y python3-pip \ - && apt install -y python3-venv +RUN apt install -y python${PYTHON_VERSION} \ + && apt install -y python${PYTHON_VERSION}-dev \ + && apt install -y python${PYTHON_VERSION}-venv \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \ + && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} RUN apt install -y gfortran gcc \ && apt install -y liblapack-dev libblas-dev @@ -44,32 +46,32 @@ RUN apt install -y g++ liblapack3 cmake cmake-curses-gui zlib1g-dev libnetcdf-de && export CXX=`which g++` # install three versions of struphy -RUN git clone https://github.com/struphy-hub/struphy.git struphy_c_ \ - && cd struphy_c_ \ - && python3 -m venv env_c_ \ - && . env_c_/bin/activate \ - && pip install -U pip \ - && pip install -e .[phys,mpi,doc] --no-cache-dir \ - && struphy compile \ - && deactivate +# RUN git clone https://github.com/struphy-hub/struphy.git struphy_c_ \ +# && cd struphy_c_ \ +# && python${PYTHON_VERSION} -m venv env_c_ \ +# && . env_c_/bin/activate \ +# && pip install -U pip \ +# && pip install -e .[phys,mpi,doc] --no-cache-dir \ +# && struphy compile \ +# && deactivate RUN git clone https://github.com/struphy-hub/struphy.git struphy_fortran_\ && cd struphy_fortran_ \ - && python3 -m venv env_fortran_ \ + && python${PYTHON_VERSION} -m venv env_fortran_ \ && . env_fortran_/bin/activate \ && pip install -U pip \ && pip install -e .[phys,mpi,doc] --no-cache-dir \ && struphy compile --language fortran -y \ && deactivate -RUN git clone https://github.com/struphy-hub/struphy.git struphy_fortran_--omp-pic\ - && cd struphy_fortran_--omp-pic \ - && python3 -m venv env_fortran_--omp-pic \ - && . env_fortran_--omp-pic/bin/activate \ - && pip install -U pip \ - && pip install -e .[phys,mpi,doc] --no-cache-dir \ - && struphy compile --language fortran --omp-pic -y \ - && deactivate +# RUN git clone https://github.com/struphy-hub/struphy.git struphy_fortran_--omp-pic\ +# && cd struphy_fortran_--omp-pic \ +# && python${PYTHON_VERSION} -m venv env_fortran_--omp-pic \ +# && . env_fortran_--omp-pic/bin/activate \ +# && pip install -U pip \ +# && pip install -e .[phys,mpi,doc] --no-cache-dir \ +# && struphy compile --language fortran --omp-pic -y \ +# && deactivate # allow mpirun as root ENV OMPI_ALLOW_RUN_AS_ROOT=1 From db1ada4026e10af4358a151a5f131888b710a3ad Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Tue, 12 May 2026 15:39:19 +0200 Subject: [PATCH 19/21] do not push to ghcr --- .github/workflows/ghcr.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index b0847024b..17a0706bc 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -6,9 +6,6 @@ on: push: branches: - devel - pull_request: - branches: - - devel # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: From dd167b8d7ff5936458695deaf6b78a7c0165786d Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Tue, 12 May 2026 16:58:55 +0200 Subject: [PATCH 20/21] revert some changes --- .../actions/install/struphy_in_container/action.yml | 3 ++- .github/workflows/reusable-scheduled.yml | 4 ++-- .github/workflows/reusable-unit-testing.yml | 12 +++++------- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/actions/install/struphy_in_container/action.yml b/.github/actions/install/struphy_in_container/action.yml index 7d371974f..fb591491c 100644 --- a/.github/actions/install/struphy_in_container/action.yml +++ b/.github/actions/install/struphy_in_container/action.yml @@ -26,7 +26,8 @@ runs: git status source env_fortran_/bin/activate which python3 - pip install -U --upgrade-strategy eager -e ".[phys,mpi,doc]" + pip install -U --upgrade-strategy eager -e ".[phys,mpi]" + pip install -e ".[doc]" PYTHON=$(which python) STRUPHY_PATH=$($PYTHON -c "import struphy; print(struphy.__path__[0])") echo "Struphy is installed at: $STRUPHY_PATH" diff --git a/.github/workflows/reusable-scheduled.yml b/.github/workflows/reusable-scheduled.yml index 4f678ca95..fa12f281a 100644 --- a/.github/workflows/reusable-scheduled.yml +++ b/.github/workflows/reusable-scheduled.yml @@ -104,10 +104,10 @@ jobs: run: | case "${{ matrix.shard }}" in shard-1) - TEST_FILES="bsplines/tests/ console/tests/ fields_background/tests/ geometry/tests/ initial/tests/ kinetic_background/tests/ linear_algebra/tests/ ode/tests/ polar/tests/ post_processing/tests/" + TEST_FILES="bsplines/tests/ console/tests/ fields_background/tests/ geometry/tests/ initial/tests/ kinetic_background/tests/ linear_algebra/tests/" ;; shard-2) - TEST_FILES="propagators/tests/" + TEST_FILES="propagators/tests/ ode/tests/ polar/tests/ post_processing/tests/" ;; shard-3) TEST_FILES="feec/tests/" diff --git a/.github/workflows/reusable-unit-testing.yml b/.github/workflows/reusable-unit-testing.yml index 9a2215b7f..6fbf21f07 100644 --- a/.github/workflows/reusable-unit-testing.yml +++ b/.github/workflows/reusable-unit-testing.yml @@ -109,13 +109,13 @@ jobs: - name: Install Struphy in Container uses: ./.github/actions/install/struphy_in_container - # - name: Get submodule diff - # uses: ./.github/actions/submodule-diff - # with: - # start-dir: /struphy_fortran_ + - name: Get submodule diff + uses: ./.github/actions/submodule-diff + with: + start-dir: /struphy_fortran_ - name: Reinstall feectools from submodule - # if: env.SUBMOD_CHANGED == 'true' + if: env.SUBMOD_CHANGED == 'true' uses: ./.github/actions/install/feectools-submodule with: env-name: /struphy_fortran_/env_fortran_ @@ -142,7 +142,5 @@ jobs: echo "Running $UNINSTALL_MPI" $UNINSTALL_MPI git status || true - free -h - grep -E 'MemTotal|MemAvailable|SwapTotal|SwapFree' /proc/meminfo echo "Running $PYTEST_CMD $TEST_FILES" $PYTEST_CMD $TEST_FILES \ No newline at end of file From 376106e96cf2f33800cc76df16a93f024759abc7 Mon Sep 17 00:00:00 2001 From: Stefan Possanner Date: Tue, 12 May 2026 17:06:06 +0200 Subject: [PATCH 21/21] run memory heavy sph unit tests again --- src/struphy/pic/tests/test_sph.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/struphy/pic/tests/test_sph.py b/src/struphy/pic/tests/test_sph.py index 2d525f1e3..4a1ea8cbd 100644 --- a/src/struphy/pic/tests/test_sph.py +++ b/src/struphy/pic/tests/test_sph.py @@ -60,7 +60,7 @@ def test_sph_evaluation_1d( if derivative == 0: ppb = 1000 else: - return # skip non-tesselation test for derivative in 1D + # return # skip non-tesselation test for derivative in 1D ppb = 20000 loading_params = LoadingParameters(ppb=ppb, seed=223) @@ -544,7 +544,7 @@ def test_evaluation_SPH_h_convergence_1d(boxes_per_dim, bc_x, eval_pts, tesselat ppb = 160 loading_params = LoadingParameters(ppb=ppb, loading="tesselation") else: - return # skip non-tesselation test for h-convergence in 1D + # return # skip non-tesselation test for h-convergence in 1D Np = 160000 ppb = None loading_params = LoadingParameters(Np=Np, ppb=ppb, seed=1607) @@ -662,7 +662,7 @@ def test_evaluation_mc_Np_and_h_convergence_1d(boxes_per_dim, bc_x, eval_pts, te ppbs = [4, 8, 16, 32, 64] Nps = [None] * len(ppbs) else: - return # skip non-tesselation test for convergence in 1D + # return # skip non-tesselation test for convergence in 1D Nps = [(2**k) * 10**3 for k in range(-2, 9)] ppbs = [None] * len(Nps)