Skip to content

Commit bb4d79b

Browse files
spossannspossann
authored
CI memory shortage (#243)
**Solves the following issue(s):** Closes #242 I guess this is because Github changed memory limits? Is there any source for this? The reason is most like the new garbage collector introduced in Python 3.14. It can cause problems in large pytest runs, since packages like numpy, pandas etc have not yet adopted it. Actually, the new GC has been rolled back in Python 3.14.5, see https://www.igorslab.de/en/python-3-14-5-released-maintenance-update-old-garbage-collector/ **Implemented solution:** Pin Python to 3.13 in the docker image `ubunte-latest-with-struphy`. The other image`ubuntu-with-reqs` still has the newest Python version and can be used for scheduled testing. **Other changes:** Implemented sharding of unit tests (4 shards) to have quicker concurrent workflows.
1 parent e0c1813 commit bb4d79b

8 files changed

Lines changed: 153 additions & 123 deletions

File tree

‎.github/actions/environment/unit-mpi/action.yml‎

Lines changed: 0 additions & 20 deletions
This file was deleted.

‎.github/actions/environment/unit-serial/action.yml‎

Lines changed: 0 additions & 16 deletions
This file was deleted.

‎.github/actions/tests/unit/action.yml‎

Lines changed: 0 additions & 45 deletions
This file was deleted.

‎.github/workflows/reusable-scheduled.yml‎

Lines changed: 59 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ on:
99

1010
jobs:
1111
test:
12+
name: ${{ matrix.test-type }} (${{ matrix.compile-language }}, ${{ matrix.shard }})
1213
runs-on: ${{ inputs.os }}
1314
env:
1415
OMPI_MCA_rmaps_base_oversubscribe: 1 # Linux
@@ -19,6 +20,20 @@ jobs:
1920
python-version: ["3.12"]
2021
compile-language: ["fortran", "c"]
2122
test-type: ["unit", "model", "verification"] #"quickstart", "tutorials"]
23+
shard: ["shard-1", "shard-2", "shard-3", "shard-4"]
24+
exclude:
25+
- test-type: model
26+
shard: shard-2
27+
- test-type: model
28+
shard: shard-3
29+
- test-type: model
30+
shard: shard-4
31+
- test-type: verification
32+
shard: shard-2
33+
- test-type: verification
34+
shard: shard-3
35+
- test-type: verification
36+
shard: shard-4
2237

2338
steps:
2439
# Checkout the repository
@@ -74,22 +89,60 @@ jobs:
7489
language: ${{ matrix.compile-language }}
7590

7691
- name: Set environment for mpi run
77-
uses: ./.github/actions/environment/unit-mpi
92+
run: |
93+
{
94+
echo 'TESTMON_KEY=testmon-unit-mpi'
95+
echo 'RUN_MAXWELL=mpirun -n 1 pytest -m single --testmon-forceselect -v --with-mpi --model-name Maxwell'
96+
echo 'UNINSTALL_MPI='
97+
echo 'PYTEST_CMD=mpirun -n 2 pytest -v --testmon --with-mpi'
98+
} >> "$GITHUB_ENV"
7899
79100
- name: Run unit tests with MPI
80101
if: matrix.test-type == 'unit'
81-
uses: ./.github/actions/tests/unit
82-
with:
83-
struphy-path: ${{ env.STRUPHY_PATH }}
102+
env:
103+
TESTMON_DATAFILE: ${{ github.workspace }}/.testmondata-unit-${{ matrix.shard }}
104+
run: |
105+
case "${{ matrix.shard }}" in
106+
shard-1)
107+
TEST_FILES="bsplines/tests/ console/tests/ fields_background/tests/ geometry/tests/ initial/tests/ kinetic_background/tests/ linear_algebra/tests/"
108+
;;
109+
shard-2)
110+
TEST_FILES="propagators/tests/ ode/tests/ polar/tests/ post_processing/tests/"
111+
;;
112+
shard-3)
113+
TEST_FILES="feec/tests/"
114+
;;
115+
shard-4)
116+
TEST_FILES="pic/tests/"
117+
;;
118+
*)
119+
echo "Unknown test shard: ${{ matrix.shard }}" >&2
120+
exit 1
121+
;;
122+
esac
123+
set +e
124+
source /bin/activate || true
125+
set -e
126+
pip show struphy
127+
struphy compile --status
128+
cd ${{ env.STRUPHY_PATH }}
129+
pwd
130+
echo "Running $RUN_MAXWELL"
131+
$RUN_MAXWELL
132+
echo "Running $UNINSTALL_MPI"
133+
$UNINSTALL_MPI
134+
git status || true
135+
echo "Running $PYTEST_CMD $TEST_FILES"
136+
$PYTEST_CMD $TEST_FILES
84137
85138
- name: Run model tests with MPI
86-
if: matrix.test-type == 'model'
139+
if: matrix.test-type == 'model' && matrix.shard == 'shard-1'
87140
shell: bash
88141
run: |
89142
mpirun -oversubscribe -n 2 pytest -x -m models --with-mpi $STRUPHY_PATH/models/tests/default_params/
90143
91144
- name: Run verification tests with MPI
92-
if: matrix.test-type == 'verification'
145+
if: matrix.test-type == 'verification' && matrix.shard == 'shard-1'
93146
shell: bash
94147
run: |
95148
mpirun --oversubscribe -n 2 pytest -x --with-mpi $STRUPHY_PATH/models/tests/verification/

‎.github/workflows/reusable-unit-testing.yml‎

Lines changed: 63 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,29 @@ permissions:
2121
contents: read
2222

2323
jobs:
24+
preflight:
25+
runs-on: ${{ inputs.os }}
26+
steps:
27+
- name: Check memory
28+
run: |
29+
free -h
30+
grep -E 'MemTotal|MemAvailable|SwapTotal|SwapFree' /proc/meminfo
31+
2432
unit-tests-in-container-with-struphy:
33+
name: Unit tests (${{ matrix.shard }})
2534
runs-on: ${{ inputs.os }}
35+
strategy:
36+
fail-fast: false
37+
matrix:
38+
include:
39+
- shard: shard-1
40+
test_files: bsplines/tests/ console/tests/ fields_background/tests/ geometry/tests/ initial/tests/ kinetic_background/tests/ linear_algebra/tests/
41+
- shard: shard-2
42+
test_files: propagators/tests/ ode/tests/ polar/tests/ post_processing/tests/
43+
- shard: shard-3
44+
test_files: feec/tests/
45+
- shard: shard-4
46+
test_files: pic/tests/
2647

2748
container:
2849
image: ghcr.io/struphy-hub/struphy/ubuntu-with-struphy:latest
@@ -34,6 +55,11 @@ jobs:
3455
- name: Check for dockerenv file
3556
run: (ls /.dockerenv && echo Found dockerenv) || (echo No dockerenv)
3657

58+
- name: Memory inside container
59+
run: |
60+
free -h
61+
grep -E 'MemTotal|MemAvailable|SwapTotal|SwapFree' /proc/meminfo
62+
3763
- name: Checkout repo
3864
uses: actions/checkout@v4
3965
with:
@@ -42,11 +68,23 @@ jobs:
4268

4369
- name: Set environment for serial run
4470
if: inputs.n-procs == 1
45-
uses: ./.github/actions/environment/unit-serial
71+
run: |
72+
{
73+
echo 'TESTMON_KEY=testmon-unit'
74+
echo 'RUN_MAXWELL='
75+
echo 'UNINSTALL_MPI=pip uninstall -y mpi4py'
76+
echo 'PYTEST_CMD=pytest -v --testmon'
77+
} >> "$GITHUB_ENV"
4678
4779
- name: Set environment for mpi run
4880
if: inputs.n-procs > 1
49-
uses: ./.github/actions/environment/unit-mpi
81+
run: |
82+
{
83+
echo 'TESTMON_KEY=testmon-unit-mpi'
84+
echo 'RUN_MAXWELL=mpirun -n 1 pytest -m single --testmon-forceselect -v --with-mpi --model-name Maxwell'
85+
echo 'UNINSTALL_MPI='
86+
echo 'PYTEST_CMD=mpirun -n ${{ inputs.n-procs }} pytest -v --testmon --with-mpi'
87+
} >> "$GITHUB_ENV"
5088
5189
- name: Check .testmondata 1
5290
run: |
@@ -56,12 +94,12 @@ jobs:
5694
uses: actions/cache@v4
5795
with:
5896
path: |
59-
.testmondata-unit
60-
key: ${{ env.TESTMON_KEY }}-${{ github.ref_name }}-${{ github.event.number }}-${{ github.run_number }}
97+
.testmondata-unit-${{ matrix.shard }}
98+
key: ${{ env.TESTMON_KEY }}-${{ matrix.shard }}-${{ github.ref_name }}-${{ github.event.number }}-${{ github.run_number }}
6199
restore-keys: |
62-
${{ env.TESTMON_KEY }}-${{ github.ref_name }}-${{ github.event.number }}-
63-
${{ env.TESTMON_KEY }}-${{ github.ref_name }}
64-
${{ env.TESTMON_KEY }}-devel
100+
${{ env.TESTMON_KEY }}-${{ matrix.shard }}-${{ github.ref_name }}-${{ github.event.number }}-
101+
${{ env.TESTMON_KEY }}-${{ matrix.shard }}-${{ github.ref_name }}
102+
${{ env.TESTMON_KEY }}-${{ matrix.shard }}-devel
65103
${{ env.TESTMON_KEY }}-
66104
67105
- name: Check .testmondata 2
@@ -88,7 +126,21 @@ jobs:
88126
env-name: /struphy_fortran_/env_fortran_
89127

90128
- name: Run unit tests
91-
uses: ./.github/actions/tests/unit
92-
with:
93-
struphy-path: ${{ env.STRUPHY_PATH }}
94-
env-name: /struphy_fortran_/env_fortran_
129+
env:
130+
TESTMON_DATAFILE: ${{ github.workspace }}/.testmondata-unit-${{ matrix.shard }}
131+
TEST_FILES: ${{ matrix.test_files }}
132+
run: |
133+
set +e
134+
source /struphy_fortran_/env_fortran_/bin/activate || true
135+
set -e
136+
pip show struphy
137+
struphy compile --status
138+
cd ${{ env.STRUPHY_PATH }}
139+
pwd
140+
echo "Running $RUN_MAXWELL"
141+
$RUN_MAXWELL
142+
echo "Running $UNINSTALL_MPI"
143+
$UNINSTALL_MPI
144+
git status || true
145+
echo "Running $PYTEST_CMD $TEST_FILES"
146+
$PYTEST_CMD $TEST_FILES

‎docker/ubuntu-latest-with-struphy.dockerfile‎

Lines changed: 23 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,19 @@
1212
FROM ubuntu:latest
1313

1414
ARG DEBIAN_FRONTEND=noninteractive
15+
ARG PYTHON_VERSION=3.13
1516

1617
# install linux packages
1718
RUN apt update -y && apt clean \
1819
&& apt install -y software-properties-common \
1920
&& add-apt-repository -y ppa:deadsnakes/ppa \
2021
&& apt update -y
2122

22-
RUN apt install -y python3 \
23-
&& apt install -y python3-dev \
24-
&& apt install -y python3-pip \
25-
&& apt install -y python3-venv
23+
RUN apt install -y python${PYTHON_VERSION} \
24+
&& apt install -y python${PYTHON_VERSION}-dev \
25+
&& apt install -y python${PYTHON_VERSION}-venv \
26+
&& update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
27+
&& update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION}
2628

2729
RUN apt install -y gfortran gcc \
2830
&& apt install -y liblapack-dev libblas-dev
@@ -44,32 +46,32 @@ RUN apt install -y g++ liblapack3 cmake cmake-curses-gui zlib1g-dev libnetcdf-de
4446
&& export CXX=`which g++`
4547

4648
# install three versions of struphy
47-
RUN git clone https://github.com/struphy-hub/struphy.git struphy_c_ \
48-
&& cd struphy_c_ \
49-
&& python3 -m venv env_c_ \
50-
&& . env_c_/bin/activate \
51-
&& pip install -U pip \
52-
&& pip install -e .[phys,mpi,doc] --no-cache-dir \
53-
&& struphy compile \
54-
&& deactivate
49+
# RUN git clone https://github.com/struphy-hub/struphy.git struphy_c_ \
50+
# && cd struphy_c_ \
51+
# && python${PYTHON_VERSION} -m venv env_c_ \
52+
# && . env_c_/bin/activate \
53+
# && pip install -U pip \
54+
# && pip install -e .[phys,mpi,doc] --no-cache-dir \
55+
# && struphy compile \
56+
# && deactivate
5557

5658
RUN git clone https://github.com/struphy-hub/struphy.git struphy_fortran_\
5759
&& cd struphy_fortran_ \
58-
&& python3 -m venv env_fortran_ \
60+
&& python${PYTHON_VERSION} -m venv env_fortran_ \
5961
&& . env_fortran_/bin/activate \
6062
&& pip install -U pip \
6163
&& pip install -e .[phys,mpi,doc] --no-cache-dir \
6264
&& struphy compile --language fortran -y \
6365
&& deactivate
6466

65-
RUN git clone https://github.com/struphy-hub/struphy.git struphy_fortran_--omp-pic\
66-
&& cd struphy_fortran_--omp-pic \
67-
&& python3 -m venv env_fortran_--omp-pic \
68-
&& . env_fortran_--omp-pic/bin/activate \
69-
&& pip install -U pip \
70-
&& pip install -e .[phys,mpi,doc] --no-cache-dir \
71-
&& struphy compile --language fortran --omp-pic -y \
72-
&& deactivate
67+
# RUN git clone https://github.com/struphy-hub/struphy.git struphy_fortran_--omp-pic\
68+
# && cd struphy_fortran_--omp-pic \
69+
# && python${PYTHON_VERSION} -m venv env_fortran_--omp-pic \
70+
# && . env_fortran_--omp-pic/bin/activate \
71+
# && pip install -U pip \
72+
# && pip install -e .[phys,mpi,doc] --no-cache-dir \
73+
# && struphy compile --language fortran --omp-pic -y \
74+
# && deactivate
7375

7476
# allow mpirun as root
7577
ENV OMPI_ALLOW_RUN_AS_ROOT=1

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ dependencies = [
3737
"ipywidgets<=8.1.8",
3838
"plotly<=6.7.0",
3939
"pyvista<=0.48.0",
40-
"trame<=3.12.0",
40+
"trame<=3.13.0",
4141
"trame-vtk<=2.11.8",
4242
"trame-vuetify<=3.2.2",
4343
"nest_asyncio2<=1.7.2",

0 commit comments

Comments
 (0)