Skip to content

Commit a7c1d9a

Browse files
harryswift01harryswift01
authored
Merge pull request #305 from CCPBioSim/304-feature-introduce-scenario-based-regression-testing
Introduce scenario-based regression testing framework with expanded configuration coverage
2 parents 5e46977 + 54884c6 commit a7c1d9a

147 files changed

Lines changed: 4384 additions & 536 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/daily.yaml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,16 @@ concurrency:
1111

1212
jobs:
1313
unit:
14-
name: Unit (${{ matrix.os }}, ${{ matrix.python-version }})
14+
name: Unit (${{ matrix.os }}, py${{ matrix.python-version }})
1515
runs-on: ${{ matrix.os }}
1616
timeout-minutes: 30
17+
1718
strategy:
1819
fail-fast: false
1920
matrix:
2021
os: [ubuntu-24.04, macos-15, windows-2025]
2122
python-version: ["3.12", "3.13", "3.14"]
23+
2224
steps:
2325
- name: Checkout
2426
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
@@ -29,10 +31,10 @@ jobs:
2931
python-version: ${{ matrix.python-version }}
3032
cache: pip
3133

32-
- name: Install (testing)
34+
- name: Install testing dependencies
3335
run: |
3436
python -m pip install --upgrade pip
3537
python -m pip install -e .[testing]
3638
37-
- name: Pytest (unit) • ${{ matrix.os }} • py${{ matrix.python-version }}
39+
- name: Run unit tests
3840
run: python -m pytest tests/unit

‎.github/workflows/pr.yaml‎

Lines changed: 63 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,13 @@ jobs:
1717
matrix:
1818
os: [ubuntu-24.04, macos-15, windows-2025]
1919
python-version: ["3.12", "3.13", "3.14"]
20+
2021
steps:
2122
- name: Checkout
22-
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
23+
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
2324

2425
- name: Set up Python ${{ matrix.python-version }}
25-
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
26+
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
2627
with:
2728
python-version: ${{ matrix.python-version }}
2829
cache: pip
@@ -35,17 +36,50 @@ jobs:
3536
- name: Pytest (unit) • ${{ matrix.os }}, ${{ matrix.python-version }}
3637
run: python -m pytest tests/unit
3738

39+
discover-systems:
40+
name: Discover regression systems
41+
runs-on: ubuntu-24.04
42+
outputs:
43+
systems: ${{ steps.set-systems.outputs.systems }}
44+
45+
steps:
46+
- name: Checkout
47+
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
48+
49+
- name: Set up Python 3.14
50+
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
51+
with:
52+
python-version: "3.14"
53+
cache: pip
54+
55+
- name: Install testing dependencies
56+
run: |
57+
python -m pip install --upgrade pip
58+
python -m pip install -e .[testing]
59+
60+
- name: Discover systems
61+
id: set-systems
62+
run: |
63+
SYSTEMS=$(python -m tests.regression.list_systems)
64+
echo "systems=$SYSTEMS" >> $GITHUB_OUTPUT
65+
3866
regression-quick:
39-
name: Regression (quick)
40-
needs: unit
67+
name: Regression (fast) • ${{ matrix.system }}
68+
needs: [unit, discover-systems]
4169
runs-on: ubuntu-24.04
4270
timeout-minutes: 35
71+
72+
strategy:
73+
fail-fast: false
74+
matrix:
75+
system: ${{ fromJson(needs.discover-systems.outputs.systems) }}
76+
4377
steps:
4478
- name: Checkout
45-
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
79+
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
4680

4781
- name: Set up Python 3.14
48-
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
82+
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
4983
with:
5084
python-version: "3.14"
5185
cache: pip
@@ -54,28 +88,31 @@ jobs:
5488
uses: actions/cache@v4
5589
with:
5690
path: .testdata
57-
key: codeentropy-testdata-v1-${{ runner.os }}-py3.14
91+
key: codeentropy-testdata-${{ runner.os }}-py314
5892

59-
- name: Install (testing)
93+
- name: Install testing dependencies
6094
run: |
6195
python -m pip install --upgrade pip
6296
python -m pip install -e .[testing]
6397
64-
- name: Pytest (regression quick)
65-
run: python -m pytest tests/regression
98+
- name: Run fast regression tests (per system)
99+
run: |
100+
python -m pytest tests/regression \
101+
-m "not slow" \
102+
-n auto \
103+
--dist=loadscope \
104+
-k "${{ matrix.system }}" \
105+
-vv \
106+
--durations=20
66107
67108
- name: Upload artifacts (failure)
68109
if: failure()
69110
uses: actions/upload-artifact@v4
70111
with:
71-
name: quick-regression-failure
112+
name: quick-regression-failure-${{ matrix.system }}
72113
path: |
73114
.testdata/**
74-
tests/regression/**/.pytest_cache/**
75-
/tmp/pytest-of-*/pytest-*/**/config.yaml
76-
/tmp/pytest-of-*/pytest-*/**/codeentropy_stdout.txt
77-
/tmp/pytest-of-*/pytest-*/**/codeentropy_stderr.txt
78-
/tmp/pytest-of-*/pytest-*/**/codeentropy_output.json
115+
/tmp/pytest-of-*/pytest-*/**
79116
80117
docs:
81118
name: Docs
@@ -93,7 +130,7 @@ jobs:
93130
python-version: "3.14"
94131
cache: pip
95132

96-
- name: Install
133+
- name: Install docs dependencies
97134
run: |
98135
python -m pip install --upgrade pip
99136
python -m pip install -e .[docs]
@@ -114,23 +151,23 @@ jobs:
114151
name: Pre-commit
115152
runs-on: ubuntu-24.04
116153
timeout-minutes: 15
154+
117155
steps:
118156
- name: Checkout
119-
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
157+
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
120158

121159
- name: Set up Python 3.14
122-
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
160+
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
123161
with:
124162
python-version: "3.14"
125163
cache: pip
126164

127-
- name: Install (pre-commit)
165+
- name: Install pre-commit dependencies
128166
run: |
129167
python -m pip install --upgrade pip
130168
python -m pip install -e .[pre-commit]
131169
132170
- name: Run pre-commit
133-
shell: bash
134171
run: |
135172
pre-commit install
136173
pre-commit run --all-files || {
@@ -144,8 +181,9 @@ jobs:
144181
needs: unit
145182
runs-on: ubuntu-24.04
146183
timeout-minutes: 30
184+
147185
steps:
148-
- name: Checkout repo
186+
- name: Checkout
149187
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
150188

151189
- name: Set up Python 3.14
@@ -154,20 +192,20 @@ jobs:
154192
python-version: "3.14"
155193
cache: pip
156194

157-
- name: Install (testing)
195+
- name: Install testing dependencies
158196
run: |
159197
python -m pip install --upgrade pip
160198
python -m pip install -e .[testing]
161199
162-
- name: Run unit test suite with coverage
200+
- name: Run coverage
163201
run: |
164202
pytest tests/unit \
165203
--cov CodeEntropy \
166204
--cov-report term-missing \
167205
--cov-report xml \
168206
-q
169207
170-
- name: Upload to Coveralls
208+
- name: Upload coverage to Coveralls
171209
uses: coverallsapp/github-action@5cbfd81b66ca5d10c19b062c04de0199c215fb6e
172210
with:
173211
github-token: ${{ secrets.GITHUB_TOKEN }}

‎.github/workflows/weekly-regression.yaml‎

Lines changed: 57 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,50 +2,89 @@ name: Weekly Regression Tests
22

33
on:
44
schedule:
5-
- cron: '0 8 * * 1' # Weekly Monday checks
5+
- cron: '0 8 * * 1'
66
workflow_dispatch:
77

88
concurrency:
99
group: weekly-regression-${{ github.ref }}
1010
cancel-in-progress: true
1111

1212
jobs:
13+
discover:
14+
name: Discover regression systems
15+
runs-on: ubuntu-24.04
16+
17+
outputs:
18+
systems: ${{ steps.set.outputs.systems }}
19+
20+
steps:
21+
- name: Checkout
22+
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
23+
24+
- name: Set up Python 3.14
25+
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
26+
with:
27+
python-version: "3.14"
28+
cache: pip
29+
30+
- name: Install testing dependencies
31+
run: |
32+
python -m pip install --upgrade pip
33+
python -m pip install -e .[testing]
34+
35+
- name: Generate system matrix
36+
id: set
37+
run: |
38+
systems=$(python -m tests.regression.list_systems)
39+
echo "systems=$systems" >> $GITHUB_OUTPUT
40+
1341
regression:
14-
name: Regression tests (including slow)
42+
name: Regression (${{ matrix.system }})
43+
needs: discover
1544
runs-on: ubuntu-24.04
45+
timeout-minutes: 360
46+
47+
strategy:
48+
fail-fast: false
49+
matrix:
50+
system: ${{ fromJson(needs.discover.outputs.systems) }}
51+
1652
steps:
17-
- name: Checkout repo
18-
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
53+
- name: Checkout
54+
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
1955

2056
- name: Set up Python 3.14
21-
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
57+
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
2258
with:
2359
python-version: "3.14"
2460
cache: pip
2561

26-
- name: Cache regression test data downloads
62+
- name: Cache regression test data
2763
uses: actions/cache@v4
2864
with:
2965
path: .testdata
3066
key: codeentropy-testdata-${{ runner.os }}-py314
3167

32-
- name: Install CodeEntropy and its testing dependencies
68+
- name: Install testing dependencies
3369
run: |
34-
pip install --upgrade pip
35-
pip install -e .[testing]
70+
python -m pip install --upgrade pip
71+
python -m pip install -e .[testing]
3672
37-
- name: Run regression test suite
38-
run: pytest tests/regression --run-slow
73+
- name: Run regression tests (slow, per system)
74+
run: |
75+
pytest tests/regression \
76+
-k "${{ matrix.system }}" \
77+
--run-slow \
78+
-n auto \
79+
--dist=loadscope \
80+
-vv \
81+
--durations=20
3982
40-
- name: Upload regression artifacts on failure
83+
- name: Upload regression artifacts (failure)
4184
if: failure()
4285
uses: actions/upload-artifact@v4
4386
with:
44-
name: regression-failure-artifacts
87+
name: regression-${{ matrix.system }}-artifacts
4588
path: |
4689
.testdata/**
47-
tests/regression/**/.pytest_cache/**
48-
/tmp/pytest-of-*/pytest-*/**/config.yaml
49-
/tmp/pytest-of-*/pytest-*/**/codeentropy_stdout.txt
50-
/tmp/pytest-of-*/pytest-*/**/codeentropy_stderr.txt
51-
/tmp/pytest-of-*/pytest-*/**/codeentropy_output.json
90+
/tmp/pytest-of-*/pytest-*/**

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,4 +129,4 @@ job*
129129
.testdata/
130130

131131
!tests/regression/baselines/
132-
!tests/regression/baselines/*.json
132+
!tests/regression/baselines/*/*.json

‎conda-recipe/meta.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ requirements:
3838
- dask >=2026.1.2,<2026.2.0
3939
- distributed >=2026.1.2,<2026.2.0
4040
- dask-jobqueue >=0.9,<0.10
41+
- pytest-xdist >=3.8, <3.9
4142

4243
test:
4344
imports:

‎docs/developer_guide.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ Run regression tests excluding slow systems::
5454

5555
Run slow regression tests::
5656

57-
pytest tests/regression -m slow
57+
pytest tests/regression --run-slow
5858

5959
Run tests with coverage::
6060

‎pyproject.toml‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,8 @@ Documentation = "https://codeentropy.readthedocs.io"
6161
testing = [
6262
"pytest>=9.0,<10.0",
6363
"pytest-cov>=7.0,<8.0",
64-
"pytest-sugar>=1.1,<2.0"
64+
"pytest-sugar>=1.1,<2.0",
65+
"pytest-xdist"
6566
]
6667
pre-commit = [
6768
"pre-commit>=4.5,<5.0",
@@ -83,6 +84,16 @@ docs = [
8384
[project.scripts]
8485
CodeEntropy = "CodeEntropy.cli:main"
8586

87+
[tool.pytest.ini_options]
88+
testpaths = ["tests"]
89+
90+
addopts = "-n auto -vv -ra"
91+
92+
markers = [
93+
"regression: end-to-end regression tests against baselines",
94+
"slow: long-running regression tests (20-30+ minutes)",
95+
]
96+
8697
[tool.ruff]
8798
line-length = 88
8899
target-version = "py311"

‎tests/__init__.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1 @@
1-
"""
2-
Empty init file in case you choose a package besides PyTest such as Nose which may look
3-
for such a file.
4-
"""
1+
""" """

‎tests/pytest.ini‎

Lines changed: 0 additions & 10 deletions
This file was deleted.

0 commit comments

Comments
 (0)