Skip to content

Commit 444db41

Browse files
JovonniJovonni
committed
Add Python SDK & model runner test framework (186 tests)
- 29 test files across 4 categories: SDK unit (119), framework integration (22), model runner (37), E2E integration (8) - Shared conftest.py with HTTP mocking, env fixtures, sample models, singleton reset - CI workflow with 4 parallel jobs: SDK unit (Python 3.9/3.11/3.12 matrix), framework, model runner, integration - Makefile targets: test-sdk, test-sdk-unit, test-sdk-frameworks, test-sdk-cov - pyproject.toml test extras
1 parent ff90ec9 commit 444db41

39 files changed

Lines changed: 3969 additions & 3 deletions

‎.github/workflows/test-python.yml‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
name: Python SDK & Model Runner Tests
2+
3+
on:
4+
push:
5+
branches: [main, "dev/**"]
6+
paths:
7+
- "sdk/python/**"
8+
- "model-runner/python/**"
9+
- "tests/python/**"
10+
pull_request:
11+
branches: [main]
12+
paths:
13+
- "sdk/python/**"
14+
- "model-runner/python/**"
15+
- "tests/python/**"
16+
workflow_dispatch:
17+
18+
concurrency:
19+
group: python-tests-${{ github.ref }}
20+
cancel-in-progress: true
21+
22+
jobs:
23+
sdk-unit-tests:
24+
name: SDK Unit Tests (Python ${{ matrix.python-version }})
25+
runs-on: ubuntu-latest
26+
strategy:
27+
matrix:
28+
python-version: ["3.9", "3.11", "3.12"]
29+
steps:
30+
- uses: actions/checkout@v4
31+
32+
- name: Setup Python ${{ matrix.python-version }}
33+
uses: actions/setup-python@v5
34+
with:
35+
python-version: ${{ matrix.python-version }}
36+
37+
- name: Cache pip
38+
uses: actions/cache@v4
39+
with:
40+
path: ~/.cache/pip
41+
key: sdk-test-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('tests/python/requirements-test.txt') }}
42+
restore-keys: sdk-test-${{ runner.os }}-py${{ matrix.python-version }}-
43+
44+
- name: Install dependencies
45+
run: |
46+
pip install --upgrade pip
47+
pip install pytest pytest-cov pytest-mock pytest-timeout responses
48+
pip install ./sdk/python
49+
50+
- name: Run SDK unit tests
51+
run: |
52+
cd tests/python
53+
python -m pytest sdk/ -v --tb=short --junitxml=../../test-results/sdk-unit.xml
54+
timeout-minutes: 5
55+
56+
- name: Upload test results
57+
if: always()
58+
uses: actions/upload-artifact@v4
59+
with:
60+
name: sdk-unit-results-py${{ matrix.python-version }}
61+
path: test-results/sdk-unit.xml
62+
63+
framework-tests:
64+
name: Framework Integration Tests
65+
runs-on: ubuntu-latest
66+
steps:
67+
- uses: actions/checkout@v4
68+
69+
- name: Setup Python 3.11
70+
uses: actions/setup-python@v5
71+
with:
72+
python-version: "3.11"
73+
74+
- name: Cache pip
75+
uses: actions/cache@v4
76+
with:
77+
path: ~/.cache/pip
78+
key: framework-test-${{ runner.os }}-${{ hashFiles('tests/python/requirements-test.txt') }}
79+
restore-keys: framework-test-${{ runner.os }}-
80+
81+
- name: Install dependencies
82+
run: |
83+
pip install --upgrade pip
84+
pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
85+
pip install -r tests/python/requirements-test.txt
86+
pip install ./sdk/python
87+
88+
- name: Run framework integration tests
89+
run: |
90+
cd tests/python
91+
python -m pytest frameworks/ -v --tb=short --junitxml=../../test-results/framework.xml
92+
timeout-minutes: 10
93+
94+
- name: Upload test results
95+
if: always()
96+
uses: actions/upload-artifact@v4
97+
with:
98+
name: framework-results
99+
path: test-results/framework.xml
100+
101+
model-runner-tests:
102+
name: Model Runner Tests
103+
runs-on: ubuntu-latest
104+
steps:
105+
- uses: actions/checkout@v4
106+
107+
- name: Setup Python 3.11
108+
uses: actions/setup-python@v5
109+
with:
110+
python-version: "3.11"
111+
112+
- name: Cache pip
113+
uses: actions/cache@v4
114+
with:
115+
path: ~/.cache/pip
116+
key: runner-test-${{ runner.os }}-${{ hashFiles('tests/python/requirements-test.txt') }}
117+
restore-keys: runner-test-${{ runner.os }}-
118+
119+
- name: Install dependencies
120+
run: |
121+
pip install --upgrade pip
122+
pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
123+
pip install -r tests/python/requirements-test.txt
124+
125+
- name: Run model runner tests
126+
run: |
127+
cd tests/python
128+
PYTHONPATH=${{ github.workspace }}/model-runner/python:$PYTHONPATH \
129+
python -m pytest model_runner/ -v --tb=short --junitxml=../../test-results/runner.xml
130+
timeout-minutes: 5
131+
132+
- name: Upload test results
133+
if: always()
134+
uses: actions/upload-artifact@v4
135+
with:
136+
name: runner-results
137+
path: test-results/runner.xml
138+
139+
integration-tests:
140+
name: Integration (E2E Flow) Tests
141+
runs-on: ubuntu-latest
142+
needs: [sdk-unit-tests, framework-tests]
143+
steps:
144+
- uses: actions/checkout@v4
145+
146+
- name: Setup Python 3.11
147+
uses: actions/setup-python@v5
148+
with:
149+
python-version: "3.11"
150+
151+
- name: Install dependencies
152+
run: |
153+
pip install --upgrade pip
154+
pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
155+
pip install -r tests/python/requirements-test.txt
156+
pip install ./sdk/python
157+
158+
- name: Run integration tests
159+
run: |
160+
cd tests/python
161+
python -m pytest integration/ -v --tb=short --junitxml=../../test-results/integration.xml
162+
timeout-minutes: 10
163+
164+
- name: Upload test results
165+
if: always()
166+
uses: actions/upload-artifact@v4
167+
with:
168+
name: integration-results
169+
path: test-results/integration.xml

‎Makefile‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
k8s-forward k8s-forward-db k8s-forward-api k8s-forward-gql k8s-forward-jupyter \
1010
build build-api build-frontend build-postgraphile build-model-runner-python build-workspace build-model-runner-rust build-all-images \
1111
dev dev-api dev-frontend dev-db dev-stop \
12-
test test-api test-frontend test-e2e test-pipelines test-all \
12+
test test-api test-frontend test-sdk test-sdk-cov test-sdk-frameworks test-sdk-unit test-e2e test-pipelines test-all \
1313
lint lint-api lint-frontend \
1414
db-init db-seed db-migrate db-reset reset-db \
1515
pipeline-run pipeline-test \
@@ -435,9 +435,9 @@ db-reset: db-init db-seed ## Reset local database (init + seed)
435435
# Testing
436436
# =====================================================================
437437

438-
test: test-api test-frontend ## Run API + frontend tests
438+
test: test-api test-frontend test-sdk ## Run API + frontend + SDK tests
439439

440-
test-all: test-api test-frontend test-e2e test-pipelines ## Run ALL tests (unit + e2e + pipelines)
440+
test-all: test-api test-frontend test-sdk test-e2e test-pipelines ## Run ALL tests (unit + e2e + pipelines)
441441

442442
test-api: ## Run Rust API tests
443443
@echo "$(CYAN)Running API tests...$(RESET)"
@@ -469,6 +469,27 @@ test-pipelines: ## Run pipeline tests
469469
fi; \
470470
done
471471

472+
PYTHON_TEST_DIR := $(TEST_DIR)/python
473+
474+
test-sdk: ## Run Python SDK & model runner tests
475+
@echo "$(CYAN)Running Python SDK & model runner tests...$(RESET)"
476+
cd $(PYTHON_TEST_DIR) && python3 -m pytest -v --tb=short -x
477+
478+
test-sdk-cov: ## Run Python tests with coverage report
479+
@echo "$(CYAN)Running Python tests with coverage...$(RESET)"
480+
cd $(PYTHON_TEST_DIR) && python3 -m pytest -v --tb=short \
481+
--cov=$(ROOT_DIR)/sdk/python/openmodelstudio \
482+
--cov=$(ROOT_DIR)/model-runner/python \
483+
--cov-report=term-missing
484+
485+
test-sdk-frameworks: ## Run only framework integration tests (sklearn, pytorch, tf)
486+
@echo "$(CYAN)Running framework integration tests...$(RESET)"
487+
cd $(PYTHON_TEST_DIR) && python3 -m pytest frameworks/ -v --tb=short
488+
489+
test-sdk-unit: ## Run only SDK unit tests (fast, no ML deps)
490+
@echo "$(CYAN)Running SDK unit tests...$(RESET)"
491+
cd $(PYTHON_TEST_DIR) && python3 -m pytest sdk/ -v --tb=short
492+
472493

473494
# =====================================================================
474495
# Linting

‎sdk/python/pyproject.toml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,18 @@ dependencies = [
3030
"requests>=2.28",
3131
]
3232

33+
[project.optional-dependencies]
34+
test = [
35+
"pytest>=7.4.0",
36+
"pytest-cov>=4.1.0",
37+
"pytest-mock>=3.11.0",
38+
"pytest-timeout>=2.2.0",
39+
"responses>=0.24.0",
40+
"scikit-learn>=1.3.0",
41+
"pandas>=2.0.0",
42+
"numpy>=1.24.0",
43+
]
44+
3345
[project.urls]
3446
Homepage = "https://github.com/GACWR/OpenModelStudio"
3547
Repository = "https://github.com/GACWR/OpenModelStudio"

0 commit comments

Comments
 (0)