Skip to content

Commit 9836fab

Browse files
jonnorjonnor
authored
Merge pull request #55 from emlearn/tests-all-runner
tests: Improve running
2 parents 80fa2e6 + 5d91d37 commit 9836fab

13 files changed

Lines changed: 196 additions & 91 deletions

‎Makefile‎

Lines changed: 37 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -28,62 +28,40 @@ PORT_DIST_DIR=./dist/ports/$(PORT)/$(BOARD)
2828

2929
UNIX_MICROPYTHON = ./dist/ports/unix/micropython
3030

31-
$(MODULES_PATH)/emlearn_trees.mpy:
32-
make -C src/emlearn_trees/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} V=1 clean dist
3331

34-
$(MODULES_PATH)/emlearn_neighbors.mpy:
35-
make -C src/emlearn_neighbors/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} V=1 clean dist
36-
37-
$(MODULES_PATH)/emlearn_iir.mpy:
38-
make -C src/emlearn_iir/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} V=1 clean dist
39-
40-
$(MODULES_PATH)/emlearn_fft.mpy:
41-
make -C src/emlearn_fft/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} V=1 clean dist
42-
43-
$(MODULES_PATH)/emlearn_cnn_int8.mpy:
44-
make -C src/tinymaix_cnn/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} V=1 CONFIG=int8 clean dist
45-
46-
$(MODULES_PATH)/emlearn_cnn_fp32.mpy:
47-
make -C src/tinymaix_cnn/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} V=1 CONFIG=fp32 clean dist
48-
49-
$(MODULES_PATH)/emlearn_kmeans.mpy:
50-
make -C src/emlearn_kmeans/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} V=1 clean dist
51-
52-
$(MODULES_PATH)/emlearn_iir_q15.mpy:
53-
make -C src/emlearn_iir_q15/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} V=1 clean dist
54-
55-
$(MODULES_PATH)/emlearn_arrayutils.mpy:
56-
make -C src/emlearn_arrayutils/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} V=1 clean dist
57-
58-
$(MODULES_PATH)/emlearn_linreg.mpy:
59-
make -C src/emlearn_linreg/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} V=1 clean dist
60-
61-
emlearn_trees.results: $(MODULES_PATH)/emlearn_trees.mpy
62-
MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_trees.py
63-
64-
emlearn_neighbors.results: $(MODULES_PATH)/emlearn_neighbors.mpy
65-
MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_neighbors.py
66-
67-
emlearn_iir.results: $(MODULES_PATH)/emlearn_iir.mpy
68-
MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_iir.py
69-
70-
emlearn_fft.results: $(MODULES_PATH)/emlearn_fft.mpy
71-
MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_fft.py
72-
73-
emlearn_cnn.results: $(MODULES_PATH)/emlearn_cnn_int8.mpy $(MODULES_PATH)/emlearn_cnn_fp32.mpy
74-
MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_cnn.py
75-
76-
emlearn_kmeans.results: $(MODULES_PATH)/emlearn_kmeans.mpy
77-
MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_kmeans.py
78-
79-
emlearn_iir_q15.results: $(MODULES_PATH)/emlearn_iir_q15.mpy
80-
MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_iir_q15.py
81-
82-
emlearn_arrayutils.results: $(MODULES_PATH)/emlearn_arrayutils.mpy
83-
MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_arrayutils.py
84-
85-
emlearn_linreg.results: $(MODULES_PATH)/emlearn_linreg.mpy
86-
MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_linreg.py
32+
# List of modules
33+
MODULES = emlearn_trees \
34+
emlearn_neighbors \
35+
emlearn_iir \
36+
emlearn_fft \
37+
emlearn_kmeans \
38+
emlearn_iir_q15 \
39+
emlearn_arrayutils \
40+
emlearn_linreg \
41+
emlearn_cnn_int8 \
42+
emlearn_cnn_fp32
43+
44+
# Generate list of .mpy files
45+
MODULE_MPYS = $(addprefix $(MODULES_PATH)/,$(addsuffix .mpy,$(MODULES)))
46+
47+
# Special cases
48+
emlearn_cnn_int8_SRC = src/tinymaix_cnn
49+
emlearn_cnn_int8_CONFIG = CONFIG=int8
50+
emlearn_cnn_fp32_SRC = src/tinymaix_cnn
51+
emlearn_cnn_fp32_CONFIG = CONFIG=fp32
52+
53+
# Generate list of .mpy files
54+
MODULE_MPYS = $(addprefix $(MODULES_PATH)/,$(addsuffix .mpy,$(MODULES)))
55+
56+
# Build dynamic native module
57+
# defaults to
58+
$(MODULES_PATH)/%.mpy:
59+
make -C $(or $($(*)_SRC),src/$*) \
60+
ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} \
61+
V=1 $($(*)_CONFIG) clean dist
62+
63+
check_unix_natmod: $(MODULE_MPYS)
64+
MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_all.py
8765

8866
$(PORT_DIR):
8967
mkdir -p $@
@@ -95,11 +73,8 @@ $(UNIX_MICROPYTHON): $(PORT_DIR)
9573
unix: $(UNIX_MICROPYTHON)
9674

9775
check_unix: $(UNIX_MICROPYTHON)
98-
$(UNIX_MICROPYTHON) tests/test_trees.py
99-
$(UNIX_MICROPYTHON) tests/test_iir.py
100-
$(UNIX_MICROPYTHON) tests/test_fft.py
101-
$(UNIX_MICROPYTHON) tests/test_arrayutils.py
102-
echo SKIP $(UNIX_MICROPYTHON) tests/test_cnn.py
76+
$(UNIX_MICROPYTHON) tests/test_all.py test_iir,test_fft,test_arrayutils
77+
# TODO: enable more modules
10378

10479
rp2: $(PORT_DIR)
10580
make -C $(MPY_DIR)/ports/rp2 V=1 USER_C_MODULES=$(C_MODULES_SRC_PATH)/micropython.cmake FROZEN_MANIFEST=$(MANIFEST_PATH) CFLAGS_EXTRA='-Wno-unused-function -Wno-unused-function' -j4
@@ -130,8 +105,8 @@ release:
130105
zip -r $(RELEASE_NAME).zip $(RELEASE_NAME)
131106
#cp $(RELEASE_NAME).zip emlearn-micropython-latest.zip
132107

133-
check: emlearn_trees.results emlearn_neighbors.results emlearn_iir.results emlearn_iir_q15.results emlearn_fft.results emlearn_kmeans.results emlearn_arrayutils.results emlearn_cnn.results emlearn_linreg.results
108+
check: check_unix_natmod
134109

135-
dist: $(MODULES_PATH)/emlearn_trees.mpy $(MODULES_PATH)/emlearn_neighbors.mpy $(MODULES_PATH)/emlearn_iir.mpy $(MODULES_PATH)/emlearn_iir_q15.mpy $(MODULES_PATH)/emlearn_fft.mpy $(MODULES_PATH)/emlearn_kmeans.mpy $(MODULES_PATH)/emlearn_arrayutils.mpy $(MODULES_PATH)/emlearn_cnn_int8.mpy $(MODULES_PATH)/emlearn_cnn_fp32.mpy $(MODULES_PATH)/emlearn_linreg.mpy
110+
dist: $(MODULE_MPYS)
136111

137112

‎docs/developing.md‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,15 @@ NOTE: Tested on Linux and Mac OS. Not tested on Windows Subsystem for Linux (WSL
6161
make check_unix
6262
```
6363

64+
You should see each of the test functions in tests/ being ran,
65+
and then a summary at the end with something like:
66+
67+
```
68+
...
69+
Passed: 17
70+
Failed: 0
71+
```
72+
6473
#### Run tests on PC using dynamic native modules
6574

6675
This runs tests by building as dynamic native modules (.mpy files),
@@ -75,6 +84,14 @@ To build and run tests of dynamic native modules on host
7584
make check
7685
```
7786

87+
You should see each of the test functions in tests/ being ran,
88+
and then a summary at the end with something like:
89+
90+
```
91+
...
92+
Passed: 17
93+
Failed: 0
94+
```
7895

7996
#### Build for device
8097

@@ -91,6 +108,24 @@ Install it on device
91108
mpremote cp dist/armv6m*/emlearn_trees.mpy :emlearn_trees.mpy
92109
```
93110

111+
#### Running tests on device
112+
113+
NOTE: Assumes that the .mpy files have been built first (in dist/).
114+
115+
We use `mpremote mount`, which allows the device to access files on the PC/host filesystem. This means we do not have to copy the modules or files across.
116+
117+
```
118+
mpremote mount . run tests/test_all.py
119+
```
120+
121+
You should see each of the test functions in tests/ being ran,
122+
and then a summary at the end with something like:
123+
124+
```
125+
...
126+
Passed: 17
127+
Failed: 0
128+
```
94129

95130
## Building documentation
96131

‎requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ scikit-learn>=1.0.0
33
ar>=1.0.0
44
pyelftools>=0.31
55
setuptools>=71.0.0
6+
mpremote>=1.25.0

‎tests/test_all.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
2+
import sys
3+
4+
# Find the module path (architecture+version specific)
5+
sys_mpy = sys.implementation._mpy
6+
mpy_arch = [None, 'x86', 'x64',
7+
'armv6', 'armv6m', 'armv7m', 'armv7em', 'armv7emsp', 'armv7emdp',
8+
'xtensa', 'xtensawin', 'rv32imc'][sys_mpy >> 10]
9+
mpy_major = sys_mpy & 0xff
10+
mpy_minor = sys_mpy >> 8 & 3
11+
12+
module_dir = f'{mpy_arch}_{mpy_major}.{mpy_minor}'
13+
14+
# make sure we can import .mpy modules
15+
sys.path.insert(0, './dist/'+module_dir)
16+
17+
# make sure we can import test files
18+
sys.path.insert(0, './tests')
19+
20+
21+
TEST_MODULES=[
22+
'test_arrayutils',
23+
'test_cnn',
24+
'test_fft',
25+
'test_iir',
26+
'test_iir_q15',
27+
'test_kmeans',
28+
'test_linreg',
29+
#'test_linreg_california',
30+
'test_neighbors',
31+
'test_trees',
32+
]
33+
34+
def main():
35+
36+
# Find which tests are enabled
37+
# Default: all
38+
39+
modules = TEST_MODULES
40+
if len(sys.argv) >= 2:
41+
modules = sys.argv[1].split(',')
42+
43+
passed = 0
44+
failed = 0
45+
46+
for module_name in modules:
47+
mod = None
48+
print(f'{module_name}:')
49+
try:
50+
mod = __import__(module_name)
51+
except Exception as e:
52+
print(f'Error while importing {module_name}:')
53+
sys.print_exception(e)
54+
print() # spacing for readability
55+
failed += 1
56+
continue
57+
58+
module_attributes = dir(mod)
59+
tests = [ o for o in module_attributes if o.startswith('test_') ]
60+
for test_name in tests:
61+
test_function = getattr(mod, test_name)
62+
print(f'{module_name}.py/{test_name}:')
63+
try:
64+
test_function()
65+
except Exception as e:
66+
print(f'\tFAIL')
67+
sys.print_exception(e)
68+
print() # spacing for readability
69+
failed += 1
70+
continue
71+
72+
print(f'\t PASS')
73+
passed += 1
74+
75+
print(f'Passed: {passed}')
76+
print(f'Failed: {failed}')
77+
78+
# Let status code reflect number of failures
79+
return failed
80+
81+
if __name__ == '__main__':
82+
sys.exit(main())

‎tests/test_arrayutils.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@ def test_arrayutils_linear_map_float_int16():
3737
print('linear_map int16>float>int16', length, d/1000.0)
3838
assert_almost_equal(out, int16)
3939

40-
41-
test_arrayutils_linear_map_float_int16()
40+
if __name__ == '__main__':
41+
test_arrayutils_linear_map_float_int16()
4242

4343
# Other functionality
4444
# reinterpret

‎tests/test_cnn.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -83,8 +83,8 @@ def test_cnn_mnist_int8():
8383

8484
def test_cnn_mnist_fp32():
8585
check_cnn_mnist(emlearn_cnn_fp32, MNIST_MODEL_FP32)
86-
8786

88-
test_cnn_create()
89-
test_cnn_mnist_int8()
90-
test_cnn_mnist_fp32()
87+
if __name__ == '__main__':
88+
test_cnn_create()
89+
test_cnn_mnist_int8()
90+
test_cnn_mnist_fp32()

‎tests/test_fft.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,5 +43,6 @@ def test_fft_run():
4343

4444
# FIXME: use some reasonable input data and assert the output data
4545

46-
test_fft_del()
47-
test_fft_run()
46+
if __name__ == '__main__':
47+
test_fft_del()
48+
test_fft_run()

‎tests/test_iir.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,5 +39,5 @@ def test_iir_del():
3939
print(before_new, after_new, after_del)
4040
#assert diff == 0, diff
4141

42-
43-
test_iir_del()
42+
if __name__ == '__main__':
43+
test_iir_del()

‎tests/test_iir_q15.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,6 @@ def test_iir_del():
5151
#assert diff == 0, diff
5252

5353
# TODO: add a test that actually runs
54-
55-
test_convert_coefficients()
56-
test_iir_del()
54+
if __name__ == '__main__':
55+
test_convert_coefficients()
56+
test_iir_del()

‎tests/test_kmeans.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ def test_kmeans_many_features():
6262
assert min(assignments) >= 0
6363
assert max(assignments) < n_clusters
6464

65-
test_kmeans_two_clusters()
66-
test_kmeans_many_features()
65+
if __name__ == '__main__':
66+
test_kmeans_two_clusters()
67+
test_kmeans_many_features()
6768

0 commit comments

Comments
 (0)