diff --git a/Makefile b/Makefile index d4323812..88d9739a 100644 --- a/Makefile +++ b/Makefile @@ -28,62 +28,40 @@ PORT_DIST_DIR=./dist/ports/$(PORT)/$(BOARD) UNIX_MICROPYTHON = ./dist/ports/unix/micropython -$(MODULES_PATH)/emlearn_trees.mpy: - make -C src/emlearn_trees/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} V=1 clean dist -$(MODULES_PATH)/emlearn_neighbors.mpy: - make -C src/emlearn_neighbors/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} V=1 clean dist - -$(MODULES_PATH)/emlearn_iir.mpy: - make -C src/emlearn_iir/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} V=1 clean dist - -$(MODULES_PATH)/emlearn_fft.mpy: - make -C src/emlearn_fft/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} V=1 clean dist - -$(MODULES_PATH)/emlearn_cnn_int8.mpy: - make -C src/tinymaix_cnn/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} V=1 CONFIG=int8 clean dist - -$(MODULES_PATH)/emlearn_cnn_fp32.mpy: - make -C src/tinymaix_cnn/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} V=1 CONFIG=fp32 clean dist - -$(MODULES_PATH)/emlearn_kmeans.mpy: - make -C src/emlearn_kmeans/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} V=1 clean dist - -$(MODULES_PATH)/emlearn_iir_q15.mpy: - make -C src/emlearn_iir_q15/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} V=1 clean dist - -$(MODULES_PATH)/emlearn_arrayutils.mpy: - make -C src/emlearn_arrayutils/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} V=1 clean dist - -$(MODULES_PATH)/emlearn_linreg.mpy: - make -C src/emlearn_linreg/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} V=1 clean dist - -emlearn_trees.results: $(MODULES_PATH)/emlearn_trees.mpy - MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_trees.py - -emlearn_neighbors.results: $(MODULES_PATH)/emlearn_neighbors.mpy - MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_neighbors.py - -emlearn_iir.results: $(MODULES_PATH)/emlearn_iir.mpy - MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_iir.py - -emlearn_fft.results: $(MODULES_PATH)/emlearn_fft.mpy - MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_fft.py - -emlearn_cnn.results: $(MODULES_PATH)/emlearn_cnn_int8.mpy $(MODULES_PATH)/emlearn_cnn_fp32.mpy - MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_cnn.py - -emlearn_kmeans.results: $(MODULES_PATH)/emlearn_kmeans.mpy - MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_kmeans.py - -emlearn_iir_q15.results: $(MODULES_PATH)/emlearn_iir_q15.mpy - MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_iir_q15.py - -emlearn_arrayutils.results: $(MODULES_PATH)/emlearn_arrayutils.mpy - MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_arrayutils.py - -emlearn_linreg.results: $(MODULES_PATH)/emlearn_linreg.mpy - MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_linreg.py +# List of modules +MODULES = emlearn_trees \ + emlearn_neighbors \ + emlearn_iir \ + emlearn_fft \ + emlearn_kmeans \ + emlearn_iir_q15 \ + emlearn_arrayutils \ + emlearn_linreg \ + emlearn_cnn_int8 \ + emlearn_cnn_fp32 + +# Generate list of .mpy files +MODULE_MPYS = $(addprefix $(MODULES_PATH)/,$(addsuffix .mpy,$(MODULES))) + +# Special cases +emlearn_cnn_int8_SRC = src/tinymaix_cnn +emlearn_cnn_int8_CONFIG = CONFIG=int8 +emlearn_cnn_fp32_SRC = src/tinymaix_cnn +emlearn_cnn_fp32_CONFIG = CONFIG=fp32 + +# Generate list of .mpy files +MODULE_MPYS = $(addprefix $(MODULES_PATH)/,$(addsuffix .mpy,$(MODULES))) + +# Build dynamic native module +# defaults to +$(MODULES_PATH)/%.mpy: + make -C $(or $($(*)_SRC),src/$*) \ + ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} \ + V=1 $($(*)_CONFIG) clean dist + +check_unix_natmod: $(MODULE_MPYS) + MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_all.py $(PORT_DIR): mkdir -p $@ @@ -95,11 +73,8 @@ $(UNIX_MICROPYTHON): $(PORT_DIR) unix: $(UNIX_MICROPYTHON) check_unix: $(UNIX_MICROPYTHON) - $(UNIX_MICROPYTHON) tests/test_trees.py - $(UNIX_MICROPYTHON) tests/test_iir.py - $(UNIX_MICROPYTHON) tests/test_fft.py - $(UNIX_MICROPYTHON) tests/test_arrayutils.py - echo SKIP $(UNIX_MICROPYTHON) tests/test_cnn.py + $(UNIX_MICROPYTHON) tests/test_all.py test_iir,test_fft,test_arrayutils + # TODO: enable more modules rp2: $(PORT_DIR) make -C $(MPY_DIR)/ports/rp2 V=1 USER_C_MODULES=$(C_MODULES_SRC_PATH)/micropython.cmake FROZEN_MANIFEST=$(MANIFEST_PATH) CFLAGS_EXTRA='-Wno-unused-function -Wno-unused-function' -j4 @@ -130,8 +105,8 @@ release: zip -r $(RELEASE_NAME).zip $(RELEASE_NAME) #cp $(RELEASE_NAME).zip emlearn-micropython-latest.zip -check: emlearn_trees.results emlearn_neighbors.results emlearn_iir.results emlearn_iir_q15.results emlearn_fft.results emlearn_kmeans.results emlearn_arrayutils.results emlearn_cnn.results emlearn_linreg.results +check: check_unix_natmod -dist: $(MODULES_PATH)/emlearn_trees.mpy $(MODULES_PATH)/emlearn_neighbors.mpy $(MODULES_PATH)/emlearn_iir.mpy $(MODULES_PATH)/emlearn_iir_q15.mpy $(MODULES_PATH)/emlearn_fft.mpy $(MODULES_PATH)/emlearn_kmeans.mpy $(MODULES_PATH)/emlearn_arrayutils.mpy $(MODULES_PATH)/emlearn_cnn_int8.mpy $(MODULES_PATH)/emlearn_cnn_fp32.mpy $(MODULES_PATH)/emlearn_linreg.mpy +dist: $(MODULE_MPYS) diff --git a/docs/developing.md b/docs/developing.md index 2537c5ea..8561d066 100644 --- a/docs/developing.md +++ b/docs/developing.md @@ -61,6 +61,15 @@ NOTE: Tested on Linux and Mac OS. Not tested on Windows Subsystem for Linux (WSL make check_unix ``` +You should see each of the test functions in tests/ being ran, +and then a summary at the end with something like: + +``` +... +Passed: 17 +Failed: 0 +``` + #### Run tests on PC using dynamic native modules This runs tests by building as dynamic native modules (.mpy files), @@ -75,6 +84,14 @@ To build and run tests of dynamic native modules on host make check ``` +You should see each of the test functions in tests/ being ran, +and then a summary at the end with something like: + +``` +... +Passed: 17 +Failed: 0 +``` #### Build for device @@ -91,6 +108,24 @@ Install it on device mpremote cp dist/armv6m*/emlearn_trees.mpy :emlearn_trees.mpy ``` +#### Running tests on device + +NOTE: Assumes that the .mpy files have been built first (in dist/). + +We use `mpremote mount`, which allows the device to access files on the PC/host filesystem. This means we do not have to copy the modules or files across. + +``` +mpremote mount . run tests/test_all.py +``` + +You should see each of the test functions in tests/ being ran, +and then a summary at the end with something like: + +``` +... +Passed: 17 +Failed: 0 +``` ## Building documentation diff --git a/requirements.txt b/requirements.txt index 9c113831..ae9c64eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ scikit-learn>=1.0.0 ar>=1.0.0 pyelftools>=0.31 setuptools>=71.0.0 +mpremote>=1.25.0 diff --git a/tests/test_all.py b/tests/test_all.py new file mode 100644 index 00000000..2ec72590 --- /dev/null +++ b/tests/test_all.py @@ -0,0 +1,82 @@ + +import sys + +# Find the module path (architecture+version specific) +sys_mpy = sys.implementation._mpy +mpy_arch = [None, 'x86', 'x64', + 'armv6', 'armv6m', 'armv7m', 'armv7em', 'armv7emsp', 'armv7emdp', + 'xtensa', 'xtensawin', 'rv32imc'][sys_mpy >> 10] +mpy_major = sys_mpy & 0xff +mpy_minor = sys_mpy >> 8 & 3 + +module_dir = f'{mpy_arch}_{mpy_major}.{mpy_minor}' + +# make sure we can import .mpy modules +sys.path.insert(0, './dist/'+module_dir) + +# make sure we can import test files +sys.path.insert(0, './tests') + + +TEST_MODULES=[ + 'test_arrayutils', + 'test_cnn', + 'test_fft', + 'test_iir', + 'test_iir_q15', + 'test_kmeans', + 'test_linreg', + #'test_linreg_california', + 'test_neighbors', + 'test_trees', +] + +def main(): + + # Find which tests are enabled + # Default: all + + modules = TEST_MODULES + if len(sys.argv) >= 2: + modules = sys.argv[1].split(',') + + passed = 0 + failed = 0 + + for module_name in modules: + mod = None + print(f'{module_name}:') + try: + mod = __import__(module_name) + except Exception as e: + print(f'Error while importing {module_name}:') + sys.print_exception(e) + print() # spacing for readability + failed += 1 + continue + + module_attributes = dir(mod) + tests = [ o for o in module_attributes if o.startswith('test_') ] + for test_name in tests: + test_function = getattr(mod, test_name) + print(f'{module_name}.py/{test_name}:') + try: + test_function() + except Exception as e: + print(f'\tFAIL') + sys.print_exception(e) + print() # spacing for readability + failed += 1 + continue + + print(f'\t PASS') + passed += 1 + + print(f'Passed: {passed}') + print(f'Failed: {failed}') + + # Let status code reflect number of failures + return failed + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/test_arrayutils.py b/tests/test_arrayutils.py index 9617e4b8..84d341ca 100644 --- a/tests/test_arrayutils.py +++ b/tests/test_arrayutils.py @@ -37,8 +37,8 @@ def test_arrayutils_linear_map_float_int16(): print('linear_map int16>float>int16', length, d/1000.0) assert_almost_equal(out, int16) - -test_arrayutils_linear_map_float_int16() +if __name__ == '__main__': + test_arrayutils_linear_map_float_int16() # Other functionality # reinterpret diff --git a/tests/test_cnn.py b/tests/test_cnn.py index 190ed5ec..a8237a52 100644 --- a/tests/test_cnn.py +++ b/tests/test_cnn.py @@ -83,8 +83,8 @@ def test_cnn_mnist_int8(): def test_cnn_mnist_fp32(): check_cnn_mnist(emlearn_cnn_fp32, MNIST_MODEL_FP32) - -test_cnn_create() -test_cnn_mnist_int8() -test_cnn_mnist_fp32() +if __name__ == '__main__': + test_cnn_create() + test_cnn_mnist_int8() + test_cnn_mnist_fp32() diff --git a/tests/test_fft.py b/tests/test_fft.py index ba398077..70aa5cb8 100644 --- a/tests/test_fft.py +++ b/tests/test_fft.py @@ -43,5 +43,6 @@ def test_fft_run(): # FIXME: use some reasonable input data and assert the output data -test_fft_del() -test_fft_run() +if __name__ == '__main__': + test_fft_del() + test_fft_run() diff --git a/tests/test_iir.py b/tests/test_iir.py index b8f165e8..be3b7c41 100644 --- a/tests/test_iir.py +++ b/tests/test_iir.py @@ -39,5 +39,5 @@ def test_iir_del(): print(before_new, after_new, after_del) #assert diff == 0, diff - -test_iir_del() +if __name__ == '__main__': + test_iir_del() diff --git a/tests/test_iir_q15.py b/tests/test_iir_q15.py index 43789cc1..5f34411e 100644 --- a/tests/test_iir_q15.py +++ b/tests/test_iir_q15.py @@ -51,6 +51,6 @@ def test_iir_del(): #assert diff == 0, diff # TODO: add a test that actually runs - -test_convert_coefficients() -test_iir_del() +if __name__ == '__main__': + test_convert_coefficients() + test_iir_del() diff --git a/tests/test_kmeans.py b/tests/test_kmeans.py index 3759ef29..472c756e 100644 --- a/tests/test_kmeans.py +++ b/tests/test_kmeans.py @@ -62,6 +62,7 @@ def test_kmeans_many_features(): assert min(assignments) >= 0 assert max(assignments) < n_clusters -test_kmeans_two_clusters() -test_kmeans_many_features() +if __name__ == '__main__': + test_kmeans_two_clusters() + test_kmeans_many_features() diff --git a/tests/test_linreg.py b/tests/test_linreg.py index d7853889..a6db9238 100644 --- a/tests/test_linreg.py +++ b/tests/test_linreg.py @@ -2,15 +2,22 @@ import emlearn_linreg import array -model = emlearn_linreg.new(4, 0.1, 0.5, 0.01) +def test_linreg_train_trivial(): -# Training data (float32 arrays) -X = array.array('f', [1,2,3,4, 2,3,4,5]) # flattened -y = array.array('f', [10, 15]) + model = emlearn_linreg.new(4, 0.1, 0.5, 0.01) -# Train -emlearn_linreg.train(model, X, y, max_iterations=100, tolerance=1e-6, verbose=0) + # Training data (float32 arrays) + X = array.array('f', [1,2,3,4, 2,3,4,5]) # flattened + y = array.array('f', [10, 15]) -# Predict -prediction = model.predict(array.array('f', [1,2,3,4])) -print(prediction) + # Train + emlearn_linreg.train(model, X, y, max_iterations=100, tolerance=1e-6, verbose=0) + + # Predict + prediction = model.predict(array.array('f', [1,2,3,4])) + + error = prediction - 10.0 + assert abs(error) < 0.80, error + +if __name__ == '__main__': + test_linreg_train_trivial() diff --git a/tests/test_neighbors.py b/tests/test_neighbors.py index f89ef2a3..8b0d18d9 100644 --- a/tests/test_neighbors.py +++ b/tests/test_neighbors.py @@ -78,6 +78,8 @@ def test_neighbors_get_results(): assert model.getresult(0)[2] == 0 assert model.getresult(3)[2] == 1 -test_neighbors_del() -test_neighbors_trivial() -test_neighbors_get_results() +if __name__ == '__main__': + test_neighbors_del() + test_neighbors_trivial() + test_neighbors_get_results() + diff --git a/tests/test_trees.py b/tests/test_trees.py index a968f645..2444746b 100644 --- a/tests/test_trees.py +++ b/tests/test_trees.py @@ -63,6 +63,7 @@ def test_trees_xor(): result = argmax(out) assert result == expect, (ex, expect, result) -test_trees_del() -test_trees_xor() +if __name__ == '__main__': + test_trees_del() + test_trees_xor()