Skip to content

Commit cafa075

Browse files
jonnorjonnor
authored
Merge pull request #70 from emlearn/makefile-dependencies
Improve makefile dependencies
2 parents 757c063 + 53d0543 commit cafa075

8 files changed

Lines changed: 129 additions & 22 deletions

File tree

‎Makefile‎

Lines changed: 67 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -50,51 +50,100 @@ emlearn_cnn_int8_CONFIG = CONFIG=int8
5050
emlearn_cnn_fp32_SRC = src/tinymaix_cnn
5151
emlearn_cnn_fp32_CONFIG = CONFIG=fp32
5252

53+
# Source directories for each module
54+
emlearn_trees_SRC = src/emlearn_trees
55+
emlearn_neighbors_SRC = src/emlearn_neighbors
56+
emlearn_iir_SRC = src/emlearn_iir
57+
emlearn_fft_SRC = src/emlearn_fft
58+
emlearn_kmeans_SRC = src/emlearn_kmeans
59+
emlearn_iir_q15_SRC = src/emlearn_iir_q15
60+
emlearn_arrayutils_SRC = src/emlearn_arrayutils
61+
emlearn_linreg_SRC = src/emlearn_linreg
62+
emlearn_logreg_SRC = src/emlearn_logreg
63+
64+
# Dependencies for each .mpy file: .c, .h, .py files, and Makefile
65+
$(foreach mod,$(MODULES),\
66+
$(eval $(MODULES_PATH)/$(mod).mpy: \
67+
$(wildcard $($(mod)_SRC)/*.c) \
68+
$(wildcard $($(mod)_SRC)/*.h) \
69+
$(wildcard $($(mod)_SRC)/*.py) \
70+
$($(mod)_SRC)/Makefile))
71+
72+
# CNN modules share the same build directory, so they must build sequentially
73+
# Ensure int8 builds after fp32 to avoid race conditions
74+
$(MODULES_PATH)/emlearn_cnn_int8.mpy: $(MODULES_PATH)/emlearn_cnn_fp32.mpy
75+
5376
# Generate list of .mpy files
5477
MODULE_MPYS = $(addprefix $(MODULES_PATH)/,$(addsuffix .mpy,$(MODULES)))
5578

56-
# Build dynamic native module
57-
# defaults to
79+
# Build dynamic native module (without forced clean)
5880
$(MODULES_PATH)/%.mpy:
59-
make -C $(or $($(*)_SRC),src/$*) \
60-
ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=${CFLAGS_EXTRA} \
61-
V=1 $($(*)_CONFIG) clean dist
62-
63-
check_unix_natmod: $(MODULE_MPYS)
81+
$(MAKE) -C $(or $($(*)_SRC),src/$*) \
82+
ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=$(CFLAGS_EXTRA) \
83+
V=1 $($(*)_CONFIG) dist
84+
85+
# CNN modules need clean build due to shared build directory
86+
# They must also build sequentially (fp32 first, then int8)
87+
$(MODULES_PATH)/emlearn_cnn_fp32.mpy:
88+
$(MAKE) -C src/tinymaix_cnn \
89+
ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=$(CFLAGS_EXTRA) \
90+
V=1 CONFIG=fp32 clean
91+
$(MAKE) -C src/tinymaix_cnn \
92+
ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=$(CFLAGS_EXTRA) \
93+
V=1 CONFIG=fp32 dist
94+
95+
$(MODULES_PATH)/emlearn_cnn_int8.mpy: $(MODULES_PATH)/emlearn_cnn_fp32.mpy
96+
$(MAKE) -C src/tinymaix_cnn \
97+
ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=$(CFLAGS_EXTRA) \
98+
V=1 CONFIG=int8 clean
99+
$(MAKE) -C src/tinymaix_cnn \
100+
ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) CFLAGS_EXTRA=$(CFLAGS_EXTRA) \
101+
V=1 CONFIG=int8 dist
102+
103+
# Collect test files for dependency tracking
104+
TEST_PY := $(wildcard tests/test_*.py)
105+
106+
check_unix_natmod: $(MODULE_MPYS) $(TEST_PY)
64107
MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_all.py
65108

66109
$(PORT_DIR):
67110
mkdir -p $@
68111

69-
$(UNIX_MICROPYTHON): $(PORT_DIR)
70-
make -C $(MPY_DIR)/ports/unix V=1 MICROPY_PY_FFI=0 USER_C_MODULES=$(C_MODULES_SRC_PATH) FROZEN_MANIFEST=$(MANIFEST_PATH) CFLAGS_EXTRA="-Wno-unused-function -Wno-unused-function ${CFLAGS_EXTRA}" -j4
112+
# Collect all source and build files under src/ for port builds
113+
SRC_C := $(shell find src -name "*.c" 2>/dev/null)
114+
SRC_H := $(shell find src -name "*.h" 2>/dev/null)
115+
SRC_PY := $(shell find src -name "*.py" 2>/dev/null)
116+
SRC_BUILD := src/micropython.cmake src/dynmodule.mk $(wildcard src/*/micropython.mk)
117+
SRC_ALL = $(SRC_C) $(SRC_H) $(SRC_PY) $(SRC_BUILD)
118+
119+
$(UNIX_MICROPYTHON): $(PORT_DIR) $(SRC_ALL) src/manifest_unix.py
120+
$(MAKE) -C $(MPY_DIR)/ports/unix V=1 MICROPY_PY_FFI=0 USER_C_MODULES=$(C_MODULES_SRC_PATH) FROZEN_MANIFEST=$(MANIFEST_PATH) CFLAGS_EXTRA="-Wno-unused-function -Wno-unused-function $(CFLAGS_EXTRA)" -j4
71121
cp $(MPY_DIR)/ports/unix/build-standard/micropython $@
72122

73123
unix: $(UNIX_MICROPYTHON)
74124

75-
$(WEBASSEMBLY_MICROPYTHON): $(PORT_DIR)
125+
$(WEBASSEMBLY_MICROPYTHON): $(PORT_DIR) $(SRC_ALL) src/manifest_webassembly.py
76126
emcc --version
77127
mkdir -p $(PORT_DIR)/../webassembly
78-
make -C $(MPY_DIR)/ports/webassembly VARIANT=pyscript V=1 USER_C_MODULES=$(C_MODULES_SRC_PATH) FROZEN_MANIFEST=$(WEBASSEMBLY_MANIFEST_PATH) CFLAGS_EXTRA="-Wno-unused-function -Wno-unused-function ${CFLAGS_EXTRA}" -j4
128+
$(MAKE) -C $(MPY_DIR)/ports/webassembly VARIANT=pyscript V=1 USER_C_MODULES=$(C_MODULES_SRC_PATH) FROZEN_MANIFEST=$(WEBASSEMBLY_MANIFEST_PATH) CFLAGS_EXTRA="-Wno-unused-function -Wno-unused-function $(CFLAGS_EXTRA)" -j4
79129
cp $(MPY_DIR)/ports/webassembly/build-pyscript/micropython.mjs $@
80130
cp $(MPY_DIR)/ports/webassembly/build-pyscript/micropython.wasm dist/ports/webassembly/
81131

82132

83133
webassembly: $(WEBASSEMBLY_MICROPYTHON)
84134

85135

86-
check_unix: $(UNIX_MICROPYTHON)
87-
$(UNIX_MICROPYTHON) tests/test_all.py test_iir,test_fft,test_arrayutils,test_linreg,test_logreg
88-
# TODO: enable more modules
136+
check_unix: $(UNIX_MICROPYTHON) $(TEST_PY)
137+
$(UNIX_MICROPYTHON) tests/test_all.py -test_cnn
89138

90-
rp2: $(PORT_DIR)
91-
make -C $(MPY_DIR)/ports/rp2 V=1 USER_C_MODULES=$(C_MODULES_SRC_PATH)/micropython.cmake FROZEN_MANIFEST=$(MANIFEST_PATH) CFLAGS_EXTRA='-Wno-unused-function -Wno-unused-function' -j4
139+
rp2: $(PORT_DIR) $(SRC_ALL) src/manifest_unix.py
140+
$(MAKE) -C $(MPY_DIR)/ports/rp2 V=1 USER_C_MODULES=$(C_MODULES_SRC_PATH)/micropython.cmake FROZEN_MANIFEST=$(MANIFEST_PATH) CFLAGS_EXTRA='-Wno-unused-function -Wno-unused-function' -j4
92141
mkdir -p ./dist/ports/rp2/RPI_PICO
93142
cp -r $(MPY_DIR)/ports/rp2/build-RPI_PICO/firmware* ./dist/ports/rp2/RPI_PICO/
94143

95144

96-
extmod:
97-
make -C $(MPY_DIR)/ports/esp32 V=1 BOARD=$(BOARD) USER_C_MODULES=$(C_MODULES_SRC_PATH)/micropython.cmake FROZEN_MANIFEST=$(MANIFEST_PATH) CFLAGS_EXTRA='-Wno-unused-function -Wno-unused-function' -j4
145+
extmod: $(SRC_ALL) src/manifest_unix.py
146+
$(MAKE) -C $(MPY_DIR)/ports/esp32 V=1 BOARD=$(BOARD) USER_C_MODULES=$(C_MODULES_SRC_PATH)/micropython.cmake FROZEN_MANIFEST=$(MANIFEST_PATH) CFLAGS_EXTRA='-Wno-unused-function -Wno-unused-function' -j4
98147
mkdir -p $(PORT_DIST_DIR)
99148
cp -r $(PORT_BUILD_DIR)/firmware* $(PORT_DIST_DIR)
100149
cp -r $(PORT_BUILD_DIR)/micropython* $(PORT_DIST_DIR)

‎src/emlearn_kmeans/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ DIST_DIR := ../../dist/$(ARCH)_$(MPY_ABI_VERSION)
1313
MOD = emlearn_kmeans
1414

1515
# Source files (.c or .py)
16-
SRC = kmeans.c kmeans.py
16+
SRC = kmeans.c emlearn_kmeans.py
1717

1818
# enable linking of libm etc
1919
LINK_RUNTIME=1

src/emlearn_kmeans/kmeans.py renamed to src/emlearn_kmeans/emlearn_kmeans.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,12 @@
11

2+
# When used as external C module, the .py is the top-level import,
3+
# and we need to merge the native module symbols at import time
4+
# When used as dynamic native modules (.mpy), .py and native code is merged at build time
5+
try:
6+
from emlearn_kmeans_c import *
7+
except ImportError as e:
8+
pass
9+
210
import array
311

412
# FIXME: get Exception: can't merge files when more than one contains native code

‎src/emlearn_kmeans/kmeans.c‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
// Include the header file to get access to the MicroPython API
2+
#ifdef MICROPY_ENABLE_DYNRUNTIME
23
#include "py/dynruntime.h"
4+
#else
5+
#include "py/runtime.h"
6+
#endif
37

48
#include <string.h>
59
#include <stdint.h>
@@ -91,9 +95,10 @@ euclidean_argmin(mp_obj_t vectors_obj, mp_obj_t point_obj) {
9195
mp_obj_new_int(min_index),
9296
mp_obj_new_int(min_dist),
9397
}));
94-
}
98+
}
9599
static MP_DEFINE_CONST_FUN_OBJ_2(euclidian_argmin_obj, euclidean_argmin);
96100

101+
#ifdef MICROPY_ENABLE_DYNRUNTIME
97102

98103
// This is the entry point and is called when the module is imported
99104
mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
@@ -106,4 +111,22 @@ mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *a
106111
MP_DYNRUNTIME_INIT_EXIT
107112
}
108113

114+
#else
115+
116+
// Define module object.
117+
static const mp_rom_map_elem_t emlearn_kmeans_globals_table[] = {
118+
{ MP_ROM_QSTR(MP_QSTR_euclidean_argmin), MP_ROM_PTR(&euclidian_argmin_obj) },
119+
};
120+
static MP_DEFINE_CONST_DICT(emlearn_kmeans_globals, emlearn_kmeans_globals_table);
121+
122+
const mp_obj_module_t emlearn_kmeans_cmodule = {
123+
.base = { &mp_type_module },
124+
.globals = (mp_obj_dict_t *)&emlearn_kmeans_globals,
125+
};
126+
127+
// External module name is XXX_c to allow .py file to be the entrypoint
128+
MP_REGISTER_MODULE(MP_QSTR_emlearn_kmeans_c, emlearn_kmeans_cmodule);
129+
130+
#endif
131+
109132

‎src/emlearn_kmeans/micropython.cmake‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
add_library(usermod_emlearn_kmeans INTERFACE)
2+
3+
target_sources(usermod_emlearn_kmeans INTERFACE
4+
${CMAKE_CURRENT_LIST_DIR}/kmeans.c
5+
)
6+
7+
target_include_directories(usermod_emlearn_kmeans INTERFACE
8+
${CMAKE_CURRENT_LIST_DIR}
9+
)
10+
11+
target_link_libraries(usermod INTERFACE usermod_emlearn_kmeans)

‎src/emlearn_kmeans/micropython.mk‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
MOD_DIR := $(USERMOD_DIR)
2+
3+
# Add all C files to SRC_USERMOD.
4+
SRC_USERMOD_C += $(MOD_DIR)/kmeans.c
5+
6+
# We can add our module folder to include paths if needed
7+
CFLAGS_USERMOD += -I$(MOD_DIR)

‎src/manifest_unix.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
# NOTE: this is a different mechanism than
44
# Ref https://docs.micropython.org/en/latest/reference/manifest.html
55
module("emlearn_trees.py", base_path='./emlearn_trees')
6+
module("emlearn_kmeans.py", base_path='./emlearn_kmeans')
67
module("emlearn_fft.py", base_path='./emlearn_fft')
78
module("emlearn_linreg.py", base_path='./emlearn_linreg')
89
module("emlearn_logreg.py", base_path='./emlearn_logreg')

‎tests/test_all.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
'test_cnn',
2424
'test_fft',
2525
'test_iir',
26-
'test_iir_q15',
26+
#'test_iir_q15', # skip, not functional
2727
'test_kmeans',
2828
'test_linreg',
2929
'test_linreg_california',
@@ -40,7 +40,15 @@ def main():
4040

4141
modules = TEST_MODULES
4242
if len(sys.argv) >= 2:
43-
modules = sys.argv[1].split(',')
43+
config = sys.argv[1].split(',')
44+
skip = [ m[1:] for m in config if m[0] == '-' ]
45+
add = [ m for m in config if m[0] != '-' ]
46+
if len(skip):
47+
modules = [m for m in TEST_MODULES if not m in skip ]
48+
print('SKIPPING', skip)
49+
if len(add):
50+
modules = add
51+
print('RUN ONLY', add)
4452

4553
passed = 0
4654
failed = 0

0 commit comments

Comments
 (0)