diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras index 9f0cf6821b8..645b8aa77bf 100644 --- a/.wolfssl_known_macro_extras +++ b/.wolfssl_known_macro_extras @@ -660,6 +660,7 @@ WC_FORCE_LINUXKM_FORTIFY_SOURCE WC_HASH_CUSTOM_MAX_BLOCK_SIZE WC_HASH_CUSTOM_MAX_DIGEST_SIZE WC_HASH_CUSTOM_MIN_DIGEST_SIZE +WC_LINUXKM_NO_USE_HEAP_WRAPPERS WC_MLKEM_KERNEL_ASM WC_NO_ASYNC_SLEEP WC_NO_RNG_SIMPLE diff --git a/linuxkm/linuxkm_wc_port.h b/linuxkm/linuxkm_wc_port.h index 9c28e771de0..892f323a8ab 100644 --- a/linuxkm/linuxkm_wc_port.h +++ b/linuxkm/linuxkm_wc_port.h @@ -249,6 +249,13 @@ #define LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT #endif + #if (LINUX_VERSION_CODE >= KERNEL_VERSION(7, 1, 0)) && \ + defined(WC_SYM_RELOC_TABLES) && \ + !defined(WC_LINUXKM_NO_USE_HEAP_WRAPPERS) && \ + !defined(WC_LINUXKM_USE_HEAP_WRAPPERS) + #define WC_LINUXKM_USE_HEAP_WRAPPERS + #endif + #ifdef BUILDING_WOLFSSL #if ((LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)) || \ @@ -521,6 +528,13 @@ #include #endif + #ifdef WC_LINUXKM_USE_HEAP_WRAPPERS + WOLFSSL_API void *wc_linuxkm_malloc(size_t size); + WOLFSSL_API void wc_linuxkm_free(void *ptr); + WOLFSSL_API void *wc_linuxkm_realloc(void *ptr, size_t newsize); + WOLFSSL_API size_t wc_linuxkm_malloc_usable_size(void *ptr); + #endif + #ifndef WC_CONTAINERIZE_THIS #include #include @@ -973,6 +987,12 @@ const unsigned char *_ctype; +#ifdef WC_LINUXKM_USE_HEAP_WRAPPERS + typeof(wc_linuxkm_malloc) *wc_linuxkm_malloc; + typeof(wc_linuxkm_free) *wc_linuxkm_free; + typeof(wc_linuxkm_realloc) *wc_linuxkm_realloc; + typeof(wc_linuxkm_malloc_usable_size) *wc_linuxkm_malloc_usable_size; +#else /* !WC_LINUXKM_USE_HEAP_WRAPPERS */ #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0) typeof(kmalloc_noprof) *kmalloc_noprof; typeof(krealloc_node_align_noprof) *krealloc_node_align_noprof; @@ -1023,6 +1043,7 @@ #endif typeof(kfree) *kfree; typeof(ksize) *ksize; +#endif /* !WC_LINUXKM_USE_HEAP_WRAPPERS */ #ifndef LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT typeof(get_random_bytes) *get_random_bytes; @@ -1313,6 +1334,11 @@ #define _ctype WC_PIE_INDIRECT_SYM(_ctype) +#ifdef WC_LINUXKM_USE_HEAP_WRAPPERS + /* no native heap call masking -- wc_linuxkm wrapper functions bound + * directly to malloc() and friends. + */ +#else /* !WC_LINUXKM_USE_HEAP_WRAPPERS */ #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0) /* see include/linux/alloc_tag.h and include/linux/slab.h */ #define kmalloc_noprof WC_PIE_INDIRECT_SYM(kmalloc_noprof) @@ -1368,6 +1394,7 @@ #define kvfree WC_PIE_INDIRECT_SYM(kvfree) #endif #define ksize WC_PIE_INDIRECT_SYM(ksize) +#endif /* !WC_LINUXKM_USE_HEAP_WRAPPERS */ #ifndef LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT #define get_random_bytes WC_PIE_INDIRECT_SYM(get_random_bytes) @@ -1705,6 +1732,12 @@ #error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture. #endif /* !CONFIG_X86 */ #endif /* WOLFSSL_USE_SAVE_VECTOR_REGISTERS */ + #ifdef WC_LINUXKM_USE_HEAP_WRAPPERS + WOLFSSL_API extern void *wc_linuxkm_malloc(size_t size); + WOLFSSL_API void wc_linuxkm_free(void *ptr); + WOLFSSL_API void *wc_linuxkm_realloc(void *ptr, size_t newsize); + WOLFSSL_API size_t wc_linuxkm_malloc_usable_size(void *ptr); + #endif #endif /* !BUILDING_WOLFSSL */ /* Copied from wc_port.h */ @@ -1859,9 +1892,25 @@ ((sizeof(_alloc_sz) * 8UL) - __builtin_clzl(_alloc_sz - 1)); \ _alloc_sz; \ }) + + #ifdef WC_LINUXKM_USE_HEAP_WRAPPERS + #ifdef WC_CONTAINERIZE_THIS + #define malloc(size) WC_PIE_INDIRECT_SYM(wc_linuxkm_malloc)(size) + #define free(ptr) WC_PIE_INDIRECT_SYM(wc_linuxkm_free)(ptr) + #define realloc(ptr, newsize) WC_PIE_INDIRECT_SYM(wc_linuxkm_realloc)(ptr, newsize) + #else + #define malloc(size) wc_linuxkm_malloc(size) + #define free(ptr) wc_linuxkm_free(ptr) + #define realloc(ptr, newsize) wc_linuxkm_realloc(ptr, newsize) + #endif + #else /* !WC_LINUXKM_USE_HEAP_WRAPPERS */ #ifdef USE_KVMALLOC #define malloc(size) kvmalloc_node(WC_LINUXKM_ROUND_UP_P_OF_2(size), (preempt_count() == 0 ? GFP_KERNEL : GFP_ATOMIC), NUMA_NO_NODE) - #define free(ptr) kvfree(ptr) + #if LINUX_VERSION_CODE >= KERNEL_VERSION(7, 2, 0) + #define free(ptr) (preempt_count() == 0 ? kvfree(ptr) : kvfree_atomic(ptr)) + #else + #define free(ptr) kvfree(ptr) + #endif #ifdef USE_KVREALLOC #define realloc(ptr, newsize) kvrealloc(ptr, WC_LINUXKM_ROUND_UP_P_OF_2(newsize), (preempt_count() == 0 ? GFP_KERNEL : GFP_ATOMIC)) #else @@ -1872,6 +1921,7 @@ #define free(ptr) kfree(ptr) #define realloc(ptr, newsize) krealloc(ptr, WC_LINUXKM_ROUND_UP_P_OF_2(newsize), (preempt_count() == 0 ? GFP_KERNEL : GFP_ATOMIC)) #endif + #endif /* !WC_LINUXKM_USE_HEAP_WRAPPERS */ #ifndef static_assert #define static_assert(expr, ...) __static_assert(expr, ##__VA_ARGS__, #expr) diff --git a/linuxkm/module_hooks.c b/linuxkm/module_hooks.c index 8ef3ea0029f..be1fd01d7b8 100644 --- a/linuxkm/module_hooks.c +++ b/linuxkm/module_hooks.c @@ -1275,6 +1275,41 @@ static const struct wc_reloc_table_segments seg_map = { #endif /* !WC_SYM_RELOC_TABLES && HAVE_FIPS */ +#ifdef WC_LINUXKM_USE_HEAP_WRAPPERS + +#ifndef USE_KVREALLOC + #error WC_LINUXKM_USE_HEAP_WRAPPERS requires USE_KVREALLOC +#endif + +void *wc_linuxkm_malloc(size_t size) +{ + return kvmalloc_node(WC_LINUXKM_ROUND_UP_P_OF_2(size), (preempt_count() == 0 ? GFP_KERNEL : GFP_ATOMIC), NUMA_NO_NODE); +} + +void wc_linuxkm_free(void *ptr) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(7, 2, 0) + if (preempt_count() == 0) + kvfree(ptr); + else + kvfree_atomic(ptr); +#else + kvfree(ptr); +#endif +} + +void *wc_linuxkm_realloc(void *ptr, size_t newsize) +{ + return kvrealloc(ptr, WC_LINUXKM_ROUND_UP_P_OF_2(newsize), (preempt_count() == 0 ? GFP_KERNEL : GFP_ATOMIC)); +} + +size_t wc_linuxkm_malloc_usable_size(void *ptr) +{ + return ksize(ptr); +} + +#endif /* WC_LINUXKM_USE_HEAP_WRAPPERS */ + #ifdef WC_SYM_RELOC_TABLES /* get_current() is an inline or macro, depending on the target -- sidestep the @@ -1361,6 +1396,13 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) { wolfssl_linuxkm_pie_redirect_table._ctype = _ctype; +#ifdef WC_LINUXKM_USE_HEAP_WRAPPERS + wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_malloc = wc_linuxkm_malloc; + wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_free = wc_linuxkm_free; + wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_realloc = wc_linuxkm_realloc; + wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_malloc_usable_size = wc_linuxkm_malloc_usable_size; +#else /* !WC_LINUXKM_USE_HEAP_WRAPPERS */ + #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0) wolfssl_linuxkm_pie_redirect_table.kmalloc_noprof = kmalloc_noprof; wolfssl_linuxkm_pie_redirect_table.krealloc_node_align_noprof = krealloc_node_align_noprof; @@ -1416,6 +1458,8 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) { wolfssl_linuxkm_pie_redirect_table.kvfree = kvfree; #endif +#endif /* !WC_LINUXKM_USE_HEAP_WRAPPERS */ + #ifndef LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT wolfssl_linuxkm_pie_redirect_table.get_random_bytes = get_random_bytes; #endif