feat: enhance WIT stub generation with parallel processing and CMake support

Signed-off-by: Gordon Smith <GordonJSmith@gmail.com>

Author: GordonSmith

include/wamr.hpp

@@ -246,19 +246,22 @@ namespace cmcpp
     inline LiftLowerContext create_lift_lower_context(
         wasm_module_inst_t module_inst,
         wasm_exec_env_t exec_env,
-        wasm_function_inst_t cabi_realloc,
         Encoding encoding = Encoding::Utf8)
     {
         wasm_memory_inst_t memory = wasm_runtime_lookup_memory(module_inst, "memory");
         if (!memory)
         {
             throw std::runtime_error("Failed to lookup memory instance");
         }
-
         uint8_t *mem_start_addr = static_cast<uint8_t *>(wasm_memory_get_base_address(memory));
         uint8_t *mem_end_addr = nullptr;
         wasm_runtime_get_native_addr_range(module_inst, mem_start_addr, nullptr, &mem_end_addr);
 
+        wasm_function_inst_t cabi_realloc = wasm_runtime_lookup_function(module_inst, "cabi_realloc");
+        if (!cabi_realloc)
+        {
+            throw std::runtime_error("Failed to lookup cabi_realloc function");
+        }
         GuestRealloc realloc = create_guest_realloc(exec_env, cabi_realloc);
         LiftLowerOptions opts(encoding, std::span<uint8_t>(mem_start_addr, mem_end_addr - mem_start_addr), realloc);
 
@@ -280,7 +283,7 @@ namespace cmcpp
         wasm_function_inst_t cabi_realloc = wasm_runtime_lookup_function(module_inst, "cabi_realloc");
 
         // Use the helper function to create LiftLowerContext
-        LiftLowerContext liftLowerContext = create_lift_lower_context(module_inst, exec_env, cabi_realloc);
+        LiftLowerContext liftLowerContext = create_lift_lower_context(module_inst, exec_env);
         auto params = lift_flat_values<params_t>(liftLowerContext, MAX_FLAT_PARAMS, lower_params);
 
         if constexpr (ValTrait<result_t>::flat_types.size() > 0)

samples/wamr/generated/sample_wamr.cpp

@@ -97,7 +97,4 @@ namespace wasm_utils {
 const uint32_t DEFAULT_STACK_SIZE = 8192;
 const uint32_t DEFAULT_HEAP_SIZE = 8192;
 
-// Note: create_guest_realloc() and create_lift_lower_context() have been
-// moved to <wamr.hpp> in the cmcpp namespace and are available for use directly
-
 } // namespace wasm_utils

samples/wamr/main.cpp

@@ -104,19 +104,14 @@ int main(int argc, char **argv)
 
     std::cout << "Using WASM file: " << wasm_path << std::endl;
 
-    char *buffer, error_buf[128];
-    wasm_module_t module;
-    wasm_module_inst_t module_inst;
-    wasm_function_inst_t cabi_realloc;
-    wasm_exec_env_t exec_env;
     uint32_t size, stack_size = wasm_utils::DEFAULT_STACK_SIZE, heap_size = wasm_utils::DEFAULT_HEAP_SIZE;
 
     /* initialize the wasm runtime by default configurations */
     wasm_runtime_init();
     std::cout << "WAMR runtime initialized successfully" << std::endl;
 
     /* read WASM file into a memory buffer */
-    buffer = read_wasm_binary_to_buffer(wasm_path, &size);
+    char *buffer = read_wasm_binary_to_buffer(wasm_path, &size);
     if (!buffer)
     {
         std::cerr << "Failed to read WASM file" << std::endl;
@@ -139,7 +134,8 @@ int main(int argc, char **argv)
     std::cout << "\nTotal: " << (registered_count + 1) << " host functions registered (all imports from guest perspective)" << std::endl;
 
     // Parse the WASM file from buffer and create a WASM module
-    module = wasm_runtime_load((uint8_t *)buffer, size, error_buf, sizeof(error_buf));
+    char error_buf[128];
+    wasm_module_t module = wasm_runtime_load((uint8_t *)buffer, size, error_buf, sizeof(error_buf));
     if (!module)
     {
         std::cerr << "Failed to load WASM module: " << error_buf << std::endl;
@@ -150,7 +146,7 @@ int main(int argc, char **argv)
     std::cout << "\nSuccessfully loaded WASM module" << std::endl;
 
     // Create an instance of the WASM module (WASM linear memory is ready)
-    module_inst = wasm_runtime_instantiate(module, stack_size, heap_size, error_buf, sizeof(error_buf));
+    wasm_module_inst_t module_inst = wasm_runtime_instantiate(module, stack_size, heap_size, error_buf, sizeof(error_buf));
     if (!module_inst)
     {
         std::cerr << "Failed to instantiate WASM module: " << error_buf << std::endl;
@@ -161,21 +157,10 @@ int main(int argc, char **argv)
     }
     std::cout << "Successfully instantiated WASM module" << std::endl;
 
-    cabi_realloc = wasm_runtime_lookup_function(module_inst, "cabi_realloc");
-    if (!cabi_realloc)
-    {
-        std::cerr << "Failed to lookup cabi_realloc function" << std::endl;
-        wasm_runtime_deinstantiate(module_inst);
-        wasm_runtime_unload(module);
-        delete[] buffer;
-        wasm_runtime_destroy();
-        return 1;
-    }
-
-    exec_env = wasm_runtime_create_exec_env(module_inst, stack_size);
+    wasm_exec_env_t exec_env = wasm_runtime_create_exec_env(module_inst, stack_size);
 
     // Use the helper from wamr.hpp to create the LiftLowerContext
-    LiftLowerContext liftLowerContext = cmcpp::create_lift_lower_context(module_inst, exec_env, cabi_realloc);
+    LiftLowerContext liftLowerContext = cmcpp::create_lift_lower_context(module_inst, exec_env);
 
     std::cout << "\n=== Testing Guest Functions (Exports) ===" << std::endl;
     std::cout << "Note: These are functions the GUEST implements, HOST calls them\n"

test/README.md

@@ -83,8 +83,56 @@ cd test && ./generate_test_stubs.sh
 
 # Filter specific files
 ./generate_test_stubs.py -f "streams"
+
+# Skip CMakeLists.txt generation (if you don't need them)
+./generate_test_stubs.py --no-cmake
+```
+
+#### Individual Stub Compilation
+
+Each generated stub includes its own `CMakeLists.txt` in a dedicated subdirectory for standalone compilation testing:
+
+```bash
+# Generate stubs with CMakeLists.txt (default, parallelized)
+cmake --build build --target generate-test-stubs
+
+# Navigate to a specific stub directory
+cd build/test/generated_stubs/simple-functions
+
+# Build the individual stub
+cmake -S . -B build
+cmake --build build
 ```
 
+**Performance:** Generation is parallelized using all available CPU cores by default. For 199 WIT files:
+- Sequential: ~20 seconds
+- Parallel (32 cores): ~4 seconds (5x faster)
+
+Control parallelization:
+```bash
+python test/generate_test_stubs.py -j 8        # Use 8 parallel jobs
+python test/generate_test_stubs.py -j 1        # Sequential (for debugging)
+python test/generate_test_stubs.py --no-cmake  # Skip CMakeLists.txt generation
+```
+
+**Structure:** Each stub gets its own directory with all files:
+```
+generated_stubs/
+├── simple-functions/
+│   ├── CMakeLists.txt
+│   ├── simple-functions.hpp
+│   ├── simple-functions_wamr.hpp
+│   └── simple-functions_wamr.cpp
+├── integers/
+│   ├── CMakeLists.txt
+│   ├── integers.hpp
+│   ├── integers_wamr.hpp
+│   └── integers_wamr.cpp
+...
+```
+
+**Note:** Stubs with `_wamr.cpp` files require WAMR (WebAssembly Micro Runtime) headers. The generated CMakeLists.txt includes helpful comments about dependencies and will automatically find the local cmcpp headers.
+
 #### Code Generation Validation
 
 The framework also validates generated code by attempting to compile it:

test/StubGenerationTests.cmake

@@ -103,19 +103,23 @@ set(TEST_STUBS_TO_COMPILE
 )
 
 # Collect source files that should exist after generation
+# NOTE: Files are now organized in subdirectories, one per stub
 set(STUB_SOURCES "")
 set(STUB_HEADERS "")
 foreach(stub_name ${TEST_STUBS_TO_COMPILE})
+    # Each stub gets its own subdirectory
+    set(stub_dir "${STUB_OUTPUT_DIR}/${stub_name}")
+    
     # Add main header
-    set(stub_header "${STUB_OUTPUT_DIR}/${stub_name}.hpp")
+    set(stub_header "${stub_dir}/${stub_name}.hpp")
     list(APPEND STUB_HEADERS ${stub_header})
 
     # Add WAMR header
-    set(stub_wamr_hpp "${STUB_OUTPUT_DIR}/${stub_name}_wamr.hpp")
+    set(stub_wamr_hpp "${stub_dir}/${stub_name}_wamr.hpp")
     list(APPEND STUB_HEADERS ${stub_wamr_hpp})
 
     # Add WAMR implementation
-    set(stub_wamr_cpp "${STUB_OUTPUT_DIR}/${stub_name}_wamr.cpp")
+    set(stub_wamr_cpp "${stub_dir}/${stub_name}_wamr.cpp")
     list(APPEND STUB_SOURCES ${stub_wamr_cpp})
 endforeach()
 
@@ -136,10 +140,16 @@ target_link_libraries(test-stubs-compiled
 )
 
 # Add include directories
+# Since each stub is in its own subdirectory, we need to add each one
 target_include_directories(test-stubs-compiled PRIVATE
     ${CMAKE_SOURCE_DIR}/include
-    ${STUB_OUTPUT_DIR}
 )
+# Add each stub directory to the include path
+foreach(stub_name ${TEST_STUBS_TO_COMPILE})
+    target_include_directories(test-stubs-compiled PRIVATE
+        "${STUB_OUTPUT_DIR}/${stub_name}"
+    )
+endforeach()
 
 # Set C++ standard
 target_compile_features(test-stubs-compiled PUBLIC cxx_std_20)