RelNode -> Substrait Plan -> JNI -> Substrait Plan

Signed-off-by: Lantao Jin <ltjin@amazon.com>

Author: LantaoJin

build-and-run-demo.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Build and run the Java-C++ integration demo
+
+set -e
+
+echo "=== Building OpenSearch SQL Native Engine Demo ==="
+
+# Check required tools
+if ! command -v cmake &> /dev/null; then
+    echo "Error: cmake is required but not installed."
+    exit 1
+fi
+
+if ! command -v make &> /dev/null; then
+    echo "Error: make is required but not installed."
+    exit 1
+fi
+
+# Build project
+echo "Building native module..."
+./gradlew :native:build
+
+# Run demo
+echo ""
+echo "Running demo..."
+./gradlew :native:run -PmainClass=org.opensearch.sql.NativeEngineDemo
+
+echo ""
+echo "=== Build and demo completed ==="

common/build.gradle

@@ -63,7 +63,7 @@ configurations.all {
     resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
     resolutionStrategy.force "org.apache.httpcomponents:httpcore:4.4.13"
     resolutionStrategy.force "joda-time:joda-time:2.10.12"
-    resolutionStrategy.force "org.slf4j:slf4j-api:1.7.36"
+    resolutionStrategy.force "org.slf4j:slf4j-api:2.0.17"
 }
 
 spotless {

core/src/main/java/org/opensearch/sql/ast/statement/Explain.java

@@ -41,7 +41,9 @@ public enum ExplainFormat {
     EXTENDED,
     COST,
     /** Formats explain output in yaml format. */
-    YAML
+    YAML,
+    /** Explain output to substrait plan. */
+    SUBSTRAIT
   }
 
   public static ExplainFormat format(String format) {

core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java

@@ -25,9 +25,11 @@
 import org.apache.calcite.rel.RelHomogeneousShuttle;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.RelShuttle;
+import org.apache.calcite.rel.RelVisitor;
 import org.apache.calcite.rel.core.Project;
 import org.apache.calcite.rel.core.Sort;
 import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rel.logical.LogicalAggregate;
 import org.apache.calcite.rel.logical.LogicalProject;
 import org.apache.calcite.rel.logical.LogicalSort;
 import org.apache.calcite.rel.type.RelDataType;
@@ -511,6 +513,24 @@ static boolean containsRexOver(LogicalProject project) {
     return project.getProjects().stream().anyMatch(RexOver::containsOver);
   }
 
+  static boolean containsAggregate(RelNode node) {
+    try {
+      (new RelVisitor() {
+            public void visit(RelNode node, int ordinal, RelNode parent) {
+              if (node instanceof LogicalAggregate) {
+                throw Util.FoundOne.NULL;
+              } else {
+                super.visit(node, ordinal, parent);
+              }
+            }
+          })
+          .go(node);
+      return false;
+    } catch (Util.FoundOne var3) {
+      return true;
+    }
+  }
+
   /**
    * The LogicalSort is a LIMIT that should be pushed down when its fetch field is not null and its
    * collation is empty. For example: <code>sort name | head 5</code> should not be pushed down

cpp/CMakeLists.txt

@@ -0,0 +1,54 @@
+cmake_minimum_required(VERSION 3.16)
+project(opensearch-sql-native)
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+# Set output directories
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+
+# Find JNI
+find_package(JNI REQUIRED)
+if(JNI_FOUND)
+    message(STATUS "JNI_INCLUDE_DIRS=${JNI_INCLUDE_DIRS}")
+    message(STATUS "JNI_LIBRARIES=${JNI_LIBRARIES}")
+endif()
+
+# Find Protobuf
+find_package(Protobuf REQUIRED)
+
+# Generate protobuf files
+set(PROTO_DIR ${CMAKE_SOURCE_DIR}/proto)
+set(GENERATED_DIR ${CMAKE_BINARY_DIR}/generated)
+file(MAKE_DIRECTORY ${GENERATED_DIR})
+
+# Generate protobuf C++ files
+execute_process(
+    COMMAND ${Protobuf_PROTOC_EXECUTABLE} 
+        --proto_path=${PROTO_DIR}
+        --cpp_out=${GENERATED_DIR}
+        ${PROTO_DIR}/substrait/plan.proto
+        ${PROTO_DIR}/substrait/algebra.proto
+        ${PROTO_DIR}/substrait/type.proto
+        ${PROTO_DIR}/substrait/extensions/extensions.proto
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+)
+
+# Collect generated files
+file(GLOB PROTO_SRCS "${GENERATED_DIR}/substrait/*.pb.cc")
+file(GLOB PROTO_HDRS "${GENERATED_DIR}/substrait/*.pb.h")
+file(GLOB PROTO_EXT_SRCS "${GENERATED_DIR}/substrait/extensions/*.pb.cc")
+file(GLOB PROTO_EXT_HDRS "${GENERATED_DIR}/substrait/extensions/*.pb.h")
+
+# Combine all proto sources
+set(ALL_PROTO_SRCS ${PROTO_SRCS} ${PROTO_EXT_SRCS})
+
+# Include directories
+include_directories(${JNI_INCLUDE_DIRS})
+include_directories(${Protobuf_INCLUDE_DIRS})
+include_directories(${GENERATED_DIR})
+include_directories(src)
+
+# Add subdirectories
+add_subdirectory(src)

cpp/README.md

@@ -0,0 +1,112 @@
+# OpenSearch SQL Native Engine POC
+
+This is a proof-of-concept (POC) for a Java+C++ hybrid project, demonstrating how to integrate C++ native code into the OpenSearch SQL project.
+
+## Project Structure
+
+```
+cpp/
+├── CMakeLists.txt              # Main CMake configuration file
+├── src/
+│   ├── CMakeLists.txt         # Source CMake configuration
+│   ├── core/
+│   │   ├── Calculator.h       # C++ core calculation class header
+│   │   └── Calculator.cpp     # C++ core calculation class implementation
+│   └── jni/
+│       └── NativeEngine.cpp   # JNI wrapper implementation
+└── build/                     # CMake build directory (auto-generated)
+
+native/
+└── src/main/java/org/opensearch/sql/native/
+    ├── NativeEngine.java      # Java JNI wrapper
+    └── NativeEngineDemo.java  # Demo program
+```
+
+## Features
+
+### 1. Java Calling C++
+- `nativeAdd(int a, int b)` - Addition operation implemented in C++
+- `nativeMultiply(int a, int b)` - Multiplication operation implemented in C++
+
+### 2. C++ Calling Java
+- `nativeCallJava()` - C++ calls Java callback method
+- `onNativeCallback(String message)` - Java callback method
+
+## Build Requirements
+
+- Java 11+
+- CMake 3.16+
+- C++17 compatible compiler (GCC/Clang/MSVC)
+- Make (Linux/macOS) or Visual Studio (Windows)
+
+## Quick Start
+
+### 1. Build Project
+```bash
+# Execute in project root directory
+./gradlew :native:build
+```
+
+### 2. Run Demo
+```bash
+./gradlew :native:run
+```
+
+### 3. Run Tests
+```bash
+./gradlew :native:test
+```
+
+### 4. One-click Build and Run
+```bash
+./build-and-run-demo.sh
+```
+
+## Build Process
+
+1. **C++ Compilation**: Gradle calls CMake to compile C++ code and generate dynamic library
+2. **Library Copy**: Copy generated dynamic library to Java project's libs directory
+3. **Java Compilation**: Compile Java code including JNI interfaces
+4. **Runtime Loading**: Java runtime loads C++ library via `System.loadLibrary()`
+
+## Extension Guide
+
+### Adding New C++ Features
+1. Add new C++ classes in `cpp/src/core/`
+2. Add JNI wrapper functions in `cpp/src/jni/NativeEngine.cpp`
+3. Declare corresponding native methods in `native/src/main/java/.../NativeEngine.java`
+
+### JNI Method Naming Convention
+```cpp
+JNIEXPORT <return_type> JNICALL 
+Java_<package_name_with_underscores>_<class_name>_<method_name>(JNIEnv *env, ...)
+```
+
+## Notes
+
+- Ensure JNI method signatures match Java declarations exactly
+- Pay attention to memory management to avoid memory leaks
+- Exception handling needs to be considered on both C++ and Java sides
+- Cross-platform compatibility needs to be configured in CMakeLists.txt
+
+## Troubleshooting
+
+### Common Issues
+
+1. **UnsatisfiedLinkError**: 
+   - Check if dynamic library is correctly generated and copied
+   - Confirm `java.library.path` is set correctly
+
+2. **CMake cannot find JNI**:
+   - Ensure JAVA_HOME environment variable is set correctly
+   - Check if JDK installation is complete
+
+3. **Compilation errors**:
+   - Confirm C++ compiler supports C++17
+   - Check if CMake version meets requirements
+
+## Performance Considerations
+
+- JNI calls have overhead, suitable for compute-intensive tasks
+- Avoid frequent small data JNI calls
+- Consider batch processing to reduce JNI boundary crossings

cpp/proto/substrait

@@ -0,0 +1 @@
+../../native/src/main/resources/proto/substrait

cpp/src/CMakeLists.txt

@@ -0,0 +1,16 @@
+# Create JNI library
+add_library(opensearch-sql-native SHARED
+    jni/NativeEngine.cpp
+    core/Calculator.cpp
+    velox/SubstraitPlanProcessor.cpp
+    ${ALL_PROTO_SRCS}
+)
+
+# Link JNI libraries and protobuf
+target_link_libraries(opensearch-sql-native ${JNI_LIBRARIES} ${Protobuf_LIBRARIES})
+
+# Set library output name
+set_target_properties(opensearch-sql-native PROPERTIES
+    OUTPUT_NAME "opensearch-sql-native"
+    PREFIX ""
+)

cpp/src/core/Calculator.cpp

@@ -0,0 +1,30 @@
+#include "Calculator.h"
+#include <jni.h>
+#include <iostream>
+
+int Calculator::add(int a, int b) {
+    return a + b;
+}
+
+int Calculator::multiply(int a, int b) {
+    return a * b;
+}
+
+void Calculator::callJavaMethod(void* env, void* obj) {
+    JNIEnv* jenv = static_cast<JNIEnv*>(env);
+    jobject jobj = static_cast<jobject>(obj);
+    
+    jclass clazz = jenv->GetObjectClass(jobj);
+    
+    jmethodID methodId = jenv->GetMethodID(clazz, "onNativeCallback", "(Ljava/lang/String;)V");
+    
+    if (methodId != nullptr) {
+        jstring message = jenv->NewStringUTF("Hello from C++!");
+        
+        jenv->CallVoidMethod(jobj, methodId, message);
+        
+        jenv->DeleteLocalRef(message);
+    }
+    
+    jenv->DeleteLocalRef(clazz);
+}

cpp/src/core/Calculator.h

@@ -0,0 +1,8 @@
+#pragma once
+
+class Calculator {
+public:
+    static int add(int a, int b);
+    static int multiply(int a, int b);
+    static void callJavaMethod(void* env, void* obj);
+};