[PECOBLR-1121] Arrow patch to circumvent Arrow issues with JDk 16+ (databricks#1243)

Databricks server shares query results in Arrow format for easy cross
language functionality. The JDBC driver experiences compatibility issues
with JDK 16 and later versions when processing Arrow results.

This problem arises from stricter encapsulation of internal APIs in
newer Java versions, which affects the driver's use of the Apache Arrow
result format consumption with the Apache Arrow library. The JDBC driver
is used in partner solutions, where they do not have control of the
runtime environment, and the workaround of setting JVM arguments is not
feasible.

This PR patches some of the Arrow code to provide alternative JVM Heap
based byte allocators that do not use native `MemoryUtil` based direct
reads from off-heap memory. This implementation uses the native Arrow
code path if feasible, else falls back to the patched code.

All the code has been tested for read compatibility with all Arrow
types, latency benchmarks have been tested, and automated tests have
been added as well.

During the course of this change it became necessary to also convert the
project into a multi-module maven project

Author: tejassp-db

.github/workflows/bugCatcher.yml

@@ -30,7 +30,7 @@ jobs:
           restore-keys: ${{ runner.os }}-m2
 
       - name: Run Integration Tests
-        run: mvn -B test -Dtest=*e2e/OAuthTests*
+        run: mvn -pl jdbc-core -B test -Dtest='*e2e/OAuthTests*'
         env:
           DATABRICKS_HOST: ${{ secrets.JDBC_PAT_TEST_HOST_NAME }}
           DATABRICKS_HTTP_PATH: ${{ secrets.JDBC_PAT_TEST_HTTP_PATH }}

.github/workflows/concurrencyExecutionTests.yml

@@ -48,7 +48,7 @@ jobs:
           restore-keys: ${{ runner.os }}-m2
 
       - name: Run Concurrency Execution Tests
-        run: mvn -B test -Dtest=com.databricks.jdbc.integration.e2e.ConcurrentExecutionTests -DargLine="-ea"
+        run: mvn -pl jdbc-core -B test -Dtest=com.databricks.jdbc.integration.e2e.ConcurrentExecutionTests -DargLine="-ea"
         env:
           DATABRICKS_TOKEN: ${{ secrets.JDBC_PAT_TEST_TOKEN }}
           DATABRICKS_USER: ${{ secrets.DATABRICKS_USER }}

.github/workflows/coverageReport.yml

@@ -34,7 +34,7 @@ jobs:
           ${{ runner.os }}-m2-
         
     - name: Run tests with coverage
-      run: mvn clean test jacoco:report
+      run: mvn -pl jdbc-core clean test -Dgroups='!Jvm17PlusAndArrowToNioReflectionDisabled' jacoco:report
 
     - name: Check for coverage override
       id: override
@@ -53,7 +53,7 @@ jobs:
     - name: Check coverage percentage
       if: steps.override.outputs.override == 'false'
       run: |
-        COVERAGE_FILE="target/site/jacoco/jacoco.xml"
+        COVERAGE_FILE="jdbc-core/target/site/jacoco/jacoco.xml"
         if [ ! -f "$COVERAGE_FILE" ]; then
           echo "ERROR: Coverage file not found at $COVERAGE_FILE"
           exit 1

.github/workflows/loggingTesting.yml

@@ -45,13 +45,13 @@ jobs:
       - name: Find JAR file
         shell: bash
         run: |
-          # Find the main JAR file dynamically (fat JAR, not thin, not tests)
-          MAIN_JAR=$(find target -maxdepth 1 -name "databricks-jdbc-*.jar" \
+          # Find the main JAR file dynamically (uber JAR from assembly-uber module)
+          MAIN_JAR=$(find assembly-uber/target -maxdepth 1 -name "databricks-jdbc-*.jar" \
             -not -name "*-thin.jar" \
             -not -name "*-tests.jar" | head -1)
           if [ -z "$MAIN_JAR" ]; then
-            echo "ERROR: Could not find main JAR file in target directory"
-            ls -la target/
+            echo "ERROR: Could not find main JAR file in assembly-uber/target directory"
+            ls -la assembly-uber/target/
             exit 1
           fi
           echo "Using JAR file: $MAIN_JAR"
@@ -88,18 +88,18 @@ jobs:
       - name: Clean & Compile LoggingTest
         shell: bash
         run: |
-          rm -rf target/test-classes
-          mkdir -p target/test-classes
+          rm -rf jdbc-core/target/test-classes
+          mkdir -p jdbc-core/target/test-classes
           
           echo "Using JAR file: $MAIN_JAR"
           
           javac \
             -cp "$MAIN_JAR" \
-            -d target/test-classes \
-            src/test/java/com/databricks/client/jdbc/LoggingTest.java
+            -d jdbc-core/target/test-classes \
+            jdbc-core/src/test/java/com/databricks/client/jdbc/LoggingTest.java
           
           echo "==== Checking compiled classes ===="
-          find target/test-classes -type f
+          find jdbc-core/target/test-classes -type f
 
       - name: Run LoggingTest
         shell: bash
@@ -110,7 +110,7 @@ jobs:
           echo "Using classpath separator: '$SEP'"
           echo "Using JAR file: $MAIN_JAR"
           
-          CP="target/test-classes${SEP}$MAIN_JAR"
+          CP="jdbc-core/target/test-classes${SEP}$MAIN_JAR"
           
           java \
             --add-opens=java.base/java.nio=ALL-UNNAMED \

.github/workflows/prCheck.yml

@@ -76,21 +76,51 @@ jobs:
           key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
           restore-keys: ${{ runner.os }}-m2
 
+      - name: Set up Maven Toolchains
+        shell: bash
+        run: |
+          mkdir -p ~/.m2
+          cat > ~/.m2/toolchains.xml <<EOF
+          <?xml version="1.0" encoding="UTF-8"?>
+          <toolchains>
+            <toolchain>
+              <type>jdk</type>
+              <provides>
+                <version>${{ matrix.java-version }}</version>
+              </provides>
+              <configuration>
+                <jdkHome>$JAVA_HOME</jdkHome>
+              </configuration>
+            </toolchain>
+          </toolchains>
+          EOF
+
+      - name: Check Arrow Patch Tests
+        shell: bash
+        if: matrix.java-version >= 17
+        run: mvn -Pjdk${{ matrix.java-version }}-NioNotOpen -pl jdbc-core test -Dgroups='Jvm17PlusAndArrowToNioReflectionDisabled'
+
+      - name: Check Arrow Allocator Manager Tests
+        shell: bash
+        if: matrix.java-version >= 17
+        run: mvn -Pjdk${{ matrix.java-version }}-NioNotOpen -pl jdbc-core test -Dgroups='Jvm17PlusAndArrowToNioReflectionDisabled' -Dtest="ArrowBufferAllocatorNettyManagerTest,ArrowBufferAllocatorUnsafeManagerTest,ArrowBufferAllocatorUnknownManagerTest" -DforkCount=1 -DreuseForks=false
+
+      - name: Check Arrow Memory Tests
+        shell: bash
+        run: mvn -Plow-memory -pl jdbc-core test -Dtest='DatabricksArrowPatchMemoryUsageTest'
+
       - name: Check Unit Tests
         shell: bash
-        run: mvn test -Dtest='!**/integration/**,!**/DatabricksDriverExamples.java,!**/ProxyTest.java,!**/LoggingTest.java,!**/SSLTest.java'
+        run: mvn -pl jdbc-core clean test -Dgroups='!Jvm17PlusAndArrowToNioReflectionDisabled' jacoco:report
 
       - name: Install xmllint
         if: runner.os == 'Linux'
         run: sudo apt-get update && sudo apt-get install -y libxml2-utils
 
-      - name: JaCoCo report
-        run: mvn --batch-mode --errors jacoco:report --file pom.xml
-
       - name: Extract codeCov percentage
         shell: bash
         run: |
-          COVERAGE_FILE="target/site/jacoco/jacoco.xml"
+          COVERAGE_FILE="jdbc-core/target/site/jacoco/jacoco.xml"
           COVERED=$(xmllint --xpath "string(//report/counter[@type='INSTRUCTION']/@covered)" "$COVERAGE_FILE")
           MISSED=$(xmllint --xpath "string(//report/counter[@type='INSTRUCTION']/@missed)" "$COVERAGE_FILE")
           TOTAL=$((COVERED + MISSED))
@@ -114,4 +144,80 @@ jobs:
             exit 1
           else
             echo "Coverage is equal to or greater than 85%"
-          fi
+          fi
+
+  packaging-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        java-version: [ 17 ]
+        github-runner: [ linux-ubuntu-latest, windows-server-latest ]
+
+    runs-on:
+      group: databricks-protected-runner-group
+      labels: ${{ matrix.github-runner }}
+
+    steps:
+      - name: Set up JDK ${{ matrix.java-version }}
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: 'adopt'
+
+      - name: Enable long paths
+        if: runner.os == 'Windows'
+        run: git config --system core.longpaths true
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
+          repository: ${{ github.event.pull_request.head.repo.full_name ||  github.repository }}
+
+      - name: Cache Maven packages
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2
+          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          restore-keys: ${{ runner.os }}-m2
+
+      - name: Set up Maven Toolchains
+        shell: bash
+        run: |
+          mkdir -p ~/.m2
+          cat > ~/.m2/toolchains.xml <<EOF
+          <?xml version="1.0" encoding="UTF-8"?>
+          <toolchains>
+            <toolchain>
+              <type>jdk</type>
+              <provides>
+                <version>${{ matrix.java-version }}</version>
+              </provides>
+              <configuration>
+                <jdkHome>$JAVA_HOME</jdkHome>
+              </configuration>
+            </toolchain>
+          </toolchains>
+          EOF
+
+      - name: Install JDBC artifacts into maven local
+        shell: bash
+        run: mvn -B -pl jdbc-core,assembly-uber,assembly-thin install -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true -Ddependency-check.skip=true
+
+      - name: Check Uber Jar Packaging
+        shell: bash
+        run: mvn -pl test-assembly-uber test
+        env:
+          DATABRICKS_HOST: ${{ secrets.JDBC_PAT_TEST_HOST_NAME }}
+          DATABRICKS_HTTP_PATH: ${{ secrets.JDBC_PAT_TEST_HTTP_PATH }}
+          DATABRICKS_USER: ${{ secrets.DATABRICKS_USER }}
+          DATABRICKS_TOKEN: ${{ secrets.JDBC_PAT_TEST_TOKEN }}
+
+      - name: Check Thin Jar Packaging
+        shell: bash
+        run:  mvn -pl test-assembly-thin test
+        env:
+          DATABRICKS_HOST: ${{ secrets.JDBC_PAT_TEST_HOST_NAME }}
+          DATABRICKS_HTTP_PATH: ${{ secrets.JDBC_PAT_TEST_HTTP_PATH }}
+          DATABRICKS_USER: ${{ secrets.DATABRICKS_USER }}
+          DATABRICKS_TOKEN: ${{ secrets.JDBC_PAT_TEST_TOKEN }}

.github/workflows/prCheckJDK8.yml

@@ -48,4 +48,8 @@ jobs:
           restore-keys: ${{ runner.os }}-jdk8-m2
 
       - name: Run Unit Tests
-        run: mvn clean test
+        run: mvn -pl jdbc-core clean test -Dgroups='!Jvm17PlusAndArrowToNioReflectionDisabled'
+
+      - name: Check Arrow Memory Tests
+        shell: bash
+        run: mvn -Plow-memory -pl jdbc-core test -Dtest='DatabricksArrowPatchMemoryUsageTest'

.github/workflows/prIntegrationTests.yml

@@ -16,10 +16,10 @@ jobs:
         include:
           # SQL_EXEC mode: Tests SEA client behavior
           # Note: CircuitBreakerIntegrationTests requires THRIFT_SERVER mode (tested in second matrix entry)
-          - test-command: mvn -B compile test -Dtest=*IntegrationTests,!M2MPrivateKeyCredentialsIntegrationTests,!M2MAuthIntegrationTests,!CircuitBreakerIntegrationTests,!ThriftCloudFetchFakeIntegrationTests
+          - test-command: mvn -pl jdbc-core -B compile test -Dtest=*IntegrationTests,!M2MPrivateKeyCredentialsIntegrationTests,!M2MAuthIntegrationTests,!CircuitBreakerIntegrationTests,!ThriftCloudFetchFakeIntegrationTests
             fake-service-type: 'SQL_EXEC'
           # THRIFT_SERVER mode: Tests Thrift client behavior and circuit breaker fallback
-          - test-command: mvn -B compile test -Dtest=*IntegrationTests,!M2MPrivateKeyCredentialsIntegrationTests,!SqlExecApiHybridResultsIntegrationTests,!DBFSVolumeIntegrationTests,!M2MAuthIntegrationTests,!UCVolumeIntegrationTests,!SqlExecApiIntegrationTests
+          - test-command: mvn -pl jdbc-core -B compile test -Dtest=*IntegrationTests,!M2MPrivateKeyCredentialsIntegrationTests,!SqlExecApiHybridResultsIntegrationTests,!DBFSVolumeIntegrationTests,!M2MAuthIntegrationTests,!UCVolumeIntegrationTests,!SqlExecApiIntegrationTests
             fake-service-type: 'THRIFT_SERVER'
     steps:
       - name: Checkout PR

.github/workflows/proxyTesting.yml

@@ -157,7 +157,7 @@ jobs:
       ################################################################
       - name: Run ProxyTest
         run: |
-          mvn test -Dtest=**/ProxyTest.java
+          mvn -pl jdbc-core test -Dtest=**/ProxyTest.java
 
       ################################################################
       # 14) Cleanup

.github/workflows/release-thin.yml

@@ -31,111 +31,37 @@ jobs:
 
       - name: Set up Java for publishing to Maven Central Repository
         uses: actions/setup-java@v4
+        env:
+          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
         with:
           java-version: 11
-          distribution: "adopt"
           server-id: central
+          distribution: "adopt"
           server-username: MAVEN_CENTRAL_USERNAME
           server-password: MAVEN_CENTRAL_PASSWORD
           gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
           gpg-passphrase: GPG_PASSPHRASE
 
-      - name: Configure GPG
+      # Step 1: Build and install dependencies to local Maven repository
+      # This builds jdbc-core (and parent) without publishing them.
+      # The -am flag builds all dependencies needed by assembly-thin.
+      # We use -Prelease here to generate sources/javadoc JARs for jdbc-core,
+      # which assembly-thin needs for its own sources/javadoc artifacts.
+      # GPG signing is skipped since we're only installing locally, not publishing.
+      - name: Build dependencies
         run: |
-          echo "allow-loopback-pinentry" >> ~/.gnupg/gpg-agent.conf
-          echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
-          gpg-connect-agent reloadagent /bye
+          mvn -Prelease clean install --batch-mode -pl jdbc-core -am -Dgpg.skip=true
 
-      - name: Build thin JAR with sources and javadocs
+      # Step 2: Deploy only the thin JAR module to Maven Central
+      # We don't use -am here to avoid the central-publishing-maven-plugin
+      # from collecting parent/jdbc-core artifacts into the deployment bundle.
+      # The jdbc-core dependency is already available from Step 1.
+      - name: Publish thin JAR to Maven Central
         run: |
-          # Build main artifacts including sources and javadocs
-          mvn -B -DskipTests package source:jar javadoc:jar
-
-      - name: Sign all thin JAR artifacts
-        run: |
-          VERSION=$(grep -m1 '<version>' pom.xml | sed 's/.*<version>\(.*\)<\/version>.*/\1/')
-
-          # Sign thin JAR
-          echo "$GPG_PASSPHRASE" | gpg --batch --yes --passphrase-fd 0 --pinentry-mode loopback \
-            --armor --detach-sign "target/databricks-jdbc-${VERSION}-thin.jar"
-
-          # Sign sources JAR
-          echo "$GPG_PASSPHRASE" | gpg --batch --yes --passphrase-fd 0 --pinentry-mode loopback \
-            --armor --detach-sign "target/databricks-jdbc-${VERSION}-sources.jar"
-
-          # Sign javadoc JAR
-          echo "$GPG_PASSPHRASE" | gpg --batch --yes --passphrase-fd 0 --pinentry-mode loopback \
-            --armor --detach-sign "target/databricks-jdbc-${VERSION}-javadoc.jar"
-        env:
-          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
-
-      - name: Verify all required artifacts exist
-        run: |
-          VERSION=$(grep -m1 '<version>' pom.xml | sed 's/.*<version>\(.*\)<\/version>.*/\1/')
-          test -f "target/databricks-jdbc-${VERSION}-thin.jar"
-          test -f "target/databricks-jdbc-${VERSION}-thin.jar.asc"
-          test -f "target/databricks-jdbc-${VERSION}-sources.jar"
-          test -f "target/databricks-jdbc-${VERSION}-sources.jar.asc"
-          test -f "target/databricks-jdbc-${VERSION}-javadoc.jar"
-          test -f "target/databricks-jdbc-${VERSION}-javadoc.jar.asc"
-
-      - name: Publish Thin JAR as Separate Artifact to Maven Central
-        run: |
-          VERSION=$(grep -m1 '<version>' pom.xml | sed 's/.*<version>\(.*\)<\/version>.*/\1/')
-
-          echo "Creating deployment bundle for thin JAR..."
-
-          # Create staging directory
-          mkdir -p target/thin-staging/com/databricks/databricks-jdbc-thin/${VERSION}
-
-          # Copy thin JAR and its signature
-          cp "target/databricks-jdbc-${VERSION}-thin.jar" \
-            target/thin-staging/com/databricks/databricks-jdbc-thin/${VERSION}/databricks-jdbc-thin-${VERSION}.jar
-          cp "target/databricks-jdbc-${VERSION}-thin.jar.asc" \
-            target/thin-staging/com/databricks/databricks-jdbc-thin/${VERSION}/databricks-jdbc-thin-${VERSION}.jar.asc
-
-          # Copy sources JAR and its signature
-          cp "target/databricks-jdbc-${VERSION}-sources.jar" \
-            target/thin-staging/com/databricks/databricks-jdbc-thin/${VERSION}/databricks-jdbc-thin-${VERSION}-sources.jar
-          cp "target/databricks-jdbc-${VERSION}-sources.jar.asc" \
-            target/thin-staging/com/databricks/databricks-jdbc-thin/${VERSION}/databricks-jdbc-thin-${VERSION}-sources.jar.asc
-
-          # Copy javadoc JAR and its signature
-          cp "target/databricks-jdbc-${VERSION}-javadoc.jar" \
-            target/thin-staging/com/databricks/databricks-jdbc-thin/${VERSION}/databricks-jdbc-thin-${VERSION}-javadoc.jar
-          cp "target/databricks-jdbc-${VERSION}-javadoc.jar.asc" \
-            target/thin-staging/com/databricks/databricks-jdbc-thin/${VERSION}/databricks-jdbc-thin-${VERSION}-javadoc.jar.asc
-
-          # Copy POM and sign it
-          cp thin_public_pom.xml target/thin-staging/com/databricks/databricks-jdbc-thin/${VERSION}/databricks-jdbc-thin-${VERSION}.pom
-          echo "$GPG_PASSPHRASE" | gpg --batch --yes --passphrase-fd 0 --pinentry-mode loopback \
-            --armor --detach-sign \
-            target/thin-staging/com/databricks/databricks-jdbc-thin/${VERSION}/databricks-jdbc-thin-${VERSION}.pom
-
-          # Generate checksums for all files
-          cd target/thin-staging/com/databricks/databricks-jdbc-thin/${VERSION}
-          for file in databricks-jdbc-thin-*; do
-            md5sum "$file" | awk '{print $1}' > "${file}.md5"
-            sha1sum "$file" | awk '{print $1}' > "${file}.sha1"
-          done
-          cd $GITHUB_WORKSPACE
-
-          # Create bundle ZIP
-          cd target/thin-staging
-          zip -r ../central-thin-bundle.zip com/
-          cd $GITHUB_WORKSPACE
-
-          echo "Uploading bundle to Maven Central Portal..."
-
-          # Upload to new Maven Central Portal
-          curl -X POST \
-            -u "$MAVEN_CENTRAL_USERNAME:$MAVEN_CENTRAL_PASSWORD" \
-            -F "bundle=@target/central-thin-bundle.zip" \
-            -F "publishingType=AUTOMATIC" \
-            -w "\nHTTP_STATUS:%{http_code}\n" \
-            https://central.sonatype.com/api/v1/publisher/upload
-
-          echo "Thin JAR published successfully!"
+          mvn -Prelease deploy --batch-mode -pl assembly-thin \
+                      -Dnvd.api.key=${{ secrets.NVD_API_KEY }} \
+                      -Dossindex.username=${{ secrets.OSSINDEX_USERNAME }} \
+                      -Dossindex.password=${{ secrets.OSSINDEX_PASSWORD }}
         env:
           GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
           MAVEN_CENTRAL_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }}
@@ -174,5 +100,4 @@ jobs:
         with:
           tag_name: ${{ steps.get_tag.outputs.tag }}
           files: |
-            target/*-thin.jar
-
+            assembly-thin/target/databricks-jdbc-thin-*.jar