Merge branch 'main' into dependabot/go_modules/go/adbc/google.golang.org/api-0.228.0

Author: zeroshade

.github/workflows/java.yml

@@ -46,7 +46,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        java: ['11', '17', '21', '22']
+        java: ['11', '17', '21', '24']
     steps:
       - uses: actions/checkout@v4
         with:
@@ -102,3 +102,118 @@ jobs:
           HERE
           # Our linters are compile-time, no point re-running tests
           mvn -P errorprone install -DskipTests
+
+  java-jni-artifacts:
+    name: "JNI Libraries/${{ matrix.os }} ${{ matrix.arch }}"
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        include:
+          - { os: Linux, arch: amd64, vcpkg_arch: x64, runner: ubuntu-latest }
+          - { os: macOS, arch: arm64v8, vcpkg_arch: arm64, runner: macos-latest }
+    env:
+      VCPKG_ROOT: "${{ github.workspace }}/vcpkg"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: actions/setup-java@v4
+        with:
+          cache: "maven"
+          distribution: "temurin"
+          java-version: 11
+      - name: Retrieve Go, VCPKG version from .env
+        run: |
+          (. .env && echo "GO_VERSION=${GO}") >> $GITHUB_ENV
+          (. .env && echo "VCPKG_VERSION=${VCPKG}") >> $GITHUB_ENV
+      - name: Install vcpkg
+        run: |
+          ./ci/scripts/install_vcpkg.sh $VCPKG_ROOT $VCPKG_VERSION
+      - uses: actions/setup-go@v5
+        with:
+          go-version: "${{ env.GO_VERSION }}"
+          check-latest: true
+          cache: true
+          cache-dependency-path: adbc/go/adbc/go.sum
+      - name: Install Homebrew dependencies
+        if: matrix.os == 'macOS'
+        run: brew install autoconf bash pkg-config ninja
+      - name: Build artifacts
+        if: matrix.os != 'macOS'
+        run: |
+          docker compose run python-wheel-manylinux-build
+      - name: Build artifacts (macOS)
+        if: matrix.os == 'macOS'
+        run: |
+          export ADBC_BUILD_STATIC=ON
+          export ADBC_BUILD_TESTS=OFF
+          export ADBC_USE_ASAN=OFF
+          export ADBC_USE_UBSAN=OFF
+          ./ci/scripts/python_wheel_unix_build.sh ${{ matrix.arch }} $(pwd) $(pwd)/build
+      - name: Build JNI artifacts
+        run: |
+          export ADBC_BUILD_STATIC=ON
+          export ADBC_BUILD_TESTS=OFF
+          export ADBC_USE_ASAN=OFF
+          export ADBC_USE_UBSAN=OFF
+          ./ci/scripts/java_build.sh $(pwd)
+          ./ci/scripts/java_jni_build.sh $(pwd) $(pwd)/build_jni $(pwd)/build/${{ matrix.vcpkg_arch }}
+      - name: Assemble artifacts
+        run: |
+          mkdir artifacts
+          cp -r java/driver/jni/src/main/resources/ artifacts/jni
+          cp -r build/${{ matrix.vcpkg_arch }}/lib artifacts/driver
+          ls -laR artifacts
+          mv artifacts artifacts-${{ matrix.os }}-${{ matrix.arch }}
+          tar czf artifacts-${{ matrix.os }}-${{ matrix.arch }}.tgz artifacts-${{ matrix.os }}-${{ matrix.arch }}
+      - uses: actions/upload-artifact@v4
+        with:
+          name: jni-artifacts-${{ matrix.os }}-${{ matrix.arch }}
+          retention-days: 7
+          path: |
+            artifacts-${{ matrix.os }}-${{ matrix.arch }}.tgz
+
+  java-jni:
+    name: "Java ${{ matrix.java }} JNI/${{ matrix.os }} ${{ matrix.arch }}"
+    runs-on: ${{ matrix.runner }}
+    needs:
+      - java-jni-artifacts
+    strategy:
+      matrix:
+        java: ['11', '24']
+        include:
+          - { os: Linux, arch: amd64, vcpkg_arch: x64, runner: ubuntu-latest }
+          - { os: macOS, arch: arm64v8, vcpkg_arch: arm64, runner: macos-latest }
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+          submodules: recursive
+      - uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+          pattern: jni-artifacts-*
+          merge-multiple: true
+      - uses: actions/setup-java@v4
+        with:
+          cache: "maven"
+          distribution: "temurin"
+          java-version: 11
+      - name: Build/Test
+        run: |
+          set -x
+          pushd artifacts
+          for archive in artifacts*.tgz; do
+            tar xvf $archive
+          done
+          popd
+          cp -r artifacts/*/jni/adbc_driver_jni java/driver/jni/src/main/resources
+          for driver in artifacts/*/driver; do
+            export LD_LIBRARY_PATH=$(pwd)/$driver:${LD_LIBRARY_PATH:-}
+            export DYLD_LIBRARY_PATH=$(pwd)/$driver:${DYLD_LIBRARY_PATH:-}
+          done
+          env BUILD_JNI=ON ./ci/scripts/java_build.sh $(pwd)
+          cd java
+          mvn -B -Pjni test -pl :adbc-driver-jni

.github/workflows/packaging.yml

@@ -99,7 +99,6 @@ jobs:
     strategy:
       matrix:
         os: ["ubuntu-latest", "windows-latest", "macos-13"]
-        go-version: [1.21.8] # Customize Go versions as needed
 
     steps:
       - uses: actions/download-artifact@v4
@@ -116,10 +115,13 @@ jobs:
           tar xf apache-arrow-adbc-${VERSION}.tar.gz
           mv apache-arrow-adbc-${VERSION} adbc
 
+      - name: Get required Go version
+        run: |
+          (. adbc/.env && echo "GO_VERSION=${GO}") >> $GITHUB_ENV
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: ${{ matrix.go-version }}
+          go-version: "${{ env.GO_VERSION }}"
           check-latest: true
           cache: true
           cache-dependency-path: adbc/go/adbc/go.sum
@@ -342,7 +344,7 @@ jobs:
           mv apache-arrow-adbc-$VERSION.tar.gz adbc/ci/linux-packages/
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@1a615958ad9d422dd932dc1d5823942ee002799f  # v1.227.0
+        uses: ruby/setup-ruby@e5ac7b085f6e63d49c8973eb0c6e04d876b881f1  # v1.230.0
         with:
           ruby-version: ruby
 
@@ -721,9 +723,13 @@ jobs:
           ci/scripts/install_vcpkg.sh $VCPKG_ROOT $VCPKG_VERSION
           popd
 
+      - name: Get required Go version
+        run: |
+          (. adbc/.env && echo "GO_VERSION=${GO}") >> $GITHUB_ENV
+
       - uses: actions/setup-go@v5
         with:
-          go-version: 1.21.8
+          go-version: "${{ env.GO_VERSION }}"
           check-latest: true
           cache: true
           cache-dependency-path: adbc/go/adbc/go.sum

.github/workflows/r-basic.yml

@@ -38,15 +38,29 @@ permissions:
   contents: read
 
 jobs:
-  check:
+  check-c:
     strategy:
       matrix:
         os: [ubuntu]
-        pkg: [adbcdrivermanager, adbcsqlite, adbcpostgresql, adbcflightsql, adbcsnowflake]
+        pkg: [adbcdrivermanager, adbcsqlite, adbcpostgresql]
 
     uses: ./.github/workflows/r-check.yml
     with:
       os: ${{ matrix.os }}
       pkg: ${{ matrix.pkg }}
+
+  check-go:
+    strategy:
+      matrix:
+        os: [ubuntu]
+        pkg: [adbcflightsql, adbcsnowflake, adbcbigquery]
+
+    uses: ./.github/workflows/r-check.yml
+    with:
+      os: ${{ matrix.os }}
+      pkg: ${{ matrix.pkg }}
+      # For the r-basic check (that runs on many PRs), just check ERRORs
+      # (e.g., build failure, test failure)
+      error-on: error
     secrets:
       SNOWFLAKE_URI: ${{ secrets.SNOWFLAKE_URI }}

.github/workflows/r-check.yml

@@ -31,6 +31,10 @@ on:
         required: false
         default: release
         type: string
+      error-on:
+        required: false
+        default: warning
+        type: string
     secrets:
       SNOWFLAKE_URI:
         required: false
@@ -100,6 +104,7 @@ jobs:
           R_KEEP_PKG_SOURCE: yes
         with:
           working-directory: r/${{ inputs.pkg }}
+          error-on: '"${{ inputs.error-on }}"'
 
       - name: Shutdown docker compose services
         if: runner.os == 'Linux'

.github/workflows/r-extended.yml

@@ -39,48 +39,65 @@ permissions:
 jobs:
 
   # Runs R CMD check on the same platforms/R versions CRAN does
-  cran:
+  cran-c:
     strategy:
       matrix:
         rversion: [oldrel, release, devel]
         os: [macOS, windows, ubuntu]
-        pkg: [adbcdrivermanager, adbcsqlite, adbcpostgresql, adbcflightsql, adbcsnowflake, adbcbigquery]
+        pkg: [adbcdrivermanager, adbcsqlite, adbcpostgresql]
+      fail-fast: false
+
+    uses: ./.github/workflows/r-check.yml
+    with:
+      os: ${{ matrix.os }}
+      pkg: ${{ matrix.pkg }}
+      rversion: ${{ matrix.rversion }}
+
+  # Go-based drivers are not distributed on CRAN and are checked with error-on error only
+  cran-go:
+    strategy:
+      matrix:
+        rversion: [oldrel, release, devel]
+        os: [macOS, windows, ubuntu]
+        pkg: [adbcflightsql, adbcsnowflake, adbcbigquery]
       fail-fast: false
 
     uses: ./.github/workflows/r-check.yml
     with:
       os: ${{ matrix.os }}
       pkg: ${{ matrix.pkg }}
       rversion: ${{ matrix.rversion }}
+      # Go based drivers generate WARNINGs because some symbols
+      # are linked in that R CMD check does not allow.
+      error-on: error
     secrets:
       SNOWFLAKE_URI: ${{ secrets.SNOWFLAKE_URI }}
 
   # Check older versions of R on Linux. This catches accidental use of newer R functions.
+  # We don't check Go drivers here because they don't have much R API surface area.
   rversions:
     strategy:
       matrix:
         rversion: ["3.6", "4.0", "4.1"]
         os: [ubuntu]
-        pkg: [adbcdrivermanager, adbcsqlite, adbcpostgresql, adbcflightsql, adbcsnowflake, adbcbigquery]
+        pkg: [adbcdrivermanager, adbcsqlite, adbcpostgresql]
       fail-fast: false
 
     uses: ./.github/workflows/r-check.yml
     with:
       os: ${{ matrix.os }}
       pkg: ${{ matrix.pkg }}
       rversion: ${{ matrix.rversion }}
-    secrets:
-      SNOWFLAKE_URI: ${{ secrets.SNOWFLAKE_URI }}
 
   # Checks on older verions of R on Windows. The Windows build system changed
   # several times so we need to check packages on every version. Go-based
   # drivers aren't supported before 4.2, so we don't check them here.
-  # We don't need to check R 4.1 because the build system for R 4.0 and R 4.1
-  # are the same.
+  # The test dependencies support the released version and the four previous
+  # versions (e.g., 4.1 is the minimum supported version as of R 4.5).
   winrversions:
     strategy:
       matrix:
-        rversion: ["3.6", "4.0"]
+        rversion: ["4.1"]
         os: [windows]
         pkg: [adbcdrivermanager, adbcsqlite, adbcpostgresql]
       fail-fast: false
@@ -90,8 +107,6 @@ jobs:
       os: ${{ matrix.os }}
       pkg: ${{ matrix.pkg }}
       rversion: ${{ matrix.rversion }}
-    secrets:
-      SNOWFLAKE_URI: ${{ secrets.SNOWFLAKE_URI }}
 
   # Runs tests with valgrind. Go does not support valgrind, so we don't run
   # those tests here.

.github/workflows/r-standard.yml

@@ -46,15 +46,30 @@ permissions:
   contents: read
 
 jobs:
-  check:
+  check-c:
     strategy:
       matrix:
         os: [ubuntu, macOS, windows]
-        pkg: [adbcdrivermanager, adbcsqlite, adbcpostgresql, adbcflightsql, adbcsnowflake, adbcbigquery]
+        pkg: [adbcdrivermanager, adbcsqlite, adbcpostgresql]
 
     uses: ./.github/workflows/r-check.yml
     with:
       os: ${{ matrix.os }}
       pkg: ${{ matrix.pkg }}
+
+  # Go-based drivers are not distributed on CRAN and are checked with error-on error only
+  check-go:
+    strategy:
+      matrix:
+        os: [ubuntu, macOS, windows]
+        pkg: [adbcflightsql, adbcsnowflake, adbcbigquery]
+
+    uses: ./.github/workflows/r-check.yml
+    with:
+      os: ${{ matrix.os }}
+      pkg: ${{ matrix.pkg }}
+      # Go based drivers generate WARNINGs because some symbols
+      # are linked in that R CMD check does not allow.
+      error-on: error
     secrets:
       SNOWFLAKE_URI: ${{ secrets.SNOWFLAKE_URI }}

CONTRIBUTING.md

@@ -316,6 +316,43 @@ mvn install -Perrorprone
 [checker-framework]: https://checkerframework.org/
 [errorprone]: https://errorprone.info/
 
+#### JNI
+
+To build the JNI bridge, the native components must be built.
+
+```
+# Build the driver manager
+export ADBC_BUILD_STATIC=ON
+export ADBC_BUILD_TESTS=OFF
+export ADBC_USE_ASAN=OFF
+export ADBC_USE_UBSAN=OFF
+export BUILD_ALL=OFF
+export BUILD_DRIVER_MANAGER=ON
+export BUILD_DRIVER_SQLITE=ON
+./ci/scripts/cpp_build.sh $(pwd) $(pwd)/build $(pwd)/local
+
+# Build the JNI libraries
+./ci/scripts/java_jni_build.sh $(pwd) $(pwd)/java/build $(pwd)/local
+```
+
+Now build the Java code with the `jni` Maven profile enabled.  To run tests,
+the SQLite driver must also be present in (DY)LD_LIBRARY_PATH.
+
+```
+export LD_LIBRARY_PATH=$(pwd)/local/lib
+pushd java
+mvn install -Pjni
+popd
+```
+
+This will build a JAR with native libraries for a single platform.  If the
+native libraries are built for multiple platforms, they can all be copied to
+appropriate paths in the resources directory to build a single JAR that works
+across multiple platforms.
+
+You can also build and test in IntelliJ; simply edit the run/test
+configuration to add `LD_LIBRARY_PATH` to the environment.
+
 ### Python
 
 Python libraries are managed with [setuptools][setuptools].  See