ci: verify source using OS-installed dependencies (#2718)

We've only really tested Conda so far.  Validate non-Conda builds too.

This only tests Ubuntu/APT but eventually we should also test
Debian/APT, macOS/Homebrew, Fedora/RPM, and AlmaLinux/RPM.

---------

Co-authored-by: Sutou Kouhei <kou@cozmixng.org>
Co-authored-by: Dewey Dunnington <dewey@dunnington.ca>

Author: 3 people

.github/workflows/nightly-verify.yml

@@ -39,6 +39,10 @@ defaults:
     # 'bash' will expand to -eo pipefail
     shell: bash
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   source:
     # For cron: only run on the main repo, not forks
@@ -201,3 +205,30 @@ jobs:
         run: |
           pushd arrow-adbc
           docker compose run -e PYTHON=3.12 --rm python-debug
+
+  source-verify-docker:
+    name: "Verify Source (OS)/${{ matrix.os }} ${{ matrix.version }}"
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 2
+      matrix:
+        include:
+          - os: ubuntu
+            version: "22.04"
+          - os: ubuntu
+            version: "24.04"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          path: arrow-adbc
+          persist-credentials: false
+          submodules: recursive
+
+      - name: Verify
+        env:
+          OS: ${{ matrix.os }}
+          OS_VERSION: ${{ matrix.version }}
+        run: |
+          pushd arrow-adbc
+          env ${OS@U}=${OS_VERSION} docker compose run --rm verify-all-${OS}

ci/scripts/verify_ubuntu.sh

@@ -0,0 +1,111 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Test the validation script using system dependencies instead of Conda.  This
+# script is meant for automation (e.g. Docker), not a local/developer machine
+# (except via Docker).
+
+set -euo pipefail
+
+main() {
+    local -r source_dir="${1}"
+
+    # Install all the dependencies we need for all the subprojects
+
+    # When installing tzdata, don't block and wait for user input
+    export DEBIAN_FRONTEND=noninteractive
+    export TZ=Etc/UTC
+
+    apt update
+    apt install -y \
+        apt-transport-https \
+        build-essential \
+        ca-certificates \
+        cmake \
+        curl \
+        dirmngr \
+        git \
+        gobject-introspection \
+        gpg \
+        libgirepository1.0-dev \
+        libglib2.0-dev \
+        libgmock-dev \
+        libgtest-dev \
+        libpq-dev \
+        libsqlite3-dev \
+        lsb-release \
+        ninja-build \
+        pkg-config \
+        protobuf-compiler \
+        python3 \
+        python3-dev \
+        python3-pip \
+        python3-venv \
+        r-base \
+        ruby-full \
+        software-properties-common \
+        wget
+
+    # Install Java
+
+    wget -qO - https://packages.adoptium.net/artifactory/api/gpg/key/public | \
+        gpg --dearmor | \
+        tee /etc/apt/trusted.gpg.d/adoptium.gpg > /dev/null
+
+    echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | \
+        tee /etc/apt/sources.list.d/adoptium.list
+
+    # Install Arrow GLib
+    wget https://packages.apache.org/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+    apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+
+    apt update
+    apt install -y \
+        libarrow-dev \
+        libarrow-glib-dev \
+        temurin-21-jdk
+
+    # Install Maven
+    wget https://dlcdn.apache.org/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz
+    mkdir -p /opt/maven
+    tar -C /opt/maven -xzvf apache-maven-3.9.9-bin.tar.gz --strip-components=1
+    export PATH=/opt/maven/bin:$PATH
+
+    # Check if protoc is too old (Ubuntu 22.04).  If so, install it from
+    # upstream instead.
+    if ! protoc --version | \
+            awk '{print $2}{print "3.15"}' | \
+            sort --version-sort | \
+            head -n1 | \
+            grep -E '^3\.15.*$' >/dev/null ; then
+        echo "protoc is too old"
+
+        wget -O protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v30.2/protoc-30.2-linux-x86_64.zip
+        unzip -o protoc.zip -d /usr/local -x readme.txt
+    fi
+
+    # We run under Docker and this is necessary since the source dir is
+    # typically mounted as a volume
+    git config --global --add safe.directory "${source_dir}"
+
+    "${source_dir}/dev/release/verify-release-candidate.sh"
+}
+
+main "$@"

compose.yaml

@@ -275,3 +275,12 @@ services:
       retries: 5
     ports:
       - "5432:5432"
+
+  ################################ Verification ################################
+
+  verify-all-ubuntu:
+    image: ubuntu:${UBUNTU}
+    volumes:
+      - .:/adbc:delegated
+    command: |
+      /adbc/ci/scripts/verify_ubuntu.sh /adbc

dev/release/verify-release-candidate.sh

@@ -246,38 +246,42 @@ install_dotnet() {
     return 0
   fi
 
-  show_info "Ensuring that .NET is installed..."
-
-  if dotnet --version | grep 8\.0 > /dev/null 2>&1; then
-    local csharp_bin=$(dirname $(which dotnet))
-    show_info "Found C# at $(which csharp) (.NET $(dotnet --version))"
-  else
-    if which dotnet > /dev/null 2>&1; then
-      show_info "dotnet found but it is the wrong version and will be ignored."
+  if command -v dotnet; then
+    show_info "Found $(dotnet --version) at $(which dotnet)"
+
+    if dotnet --version | grep --quiet --fixed-string 8.0; then
+        local csharp_bin=$(dirname $(which dotnet))
+        show_info "Found C# at $(which csharp) (.NET $(dotnet --version))"
+        DOTNET_ALREADY_INSTALLED=1
+        return 0
     fi
-    local csharp_bin=${ARROW_TMPDIR}/csharp/bin
-    local dotnet_version=8.0.204
-    local dotnet_platform=
-    case "$(uname)" in
+  fi
+
+  show_info "dotnet found but it is the wrong version or dotnet not found"
+
+  local csharp_bin=${ARROW_TMPDIR}/csharp/bin
+  local dotnet_version=8.0.204
+  local dotnet_platform=
+  case "$(uname)" in
       Linux)
-        dotnet_platform=linux
-        ;;
+          dotnet_platform=linux
+          ;;
       Darwin)
-        dotnet_platform=macos
-        ;;
-    esac
-    local dotnet_download_thank_you_url=https://dotnet.microsoft.com/download/thank-you/dotnet-sdk-${dotnet_version}-${dotnet_platform}-x64-binaries
-    local dotnet_download_url=$( \
-      curl -sL ${dotnet_download_thank_you_url} | \
-        grep 'directLink' | \
-        grep -E -o 'https://download[^"]+' | \
-        sed -n 2p)
-    mkdir -p ${csharp_bin}
-    curl -sL ${dotnet_download_url} | \
+          dotnet_platform=macos
+          ;;
+  esac
+  local dotnet_download_thank_you_url=https://dotnet.microsoft.com/download/thank-you/dotnet-sdk-${dotnet_version}-${dotnet_platform}-x64-binaries
+  show_info "Getting .NET download URL from ${dotnet_download_thank_you_url}"
+  local dotnet_download_url=$(curl -sL ${dotnet_download_thank_you_url} | \
+                                  grep 'directLink' | \
+                                  grep -E -o 'https://download[^"]+' | \
+                                  sed -n 2p)
+  show_info "Downloading .NET from ${dotnet_download_url}"
+  mkdir -p ${csharp_bin}
+  curl -sL ${dotnet_download_url} | \
       tar xzf - -C ${csharp_bin}
-    PATH=${csharp_bin}:${PATH}
-    show_info "Installed C# at $(which csharp) (.NET $(dotnet --version))"
-  fi
+  PATH=${csharp_bin}:${PATH}
+  show_info "Installed C# at $(which csharp) (.NET $(dotnet --version))"
 
   DOTNET_ALREADY_INSTALLED=1
 }
@@ -296,6 +300,18 @@ install_go() {
     return 0
   fi
 
+  local prefix=${ARROW_TMPDIR}/go
+  mkdir -p $prefix
+
+  if [ -f "${prefix}/go/bin/go" ]; then
+    export GOROOT=${prefix}/go
+    export GOPATH=${prefix}/gopath
+    export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
+    show_info "Found $(go version) at ${prefix}/go/bin/go"
+    GO_ALREADY_INSTALLED=1
+    return 0
+  fi
+
   local version=1.24.2
   show_info "Installing go version ${version}..."
 
@@ -315,8 +331,6 @@ install_go() {
   local archive="go${version}.${os}-${arch}.tar.gz"
   curl -sLO https://go.dev/dl/$archive
 
-  local prefix=${ARROW_TMPDIR}/go
-  mkdir -p $prefix
   tar -xzf $archive -C $prefix
   rm -f $archive
 
@@ -549,7 +563,7 @@ test_python() {
   show_header "Build and test Python libraries"
 
   # Build and test Python
-  maybe_setup_virtualenv cython duckdb pandas protobuf pyarrow pytest setuptools_scm setuptools importlib_resources || exit 1
+  maybe_setup_virtualenv cython duckdb pandas polars protobuf pyarrow pytest setuptools_scm setuptools importlib_resources || exit 1
   # XXX: pin Python for now since various other packages haven't caught up
   maybe_setup_conda --file "${ADBC_DIR}/ci/conda_env_python.txt" python=3.12 || exit 1
 

go/adbc/driver/flightsql/flightsql_adbc_server_test.go

@@ -1660,8 +1660,7 @@ func (ts *TimeoutTests) TestDoActionTimeout() {
 	var adbcErr adbc.Error
 	ts.ErrorAs(stmt.Prepare(context.Background()), &adbcErr)
 	ts.Equal(adbc.StatusTimeout, adbcErr.Code, adbcErr.Error())
-	// Exact match - we don't want extra fluff in the message
-	ts.Equal("[FlightSQL] context deadline exceeded (DeadlineExceeded; Prepare)", adbcErr.Msg)
+	// It seems gRPC isn't stable about the error message, unfortunately
 }
 
 func (ts *TimeoutTests) TestDoGetTimeout() {