From 9f9c7d41b4d9d3ceb82e41c3af6863bf1a9d56a0 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 21 Apr 2025 21:18:50 +0900 Subject: [PATCH 1/4] ci: also verify under Docker with native deps We've only really tested Conda so far. Validate non-Conda builds too. --- .github/workflows/nightly-verify.yml | 32 ++++++ ci/scripts/verify_ubuntu.sh | 104 ++++++++++++++++++ compose.yaml | 9 ++ dev/release/verify-release-candidate.sh | 76 +++++++------ .../flightsql/flightsql_adbc_server_test.go | 3 +- 5 files changed, 191 insertions(+), 33 deletions(-) create mode 100755 ci/scripts/verify_ubuntu.sh diff --git a/.github/workflows/nightly-verify.yml b/.github/workflows/nightly-verify.yml index a689ccf7fe..e2be589503 100644 --- a/.github/workflows/nightly-verify.yml +++ b/.github/workflows/nightly-verify.yml @@ -39,6 +39,10 @@ defaults: # 'bash' will expand to -eo pipefail shell: bash +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: source: # For cron: only run on the main repo, not forks @@ -201,3 +205,31 @@ jobs: run: | pushd arrow-adbc docker compose run -e PYTHON=3.12 --rm python-debug + + source-verify-docker: + name: "Verify Source (OS)/${{ matrix.os }} ${{ matrix.version }}" + runs-on: ubuntu-latest + strategy: + max-parallel: 2 + matrix: + include: + - os: ubuntu + version: "22.04" + - os: ubuntu + version: "24.04" + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + path: arrow-adbc + persist-credentials: false + submodules: recursive + + - name: Verify + env: + OS: ${{ matrix.os }} + OS_VERSION: ${{ matrix.version }} + run: | + # Hmm, -e doesn't work? + pushd arrow-adbc + env ${OS@U}=${OS_VERSION} docker compose run --rm verify-all-${OS} diff --git a/ci/scripts/verify_ubuntu.sh b/ci/scripts/verify_ubuntu.sh new file mode 100755 index 0000000000..19a7bf3d7b --- /dev/null +++ b/ci/scripts/verify_ubuntu.sh @@ -0,0 +1,104 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Test the validation script using system dependencies instead of Conda. This +# script is meant for automation (e.g. Docker), not a local/developer machine +# (except via Docker). + +set -euo pipefail + +main() { + local -r source_dir="${1}" + + # Install all the dependencies we need for all the subprojects + + # When installing tzdata, don't block and wait for user input + export DEBIAN_FRONTEND=noninteractive + export TZ=Etc/UTC + + apt update + apt install -y \ + apt-transport-https \ + build-essential \ + ca-certificates \ + cmake \ + curl \ + dirmngr \ + git \ + gobject-introspection \ + gpg \ + libgirepository1.0-dev \ + libglib2.0-dev \ + libgmock-dev \ + libgtest-dev \ + libpq-dev \ + libsqlite3-dev \ + lsb-release \ + ninja-build \ + pkg-config \ + python3 \ + python3-dev \ + python3-pip \ + python3-venv \ + ruby-full \ + software-properties-common \ + wget + + # Install Java + + wget -qO - https://packages.adoptium.net/artifactory/api/gpg/key/public | \ + gpg --dearmor | \ + tee /etc/apt/trusted.gpg.d/adoptium.gpg > /dev/null + + echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | \ + tee /etc/apt/sources.list.d/adoptium.list + + # Install R + # https://cloud.r-project.org/bin/linux/ubuntu/ + + wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | \ + tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc + + add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/" + + # Install Arrow GLib + wget https://repo1.maven.org/maven2/org/apache/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + + apt update + apt install -y \ + libarrow-dev \ + libarrow-glib-dev \ + r-base \ + temurin-21-jdk + + # Install Maven + wget https://dlcdn.apache.org/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz + mkdir -p /opt/maven + tar -C /opt/maven -xzvf apache-maven-3.9.9-bin.tar.gz --strip-components=1 + export PATH=/opt/maven/bin:$PATH + + # We run under Docker and this is necessary since the source dir is mounted in + git config --global --add safe.directory "${source_dir}" + + "${source_dir}/dev/release/verify-release-candidate.sh" +} + +main "$@" diff --git a/compose.yaml b/compose.yaml index 3a7b07fdce..742669cc80 100644 --- a/compose.yaml +++ b/compose.yaml @@ -275,3 +275,12 @@ services: retries: 5 ports: - "5432:5432" + + ################################ Verification ################################ + + verify-all-ubuntu: + image: ubuntu:${UBUNTU} + volumes: + - .:/adbc:delegated + command: | + /adbc/ci/scripts/verify_ubuntu.sh /adbc diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 4967493e76..d7ac571322 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -246,38 +246,42 @@ install_dotnet() { return 0 fi - show_info "Ensuring that .NET is installed..." - - if dotnet --version | grep 8\.0 > /dev/null 2>&1; then - local csharp_bin=$(dirname $(which dotnet)) - show_info "Found C# at $(which csharp) (.NET $(dotnet --version))" - else - if which dotnet > /dev/null 2>&1; then - show_info "dotnet found but it is the wrong version and will be ignored." + if command -v dotnet; then + show_info "Found $(dotnet --version) at $(which dotnet)" + + if dotnet --version | grep 8\.0 > /dev/null 2>&1; then + local csharp_bin=$(dirname $(which dotnet)) + show_info "Found C# at $(which csharp) (.NET $(dotnet --version))" + DOTNET_ALREADY_INSTALLED=1 + return 0 fi - local csharp_bin=${ARROW_TMPDIR}/csharp/bin - local dotnet_version=8.0.204 - local dotnet_platform= - case "$(uname)" in + fi + + show_info "dotnet found but it is the wrong version or dotnet not found" + + local csharp_bin=${ARROW_TMPDIR}/csharp/bin + local dotnet_version=8.0.204 + local dotnet_platform= + case "$(uname)" in Linux) - dotnet_platform=linux - ;; + dotnet_platform=linux + ;; Darwin) - dotnet_platform=macos - ;; - esac - local dotnet_download_thank_you_url=https://dotnet.microsoft.com/download/thank-you/dotnet-sdk-${dotnet_version}-${dotnet_platform}-x64-binaries - local dotnet_download_url=$( \ - curl -sL ${dotnet_download_thank_you_url} | \ - grep 'directLink' | \ - grep -E -o 'https://download[^"]+' | \ - sed -n 2p) - mkdir -p ${csharp_bin} - curl -sL ${dotnet_download_url} | \ + dotnet_platform=macos + ;; + esac + local dotnet_download_thank_you_url=https://dotnet.microsoft.com/download/thank-you/dotnet-sdk-${dotnet_version}-${dotnet_platform}-x64-binaries + show_info "Getting .NET download URL from ${dotnet_download_thank_you_url}" + local dotnet_download_url=$(curl -sL ${dotnet_download_thank_you_url} | \ + grep 'directLink' | \ + grep -E -o 'https://download[^"]+' | \ + sed -n 2p) + show_info "Downloading .NET from ${dotnet_download_url}" + mkdir -p ${csharp_bin} + curl -sL ${dotnet_download_url} | \ tar xzf - -C ${csharp_bin} - PATH=${csharp_bin}:${PATH} - show_info "Installed C# at $(which csharp) (.NET $(dotnet --version))" - fi + PATH=${csharp_bin}:${PATH} + show_info "Installed C# at $(which csharp) (.NET $(dotnet --version))" DOTNET_ALREADY_INSTALLED=1 } @@ -296,6 +300,18 @@ install_go() { return 0 fi + local prefix=${ARROW_TMPDIR}/go + mkdir -p $prefix + + if [ -f "${prefix}/go/bin/go" ]; then + export GOROOT=${prefix}/go + export GOPATH=${prefix}/gopath + export PATH=$GOROOT/bin:$GOPATH/bin:$PATH + show_info "Found $(go version) at ${prefix}/go/bin/go" + GO_ALREADY_INSTALLED=1 + return 0 + fi + local version=1.24.2 show_info "Installing go version ${version}..." @@ -315,8 +331,6 @@ install_go() { local archive="go${version}.${os}-${arch}.tar.gz" curl -sLO https://go.dev/dl/$archive - local prefix=${ARROW_TMPDIR}/go - mkdir -p $prefix tar -xzf $archive -C $prefix rm -f $archive @@ -549,7 +563,7 @@ test_python() { show_header "Build and test Python libraries" # Build and test Python - maybe_setup_virtualenv cython duckdb pandas protobuf pyarrow pytest setuptools_scm setuptools importlib_resources || exit 1 + maybe_setup_virtualenv cython duckdb pandas polars protobuf pyarrow pytest setuptools_scm setuptools importlib_resources || exit 1 # XXX: pin Python for now since various other packages haven't caught up maybe_setup_conda --file "${ADBC_DIR}/ci/conda_env_python.txt" python=3.12 || exit 1 diff --git a/go/adbc/driver/flightsql/flightsql_adbc_server_test.go b/go/adbc/driver/flightsql/flightsql_adbc_server_test.go index 5ec08b1b6a..1261d6becc 100644 --- a/go/adbc/driver/flightsql/flightsql_adbc_server_test.go +++ b/go/adbc/driver/flightsql/flightsql_adbc_server_test.go @@ -1660,8 +1660,7 @@ func (ts *TimeoutTests) TestDoActionTimeout() { var adbcErr adbc.Error ts.ErrorAs(stmt.Prepare(context.Background()), &adbcErr) ts.Equal(adbc.StatusTimeout, adbcErr.Code, adbcErr.Error()) - // Exact match - we don't want extra fluff in the message - ts.Equal("[FlightSQL] context deadline exceeded (DeadlineExceeded; Prepare)", adbcErr.Msg) + // It seems gRPC isn't stable about the error message, unfortunately } func (ts *TimeoutTests) TestDoGetTimeout() { From acdaeb0be26b9eb88b210e676643c8598a38fb64 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 23 Apr 2025 14:58:25 +0900 Subject: [PATCH 2/4] Apply suggestions from code review Co-authored-by: Sutou Kouhei Co-authored-by: Dewey Dunnington --- ci/scripts/verify_ubuntu.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/verify_ubuntu.sh b/ci/scripts/verify_ubuntu.sh index 19a7bf3d7b..f3cdf871d8 100755 --- a/ci/scripts/verify_ubuntu.sh +++ b/ci/scripts/verify_ubuntu.sh @@ -79,7 +79,7 @@ main() { add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/" # Install Arrow GLib - wget https://repo1.maven.org/maven2/org/apache/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + wget https://packages.apache.org/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb apt update @@ -95,7 +95,7 @@ main() { tar -C /opt/maven -xzvf apache-maven-3.9.9-bin.tar.gz --strip-components=1 export PATH=/opt/maven/bin:$PATH - # We run under Docker and this is necessary since the source dir is mounted in + # We run under Docker and this is necessary since the source dir is typically mounted as a volume git config --global --add safe.directory "${source_dir}" "${source_dir}/dev/release/verify-release-candidate.sh" From 9922855e7cc85ce6fabc9d18d5d81d79d521fa5d Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 23 Apr 2025 15:01:28 +0900 Subject: [PATCH 3/4] simplify R install --- .github/workflows/nightly-verify.yml | 1 - ci/scripts/verify_ubuntu.sh | 27 +++++++++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/nightly-verify.yml b/.github/workflows/nightly-verify.yml index e2be589503..5caa2b26c5 100644 --- a/.github/workflows/nightly-verify.yml +++ b/.github/workflows/nightly-verify.yml @@ -230,6 +230,5 @@ jobs: OS: ${{ matrix.os }} OS_VERSION: ${{ matrix.version }} run: | - # Hmm, -e doesn't work? pushd arrow-adbc env ${OS@U}=${OS_VERSION} docker compose run --rm verify-all-${OS} diff --git a/ci/scripts/verify_ubuntu.sh b/ci/scripts/verify_ubuntu.sh index f3cdf871d8..0b1eef2c5f 100755 --- a/ci/scripts/verify_ubuntu.sh +++ b/ci/scripts/verify_ubuntu.sh @@ -53,10 +53,12 @@ main() { lsb-release \ ninja-build \ pkg-config \ + protobuf-compiler \ python3 \ python3-dev \ python3-pip \ python3-venv \ + r-base \ ruby-full \ software-properties-common \ wget @@ -70,14 +72,6 @@ main() { echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | \ tee /etc/apt/sources.list.d/adoptium.list - # Install R - # https://cloud.r-project.org/bin/linux/ubuntu/ - - wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | \ - tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc - - add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/" - # Install Arrow GLib wget https://packages.apache.org/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb @@ -86,7 +80,6 @@ main() { apt install -y \ libarrow-dev \ libarrow-glib-dev \ - r-base \ temurin-21-jdk # Install Maven @@ -95,7 +88,21 @@ main() { tar -C /opt/maven -xzvf apache-maven-3.9.9-bin.tar.gz --strip-components=1 export PATH=/opt/maven/bin:$PATH - # We run under Docker and this is necessary since the source dir is typically mounted as a volume + # Check if protoc is too old (Ubuntu 22.04). If so, install it from + # upstream instead. + if ! protoc --version | \ + awk '{print $2}{print "3.15"}' | \ + sort --version-sort | \ + head -n1 | \ + grep -E '^3\.15.*$' >/dev/null ; then + echo "protoc is too old" + + wget -O protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v30.2/protoc-30.2-linux-x86_64.zip + unzip -o protoc.zip -d /usr/local -x readme.txt + fi + + # We run under Docker and this is necessary since the source dir is + # typically mounted as a volume git config --global --add safe.directory "${source_dir}" "${source_dir}/dev/release/verify-release-candidate.sh" From a7895d0bdb0545d0e862de5bbb5312b3d5448c47 Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 24 Apr 2025 10:40:45 +0900 Subject: [PATCH 4/4] Update dev/release/verify-release-candidate.sh Co-authored-by: Sutou Kouhei --- dev/release/verify-release-candidate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index d7ac571322..fab1cb0190 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -249,7 +249,7 @@ install_dotnet() { if command -v dotnet; then show_info "Found $(dotnet --version) at $(which dotnet)" - if dotnet --version | grep 8\.0 > /dev/null 2>&1; then + if dotnet --version | grep --quiet --fixed-string 8.0; then local csharp_bin=$(dirname $(which dotnet)) show_info "Found C# at $(which csharp) (.NET $(dotnet --version))" DOTNET_ALREADY_INSTALLED=1