Skip to content

Commit a96e753

Browse files
committed
Merge remote-tracking branch 'origin/main' into qwp_query_reset_dict_flag
# Conflicts: # core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java
2 parents a96fed7 + 6c336d1 commit a96e753

197 files changed

Lines changed: 19093 additions & 5494 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.claude/skills/review-pr/SKILL.md

Lines changed: 155 additions & 10 deletions
Large diffs are not rendered by default.
Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#!/usr/bin/env bash
2+
# Assert the glibc runtime floor of a Linux native library.
3+
#
4+
# Usage: check-glibc-floor.sh <path-to-libquestdb.so> <max-glibc-version>
5+
# e.g. check-glibc-floor.sh core/.../linux-x86-64/libquestdb.so 2.14
6+
# check-glibc-floor.sh core/.../linux-aarch64/libquestdb.so 2.17
7+
#
8+
# The dynamic linker resolves .gnu.version_r at load time, so the HIGHEST
9+
# GLIBC_x.y version node the library imports is its hard load floor: a host
10+
# whose glibc is older than that node fails System.loadLibrary/dlopen with
11+
# `version 'GLIBC_x.y' not found`. This script extracts every versioned import
12+
# and fails if the highest one exceeds the allowed floor.
13+
#
14+
# Why the floors are what they are:
15+
# * linux-x86-64 -> 2.14. The oldest node we intentionally keep is
16+
# memcpy@GLIBC_2.14; clock_gettime is pinned back to GLIBC_2.2.5 by
17+
# src/main/c/share/glibc_compat.h, and stat/fstat resolve to the inline
18+
# __xstat/__fxstat@GLIBC_2.2.5 wrappers when built in a low-glibc container.
19+
# A build on a modern host (glibc >= 2.33) instead emits stat@GLIBC_2.33 /
20+
# fstat@GLIBC_2.33 and trips this guard -- that is exactly the regression it
21+
# exists to catch.
22+
# * linux-aarch64 -> 2.17. glibc gained aarch64 support in 2.17, so 2.17 is
23+
# the lowest floor physically achievable on that architecture.
24+
#
25+
# Portable to bash 3.2 (no mapfile / no negative array indices) so it can be run
26+
# locally on macOS as well as in the glibc build containers.
27+
set -euo pipefail
28+
29+
lib="${1:?usage: check-glibc-floor.sh <lib.so> <max-glibc-version>}"
30+
floor="${2:?usage: check-glibc-floor.sh <lib.so> <max-glibc-version>}"
31+
32+
if [ ! -f "$lib" ]; then
33+
echo "::error::check-glibc-floor: library not found: $lib"
34+
exit 1
35+
fi
36+
37+
# All distinct versioned GLIBC nodes (e.g. 2.14, 2.2.5), sorted ascending.
38+
# objdump prints them as (GLIBC_x.y) or GLIBC_x.y depending on the toolchain;
39+
# the -o regex captures the token regardless of surrounding parentheses.
40+
# GLIBC_PRIVATE has no digit after the underscore, so it is naturally excluded.
41+
versions="$(
42+
objdump -T "$lib" \
43+
| grep -oE 'GLIBC_[0-9]+(\.[0-9]+)+' \
44+
| sed 's/^GLIBC_//' \
45+
| sort -Vu
46+
)"
47+
48+
if [ -z "$versions" ]; then
49+
echo "::error::check-glibc-floor: no versioned GLIBC symbols found in $lib (unexpected)."
50+
exit 1
51+
fi
52+
53+
highest="$(printf '%s\n' "$versions" | tail -n1)"
54+
55+
echo "GLIBC version nodes required by $lib:"
56+
printf '%s\n' "$versions" | sed 's/^/ GLIBC_/'
57+
echo "Highest required: GLIBC_${highest} (allowed floor: GLIBC_${floor})"
58+
59+
# leq A B -> succeeds when version A <= version B: sorting {A, B} with -V puts B
60+
# last, or they are equal.
61+
leq() {
62+
[ "$1" = "$2" ] && return 0
63+
[ "$(printf '%s\n%s\n' "$1" "$2" | sort -V | tail -n1)" = "$2" ]
64+
}
65+
66+
if leq "$highest" "$floor"; then
67+
echo "OK: $lib floor is GLIBC_${highest} (<= GLIBC_${floor})."
68+
exit 0
69+
fi
70+
71+
echo "::error::GLIBC floor regression in $lib: requires GLIBC_${highest}, above the GLIBC_${floor} floor."
72+
echo "::error::This library will fail to load on hosts with glibc < ${highest}."
73+
echo "Offending nodes above the floor and the symbols that pull them in:"
74+
printf '%s\n' "$versions" | while IFS= read -r v; do
75+
if ! leq "$v" "$floor"; then
76+
echo " GLIBC_${v}:"
77+
objdump -T "$lib" | grep -E "GLIBC_${v//./\\.}([^0-9]|\$)" | awk '{print " " $NF}' | sort -u
78+
fi
79+
done
80+
exit 1

.github/scripts/stage-native-artifacts.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,10 @@ downloaded="core/target/downloaded-native-artifacts"
99
staged="core/target/native-libs/io/questdb/client/bin"
1010

1111
# platform -> library filename
12+
# darwin-x86-64 is intentionally not shipped: no CI runs the test suite on x64
13+
# macOS, so the release does not build or bundle a binary it cannot test.
1214
declare -A libs=(
1315
[darwin-aarch64]=libquestdb.dylib
14-
[darwin-x86-64]=libquestdb.dylib
1516
[linux-aarch64]=libquestdb.so
1617
[linux-x86-64]=libquestdb.so
1718
[windows-x86-64]=libquestdb.dll

.github/workflows/ci.yml

Lines changed: 104 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,20 @@ defaults:
1717
jobs:
1818
# JDK 8 is the source of truth: the client ships as a Java 8 artifact
1919
# (io.questdb:questdb-client) and is released from JDK 8, so on JDK 8 it must
20-
# compile, the full test suite must pass against the committed native
21-
# libraries, and the javadoc jar must build (-P javadoc attaches it at the
22-
# package phase). The committed native .so/.dylib/.dll are enough -- the only
23-
# git submodule (zstd) is needed solely for C++ native rebuilds, not here.
20+
# compile, the full test suite must pass, and the javadoc jar must build
21+
# (-P javadoc attaches it at the package phase). The native libraries are no
22+
# longer committed, so this job compiles libquestdb.so from source (hence the
23+
# zstd submodule + cmake/nasm/build-essential toolchain) before the tests run.
2424
build-jdk8:
2525
name: Build, test & javadoc (JDK 8)
2626
runs-on: ubuntu-latest
2727
timeout-minutes: 45
2828
steps:
2929
- name: Check out
3030
uses: actions/checkout@v4
31+
with:
32+
# zstd is required to compile the native library.
33+
submodules: recursive
3134

3235
- name: Set up JDK 8
3336
uses: actions/setup-java@v4
@@ -36,6 +39,26 @@ jobs:
3639
java-version: "8"
3740
cache: maven
3841

42+
- name: Install native build toolchain
43+
run: sudo apt-get update && sudo apt-get install -y cmake nasm build-essential
44+
45+
- name: Build native libquestdb.so
46+
# JAVA_HOME points at the JDK 8 above, so the lib is compiled against the
47+
# Java 8 JNI headers -- the artifact's Java floor. Copy it into src
48+
# resources (not target/) so it survives the `mvn clean` in the next step
49+
# and gets packaged + loaded via the production bin/<platform> path.
50+
# NOTE: this builds on ubuntu-latest for FUNCTIONAL testing only; the
51+
# library's glibc runtime floor is validated separately by the
52+
# `glibc-floor` job, which rebuilds in the release low-glibc container.
53+
run: |
54+
cd core
55+
cmake -DCMAKE_BUILD_TYPE=Release -B cmake-build-release -S.
56+
cmake --build cmake-build-release --config Release
57+
test -f target/classes/io/questdb/client/bin-local/libquestdb.so
58+
mkdir -p src/main/resources/io/questdb/client/bin/linux-x86-64
59+
cp target/classes/io/questdb/client/bin-local/libquestdb.so \
60+
src/main/resources/io/questdb/client/bin/linux-x86-64/libquestdb.so
61+
3962
- name: Compile, test, and build javadoc
4063
run: mvn -B -ntp -P javadoc clean install
4164

@@ -61,3 +84,80 @@ jobs:
6184

6285
- name: Compile (main + test) and build javadoc (no tests run)
6386
run: mvn -B -ntp -P javadoc -DskipTests clean package
87+
88+
# GLIBC floor guard. The native libraries are built at release time in
89+
# low-glibc manylinux containers (see maven_central_release.yml) and are NOT
90+
# committed, so a floor regression is invisible to the functional test job
91+
# above (it builds on ubuntu-latest, whose glibc is new enough to load almost
92+
# anything). This job rebuilds the linux libraries in the SAME low-glibc
93+
# environment as release and asserts the runtime floor with objdump, so a
94+
# change that raises the floor (e.g. a new stat/fstat call pulling in
95+
# stat@GLIBC_2.33 on a modern build host) fails the PR instead of silently
96+
# shipping a library that cannot load on older distros.
97+
#
98+
# * linux-x86-64 -> GLIBC_2.14 (the intended floor: memcpy@GLIBC_2.14).
99+
# * linux-aarch64 -> GLIBC_2.17 (the lowest floor glibc offers on aarch64).
100+
#
101+
# Uses manylinux_2_28 for both arches (stock Node 24, no glibc-2.17 shadow
102+
# hack). The x86-64 floor is identical in manylinux2014 (2.17) and
103+
# manylinux_2_28 (2.28) -- both resolve stat/fstat to the inline
104+
# __xstat/__fxstat@GLIBC_2.2.5 wrappers -- so this validates the real shipped
105+
# floor without the heavier manylinux2014 release toolchain.
106+
glibc-floor:
107+
name: GLIBC floor guard (${{ matrix.platform }})
108+
strategy:
109+
fail-fast: false
110+
matrix:
111+
include:
112+
- platform: linux-x86-64
113+
os: ubuntu-latest
114+
image: quay.io/pypa/manylinux_2_28_x86_64
115+
jdk_arch: x64
116+
floor: "2.14"
117+
cmake_args: ""
118+
build_dir: cmake-build-release
119+
- platform: linux-aarch64
120+
os: ubuntu-22.04-arm
121+
image: quay.io/pypa/manylinux_2_28_aarch64
122+
jdk_arch: aarch64
123+
floor: "2.17"
124+
cmake_args: "-DCMAKE_TOOLCHAIN_FILE=./src/main/c/toolchains/linux-arm64.cmake"
125+
build_dir: cmake-build-release-arm64
126+
runs-on: ${{ matrix.os }}
127+
timeout-minutes: 45
128+
container:
129+
image: ${{ matrix.image }}
130+
steps:
131+
- name: Check out
132+
uses: actions/checkout@v4
133+
with:
134+
# zstd is required to compile the native library.
135+
submodules: recursive
136+
137+
- name: Install tooling
138+
# binutils provides objdump for the floor check; nasm/zstd are build deps.
139+
run: |
140+
yum update -y
141+
yum install -y wget nasm zstd binutils
142+
143+
- name: Install Temurin JDK 8 (for jni.h)
144+
# Build against the Java 8 JNI headers -- JDK 8 is the artifact's floor.
145+
# The JDK version does not affect the glibc floor; it only supplies jni.h.
146+
run: |
147+
wget -v --timeout=180 -O jdk8.tar.gz \
148+
"https://api.adoptium.net/v3/binary/latest/8/ga/linux/${{ matrix.jdk_arch }}/jdk/hotspot/normal/eclipse"
149+
mkdir jdk8
150+
tar xfz jdk8.tar.gz -C jdk8 --strip-components=1
151+
echo "JAVA_HOME=$(pwd)/jdk8" >> "$GITHUB_ENV"
152+
153+
- name: Build native libquestdb.so
154+
run: |
155+
cd core
156+
cmake ${{ matrix.cmake_args }} -DCMAKE_BUILD_TYPE=Release -B ${{ matrix.build_dir }} -S.
157+
cmake --build ${{ matrix.build_dir }} --config Release
158+
159+
- name: Assert GLIBC floor
160+
run: |
161+
./.github/scripts/check-glibc-floor.sh \
162+
core/target/classes/io/questdb/client/bin-local/libquestdb.so \
163+
"${{ matrix.floor }}"

0 commit comments

Comments
 (0)