Skip to content

Commit 3a03cb5

Browse files
authored
Merge branch 'main' into fix/tls-unwrap-buffering
2 parents a85a194 + 666e234 commit 3a03cb5

100 files changed

Lines changed: 20981 additions & 325 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/gitleaks.yml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
name: Gitleaks
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- main
8+
9+
jobs:
10+
gitleaks:
11+
runs-on: ubuntu-latest
12+
env:
13+
HAS_GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE != '' }}
14+
steps:
15+
- uses: actions/checkout@v4
16+
with:
17+
fetch-depth: 0
18+
- uses: gitleaks/gitleaks-action@ff98106e4c7b2bc287b24eaf42907196329070c7
19+
if: ${{ env.HAS_GITLEAKS_LICENSE == 'true' }}
20+
env:
21+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
22+
GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE }}

.github/workflows/rebuild_native_libs.yml

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -109,11 +109,12 @@ jobs:
109109
wget -nv https://github.com/Kitware/CMake/releases/download/v3.29.2/cmake-3.29.2-linux-x86_64.tar.gz
110110
tar -zxf cmake-3.29.2-linux-x86_64.tar.gz
111111
echo "PATH=`pwd`/cmake-3.29.2-linux-x86_64/bin/:$PATH" >> "$GITHUB_ENV"
112-
- name: Install JDK17 (for jni.h)
112+
- name: Install GraalVM JDK 25 (for jni.h)
113113
run: |
114-
wget -nv https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.11%2B9/OpenJDK17U-jdk_x64_linux_hotspot_17.0.11_9.tar.gz
115-
tar xfz OpenJDK17U-jdk_x64_linux_hotspot_17.0.11_9.tar.gz
116-
echo "JAVA_HOME=`pwd`/jdk-17.0.11+9/" >> "$GITHUB_ENV"
114+
wget -nv -O graalvm.tar.gz https://download.oracle.com/graalvm/25/latest/graalvm-jdk-25_linux-x64_bin.tar.gz
115+
mkdir graalvm
116+
tar xfz graalvm.tar.gz -C graalvm --strip-components=1
117+
echo "JAVA_HOME=`pwd`/graalvm" >> "$GITHUB_ENV"
117118
- name: Generate Makefiles
118119
run: |
119120
cd ./core
@@ -144,11 +145,12 @@ jobs:
144145
run: |
145146
yum update -y
146147
yum install wget nasm zstd -y
147-
- name: Install JDK17 (for jni.h)
148+
- name: Install GraalVM JDK 25 (for jni.h)
148149
run: |
149-
wget -v --timeout=180 https://api.adoptium.net/v3/binary/version/jdk-17.0.11+9/linux/aarch64/jdk/hotspot/normal/eclipse
150-
tar xfvz eclipse
151-
echo "JAVA_HOME=`pwd`/jdk-17.0.11+9/" >> "$GITHUB_ENV"
150+
wget -v --timeout=180 -O graalvm.tar.gz https://download.oracle.com/graalvm/25/latest/graalvm-jdk-25_linux-aarch64_bin.tar.gz
151+
mkdir graalvm
152+
tar xfz graalvm.tar.gz -C graalvm --strip-components=1
153+
echo "JAVA_HOME=`pwd`/graalvm" >> "$GITHUB_ENV"
152154
- name: CMAKE linux-aarch64
153155
run: |
154156
cd ./core
@@ -177,10 +179,16 @@ jobs:
177179
sudo sysctl -w fs.file-max=500000
178180
sudo apt-get update -y
179181
sudo apt-get install -y nasm gcc-mingw-w64 g++-mingw-w64
180-
- name: Download windows jni_md.h from JDK 11
182+
- name: Install GraalVM JDK 25 (for jni.h)
183+
run: |
184+
wget -nv -O graalvm.tar.gz https://download.oracle.com/graalvm/25/latest/graalvm-jdk-25_linux-x64_bin.tar.gz
185+
mkdir graalvm
186+
tar xfz graalvm.tar.gz -C graalvm --strip-components=1
187+
echo "JAVA_HOME=`pwd`/graalvm" >> "$GITHUB_ENV"
188+
- name: Download windows jni_md.h from JDK 25
181189
run: |
182190
cd core
183-
curl https://raw.githubusercontent.com/AdoptOpenJDK/openjdk-jdk11/master/src/java.base/windows/native/include/jni_md.h > $JAVA_HOME/include/jni_md.h
191+
curl https://raw.githubusercontent.com/openjdk/jdk25u/master/src/java.base/windows/native/include/jni_md.h > $JAVA_HOME/include/jni_md.h
184192
- name: CMake Windows
185193
run: |
186194
cd core

.gitmodules

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[submodule "core/src/main/c/share/zstd"]
2+
path = core/src/main/c/share/zstd
3+
url = https://github.com/facebook/zstd.git

ci/run_tests_pipeline.yaml

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,9 @@ stages:
2323
pool:
2424
name: "Azure Pipelines"
2525
vmImage: "ubuntu-latest"
26+
variables:
27+
jdkArch: "x64"
28+
imageName: "ubuntu-latest"
2629
steps:
2730
- checkout: self
2831
fetchDepth: 0
@@ -33,12 +36,7 @@ stages:
3336
git checkout FETCH_HEAD
3437
displayName: "Checkout PR source branch"
3538
condition: eq(variables['Build.Reason'], 'PullRequest')
36-
- task: JavaToolInstaller@0
37-
displayName: "Install Java 17"
38-
inputs:
39-
versionSpec: "17"
40-
jdkArchitectureOption: "x64"
41-
jdkSourceOption: "PreInstalled"
39+
- template: setup.yaml
4240
- bash: mvn -f core/pom.xml javadoc:javadoc -Pjavadoc --batch-mode
4341
displayName: "Verify Javadoc"
4442

ci/setup.yaml

Lines changed: 44 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,48 @@
11
steps:
2-
- task: JavaToolInstaller@0
3-
displayName: "Install Java 17"
4-
inputs:
5-
versionSpec: "17"
6-
jdkArchitectureOption: "$(jdkArch)"
7-
jdkSourceOption: "PreInstalled"
2+
- bash: |
3+
set -eux
4+
case "$(jdkArch)" in
5+
arm64) ARCH="aarch64" ;;
6+
*) ARCH="x64" ;;
7+
esac
8+
curl -fsSL -o "$HOME/graalvm.tar.gz" \
9+
"https://download.oracle.com/graalvm/25/latest/graalvm-jdk-25_linux-${ARCH}_bin.tar.gz"
10+
mkdir -p "$HOME/graalvm"
11+
tar -xzf "$HOME/graalvm.tar.gz" -C "$HOME/graalvm" --strip-components=1
12+
echo "##vso[task.setvariable variable=JAVA_HOME]$HOME/graalvm"
13+
echo "##vso[task.prependpath]$HOME/graalvm/bin"
14+
displayName: "Install GraalVM JDK 25 (Linux)"
15+
condition: eq(variables['Agent.OS'], 'Linux')
16+
17+
- bash: |
18+
set -eux
19+
case "$(jdkArch)" in
20+
arm64) ARCH="aarch64" ;;
21+
*) ARCH="x64" ;;
22+
esac
23+
curl -fsSL -o "$HOME/graalvm.tar.gz" \
24+
"https://download.oracle.com/graalvm/25/latest/graalvm-jdk-25_macos-${ARCH}_bin.tar.gz"
25+
mkdir -p "$HOME/graalvm"
26+
tar -xzf "$HOME/graalvm.tar.gz" -C "$HOME/graalvm" --strip-components=1
27+
echo "##vso[task.setvariable variable=JAVA_HOME]$HOME/graalvm/Contents/Home"
28+
echo "##vso[task.prependpath]$HOME/graalvm/Contents/Home/bin"
29+
displayName: "Install GraalVM JDK 25 (macOS)"
30+
condition: eq(variables['Agent.OS'], 'Darwin')
31+
32+
- powershell: |
33+
$ErrorActionPreference = "Stop"
34+
$url = "https://download.oracle.com/graalvm/25/latest/graalvm-jdk-25_windows-x64_bin.zip"
35+
$zip = Join-Path $env:USERPROFILE "graalvm.zip"
36+
$dest = Join-Path $env:USERPROFILE "graalvm"
37+
Invoke-WebRequest -Uri $url -OutFile $zip -UseBasicParsing
38+
if (Test-Path $dest) { Remove-Item -Recurse -Force $dest }
39+
Expand-Archive -Path $zip -DestinationPath $dest -Force
40+
$jdkDir = Get-ChildItem -Path $dest -Directory | Select-Object -First 1
41+
$javaHome = $jdkDir.FullName
42+
Write-Host "##vso[task.setvariable variable=JAVA_HOME]$javaHome"
43+
Write-Host "##vso[task.prependpath]$javaHome\bin"
44+
displayName: "Install GraalVM JDK 25 (Windows)"
45+
condition: eq(variables['Agent.OS'], 'Windows_NT')
846
947
- bash: sudo sysctl -w fs.file-max=500000
1048
displayName: "Increase file count on Linux"

core/CMakeLists.txt

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
cmake_minimum_required(VERSION 3.5)
22
project(questdb)
33

4+
# Required for zstd's huf_decompress_amd64.S to be assembled. Without this,
5+
# CMake silently drops the .S file from the build and the link fails at
6+
# _HUF_decompress4X1_usingDTable_internal_fast_asm_loop etc. (The asmlib
7+
# subdirectory enables ASM_NASM separately for Agner Fog's .asm files.)
8+
enable_language(ASM)
9+
410
include(ExternalProject)
511

612
set(CMAKE_CXX_STANDARD 17)
@@ -49,6 +55,42 @@ set(
4955
src/main/c/share/byte_sink.h
5056
)
5157

58+
# libzstd is included via a git submodule at src/main/c/share/zstd (pinned to
59+
# upstream tag v1.5.7). Covers the client side of the QWP egress compression
60+
# feature; the server-side compressor lives in the Rust qdbr crate and isn't
61+
# linked into this library. Only the decompress-only subset of upstream is
62+
# compiled -- the compress/ directory is left out entirely. zstd_jni.c is our
63+
# JNI glue and lives alongside the submodule (not inside) so upstream resets
64+
# don't disturb it.
65+
if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/src/main/c/share/zstd/lib/zstd.h)
66+
message(FATAL_ERROR
67+
"libzstd submodule not initialised. Run:\n"
68+
" git submodule update --init --recursive\n"
69+
"from java-questdb-client/.")
70+
endif ()
71+
set(
72+
ZSTD_FILES
73+
src/main/c/share/zstd_jni.c
74+
src/main/c/share/zstd/lib/common/debug.c
75+
src/main/c/share/zstd/lib/common/entropy_common.c
76+
src/main/c/share/zstd/lib/common/error_private.c
77+
src/main/c/share/zstd/lib/common/fse_decompress.c
78+
src/main/c/share/zstd/lib/common/pool.c
79+
src/main/c/share/zstd/lib/common/threading.c
80+
src/main/c/share/zstd/lib/common/xxhash.c
81+
src/main/c/share/zstd/lib/common/zstd_common.c
82+
src/main/c/share/zstd/lib/decompress/huf_decompress.c
83+
src/main/c/share/zstd/lib/decompress/zstd_ddict.c
84+
src/main/c/share/zstd/lib/decompress/zstd_decompress_block.c
85+
src/main/c/share/zstd/lib/decompress/zstd_decompress.c
86+
)
87+
# x86_64-only hand-tuned Huffman decoder; C fallback kicks in when
88+
# ZSTD_DISABLE_ASM is set.
89+
if (ARCH_AMD64 AND NOT WIN32)
90+
list(APPEND ZSTD_FILES src/main/c/share/zstd/lib/decompress/huf_decompress_amd64.S)
91+
endif ()
92+
list(APPEND SOURCE_FILES ${ZSTD_FILES})
93+
5294
# JNI includes
5395
include_directories($ENV{JAVA_HOME}/include/)
5496

@@ -111,6 +153,20 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${OUTPUT})
111153

112154
add_library(questdb SHARED ${SOURCE_FILES})
113155

156+
# libzstd public header is at zstd/lib/zstd.h; internal headers live under
157+
# zstd/lib/common/. Both directories go on the include path so zstd_jni.c can
158+
# use the short include "zstd.h" and the upstream .c files can find their own
159+
# siblings without patching.
160+
target_include_directories(questdb PRIVATE
161+
${CMAKE_CURRENT_SOURCE_DIR}/src/main/c/share/zstd/lib
162+
${CMAKE_CURRENT_SOURCE_DIR}/src/main/c/share/zstd/lib/common)
163+
164+
# Drop the zstd-internal hand-written amd64 assembly on platforms that can't
165+
# assemble it; libzstd falls back to a C implementation when this is set.
166+
if (NOT ARCH_AMD64 OR WIN32)
167+
target_compile_definitions(questdb PRIVATE ZSTD_DISABLE_ASM=1)
168+
endif ()
169+
114170
set(COMMON_OPTIONS "-Wno-gnu-anonymous-struct;-Wno-nested-anon-types;-Wno-unused-parameter;-fPIC;-fno-rtti;-fno-exceptions")
115171

116172
set(DEBUG_OPTIONS "-Wall;-pedantic;-Wextra;-g;-O0")

core/src/main/c/share/zstd

Submodule zstd added at f8745da

core/src/main/c/share/zstd_jni.c

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
/*+*****************************************************************************
2+
* ___ _ ____ ____
3+
* / _ \ _ _ ___ ___| |_| _ \| __ )
4+
* | | | | | | |/ _ \/ __| __| | | | _ \
5+
* | |_| | |_| | __/\__ \ |_| |_| | |_) |
6+
* \__\_\\__,_|\___||___/\__|____/|____/
7+
*
8+
* Copyright (c) 2014-2019 Appsicle
9+
* Copyright (c) 2019-2026 QuestDB
10+
*
11+
* Licensed under the Apache License, Version 2.0 (the "License");
12+
* you may not use this file except in compliance with the License.
13+
* You may obtain a copy of the License at
14+
*
15+
* http://www.apache.org/licenses/LICENSE-2.0
16+
*
17+
* Unless required by applicable law or agreed to in writing, software
18+
* distributed under the License is distributed on an "AS IS" BASIS,
19+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20+
* See the License for the specific language governing permissions and
21+
* limitations under the License.
22+
*
23+
******************************************************************************/
24+
25+
/*
26+
* JNI wrapper over the bundled libzstd (decompression only). The server ships
27+
* compression support in the Rust qdbr crate; this file covers the client
28+
* decompression path so RESULT_BATCH frames with FLAG_ZSTD can be decoded
29+
* without any external native dependency.
30+
*
31+
* libzstd is vendored as a git submodule at share/zstd/ pinned to v1.5.7;
32+
* this file lives alongside (not inside) the submodule so upstream resets
33+
* don't nuke our JNI glue.
34+
*/
35+
36+
#include <jni.h>
37+
#include <stdint.h>
38+
#include <stdlib.h>
39+
#include "zstd.h"
40+
41+
JNIEXPORT jlong JNICALL Java_io_questdb_client_std_Zstd_createDCtx(
42+
JNIEnv *env, jclass cls) {
43+
return (jlong) (uintptr_t) ZSTD_createDCtx();
44+
}
45+
46+
JNIEXPORT jlong JNICALL Java_io_questdb_client_std_Zstd_getFrameContentSize(
47+
JNIEnv *env, jclass cls,
48+
jlong src_addr, jlong src_len) {
49+
/*
50+
* Peeks the zstd frame header at src_addr to recover the declared
51+
* uncompressed size. Returns:
52+
* positive -- declared content size in bytes
53+
* -1 -- frame valid, content size not stored (ZSTD_CONTENTSIZE_UNKNOWN)
54+
* -2 -- invalid frame, truncated header, or size > INT64_MAX
55+
*
56+
* Lets the Java caller size the destination buffer in a single allocation
57+
* instead of retrying decompress on dst-too-small. Crucially, it also lets
58+
* a corrupt frame fail BEFORE any output buffer growth, eliminating a
59+
* memory-amplification vector where one bad frame would have driven
60+
* scratch growth all the way to the 64 MiB cap.
61+
*/
62+
if (src_len < 0 || (src_len > 0 && src_addr == 0)) {
63+
return -2;
64+
}
65+
unsigned long long size = ZSTD_getFrameContentSize(
66+
(const void *) (uintptr_t) src_addr, (size_t) src_len);
67+
if (size == ZSTD_CONTENTSIZE_UNKNOWN) {
68+
return -1;
69+
}
70+
if (size == ZSTD_CONTENTSIZE_ERROR) {
71+
return -2;
72+
}
73+
if (size > (unsigned long long) INT64_MAX) {
74+
/* Cast to jlong would wrap to negative and look like an error code;
75+
* reject upfront so the caller doesn't double-interpret. */
76+
return -2;
77+
}
78+
return (jlong) size;
79+
}
80+
81+
JNIEXPORT void JNICALL Java_io_questdb_client_std_Zstd_freeDCtx(
82+
JNIEnv *env, jclass cls, jlong ptr) {
83+
if (ptr != 0) {
84+
ZSTD_freeDCtx((ZSTD_DCtx *) (uintptr_t) ptr);
85+
}
86+
}
87+
88+
JNIEXPORT jlong JNICALL Java_io_questdb_client_std_Zstd_decompress(
89+
JNIEnv *env, jclass cls,
90+
jlong ctx,
91+
jlong src_addr, jlong src_len,
92+
jlong dst_addr, jlong dst_cap) {
93+
if (ctx == 0) {
94+
return -1;
95+
}
96+
ZSTD_DCtx *dctx = (ZSTD_DCtx *) (uintptr_t) ctx;
97+
size_t n = ZSTD_decompressDCtx(
98+
dctx,
99+
(void *) (uintptr_t) dst_addr, (size_t) dst_cap,
100+
(const void *) (uintptr_t) src_addr, (size_t) src_len);
101+
if (ZSTD_isError(n)) {
102+
unsigned code = ZSTD_getErrorCode(n);
103+
return -(jlong) code;
104+
}
105+
return (jlong) n;
106+
}

0 commit comments

Comments
 (0)