Skip to content

Commit 6d5cdfb

Browse files
yashmayyayashmayya
authored
Verify the Java 11 pinned client artifacts actually run on a Java 11 JVM in CI (#19102)
1 parent 2506545 commit 6d5cdfb

9 files changed

Lines changed: 1982 additions & 0 deletions

File tree

‎.github/workflows/pinot_java11_client_compatibility.yml‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: Pinot Java 11 Client Compatibility
21+
22+
on:
23+
push:
24+
branches:
25+
- master
26+
paths-ignore:
27+
- "contrib/**"
28+
- "docs/**"
29+
- "docker/**"
30+
- "kubernetes/**"
31+
- "licenses/**"
32+
- "licenses-binary/**"
33+
- "**.md"
34+
pull_request:
35+
branches:
36+
- master
37+
paths-ignore:
38+
- "contrib/**"
39+
- "docs/**"
40+
- "docker/**"
41+
- "kubernetes/**"
42+
- "licenses/**"
43+
- "licenses-binary/**"
44+
- "**.md"
45+
46+
concurrency:
47+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
48+
cancel-in-progress: true
49+
50+
jobs:
51+
# pinot-spi, pinot-segment-spi, pinot-timeseries-spi, pinot-common, pinot-java-client and
52+
# pinot-jdbc-client hard-code <release>11</release> so that third-party plugins and applications
53+
# embedding the Java/JDBC client are not forced onto the JDK that Pinot's services require.
54+
#
55+
# --release 11 keeps Pinot's own code in those modules Java 11 clean, but nothing checks their
56+
# transitive dependency closure. A routine dependency bump can drop a Java 17+ jar into the client
57+
# classpath, and today the first sign of that would be a user reporting
58+
# UnsupportedClassVersionError. This job loads and exercises the clients on a real Java 11 JVM.
59+
java11-client-compatibility:
60+
if: github.repository == 'apache/pinot'
61+
runs-on: ubuntu-latest
62+
strategy:
63+
fail-fast: false
64+
matrix:
65+
# build-java is the JDK the artifacts are compiled with; the verifier always runs on the
66+
# Java version the clients are pinned to.
67+
build-java: [ 25 ]
68+
distribution: [ "temurin" ]
69+
name: Pinot Java 11 Client Compatibility (built on JDK ${{ matrix.build-java }}-${{ matrix.distribution }})
70+
steps:
71+
- uses: actions/checkout@v7
72+
# Installed first so that the build JDK below ends up as JAVA_HOME. setup-java also exports
73+
# JAVA_HOME_11_<arch>, which the script falls back on if this step's output is unavailable.
74+
- name: Set up Java 11 (verification runtime)
75+
id: java11
76+
uses: actions/setup-java@v5
77+
with:
78+
java-version: 11
79+
distribution: ${{ matrix.distribution }}
80+
- name: Set up JDK ${{ matrix.build-java }} (build)
81+
uses: actions/setup-java@v5
82+
with:
83+
java-version: ${{ matrix.build-java }}
84+
distribution: ${{ matrix.distribution }}
85+
cache: 'maven'
86+
- uses: actions/cache@v6
87+
env:
88+
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
89+
with:
90+
path: ~/.m2/repository
91+
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
92+
restore-keys: |
93+
${{ runner.os }}-maven-
94+
- name: Verify the Java 11 clients on a Java 11 JVM
95+
timeout-minutes: 40
96+
env:
97+
JAVA11_HOME: ${{ steps.java11.outputs.path }}
98+
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
99+
# Same download hardening the other Maven jobs use, so a transient Central hiccup retries
100+
# here too instead of failing the job.
101+
#
102+
# Only JVM arguments and -D system properties belong here: MAVEN_OPTS is handed to the JVM
103+
# that runs Maven, not to Maven itself. The peer jobs also list -B and -ntp, which the JVM
104+
# rejects outright ("Unrecognized option: -B") and which only survive there because those
105+
# jobs additionally pass -XX:+IgnoreUnrecognizedVMOptions. The script passes -B -ntp on the
106+
# mvn command line where they belong, so they are left out rather than masked.
107+
#
108+
# Deliberately no -DskipShade either: the script passes -Dshade.phase.prop=none, which
109+
# disables shading for both client modules (-DskipShade only covers pinot-jdbc-client).
110+
MAVEN_OPTS: >
111+
-Xmx2G -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
112+
-Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
113+
run: |
114+
.github/workflows/scripts/.pinot_java11_client_compat.sh

‎.github/workflows/scripts/.pinot_java11_client_compat.sh‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#!/bin/bash -x
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
#
20+
# Verifies that Pinot's Java-11-pinned client and SPI artifacts work on a Java 11 JVM.
21+
#
22+
# The build cannot run on Java 11 -- the root pom enforces requireJavaVersion [25,) and Pinot's
23+
# services have a genuine Java 25 floor. So this is necessarily a two-JDK job: build the artifacts
24+
# with the build JDK, then run the verifier under Java 11.
25+
#
26+
# Environment:
27+
# JAVA11_HOME Optional. Home of the JVM to verify against. Falls back to the
28+
# JAVA_HOME_11_<arch> variables that GitHub-hosted runners export.
29+
# MVN Optional. Maven command, defaults to "mvn".
30+
31+
TARGET_JAVA_VERSION=11
32+
VERIFIER_MODULE="pinot-java11-client-verifier"
33+
VERIFIER_MAIN_CLASS="org.apache.pinot.java11.Java11CompatibilityVerifier"
34+
MVN="${MVN:-mvn}"
35+
36+
# Build JDK, for the record.
37+
java -version
38+
39+
# GitHub-hosted runners export JAVA_HOME_<version>_<arch> for every JDK that setup-java installed.
40+
# The arch suffix differs between x64 and arm64 runners, so try both rather than assuming.
41+
if [ -z "${JAVA11_HOME}" ]; then
42+
JAVA11_HOME="${JAVA_HOME_11_X64:-${JAVA_HOME_11_ARM64:-}}"
43+
fi
44+
if [ -z "${JAVA11_HOME}" ]; then
45+
echo "No Java ${TARGET_JAVA_VERSION} JVM found. Set JAVA11_HOME, or install one with actions/setup-java" \
46+
"and pass its path through."
47+
exit 1
48+
fi
49+
50+
JAVA11_BIN="${JAVA11_HOME}/bin/java"
51+
if [ ! -x "${JAVA11_BIN}" ]; then
52+
echo "Not an executable JVM launcher: ${JAVA11_BIN}"
53+
exit 1
54+
fi
55+
"${JAVA11_BIN}" -version || exit 1
56+
57+
# Build the verifier and everything it depends on, which is exactly the six Java-11-pinned modules
58+
# and the third-party closure underneath them. Linting is covered by the linter job, so skip it here.
59+
#
60+
# -Dshade.phase.prop=none matters: pinot-java-client and pinot-jdbc-client each produce a ~150 MB
61+
# shaded jar, and shadedArtifactAttached=true means those jars never even appear on the runtime
62+
# classpath this job verifies. Without the flag the job spends minutes building them and then pushes
63+
# 300 MB into ~/.m2, which actions/cache uploads under a key the other Maven jobs share. Note that
64+
# -DskipShade=true is not enough: it only deactivates the pinot-jdbc-client profile, while
65+
# pinot-java-client sets shade.phase.prop=package unconditionally.
66+
${MVN} clean install -B -ntp -T1C -pl "${VERIFIER_MODULE}" -am \
67+
-DskipTests \
68+
-Dshade.phase.prop=none \
69+
-Dmaven.javadoc.skip=true \
70+
-Dlicense.skip=true \
71+
-Dcheckstyle.skip=true \
72+
-Dspotless.check.skip=true || exit 1
73+
74+
CLASSPATH_FILE="${VERIFIER_MODULE}/target/runtime-classpath.txt"
75+
if [ ! -s "${CLASSPATH_FILE}" ]; then
76+
echo "Expected the build to write the resolved runtime closure to ${CLASSPATH_FILE}"
77+
exit 1
78+
fi
79+
80+
VERIFIER_CLASSPATH="${VERIFIER_MODULE}/target/classes:$(cat "${CLASSPATH_FILE}")"
81+
82+
# Tracing off for the run itself: echoing a few hundred jar paths buries the verifier's own output.
83+
set +x
84+
echo "Running ${VERIFIER_MAIN_CLASS} on Java ${TARGET_JAVA_VERSION} against a closure of" \
85+
"$(tr ':' '\n' <<< "${VERIFIER_CLASSPATH}" | wc -l | tr -d ' ') classpath entries"
86+
87+
# No --add-opens or -Dio.netty.tryReflectionSetAccessible here on purpose. Those flags exist in
88+
# Pinot's own launch scripts for JDK 17+, and adding them would paper over exactly the kind of
89+
# runtime breakage this job is meant to catch. The verifier asserts it is really running on Java
90+
# ${TARGET_JAVA_VERSION}, so a mis-wired JDK fails the job instead of passing it vacuously.
91+
if ! "${JAVA11_BIN}" -cp "${VERIFIER_CLASSPATH}" "${VERIFIER_MAIN_CLASS}" "${TARGET_JAVA_VERSION}"; then
92+
# Hosted runners discard the workspace, so dump the closure that was verified while we still can.
93+
echo
94+
echo "Verification failed. The runtime closure that was verified:"
95+
tr ':' '\n' <<< "${VERIFIER_CLASSPATH}"
96+
exit 1
97+
fi

‎pinot-java11-client-verifier/pom.xml‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
<?xml version="1.0"?>
2+
<!--
3+
4+
Licensed to the Apache Software Foundation (ASF) under one
5+
or more contributor license agreements. See the NOTICE file
6+
distributed with this work for additional information
7+
regarding copyright ownership. The ASF licenses this file
8+
to you under the Apache License, Version 2.0 (the
9+
"License"); you may not use this file except in compliance
10+
with the License. You may obtain a copy of the License at
11+
12+
http://www.apache.org/licenses/LICENSE-2.0
13+
14+
Unless required by applicable law or agreed to in writing,
15+
software distributed under the License is distributed on an
16+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
KIND, either express or implied. See the License for the
18+
specific language governing permissions and limitations
19+
under the License.
20+
21+
-->
22+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
23+
<modelVersion>4.0.0</modelVersion>
24+
<parent>
25+
<artifactId>pinot</artifactId>
26+
<groupId>org.apache.pinot</groupId>
27+
<version>1.6.0-SNAPSHOT</version>
28+
</parent>
29+
<artifactId>pinot-java11-client-verifier</artifactId>
30+
<name>Pinot Java 11 Client Verifier</name>
31+
<url>https://pinot.apache.org/</url>
32+
<packaging>jar</packaging>
33+
34+
<properties>
35+
<pinot.root>${basedir}/..</pinot.root>
36+
<!--
37+
A CI harness, not something a consumer should depend on. Keep it out of the published
38+
artifacts; nothing in the reactor depends on it either.
39+
-->
40+
<maven.deploy.skip>true</maven.deploy.skip>
41+
</properties>
42+
43+
<build>
44+
<plugins>
45+
<plugin>
46+
<groupId>org.apache.maven.plugins</groupId>
47+
<artifactId>maven-compiler-plugin</artifactId>
48+
<configuration>
49+
<!-- Load-bearing rather than copied boilerplate: .pinot_java11_client_compat.sh launches
50+
this module's own classes on a Java 11 JVM, so its bytecode has to be Java 11 as well.
51+
Hard-coded literal so -Djdk.version=21 on the CLI cannot silently bump the level. -->
52+
<release>11</release>
53+
<source>11</source>
54+
<target>11</target>
55+
</configuration>
56+
</plugin>
57+
<plugin>
58+
<!--
59+
Writes the resolved runtime closure of this module (the union of the pinot-java-client and
60+
pinot-jdbc-client closures) to a file, so the CI driver script can hand exactly that
61+
classpath to a Java 11 JVM.
62+
-->
63+
<groupId>org.apache.maven.plugins</groupId>
64+
<artifactId>maven-dependency-plugin</artifactId>
65+
<executions>
66+
<execution>
67+
<id>write-runtime-classpath</id>
68+
<phase>prepare-package</phase>
69+
<goals>
70+
<goal>build-classpath</goal>
71+
</goals>
72+
<configuration>
73+
<includeScope>runtime</includeScope>
74+
<outputFile>${project.build.directory}/runtime-classpath.txt</outputFile>
75+
<regenerateFile>true</regenerateFile>
76+
</configuration>
77+
</execution>
78+
</executions>
79+
</plugin>
80+
</plugins>
81+
</build>
82+
83+
<dependencies>
84+
<!-- The two consumer-facing clients. Depending on both makes this module's runtime closure the
85+
union of theirs, which is what gets scanned for bytecode a Java 11 JVM cannot load. -->
86+
<dependency>
87+
<groupId>org.apache.pinot</groupId>
88+
<artifactId>pinot-java-client</artifactId>
89+
</dependency>
90+
<dependency>
91+
<groupId>org.apache.pinot</groupId>
92+
<artifactId>pinot-jdbc-client</artifactId>
93+
</dependency>
94+
95+
<!-- The remaining Java-11-pinned modules. All four already arrive transitively, so declaring
96+
them changes nothing about the closure; it is done because the verifier imports them
97+
directly, and so that dropping one breaks the build instead of silently shrinking coverage.
98+
Java11CompatibilityVerifier also asserts at runtime that all six are on the classpath. -->
99+
<dependency>
100+
<groupId>org.apache.pinot</groupId>
101+
<artifactId>pinot-common</artifactId>
102+
</dependency>
103+
<dependency>
104+
<groupId>org.apache.pinot</groupId>
105+
<artifactId>pinot-spi</artifactId>
106+
</dependency>
107+
<dependency>
108+
<groupId>org.apache.pinot</groupId>
109+
<artifactId>pinot-segment-spi</artifactId>
110+
</dependency>
111+
<dependency>
112+
<groupId>org.apache.pinot</groupId>
113+
<artifactId>pinot-timeseries-spi</artifactId>
114+
</dependency>
115+
116+
<!-- Third-party types the verifier constructs directly. -->
117+
<dependency>
118+
<groupId>org.apache.helix</groupId>
119+
<artifactId>helix-core</artifactId>
120+
</dependency>
121+
<dependency>
122+
<groupId>com.google.protobuf</groupId>
123+
<artifactId>protobuf-java</artifactId>
124+
</dependency>
125+
<dependency>
126+
<groupId>com.fasterxml.jackson.core</groupId>
127+
<artifactId>jackson-databind</artifactId>
128+
</dependency>
129+
<dependency>
130+
<groupId>io.grpc</groupId>
131+
<artifactId>grpc-api</artifactId>
132+
</dependency>
133+
134+
<dependency>
135+
<groupId>org.testng</groupId>
136+
<artifactId>testng</artifactId>
137+
<scope>test</scope>
138+
</dependency>
139+
</dependencies>
140+
</project>

0 commit comments

Comments
 (0)