Skip to content

Commit 38a618d

Browse files
committed
Pass --add-opens JVM flags to Spark job server
1 parent 477c747 commit 38a618d

3 files changed

Lines changed: 24 additions & 3 deletions

File tree

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"trigger-2026-04-04": "portable_runner expand_sdf opt-in"
2+
"trigger-2026-07-08": "portable_runner expand_sdf opt-in 2"
33
}

runners/spark/job-server/spark_job_server.gradle

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -267,13 +267,28 @@ tasks.register("validatesRunnerSickbay", Test) {
267267
def jobPort = BeamModulePlugin.getRandomPort()
268268
def artifactPort = BeamModulePlugin.getRandomPort()
269269

270+
def sparkJobServerJvmArgs() {
271+
def testJavaVer = project.findProperty('testJavaVersion') ? (project.property('testJavaVersion') as int) : JavaVersion.current().majorVersion.toInteger()
272+
if (testJavaVer >= 17) {
273+
return [
274+
"--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
275+
"--add-opens=java.base/java.nio=ALL-UNNAMED",
276+
"--add-opens=java.base/java.util=ALL-UNNAMED",
277+
"--add-opens=java.base/java.lang.invoke=ALL-UNNAMED"
278+
]
279+
}
280+
return []
281+
}
282+
270283
def setupTask = project.tasks.register("sparkJobServerSetup", Exec) {
271284
dependsOn shadowJar
272285
def pythonDir = project.project(":sdks:python").projectDir
273286
def sparkJobServerJar = shadowJar.archivePath
287+
def jvmArgs = sparkJobServerJvmArgs().join(' ')
288+
def jvmArgsOpt = jvmArgs ? "--jvm_args \"${jvmArgs}\"" : ""
274289

275290
executable 'sh'
276-
args '-c', "$pythonDir/scripts/run_job_server.sh stop --group_id ${project.name} && $pythonDir/scripts/run_job_server.sh start --group_id ${project.name} --job_port ${jobPort} --artifact_port ${artifactPort} --job_server_jar ${sparkJobServerJar}"
291+
args '-c', "$pythonDir/scripts/run_job_server.sh stop --group_id ${project.name} && $pythonDir/scripts/run_job_server.sh start --group_id ${project.name} --job_port ${jobPort} --artifact_port ${artifactPort} --job_server_jar ${sparkJobServerJar} ${jvmArgsOpt}"
277292
}
278293

279294
def cleanupTask = project.tasks.register("sparkJobServerCleanup", Exec) {

sdks/python/scripts/run_job_server.sh

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ Options:
2323
--job_port [port for job endpoint, default 8099]
2424
--artifact_port [port for artifact service, default 8098]
2525
--job_server_jar [path to job server jar]
26+
--jvm_args [additional JVM arguments, e.g. --add-opens flags]
2627
END
2728

2829
JOB_PORT=8099
@@ -61,6 +62,11 @@ while [[ $# -gt 0 ]]; do
6162
shift
6263
shift
6364
;;
65+
--jvm_args)
66+
JVM_ARGS="$2"
67+
shift
68+
shift
69+
;;
6470
start)
6571
STARTSTOP="$1"
6672
shift
@@ -107,7 +113,7 @@ case $STARTSTOP in
107113
fi
108114

109115
echo "Launching job server @ $JOB_PORT ..."
110-
"$JAVA_CMD" -jar $JOB_SERVER_JAR --job-port=$JOB_PORT --artifact-port=$ARTIFACT_PORT --expansion-port=0 $ADDITIONAL_ARGS >$TEMP_DIR/$FILE_BASE.log 2>&1 </dev/null &
116+
"$JAVA_CMD" $JVM_ARGS -jar $JOB_SERVER_JAR --job-port=$JOB_PORT --artifact-port=$ARTIFACT_PORT --expansion-port=0 $ADDITIONAL_ARGS >$TEMP_DIR/$FILE_BASE.log 2>&1 </dev/null &
111117
mypid=$!
112118
if kill -0 $mypid >/dev/null 2>&1; then
113119
echo $mypid >> $pid

0 commit comments

Comments
 (0)