Skip to content

Commit cb5467f

Browse files
sbernauerrazvan
sbernauer
and
razvan
authored
fix: Correctly propagate user arguments to Spark connect server (#696)
* fix: Correctly propage user arguments to Spark connect server * update assert --------- Co-authored-by: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>
1 parent 5e472b0 commit cb5467f

17 files changed

Lines changed: 205 additions & 46 deletions

‎docs/modules/spark-k8s/examples/example-spark-connect.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ spec:
88
productVersion: "3.5.8" # <2>
99
pullPolicy: IfNotPresent
1010
args:
11-
- "--package org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1" # <3>
11+
- "--packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.11.0" # <3>
1212
server:
1313
podOverrides:
1414
spec:

‎docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -113,14 +113,13 @@ metadata:
113113
spec:
114114
sparkConf:
115115
spark.sql.extensions: org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
116-
spark.sql.catalog.spark_catalog: org.apache.iceberg.spark.SparkSessionCatalog
117-
spark.sql.catalog.spark_catalog.type: hive
118-
spark.sql.catalog.local: org.apache.iceberg.spark.SparkCatalog
119-
spark.sql.catalog.local.type: hadoop
120-
spark.sql.catalog.local.warehouse: /tmp/warehouse
116+
spark.sql.catalog.lakehouse: org.apache.iceberg.spark.SparkCatalog
117+
spark.sql.catalog.lakehouse.type: hive
118+
spark.sql.catalog.lakehouse.uri: thrift://hive-iceberg-metastore:9083
119+
spark.sql.defaultCatalog: lakehouse
121120
deps:
122121
packages:
123-
- org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1 # <1>
122+
- org.apache.iceberg:iceberg-spark-runtime-4.1_2.13:1.11.0 # <1>
124123
...
125124
----
126125

‎rust/operator-binary/src/connect/server.rs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -432,7 +432,7 @@ pub(crate) fn build_stateful_set(
432432

433433
#[allow(clippy::result_large_err)]
434434
pub(crate) fn command_args(user_args: &[String]) -> Vec<String> {
435-
let mut command = vec![formatdoc! { "
435+
let mut command = formatdoc! { "
436436
containerdebug --output={VOLUME_MOUNT_PATH_LOG}/containerdebug-state.json --loop &
437437
438438
cp {VOLUME_MOUNT_PATH_CONFIG}/{SPARK_DEFAULTS_FILE_NAME} /tmp/spark.properties
@@ -441,13 +441,13 @@ pub(crate) fn command_args(user_args: &[String]) -> Vec<String> {
441441
/stackable/spark/sbin/start-connect-server.sh \\
442442
--deploy-mode client \\
443443
--master k8s://https://${{KUBERNETES_SERVICE_HOST}}:${{KUBERNETES_SERVICE_PORT_HTTPS}} \\
444-
--properties-file /tmp/spark.properties
445-
" }];
444+
--properties-file /tmp/spark.properties"};
446445

447-
// User provided command line arguments
448-
command.extend_from_slice(user_args);
449-
450-
vec![command.join(" ")]
446+
// Append user-provided command line arguments as continuation lines.
447+
for user_arg in user_args {
448+
command.push_str(&format!(" \\\n{user_arg}"));
449+
}
450+
vec![command]
451451
}
452452

453453
#[allow(clippy::result_large_err)]

‎tests/release.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ releases:
1414
operatorVersion: 0.0.0-dev
1515
zookeeper:
1616
operatorVersion: 0.0.0-dev
17+
hive:
18+
operatorVersion: 0.0.0-dev
1719
hdfs:
1820
operatorVersion: 0.0.0-dev
1921
hbase:

‎tests/templates/kuttl/iceberg/10-deploy-spark-app.yaml.j2‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,9 @@ spec:
4242
deps:
4343
packages:
4444
{% if test_scenario['values']['spark-iceberg'].startswith("4") %}
45-
- org.apache.iceberg:iceberg-spark-runtime-{{ ".".join(test_scenario['values']['spark-iceberg'].split('.')[:2]) }}_2.13:1.10.1
45+
- org.apache.iceberg:iceberg-spark-runtime-{{ ".".join(test_scenario['values']['spark-iceberg'].split('.')[:2]) }}_2.13:{{ test_scenario['values']['iceberg-latest'] }}
4646
{% else %}
47-
- org.apache.iceberg:iceberg-spark-runtime-{{ ".".join(test_scenario['values']['spark-iceberg'].split('.')[:2]) }}_2.12:1.10.1
47+
- org.apache.iceberg:iceberg-spark-runtime-{{ ".".join(test_scenario['values']['spark-iceberg'].split('.')[:2]) }}_2.12:{{ test_scenario['values']['iceberg-latest'] }}
4848
{% endif %}
4949
volumes:
5050
- name: script

tests/templates/kuttl/spark-connect/04-minio-secrets.yaml renamed to tests/templates/kuttl/spark-connect/04-minio-secrets.yaml.j2

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,3 +41,30 @@ metadata:
4141
stringData:
4242
accessKey: spark
4343
secretKey: sparkspark
44+
---
45+
apiVersion: s3.stackable.tech/v1alpha1
46+
kind: S3Connection
47+
metadata:
48+
name: minio
49+
spec:
50+
host: minio
51+
port: 9000
52+
accessStyle: Path
53+
credentials:
54+
secretClass: minio-credentials-class
55+
{% if test_scenario['values']['s3-use-tls'] == 'true' %}
56+
tls:
57+
verification:
58+
server:
59+
caCert:
60+
secretClass: minio-tls-ca
61+
{% endif %}
62+
---
63+
apiVersion: s3.stackable.tech/v1alpha1
64+
kind: S3Bucket
65+
metadata:
66+
name: ingest-bucket
67+
spec:
68+
bucketName: ingest-bucket
69+
connection:
70+
reference: minio

‎tests/templates/kuttl/spark-connect/06-assert.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
timeout: 600
5+
---
6+
apiVersion: apps/v1
7+
kind: StatefulSet
8+
metadata:
9+
name: postgresql-hive
10+
status:
11+
readyReplicas: 1
12+
replicas: 1

‎tests/templates/kuttl/spark-connect/06-install-hive-postgres.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestStep
4+
timeout: 300
5+
commands:
6+
- script: >-
7+
helm upgrade postgresql-hive
8+
--install
9+
--version=12.5.6
10+
--namespace $NAMESPACE
11+
-f 06_helm-bitnami-postgresql-values.yaml
12+
--repo https://charts.bitnami.com/bitnami postgresql

‎tests/templates/kuttl/spark-connect/06_helm-bitnami-postgresql-values.yaml.j2‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
---
2+
global:
3+
security:
4+
allowInsecureImages: true # needed starting with Chart version 16.3.0 if modifying images
5+
6+
image:
7+
repository: bitnamilegacy/postgresql
8+
9+
volumePermissions:
10+
enabled: false
11+
image:
12+
repository: bitnamilegacy/os-shell
13+
securityContext:
14+
runAsUser: auto
15+
16+
metrics:
17+
image:
18+
repository: bitnamilegacy/postgres-exporter
19+
20+
primary:
21+
extendedConfiguration: |
22+
password_encryption=md5
23+
podSecurityContext:
24+
{% if test_scenario['values']['openshift'] == 'true' %}
25+
enabled: false
26+
{% else %}
27+
enabled: true
28+
{% endif %}
29+
containerSecurityContext:
30+
enabled: false
31+
resources:
32+
requests:
33+
memory: "512Mi"
34+
cpu: "512m"
35+
limits:
36+
memory: "512Mi"
37+
cpu: "1"
38+
39+
auth:
40+
username: hive
41+
password: hive
42+
database: hive

‎tests/templates/kuttl/spark-connect/07-assert.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
timeout: 900
5+
---
6+
apiVersion: apps/v1
7+
kind: StatefulSet
8+
metadata:
9+
name: hive-metastore-default
10+
status:
11+
readyReplicas: 1
12+
replicas: 1

0 commit comments

Comments
 (0)