feat: add openlineage for 3.5.8 image

Author: sweb

CHANGELOG.md

@@ -24,7 +24,7 @@ All notable changes to this project will be documented in this file.
 - druid: Add `37.0.0` ([#1535]).
 - hbase: Add `2.6.6` ([#1547]).
 - spark: Add `4.1.2` ([#1550]).
-- spark: Bake the OpenLineage Spark listener (`openlineage-spark_2.13` `1.51.0`) into the `4.1.x` images under the dedicated directory `/stackable/spark/openlineage/` (off the `extra-jars` classpath), enabling the OpenLineage MVP in the operator ([#XXXX]).
+- spark: Bake the OpenLineage Spark listener (`1.51.0`) into the Spark images under the dedicated directory `/stackable/spark/openlineage/` (off the `extra-jars` classpath), enabling the OpenLineage MVP in the operator. The Scala 2.13 build (`openlineage-spark_2.13`) is used for the `4.1.x` images and the Scala 2.12 build (`openlineage-spark_2.12`) for the `3.5.x` images ([#XXXX]).
 - opensearch: Add `3.6.0` ([#1549]).
 - opensearch-dashboards: Add `3.6.0` ([#1551]).
 

spark-k8s/Dockerfile.3

@@ -47,6 +47,7 @@ ARG HBASE_VERSION
 ARG JACKSON_DATAFORMAT_XML_VERSION
 ARG STAX2_API_VERSION
 ARG WOODSTOX_CORE_VERSION
+ARG OPENLINEAGE_SPARK_VERSION
 ARG JMX_EXPORTER_VERSION
 ARG TARGETARCH
 ARG TINI_VERSION
@@ -148,6 +149,17 @@ mvn dependency:get -Dartifact=mvn dependency:get -Dartifact=com.fasterxml.woodst
 cp /root/.m2/repository/com/fasterxml/woodstox/woodstox-core/${WOODSTOX_CORE_VERSION}/woodstox-core-${WOODSTOX_CORE_VERSION}.jar \
   /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/extra-jars/woodstox-core-${WOODSTOX_CORE_VERSION}.jar
 
+# Download the OpenLineage Spark listener jar into a DEDICATED directory (dist/openlineage),
+# NOT dist/extra-jars. The operator references it via `--jars local:///stackable/spark/openlineage/...`
+# so it loads on Spark's child/user classloader, sharing it with connectors delivered via `--packages`
+# (e.g. Iceberg). If it landed on extra-jars it would be added to the system classloader via
+# extraClassPath, OpenLineage's connector probe would fail, and no datasets would be emitted.
+# Spark 3.5.x is a Scala 2.12 build, so the 2.12 artifact is used (Spark 4.x uses 2.13).
+mkdir -p /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage
+mvn dependency:get -Dartifact=io.openlineage:openlineage-spark_2.12:${OPENLINEAGE_SPARK_VERSION}
+cp /root/.m2/repository/io/openlineage/openlineage-spark_2.12/${OPENLINEAGE_SPARK_VERSION}/openlineage-spark_2.12-${OPENLINEAGE_SPARK_VERSION}.jar \
+  /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage/openlineage-spark_2.12-${OPENLINEAGE_SPARK_VERSION}.jar
+
 # Get the correct `tini` binary for our architecture.
 curl --fail "https://repo.stackable.tech/repository/packages/tini/tini-${TINI_VERSION}-${TARGETARCH}" \
   -o /usr/bin/tini

spark-k8s/boil-config.toml

@@ -20,6 +20,7 @@ woodstox-core-version = "6.5.1"           # Needs to match https://mvnrepository
 jmx-exporter-version = "1.3.0"
 tini-version = "0.19.0"
 hbase-connector-version = "1.0.1_3.5.8"
+openlineage-spark-version = "1.51.0"      # Scala 2.12 build for Spark 3.5.x; see https://mvnrepository.com/artifact/io.openlineage/openlineage-spark_2.12
 
 # Deprecated since 26.7
 [versions."4.1.1"]