Skip to content

Commit 24dfb75

Browse files
razvanrazvan
committed
feat: add tls field
1 parent 6cdec1a commit 24dfb75

4 files changed

Lines changed: 198 additions & 18 deletions

File tree

‎docs/modules/spark-k8s/pages/usage-guide/openlineage.adoc‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ When configured, the operator injects everything required to make the https://op
1414
== Enabling OpenLineage
1515

1616
The backend endpoint is configured directly on the `SparkApplication` through the `host` and `port` fields.
17-
The operator points the HTTP transport at `http://<host>:<port>`:
17+
The operator points the transport at `<scheme>://<host>:<port>`:
1818

1919
[source,yaml]
2020
----
@@ -28,6 +28,11 @@ spec:
2828
port: 5000 # <3>
2929
# namespace: orders-lineage # <4>
3030
# appName: orders-by-nation # <5>
31+
# tls: # <6>
32+
# verification:
33+
# server:
34+
# caCert:
35+
# secretClass: marquez-ca
3136
...
3237
----
3338
<1> Adding the `openLineage` block enables OpenLineage event emission. Omit it entirely to leave OpenLineage off (the default), in which case the operator emits nothing OpenLineage-related and behaviour is unchanged.
@@ -36,8 +41,11 @@ spec:
3641
<4> Optional. The OpenLineage namespace events are reported under.
3742
Defaults to the application's Kubernetes namespace.
3843
<5> Optional. The stable OpenLineage job name (see <<job-name>>).
44+
<6> Optional. TLS configuration, mirroring the field of the same name on an S3 connection.
45+
When `tls.verification.server` is set the transport uses `https` instead of `http`.
3946

40-
The transport protocol is always `http`; explicit `https` support will be added later.
47+
The transport scheme is `https` when `tls.verification.server` is configured, otherwise `http`.
48+
Explicit certificate handling beyond selecting the scheme will be added later.
4149

4250
You can still override any of the injected values — including the full `spark.openlineage.transport.url` — through `sparkConf`.
4351

‎extra/crds.yaml‎

Lines changed: 100 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1970,12 +1970,14 @@ spec:
19701970
A stable OpenLineage job/application name. Setting this prevents fragmented run history
19711971
(and the intermittent `unknown` job-name bug). If unset, the operator resolves it from
19721972
`spark.app.name`, falling back to `metadata.name` (with a warning event).
1973+
TODO: maybe rename to job_name as it would be more appropriate. Trino users can put
1974+
placeholders like $QUERY_ID, $USER, $SOURCE, $CLIENT_IP.
19731975
nullable: true
19741976
type: string
19751977
host:
19761978
description: |-
1977-
Host of the OpenLineage backend the HTTP transport points at (e.g. `marquez`).
1978-
Combined with `port` into the transport URL `http://<host>:<port>`.
1979+
Host of the OpenLineage backend the transport points at (e.g. `marquez`).
1980+
Combined with `port` into the transport URL `<scheme>://<host>:<port>`.
19791981
type: string
19801982
namespace:
19811983
description: |-
@@ -1986,11 +1988,56 @@ spec:
19861988
port:
19871989
description: |-
19881990
Port of the OpenLineage backend (e.g. `5000`).
1989-
Combined with `host` into the transport URL `http://<host>:<port>`.
1991+
Combined with `host` into the transport URL `<scheme>://<host>:<port>`.
19901992
format: uint16
19911993
maximum: 65535.0
19921994
minimum: 0.0
19931995
type: integer
1996+
tls:
1997+
description: Use a TLS connection. If not specified no TLS will be used.
1998+
nullable: true
1999+
properties:
2000+
verification:
2001+
description: The verification method used to verify the certificates of the server and/or the client.
2002+
oneOf:
2003+
- required:
2004+
- none
2005+
- required:
2006+
- server
2007+
properties:
2008+
none:
2009+
description: Use TLS but don't verify certificates.
2010+
type: object
2011+
server:
2012+
description: Use TLS and a CA certificate to verify the server.
2013+
properties:
2014+
caCert:
2015+
description: CA cert to verify the server.
2016+
oneOf:
2017+
- required:
2018+
- webPki
2019+
- required:
2020+
- secretClass
2021+
properties:
2022+
secretClass:
2023+
description: |-
2024+
Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate.
2025+
Note that a SecretClass does not need to have a key but can also work with just a CA certificate,
2026+
so if you got provided with a CA cert but don't have access to the key you can still use this method.
2027+
type: string
2028+
webPki:
2029+
description: |-
2030+
Use TLS and the CA certificates trusted by the common web browsers to verify the server.
2031+
This can be useful when you e.g. use public AWS S3 or other public available services.
2032+
type: object
2033+
type: object
2034+
required:
2035+
- caCert
2036+
type: object
2037+
type: object
2038+
required:
2039+
- verification
2040+
type: object
19942041
required:
19952042
- host
19962043
- port
@@ -6614,12 +6661,14 @@ spec:
66146661
A stable OpenLineage job/application name. Setting this prevents fragmented run history
66156662
(and the intermittent `unknown` job-name bug). If unset, the operator resolves it from
66166663
`spark.app.name`, falling back to `metadata.name` (with a warning event).
6664+
TODO: maybe rename to job_name as it would be more appropriate. Trino users can put
6665+
placeholders like $QUERY_ID, $USER, $SOURCE, $CLIENT_IP.
66176666
nullable: true
66186667
type: string
66196668
host:
66206669
description: |-
6621-
Host of the OpenLineage backend the HTTP transport points at (e.g. `marquez`).
6622-
Combined with `port` into the transport URL `http://<host>:<port>`.
6670+
Host of the OpenLineage backend the transport points at (e.g. `marquez`).
6671+
Combined with `port` into the transport URL `<scheme>://<host>:<port>`.
66236672
type: string
66246673
namespace:
66256674
description: |-
@@ -6630,11 +6679,56 @@ spec:
66306679
port:
66316680
description: |-
66326681
Port of the OpenLineage backend (e.g. `5000`).
6633-
Combined with `host` into the transport URL `http://<host>:<port>`.
6682+
Combined with `host` into the transport URL `<scheme>://<host>:<port>`.
66346683
format: uint16
66356684
maximum: 65535.0
66366685
minimum: 0.0
66376686
type: integer
6687+
tls:
6688+
description: Use a TLS connection. If not specified no TLS will be used.
6689+
nullable: true
6690+
properties:
6691+
verification:
6692+
description: The verification method used to verify the certificates of the server and/or the client.
6693+
oneOf:
6694+
- required:
6695+
- none
6696+
- required:
6697+
- server
6698+
properties:
6699+
none:
6700+
description: Use TLS but don't verify certificates.
6701+
type: object
6702+
server:
6703+
description: Use TLS and a CA certificate to verify the server.
6704+
properties:
6705+
caCert:
6706+
description: CA cert to verify the server.
6707+
oneOf:
6708+
- required:
6709+
- webPki
6710+
- required:
6711+
- secretClass
6712+
properties:
6713+
secretClass:
6714+
description: |-
6715+
Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate.
6716+
Note that a SecretClass does not need to have a key but can also work with just a CA certificate,
6717+
so if you got provided with a CA cert but don't have access to the key you can still use this method.
6718+
type: string
6719+
webPki:
6720+
description: |-
6721+
Use TLS and the CA certificates trusted by the common web browsers to verify the server.
6722+
This can be useful when you e.g. use public AWS S3 or other public available services.
6723+
type: object
6724+
type: object
6725+
required:
6726+
- caCert
6727+
type: object
6728+
type: object
6729+
required:
6730+
- verification
6731+
type: object
66386732
required:
66396733
- host
66406734
- port

‎rust/operator-binary/src/crd/mod.rs‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1751,6 +1751,65 @@ spec:
17511751
assert!(command.contains(r#"--conf "spark.openlineage.namespace=custom-ns""#));
17521752
}
17531753

1754+
#[test]
1755+
fn test_openlineage_uses_https_when_tls_verification_set() {
1756+
let yaml = indoc! {r#"
1757+
---
1758+
apiVersion: spark.stackable.tech/v1alpha1
1759+
kind: SparkApplication
1760+
metadata:
1761+
name: spark-examples
1762+
namespace: default
1763+
spec:
1764+
mode: cluster
1765+
mainApplicationFile: test.py
1766+
sparkImage:
1767+
productVersion: 1.2.3
1768+
openLineage:
1769+
host: marquez
1770+
port: 5000
1771+
tls:
1772+
verification:
1773+
server:
1774+
caCert:
1775+
secretClass: marquez-ca
1776+
"#};
1777+
let command = build_command_with_openlineage(yaml);
1778+
1779+
assert!(
1780+
command.contains(r#"--conf "spark.openlineage.transport.url=https://marquez:5000""#)
1781+
);
1782+
}
1783+
1784+
#[test]
1785+
fn test_openlineage_stays_http_without_tls_verification() {
1786+
// TLS present but with `verification: none` → no server verification → still `http`.
1787+
let yaml = indoc! {r#"
1788+
---
1789+
apiVersion: spark.stackable.tech/v1alpha1
1790+
kind: SparkApplication
1791+
metadata:
1792+
name: spark-examples
1793+
namespace: default
1794+
spec:
1795+
mode: cluster
1796+
mainApplicationFile: test.py
1797+
sparkImage:
1798+
productVersion: 1.2.3
1799+
openLineage:
1800+
host: marquez
1801+
port: 5000
1802+
tls:
1803+
verification:
1804+
none: {}
1805+
"#};
1806+
let command = build_command_with_openlineage(yaml);
1807+
1808+
assert!(
1809+
command.contains(r#"--conf "spark.openlineage.transport.url=http://marquez:5000""#)
1810+
);
1811+
}
1812+
17541813
#[rstest]
17551814
#[case::explicit_app_name(
17561815
indoc! {r#"

‎rust/operator-binary/src/openlineage.rs‎

Lines changed: 29 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,31 +6,39 @@ use std::collections::BTreeMap;
66

77
use serde::{Deserialize, Serialize};
88
use snafu::OptionExt;
9-
use stackable_operator::schemars::{self, JsonSchema};
9+
use stackable_operator::{
10+
commons::tls_verification::TlsClientDetails,
11+
schemars::{self, JsonSchema},
12+
};
1013

1114
use crate::crd::{Error, ObjectHasNoNameSnafu, v1alpha1};
1215

1316
/// OpenLineage lineage emission for a [`SparkApplication`].
1417
///
1518
/// Adding this block enables OpenLineage: the operator injects the OpenLineage Spark listener,
16-
/// points its HTTP transport at `http://<host>:<port>`, and sets a stable job name. Omit the block
19+
/// points its transport at `<scheme>://<host>:<port>`, and sets a stable job name. Omit the block
1720
/// to leave OpenLineage off. The injected `namespace` and `appName` are defaults that can be
1821
/// overridden via `sparkConf`.
1922
///
20-
/// The transport protocol is always `http`; explicit `https` support will be added later.
23+
/// The transport scheme is `https` when `tls.verification.server` is configured, otherwise `http`.
2124
///
2225
/// [`SparkApplication`]: crate::crd::v1alpha1::SparkApplication
23-
#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)]
26+
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
2427
#[serde(rename_all = "camelCase")]
2528
pub struct OpenLineageSpec {
26-
/// Host of the OpenLineage backend the HTTP transport points at (e.g. `marquez`).
27-
/// Combined with `port` into the transport URL `http://<host>:<port>`.
29+
/// Host of the OpenLineage backend the transport points at (e.g. `marquez`).
30+
/// Combined with `port` into the transport URL `<scheme>://<host>:<port>`.
2831
pub host: String,
2932

3033
/// Port of the OpenLineage backend (e.g. `5000`).
31-
/// Combined with `host` into the transport URL `http://<host>:<port>`.
34+
/// Combined with `host` into the transport URL `<scheme>://<host>:<port>`.
3235
pub port: u16,
3336

37+
/// TLS configuration for the connection to the OpenLineage backend. When
38+
/// `tls.verification.server` is set, the transport uses `https` instead of `http`.
39+
#[serde(flatten)]
40+
pub tls: TlsClientDetails,
41+
3442
/// The OpenLineage namespace lineage is reported under.
3543
/// Defaults to the application's Kubernetes namespace (`metadata.namespace`).
3644
#[serde(default, skip_serializing_if = "Option::is_none")]
@@ -39,15 +47,26 @@ pub struct OpenLineageSpec {
3947
/// A stable OpenLineage job/application name. Setting this prevents fragmented run history
4048
/// (and the intermittent `unknown` job-name bug). If unset, the operator resolves it from
4149
/// `spark.app.name`, falling back to `metadata.name` (with a warning event).
50+
/// TODO: maybe rename to job_name as it would be more appropriate. Trino users can put
51+
/// placeholders like $QUERY_ID, $USER, $SOURCE, $CLIENT_IP.
4252
#[serde(default, skip_serializing_if = "Option::is_none")]
4353
pub app_name: Option<String>,
4454
}
4555

4656
impl OpenLineageSpec {
47-
/// The OpenLineage HTTP transport URL, built from `host` and `port`. The protocol is always
48-
/// `http` (explicit `https` support will be added later).
57+
/// The OpenLineage transport URL, built from `host` and `port`. The scheme is `https` when
58+
/// `tls.verification.server` is configured, otherwise `http`.
4959
pub fn transport_url(&self) -> String {
50-
format!("http://{host}:{port}", host = self.host, port = self.port)
60+
let scheme = if self.tls.uses_tls_verification() {
61+
"https"
62+
} else {
63+
"http"
64+
};
65+
format!(
66+
"{scheme}://{host}:{port}",
67+
host = self.host,
68+
port = self.port
69+
)
5170
}
5271
}
5372

0 commit comments

Comments
 (0)