add authentication with bearer token

Author: razvan

Cargo.lock

@@ -193,6 +193,24 @@ rules:
       - get
       - list
       - watch
+  # OpenLineage backend connections referenced by SparkApplications.
+  - apiGroups:
+      - openlineage.stackable.tech
+    resources:
+      - openlineageconnections
+    verbs:
+      - get
+      - list
+      - watch
+  # AuthenticationClasses used to authenticate against an OpenLineage backend.
+  - apiGroups:
+      - authentication.stackable.tech
+    resources:
+      - authenticationclasses
+    verbs:
+      - get
+      - list
+      - watch
   # Required for managing how the History Server and Connect Server are exposed
   # outside of the cluster.
   - apiGroups:

Cargo.nix

@@ -68,6 +68,65 @@ trust store, so the OpenLineage listener trusts the backend's certificate.
 
 You can still override any of the injected values — including the full `spark.openlineage.transport.url` — through `sparkConf`.
 
+[#authentication]
+== Authentication
+
+Backends that require authentication (for example DataHub, or a Marquez instance behind a gateway) expect a bearer token on each request.
+Configure this by referencing an xref:concepts:authentication.adoc[`AuthenticationClass`] from the connection via `authenticationClassRef`:
+
+[source,yaml]
+----
+apiVersion: spark.stackable.tech/v1alpha1
+kind: SparkApplication
+metadata:
+  name: orders-by-nation
+spec:
+  openLineage:
+    connection:
+      inline:
+        host: marquez
+        port: 5000
+        authenticationClassRef: openlineage-auth # <1>
+  ...
+---
+apiVersion: authentication.stackable.tech/v1alpha1
+kind: AuthenticationClass
+metadata:
+  name: openlineage-auth
+spec:
+  provider:
+    static: # <2>
+      userCredentialsSecret:
+        name: openlineage-token # <3>
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: openlineage-token
+  namespace: default # <4>
+stringData:
+  apiKey: "s3cr3t-bearer-token" # <5>
+----
+<1> References an `AuthenticationClass` by name (cluster-scoped). If omitted, no authentication is used.
+<2> Only the `static` provider is supported for OpenLineage. Other providers (LDAP, OIDC, TLS) are rejected with an error.
+<3> Name of the Secret holding the token. Must exist in the same namespace as the `SparkApplication`.
+<4> The Secret must be in the workload's namespace so it can be projected into the driver.
+<5> The token must be stored under the fixed key `apiKey`. The operator never reads the Secret; it is injected into the driver's OpenLineage listener via a `secretKeyRef` environment variable, so the token never appears in the spark-submit arguments or the pod spec.
+
+The operator translates this into the OpenLineage HTTP transport's `api_key` (bearer) authentication.
+Combine it with the `tls` block above when the backend also needs TLS server verification.
+
+[NOTE]
+====
+When `authenticationClassRef` is set, the operator delivers the *entire* transport configuration
+(type, URL and auth) to the driver through `OPENLINEAGE__*` environment variables rather than
+`spark.openlineage.transport.*` `--conf` properties.
+This is required: OpenLineage resolves the `transport` from a single source, so a transport defined
+via `sparkConf` would discard the environment-provided authentication.
+Consequently, do **not** override `spark.openlineage.transport.*` via `sparkConf` when using
+`authenticationClassRef` — doing so drops the credentials and the backend will reject the events.
+====
+
 [#job-name]
 == Job name
 

crate-hashes.json

@@ -1987,6 +1987,15 @@ spec:
                           OpenLineage connection definition as a resource.
                           Learn more about [OpenLineage](https://openlineage.io/).
                         properties:
+                          authenticationClassRef:
+                            description: |-
+                              Name of an [`AuthenticationClass`](https://docs.stackable.tech/home/nightly/concepts/authentication) used
+                              to authenticate against the OpenLineage backend. The `AuthenticationClass` is cluster-scoped
+                              and referenced by name; it is resolved at runtime via
+                              [`OpenLineageConnectionSpec::resolve_authentication_class`]. If not specified, no
+                              authentication is used.
+                            nullable: true
+                            type: string
                           host:
                             description: 'Host of the OpenLineage backend without any protocol or port. For example: `marquez`.'
                             type: string
@@ -6693,6 +6702,15 @@ spec:
                           OpenLineage connection definition as a resource.
                           Learn more about [OpenLineage](https://openlineage.io/).
                         properties:
+                          authenticationClassRef:
+                            description: |-
+                              Name of an [`AuthenticationClass`](https://docs.stackable.tech/home/nightly/concepts/authentication) used
+                              to authenticate against the OpenLineage backend. The `AuthenticationClass` is cluster-scoped
+                              and referenced by name; it is resolved at runtime via
+                              [`OpenLineageConnectionSpec::resolve_authentication_class`]. If not specified, no
+                              authentication is used.
+                            nullable: true
+                            type: string
                           host:
                             description: 'Host of the OpenLineage backend without any protocol or port. For example: `marquez`.'
                             type: string

deploy/helm/spark-k8s-operator/templates/roles.yaml

@@ -126,6 +126,37 @@ pub const OPENLINEAGE_JAR_LOCAL_URI: &str =
 /// Appended to both driver and executor `extraJavaOptions`.
 pub const OPENLINEAGE_ADD_OPENS: &str = "--add-opens java.base/java.security=ALL-UNNAMED";
 
+/// Env vars delivering the OpenLineage HTTP transport config via the OpenLineage Java client's
+/// `OPENLINEAGE__` env-var configuration (`__` separates the config hierarchy).
+///
+/// When an `AuthenticationClass` is configured, the operator delivers the **whole** transport this
+/// way rather than via `spark.openlineage.transport.*` `--conf`. This is required, not cosmetic:
+/// OpenLineage resolves `transport` from a single source, so a `--conf`-defined transport would
+/// drop the env-provided `auth` sub-tree entirely (verified against openlineage-spark 1.51.0).
+/// Delivering the token as a `secretKeyRef` env var also keeps it out of the spark-submit args.
+pub const OPENLINEAGE_TRANSPORT_TYPE_ENV: &str = "OPENLINEAGE__TRANSPORT__TYPE";
+
+/// Env var carrying the OpenLineage HTTP transport URL. See [`OPENLINEAGE_TRANSPORT_TYPE_ENV`].
+pub const OPENLINEAGE_TRANSPORT_URL_ENV: &str = "OPENLINEAGE__TRANSPORT__URL";
+
+/// The OpenLineage transport type the operator uses (the HTTP transport).
+pub const OPENLINEAGE_TRANSPORT_TYPE_HTTP: &str = "http";
+
+/// Env var selecting the OpenLineage HTTP transport auth type. See [`OPENLINEAGE_TRANSPORT_TYPE_ENV`].
+pub const OPENLINEAGE_AUTH_TYPE_ENV: &str = "OPENLINEAGE__TRANSPORT__AUTH__TYPE";
+
+/// Env var carrying the OpenLineage HTTP transport bearer token. Sourced from the referenced
+/// Secret via `secretKeyRef` (key [`OPENLINEAGE_AUTH_SECRET_KEY`]), so the operator never reads
+/// the token and it stays out of the Job/pod spec.
+pub const OPENLINEAGE_AUTH_API_KEY_ENV: &str = "OPENLINEAGE__TRANSPORT__AUTH__API_KEY";
+
+/// The only OpenLineage HTTP transport auth type the operator supports: a static bearer token.
+pub const OPENLINEAGE_AUTH_TYPE_API_KEY: &str = "api_key";
+
+/// Fixed key that must hold the bearer token inside the Secret referenced by the Static
+/// `AuthenticationClass`.
+pub const OPENLINEAGE_AUTH_SECRET_KEY: &str = "apiKey";
+
 /// The JVM `security.properties` entries the operator sets by default (DNS cache TTLs).
 pub fn default_jvm_security_properties() -> BTreeMap<String, String> {
     [