Skip to content

Commit c448ebc

Browse files
razvanrazvan
committed
test: add integration test for openlineage events
1 parent 93f09bd commit c448ebc

13 files changed

Lines changed: 369 additions & 0 deletions

‎tests/templates/kuttl/openlineage/00-assert.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
timeout: 900
5+
---
6+
apiVersion: v1
7+
kind: ServiceAccount
8+
metadata:
9+
name: integration-tests-sa

‎tests/templates/kuttl/openlineage/00-patch-ns.yaml.j2‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{% if test_scenario['values']['openshift'] == 'true' %}
2+
# see https://github.com/stackabletech/issues/issues/566
3+
---
4+
apiVersion: kuttl.dev/v1beta1
5+
kind: TestStep
6+
commands:
7+
- script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}'
8+
timeout: 120
9+
{% endif %}

‎tests/templates/kuttl/openlineage/00-serviceaccount.yaml.j2‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
---
2+
kind: Role
3+
apiVersion: rbac.authorization.k8s.io/v1
4+
metadata:
5+
name: use-integration-tests-scc
6+
rules:
7+
{% if test_scenario['values']['openshift'] == "true" %}
8+
- apiGroups: ["security.openshift.io"]
9+
resources: ["securitycontextconstraints"]
10+
resourceNames: ["privileged"]
11+
verbs: ["use"]
12+
{% endif %}
13+
---
14+
apiVersion: v1
15+
kind: ServiceAccount
16+
metadata:
17+
name: integration-tests-sa
18+
---
19+
kind: RoleBinding
20+
apiVersion: rbac.authorization.k8s.io/v1
21+
metadata:
22+
name: use-integration-tests-scc
23+
subjects:
24+
- kind: ServiceAccount
25+
name: integration-tests-sa
26+
roleRef:
27+
kind: Role
28+
name: use-integration-tests-scc
29+
apiGroup: rbac.authorization.k8s.io

‎tests/templates/kuttl/openlineage/01-assert.yaml.j2‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
{% if lookup('env', 'VECTOR_AGGREGATOR') %}
5+
---
6+
apiVersion: v1
7+
kind: ConfigMap
8+
metadata:
9+
name: vector-aggregator-discovery
10+
{% endif %}

‎tests/templates/kuttl/openlineage/01-install-vector-aggregator-discovery-configmap.yaml.j2‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{% if lookup('env', 'VECTOR_AGGREGATOR') %}
2+
---
3+
apiVersion: v1
4+
kind: ConfigMap
5+
metadata:
6+
name: vector-aggregator-discovery
7+
data:
8+
ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }}
9+
{% endif %}

‎tests/templates/kuttl/openlineage/10-assert.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
timeout: 300
5+
---
6+
apiVersion: apps/v1
7+
kind: Deployment
8+
metadata:
9+
name: openlineage-receiver
10+
status:
11+
availableReplicas: 1
12+
readyReplicas: 1

‎tests/templates/kuttl/openlineage/10-install-receiver.yaml.j2‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestStep
4+
commands:
5+
# `envsubst` injects the kuttl-assigned $NAMESPACE (only $NAMESPACE, so the nginx config's own
6+
# `$` tokens are left untouched) into the SecretClass name / cert scope.
7+
- script: envsubst '$NAMESPACE' < 10_receiver.yaml | kubectl apply -n $NAMESPACE -f -

‎tests/templates/kuttl/openlineage/10_receiver.yaml.j2‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
# Lightweight OpenLineage receiver.
2+
#
3+
# Instead of a full Marquez, this is a plain nginx that accepts the OpenLineage HTTP transport's
4+
# `POST /api/v1/lineage` requests and persists each request body as a flat file. The assertion in
5+
# `30-assert.yaml` then greps those files to prove events were delivered (and, in the TLS scenario,
6+
# that the driver trusted the secret-operator-issued server cert end-to-end).
7+
#
8+
# nginx cannot write a request body to disk on a `return` (the body is discarded), so the receive
9+
# location sets `client_body_in_file_only on` and `proxy_pass`es to a local sink server; nginx then
10+
# buffers the full body to a file (kept, not cleaned) before forwarding. Bodies land in
11+
# `/tmp/lineage/`.
12+
#
13+
# `$NAMESPACE` is substituted by the `10-install-receiver` step via `envsubst` (kuttl assigns the
14+
# namespace at run time; beku leaves the literal in place).
15+
{% if test_scenario['values']['openlineage-use-tls'] == 'true' %}
16+
---
17+
# A genuine autoTls SecretClass: the secret operator generates the CA and issues the nginx server
18+
# cert. Suffixed with the (cluster-scoped) namespace to avoid collisions across parallel scenarios.
19+
apiVersion: secrets.stackable.tech/v1alpha1
20+
kind: SecretClass
21+
metadata:
22+
name: openlineage-tls-$NAMESPACE
23+
spec:
24+
backend:
25+
autoTls:
26+
ca:
27+
autoGenerate: true
28+
secret:
29+
name: openlineage-tls-ca
30+
namespace: $NAMESPACE
31+
{% endif %}
32+
---
33+
apiVersion: v1
34+
kind: ConfigMap
35+
metadata:
36+
name: openlineage-receiver-config
37+
data:
38+
nginx.conf: |
39+
worker_processes 1;
40+
error_log /dev/stderr notice;
41+
pid /tmp/nginx.pid;
42+
events { worker_connections 1024; }
43+
http {
44+
access_log /dev/stdout;
45+
client_body_temp_path /tmp/nginx-body;
46+
proxy_temp_path /tmp/nginx-proxy;
47+
fastcgi_temp_path /tmp/nginx-fastcgi;
48+
uwsgi_temp_path /tmp/nginx-uwsgi;
49+
scgi_temp_path /tmp/nginx-scgi;
50+
51+
# Internal sink: acknowledges the forwarded lineage event.
52+
server {
53+
listen 127.0.0.1:8080;
54+
location / { return 201; }
55+
}
56+
57+
server {
58+
listen 5000{% if test_scenario['values']['openlineage-use-tls'] == 'true' %} ssl{% endif %};
59+
{% if test_scenario['values']['openlineage-use-tls'] == 'true' %}
60+
ssl_certificate /stackable/tls/tls.crt;
61+
ssl_certificate_key /stackable/tls/tls.key;
62+
{% endif %}
63+
client_max_body_size 16m;
64+
65+
# Persist each OpenLineage event body to /tmp/lineage/, then acknowledge via the sink.
66+
location = /api/v1/lineage {
67+
client_body_in_file_only on;
68+
client_body_temp_path /tmp/lineage;
69+
proxy_pass http://127.0.0.1:8080;
70+
}
71+
72+
location / { return 204; }
73+
}
74+
}
75+
---
76+
apiVersion: apps/v1
77+
kind: Deployment
78+
metadata:
79+
name: openlineage-receiver
80+
labels:
81+
app: openlineage-receiver
82+
spec:
83+
replicas: 1
84+
selector:
85+
matchLabels:
86+
app: openlineage-receiver
87+
template:
88+
metadata:
89+
labels:
90+
app: openlineage-receiver
91+
spec:
92+
{% if test_scenario['values']['openshift'] != 'true' %}
93+
# On plain Kubernetes, make mounted emptyDirs writable by the nginx-unprivileged group (101).
94+
# On OpenShift this is omitted so the restricted SCC can assign an fsGroup from the project range.
95+
securityContext:
96+
fsGroup: 101
97+
{% endif %}
98+
containers:
99+
- name: nginx
100+
image: docker.io/nginxinc/nginx-unprivileged:1.27-alpine
101+
ports:
102+
- containerPort: 5000
103+
volumeMounts:
104+
- name: config
105+
mountPath: /etc/nginx/nginx.conf
106+
subPath: nginx.conf
107+
- name: tmp
108+
mountPath: /tmp
109+
{% if test_scenario['values']['openlineage-use-tls'] == 'true' %}
110+
- name: tls
111+
mountPath: /stackable/tls
112+
{% endif %}
113+
resources:
114+
requests:
115+
cpu: 100m
116+
memory: 128Mi
117+
limits:
118+
cpu: 500m
119+
memory: 128Mi
120+
volumes:
121+
- name: config
122+
configMap:
123+
name: openlineage-receiver-config
124+
- name: tmp
125+
emptyDir: {}
126+
{% if test_scenario['values']['openlineage-use-tls'] == 'true' %}
127+
- name: tls
128+
ephemeral:
129+
volumeClaimTemplate:
130+
metadata:
131+
annotations:
132+
secrets.stackable.tech/class: openlineage-tls-$NAMESPACE
133+
# `service` scope puts openlineage-receiver.<ns>.svc.cluster.local in the cert SANs,
134+
# which is exactly the host the Spark driver dials (see the SparkApplication).
135+
secrets.stackable.tech/scope: service=openlineage-receiver
136+
secrets.stackable.tech/format: tls-pem
137+
spec:
138+
accessModes: ["ReadWriteOnce"]
139+
resources:
140+
requests:
141+
storage: "1"
142+
storageClassName: secrets.stackable.tech
143+
{% endif %}
144+
---
145+
apiVersion: v1
146+
kind: Service
147+
metadata:
148+
name: openlineage-receiver
149+
labels:
150+
app: openlineage-receiver
151+
spec:
152+
selector:
153+
app: openlineage-receiver
154+
ports:
155+
- name: http
156+
port: 5000
157+
targetPort: 5000

‎tests/templates/kuttl/openlineage/20-assert.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
timeout: 900
5+
---
6+
apiVersion: spark.stackable.tech/v1alpha1
7+
kind: SparkApplication
8+
metadata:
9+
name: openlineage
10+
status:
11+
phase: Succeeded

‎tests/templates/kuttl/openlineage/20-deploy-spark-app.yaml.j2‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestStep
4+
commands:
5+
# `envsubst '$NAMESPACE'` fills in the connection host FQDN (and the TLS secretClass name) while
6+
# leaving any other `$` tokens in the PySpark script untouched.
7+
- script: envsubst '$NAMESPACE' < 20_spark-app.yaml | kubectl apply -n $NAMESPACE -f -

0 commit comments

Comments
 (0)