Skip to content

Commit 148a5b9

Browse files
razvanrazvan
committed
feat: add openlineage kuttl test
1 parent 43a7908 commit 148a5b9

18 files changed

Lines changed: 549 additions & 0 deletions

‎tests/templates/kuttl/openlineage/00-limit-range.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
---
2+
apiVersion: v1
3+
kind: LimitRange
4+
metadata:
5+
name: limit-request-ratio
6+
spec:
7+
limits:
8+
- type: "Container"
9+
maxLimitRequestRatio:
10+
cpu: 5
11+
memory: 1

‎tests/templates/kuttl/openlineage/00-patch-ns.yaml.j2‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{% if test_scenario['values']['openshift'] == 'true' %}
2+
# see https://github.com/stackabletech/issues/issues/566
3+
---
4+
apiVersion: kuttl.dev/v1beta1
5+
kind: TestStep
6+
commands:
7+
- script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}'
8+
timeout: 120
9+
{% endif %}

‎tests/templates/kuttl/openlineage/00-rbac.yaml.j2‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
---
2+
kind: Role
3+
apiVersion: rbac.authorization.k8s.io/v1
4+
metadata:
5+
name: use-integration-tests-scc
6+
rules:
7+
- apiGroups:
8+
- ""
9+
resources:
10+
- configmaps
11+
- secrets
12+
verbs:
13+
- create
14+
- get
15+
- patch
16+
{% if test_scenario['values']['openshift'] == "true" %}
17+
- apiGroups: ["security.openshift.io"]
18+
resources: ["securitycontextconstraints"]
19+
resourceNames: ["privileged"]
20+
verbs: ["use"]
21+
{% endif %}
22+
---
23+
apiVersion: v1
24+
kind: ServiceAccount
25+
metadata:
26+
name: integration-tests-sa
27+
---
28+
kind: RoleBinding
29+
apiVersion: rbac.authorization.k8s.io/v1
30+
metadata:
31+
name: use-integration-tests-scc
32+
subjects:
33+
- kind: ServiceAccount
34+
name: integration-tests-sa
35+
roleRef:
36+
kind: Role
37+
name: use-integration-tests-scc
38+
apiGroup: rbac.authorization.k8s.io

‎tests/templates/kuttl/openlineage/10-assert.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
metadata:
5+
name: test-superset-postgresql
6+
timeout: 480
7+
---
8+
apiVersion: apps/v1
9+
kind: StatefulSet
10+
metadata:
11+
name: superset-postgresql
12+
status:
13+
readyReplicas: 1
14+
replicas: 1

‎tests/templates/kuttl/openlineage/10-install-postgresql.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestStep
4+
commands:
5+
- script: >-
6+
helm install superset-postgresql
7+
--namespace $NAMESPACE
8+
--version 12.5.6
9+
-f helm-bitnami-postgresql-values.yaml
10+
--repo https://charts.bitnami.com/bitnami postgresql
11+
--wait
12+
timeout: 600

‎tests/templates/kuttl/openlineage/20-assert.yaml.j2‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
{% if lookup('env', 'VECTOR_AGGREGATOR') %}
5+
---
6+
apiVersion: v1
7+
kind: ConfigMap
8+
metadata:
9+
name: vector-aggregator-discovery
10+
{% endif %}

‎tests/templates/kuttl/openlineage/20-install-vector-aggregator-discovery-configmap.yaml.j2‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{% if lookup('env', 'VECTOR_AGGREGATOR') %}
2+
---
3+
apiVersion: v1
4+
kind: ConfigMap
5+
metadata:
6+
name: vector-aggregator-discovery
7+
data:
8+
ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }}
9+
{% endif %}

‎tests/templates/kuttl/openlineage/30-assert.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
metadata:
5+
name: openlineage-receiver
6+
timeout: 240
7+
---
8+
apiVersion: apps/v1
9+
kind: Deployment
10+
metadata:
11+
name: openlineage-receiver
12+
status:
13+
readyReplicas: 1
14+
replicas: 1

‎tests/templates/kuttl/openlineage/30-install-openlineage-receiver.yaml.j2‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
---
2+
apiVersion: v1
3+
kind: ConfigMap
4+
metadata:
5+
name: openlineage-receiver-script
6+
data:
7+
receiver.py: |
8+
#!/usr/bin/env python
9+
# Minimal OpenLineage receiver used for integration testing.
10+
# The OpenLineage HTTP transport POSTs run events to /api/v1/lineage; this handler
11+
# appends every POST body to a log regardless of path. GET returns everything received
12+
# so far, so the test can assert lineage was emitted.
13+
import http.server
14+
import os
15+
16+
EVENTS = "/tmp/events.log"
17+
PORT = int(os.environ.get("PORT", "5000"))
18+
19+
class Handler(http.server.BaseHTTPRequestHandler):
20+
def do_POST(self):
21+
length = int(self.headers.get("Content-Length", 0))
22+
body = self.rfile.read(length) if length else b""
23+
with open(EVENTS, "ab") as f:
24+
f.write(body + b"\n")
25+
self.send_response(201)
26+
self.end_headers()
27+
28+
def do_GET(self):
29+
try:
30+
with open(EVENTS, "rb") as f:
31+
data = f.read()
32+
except FileNotFoundError:
33+
data = b""
34+
self.send_response(200)
35+
self.end_headers()
36+
self.wfile.write(data)
37+
38+
def log_message(self, *args):
39+
pass
40+
41+
httpd = http.server.HTTPServer(("0.0.0.0", PORT), Handler)
42+
httpd.serve_forever()
43+
---
44+
apiVersion: apps/v1
45+
kind: Deployment
46+
metadata:
47+
name: openlineage-receiver
48+
spec:
49+
replicas: 1
50+
selector:
51+
matchLabels:
52+
app: openlineage-receiver
53+
template:
54+
metadata:
55+
labels:
56+
app: openlineage-receiver
57+
spec:
58+
serviceAccountName: integration-tests-sa
59+
securityContext:
60+
runAsUser: 1000
61+
runAsGroup: 1000
62+
fsGroup: 1000
63+
containers:
64+
- name: receiver
65+
image: oci.stackable.tech/sdp/testing-tools:0.3.0-stackable0.0.0-dev
66+
command: ["python", "/scripts/receiver.py"]
67+
env:
68+
- name: PORT
69+
value: "5000"
70+
ports:
71+
- containerPort: 5000
72+
volumeMounts:
73+
- name: script
74+
mountPath: /scripts
75+
resources:
76+
requests:
77+
memory: "128Mi"
78+
cpu: "256m"
79+
limits:
80+
memory: "128Mi"
81+
cpu: "1"
82+
volumes:
83+
- name: script
84+
configMap:
85+
name: openlineage-receiver-script
86+
---
87+
apiVersion: v1
88+
kind: Service
89+
metadata:
90+
name: openlineage-receiver
91+
spec:
92+
selector:
93+
app: openlineage-receiver
94+
ports:
95+
- port: 5000
96+
targetPort: 5000

‎tests/templates/kuttl/openlineage/40-assert.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
metadata:
5+
name: install-superset
6+
timeout: 300
7+
commands:
8+
- script: kubectl -n $NAMESPACE wait --for=condition=available=true supersetclusters.superset.stackable.tech/superset --timeout 301s

0 commit comments

Comments
 (0)