Skip to content

Commit 3c7850f

Browse files
labrenbelabrenbe
committed
fix integration tests
1 parent eb06291 commit 3c7850f

9 files changed

Lines changed: 106 additions & 69 deletions

File tree

‎rust/operator-binary/src/reporting_task/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
//! Due to changes in the JWT validation in 1.25.0, the issuer refers to the FQDN of the Pod that was created, e.g.:
99
//! {
1010
//! "sub": "admin",
11-
//! "iss": "test-nifi-node-default-0.test-nifi-node-default.default.svc.cluster.local:8443",
11+
//! "iss": "test-nifi-node-default-0.test-nifi-node-default-metrics.default.svc.cluster.local:8443",
1212
//! }
1313
//! which was different in e.g. 1.23.2
1414
//! {

‎tests/templates/kuttl/custom-components-git-sync/20-assert.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ timeout: 600
66
apiVersion: apps/v1
77
kind: StatefulSet
88
metadata:
9-
name: zookeeper-server-default
9+
name: test-zk-server-default
1010
status:
1111
readyReplicas: 1
1212
replicas: 1

‎tests/templates/kuttl/iceberg/34_trino.yaml.j2‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,14 @@ spec:
1616
catalogLabelSelector:
1717
matchLabels:
1818
trino: trino
19+
listenerClass: external-unstable
1920
{% if lookup('env', 'VECTOR_AGGREGATOR') %}
2021
vectorAggregatorConfigMapName: vector-aggregator-discovery
2122
{% endif %}
2223
coordinators:
2324
config:
2425
logging:
2526
enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
26-
roleConfig:
27-
listenerClass: external-unstable
2827
{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %}
2928
podOverrides: &podOverrides
3029
spec:

‎tests/templates/kuttl/ldap/test_nifi.py‎

Lines changed: 35 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -8,61 +8,71 @@
88

99
def get_token(nifi_host, username, password):
1010
nifi_headers = {
11-
'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
11+
"content-type": "application/x-www-form-urlencoded; charset=UTF-8",
1212
}
13-
data = {'username': username, 'password': password}
13+
data = {"username": username, "password": password}
1414

1515
# TODO: handle actual errors when connecting properly
16-
nifi_url = nifi_host + '/nifi-api/access/token'
17-
response = requests.post(nifi_url, headers=nifi_headers, data=data, verify=False) # , cert='./tmp/cacert.pem')
16+
nifi_url = nifi_host + "/nifi-api/access/token"
17+
response = requests.post(
18+
nifi_url, headers=nifi_headers, data=data, verify=False
19+
) # , cert='./tmp/cacert.pem')
1820

1921
if response.ok:
20-
nifi_token = response.content.decode('utf-8')
22+
nifi_token = response.content.decode("utf-8")
2123
return "Bearer " + nifi_token
2224
else:
2325
print(f"Failed to get token: {response.status_code}: {response.content}")
2426
exit(-1)
2527

2628

27-
if __name__ == '__main__':
29+
if __name__ == "__main__":
2830
# Construct an argument parser
2931
all_args = argparse.ArgumentParser()
3032

3133
# Add arguments to the parser
32-
all_args.add_argument("-u", "--user", required=True,
33-
help="Username to connect as")
34-
all_args.add_argument("-p", "--password", required=True,
35-
help="Password for the user")
36-
all_args.add_argument("-n", "--namespace", required=True,
37-
help="Namespace the test is running in")
38-
all_args.add_argument("-c", "--count", required=True,
39-
help="The expected number of Nodes")
34+
all_args.add_argument("-u", "--user", required=True, help="Username to connect as")
35+
all_args.add_argument(
36+
"-p", "--password", required=True, help="Password for the user"
37+
)
38+
all_args.add_argument(
39+
"-n", "--namespace", required=True, help="Namespace the test is running in"
40+
)
41+
all_args.add_argument(
42+
"-c", "--count", required=True, help="The expected number of Nodes"
43+
)
4044
args = vars(all_args.parse_args())
4145

4246
# disable warnings as we have specified non-verified https connections
4347
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
4448

45-
host = f"https://test-nifi-node-default-1.test-nifi-node-default.{args['namespace']}.svc.cluster.local:8443"
46-
token = get_token(host, args['user'], args['password'])
47-
headers = {'Authorization': token}
48-
node_count = int(args['count'])
49+
host = f"https://test-nifi-node-default-1.test-nifi-node-default-metrics.{args['namespace']}.svc.cluster.local:8443"
50+
token = get_token(host, args["user"], args["password"])
51+
headers = {"Authorization": token}
52+
node_count = int(args["count"])
4953

5054
x = 0
5155
while x < 15:
52-
url = host + '/nifi-api/controller/cluster'
53-
cluster = requests.get(url, headers=headers, verify=False) # , cert='/tmp/cacert.pem')
56+
url = host + "/nifi-api/controller/cluster"
57+
cluster = requests.get(
58+
url, headers=headers, verify=False
59+
) # , cert='/tmp/cacert.pem')
5460
if cluster.status_code != 200:
5561
print("Waiting for cluster...")
5662
else:
57-
cluster_data = json.loads(cluster.content.decode('utf-8'))
58-
nodes = cluster_data['cluster']['nodes']
63+
cluster_data = json.loads(cluster.content.decode("utf-8"))
64+
nodes = cluster_data["cluster"]["nodes"]
5965
if len(nodes) != node_count:
60-
print(f"Cluster should have {node_count} nodes at this stage, but has: {len(nodes)}")
66+
print(
67+
f"Cluster should have {node_count} nodes at this stage, but has: {len(nodes)}"
68+
)
6169
else:
6270
connected = True
6371
for node in nodes:
64-
if node['status'] != "CONNECTED":
65-
print(f"Node {node['nodeId']} is in state {node['status']} but should have been CONNECTED")
72+
if node["status"] != "CONNECTED":
73+
print(
74+
f"Node {node['nodeId']} is in state {node['status']} but should have been CONNECTED"
75+
)
6676
connected = False
6777
if connected:
6878
print("Test succeeded!")

‎tests/templates/kuttl/oidc-opa/test.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
namespace = os.environ["NAMESPACE"]
1818
tls = os.environ["OIDC_USE_TLS"]
1919
nifi_version = os.environ["NIFI_VERSION"]
20-
nifi = f"test-nifi-node-default-0.test-nifi-node-default.{namespace}.svc.cluster.local"
20+
nifi = f"test-nifi-node-default-0.test-nifi-node-default-metrics.{namespace}.svc.cluster.local"
2121
keycloak_service = f"keycloak.{namespace}.svc.cluster.local"
2222

2323
keycloak_base_url = (

‎tests/templates/kuttl/upgrade/04-assert.yaml.j2‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@ commands:
88
- script: kubectl exec -n $NAMESPACE test-nifi-0 -- python /tmp/test_nifi_metrics.py -n $NAMESPACE
99
{% endif %}
1010
{% if test_scenario['values']['nifi_old'].split(',')[0] == '2.0.0' %}
11-
- script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default.$NAMESPACE.svc.cluster.local:8443 run json /tmp/generate-and-log-flowfiles.json > /tmp/old_input"
11+
- script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default-metrics.$NAMESPACE.svc.cluster.local:8443 run json /tmp/generate-and-log-flowfiles.json > /tmp/old_input"
1212
{% else %}
13-
- script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default.$NAMESPACE.svc.cluster.local:8443 run template /tmp/generate-and-log-flowfiles.xml > /tmp/old_input"
13+
- script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default-metrics.$NAMESPACE.svc.cluster.local:8443 run template /tmp/generate-and-log-flowfiles.xml > /tmp/old_input"
1414
{% endif %}
1515
# This tests if the output contains an Error or zero flow files are queued, which also indicates that something went wrong
1616
- script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "cat /tmp/old_input | grep -Eov 'Error|\b0\b'"

‎tests/templates/kuttl/upgrade/07-assert.yaml.j2‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ commands:
99
{% if test_scenario['values']['nifi_new'].split(',')[0].startswith('1.') %}
1010
- script: kubectl exec -n $NAMESPACE test-nifi-0 -- python /tmp/test_nifi_metrics.py -n $NAMESPACE
1111
{% endif %}
12-
- script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default.$NAMESPACE.svc.cluster.local:8443 query > /tmp/new_input"
12+
- script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default-metrics.$NAMESPACE.svc.cluster.local:8443 query > /tmp/new_input"
1313
# This tests if the output contains an Error or zero flow files are queued, which also indicates that something went wrong
1414
- script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "cat /tmp/new_input | grep -Eov 'Error|\b0\b'"
1515
# This tests that the number of input records stays the same after the upgrade.

‎tests/templates/kuttl/upgrade/test_nifi.py‎

Lines changed: 35 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -8,61 +8,71 @@
88

99
def get_token(nifi_host, username, password):
1010
nifi_headers = {
11-
'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
11+
"content-type": "application/x-www-form-urlencoded; charset=UTF-8",
1212
}
13-
data = {'username': username, 'password': password}
13+
data = {"username": username, "password": password}
1414

1515
# TODO: handle actual errors when connecting properly
16-
nifi_url = nifi_host + '/nifi-api/access/token'
17-
response = requests.post(nifi_url, headers=nifi_headers, data=data, verify=False) # , cert='./tmp/cacert.pem')
16+
nifi_url = nifi_host + "/nifi-api/access/token"
17+
response = requests.post(
18+
nifi_url, headers=nifi_headers, data=data, verify=False
19+
) # , cert='./tmp/cacert.pem')
1820

1921
if response.ok:
20-
nifi_token = response.content.decode('utf-8')
22+
nifi_token = response.content.decode("utf-8")
2123
return "Bearer " + nifi_token
2224
else:
2325
print(f"Failed to get token: {response.status_code}: {response.content}")
2426
exit(-1)
2527

2628

27-
if __name__ == '__main__':
29+
if __name__ == "__main__":
2830
# Construct an argument parser
2931
all_args = argparse.ArgumentParser()
3032

3133
# Add arguments to the parser
32-
all_args.add_argument("-u", "--user", required=True,
33-
help="Username to connect as")
34-
all_args.add_argument("-p", "--password", required=True,
35-
help="Password for the user")
36-
all_args.add_argument("-n", "--namespace", required=True,
37-
help="Namespace the test is running in")
38-
all_args.add_argument("-c", "--count", required=True,
39-
help="The expected number of Nodes")
34+
all_args.add_argument("-u", "--user", required=True, help="Username to connect as")
35+
all_args.add_argument(
36+
"-p", "--password", required=True, help="Password for the user"
37+
)
38+
all_args.add_argument(
39+
"-n", "--namespace", required=True, help="Namespace the test is running in"
40+
)
41+
all_args.add_argument(
42+
"-c", "--count", required=True, help="The expected number of Nodes"
43+
)
4044
args = vars(all_args.parse_args())
4145

4246
# disable warnings as we have specified non-verified https connections
4347
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
4448

45-
host = f"https://test-nifi-node-default-1.test-nifi-node-default.{args['namespace']}.svc.cluster.local:8443"
46-
token = get_token(host, args['user'], args['password'])
47-
headers = {'Authorization': token}
48-
node_count = int(args['count'])
49+
host = f"https://test-nifi-node-default-1.test-nifi-node-default-metrics.{args['namespace']}.svc.cluster.local:8443"
50+
token = get_token(host, args["user"], args["password"])
51+
headers = {"Authorization": token}
52+
node_count = int(args["count"])
4953

5054
x = 0
5155
while x < 15:
52-
url = host + '/nifi-api/controller/cluster'
53-
cluster = requests.get(url, headers=headers, verify=False) # , cert='/tmp/cacert.pem')
56+
url = host + "/nifi-api/controller/cluster"
57+
cluster = requests.get(
58+
url, headers=headers, verify=False
59+
) # , cert='/tmp/cacert.pem')
5460
if cluster.status_code != 200:
5561
print("Waiting for cluster...")
5662
else:
57-
cluster_data = json.loads(cluster.content.decode('utf-8'))
58-
nodes = cluster_data['cluster']['nodes']
63+
cluster_data = json.loads(cluster.content.decode("utf-8"))
64+
nodes = cluster_data["cluster"]["nodes"]
5965
if len(nodes) != node_count:
60-
print(f"Cluster should have {node_count} nodes at this stage, but has: {len(nodes)}")
66+
print(
67+
f"Cluster should have {node_count} nodes at this stage, but has: {len(nodes)}"
68+
)
6169
else:
6270
connected = True
6371
for node in nodes:
64-
if node['status'] != "CONNECTED":
65-
print(f"Node {node['nodeId']} is in state {node['status']} but should have been CONNECTED")
72+
if node["status"] != "CONNECTED":
73+
print(
74+
f"Node {node['nodeId']} is in state {node['status']} but should have been CONNECTED"
75+
)
6676
connected = False
6777
if connected:
6878
print("Test succeeded!")

‎tests/templates/kuttl/upgrade/test_nifi_metrics.py‎

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,26 +4,42 @@
44
import time
55
from requests.exceptions import ConnectionError
66

7-
if __name__ == '__main__':
7+
if __name__ == "__main__":
88
# Construct an argument parser
99
all_args = argparse.ArgumentParser()
1010
# Add arguments to the parser
11-
all_args.add_argument("-m", "--metric", required=False, default="nifi_amount_bytes_read",
12-
help="The name of a certain metric to check")
13-
all_args.add_argument("-n", "--namespace", required=True,
14-
help="The namespace the test is running in")
15-
all_args.add_argument("-p", "--port", required=False, default="8081",
16-
help="The port where metrics are exposed")
17-
all_args.add_argument("-t", "--timeout", required=False, default="120",
18-
help="The timeout in seconds to wait for the metrics port to be opened")
11+
all_args.add_argument(
12+
"-m",
13+
"--metric",
14+
required=False,
15+
default="nifi_amount_bytes_read",
16+
help="The name of a certain metric to check",
17+
)
18+
all_args.add_argument(
19+
"-n", "--namespace", required=True, help="The namespace the test is running in"
20+
)
21+
all_args.add_argument(
22+
"-p",
23+
"--port",
24+
required=False,
25+
default="8081",
26+
help="The port where metrics are exposed",
27+
)
28+
all_args.add_argument(
29+
"-t",
30+
"--timeout",
31+
required=False,
32+
default="120",
33+
help="The timeout in seconds to wait for the metrics port to be opened",
34+
)
1935

2036
args = vars(all_args.parse_args())
2137
metric_name = args["metric"]
2238
namespace = args["namespace"]
2339
port = args["port"]
2440
timeout = int(args["timeout"])
2541

26-
url = f"http://test-nifi-node-default-0.test-nifi-node-default.{namespace}.svc.cluster.local:{port}/metrics"
42+
url = f"http://test-nifi-node-default-0.test-nifi-node-default-metrics.{namespace}.svc.cluster.local:{port}/metrics"
2743

2844
# wait for 'timeout' seconds
2945
t_end = time.time() + timeout
@@ -35,7 +51,9 @@
3551
print("Test metrics succeeded!")
3652
exit(0)
3753
else:
38-
print(f"Could not find metric [{metric_name}] in response:\n {response.text}")
54+
print(
55+
f"Could not find metric [{metric_name}] in response:\n {response.text}"
56+
)
3957
time.sleep(timeout)
4058
except ConnectionError:
4159
# NewConnectionError is expected until metrics are available

0 commit comments

Comments
 (0)