Skip to content

Commit 3753ede

Browse files
maltesandermaltesander
committed
start fixing smoke tests
1 parent 7cc2e2f commit 3753ede

7 files changed

Lines changed: 181 additions & 120 deletions

File tree

‎tests/templates/kuttl/smoke/30-assert.yaml.j2‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,100 @@ status:
7878
{% if test_scenario['values']['datanode-pvcs'] == '2hdd-1ssd' %}
7979
---
8080
apiVersion: v1
81+
kind: Service
82+
metadata:
83+
name: hdfs-namenode-default-headless
84+
spec:
85+
ports:
86+
- name: rpc
87+
port: 8020
88+
protocol: TCP
89+
targetPort: 8020
90+
- name: http
91+
port: 9070
92+
protocol: TCP
93+
targetPort: 9070
94+
---
95+
apiVersion: v1
96+
kind: Service
97+
metadata:
98+
name: hdfs-namenode-default-metrics
99+
spec:
100+
ports:
101+
- name: metrics
102+
port: 9870
103+
protocol: TCP
104+
targetPort: 9870
105+
- name: jmx-metrics
106+
port: 8183
107+
protocol: TCP
108+
targetPort: 8183
109+
---
110+
apiVersion: v1
111+
kind: Service
112+
metadata:
113+
name: hdfs-datanode-default-headless
114+
spec:
115+
ports:
116+
- name: data
117+
port: 9866
118+
protocol: TCP
119+
targetPort: 9866
120+
- name: ipc
121+
port: 9867
122+
protocol: TCP
123+
targetPort: 9867
124+
- name: http
125+
port: 9864
126+
protocol: TCP
127+
targetPort: 9864
128+
---
129+
apiVersion: v1
130+
kind: Service
131+
metadata:
132+
name: hdfs-datanode-default-metrics
133+
spec:
134+
ports:
135+
- name: metrics
136+
port: 9864
137+
protocol: TCP
138+
targetPort: 9864
139+
- name: jmx-metrics
140+
port: 8082
141+
protocol: TCP
142+
targetPort: 8082
143+
---
144+
apiVersion: v1
145+
kind: Service
146+
metadata:
147+
name: hdfs-journal-default-headless
148+
spec:
149+
ports:
150+
- name: rpc
151+
port: 8485
152+
protocol: TCP
153+
targetPort: 8485
154+
- name: http
155+
port: 8480
156+
protocol: TCP
157+
targetPort: 8480
158+
---
159+
apiVersion: v1
160+
kind: Service
161+
metadata:
162+
name: hdfs-journal-default-metrics
163+
spec:
164+
ports:
165+
- name: metrics
166+
port: 8480
167+
protocol: TCP
168+
targetPort: 8480
169+
- name: jmx-metrics
170+
port: 8081
171+
protocol: TCP
172+
targetPort: 8081
173+
---
174+
apiVersion: v1
81175
kind: PersistentVolumeClaim
82176
metadata:
83177
name: hdd-hdfs-datanode-default-0

‎tests/templates/kuttl/smoke/51-assert.yaml.j2‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,8 @@ commands:
88
{% else %}
99
PRODUCT_VERSION={{ test_scenario['values']['hadoop'] }}
1010
{% endif %}
11-
# Test JMX exported metrics
11+
# Test exported metrics
1212
kubectl exec --namespace=$NAMESPACE test-runner-0 -- \
13-
python /tmp/test_metrics.py $NAMESPACE $PRODUCT_VERSION
14-
# Test Prometheus metrics
13+
python /tmp/test_jmx_metrics.py $NAMESPACE $PRODUCT_VERSION
1514
kubectl exec --namespace=$NAMESPACE test-runner-0 -- \
16-
python /tmp/test_prometheus_metrics.py $NAMESPACE $PRODUCT_VERSION
15+
python /tmp/test_native_metrics.py $NAMESPACE

‎tests/templates/kuttl/smoke/51-copy-metrics-test-script.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
apiVersion: kuttl.dev/v1beta1
33
kind: TestStep
44
commands:
5-
- script: kubectl cp -n $NAMESPACE ./test_metrics.py test-runner-0:/tmp
6-
- script: kubectl cp -n $NAMESPACE ./test_prometheus_metrics.py test-runner-0:/tmp
5+
- script: kubectl cp -n $NAMESPACE ./test_jmx_metrics.py test-runner-0:/tmp
6+
- script: kubectl cp -n $NAMESPACE ./test_native_metrics.py test-runner-0:/tmp

tests/templates/kuttl/smoke/test_metrics.py renamed to tests/templates/kuttl/smoke/test_jmx_metrics.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ def check_metrics(
1111
namespace: str, role: str, port: int, expected_metrics: list[str]
1212
) -> None:
1313
response: requests.Response = requests.get(
14-
f"http://hdfs-{role}-default-0.hdfs-{role}-default.{namespace}.svc.cluster.local:{port}/metrics",
14+
f"http://hdfs-{role}-default-metrics.{namespace}.svc.cluster.local:{port}/metrics",
1515
timeout=10,
1616
)
1717
assert response.ok, "Requesting metrics failed"
@@ -65,9 +65,9 @@ def check_datanode_metrics(
6565
# Kind "FSDatasetState"
6666
'hadoop_datanode_capacity{fsdatasetid=".+",kind="FSDatasetState",role="DataNode",service="HDFS"}',
6767
# Kind "DataNodeActivity" suffixed with "_info"
68-
'hadoop_datanode_blocks_get_local_path_info_{host="hdfs-datanode-default-0\\.hdfs-datanode-default\\..+\\.svc\\.cluster\\.local",kind="DataNodeActivity",port="9866",role="DataNode",service="HDFS"}',
68+
'hadoop_datanode_blocks_get_local_path_info_{host="hdfs-datanode-default-0\\.hdfs-datanode-default-headless\\..+\\.svc\\.cluster\\.local",kind="DataNodeActivity",port="9866",role="DataNode",service="HDFS"}',
6969
# Kind "DataNodeActivity"
70-
'hadoop_datanode_blocks_read{host="hdfs-datanode-default-0\\.hdfs-datanode-default\\..+\\.svc\\.cluster\\.local",kind="DataNodeActivity",port="9866",role="DataNode",service="HDFS"}',
70+
'hadoop_datanode_blocks_read{host="hdfs-datanode-default-0\\.hdfs-datanode-default-headless\\..+\\.svc\\.cluster\\.local",kind="DataNodeActivity",port="9866",role="DataNode",service="HDFS"}',
7171
# Counter suffixed with "_total"
7272
'hadoop_datanode_estimated_capacity_lost_total{kind="FSDatasetState",role="DataNode",service="HDFS"}',
7373
# Boolean metric

‎tests/templates/kuttl/smoke/test_native_metrics.py‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# Every rule in the JMX configuration is covered by one expected metric.
2+
3+
import re
4+
import sys
5+
import logging
6+
7+
import requests
8+
9+
10+
def check_metrics(
11+
namespace: str, role: str, port: int, expected_metrics: list[str]
12+
) -> None:
13+
response: requests.Response = requests.get(
14+
f"http://hdfs-{role}-default-metrics.{namespace}.svc.cluster.local:{port}/prom",
15+
timeout=10,
16+
)
17+
assert response.ok, "Requesting metrics failed"
18+
19+
for metric in expected_metrics:
20+
assert re.search(f"^{metric}", response.text, re.MULTILINE) is not None, (
21+
f"Metric '{metric}' not found for {role}"
22+
)
23+
24+
25+
def check_namenode_metrics(
26+
namespace: str,
27+
) -> None:
28+
expected_metrics: list[str] = [
29+
'metrics_system_num_active_sources{context="metricssystem",hostname="hdfs-namenode-default-',
30+
'namenode_total_file_ops{processname="NameNode",sessionid="null",context="dfs",hostname="hdfs-namenode-default',
31+
'namenode_files_created{processname="NameNode",sessionid="null",context="dfs",hostname="hdfs-namenode-default-',
32+
'namenode_files_deleted{processname="NameNode",sessionid="null",context="dfs",hostname="hdfs-namenode-default-',
33+
]
34+
35+
check_metrics(namespace, "namenode", 9870, expected_metrics)
36+
37+
38+
def check_datanode_metrics(
39+
namespace: str,
40+
) -> None:
41+
expected_metrics: list[str] = [
42+
'metrics_system_num_active_sources{context="metricssystem",hostname="hdfs-datanode-default-0"}',
43+
'org_apache_hadoop_hdfs_server_datanode_fsdataset_impl_fs_dataset_impl_capacity{context="FSDatasetState",storageinfo="FSDataset{dirpath=\'[/stackable/data/data/datanode]\'}",hostname="hdfs-datanode-default-0"}',
44+
'org_apache_hadoop_hdfs_server_datanode_fsdataset_impl_fs_dataset_impl_estimated_capacity_lost_total{context="FSDatasetState",storageinfo="FSDataset{dirpath=\'[/stackable/data/data/datanode]\'}",hostname="hdfs-datanode-default-0"}',
45+
'datanode_blocks_get_local_path_info{sessionid="null",context="dfs",hostname="hdfs-datanode-default-0"}',
46+
'datanode_blocks_read{sessionid="null",context="dfs",hostname="hdfs-datanode-default-0"}',
47+
'jvm_metrics_gc_count{context="jvm",processname="DataNode",sessionid="null",hostname="hdfs-datanode-default-0"}',
48+
]
49+
50+
check_metrics(namespace, "datanode", 9864, expected_metrics)
51+
52+
53+
def check_journalnode_metrics(
54+
namespace: str,
55+
) -> None:
56+
expected_metrics: list[str] = [
57+
'metrics_system_num_active_sources{context="metricssystem",hostname="hdfs-journalnode-default-0"}',
58+
'journal_node_bytes_written{context="dfs",journalid="hdfs",hostname="hdfs-journalnode-default-0"}',
59+
]
60+
61+
check_metrics(namespace, "journalnode", 8480, expected_metrics)
62+
63+
64+
if __name__ == "__main__":
65+
namespace_arg: str = sys.argv[1]
66+
67+
logging.basicConfig(
68+
level="DEBUG",
69+
format="%(asctime)s %(levelname)s: %(message)s",
70+
stream=sys.stdout,
71+
)
72+
73+
check_namenode_metrics(namespace_arg)
74+
check_datanode_metrics(namespace_arg)
75+
check_journalnode_metrics(namespace_arg)
76+
77+
print("All expected metrics found")

‎tests/templates/kuttl/smoke/test_prometheus_metrics.py‎

Lines changed: 0 additions & 109 deletions
This file was deleted.

‎tests/templates/kuttl/smoke/webhdfs.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ def main() -> int:
1717

1818
if command == "ls":
1919
http_code = requests.get(
20-
f"http://hdfs-namenode-default-0.hdfs-namenode-default.{namespace}.svc.cluster.local:9870/webhdfs/v1/testdata.txt?user.name=stackable&op=LISTSTATUS"
20+
f"http://hdfs-namenode-default-0.hdfs-namenode-default-headless.{namespace}.svc.cluster.local:9870/webhdfs/v1/testdata.txt?user.name=stackable&op=LISTSTATUS"
2121
).status_code
2222
if http_code != 200:
2323
result = 1
@@ -31,7 +31,7 @@ def main() -> int:
3131
)
3232
}
3333
http_code = requests.put(
34-
f"http://hdfs-namenode-default-0.hdfs-namenode-default.{namespace}.svc.cluster.local:9870/webhdfs/v1/testdata.txt?user.name=stackable&op=CREATE",
34+
f"http://hdfs-namenode-default-0.hdfs-namenode-default-headless.{namespace}.svc.cluster.local:9870/webhdfs/v1/testdata.txt?user.name=stackable&op=CREATE",
3535
files=files,
3636
allow_redirects=True,
3737
).status_code

0 commit comments

Comments
 (0)