fix smoke tests

Author: maltesander

tests/templates/kuttl/smoke/test_jmx_metrics.py

@@ -65,9 +65,9 @@ def check_datanode_metrics(
         # Kind "FSDatasetState"
         'hadoop_datanode_capacity{fsdatasetid=".+",kind="FSDatasetState",role="DataNode",service="HDFS"}',
         # Kind "DataNodeActivity" suffixed with "_info"
-        'hadoop_datanode_blocks_get_local_path_info_{host="hdfs-datanode-default-0\\.hdfs-datanode-default-headless\\..+\\.svc\\.cluster\\.local",kind="DataNodeActivity",port="9866",role="DataNode",service="HDFS"}',
+        'hadoop_datanode_blocks_get_local_path_info_{host="hdfs-datanode-default-\\d+\\.hdfs-datanode-default-headless\\..+\\.svc\\.cluster\\.local",kind="DataNodeActivity",port="9866",role="DataNode",service="HDFS"}',
         # Kind "DataNodeActivity"
-        'hadoop_datanode_blocks_read{host="hdfs-datanode-default-0\\.hdfs-datanode-default-headless\\..+\\.svc\\.cluster\\.local",kind="DataNodeActivity",port="9866",role="DataNode",service="HDFS"}',
+        'hadoop_datanode_blocks_read{host="hdfs-datanode-default-\\d+\\.hdfs-datanode-default-headless\\..+\\.svc\\.cluster\\.local",kind="DataNodeActivity",port="9866",role="DataNode",service="HDFS"}',
         # Counter suffixed with "_total"
         'hadoop_datanode_estimated_capacity_lost_total{kind="FSDatasetState",role="DataNode",service="HDFS"}',
         # Boolean metric

tests/templates/kuttl/smoke/test_native_metrics.py

@@ -1,4 +1,6 @@
-# Every rule in the JMX configuration is covered by one expected metric.
+# Native Prometheus metrics test
+# We use a raw string for "expected_metrics" but still have to escape special regex characters "[", "]", "{" and "}"
+# that we expect to be in the metrics string.
 
 import re
 import sys
@@ -8,28 +10,28 @@
 
 
 def check_metrics(
-    namespace: str, role: str, port: int, expected_metrics: list[str]
+    namespace: str,
+    role: str,
+    port: int,
+    expected_metrics: list[str]
 ) -> None:
-    response: requests.Response = requests.get(
+    response = requests.get(
         f"http://hdfs-{role}-default-metrics.{namespace}.svc.cluster.local:{port}/prom",
         timeout=10,
     )
-    assert response.ok, "Requesting metrics failed"
+    assert response.ok, "Requesting metrics failed for {role}."
 
     for metric in expected_metrics:
-        assert re.search(f"^{metric}", response.text, re.MULTILINE) is not None, (
-            f"Metric '{metric}' not found for {role}"
-        )
+        regex = re.compile(metric, re.MULTILINE)
+        assert regex.search(response.text) is not None, f"Metric '{metric}' not found for {role}"
 
 
-def check_namenode_metrics(
-    namespace: str,
-) -> None:
-    expected_metrics: list[str] = [
-        'metrics_system_num_active_sources{context="metricssystem",hostname="hdfs-namenode-default-',
-        'namenode_total_file_ops{processname="NameNode",sessionid="null",context="dfs",hostname="hdfs-namenode-default',
-        'namenode_files_created{processname="NameNode",sessionid="null",context="dfs",hostname="hdfs-namenode-default-',
-        'namenode_files_deleted{processname="NameNode",sessionid="null",context="dfs",hostname="hdfs-namenode-default-',
+def check_namenode_metrics(namespace: str) -> None:
+    expected_metrics = [
+        r'metrics_system_num_active_sources\{context="metricssystem",hostname="hdfs-namenode-default-\d+"\}',
+        r'namenode_total_file_ops\{processname="NameNode",sessionid="null",context="dfs",hostname="hdfs-namenode-default-\d+"\}',
+        r'namenode_files_created\{processname="NameNode",sessionid="null",context="dfs",hostname="hdfs-namenode-default-\d+"\}',
+        r'namenode_files_deleted\{processname="NameNode",sessionid="null",context="dfs",hostname="hdfs-namenode-default-\d+"\}',
     ]
 
     check_metrics(namespace, "namenode", 9870, expected_metrics)
@@ -39,12 +41,12 @@ def check_datanode_metrics(
     namespace: str,
 ) -> None:
     expected_metrics: list[str] = [
-        'metrics_system_num_active_sources{context="metricssystem",hostname="hdfs-datanode-default-0"}',
-        'org_apache_hadoop_hdfs_server_datanode_fsdataset_impl_fs_dataset_impl_capacity{context="FSDatasetState",storageinfo="FSDataset{dirpath=\'[/stackable/data/data/datanode]\'}",hostname="hdfs-datanode-default-0"}',
-        'org_apache_hadoop_hdfs_server_datanode_fsdataset_impl_fs_dataset_impl_estimated_capacity_lost_total{context="FSDatasetState",storageinfo="FSDataset{dirpath=\'[/stackable/data/data/datanode]\'}",hostname="hdfs-datanode-default-0"}',
-        'datanode_blocks_get_local_path_info{sessionid="null",context="dfs",hostname="hdfs-datanode-default-0"}',
-        'datanode_blocks_read{sessionid="null",context="dfs",hostname="hdfs-datanode-default-0"}',
-        'jvm_metrics_gc_count{context="jvm",processname="DataNode",sessionid="null",hostname="hdfs-datanode-default-0"}',
+        r'metrics_system_num_active_sources\{context="metricssystem",hostname="hdfs-datanode-default-\d+',
+        r'org_apache_hadoop_hdfs_server_datanode_fsdataset_impl_fs_dataset_impl_capacity\{context="FSDatasetState",storageinfo="FSDataset\{dirpath=\'\[/stackable/data/data/datanode]\'\}",hostname="hdfs-datanode-default-\d+"\}',
+        r'org_apache_hadoop_hdfs_server_datanode_fsdataset_impl_fs_dataset_impl_estimated_capacity_lost_total\{context="FSDatasetState",storageinfo="FSDataset\{dirpath=\'\[/stackable/data/data/datanode]\'\}",hostname="hdfs-datanode-default-\d+"\}',
+        r'datanode_blocks_get_local_path_info\{sessionid="null",context="dfs",hostname="hdfs-datanode-default-\d+"\}',
+        r'datanode_blocks_read\{sessionid="null",context="dfs",hostname="hdfs-datanode-default-\d+"\}',
+        r'jvm_metrics_gc_count\{context="jvm",processname="DataNode",sessionid="null",hostname="hdfs-datanode-default-\d+"\}',
     ]
 
     check_metrics(namespace, "datanode", 9864, expected_metrics)
@@ -54,8 +56,8 @@ def check_journalnode_metrics(
     namespace: str,
 ) -> None:
     expected_metrics: list[str] = [
-        'metrics_system_num_active_sources{context="metricssystem",hostname="hdfs-journalnode-default-0"}',
-        'journal_node_bytes_written{context="dfs",journalid="hdfs",hostname="hdfs-journalnode-default-0"}',
+        r'metrics_system_num_active_sources\{context="metricssystem",hostname="hdfs-journalnode-default-\d+"\}',
+        r'journal_node_bytes_written\{context="dfs",journalid="hdfs",hostname="hdfs-journalnode-default-\d+"\}',
     ]
 
     check_metrics(namespace, "journalnode", 8480, expected_metrics)