HDDS-15552. Ratis events should not be published as metrics

Change-Id: I06634562bad09e1cf308b3d2f9ec93d4b6c078fe

Author: jojochuang

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMMetrics.java

@@ -180,7 +180,9 @@ public void addRatisEvent(String event) {
     }
   }
 
-  @Metric("Ratis state machine events")
+  // Ratis state machine events are multi-line logs, which should not be
+  // published as time-series metrics to metrics systems like Prometheus.
+  // Instead, they are exposed via JMX / MXBean endpoints.
   public String getRatisEvents() {
     synchronized (ratisEvents) {
       return String.join("\n", ratisEvents);

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMMXBean.java

@@ -82,4 +82,6 @@ public interface SCMMXBean extends ServiceRuntimeInfo {
    * @return the SCM hostname for the datanode.
    */
   String getHostname();
+
+  String getRatisEvents();
 }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java

@@ -2238,6 +2238,11 @@ public String getHostname() {
     return scmHostName;
   }
 
+  @Override
+  public String getRatisEvents() {
+    return metrics != null ? metrics.getRatisEvents() : "";
+  }
+
   public Collection<String> getScmAdminUsernames() {
     return scmAdmins.getAdminUsernames();
   }

hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js

@@ -30,10 +30,10 @@
         templateUrl: 'ratis-events.html',
         controller: function ($http) {
             var ctrl = this;
-            $http.get("jmx?qry=Hadoop:service=StorageContainerManager,name=SCMMetrics")
+            $http.get("jmx?qry=Hadoop:service=StorageContainerManager,name=StorageContainerManagerInfo,component=ServerRuntime")
                 .then(function (result) {
                     var metrics = result.data.beans[0];
-                  var rawEvents = metrics['tag.RatisEvents'] ? metrics['tag.RatisEvents'].split('\n') : [];
+                    var rawEvents = (metrics && metrics['RatisEvents']) ? metrics['RatisEvents'].split('\n') : [];
                     ctrl.events = rawEvents.map(function(e) {
                         var parts = e.split('|');
                         return {

hadoop-ozone/dist/src/main/compose/ozone/test.sh

@@ -24,6 +24,7 @@ export COMPOSE_DIR
 
 export SECURITY_ENABLED=false
 export OZONE_REPLICATION_FACTOR=3
+export COMPOSE_FILE=docker-compose.yaml:monitoring.yaml
 
 # shellcheck source=/dev/null
 source "$COMPOSE_DIR/../testlib.sh"
@@ -40,6 +41,7 @@ execute_robot_test scm gdpr
 execute_robot_test scm security/ozone-secure-token.robot
 
 execute_robot_test scm recon
+execute_robot_test scm prometheus
 
 execute_robot_test scm om-ratis
 

hadoop-ozone/dist/src/main/smoketest/prometheus/prometheus.robot

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       Test Prometheus monitoring integration
+Library             OperatingSystem
+Library             BuiltIn
+Resource            ../commonlib.robot
+
+*** Test Cases ***
+Verify Prometheus targets are healthy
+    Wait Until Keyword Succeeds     90sec      10sec        Check Prometheus Targets Health
+
+*** Keywords ***
+Check Prometheus Targets Health
+    ${result} =         Execute     python3 ${OZONE_DIR}/smoketest/prometheus/prometheus_check.py
+    Should Contain      ${result}   Successfully verified

hadoop-ozone/dist/src/main/smoketest/prometheus/prometheus_check.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import urllib.request
+import json
+import sys
+import socket
+
+def is_running(host, port):
+    try:
+        with socket.create_connection((host, int(port)), timeout=2):
+            return True
+    except Exception:
+        return False
+
+def main():
+    try:
+        res = urllib.request.urlopen("http://prometheus:9090/api/v1/targets")
+        data = json.loads(res.read().decode())
+        targets = data.get("data", {}).get("activeTargets", [])
+        if not targets:
+            print("No active targets found in Prometheus")
+            sys.exit(1)
+            
+        failed = False
+        checked = 0
+        for t in targets:
+            url = t.get("scrapeUrl", "")
+            # scrapeUrl is like "http://scm:9876/prom"
+            try:
+                host_port = url.split("//")[1].split("/")[0]
+                if ":" in host_port:
+                    host, port = host_port.split(":")
+                else:
+                    host = host_port
+                    port = 80
+            except Exception:
+                continue
+                
+            if is_running(host, port):
+                checked += 1
+                health = t.get("health", "")
+                print(f"Target {host}:{port} is running. Prometheus health: {health}")
+                if health != "up":
+                    print(f"Error: Target {host}:{port} is running but Prometheus health is '{health}'. Last error: {t.get('lastError')}")
+                    failed = True
+            else:
+                print(f"Target {host}:{port} is not running. Skipping check.")
+                
+        if checked == 0:
+            print("Error: No running targets were checked!")
+            sys.exit(1)
+            
+        if failed:
+            sys.exit(1)
+            
+        print(f"Successfully verified {checked} running targets.")
+    except Exception as e:
+        print(f"Exception during health check: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMMXBean.java

@@ -86,6 +86,9 @@ public void testSCMMXBean() throws Exception {
     double containerThreshold = (double) mbs.getAttribute(bean,
         "SafeModeCurrentContainerThreshold");
     assertEquals(scm.getCurrentContainerThreshold(), containerThreshold, 0);
+
+    String ratisEvents = (String) mbs.getAttribute(bean, "RatisEvents");
+    assertEquals(scm.getMetrics().getRatisEvents(), ratisEvents);
   }
 
   @Test

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMXBean.java

@@ -41,4 +41,6 @@ public interface OMMXBean extends ServiceRuntimeInfo {
    * @return the OM hostname for the datanode.
    */
   String getHostname();
+
+  String getRatisEvents();
 }

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java

@@ -1590,7 +1590,9 @@ public void addRatisEvent(String event) {
     }
   }
 
-  @Metric("Ratis state machine events")
+  // Ratis state machine events are multi-line logs, which should not be
+  // published as time-series metrics to metrics systems like Prometheus.
+  // Instead, they are exposed via JMX / MXBean endpoints.
   public String getRatisEvents() {
     synchronized (ratisEvents) {
       return String.join("\n", ratisEvents);