[CELEBORN-2380] Support JmxSink for exposing metrics as JMX MBeans

LICENSE-binary

@@ -226,6 +226,7 @@ com.zaxxer:HikariCP
 info.picocli:picocli
 io.dropwizard.metrics:metrics-core
 io.dropwizard.metrics:metrics-graphite
+io.dropwizard.metrics:metrics-jmx
 io.dropwizard.metrics:metrics-jvm
 io.netty:netty-all
 io.netty:netty-buffer

charts/celeborn/files/conf/metrics.properties

@@ -18,3 +18,6 @@
 *.sink.prometheusServlet.class=org.apache.celeborn.common.metrics.sink.PrometheusServlet
 *.sink.jsonServlet.class=org.apache.celeborn.common.metrics.sink.JsonServlet
 *.sink.loggerSink.class=org.apache.celeborn.common.metrics.sink.LoggerSink
+
+# Expose metrics as JMX MBeans.
+*.sink.jmx.class=org.apache.celeborn.common.metrics.sink.JmxSink

charts/celeborn/tests/master/statefulset_test.yaml

@@ -40,7 +40,7 @@ tests:
     asserts:
       - equal:
           path: spec.template.metadata.annotations["celeborn.apache.org/conf-hash"]
-          value: 7e9a27719ab1f2c1cea53e4879a807782abd48d87ec2a45774084990af6125b3
+          value: 035c23a53d85eb407eecf3fe864be5e623afae38e8ecdd4c10579e76d85f5c6b
 
   - it: Should change checksum annotation when celeborn config changes
     template: master/statefulset.yaml
@@ -50,10 +50,10 @@ tests:
     asserts:
       - notEqual:
           path: spec.template.metadata.annotations["celeborn.apache.org/conf-hash"]
-          value: 7e9a27719ab1f2c1cea53e4879a807782abd48d87ec2a45774084990af6125b3
+          value: 035c23a53d85eb407eecf3fe864be5e623afae38e8ecdd4c10579e76d85f5c6b
       - equal:
           path: spec.template.metadata.annotations["celeborn.apache.org/conf-hash"]
-          value: 118d5c045d52fbbd4e8f05cf0522044373165960aa57893f645a6f9e8844dd68
+          value: d15c180987eca7e3b13dc37e3277c057a4060799ddd45cf20490ebb91da21ee1
 
   - it: Should add extra pod annotations if `master.annotations` is specified
     template: master/statefulset.yaml

charts/celeborn/tests/worker/statefulset_test.yaml

@@ -40,7 +40,7 @@ tests:
     asserts:
       - equal:
           path: spec.template.metadata.annotations["celeborn.apache.org/conf-hash"]
-          value: 7e9a27719ab1f2c1cea53e4879a807782abd48d87ec2a45774084990af6125b3
+          value: 035c23a53d85eb407eecf3fe864be5e623afae38e8ecdd4c10579e76d85f5c6b
 
   - it: Should change checksum annotation when celeborn config changes
     template: worker/statefulset.yaml
@@ -50,10 +50,10 @@ tests:
     asserts:
       - notEqual:
           path: spec.template.metadata.annotations["celeborn.apache.org/conf-hash"]
-          value: 7e9a27719ab1f2c1cea53e4879a807782abd48d87ec2a45774084990af6125b3
+          value: 035c23a53d85eb407eecf3fe864be5e623afae38e8ecdd4c10579e76d85f5c6b
       - equal:
           path: spec.template.metadata.annotations["celeborn.apache.org/conf-hash"]
-          value: 118d5c045d52fbbd4e8f05cf0522044373165960aa57893f645a6f9e8844dd68
+          value: d15c180987eca7e3b13dc37e3277c057a4060799ddd45cf20490ebb91da21ee1
 
   - it: Should add extra pod annotations if `worker.annotations` is specified
     template: worker/statefulset.yaml

common/pom.xml

@@ -47,6 +47,10 @@
       <groupId>io.dropwizard.metrics</groupId>
       <artifactId>metrics-jvm</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.dropwizard.metrics</groupId>
+      <artifactId>metrics-jmx</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.yaml</groupId>
       <artifactId>snakeyaml</artifactId>

common/src/main/scala/org/apache/celeborn/common/metrics/sink/JmxSink.scala

@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.common.metrics.sink
+
+import java.util.Properties
+
+import com.codahale.metrics.MetricRegistry
+import com.codahale.metrics.jmx.JmxReporter
+
+class JmxSink(val property: Properties, val registry: MetricRegistry) extends Sink {
+
+  // Publish MBeans under a configurable, Celeborn-specific JMX domain (defaulting to
+  // `celeborn`) rather than JmxReporter's global default domain `metrics`. This avoids
+  // MBean name collisions when other components using Dropwizard metrics run in the
+  // same JVM. The domain can be overridden via `*.sink.jmx.domain=<domain>`.
+  val domain: String =
+    Option(property.getProperty(JmxSink.JMX_DOMAIN_KEY))
+      .map(_.trim)
+      .filter(_.nonEmpty)
+      .getOrElse(JmxSink.JMX_DEFAULT_DOMAIN)
+
+  val reporter: JmxReporter = JmxReporter.forRegistry(registry)
+    .inDomain(domain)
+    .build()
+
+  override def start(): Unit = {
+    reporter.start()
+  }
+
+  override def stop(): Unit = {
+    reporter.stop()
+  }
+
+  override def report(): Unit = {}
+}
+
+object JmxSink {
+  val JMX_DOMAIN_KEY = "domain"
+  val JMX_DEFAULT_DOMAIN = "celeborn"
+}

conf/metrics.properties.template

@@ -18,3 +18,8 @@
 *.sink.prometheusServlet.class=org.apache.celeborn.common.metrics.sink.PrometheusServlet
 *.sink.jsonServlet.class=org.apache.celeborn.common.metrics.sink.JsonServlet
 *.sink.loggerSink.class=org.apache.celeborn.common.metrics.sink.LoggerSink
+
+# Expose metrics as JMX MBeans. Disabled by default; uncomment to enable.
+# MBeans are published under the `celeborn` JMX domain by default; override with
+# `*.sink.jmx.domain=<domain>` if needed.
+# *.sink.jmx.class=org.apache.celeborn.common.metrics.sink.JmxSink

dev/deps/dependencies-client-flink-1.18

@@ -36,6 +36,7 @@ lz4-java/1.10.4//lz4-java-1.10.4.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
 metrics-core/4.2.25//metrics-core-4.2.25.jar
 metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jmx/4.2.25//metrics-jmx-4.2.25.jar
 metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.2.10.Final//netty-all-4.2.10.Final.jar
 netty-buffer/4.2.10.Final//netty-buffer-4.2.10.Final.jar

dev/deps/dependencies-client-flink-1.19

@@ -36,6 +36,7 @@ lz4-java/1.10.4//lz4-java-1.10.4.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
 metrics-core/4.2.25//metrics-core-4.2.25.jar
 metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jmx/4.2.25//metrics-jmx-4.2.25.jar
 metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.2.10.Final//netty-all-4.2.10.Final.jar
 netty-buffer/4.2.10.Final//netty-buffer-4.2.10.Final.jar

dev/deps/dependencies-client-flink-1.20

@@ -36,6 +36,7 @@ lz4-java/1.10.4//lz4-java-1.10.4.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
 metrics-core/4.2.25//metrics-core-4.2.25.jar
 metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jmx/4.2.25//metrics-jmx-4.2.25.jar
 metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.2.10.Final//netty-all-4.2.10.Final.jar
 netty-buffer/4.2.10.Final//netty-buffer-4.2.10.Final.jar

dev/deps/dependencies-client-flink-2.0

@@ -35,6 +35,7 @@ leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 lz4-java/1.10.4//lz4-java-1.10.4.jar
 metrics-core/4.2.25//metrics-core-4.2.25.jar
 metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jmx/4.2.25//metrics-jmx-4.2.25.jar
 metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.2.10.Final//netty-all-4.2.10.Final.jar
 netty-buffer/4.2.10.Final//netty-buffer-4.2.10.Final.jar

dev/deps/dependencies-client-flink-2.1

@@ -35,6 +35,7 @@ leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 lz4-java/1.10.4//lz4-java-1.10.4.jar
 metrics-core/4.2.25//metrics-core-4.2.25.jar
 metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jmx/4.2.25//metrics-jmx-4.2.25.jar
 metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.2.10.Final//netty-all-4.2.10.Final.jar
 netty-buffer/4.2.10.Final//netty-buffer-4.2.10.Final.jar

dev/deps/dependencies-client-flink-2.2

@@ -35,6 +35,7 @@ leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 lz4-java/1.10.4//lz4-java-1.10.4.jar
 metrics-core/4.2.25//metrics-core-4.2.25.jar
 metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jmx/4.2.25//metrics-jmx-4.2.25.jar
 metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.2.10.Final//netty-all-4.2.10.Final.jar
 netty-buffer/4.2.10.Final//netty-buffer-4.2.10.Final.jar

dev/deps/dependencies-client-flink-2.3

@@ -35,6 +35,7 @@ leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 lz4-java/1.10.4//lz4-java-1.10.4.jar
 metrics-core/4.2.25//metrics-core-4.2.25.jar
 metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jmx/4.2.25//metrics-jmx-4.2.25.jar
 metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.2.10.Final//netty-all-4.2.10.Final.jar
 netty-buffer/4.2.10.Final//netty-buffer-4.2.10.Final.jar

dev/deps/dependencies-client-mr

@@ -138,6 +138,7 @@ lz4-java/1.10.4//lz4-java-1.10.4.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
 metrics-core/4.2.25//metrics-core-4.2.25.jar
 metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jmx/4.2.25//metrics-jmx-4.2.25.jar
 metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 mssql-jdbc/6.2.1.jre7//mssql-jdbc-6.2.1.jre7.jar
 netty-all/4.2.10.Final//netty-all-4.2.10.Final.jar

dev/deps/dependencies-client-spark-3.0

@@ -36,6 +36,7 @@ lz4-java/1.7.1//lz4-java-1.7.1.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
 metrics-core/4.2.25//metrics-core-4.2.25.jar
 metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jmx/4.2.25//metrics-jmx-4.2.25.jar
 metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.2.10.Final//netty-all-4.2.10.Final.jar
 netty-buffer/4.2.10.Final//netty-buffer-4.2.10.Final.jar

dev/deps/dependencies-client-spark-3.1

@@ -36,6 +36,7 @@ lz4-java/1.7.1//lz4-java-1.7.1.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
 metrics-core/4.2.25//metrics-core-4.2.25.jar
 metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jmx/4.2.25//metrics-jmx-4.2.25.jar
 metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.2.10.Final//netty-all-4.2.10.Final.jar
 netty-buffer/4.2.10.Final//netty-buffer-4.2.10.Final.jar

dev/deps/dependencies-client-spark-3.2

@@ -36,6 +36,7 @@ lz4-java/1.7.1//lz4-java-1.7.1.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
 metrics-core/4.2.25//metrics-core-4.2.25.jar
 metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jmx/4.2.25//metrics-jmx-4.2.25.jar
 metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.2.10.Final//netty-all-4.2.10.Final.jar
 netty-buffer/4.2.10.Final//netty-buffer-4.2.10.Final.jar

dev/deps/dependencies-client-spark-3.3

@@ -36,6 +36,7 @@ lz4-java/1.8.0//lz4-java-1.8.0.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
 metrics-core/4.2.25//metrics-core-4.2.25.jar
 metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jmx/4.2.25//metrics-jmx-4.2.25.jar
 metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.2.10.Final//netty-all-4.2.10.Final.jar
 netty-buffer/4.2.10.Final//netty-buffer-4.2.10.Final.jar

dev/deps/dependencies-client-spark-3.4

@@ -36,6 +36,7 @@ lz4-java/1.8.0//lz4-java-1.8.0.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
 metrics-core/4.2.25//metrics-core-4.2.25.jar
 metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jmx/4.2.25//metrics-jmx-4.2.25.jar
 metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.2.10.Final//netty-all-4.2.10.Final.jar
 netty-buffer/4.2.10.Final//netty-buffer-4.2.10.Final.jar

dev/deps/dependencies-client-spark-3.5

@@ -36,6 +36,7 @@ lz4-java/1.8.0//lz4-java-1.8.0.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
 metrics-core/4.2.25//metrics-core-4.2.25.jar
 metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jmx/4.2.25//metrics-jmx-4.2.25.jar
 metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.2.10.Final//netty-all-4.2.10.Final.jar
 netty-buffer/4.2.10.Final//netty-buffer-4.2.10.Final.jar

dev/deps/dependencies-client-spark-4.0

@@ -35,6 +35,7 @@ leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 lz4-java/1.8.0//lz4-java-1.8.0.jar
 metrics-core/4.2.25//metrics-core-4.2.25.jar
 metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jmx/4.2.25//metrics-jmx-4.2.25.jar
 metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.2.10.Final//netty-all-4.2.10.Final.jar
 netty-buffer/4.2.10.Final//netty-buffer-4.2.10.Final.jar

dev/deps/dependencies-client-spark-4.1

@@ -35,6 +35,7 @@ leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 lz4-java/1.8.0//lz4-java-1.8.0.jar
 metrics-core/4.2.25//metrics-core-4.2.25.jar
 metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jmx/4.2.25//metrics-jmx-4.2.25.jar
 metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.2.10.Final//netty-all-4.2.10.Final.jar
 netty-buffer/4.2.10.Final//netty-buffer-4.2.10.Final.jar

dev/deps/dependencies-client-spark-4.2

@@ -35,6 +35,7 @@ leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 lz4-java/1.11.0//lz4-java-1.11.0.jar
 metrics-core/4.2.25//metrics-core-4.2.25.jar
 metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jmx/4.2.25//metrics-jmx-4.2.25.jar
 metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.2.10.Final//netty-all-4.2.10.Final.jar
 netty-buffer/4.2.10.Final//netty-buffer-4.2.10.Final.jar

dev/deps/dependencies-client-tez

@@ -111,6 +111,7 @@ lz4-java/1.10.4//lz4-java-1.10.4.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
 metrics-core/4.2.25//metrics-core-4.2.25.jar
 metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jmx/4.2.25//metrics-jmx-4.2.25.jar
 metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 netty-all/4.2.10.Final//netty-all-4.2.10.Final.jar
 netty-buffer/4.2.10.Final//netty-buffer-4.2.10.Final.jar

dev/deps/dependencies-server

@@ -83,6 +83,7 @@ lz4-java/1.10.4//lz4-java-1.10.4.jar
 maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
 metrics-core/4.2.25//metrics-core-4.2.25.jar
 metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jmx/4.2.25//metrics-jmx-4.2.25.jar
 metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
 mimepull/1.9.15//mimepull-1.9.15.jar
 mybatis/3.5.15//mybatis-3.5.15.jar

docs/monitoring.md

@@ -47,6 +47,7 @@ Each instance can report to zero or more _sinks_. Sinks are contained in the
 * `PrometheusServlet`: Adds a servlet within the existing Celeborn REST API to serve metrics data in Prometheus format.
 * `JsonServlet`: Adds a servlet within the existing Celeborn REST API to serve metrics data in JSON format.
 * `GraphiteSink`: Sends metrics to a Graphite node.
+* `JmxSink`: Registers metrics for viewing in a JMX console.
 * `LoggerSink`: Scrape metrics periodically and output them to the logger files if you have enabled
   `celeborn.metrics.loggerSink.output.enabled`. This is used as safety valve to make sure the
   metrics data won't exist in the memory for a long time. If you don't have a metrics collector to

pom.xml

@@ -282,6 +282,11 @@
         <artifactId>metrics-jvm</artifactId>
         <version>${codahale.metrics.version}</version>
       </dependency>
+      <dependency>
+        <groupId>io.dropwizard.metrics</groupId>
+        <artifactId>metrics-jmx</artifactId>
+        <version>${codahale.metrics.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.ratis</groupId>
         <artifactId>ratis-common</artifactId>

project/CelebornBuild.scala

@@ -142,6 +142,7 @@ object Dependencies {
   val ioDropwizardMetricsGraphite = "io.dropwizard.metrics" % "metrics-graphite" % metricsVersion excludeAll (
     ExclusionRule("com.rabbitmq", "amqp-client"))
   val ioDropwizardMetricsJvm = "io.dropwizard.metrics" % "metrics-jvm" % metricsVersion
+  val ioDropwizardMetricsJmx = "io.dropwizard.metrics" % "metrics-jmx" % metricsVersion
   val ioNetty = "io.netty" % "netty-all" % nettyVersion excludeAll(
     ExclusionRule("io.netty", "netty-codec-haproxy"),
     ExclusionRule("io.netty", "netty-codec-memcache"),
@@ -679,6 +680,7 @@ object CelebornCommon {
         Dependencies.ioDropwizardMetricsCore,
         Dependencies.ioDropwizardMetricsGraphite,
         Dependencies.ioDropwizardMetricsJvm,
+        Dependencies.ioDropwizardMetricsJmx,
         Dependencies.ioNetty,
         Dependencies.ioNettyEpollLinuxX8664,
         Dependencies.ioNettyEpollLinuxAarch64,

service/src/test/resources/metrics-jmx.properties

@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+*.sink.jmx.class=org.apache.celeborn.common.metrics.sink.JmxSink