From c4a611a8554eb98f7f8425f79c2692198812debe Mon Sep 17 00:00:00 2001 From: jeremyhi Date: Mon, 19 May 2025 16:10:31 +0800 Subject: [PATCH 1/3] feat: add readme for metrics module --- .../bench/BatchingWriteBenchmark.java | 5 +- .../io/greptime/bench/BulkWriteBenchmark.java | 5 +- .../bench/StreamingWriteBenchmark.java | 5 +- ingester-prometheus-metrics/README.md | 52 +++++++++++++++ .../io/greptime/metrics/ExporterOptions.java | 63 +++++++++++++++++++ .../io/greptime/metrics/MetricsExporter.java | 20 +++--- 6 files changed, 137 insertions(+), 13 deletions(-) create mode 100644 ingester-prometheus-metrics/README.md create mode 100644 ingester-prometheus-metrics/src/main/java/io/greptime/metrics/ExporterOptions.java diff --git a/ingester-example/src/main/java/io/greptime/bench/BatchingWriteBenchmark.java b/ingester-example/src/main/java/io/greptime/bench/BatchingWriteBenchmark.java index 36fd042..10e306e 100644 --- a/ingester-example/src/main/java/io/greptime/bench/BatchingWriteBenchmark.java +++ b/ingester-example/src/main/java/io/greptime/bench/BatchingWriteBenchmark.java @@ -21,6 +21,7 @@ import io.greptime.common.util.MetricsUtil; import io.greptime.common.util.ServiceLoader; import io.greptime.common.util.SystemPropertyUtil; +import io.greptime.metrics.ExporterOptions; import io.greptime.metrics.MetricsExporter; import io.greptime.models.Err; import io.greptime.models.Result; @@ -61,8 +62,8 @@ public static void main(String[] args) throws Exception { LOG.info("Max points per second: {}", maxPointsPerSecond); // Start a metrics exporter - MetricsExporter metricsExporter = new MetricsExporter(8080, MetricsUtil.metricRegistry()); - metricsExporter.init(null); + MetricsExporter metricsExporter = new MetricsExporter(MetricsUtil.metricRegistry()); + metricsExporter.init(ExporterOptions.newDefault()); GreptimeDB greptimeDB = DBConnector.connectTo(new String[] {endpoint}, dbName); diff --git a/ingester-example/src/main/java/io/greptime/bench/BulkWriteBenchmark.java b/ingester-example/src/main/java/io/greptime/bench/BulkWriteBenchmark.java index 204f74f..67e2998 100644 --- a/ingester-example/src/main/java/io/greptime/bench/BulkWriteBenchmark.java +++ b/ingester-example/src/main/java/io/greptime/bench/BulkWriteBenchmark.java @@ -22,6 +22,7 @@ import io.greptime.common.util.MetricsUtil; import io.greptime.common.util.ServiceLoader; import io.greptime.common.util.SystemPropertyUtil; +import io.greptime.metrics.ExporterOptions; import io.greptime.metrics.MetricsExporter; import io.greptime.models.Table; import io.greptime.models.TableSchema; @@ -55,8 +56,8 @@ public static void main(String[] args) throws Exception { LOG.info("Batch size: {}", batchSize); // Start a metrics exporter - MetricsExporter metricsExporter = new MetricsExporter(8080, MetricsUtil.metricRegistry()); - metricsExporter.init(null); + MetricsExporter metricsExporter = new MetricsExporter(MetricsUtil.metricRegistry()); + metricsExporter.init(ExporterOptions.newDefault()); GreptimeDB greptimeDB = DBConnector.connectTo(new String[] {endpoint}, dbName); diff --git a/ingester-example/src/main/java/io/greptime/bench/StreamingWriteBenchmark.java b/ingester-example/src/main/java/io/greptime/bench/StreamingWriteBenchmark.java index 7d99220..9fbd164 100644 --- a/ingester-example/src/main/java/io/greptime/bench/StreamingWriteBenchmark.java +++ b/ingester-example/src/main/java/io/greptime/bench/StreamingWriteBenchmark.java @@ -21,6 +21,7 @@ import io.greptime.common.util.MetricsUtil; import io.greptime.common.util.ServiceLoader; import io.greptime.common.util.SystemPropertyUtil; +import io.greptime.metrics.ExporterOptions; import io.greptime.metrics.MetricsExporter; import io.greptime.models.Table; import io.greptime.models.TableSchema; @@ -58,8 +59,8 @@ public static void main(String[] args) throws Exception { LOG.info("Max points per second: {}", maxPointsPerSecond); // Start a metrics exporter - MetricsExporter metricsExporter = new MetricsExporter(8080, MetricsUtil.metricRegistry()); - metricsExporter.init(null); + MetricsExporter metricsExporter = new MetricsExporter(MetricsUtil.metricRegistry()); + metricsExporter.init(ExporterOptions.newDefault()); GreptimeDB greptimeDB = DBConnector.connectTo(new String[] {endpoint}, dbName); diff --git a/ingester-prometheus-metrics/README.md b/ingester-prometheus-metrics/README.md new file mode 100644 index 0000000..698a266 --- /dev/null +++ b/ingester-prometheus-metrics/README.md @@ -0,0 +1,52 @@ +# Export Metrics + +Monitoring metrics is essential for maintaining a healthy ingester deployment. It enables you to: + +- Assess the current state of your ingester +- Proactively maintain your deployment +- Quickly diagnose and resolve issues when they arise + +For a comprehensive list of available metrics and their descriptions, please refer to our [detailed metrics documentation](../docs/metrics-display.md#list-of-metrics-constantly-updated). + +## Start Metrics Exporter + +To start an HTTP server that serves Prometheus metrics, initialize the metrics exporter before starting the GreptimeDB ingester: + +```java + MetricsExporter metricsExporter = new MetricsExporter(MetricsUtil.metricRegistry()); + metricsExporter.init(ExporterOptions.newDefault()); + // Start GreptimeDB ingester + // ... +``` + +## Get Metrics + +You can check the output of `curl http://:/metrics` by getting the latest metrics of Ingester. + +## Export metrics to Prometheus + +Ingester supports exporting metrics to Prometheus. Before configuring export of metrics, you need to setup Prometheus by following their official [documentation](https://prometheus.io/docs/prometheus/latest/installation/). + +To scrape metrics from Ingester, write a Prometheus configuration file and save it as `prometheus.yml`: + +```yaml +global: + scrape_interval: 15s + +scrape_configs: + - job_name: 'greptimedb ingester' + static_configs: + # Assuming that Ingester is running locally. + # The default HTTP port of 8090. + - targets: ['localhost:8090'] +``` + +Start Prometheus using the configuration file: + +```yaml +./prometheus --config.file=prometheus.yml +``` + +## Grafana Dashboard + +You can import dashboard via JSON model: [`greptimedb-ingester-dashboard.json`](../grafana/greptimedb-ingester-dashboard.json) diff --git a/ingester-prometheus-metrics/src/main/java/io/greptime/metrics/ExporterOptions.java b/ingester-prometheus-metrics/src/main/java/io/greptime/metrics/ExporterOptions.java new file mode 100644 index 0000000..4be3ac0 --- /dev/null +++ b/ingester-prometheus-metrics/src/main/java/io/greptime/metrics/ExporterOptions.java @@ -0,0 +1,63 @@ +/* + * Copyright 2023 Greptime Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.greptime.metrics; + +import io.greptime.common.Copiable; + +/** + * Exporter options. + */ +public class ExporterOptions implements Copiable { + private int port; + private boolean deamon; + + public static ExporterOptions newDefault() { + ExporterOptions opts = new ExporterOptions(); + opts.port = 8090; + opts.deamon = true; + return opts; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public boolean isDeamon() { + return deamon; + } + + public void setDeamon(boolean deamon) { + this.deamon = deamon; + } + + @Override + public ExporterOptions copy() { + ExporterOptions opts = new ExporterOptions(); + opts.port = this.port; + opts.deamon = this.deamon; + return opts; + } + + @Override + public String toString() { + return "ExporterOptions{" + "port=" + port + ", deamon=" + deamon + '}'; + } +} diff --git a/ingester-prometheus-metrics/src/main/java/io/greptime/metrics/MetricsExporter.java b/ingester-prometheus-metrics/src/main/java/io/greptime/metrics/MetricsExporter.java index 03a83a2..abcb72c 100644 --- a/ingester-prometheus-metrics/src/main/java/io/greptime/metrics/MetricsExporter.java +++ b/ingester-prometheus-metrics/src/main/java/io/greptime/metrics/MetricsExporter.java @@ -27,29 +27,30 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class MetricsExporter implements Lifecycle { +public class MetricsExporter implements Lifecycle { private static final Logger LOG = LoggerFactory.getLogger(MetricsExporter.class); private final CollectorRegistry prometheusMetricRegistry; - private final int port; private HTTPServer server; + private ExporterOptions opts; private final AtomicBoolean started = new AtomicBoolean(false); - public MetricsExporter(int port, MetricRegistry dropwizardMetricRegistry) { - this.port = port; + public MetricsExporter(MetricRegistry dropwizardMetricRegistry) { this.prometheusMetricRegistry = new CollectorRegistry(); this.prometheusMetricRegistry.register(new DropwizardExports(dropwizardMetricRegistry)); } @Override - public boolean init(Void opts) { + public boolean init(ExporterOptions opts) { if (this.started.compareAndSet(false, true)) { + this.opts = opts; try { - this.server = new HTTPServer(new InetSocketAddress(this.port), this.prometheusMetricRegistry, true); - LOG.info("Metrics exporter started at `http://localhost:{}/metrics`", this.port); + this.server = new HTTPServer( + new InetSocketAddress(opts.getPort()), this.prometheusMetricRegistry, opts.isDeamon()); + LOG.info("Metrics exporter started at `http://localhost:{}/metrics`", opts.getPort()); return true; } catch (IOException e) { this.started.set(false); @@ -69,4 +70,9 @@ public void shutdownGracefully() { } } } + + @Override + public String toString() { + return "MetricsExporter{" + "opts=" + opts + '}'; + } } From 9f1515395b69fbf255f19f5eb8caf29ee66c05f4 Mon Sep 17 00:00:00 2001 From: jeremyhi Date: Mon, 19 May 2025 16:28:15 +0800 Subject: [PATCH 2/3] chore: minor refactor the main readme --- README.md | 19 ++++++++++++++----- ingester-prometheus-metrics/README.md | 22 +++++++++++----------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 94b4fc2..bded513 100644 --- a/README.md +++ b/README.md @@ -10,14 +10,14 @@ This client offers multiple ingestion methods optimized for various performance ## Documentation - [API Reference](https://javadoc.io/doc/io.greptime/ingester-protocol/latest/index.html) -- [Examples](https://github.com/GreptimeTeam/greptimedb-ingester-java/tree/main/ingester-example) +- [Examples](./ingester-example) ## Features - Writing data using - - [Regular Write API](https://github.com/GreptimeTeam/greptimedb-ingester-java/tree/main/ingester-example#regular-write-api) - - [Batching Write](https://github.com/GreptimeTeam/greptimedb-ingester-java/tree/main/ingester-example#batching-write) - - [Streaming Write](https://github.com/GreptimeTeam/greptimedb-ingester-java/tree/main/ingester-example#streaming-write) - - [Bulk Write API](https://github.com/GreptimeTeam/greptimedb-ingester-java/tree/main/ingester-example#bulk-write-api) + - [Regular Write API](./ingester-example#regular-write-api) + - [Batching Write](./ingester-example#batching-write) + - [Streaming Write](./ingester-example#streaming-write) + - [Bulk Write API](./ingester-example#bulk-write-api) - Management API client for managing - Health check - Authorizations @@ -100,6 +100,9 @@ This client offers multiple ingestion methods optimized for various performance - [Compression Options](#compression-options) - [Write Operation Comparison](#write-operation-comparison) - [Buffer Size Optimization](#buffer-size-optimization) +- [Export metrics](#export-metrics) +- [Build Requirements](#build-requirements) +- [Contributing](#contributing) ### Installation @@ -364,6 +367,12 @@ Table.TableBufferRoot table = bulkStreamWriter.tableBufferRoot(columnBufferSize) This option can significantly improve the speed of data conversion to the underlying format. For optimal performance, we recommend setting the column buffer size to 1024 or larger, depending on your specific workload characteristics and available memory. +### Export Metrics + +The ingester exposes comprehensive metrics that enable you to monitor its performance, health, and operational status. + +For detailed information about available metrics and their usage, refer to the [Ingester Prometheus Metrics](./ingester-prometheus-metrics/) documentation. + ### Build Requirements - Java 8+ diff --git a/ingester-prometheus-metrics/README.md b/ingester-prometheus-metrics/README.md index 698a266..1fe6aae 100644 --- a/ingester-prometheus-metrics/README.md +++ b/ingester-prometheus-metrics/README.md @@ -1,12 +1,12 @@ # Export Metrics -Monitoring metrics is essential for maintaining a healthy ingester deployment. It enables you to: +Metrics monitoring is a critical component for ensuring optimal ingester performance and reliability. It provides: -- Assess the current state of your ingester -- Proactively maintain your deployment -- Quickly diagnose and resolve issues when they arise +- Real-time visibility into ingester health and performance +- Proactive monitoring capabilities for maintaining system stability +- Rapid troubleshooting and issue resolution -For a comprehensive list of available metrics and their descriptions, please refer to our [detailed metrics documentation](../docs/metrics-display.md#list-of-metrics-constantly-updated). +For detailed information about available metrics, including descriptions and usage guidelines, see our [comprehensive metrics documentation](../docs/metrics-display.md#list-of-metrics-constantly-updated). ## Start Metrics Exporter @@ -21,13 +21,13 @@ To start an HTTP server that serves Prometheus metrics, initialize the metrics e ## Get Metrics -You can check the output of `curl http://:/metrics` by getting the latest metrics of Ingester. +You can check the output of `curl http://:/metrics` by getting the latest metrics of the ingester. -## Export metrics to Prometheus +## Export Metrics to Prometheus -Ingester supports exporting metrics to Prometheus. Before configuring export of metrics, you need to setup Prometheus by following their official [documentation](https://prometheus.io/docs/prometheus/latest/installation/). +The ingester supports exporting metrics to Prometheus. Before configuring export of metrics, you need to setup Prometheus by following their official [documentation](https://prometheus.io/docs/prometheus/latest/installation/). -To scrape metrics from Ingester, write a Prometheus configuration file and save it as `prometheus.yml`: +To scrape metrics from the ingester, write a Prometheus configuration file and save it as `prometheus.yml`: ```yaml global: @@ -36,14 +36,14 @@ global: scrape_configs: - job_name: 'greptimedb ingester' static_configs: - # Assuming that Ingester is running locally. + # Assuming that the ingester is running locally. # The default HTTP port of 8090. - targets: ['localhost:8090'] ``` Start Prometheus using the configuration file: -```yaml +```bash ./prometheus --config.file=prometheus.yml ``` From 10f262779b9191def34a2f8bc599fbf9c91b2afb Mon Sep 17 00:00:00 2001 From: jeremyhi Date: Mon, 19 May 2025 19:19:24 +0800 Subject: [PATCH 3/3] chore: by comment --- .../io/greptime/metrics/ExporterOptions.java | 17 +++++++++-------- .../io/greptime/metrics/MetricsExporter.java | 8 +++++--- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/ingester-prometheus-metrics/src/main/java/io/greptime/metrics/ExporterOptions.java b/ingester-prometheus-metrics/src/main/java/io/greptime/metrics/ExporterOptions.java index 4be3ac0..9c4c288 100644 --- a/ingester-prometheus-metrics/src/main/java/io/greptime/metrics/ExporterOptions.java +++ b/ingester-prometheus-metrics/src/main/java/io/greptime/metrics/ExporterOptions.java @@ -17,27 +17,28 @@ package io.greptime.metrics; import io.greptime.common.Copiable; +import io.greptime.common.Endpoint; /** * Exporter options. */ public class ExporterOptions implements Copiable { - private int port; + private Endpoint bind_addr; private boolean deamon; public static ExporterOptions newDefault() { ExporterOptions opts = new ExporterOptions(); - opts.port = 8090; + opts.bind_addr = new Endpoint("0.0.0.0", 8090); opts.deamon = true; return opts; } - public int getPort() { - return port; + public Endpoint getBindAddr() { + return bind_addr; } - public void setPort(int port) { - this.port = port; + public void setBindAddr(Endpoint bind_addr) { + this.bind_addr = bind_addr; } public boolean isDeamon() { @@ -51,13 +52,13 @@ public void setDeamon(boolean deamon) { @Override public ExporterOptions copy() { ExporterOptions opts = new ExporterOptions(); - opts.port = this.port; + opts.bind_addr = this.bind_addr; opts.deamon = this.deamon; return opts; } @Override public String toString() { - return "ExporterOptions{" + "port=" + port + ", deamon=" + deamon + '}'; + return "ExporterOptions{" + "bind_addr=" + bind_addr + ", deamon=" + deamon + '}'; } } diff --git a/ingester-prometheus-metrics/src/main/java/io/greptime/metrics/MetricsExporter.java b/ingester-prometheus-metrics/src/main/java/io/greptime/metrics/MetricsExporter.java index abcb72c..f79aee7 100644 --- a/ingester-prometheus-metrics/src/main/java/io/greptime/metrics/MetricsExporter.java +++ b/ingester-prometheus-metrics/src/main/java/io/greptime/metrics/MetricsExporter.java @@ -17,6 +17,7 @@ package io.greptime.metrics; import com.codahale.metrics.MetricRegistry; +import io.greptime.common.Endpoint; import io.greptime.common.Lifecycle; import io.prometheus.client.CollectorRegistry; import io.prometheus.client.dropwizard.DropwizardExports; @@ -48,9 +49,10 @@ public boolean init(ExporterOptions opts) { if (this.started.compareAndSet(false, true)) { this.opts = opts; try { - this.server = new HTTPServer( - new InetSocketAddress(opts.getPort()), this.prometheusMetricRegistry, opts.isDeamon()); - LOG.info("Metrics exporter started at `http://localhost:{}/metrics`", opts.getPort()); + Endpoint bindAddr = opts.getBindAddr(); + InetSocketAddress socketAddress = new InetSocketAddress(bindAddr.getAddr(), bindAddr.getPort()); + this.server = new HTTPServer(socketAddress, this.prometheusMetricRegistry, opts.isDeamon()); + LOG.info("Metrics exporter started at `http://{}:{}/metrics`", bindAddr.getAddr(), bindAddr.getPort()); return true; } catch (IOException e) { this.started.set(false);