Skip to content

Commit e02295b

Browse files
j1wonparkj1wonpark
authored
[Feature] Add metric category filtering support to Prometheus exporter (#4084)
* Add metric category filtering support to Prometheus exporter Allow users to enable/disable metric categories (self-optimizing, optimizer-group, orphan-files, ams-jvm, table-summary) via metric-reporters.yaml properties. All categories are enabled by default for backward compatibility. Also update Helm chart values and configmap tests to reflect the new configuration options. Signed-off-by: jiwonpark <jiwonpark@example.com> Signed-off-by: Jiwon Park <jpark92@outlook.kr> * Replace category-based filtering with regex-based metric filtering Replace the fixed category enum approach with a flexible regex-based metric filter. Users can now specify include/exclude patterns via metric-filter.includes and metric-filter.excludes properties to freely configure which metrics are exported. Signed-off-by: Jiwon Park <jpark92@outlook.kr> * Handle invalid regex patterns gracefully and add includes Helm test - Catch PatternSyntaxException in MetricFilter.fromProperties() and fall back to ACCEPT_ALL with a warning log instead of failing reporter initialization - Add tests for invalid regex fallback (includes and excludes) - Add Helm unittest for metric-filter.includes configuration - Fix missing newline at end of amoro-configmap_test.yaml Signed-off-by: Jiwon Park <jpark92@outlook.kr> --------- Signed-off-by: jiwonpark <jiwonpark@example.com> Signed-off-by: Jiwon Park <jpark92@outlook.kr>
1 parent 743e6f1 commit e02295b

8 files changed

Lines changed: 395 additions & 4 deletions

File tree

‎amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricFilter.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.amoro.metrics.promethues;
20+
21+
import org.slf4j.Logger;
22+
import org.slf4j.LoggerFactory;
23+
24+
import java.util.Map;
25+
import java.util.regex.Pattern;
26+
import java.util.regex.PatternSyntaxException;
27+
28+
/** Regex-based metric filter for Prometheus exporter. */
29+
public class MetricFilter {
30+
31+
private static final Logger LOGGER = LoggerFactory.getLogger(MetricFilter.class);
32+
33+
public static final String INCLUDES_KEY = "metric-filter.includes";
34+
public static final String EXCLUDES_KEY = "metric-filter.excludes";
35+
36+
public static final MetricFilter ACCEPT_ALL = new MetricFilter(null, null);
37+
38+
private final Pattern includePattern;
39+
private final Pattern excludePattern;
40+
41+
public MetricFilter(Pattern includePattern, Pattern excludePattern) {
42+
this.includePattern = includePattern;
43+
this.excludePattern = excludePattern;
44+
}
45+
46+
/** Parse metric filter from reporter properties. */
47+
public static MetricFilter fromProperties(Map<String, String> properties) {
48+
String includes = properties.get(INCLUDES_KEY);
49+
String excludes = properties.get(EXCLUDES_KEY);
50+
51+
if (includes == null && excludes == null) {
52+
return ACCEPT_ALL;
53+
}
54+
55+
try {
56+
Pattern includePattern = includes != null ? Pattern.compile(includes) : null;
57+
Pattern excludePattern = excludes != null ? Pattern.compile(excludes) : null;
58+
return new MetricFilter(includePattern, excludePattern);
59+
} catch (PatternSyntaxException e) {
60+
LOGGER.warn(
61+
"Invalid metric filter regex pattern, falling back to accept-all. "
62+
+ "includes='{}', excludes='{}'",
63+
includes,
64+
excludes,
65+
e);
66+
return ACCEPT_ALL;
67+
}
68+
}
69+
70+
/** Check if a metric name passes the filter. */
71+
public boolean matches(String metricName) {
72+
if (includePattern != null && !includePattern.matcher(metricName).matches()) {
73+
return false;
74+
}
75+
if (excludePattern != null && excludePattern.matcher(metricName).matches()) {
76+
return false;
77+
}
78+
return true;
79+
}
80+
}

‎amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricsCollector.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,15 @@ public class MetricsCollector extends Collector {
4343
private static final Pattern NAME_PATTERN = Pattern.compile("[a-zA-Z_:][a-zA-Z0-9_:]*");
4444
private static final Pattern LABEL_PATTERN = Pattern.compile("[a-zA-Z_][a-zA-Z0-9_]*");
4545
MetricSet metrics;
46+
private final MetricFilter metricFilter;
4647

4748
public MetricsCollector(MetricSet metrics) {
49+
this(metrics, MetricFilter.ACCEPT_ALL);
50+
}
51+
52+
public MetricsCollector(MetricSet metrics, MetricFilter metricFilter) {
4853
this.metrics = metrics;
54+
this.metricFilter = metricFilter != null ? metricFilter : MetricFilter.ACCEPT_ALL;
4955
}
5056

5157
@Override
@@ -76,8 +82,14 @@ private boolean isValidMetric(MetricDefine define) {
7682
boolean valid = nameIsValid && labelIsValid;
7783
if (!valid) {
7884
LOGGER.warn("Metric {} is not a valid prometheus metric.", define);
85+
return false;
7986
}
80-
return valid;
87+
88+
if (!metricFilter.matches(define.getName())) {
89+
return false;
90+
}
91+
92+
return true;
8193
}
8294

8395
private MetricFamilySamples createFamilySample(

‎amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/PrometheusMetricsReporter.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ public class PrometheusMetricsReporter implements MetricReporter {
3232
public static final String PORT = "port";
3333

3434
private HTTPServer server;
35+
private MetricFilter metricFilter = MetricFilter.ACCEPT_ALL;
3536

3637
@Override
3738
public void open(Map<String, String> properties) {
@@ -40,6 +41,8 @@ public void open(Map<String, String> properties) {
4041
.map(Integer::valueOf)
4142
.orElseThrow(() -> new IllegalArgumentException("Lack required property: " + PORT));
4243

44+
this.metricFilter = MetricFilter.fromProperties(properties);
45+
4346
try {
4447
this.server = new HTTPServer(port);
4548
} catch (IOException e) {
@@ -59,7 +62,7 @@ public String name() {
5962

6063
@Override
6164
public void setGlobalMetricSet(MetricSet globalMetricSet) {
62-
MetricsCollector collector = new MetricsCollector(globalMetricSet);
65+
MetricsCollector collector = new MetricsCollector(globalMetricSet, metricFilter);
6366
collector.register();
6467
}
6568
}

‎amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricFilterTest.java‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.amoro.metrics.promethues;
20+
21+
import static org.junit.jupiter.api.Assertions.assertFalse;
22+
import static org.junit.jupiter.api.Assertions.assertSame;
23+
import static org.junit.jupiter.api.Assertions.assertTrue;
24+
25+
import org.junit.jupiter.api.Test;
26+
27+
import java.util.Collections;
28+
import java.util.HashMap;
29+
import java.util.Map;
30+
31+
public class MetricFilterTest {
32+
33+
@Test
34+
public void testAcceptAllMatchesEverything() {
35+
MetricFilter filter = MetricFilter.ACCEPT_ALL;
36+
assertTrue(filter.matches("table_optimizing_status_in_idle"));
37+
assertTrue(filter.matches("optimizer_group_pending_tasks"));
38+
assertTrue(filter.matches("any_metric_name"));
39+
}
40+
41+
@Test
42+
public void testFromPropertiesEmptyReturnsAcceptAll() {
43+
MetricFilter filter = MetricFilter.fromProperties(Collections.emptyMap());
44+
assertSame(MetricFilter.ACCEPT_ALL, filter);
45+
}
46+
47+
@Test
48+
public void testIncludesOnly() {
49+
Map<String, String> props = new HashMap<>();
50+
props.put("metric-filter.includes", "table_optimizing_.*|optimizer_group_.*");
51+
52+
MetricFilter filter = MetricFilter.fromProperties(props);
53+
assertTrue(filter.matches("table_optimizing_status_in_idle"));
54+
assertTrue(filter.matches("optimizer_group_pending_tasks"));
55+
assertFalse(filter.matches("table_summary_total_files"));
56+
assertFalse(filter.matches("ams_jvm_cpu_load"));
57+
}
58+
59+
@Test
60+
public void testExcludesOnly() {
61+
Map<String, String> props = new HashMap<>();
62+
props.put("metric-filter.excludes", "table_summary_.*");
63+
64+
MetricFilter filter = MetricFilter.fromProperties(props);
65+
assertTrue(filter.matches("table_optimizing_status_in_idle"));
66+
assertTrue(filter.matches("ams_jvm_cpu_load"));
67+
assertFalse(filter.matches("table_summary_total_files"));
68+
assertFalse(filter.matches("table_summary_health_score"));
69+
}
70+
71+
@Test
72+
public void testIncludesAndExcludes() {
73+
Map<String, String> props = new HashMap<>();
74+
props.put("metric-filter.includes", "table_.*");
75+
props.put("metric-filter.excludes", "table_summary_.*");
76+
77+
MetricFilter filter = MetricFilter.fromProperties(props);
78+
assertTrue(filter.matches("table_optimizing_status_in_idle"));
79+
assertTrue(filter.matches("table_orphan_content_file_cleaning_count"));
80+
assertFalse(filter.matches("table_summary_total_files"));
81+
assertFalse(filter.matches("optimizer_group_pending_tasks"));
82+
}
83+
84+
@Test
85+
public void testExcludesTakesPrecedenceOverIncludes() {
86+
Map<String, String> props = new HashMap<>();
87+
props.put("metric-filter.includes", ".*");
88+
props.put("metric-filter.excludes", "ams_jvm_.*");
89+
90+
MetricFilter filter = MetricFilter.fromProperties(props);
91+
assertTrue(filter.matches("table_optimizing_status_in_idle"));
92+
assertFalse(filter.matches("ams_jvm_cpu_load"));
93+
assertFalse(filter.matches("ams_jvm_memory_heap_used"));
94+
}
95+
96+
@Test
97+
public void testInvalidRegexFallsBackToAcceptAll() {
98+
Map<String, String> props = new HashMap<>();
99+
props.put("metric-filter.includes", "[invalid(regex");
100+
101+
MetricFilter filter = MetricFilter.fromProperties(props);
102+
assertSame(MetricFilter.ACCEPT_ALL, filter);
103+
}
104+
105+
@Test
106+
public void testInvalidExcludesRegexFallsBackToAcceptAll() {
107+
Map<String, String> props = new HashMap<>();
108+
props.put("metric-filter.includes", "table_.*");
109+
props.put("metric-filter.excludes", "[invalid(regex");
110+
111+
MetricFilter filter = MetricFilter.fromProperties(props);
112+
assertSame(MetricFilter.ACCEPT_ALL, filter);
113+
}
114+
115+
@Test
116+
public void testExactMatchPattern() {
117+
Map<String, String> props = new HashMap<>();
118+
props.put("metric-filter.includes", "ams_jvm_cpu_load");
119+
120+
MetricFilter filter = MetricFilter.fromProperties(props);
121+
assertTrue(filter.matches("ams_jvm_cpu_load"));
122+
assertFalse(filter.matches("ams_jvm_cpu_time"));
123+
}
124+
}

0 commit comments

Comments
 (0)