Add Prometheus translation strategy support (#8346)

Signed-off-by: Gregor Zeitlinger <gregor.zeitlinger@grafana.com>

Author: zeitlinger

dependencyManagement/build.gradle.kts

@@ -15,7 +15,7 @@ val jmhVersion = "1.37"
 val mockitoVersion = "4.11.0"
 val slf4jVersion = "2.0.18"
 val opencensusVersion = "0.31.1"
-val prometheusServerVersion = "1.5.1"
+val prometheusServerVersion = "1.8.0"
 val armeriaVersion = "1.39.1"
 val junitVersion = "5.14.4"
 val junitPlatformVersion = "1.14.4"

exporters/prometheus/src/main/java/io/opentelemetry/exporter/prometheus/Otel2PrometheusConverter.java

@@ -5,9 +5,6 @@
 
 package io.opentelemetry.exporter.prometheus;
 
-import static io.prometheus.metrics.model.snapshots.PrometheusNaming.prometheusName;
-import static io.prometheus.metrics.model.snapshots.PrometheusNaming.sanitizeLabelName;
-import static io.prometheus.metrics.model.snapshots.PrometheusNaming.sanitizeMetricName;
 import static java.util.Objects.requireNonNull;
 
 import io.opentelemetry.api.common.AttributeKey;
@@ -87,6 +84,7 @@ final class Otel2PrometheusConverter {
 
   private final boolean otelScopeLabelsEnabled;
   private final boolean targetInfoMetricEnabled;
+  private final TranslationStrategy translationStrategy;
   @Nullable private final Predicate<String> allowedResourceAttributesFilter;
 
   /**
@@ -107,9 +105,11 @@ final class Otel2PrometheusConverter {
   Otel2PrometheusConverter(
       boolean otelScopeLabelsEnabled,
       boolean targetInfoMetricEnabled,
+      TranslationStrategy translationStrategy,
       @Nullable Predicate<String> allowedResourceAttributesFilter) {
     this.otelScopeLabelsEnabled = otelScopeLabelsEnabled;
     this.targetInfoMetricEnabled = targetInfoMetricEnabled;
+    this.translationStrategy = translationStrategy;
     this.allowedResourceAttributesFilter = allowedResourceAttributesFilter;
     this.resourceAttributesToAllowedKeysCache =
         allowedResourceAttributesFilter != null
@@ -125,6 +125,10 @@ boolean isTargetInfoMetricEnabled() {
     return targetInfoMetricEnabled;
   }
 
+  TranslationStrategy getTranslationStrategy() {
+    return translationStrategy;
+  }
+
   @Nullable
   Predicate<String> getAllowedResourceAttributesFilter() {
     return allowedResourceAttributesFilter;
@@ -154,11 +158,24 @@ MetricSnapshots convert(@Nullable Collection<MetricData> metricDataCollection) {
 
   @Nullable
   private MetricSnapshot convert(MetricData metricData) {
+    try {
+      return doConvert(metricData);
+    } catch (IllegalArgumentException e) {
+      THROTTLING_LOGGER.log(
+          Level.WARNING,
+          "Failed to convert metric " + metricData.getName() + ". Dropping metric.",
+          e);
+      return null;
+    }
+  }
 
+  @Nullable
+  private MetricSnapshot doConvert(MetricData metricData) {
     // Note that AggregationTemporality.DELTA should never happen
     // because PrometheusMetricReader#getAggregationTemporality returns CUMULATIVE.
 
-    MetricMetadata metadata = convertMetadata(metricData);
+    boolean isCounter = isMonotonicSum(metricData);
+    MetricMetadata metadata = convertMetadata(metricData, isCounter);
     InstrumentationScopeInfo scope = metricData.getInstrumentationScopeInfo();
     switch (metricData.getType()) {
       case LONG_GAUGE:
@@ -213,6 +230,17 @@ private MetricSnapshot convert(MetricData metricData) {
     return null;
   }
 
+  private static boolean isMonotonicSum(MetricData metricData) {
+    switch (metricData.getType()) {
+      case LONG_SUM:
+        return metricData.getLongSumData().isMonotonic();
+      case DOUBLE_SUM:
+        return metricData.getDoubleSumData().isMonotonic();
+      default:
+        return false;
+    }
+  }
+
   private GaugeSnapshot convertLongGauge(
       MetricMetadata metadata,
       InstrumentationScopeInfo scope,
@@ -479,7 +507,7 @@ private static int labelSetLength(Labels labels) {
 
   private InfoSnapshot makeTargetInfo(Resource resource) {
     return new InfoSnapshot(
-        new MetricMetadata("target"),
+        MetricMetadata.builder().name("target").build(),
         Collections.singletonList(
             new InfoDataPointSnapshot(
                 convertAttributes(
@@ -582,34 +610,181 @@ private List<AttributeKey<?>> filterAllowedResourceAttributeKeys(@Nullable Resou
     return allowedAttributeKeys;
   }
 
+  private String convertLabelName(String key) {
+    if (shouldEscape(translationStrategy)) {
+      return convertLegacyLabelName(key);
+    }
+    return key;
+  }
+
   /**
-   * Convert an attribute key to a legacy Prometheus label name. {@code prometheusName} converts
-   * non-standard characters (dots, dashes, etc.) to underscores, and {@code sanitizeLabelName}
-   * strips invalid leading prefixes.
+   * Normalize an attribute name to the legacy Prometheus label scheme.
+   *
+   * <p>This mirrors {@code prometheus/otlptranslator}'s invalid-character collapsing rules, but we
+   * intentionally do not preserve {@code __...__} names because Prometheus Java rejects user label
+   * names that begin with {@code __}.
    */
-  private static String convertLabelName(String key) {
-    return sanitizeLabelName(prometheusName(key));
+  private static String convertLegacyLabelName(String key) {
+    if (key.isEmpty()) {
+      throw new IllegalArgumentException("label name is empty");
+    }
+
+    // OTel owns OTLP-to-Prometheus translation. Prometheus Java validates and serializes names,
+    // but no longer owns this naming mangling. The OTel compatibility spec requires invalid
+    // attribute-name characters and repeated underscores to collapse to a single "_". Prometheus
+    // label names beginning with "__" are reserved for internal labels like "__name__" and
+    // scrape/relabel labels, and Prometheus Java rejects them as user labels, so do not preserve
+    // "__...__" reserved-looking names here.
+    StringBuilder result = new StringBuilder(key.length());
+    boolean previousWasUnderscore = false;
+    for (int i = 0; i < key.length(); ) {
+      int codePoint = key.codePointAt(i);
+      if (isValidLegacyLabelChar(codePoint)) {
+        result.appendCodePoint(codePoint);
+        previousWasUnderscore = false;
+      } else if (!previousWasUnderscore) {
+        result.append('_');
+        previousWasUnderscore = true;
+      }
+      i += Character.charCount(codePoint);
+    }
+
+    String normalized = result.toString();
+    if (Character.isDigit(normalized.charAt(0))) {
+      normalized = "key_" + normalized;
+    }
+    if (containsOnlyUnderscores(normalized)) {
+      throw new IllegalArgumentException(
+          "normalization for label name \""
+              + key
+              + "\" resulted in invalid name \""
+              + normalized
+              + "\"");
+    }
+    return normalized;
+  }
+
+  private static boolean isValidLegacyLabelChar(int codePoint) {
+    return (codePoint >= 'a' && codePoint <= 'z')
+        || (codePoint >= 'A' && codePoint <= 'Z')
+        || (codePoint >= '0' && codePoint <= '9');
+  }
+
+  private static boolean containsOnlyUnderscores(String value) {
+    for (int i = 0; i < value.length(); i++) {
+      if (value.charAt(i) != '_') {
+        return false;
+      }
+    }
+    return true;
   }
 
-  private static MetricMetadata convertMetadata(MetricData metricData) {
-    String name = sanitizeMetricName(prometheusName(metricData.getName()));
+  private static String convertLegacyMetricName(String name) {
+    if (name.isEmpty()) {
+      throw new IllegalArgumentException("metric name is empty");
+    }
+
+    StringBuilder result = new StringBuilder(name.length());
+    boolean previousWasUnderscore = false;
+    for (int i = 0; i < name.length(); ) {
+      int codePoint = name.codePointAt(i);
+      if (isValidLegacyMetricChar(codePoint, i) && codePoint != '_') {
+        result.appendCodePoint(codePoint);
+        previousWasUnderscore = false;
+      } else if (!previousWasUnderscore) {
+        result.append('_');
+        previousWasUnderscore = true;
+      }
+      i += Character.charCount(codePoint);
+    }
+    return result.toString();
+  }
+
+  private static boolean isValidLegacyMetricChar(int codePoint, int index) {
+    return (codePoint >= 'a' && codePoint <= 'z')
+        || (codePoint >= 'A' && codePoint <= 'Z')
+        || codePoint == '_'
+        || codePoint == ':'
+        || (codePoint >= '0' && codePoint <= '9' && index > 0);
+  }
+
+  private MetricMetadata convertMetadata(MetricData metricData, boolean isCounter) {
+    switch (translationStrategy) {
+      case UNDERSCORE_ESCAPING_WITH_SUFFIXES:
+        return convertMetadataEscapedWithSuffixes(metricData);
+      case UNDERSCORE_ESCAPING_WITHOUT_SUFFIXES:
+        return convertMetadataEscapedWithoutSuffixes(metricData);
+      case NO_UTF8_ESCAPING_WITH_SUFFIXES:
+        return convertMetadataUtf8WithSuffixes(metricData, isCounter);
+      case NO_TRANSLATION:
+        return convertMetadataNoTranslation(metricData);
+    }
+    throw new IllegalStateException("Unknown strategy: " + translationStrategy);
+  }
+
+  private static MetricMetadata convertMetadataEscapedWithSuffixes(MetricData metricData) {
+    String originalName = metricData.getName();
+    String name = stripReservedMetricSuffixes(convertLegacyMetricName(originalName));
     String help = metricData.getDescription();
     Unit unit = PrometheusUnitsHelper.convertUnit(metricData.getUnit());
     if (unit != null && !name.endsWith(unit.toString())) {
       name = name + "_" + unit;
     }
-    // Repeated __ are discouraged according to spec, although this is allowed in prometheus, see
-    // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/compatibility/prometheus_and_openmetrics.md#metric-metadata-1
-    while (name.contains("__")) {
-      name = name.replace("__", "_");
+    validateNormalizedMetricName(originalName, name);
+    return MetricMetadata.builder().name(name).help(help).unit(unit).build();
+  }
+
+  private static MetricMetadata convertMetadataEscapedWithoutSuffixes(MetricData metricData) {
+    String originalName = metricData.getName();
+    String name = stripReservedMetricSuffixes(convertLegacyMetricName(originalName));
+    String help = metricData.getDescription();
+    validateNormalizedMetricName(originalName, name);
+    return MetricMetadata.builder().name(name).help(help).build();
+  }
+
+  private static MetricMetadata convertMetadataUtf8WithSuffixes(
+      MetricData metricData, boolean isCounter) {
+    String name = metricData.getName();
+    String help = metricData.getDescription();
+    Unit unit = PrometheusUnitsHelper.convertUnit(metricData.getUnit());
+    if (unit != null && !name.endsWith(unit.toString())) {
+      name = name + "_" + unit;
     }
+    return MetricMetadata.builder()
+        .name(stripReservedMetricSuffixes(name))
+        .help(help)
+        .unit(unit)
+        .counterSuffix(isCounter)
+        .build();
+  }
 
-    return new MetricMetadata(name, help, unit);
+  private static MetricMetadata convertMetadataNoTranslation(MetricData metricData) {
+    String name = stripReservedMetricSuffixes(metricData.getName());
+    String help = metricData.getDescription();
+    return MetricMetadata.builder().name(name).help(help).build();
   }
 
-  private static void putOrMerge(
-      Map<String, MetricSnapshot> snapshotsByName, MetricSnapshot snapshot) {
-    String name = snapshot.getMetadata().getPrometheusName();
+  private static String stripReservedMetricSuffixes(String name) {
+    boolean modified = true;
+    while (modified) {
+      modified = false;
+      for (String suffix : PrometheusUnitsHelper.RESERVED_SUFFIXES) {
+        if (name.equals(suffix)) {
+          return name.substring(1);
+        }
+        if (name.endsWith(suffix)) {
+          name = name.substring(0, name.length() - suffix.length());
+          modified = true;
+        }
+      }
+    }
+    return name;
+  }
+
+  private void putOrMerge(Map<String, MetricSnapshot> snapshotsByName, MetricSnapshot snapshot) {
+    MetricMetadata metadata = snapshot.getMetadata();
+    String name =
+        shouldEscape(translationStrategy) ? metadata.getPrometheusName() : metadata.getName();
     if (snapshotsByName.containsKey(name)) {
       MetricSnapshot merged = merge(snapshotsByName.get(name), snapshot);
       if (merged != null) {
@@ -620,6 +795,26 @@ private static void putOrMerge(
     }
   }
 
+  private static boolean shouldEscape(TranslationStrategy translationStrategy) {
+    return translationStrategy == TranslationStrategy.UNDERSCORE_ESCAPING_WITH_SUFFIXES
+        || translationStrategy == TranslationStrategy.UNDERSCORE_ESCAPING_WITHOUT_SUFFIXES;
+  }
+
+  private static void validateNormalizedMetricName(String originalName, String normalizedName) {
+    if (normalizedName.isEmpty()) {
+      throw new IllegalArgumentException(
+          "normalization for metric \"" + originalName + "\" resulted in empty name");
+    }
+    if (containsOnlyUnderscores(normalizedName)) {
+      throw new IllegalArgumentException(
+          "normalization for metric \""
+              + originalName
+              + "\" resulted in invalid name \""
+              + normalizedName
+              + "\"");
+    }
+  }
+
   /**
    * OpenTelemetry may use the same metric name multiple times but in different instrumentation
    * scopes. In that case, we try to merge the metrics. They will have different {@code
@@ -695,7 +890,7 @@ private static MetricMetadata mergeMetadata(MetricMetadata a, MetricMetadata b)
               + ".");
       return null;
     }
-    return new MetricMetadata(name, help, unit);
+    return MetricMetadata.builder().name(name).help(help).unit(unit).build();
   }
 
   private static String typeString(MetricSnapshot snapshot) {

exporters/prometheus/src/main/java/io/opentelemetry/exporter/prometheus/PrometheusHttpServer.java

@@ -21,6 +21,7 @@
 import io.opentelemetry.sdk.metrics.export.CollectionRegistration;
 import io.opentelemetry.sdk.metrics.export.DefaultAggregationSelector;
 import io.opentelemetry.sdk.metrics.export.MetricReader;
+import io.prometheus.metrics.config.PrometheusProperties;
 import io.prometheus.metrics.exporter.httpserver.HTTPServer;
 import io.prometheus.metrics.model.registry.PrometheusRegistry;
 import java.io.IOException;
@@ -73,6 +74,7 @@ public static PrometheusHttpServerBuilder builder() {
       @Nullable HttpHandler defaultHandler,
       DefaultAggregationSelector defaultAggregationSelector,
       @Nullable Authenticator authenticator,
+      TranslationStrategy translationStrategy,
       PrometheusMetricReader prometheusMetricReader) {
     this.host = host;
     this.port = port;
@@ -95,9 +97,17 @@ public static PrometheusHttpServerBuilder builder() {
               new LinkedBlockingQueue<>(),
               new DaemonThreadFactory("prometheus-http-server"));
     }
+    HTTPServer.Builder httpServerBuilder = HTTPServer.builder();
+    if (translationStrategy != TranslationStrategy.UNDERSCORE_ESCAPING_WITH_SUFFIXES) {
+      // Intentionally enable OM2 without content negotiation so OpenMetrics responses keep the
+      // legacy OM1 content type while using OM2 name-preservation semantics.
+      PrometheusProperties prometheusProperties =
+          PrometheusProperties.builder().enableOpenMetrics2(om2 -> {}).build();
+      httpServerBuilder = HTTPServer.builder(prometheusProperties);
+    }
     try {
       this.httpServer =
-          HTTPServer.builder()
+          httpServerBuilder
               .hostname(host)
               .port(port)
               .executorService(executor)

exporters/prometheus/src/main/java/io/opentelemetry/exporter/prometheus/PrometheusHttpServerBuilder.java

@@ -102,6 +102,20 @@ public PrometheusHttpServerBuilder setTargetInfoMetricEnabled(boolean targetInfo
     return this;
   }
 
+  /**
+   * Sets the translation strategy for metric and label name conversion.
+   *
+   * @param translationStrategy the strategy to use
+   * @return this builder
+   * @see TranslationStrategy
+   */
+  public PrometheusHttpServerBuilder setTranslationStrategy(
+      TranslationStrategy translationStrategy) {
+    requireNonNull(translationStrategy, "translationStrategy");
+    metricReaderBuilder.setTranslationStrategy(translationStrategy);
+    return this;
+  }
+
   /**
    * Set if the resource attributes should be added as labels on each exported metric.
    *
@@ -201,6 +215,7 @@ public PrometheusHttpServer build() {
         defaultHandler,
         defaultAggregationSelector,
         authenticator,
+        metricReaderBuilder.getTranslationStrategy(),
         metricReaderBuilder.build());
   }
 }