HDDS-13804. Make server tracing config dynamically reconfigurable (#10056)

Author: sravani-revuri

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingConfig.java

@@ -43,6 +43,7 @@ public class TracingConfig extends ReconfigurableConfig {
       key = "ozone.tracing.enabled",
       defaultValue = "false",
       type = ConfigType.BOOLEAN,
+      reconfigurable = true,
       tags = { ConfigTag.OZONE, ConfigTag.HDDS },
       description = "If true, tracing is initialized and spans may be exported (subject to sampling)."
   )
@@ -52,6 +53,7 @@ public class TracingConfig extends ReconfigurableConfig {
       key = "ozone.tracing.endpoint",
       defaultValue = "",
       type = ConfigType.STRING,
+      reconfigurable = true,
       tags = { ConfigTag.OZONE, ConfigTag.HDDS },
       description = "OTLP gRPC receiver endpoint URL."
   )
@@ -61,6 +63,7 @@ public class TracingConfig extends ReconfigurableConfig {
       key = "ozone.tracing.sampler",
       defaultValue = "-1",
       type = ConfigType.DOUBLE,
+      reconfigurable = true,
       tags = { ConfigTag.OZONE, ConfigTag.HDDS },
       description = "Root trace sampling ratio (0.0 to 1.0)."
   )
@@ -70,6 +73,7 @@ public class TracingConfig extends ReconfigurableConfig {
       key = "ozone.tracing.span.sampling",
       defaultValue = "",
       type = ConfigType.STRING,
+      reconfigurable = true,
       tags = { ConfigTag.OZONE, ConfigTag.HDDS },
       description = "Optional per-span sampling: comma-separated spanName:rate entries."
   )

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java

@@ -51,31 +51,59 @@ public final class TracingUtil {
 
   private static volatile boolean isInit = false;
   private static Tracer tracer = OpenTelemetry.noop().getTracer("noop");
+  private static SdkTracerProvider sdkTracerProvider;
 
   private TracingUtil() {
   }
 
   /**
    * Initialize the tracing with the given service name.
    */
-  public static void initTracing(
-      String serviceName, ConfigurationSource conf) {
-    if (!isTracingEnabled(conf) || isInit) {
+  public static synchronized void initTracing(
+      String serviceName, TracingConfig tracingConfig) {
+    if (!tracingConfig.isTracingEnabled() || isInit) {
       return;
     }
 
     try {
-      initialize(serviceName, conf);
+      initialize(serviceName, tracingConfig);
       isInit = true;
       LOG.info("Initialized tracing service: {}", serviceName);
     } catch (Exception e) {
       LOG.error("Failed to initialize tracing", e);
     }
   }
 
-  private static void initialize(String serviceName, ConfigurationSource conf) {
+  /**
+   * Receives serviceName and configurationSource.
+   * Delegates tracing initiation to {@link #initTracing(String, TracingConfig)}.
+   */
+  public static synchronized void initTracing(
+      String serviceName, ConfigurationSource conf) {
+    initTracing(serviceName, conf.getObject(TracingConfig.class));
+  }
+
+  /**
+   * Shuts down and re-initializes tracing.
+   * Called after tracing-related keys are reconfigured on OM/SCM/DN.
+   */
+  public static synchronized void reconfigureTracing(
+      String serviceName, TracingConfig tracingConfig) {
+    shutdownTracing();
+    initTracing(serviceName, tracingConfig);
+  }
+
+  private static void shutdownTracing() {
+    if (sdkTracerProvider != null) {
+      sdkTracerProvider.shutdown();
+      sdkTracerProvider = null;
+    }
+    tracer = OpenTelemetry.noop().getTracer("noop");
+    isInit = false;
+  }
+
+  private static void initialize(String serviceName, TracingConfig tracingConfig) {
     //Fetch and log the right tracing parameters based on config, environment variable and default value priority.
-    TracingConfig tracingConfig = conf.getObject(TracingConfig.class);
     String otelEndPoint = tracingConfig.getTracingEndpoint();
     double samplerRatio = tracingConfig.getTraceSamplerRatio();
     LOG.info("Sampling Trace Config = '{}'", samplerRatio);
@@ -106,10 +134,17 @@ private static void initialize(String serviceName, ConfigurationSource conf) {
         .setResource(resource)
         .setSampler(sampler)
         .build();
-    OpenTelemetry openTelemetry = OpenTelemetrySdk.builder()
-        .setTracerProvider(tracerProvider)
-        .build();
-    tracer = openTelemetry.getTracer(serviceName);
+
+    try {
+      OpenTelemetry openTelemetry = OpenTelemetrySdk.builder()
+          .setTracerProvider(tracerProvider)
+          .build();
+      tracer = openTelemetry.getTracer(serviceName);
+      sdkTracerProvider = tracerProvider;
+    } catch (RuntimeException e) {
+      tracerProvider.shutdown();
+      throw e;
+    }
   }
 
   /**

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java

@@ -65,6 +65,7 @@
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.conf.ReconfigurationHandler;
+import org.apache.hadoop.hdds.conf.TracingReconfigurationCallback;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.DatanodeID;
 import org.apache.hadoop.hdds.protocol.DiskBalancerProtocol;
@@ -78,7 +79,7 @@
 import org.apache.hadoop.hdds.server.OzoneAdmins;
 import org.apache.hadoop.hdds.server.http.HttpConfig;
 import org.apache.hadoop.hdds.server.http.RatisDropwizardExports;
-import org.apache.hadoop.hdds.tracing.TracingUtil;
+import org.apache.hadoop.hdds.tracing.TracingConfig;
 import org.apache.hadoop.hdds.utils.HddsServerUtil;
 import org.apache.hadoop.hdds.utils.HddsVersionInfo;
 import org.apache.hadoop.hdds.utils.IOUtils;
@@ -253,8 +254,10 @@ public String getNamespace() {
       datanodeDetails.setSetupTime(Time.now());
       datanodeDetails.setRevision(
           HddsVersionInfo.HDDS_VERSION_INFO.getRevision());
-      TracingUtil.initTracing(
-          "HddsDatanodeService." + datanodeDetails.getID(), conf);
+      String tracingServiceName = "HddsDatanodeService." + datanodeDetails.getID();
+      TracingConfig tracingConfig = conf.getObject(TracingConfig.class);
+      TracingReconfigurationCallback tracingReconfigurationCallback =
+          TracingReconfigurationCallback.init(tracingServiceName, tracingConfig);
       LOG.info("HddsDatanodeService {}", datanodeDetails);
       // Authenticate Hdds Datanode service if security is enabled
       if (OzoneSecurityUtil.isSecurityEnabled(conf)) {
@@ -304,6 +307,7 @@ public String getNamespace() {
 
       reconfigurationHandler =
           new ReconfigurationHandler("DN", conf, this::checkAdminPrivilege)
+              .register(tracingConfig)
               .register(HDDS_DATANODE_BLOCK_DELETE_THREAD_MAX,
                   this::reconfigBlockDeleteThreadMax)
               .register(OZONE_BLOCK_DELETING_SERVICE_WORKERS,
@@ -324,6 +328,7 @@ public String getNamespace() {
       }
 
       reconfigurationHandler.setReconfigurationCompleteCallback(reconfigurationHandler.defaultLoggingCallback());
+      reconfigurationHandler.registerCompleteCallback(tracingReconfigurationCallback);
 
       datanodeStateMachine = new DatanodeStateMachine(this, datanodeDetails, conf,
           dnCertClient, secretKeyClient, this::terminateDatanode,

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/TracingReconfigurationCallback.java

@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.conf;
+
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.tracing.TracingConfig;
+import org.apache.hadoop.hdds.tracing.TracingUtil;
+
+/**
+ * Reconfiguration hook for ozone.tracing settings.
+ * init runs TracingUtil.initTracing once and returns this callback for registerCompleteCallback.
+ * onPropertiesChanged calls TracingUtil.reconfigureTracing when any changed key starts with ozone.tracing.
+ */
+
+public final class TracingReconfigurationCallback implements ReconfigurationChangeCallback {
+
+  private static final String TRACING_KEY_PREFIX = "ozone.tracing.";
+
+  private final String serviceName;
+  private final TracingConfig tracingConfig;
+
+  private TracingReconfigurationCallback(String serviceName, TracingConfig tracingConfig) {
+    this.serviceName = serviceName;
+    this.tracingConfig = tracingConfig;
+  }
+
+  /**
+   * Initializes tracing and returns a callback for runtime reconfiguration.
+   */
+  public static TracingReconfigurationCallback init(
+      String serviceName, TracingConfig tracingConfig) {
+    TracingUtil.initTracing(serviceName, tracingConfig);
+    return new TracingReconfigurationCallback(serviceName, tracingConfig);
+  }
+
+  @Override
+  public void onPropertiesChanged(Map<String, Boolean> changedKeys, Configuration newConf) {
+    if (changedKeys.keySet().stream().anyMatch(k -> k.startsWith(TRACING_KEY_PREFIX))) {
+      TracingUtil.reconfigureTracing(serviceName, tracingConfig);
+    }
+  }
+}

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java

@@ -63,6 +63,7 @@
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.conf.ReconfigurationHandler;
+import org.apache.hadoop.hdds.conf.TracingReconfigurationCallback;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
@@ -168,6 +169,7 @@
 import org.apache.hadoop.hdds.server.events.EventQueue;
 import org.apache.hadoop.hdds.server.events.FixedThreadPoolWithAffinityExecutor;
 import org.apache.hadoop.hdds.server.http.RatisDropwizardExports;
+import org.apache.hadoop.hdds.tracing.TracingConfig;
 import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
 import org.apache.hadoop.hdds.utils.HAUtils;
 import org.apache.hadoop.hdds.utils.HddsServerUtil;
@@ -351,12 +353,15 @@ private StorageContainerManager(OzoneConfiguration conf,
     super(HddsVersionInfo.HDDS_VERSION_INFO);
 
     Objects.requireNonNull(configurator, "configurator cannot be null");
-    Objects.requireNonNull(conf, "configuration cannot be null");
+    configuration = Objects.requireNonNull(conf, "configuration cannot be null");
+    TracingConfig tracingConfig = configuration.getObject(TracingConfig.class);
+    TracingReconfigurationCallback tracingReconfigurationCallback =
+        TracingReconfigurationCallback.init("StorageContainerManager", tracingConfig);
     // It is assumed the scm --init command creates the SCM Storage Config.
     scmStorageConfig = new SCMStorageConfig(conf);
 
     scmHANodeDetails = SCMHANodeDetails.loadSCMHAConfig(conf, scmStorageConfig);
-    configuration = conf;
+
     initMetrics();
     initPerfMetrics();
 
@@ -418,13 +423,15 @@ private StorageContainerManager(OzoneConfiguration conf,
     serviceManager = new SCMServiceManager();
     reconfigurationHandler =
         new ReconfigurationHandler("SCM", conf, this::checkAdminAccess)
+            .register(tracingConfig)
             .register(OZONE_ADMINISTRATORS, this::reconfOzoneAdmins)
             .register(OZONE_READONLY_ADMINISTRATORS,
                 this::reconfOzoneReadOnlyAdmins)
             .register(HddsConfigKeys.HDDS_SCM_SAFEMODE_LOG_INTERVAL,
                 this::reconfigureSafeModeLogInterval);
 
     reconfigurationHandler.setReconfigurationCompleteCallback(reconfigurationHandler.defaultLoggingCallback());
+    reconfigurationHandler.registerCompleteCallback(tracingReconfigurationCallback);
 
     initializeSystemManagers(conf, configurator);
 

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java

@@ -24,7 +24,6 @@
 import org.apache.hadoop.hdds.cli.GenericCli;
 import org.apache.hadoop.hdds.cli.HddsVersionProvider;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.tracing.TracingUtil;
 import org.apache.hadoop.hdds.utils.HddsServerUtil;
 import org.apache.hadoop.hdds.utils.HddsVersionInfo;
 import org.apache.hadoop.ozone.common.StorageInfo;
@@ -147,7 +146,6 @@ private void startScm() throws Exception {
    */
   private void commonInit() {
     conf = getOzoneConf();
-    TracingUtil.initTracing("StorageContainerManager", conf);
 
     String[] originalArgs = getCmd().getParseResult().originalArgs()
         .toArray(new String[0]);

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestDatanodeReconfiguration.java

@@ -29,6 +29,7 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import org.apache.hadoop.conf.ReconfigurationException;
 import org.apache.hadoop.hdds.conf.ReconfigurationHandler;
+import org.apache.hadoop.hdds.tracing.TracingConfig;
 import org.apache.hadoop.ozone.HddsDatanodeService;
 import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
 import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.DeleteBlocksCommandHandler;
@@ -54,6 +55,7 @@ void reconfigurableProperties() {
         .add(OZONE_BLOCK_DELETING_SERVICE_TIMEOUT)
         .add(REPLICATION_STREAMS_LIMIT_KEY)
         .addAll(new DatanodeConfiguration().reconfigurableProperties())
+        .addAll(new TracingConfig().reconfigurableProperties())
         .build();
 
     assertProperties(getSubject(), expected);

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfiguration.java

@@ -41,6 +41,7 @@
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.hadoop.conf.ReconfigurationException;
 import org.apache.hadoop.hdds.conf.ReconfigurationHandler;
+import org.apache.hadoop.hdds.tracing.TracingConfig;
 import org.apache.hadoop.ozone.om.KeyManagerImpl;
 import org.apache.hadoop.ozone.om.OmConfig;
 import org.apache.hadoop.ozone.om.OzoneManager;
@@ -68,6 +69,7 @@ void reconfigurableProperties() {
         .add(OZONE_THREAD_NUMBER_DIR_DELETION)
         .add(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL)
         .addAll(new OmConfig().reconfigurableProperties())
+        .addAll(new TracingConfig().reconfigurableProperties())
         .add(OZONE_BLACKLIST_USERS)
         .add(OZONE_BLACKLIST_GROUPS)
         .add(OZONE_READ_BLACKLIST_USERS)

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestScmReconfiguration.java

@@ -32,6 +32,7 @@
 import org.apache.hadoop.hdds.scm.block.SCMBlockDeletingService;
 import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager.ReplicationManagerConfiguration;
 import org.apache.hadoop.hdds.scm.pipeline.WritableECContainerProvider.WritableECContainerProviderConfig;
+import org.apache.hadoop.hdds.tracing.TracingConfig;
 import org.junit.jupiter.api.Test;
 
 /**
@@ -56,6 +57,7 @@ void reconfigurableProperties() {
         .addAll(new WritableECContainerProviderConfig()
             .reconfigurableProperties())
         .addAll(new ScmConfig().reconfigurableProperties())
+        .addAll(new TracingConfig().reconfigurableProperties())
         .build();
 
     assertProperties(getSubject(), expected);

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java

@@ -178,6 +178,7 @@
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.conf.ReconfigurationHandler;
+import org.apache.hadoop.hdds.conf.TracingReconfigurationCallback;
 import org.apache.hadoop.hdds.protocol.SecretKeyProtocol;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto.ReconfigureProtocolProtos.ReconfigureProtocolService;
@@ -204,6 +205,7 @@
 import org.apache.hadoop.hdds.server.OzoneBlacklist;
 import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl;
 import org.apache.hadoop.hdds.server.http.RatisDropwizardExports;
+import org.apache.hadoop.hdds.tracing.TracingConfig;
 import org.apache.hadoop.hdds.utils.HAUtils;
 import org.apache.hadoop.hdds.utils.HddsServerUtil;
 import org.apache.hadoop.hdds.utils.IOUtils;
@@ -517,6 +519,9 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption)
     super(OzoneVersionInfo.OZONE_VERSION_INFO);
     Objects.requireNonNull(conf, "conf == null");
     setConfiguration(conf);
+    TracingConfig tracingConfig = conf.getObject(TracingConfig.class);
+    TracingReconfigurationCallback tracingReconfigurationCallback =
+        TracingReconfigurationCallback.init("OzoneManager", tracingConfig);
     // Load HA related configurations
     OMHANodeDetails omhaNodeDetails =
         OMHANodeDetails.loadOMHAConfig(configuration);
@@ -530,6 +535,7 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption)
     reconfigurationHandler =
         new ReconfigurationHandler("OM", conf, this::checkAdminUserPrivilege)
             .register(config)
+            .register(tracingConfig)
             .register(OZONE_ADMINISTRATORS, this::reconfOzoneAdmins)
             .register(OZONE_READONLY_ADMINISTRATORS,
                 this::reconfOzoneReadOnlyAdmins)
@@ -545,6 +551,7 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption)
             .register(OZONE_READ_BLACKLIST_GROUPS, this::reconfOzoneReadBlacklistGroups);
 
     reconfigurationHandler.setReconfigurationCompleteCallback(reconfigurationHandler.defaultLoggingCallback());
+    reconfigurationHandler.registerCompleteCallback(tracingReconfigurationCallback);
 
     versionManager = new OMLayoutVersionManager(omStorage.getLayoutVersion());
     upgradeFinalizer = new OMUpgradeFinalizer(versionManager);