dynamic log level (#613)

Author: SylvainJuge

agent/entrypoint/src/main/java/co/elastic/otel/agent/ElasticAgent.java

@@ -24,13 +24,18 @@
 /** Elastic agent entry point, delegates to OpenTelemetry agent */
 public class ElasticAgent {
 
+  private static final String OTEL_JAVAAGENT_LOGGING = "otel.javaagent.logging";
+  private static final String OTEL_JAVAAGENT_LOGGING_ENV = "OTEL_JAVAAGENT_LOGGING";
+  private static final String OTEL_JAVAAGENT_LOGGING_DEFAULT = "simple";
+
   /**
    * Entry point for -javaagent JVM argument attach
    *
    * @param agentArgs agent arguments
    * @param inst instrumentation
    */
   public static void premain(String agentArgs, Instrumentation inst) {
+    initLogging();
     OpenTelemetryAgent.premain(agentArgs, inst);
   }
 
@@ -41,6 +46,7 @@ public static void premain(String agentArgs, Instrumentation inst) {
    * @param inst instrumentation
    */
   public static void agentmain(String agentArgs, Instrumentation inst) {
+    initLogging();
     OpenTelemetryAgent.agentmain(agentArgs, inst);
   }
 
@@ -53,5 +59,22 @@ public static void main(String[] args) {
     OpenTelemetryAgent.main(args);
   }
 
+  private static void initLogging() {
+
+    // Do not override explicitly provided configuration unless it's using the default as the
+    // 'simple' provider is not included in this distribution and that triggers an SLF4j error.
+    if (isLoggingNotDefault(System.getProperty(OTEL_JAVAAGENT_LOGGING))
+        || isLoggingNotDefault(System.getenv(OTEL_JAVAAGENT_LOGGING_ENV))) {
+      return;
+    }
+
+    // must match value returned by ElasticLoggingCustomizer#getName
+    System.setProperty(OTEL_JAVAAGENT_LOGGING, "elastic");
+  }
+
+  private static boolean isLoggingNotDefault(String value) {
+    return value != null && !OTEL_JAVAAGENT_LOGGING_DEFAULT.equals(value);
+  }
+
   private ElasticAgent() {}
 }

buildSrc/src/main/kotlin/elastic-otel.agent-packaging-conventions.gradle.kts

@@ -182,6 +182,9 @@ tasks {
     dependsOn(isolateJavaagentLibs)
     configurations = listOf(bootstrapLibs, upstreamAgent)
 
+    // exclude slf4j-simple from the shadow jar as we use log4j2-slf4j with internal-logging instead
+    exclude("inst/io/opentelemetry/javaagent/slf4j/simple/**")
+
     from(isolateJavaagentLibs.get().outputs)
 
     archiveClassifier.set("")

buildscripts/allowed-licenses.json

@@ -8,6 +8,11 @@
       "moduleLicense": "The 2-Clause BSD License",
       "moduleVersion": ".*",
       "moduleName": "org.hdrhistogram:HdrHistogram"
+    },
+    {
+      "moduleLicense": "MIT License",
+      "moduleVersion": ".*",
+      "moduleName": "org.slf4j:slf4j-api"
     }
   ]
 }

custom/build.gradle.kts

@@ -11,6 +11,7 @@ dependencies {
   implementation(project(":inferred-spans"))
   implementation(project(":universal-profiling-integration"))
   implementation(project(":resources"))
+  implementation(project(":internal-logging", configuration = "shadow"))
   instrumentations.forEach {
     implementation(project(it))
   }

gradle/libs.versions.toml

@@ -4,6 +4,7 @@ jib = "3.4.5"
 spotless = "7.0.3"
 junit = "5.12.2"
 autoservice = "1.1.1"
+log4j2 = "2.24.3"
 
 # otel protocol (OTLP)
 opentelemetryProto = "1.3.2-alpha"
@@ -76,12 +77,18 @@ ant = "org.apache.ant:ant:1.10.15"
 # ASM is currently only used during compile-time, so it is okay to diverge from the version used in ByteBuddy
 asm = "org.ow2.asm:asm:9.8"
 
+slf4j-api = "org.slf4j:slf4j-api:2.0.17"
+log4j2-slf4j = { group= "org.apache.logging.log4j", name="log4j-slf4j2-impl", version.ref="log4j2"}
+log4j2-core = { group= "org.apache.logging.log4j", name="log4j-core", version.ref="log4j2"}
+
 # Instrumented libraries
 openaiClient = "com.openai:openai-java:1.4.1"
 
 [bundles]
 
 semconv = ["opentelemetrySemconv", "opentelemetrySemconvIncubating"]
+log4j2 = ["log4j2-core", "log4j2-slf4j"]
+
 
 [plugins]
 

internal-logging/build.gradle.kts

@@ -0,0 +1,38 @@
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
+import com.github.jengelman.gradle.plugins.shadow.transformers.Log4j2PluginsCacheFileTransformer
+
+plugins {
+  id("elastic-otel.library-packaging-conventions")
+  id("com.gradleup.shadow")
+}
+
+dependencies {
+
+  compileOnly("io.opentelemetry.javaagent:opentelemetry-javaagent-tooling")
+  compileOnly("io.opentelemetry.javaagent:opentelemetry-javaagent-bootstrap")
+  compileOnly(libs.slf4j.api)
+  implementation(libs.bundles.log4j2)
+
+}
+
+tasks {
+  val shadowJar by existing(ShadowJar::class) {
+    // required for META-INF/services files relocation
+    mergeServiceFiles()
+
+    // Excluding property source SPI prevents log4j system properties and env variables that might
+    // be set at the application level to change the behavior of this internal log4j instance.
+    exclude("**/*.log4j.util.PropertySource")
+
+    transform(Log4j2PluginsCacheFileTransformer::class.java)
+
+    // relocate slf4j and log4j for internal logging to prevent any conflict
+    relocate("org.slf4j", "co.elastic.otel.logging.slf4j")
+    relocate("org.apache.logging.log4j", "co.elastic.otel.logging.log4j")
+    relocate("org.apache.logging.slf4j", "co.elastic.otel.logging.log4j.slf4j")
+  }
+
+  assemble {
+    dependsOn(shadowJar)
+  }
+}

internal-logging/src/main/java/co/elastic/otel/logging/AgentLog.java

@@ -0,0 +1,69 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.otel.logging;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.core.config.Configurator;
+import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder;
+import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFactory;
+import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration;
+
+public class AgentLog {
+
+  private static final String PATTERN = "%d{DEFAULT} [%t] %-5level %logger{36} - %msg{nolookups}%n";
+
+  // logger is an empty string
+  private static final String ROOT_LOGGER_NAME = "";
+
+  private AgentLog() {}
+
+  public static void init() {
+
+    ConfigurationBuilder<BuiltConfiguration> conf =
+        ConfigurationBuilderFactory.newConfigurationBuilder();
+
+    conf.add(
+        conf.newAppender("stdout", "Console")
+            .add(conf.newLayout("PatternLayout").addAttribute("pattern", PATTERN)));
+
+    conf.add(conf.newRootLogger().add(conf.newAppenderRef("stdout")));
+
+    Configurator.initialize(conf.build(false));
+  }
+
+  /**
+   * Sets the agent log level at runtime
+   *
+   * @param level log level
+   */
+  public static void setLevel(Level level) {
+    // Using log4j2 implementation allows to change the log level programmatically at runtime
+    // which is not directly possible through the slf4j API and simple implementation used in
+    // upstream distribution.
+
+    Configurator.setAllLevels(ROOT_LOGGER_NAME, level);
+
+    // when debugging we should avoid very chatty http client debug messages
+    // this behavior is replicated from the upstream distribution.
+    if (level.intLevel() >= Level.DEBUG.intLevel()) {
+      Configurator.setLevel("okhttp3.internal.http2", Level.INFO);
+      Configurator.setLevel("okhttp3.internal.concurrent.TaskRunner", Level.INFO);
+    }
+  }
+}

internal-logging/src/main/java/co/elastic/otel/logging/ElasticLoggingCustomizer.java

@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.otel.logging;
+
+import com.google.auto.service.AutoService;
+import io.opentelemetry.javaagent.bootstrap.InternalLogger;
+import io.opentelemetry.javaagent.tooling.LoggingCustomizer;
+import io.opentelemetry.javaagent.tooling.config.EarlyInitAgentConfig;
+import org.apache.logging.log4j.Level;
+import org.slf4j.LoggerFactory;
+
+@AutoService(LoggingCustomizer.class)
+public class ElasticLoggingCustomizer implements LoggingCustomizer {
+
+  @Override
+  public String name() {
+    // must match "otel.javaagent.logging" system property for SPI lookup
+    return "elastic";
+  }
+
+  @Override
+  public void init(EarlyInitAgentConfig earlyConfig) {
+
+    // trigger loading the slf4j provider from the agent CL, this should load log4j implementation
+    LoggerFactory.getILoggerFactory();
+
+    // make the agent internal logger delegate to slf4j, which will delegate to log4j
+    InternalLogger.initialize(Slf4jInternalLogger::create);
+
+    AgentLog.init();
+
+    Level level = null;
+    if (earlyConfig.getBoolean("otel.javaagent.debug", false)) {
+      // set debug logging when enabled through configuration to behave like the upstream
+      // distribution
+      level = Level.DEBUG;
+    } else {
+      String levelConfig = earlyConfig.getString("elastic.otel.javaagent.log.level");
+      if (levelConfig != null) {
+        level = Level.getLevel(levelConfig);
+      }
+    }
+    AgentLog.setLevel(level != null ? level : Level.INFO);
+  }
+
+  @Override
+  public void onStartupSuccess() {}
+
+  @SuppressWarnings("CallToPrintStackTrace")
+  @Override
+  public void onStartupFailure(Throwable throwable) {
+    throwable.printStackTrace();
+  }
+}

internal-logging/src/main/java/co/elastic/otel/logging/Slf4jInternalLogger.java

@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.otel.logging;
+
+import io.opentelemetry.javaagent.bootstrap.InternalLogger;
+import javax.annotation.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Internal logger implementation that delegates to SLF4J */
+public class Slf4jInternalLogger implements InternalLogger {
+  private final Logger logger;
+
+  private Slf4jInternalLogger(String name) {
+    this.logger = LoggerFactory.getLogger(name);
+  }
+
+  public static InternalLogger create(String name) {
+    return new Slf4jInternalLogger(name);
+  }
+
+  @Override
+  public boolean isLoggable(Level level) {
+    switch (level) {
+      case TRACE:
+        return logger.isTraceEnabled();
+      case DEBUG:
+        return logger.isDebugEnabled();
+      case INFO:
+        return logger.isInfoEnabled();
+      case WARN:
+        return logger.isWarnEnabled();
+      case ERROR:
+        return logger.isErrorEnabled();
+      default:
+        throw new IllegalArgumentException("Unsupported level: " + level);
+    }
+  }
+
+  @Override
+  public void log(Level level, String s, @Nullable Throwable throwable) {
+    logger.atLevel(toSlf4jLevel(level)).setCause(throwable).log(s);
+  }
+
+  private static org.slf4j.event.Level toSlf4jLevel(Level level) {
+    switch (level) {
+      case TRACE:
+        return org.slf4j.event.Level.TRACE;
+      case DEBUG:
+        return org.slf4j.event.Level.DEBUG;
+      case INFO:
+        return org.slf4j.event.Level.INFO;
+      case WARN:
+        return org.slf4j.event.Level.WARN;
+      case ERROR:
+        return org.slf4j.event.Level.ERROR;
+      default:
+        throw new IllegalArgumentException("Unsupported level: " + level);
+    }
+  }
+
+  @Override
+  public String name() {
+    return logger.getName();
+  }
+}

settings.gradle.kts

@@ -19,6 +19,7 @@ include("custom")
 include("instrumentation")
 include("instrumentation:openai-client-instrumentation:instrumentation-1.1")
 include("inferred-spans")
+include("internal-logging")
 include("resources")
 include("runtime-attach")
 include("smoke-tests")