Add native tracing fields to LogEvent to eliminate async MDC overhead (#4171)

* Add tracing fields to RingBufferLogEvent and related classes

* Add W3C trace context support with pattern converters for trace ID, span ID, and trace flags

* Enhance TraceContextProviderService for exception safety and simplify trace ID retrieval

* Add tests for tracing fields serialization and pattern converters

* Refactor tracing metadata documentation and improve code comments for clarity

* Add native W3C tracing fields to LogEvent and introduce TraceContextProvider SPI

* Add benchmark for ContextDataProvider tracing approach

* Enhance TraceContextProviderService to handle SecurityManager restrictions gracefully

Author: ramanathan1504

log4j-api/src/main/java/org/apache/logging/log4j/spi/TraceContextProvider.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.logging.log4j.spi;
+
+/**
+ * Service Provider Interface (SPI) for retrieving distributed tracing metadata (such as W3C Trace Context)
+ * from the active execution context.
+ * <p>
+ * Implementing this SPI allows tracing frameworks (e.g., OpenTelemetry, Micrometer, Zipkin) to pass native
+ * trace identifiers directly to Log4j events. This completely bypasses the {@link org.apache.logging.log4j.ThreadContext}
+ * map, eliminating map-cloning and garbage collection overhead during asynchronous logging.
+ * </p>
+ * <p>
+ * Log4j locates implementations of this interface using the standard Java {@link java.util.ServiceLoader} mechanism.
+ * To register a custom provider, create a plain-text file named
+ * {@code org.apache.logging.log4j.spi.TraceContextProvider} in the {@code META-INF/services/} directory
+ * containing the fully qualified class name of the implementation.
+ * </p>
+ *
+ * @since 2.27.0
+ */
+public interface TraceContextProvider {
+
+    /**
+     * Returns the standard trace ID from the active context, or {@code null}.
+     */
+    String getTraceId();
+
+    /**
+     * Returns the standard span ID from the active context, or {@code null}.
+     */
+    String getSpanId();
+
+    /**
+     * Returns the standard trace flags from the active context, or {@code null}.
+     */
+    String getTraceFlags();
+}

log4j-api/src/main/java/org/apache/logging/log4j/spi/package-info.java

@@ -19,7 +19,7 @@
  * API classes.
  */
 @Export
-@Version("2.25.0")
+@Version("2.26.0")
 package org.apache.logging.log4j.spi;
 
 import org.osgi.annotation.bundle.Export;

log4j-core-test/src/test/java/org/apache/logging/log4j/core/async/RingBufferLogEventTest.java

@@ -16,10 +16,12 @@
  */
 package org.apache.logging.log4j.core.async;
 
+import static org.apache.logging.log4j.core.util.ClockFactory.getClock;
 import static org.assertj.core.api.Assertions.as;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatCode;
 import static org.assertj.core.api.Assertions.fail;
+import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
@@ -31,6 +33,7 @@
 import org.apache.logging.log4j.ThreadContext;
 import org.apache.logging.log4j.ThreadContext.ContextStack;
 import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.impl.ContextDataFactory;
 import org.apache.logging.log4j.core.impl.ThrowableProxy;
 import org.apache.logging.log4j.core.time.internal.FixedPreciseClock;
 import org.apache.logging.log4j.core.util.Clock;
@@ -468,4 +471,46 @@ void testGettersAndClear() {
         // Verify interaction exhaustion
         verifyNoMoreInteractions(asyncLogger, message, throwable, contextData, contextStack);
     }
+
+    @Test
+    void testRingBufferLogEventTracingFieldsAndClear() {
+        final RingBufferLogEvent event = new RingBufferLogEvent();
+
+        // Check initial state
+        assertThat(event.getTraceId()).isNull();
+        assertThat(event.getSpanId()).isNull();
+        assertThat(event.getTraceFlags()).isNull();
+
+        // Initialize with trace fields inside 18-parameter setValues
+        event.setValues(
+                null,
+                "TestLogger",
+                null,
+                "FQCN",
+                Level.INFO,
+                new SimpleMessage("msg"),
+                null,
+                ContextDataFactory.createContextData(),
+                ThreadContext.EMPTY_STACK,
+                123L,
+                "thread-name",
+                5,
+                null,
+                getClock(),
+                new DummyNanoClock(),
+                "trace-id-ringbuffer",
+                "span-id-ringbuffer",
+                "01");
+
+        // Assert values are retrievable
+        assertThat(event.getTraceId()).isEqualTo("trace-id-ringbuffer");
+        assertThat(event.getSpanId()).isEqualTo("span-id-ringbuffer");
+        assertThat(event.getTraceFlags()).isEqualTo("01");
+
+        // Verify clearing wipes tracing state to avoid leakage on slot reuse
+        event.clear();
+        assertNull(event.getTraceId());
+        assertNull(event.getSpanId());
+        assertNull(event.getTraceFlags());
+    }
 }

log4j-core-test/src/test/java/org/apache/logging/log4j/core/impl/Log4jLogEventTest.java

@@ -18,6 +18,7 @@
 
 import static org.apache.logging.log4j.test.junit.SerialUtil.deserialize;
 import static org.apache.logging.log4j.test.junit.SerialUtil.serialize;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertInstanceOf;
@@ -29,13 +30,16 @@
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.lang.reflect.Field;
+import java.util.Map;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.Marker;
 import org.apache.logging.log4j.MarkerManager;
 import org.apache.logging.log4j.ThreadContext;
 import org.apache.logging.log4j.ThreadContext.ContextStack;
 import org.apache.logging.log4j.core.LogEvent;
 import org.apache.logging.log4j.core.config.plugins.convert.Base64Converter;
+import org.apache.logging.log4j.core.time.Instant;
+import org.apache.logging.log4j.core.time.MutableInstant;
 import org.apache.logging.log4j.core.util.Clock;
 import org.apache.logging.log4j.core.util.ClockFactory;
 import org.apache.logging.log4j.core.util.ClockFactoryTest;
@@ -45,6 +49,7 @@
 import org.apache.logging.log4j.message.ReusableMessage;
 import org.apache.logging.log4j.message.ReusableObjectMessage;
 import org.apache.logging.log4j.message.SimpleMessage;
+import org.apache.logging.log4j.util.ReadOnlyStringMap;
 import org.apache.logging.log4j.util.SortedArrayStringMap;
 import org.apache.logging.log4j.util.StringMap;
 import org.apache.logging.log4j.util.Strings;
@@ -541,4 +546,164 @@ void testToString() {
         // Throws an NPE in 2.6.2
         assertNotNull(new Log4jLogEvent().toString());
     }
+
+    @Test
+    public void testCustomLegacyLogEventDefaultBehavior() {
+        // Create an anonymous/stub class implementing LogEvent without implementing getTraceId/getSpanId/getTraceFlags
+        final LogEvent legacyEvent = new LogEvent() {
+            private static final long serialVersionUID = 1L;
+
+            @Override
+            public LogEvent toImmutable() {
+                return this;
+            }
+
+            @Override
+            @Deprecated
+            public Map<String, String> getContextMap() {
+                return java.util.Collections.emptyMap();
+            }
+
+            @Override
+            public ReadOnlyStringMap getContextData() {
+                return ContextDataFactory.emptyFrozenContextData();
+            }
+
+            @Override
+            public ThreadContext.ContextStack getContextStack() {
+                return ThreadContext.EMPTY_STACK;
+            }
+
+            @Override
+            public String getLoggerFqcn() {
+                return null;
+            }
+
+            @Override
+            public Level getLevel() {
+                return Level.INFO;
+            }
+
+            @Override
+            public String getLoggerName() {
+                return "LegacyLogger";
+            }
+
+            @Override
+            public Marker getMarker() {
+                return null;
+            }
+
+            @Override
+            public Message getMessage() {
+                return new SimpleMessage("Legacy msg");
+            }
+
+            @Override
+            public long getTimeMillis() {
+                return 0;
+            }
+
+            @Override
+            public Instant getInstant() {
+                return new MutableInstant();
+            }
+
+            @Override
+            public StackTraceElement getSource() {
+                return null;
+            }
+
+            @Override
+            public String getThreadName() {
+                return "main";
+            }
+
+            @Override
+            public long getThreadId() {
+                return 1;
+            }
+
+            @Override
+            public int getThreadPriority() {
+                return 5;
+            }
+
+            @Override
+            public Throwable getThrown() {
+                return null;
+            }
+
+            @Override
+            @Deprecated
+            public ThrowableProxy getThrownProxy() {
+                return null;
+            }
+
+            @Override
+            public boolean isEndOfBatch() {
+                return false;
+            }
+
+            @Override
+            public boolean isIncludeLocation() {
+                return false;
+            }
+
+            @Override
+            public void setEndOfBatch(boolean endOfBatch) {}
+
+            @Override
+            public void setIncludeLocation(boolean locationRequired) {}
+
+            @Override
+            public long getNanoTime() {
+                return 0;
+            }
+        };
+
+        assertNull(legacyEvent.getTraceId());
+        assertNull(legacyEvent.getSpanId());
+        assertNull(legacyEvent.getTraceFlags());
+    }
+
+    @Test
+    void testTracingFieldsInBuilderAndCopy() {
+        final Log4jLogEvent originalEvent = Log4jLogEvent.newBuilder()
+                .setLoggerName("TestLogger")
+                .setLevel(Level.DEBUG)
+                .setMessage(new org.apache.logging.log4j.message.SimpleMessage("Test message"))
+                .setTraceId("4bf92f3577b34da6a3ce929d0e0e4736")
+                .setSpanId("00f067aa0ba902b7")
+                .setTraceFlags("01")
+                .build();
+
+        assertThat(originalEvent.getTraceId()).isEqualTo("4bf92f3577b34da6a3ce929d0e0e4736");
+        assertThat(originalEvent.getSpanId()).isEqualTo("00f067aa0ba902b7");
+        assertThat(originalEvent.getTraceFlags()).isEqualTo("01");
+
+        final Log4jLogEvent copiedEvent = new Log4jLogEvent.Builder(originalEvent).build();
+        assertThat(copiedEvent.getTraceId()).isEqualTo("4bf92f3577b34da6a3ce929d0e0e4736");
+        assertThat(copiedEvent.getSpanId()).isEqualTo("00f067aa0ba902b7");
+        assertThat(copiedEvent.getTraceFlags()).isEqualTo("01");
+    }
+
+    @Test
+    void testTracingFieldsSerialization() throws Exception {
+        final Log4jLogEvent originalEvent = Log4jLogEvent.newBuilder()
+                .setLoggerName("SerializationLogger")
+                .setMessage(new org.apache.logging.log4j.message.SimpleMessage("dummy message"))
+                .setTraceId("trace-serialize-123")
+                .setSpanId("span-serialize-456")
+                .setTraceFlags("01")
+                .build();
+
+        final byte[] serializedBytes = serialize(originalEvent);
+
+        final Log4jLogEvent deserializedEvent = deserialize(serializedBytes);
+
+        assertThat(deserializedEvent.getTraceId()).isEqualTo("trace-serialize-123");
+        assertThat(deserializedEvent.getSpanId()).isEqualTo("span-serialize-456");
+        assertThat(deserializedEvent.getTraceFlags()).isEqualTo("01");
+    }
 }

log4j-core-test/src/test/java/org/apache/logging/log4j/core/impl/MutableLogEventTest.java

@@ -366,4 +366,33 @@ void testPreservesLocation() {
         final Log4jLogEvent immutable = mutable.toImmutable();
         assertThat(immutable.getSource()).isEqualTo(source);
     }
+
+    @Test
+    void testMutableLogEventTracingFieldsPropagationAndClear() {
+        final Log4jLogEvent sourceEvent = Log4jLogEvent.newBuilder()
+                .setLoggerName("SourceLogger")
+                .setTraceId("trace-xyz-123")
+                .setSpanId("span-abc-456")
+                .setTraceFlags("01")
+                .build();
+
+        final MutableLogEvent mutableEvent = new MutableLogEvent();
+
+        // Initially empty
+        assertThat(mutableEvent.getTraceId()).isNull();
+        assertThat(mutableEvent.getSpanId()).isNull();
+        assertThat(mutableEvent.getTraceFlags()).isNull();
+
+        // Verify propagation via initFrom
+        mutableEvent.initFrom(sourceEvent);
+        assertThat(mutableEvent.getTraceId()).isEqualTo("trace-xyz-123");
+        assertThat(mutableEvent.getSpanId()).isEqualTo("span-abc-456");
+        assertThat(mutableEvent.getTraceFlags()).isEqualTo("01");
+
+        // Verify clearing removes tracking state to prevent leakage on pool reuse
+        mutableEvent.clear();
+        assertThat(mutableEvent.getTraceId()).isNull();
+        assertThat(mutableEvent.getSpanId()).isNull();
+        assertThat(mutableEvent.getTraceFlags()).isNull();
+    }
 }