Capture gRPC UNKNOWN requests

trask · trask · commit 61aa2e263bd1 · 2026-02-24T20:12:02.000-08:00
diff --git a/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/rpc/RpcAttributesGetter.java b/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/rpc/RpcAttributesGetter.java
@@ -51,6 +51,17 @@ default String getRpcMethod(REQUEST request) {
     return null;
   }
 
+  /**
+   * Returns the original method name when the method reported via {@link #getRpcMethod(REQUEST)} is
+   * {@code _OTHER}. This is used to populate the {@code rpc.method_original} attribute.
+   *
+   * @return the original method name, or null if the method was not replaced with {@code _OTHER}
+   */
+  @Nullable
+  default String getRpcMethodOriginal(REQUEST request) {
+    return null;
+  }
+
   /**
    * Returns a description of a class of error the operation ended with.
    *
diff --git a/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/rpc/RpcCommonAttributesExtractor.java b/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/rpc/RpcCommonAttributesExtractor.java
@@ -16,6 +16,8 @@ abstract class RpcCommonAttributesExtractor<REQUEST, RESPONSE>
 
   // copied from RpcIncubatingAttributes
   static final AttributeKey<String> RPC_METHOD = AttributeKey.stringKey("rpc.method");
+  static final AttributeKey<String> RPC_METHOD_ORIGINAL =
+      AttributeKey.stringKey("rpc.method_original");
   static final AttributeKey<String> RPC_SERVICE = AttributeKey.stringKey("rpc.service");
   static final AttributeKey<String> RPC_SYSTEM = AttributeKey.stringKey("rpc.system");
 
@@ -31,6 +33,7 @@ public final void onStart(AttributesBuilder attributes, Context parentContext, R
     attributes.put(RPC_SYSTEM, getter.getSystem(request));
     attributes.put(RPC_SERVICE, getter.getService(request));
     attributes.put(RPC_METHOD, getter.getMethod(request));
+    attributes.put(RPC_METHOD_ORIGINAL, getter.getRpcMethodOriginal(request));
   }
 
   @Override
diff --git a/instrumentation/grpc-1.6/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/grpc/v1_6/GrpcServerBuilderInstrumentation.java b/instrumentation/grpc-1.6/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/grpc/v1_6/GrpcServerBuilderInstrumentation.java
@@ -51,6 +51,7 @@ public static CallDepth onEnter(@Advice.This ServerBuilder<?> serverBuilder) {
       }
       if (!Boolean.TRUE.equals(SERVER_BUILDER_INSTRUMENTED.get(serverBuilder))) {
         serverBuilder.intercept(GrpcSingletons.SERVER_INTERCEPTOR);
+        serverBuilder.addStreamTracerFactory(GrpcSingletons.SERVER_STREAM_TRACER_FACTORY);
         SERVER_BUILDER_INSTRUMENTED.set(serverBuilder, true);
       }
       return callDepth;
diff --git a/instrumentation/grpc-1.6/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/grpc/v1_6/GrpcSingletons.java b/instrumentation/grpc-1.6/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/grpc/v1_6/GrpcSingletons.java
@@ -12,6 +12,7 @@
 import io.grpc.ManagedChannelBuilder;
 import io.grpc.ServerBuilder;
 import io.grpc.ServerInterceptor;
+import io.grpc.ServerStreamTracer;
 import io.opentelemetry.api.GlobalOpenTelemetry;
 import io.opentelemetry.api.incubator.config.DeclarativeConfigProperties;
 import io.opentelemetry.instrumentation.api.incubator.config.internal.DeclarativeConfigUtil;
@@ -35,6 +36,8 @@ public final class GrpcSingletons {
 
   public static final ServerInterceptor SERVER_INTERCEPTOR;
 
+  public static final ServerStreamTracer.Factory SERVER_STREAM_TRACER_FACTORY;
+
   private static final AtomicReference<Context.Storage> STORAGE_REFERENCE = new AtomicReference<>();
 
   static {
@@ -66,6 +69,7 @@ public final class GrpcSingletons {
 
     CLIENT_INTERCEPTOR = telemetry.createClientInterceptor();
     SERVER_INTERCEPTOR = telemetry.createServerInterceptor();
+    SERVER_STREAM_TRACER_FACTORY = telemetry.createServerStreamTracerFactory();
   }
 
   public static Context.Storage getStorage() {
diff --git a/instrumentation/grpc-1.6/library/src/main/java/io/opentelemetry/instrumentation/grpc/v1_6/GrpcAttributesExtractor.java b/instrumentation/grpc-1.6/library/src/main/java/io/opentelemetry/instrumentation/grpc/v1_6/GrpcAttributesExtractor.java
@@ -33,7 +33,7 @@ final class GrpcAttributesExtractor implements AttributesExtractor<GrpcRequest,
 
   @Override
   public void onStart(AttributesBuilder attributes, Context parentContext, GrpcRequest request) {
-    // Request attributes captured on request end.
+    // Attributes captured on request end.
   }
 
   @Override
diff --git a/instrumentation/grpc-1.6/library/src/main/java/io/opentelemetry/instrumentation/grpc/v1_6/GrpcRequest.java b/instrumentation/grpc-1.6/library/src/main/java/io/opentelemetry/instrumentation/grpc/v1_6/GrpcRequest.java
@@ -12,7 +12,9 @@
 
 public final class GrpcRequest {
 
-  private final MethodDescriptor<?, ?> method;
+  @Nullable private volatile MethodDescriptor<?, ?> method;
+  private final String fullMethodName;
+  @Nullable private final String originalFullMethodName;
 
   @Nullable private volatile Metadata metadata;
 
@@ -29,11 +31,19 @@ public final class GrpcRequest {
       @Nullable SocketAddress peerSocketAddress,
       @Nullable String authority) {
     this.method = method;
+    this.fullMethodName = method.getFullMethodName();
+    this.originalFullMethodName = null;
     this.metadata = metadata;
     this.peerSocketAddress = peerSocketAddress;
     setLogicalAddress(authority);
   }
 
+  GrpcRequest(String fullMethodName, @Nullable String originalFullMethodName, Metadata metadata) {
+    this.fullMethodName = fullMethodName;
+    this.originalFullMethodName = originalFullMethodName;
+    this.metadata = metadata;
+  }
+
   private void setLogicalAddress(@Nullable String authority) {
     if (authority == null) {
       return;
@@ -51,10 +61,20 @@ private void setLogicalAddress(@Nullable String authority) {
     }
   }
 
+  @Nullable
   public MethodDescriptor<?, ?> getMethod() {
     return method;
   }
 
+  String getFullMethodName() {
+    return fullMethodName;
+  }
+
+  @Nullable
+  String getOriginalFullMethodName() {
+    return originalFullMethodName;
+  }
+
   @Nullable
   public Metadata getMetadata() {
     return metadata;
diff --git a/instrumentation/grpc-1.6/library/src/main/java/io/opentelemetry/instrumentation/grpc/v1_6/GrpcRpcAttributesGetter.java b/instrumentation/grpc-1.6/library/src/main/java/io/opentelemetry/instrumentation/grpc/v1_6/GrpcRpcAttributesGetter.java
@@ -26,7 +26,7 @@ public String getSystem(GrpcRequest request) {
   @Override
   @Nullable
   public String getService(GrpcRequest request) {
-    String fullMethodName = request.getMethod().getFullMethodName();
+    String fullMethodName = request.getFullMethodName();
     int slashIndex = fullMethodName.lastIndexOf('/');
     if (slashIndex == -1) {
       return null;
@@ -38,10 +38,10 @@ public String getService(GrpcRequest request) {
   @Override
   @Nullable
   public String getMethod(GrpcRequest request) {
-    String fullMethodName = request.getMethod().getFullMethodName();
+    String fullMethodName = request.getFullMethodName();
     int slashIndex = fullMethodName.lastIndexOf('/');
     if (slashIndex == -1) {
-      return null;
+      return fullMethodName;
     }
     return fullMethodName.substring(slashIndex + 1);
   }
@@ -58,6 +58,12 @@ public Long getResponseSize(GrpcRequest request) {
     return request.getResponseSize();
   }
 
+  @Override
+  @Nullable
+  public String getRpcMethodOriginal(GrpcRequest request) {
+    return request.getOriginalFullMethodName();
+  }
+
   List<String> metadataValue(GrpcRequest request, String key) {
     if (request.getMetadata() == null) {
       return emptyList();
diff --git a/instrumentation/grpc-1.6/library/src/main/java/io/opentelemetry/instrumentation/grpc/v1_6/GrpcSpanNameExtractor.java b/instrumentation/grpc-1.6/library/src/main/java/io/opentelemetry/instrumentation/grpc/v1_6/GrpcSpanNameExtractor.java
@@ -11,6 +11,6 @@
 final class GrpcSpanNameExtractor implements SpanNameExtractor<GrpcRequest> {
   @Override
   public String extract(GrpcRequest request) {
-    return request.getMethod().getFullMethodName();
+    return request.getFullMethodName();
   }
 }
diff --git a/instrumentation/grpc-1.6/library/src/main/java/io/opentelemetry/instrumentation/grpc/v1_6/GrpcTelemetry.java b/instrumentation/grpc-1.6/library/src/main/java/io/opentelemetry/instrumentation/grpc/v1_6/GrpcTelemetry.java
@@ -7,6 +7,7 @@
 
 import io.grpc.ClientInterceptor;
 import io.grpc.ServerInterceptor;
+import io.grpc.ServerStreamTracer;
 import io.grpc.Status;
 import io.opentelemetry.api.OpenTelemetry;
 import io.opentelemetry.context.propagation.ContextPropagators;
@@ -61,4 +62,14 @@ public ServerInterceptor createServerInterceptor() {
     return new TracingServerInterceptor(
         serverInstrumenter, captureExperimentalSpanAttributes, emitMessageEvents);
   }
+
+  /**
+   * Returns a new {@link ServerStreamTracer.Factory} for use with methods like {@link
+   * io.grpc.ServerBuilder#addStreamTracerFactory(ServerStreamTracer.Factory)}. This enables
+   * creating server spans for requests to unregistered services, which are not seen by server
+   * interceptors. Should be used together with {@link #createServerInterceptor()}.
+   */
+  public ServerStreamTracer.Factory createServerStreamTracerFactory() {
+    return new TracingServerStreamTracerFactory(serverInstrumenter, propagators);
+  }
 }
diff --git a/instrumentation/grpc-1.6/library/src/main/java/io/opentelemetry/instrumentation/grpc/v1_6/TracingServerInterceptor.java b/instrumentation/grpc-1.6/library/src/main/java/io/opentelemetry/instrumentation/grpc/v1_6/TracingServerInterceptor.java
@@ -70,6 +70,14 @@ public <REQUEST, RESPONSE> ServerCall.Listener<REQUEST> interceptCall(
       // field.
       authority = GrpcAuthorityStorage.getAuthority(call);
     }
+
+    // If a ServerStreamTracer is active, mark it as handled so it won't create a span for this
+    // request in streamClosed().
+    TracingServerStreamTracer streamTracer = TracingServerStreamTracer.STREAM_TRACER_KEY.get();
+    if (streamTracer != null) {
+      streamTracer.markInterceptorHandled();
+    }
+
     GrpcRequest request =
         new GrpcRequest(
             call.getMethodDescriptor(),
diff --git a/instrumentation/grpc-1.6/library/src/main/java/io/opentelemetry/instrumentation/grpc/v1_6/TracingServerStreamTracer.java b/instrumentation/grpc-1.6/library/src/main/java/io/opentelemetry/instrumentation/grpc/v1_6/TracingServerStreamTracer.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.grpc.v1_6;
+
+import io.grpc.Grpc;
+import io.grpc.Metadata;
+import io.grpc.ServerCall;
+import io.grpc.ServerStreamTracer;
+import io.grpc.Status;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.propagation.ContextPropagators;
+import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
+import io.opentelemetry.instrumentation.api.internal.InstrumenterUtil;
+import java.net.SocketAddress;
+import java.time.Instant;
+import javax.annotation.Nullable;
+
+/**
+ * A {@link ServerStreamTracer} that detects whether a gRPC server request was handled by the {@link
+ * TracingServerInterceptor}. If the interceptor does not fire (unregistered method), {@link
+ * #streamClosed(Status)} creates a span for the unhandled request.
+ */
+final class TracingServerStreamTracer extends ServerStreamTracer {
+
+  static final io.grpc.Context.Key<TracingServerStreamTracer> STREAM_TRACER_KEY =
+      io.grpc.Context.key("otel-grpc-stream-tracer");
+
+  private static final String UNKNOWN_METHOD_SPAN_NAME = "_OTHER";
+
+  private final Instrumenter<GrpcRequest, Status> instrumenter;
+  private final ContextPropagators propagators;
+  private final String fullMethodName;
+  private final Metadata headers;
+  private final Context parentContext;
+  private final Instant startTime;
+
+  private volatile boolean interceptorHandled;
+  @Nullable private volatile SocketAddress peerAddress;
+
+  TracingServerStreamTracer(
+      Instrumenter<GrpcRequest, Status> instrumenter,
+      ContextPropagators propagators,
+      String fullMethodName,
+      Metadata headers,
+      Context parentContext) {
+    this.instrumenter = instrumenter;
+    this.propagators = propagators;
+    this.fullMethodName = fullMethodName;
+    this.headers = headers;
+    this.parentContext = parentContext;
+    this.startTime = Instant.now();
+  }
+
+  void markInterceptorHandled() {
+    interceptorHandled = true;
+  }
+
+  @Override
+  public io.grpc.Context filterContext(io.grpc.Context context) {
+    return context.withValue(STREAM_TRACER_KEY, this);
+  }
+
+  @Override
+  public void serverCallStarted(ServerCall<?, ?> call) {
+    if (peerAddress == null) {
+      SocketAddress addr = call.getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR);
+      if (addr != null) {
+        peerAddress = addr;
+      }
+    }
+  }
+
+  @Override
+  public void streamClosed(Status status) {
+    if (interceptorHandled) {
+      return;
+    }
+    // Interceptor did not fire — this is an unregistered method
+    GrpcRequest request = new GrpcRequest(UNKNOWN_METHOD_SPAN_NAME, fullMethodName, headers);
+    if (peerAddress != null) {
+      request.setPeerSocketAddress(peerAddress);
+    }
+    // Extract trace context from incoming headers (e.g. W3C traceparent)
+    Context extracted =
+        propagators
+            .getTextMapPropagator()
+            .extract(parentContext, request, GrpcRequestGetter.INSTANCE);
+    if (instrumenter.shouldStart(extracted, request)) {
+      InstrumenterUtil.startAndEnd(
+          instrumenter, extracted, request, status, status.getCause(), startTime, Instant.now());
+    }
+  }
+}
diff --git a/instrumentation/grpc-1.6/library/src/main/java/io/opentelemetry/instrumentation/grpc/v1_6/TracingServerStreamTracerFactory.java b/instrumentation/grpc-1.6/library/src/main/java/io/opentelemetry/instrumentation/grpc/v1_6/TracingServerStreamTracerFactory.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.grpc.v1_6;
+
+import io.grpc.Metadata;
+import io.grpc.ServerStreamTracer;
+import io.grpc.Status;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.propagation.ContextPropagators;
+import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
+
+final class TracingServerStreamTracerFactory extends ServerStreamTracer.Factory {
+
+  private final Instrumenter<GrpcRequest, Status> instrumenter;
+  private final ContextPropagators propagators;
+
+  TracingServerStreamTracerFactory(
+      Instrumenter<GrpcRequest, Status> instrumenter, ContextPropagators propagators) {
+    this.instrumenter = instrumenter;
+    this.propagators = propagators;
+  }
+
+  @Override
+  public ServerStreamTracer newServerStreamTracer(String fullMethodName, Metadata headers) {
+    return new TracingServerStreamTracer(
+        instrumenter, propagators, fullMethodName, headers, Context.current());
+  }
+}
diff --git a/instrumentation/grpc-1.6/library/src/test/java/io/opentelemetry/instrumentation/grpc/v1_6/GrpcTest.java b/instrumentation/grpc-1.6/library/src/test/java/io/opentelemetry/instrumentation/grpc/v1_6/GrpcTest.java
@@ -44,11 +44,13 @@ class GrpcTest extends AbstractGrpcTest {
 
   @Override
   protected ServerBuilder<?> configureServer(ServerBuilder<?> server) {
-    return server.intercept(
+    GrpcTelemetry telemetry =
         GrpcTelemetry.builder(testing.getOpenTelemetry())
             .setCapturedServerRequestMetadata(singletonList(SERVER_REQUEST_METADATA_KEY))
-            .build()
-            .createServerInterceptor());
+            .build();
+    return server
+        .intercept(telemetry.createServerInterceptor())
+        .addStreamTracerFactory(telemetry.createServerStreamTracerFactory());
   }
 
   @Override
diff --git a/instrumentation/grpc-1.6/testing/src/main/java/io/opentelemetry/instrumentation/grpc/v1_6/AbstractGrpcTest.java b/instrumentation/grpc-1.6/testing/src/main/java/io/opentelemetry/instrumentation/grpc/v1_6/AbstractGrpcTest.java

Original file line number	Diff line number	Diff line change
`@@ -51,6 +51,7 @@ public static CallDepth onEnter(@Advice.This ServerBuilder<?> serverBuilder) {`
`51`	`51`	`}`
`52`	`52`	`if (!Boolean.TRUE.equals(SERVER_BUILDER_INSTRUMENTED.get(serverBuilder))) {`
`53`	`53`	`serverBuilder.intercept(GrpcSingletons.SERVER_INTERCEPTOR);`
	`54`	`+ serverBuilder.addStreamTracerFactory(GrpcSingletons.SERVER_STREAM_TRACER_FACTORY);`
`54`	`55`	`SERVER_BUILDER_INSTRUMENTED.set(serverBuilder, true);`
`55`	`56`	`}`
`56`	`57`	`return callDepth;`
Original file line number	Diff line number	Diff line change
`@@ -33,7 +33,7 @@ final class GrpcAttributesExtractor implements AttributesExtractor<GrpcRequest,`
`33`	`33`
`34`	`34`	`@Override`
`35`	`35`	`public void onStart(AttributesBuilder attributes, Context parentContext, GrpcRequest request) {`
`36`		`- // Request attributes captured on request end.`
	`36`	`+ // Attributes captured on request end.`
`37`	`37`	`}`
`38`	`38`
`39`	`39`	`@Override`
Original file line number	Diff line number	Diff line change
`@@ -11,6 +11,6 @@`
`11`	`11`	`final class GrpcSpanNameExtractor implements SpanNameExtractor<GrpcRequest> {`
`12`	`12`	`@Override`
`13`	`13`	`public String extract(GrpcRequest request) {`
`14`		`- return request.getMethod().getFullMethodName();`
	`14`	`+ return request.getFullMethodName();`
`15`	`15`	`}`
`16`	`16`	`}`