Capture gRPC UNKNOWN requests

instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/rpc/RpcAttributesGetter.java

@@ -68,6 +68,15 @@ default String getRpcMethod(REQUEST request) {
     return null;
   }
 
+  /**
+   * Returns the original method name when the method reported via {@link #getRpcMethod(REQUEST)} is
+   * set to {@code _OTHER} because the method is not recognized by the RPC framework.
+   */
+  @Nullable
+  default String getRpcMethodOriginal(REQUEST request) {
+    return null;
+  }
+
   /**
    * Returns a description of a class of error the operation ended with.
    *

instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/rpc/RpcCommonAttributesExtractor.java

@@ -19,6 +19,8 @@ abstract class RpcCommonAttributesExtractor<REQUEST, RESPONSE>
     implements AttributesExtractor<REQUEST, RESPONSE> {
 
   static final AttributeKey<String> RPC_METHOD = AttributeKey.stringKey("rpc.method");
+  static final AttributeKey<String> RPC_METHOD_ORIGINAL =
+      AttributeKey.stringKey("rpc.method_original");
 
   // Stable semconv keys
   static final AttributeKey<String> RPC_SYSTEM_NAME = AttributeKey.stringKey("rpc.system.name");
@@ -42,6 +44,7 @@ public final void onStart(AttributesBuilder attributes, Context parentContext, R
     if (emitStableRpcSemconv()) {
       attributes.put(RPC_SYSTEM_NAME, getter.getRpcSystemName(request));
       attributes.put(RPC_METHOD, getter.getRpcMethod(request));
+      attributes.put(RPC_METHOD_ORIGINAL, getter.getRpcMethodOriginal(request));
     }
 
     if (emitOldRpcSemconv()) {

instrumentation/armeria/armeria-grpc-1.14/javaagent/build.gradle.kts

@@ -18,6 +18,7 @@ dependencies {
   library("com.linecorp.armeria:armeria-grpc:1.14.0")
   implementation(project(":instrumentation:grpc-1.6:library"))
 
+  testInstrumentation(project(":instrumentation:armeria:armeria-1.3:javaagent"))
   testInstrumentation(project(":instrumentation:netty:netty-4.1:javaagent"))
   testInstrumentation(project(":instrumentation:grpc-1.6:javaagent"))
 
@@ -61,6 +62,10 @@ afterEvaluate {
 tasks {
   withType<Test>().configureEach {
     systemProperty("collectMetadata", otelProps.collectMetadata)
+    // The armeria HTTP instrumentation creates an HTTP server span, and then the gRPC
+    // interceptor creates a second server span from the same incoming context, which triggers the
+    // context leak debugger.
+    jvmArgs("-Dotel.javaagent.experimental.thread-propagation-debugger.enabled=false")
   }
 
   val testStableSemconv by registering(Test::class) {

instrumentation/armeria/armeria-grpc-1.14/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/armeria/grpc/v1_14/ArmeriaGrpcServiceBuilderInstrumentation.java

@@ -9,8 +9,6 @@
 import static net.bytebuddy.matcher.ElementMatchers.named;
 
 import com.linecorp.armeria.server.grpc.GrpcServiceBuilder;
-import io.opentelemetry.api.GlobalOpenTelemetry;
-import io.opentelemetry.instrumentation.grpc.v1_6.GrpcTelemetry;
 import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
 import io.opentelemetry.javaagent.extension.instrumentation.TypeTransformer;
 import net.bytebuddy.asm.Advice;
@@ -34,7 +32,7 @@ public static class BuildAdvice {
 
     @Advice.OnMethodEnter(suppress = Throwable.class, inline = false)
     public static void onEnter(@Advice.This GrpcServiceBuilder builder) {
-      builder.intercept(GrpcTelemetry.create(GlobalOpenTelemetry.get()).createServerInterceptor());
+      builder.intercept(ArmeriaGrpcSingletons.SERVER_INTERCEPTOR);
     }
   }
 }

instrumentation/armeria/armeria-grpc-1.14/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/armeria/grpc/v1_14/ArmeriaGrpcSingletons.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.armeria.grpc.v1_14;
+
+import io.grpc.ServerInterceptor;
+import io.opentelemetry.api.GlobalOpenTelemetry;
+import io.opentelemetry.instrumentation.grpc.v1_6.GrpcTelemetry;
+import io.opentelemetry.instrumentation.grpc.v1_6.internal.Internal;
+
+public final class ArmeriaGrpcSingletons {
+
+  public static final ServerInterceptor SERVER_INTERCEPTOR;
+
+  static {
+    GrpcTelemetry telemetry = GrpcTelemetry.create(GlobalOpenTelemetry.get());
+    SERVER_INTERCEPTOR = Internal.createServerInterceptor(telemetry);
+  }
+
+  private ArmeriaGrpcSingletons() {}
+}

instrumentation/armeria/armeria-grpc-1.14/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/armeria/grpc/v1_14/ArmeriaGrpcTest.java

@@ -8,8 +8,19 @@
 import static io.opentelemetry.instrumentation.api.internal.SemconvStability.emitOldRpcSemconv;
 import static io.opentelemetry.instrumentation.api.internal.SemconvStability.emitStableRpcSemconv;
 import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo;
+import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies;
+import static io.opentelemetry.semconv.ClientAttributes.CLIENT_ADDRESS;
+import static io.opentelemetry.semconv.HttpAttributes.HTTP_REQUEST_METHOD;
+import static io.opentelemetry.semconv.HttpAttributes.HTTP_RESPONSE_STATUS_CODE;
+import static io.opentelemetry.semconv.HttpAttributes.HTTP_ROUTE;
+import static io.opentelemetry.semconv.NetworkAttributes.NETWORK_PEER_ADDRESS;
+import static io.opentelemetry.semconv.NetworkAttributes.NETWORK_PEER_PORT;
+import static io.opentelemetry.semconv.NetworkAttributes.NETWORK_PROTOCOL_VERSION;
 import static io.opentelemetry.semconv.ServerAttributes.SERVER_ADDRESS;
 import static io.opentelemetry.semconv.ServerAttributes.SERVER_PORT;
+import static io.opentelemetry.semconv.UrlAttributes.URL_PATH;
+import static io.opentelemetry.semconv.UrlAttributes.URL_SCHEME;
+import static io.opentelemetry.semconv.UserAgentAttributes.USER_AGENT_ORIGINAL;
 import static io.opentelemetry.semconv.incubating.MessageIncubatingAttributes.MESSAGE_ID;
 import static io.opentelemetry.semconv.incubating.MessageIncubatingAttributes.MESSAGE_TYPE;
 import static io.opentelemetry.semconv.incubating.RpcIncubatingAttributes.RPC_GRPC_STATUS_CODE;
@@ -19,6 +30,7 @@
 import static io.opentelemetry.semconv.incubating.RpcIncubatingAttributes.RPC_SYSTEM;
 import static io.opentelemetry.semconv.incubating.RpcIncubatingAttributes.RPC_SYSTEM_NAME;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
 import com.linecorp.armeria.client.grpc.GrpcClients;
 import com.linecorp.armeria.server.ServerBuilder;
@@ -27,9 +39,12 @@
 import example.GreeterGrpc;
 import example.Helloworld;
 import io.grpc.Status;
+import io.grpc.StatusRuntimeException;
+import io.grpc.protobuf.services.HealthStatusManager;
 import io.grpc.stub.StreamObserver;
 import io.opentelemetry.api.trace.SpanKind;
 import io.opentelemetry.instrumentation.testing.junit.AgentInstrumentationExtension;
+import io.opentelemetry.sdk.trace.data.StatusData;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
@@ -62,6 +77,18 @@ public void sayHello(
         }
       };
 
+  @RegisterExtension
+  static final ServerExtension serverWithoutGreeter =
+      new ServerExtension() {
+        @Override
+        protected void configure(ServerBuilder sb) {
+          sb.service(
+              GrpcService.builder()
+                  .addService(new HealthStatusManager().getHealthService())
+                  .build());
+        }
+      };
+
   @SuppressWarnings("deprecation") // using deprecated semconv
   @Test
   void grpcInstrumentation() {
@@ -108,6 +135,23 @@ void grpcInstrumentation() {
                                     .hasAttributesSatisfyingExactly(
                                         equalTo(MESSAGE_TYPE, "RECEIVED"),
                                         equalTo(MESSAGE_ID, 1L))),
+                span ->
+                    span.hasName("POST /example.Greeter/SayHello")
+                        .hasKind(SpanKind.SERVER)
+                        .hasParent(trace.getSpan(1))
+                        .hasAttributesSatisfyingExactly(
+                            equalTo(HTTP_REQUEST_METHOD, "POST"),
+                            equalTo(HTTP_RESPONSE_STATUS_CODE, 200),
+                            equalTo(HTTP_ROUTE, "/example.Greeter/SayHello"),
+                            equalTo(URL_PATH, "/example.Greeter/SayHello"),
+                            equalTo(URL_SCHEME, "http"),
+                            equalTo(SERVER_ADDRESS, "127.0.0.1"),
+                            equalTo(SERVER_PORT, (long) server.httpPort()),
+                            equalTo(CLIENT_ADDRESS, "127.0.0.1"),
+                            equalTo(NETWORK_PEER_ADDRESS, "127.0.0.1"),
+                            satisfies(NETWORK_PEER_PORT, port -> port.isInstanceOf(Long.class)),
+                            equalTo(NETWORK_PROTOCOL_VERSION, "2"),
+                            satisfies(USER_AGENT_ORIGINAL, val -> val.startsWith("armeria/"))),
                 span ->
                     span.hasName("example.Greeter/SayHello")
                         .hasKind(SpanKind.SERVER)
@@ -139,4 +183,70 @@ void grpcInstrumentation() {
                                     .hasAttributesSatisfyingExactly(
                                         equalTo(MESSAGE_TYPE, "SENT"), equalTo(MESSAGE_ID, 1L)))));
   }
+
+  @SuppressWarnings("deprecation") // using deprecated semconv
+  @Test
+  void unknownService() {
+    GreeterGrpc.GreeterBlockingStub client =
+        GrpcClients.builder(serverWithoutGreeter.httpUri())
+            .build(GreeterGrpc.GreeterBlockingStub.class);
+
+    Helloworld.Request request = Helloworld.Request.newBuilder().setName("test").build();
+    assertThatThrownBy(() -> client.sayHello(request)).isInstanceOf(StatusRuntimeException.class);
+
+    // Armeria uses exact-route binding for gRPC services, so calling an unregistered service
+    // results in an HTTP 404 rather than the gRPC service layer producing an UNIMPLEMENTED
+    // response. The gRPC client maps the HTTP 404 to Status.UNIMPLEMENTED (asserted on the
+    // client span below), and the armeria HTTP instrumentation captures the request as an
+    // HTTP server span -- there is no gRPC server span because the gRPC service layer is
+    // never invoked.
+    testing.waitAndAssertTraces(
+        trace ->
+            trace.hasSpansSatisfyingExactly(
+                span ->
+                    span.hasName("example.Greeter/SayHello")
+                        .hasKind(SpanKind.CLIENT)
+                        .hasNoParent()
+                        .hasStatus(StatusData.error())
+                        .hasAttributesSatisfyingExactly(
+                            equalTo(RPC_SYSTEM, emitOldRpcSemconv() ? "grpc" : null),
+                            equalTo(RPC_SYSTEM_NAME, emitStableRpcSemconv() ? "grpc" : null),
+                            equalTo(RPC_SERVICE, emitOldRpcSemconv() ? "example.Greeter" : null),
+                            equalTo(
+                                RPC_METHOD,
+                                emitStableRpcSemconv() ? "example.Greeter/SayHello" : "SayHello"),
+                            equalTo(
+                                RPC_GRPC_STATUS_CODE,
+                                emitOldRpcSemconv()
+                                    ? (long) Status.Code.UNIMPLEMENTED.value()
+                                    : null),
+                            equalTo(
+                                RPC_RESPONSE_STATUS_CODE,
+                                emitStableRpcSemconv() ? Status.Code.UNIMPLEMENTED.name() : null),
+                            equalTo(SERVER_ADDRESS, "127.0.0.1"),
+                            equalTo(SERVER_PORT, (long) serverWithoutGreeter.httpPort()))
+                        .hasEventsSatisfyingExactly(
+                            event ->
+                                event
+                                    .hasName("message")
+                                    .hasAttributesSatisfyingExactly(
+                                        equalTo(MESSAGE_TYPE, "SENT"), equalTo(MESSAGE_ID, 1L))),
+                span ->
+                    span.hasName("POST /*")
+                        .hasKind(SpanKind.SERVER)
+                        .hasParent(trace.getSpan(0))
+                        .hasAttributesSatisfyingExactly(
+                            equalTo(HTTP_REQUEST_METHOD, "POST"),
+                            equalTo(HTTP_RESPONSE_STATUS_CODE, 404),
+                            equalTo(HTTP_ROUTE, "/*"),
+                            equalTo(URL_PATH, "/example.Greeter/SayHello"),
+                            equalTo(URL_SCHEME, "http"),
+                            equalTo(SERVER_ADDRESS, "127.0.0.1"),
+                            equalTo(SERVER_PORT, (long) serverWithoutGreeter.httpPort()),
+                            equalTo(CLIENT_ADDRESS, "127.0.0.1"),
+                            equalTo(NETWORK_PEER_ADDRESS, "127.0.0.1"),
+                            satisfies(NETWORK_PEER_PORT, port -> port.isInstanceOf(Long.class)),
+                            equalTo(NETWORK_PROTOCOL_VERSION, "2"),
+                            satisfies(USER_AGENT_ORIGINAL, val -> val.startsWith("armeria/")))));
+  }
 }

instrumentation/grpc-1.6/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/grpc/v1_6/GrpcServerBuilderInstrumentation.java

@@ -8,7 +8,6 @@
 import static io.opentelemetry.javaagent.extension.matcher.AgentElementMatchers.extendsClass;
 import static io.opentelemetry.javaagent.extension.matcher.AgentElementMatchers.hasClassesNamed;
 import static io.opentelemetry.javaagent.instrumentation.grpc.v1_6.GrpcSingletons.SERVER_BUILDER_INSTRUMENTED;
-import static io.opentelemetry.javaagent.instrumentation.grpc.v1_6.GrpcSingletons.serverInterceptor;
 import static net.bytebuddy.matcher.ElementMatchers.isPublic;
 import static net.bytebuddy.matcher.ElementMatchers.named;
 import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
@@ -50,7 +49,7 @@ public static CallDepth onEnter(@Advice.This ServerBuilder<?> serverBuilder) {
         return callDepth;
       }
       if (!Boolean.TRUE.equals(SERVER_BUILDER_INSTRUMENTED.get(serverBuilder))) {
-        serverBuilder.intercept(serverInterceptor());
+        GrpcSingletons.configureServerBuilder(serverBuilder);
         SERVER_BUILDER_INSTRUMENTED.set(serverBuilder, true);
       }
       return callDepth;

instrumentation/grpc-1.6/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/grpc/v1_6/GrpcSingletons.java

@@ -11,7 +11,6 @@
 import io.grpc.Context;
 import io.grpc.ManagedChannelBuilder;
 import io.grpc.ServerBuilder;
-import io.grpc.ServerInterceptor;
 import io.opentelemetry.api.GlobalOpenTelemetry;
 import io.opentelemetry.api.incubator.config.DeclarativeConfigProperties;
 import io.opentelemetry.instrumentation.api.incubator.config.internal.DeclarativeConfigUtil;
@@ -34,7 +33,7 @@ public class GrpcSingletons {
 
   private static final ClientInterceptor clientInterceptor;
 
-  private static final ServerInterceptor serverInterceptor;
+  private static final GrpcTelemetry grpcTelemetry;
 
   private static final AtomicReference<Context.Storage> storageReference = new AtomicReference<>();
 
@@ -57,26 +56,21 @@ public class GrpcSingletons {
             .get("server")
             .getScalarList("request", String.class, emptyList());
 
-    GrpcTelemetry telemetry =
+    grpcTelemetry =
         GrpcTelemetry.builder(GlobalOpenTelemetry.get())
             .setEmitMessageEvents(emitMessageEvents)
             .setCaptureExperimentalSpanAttributes(experimentalSpanAttributes)
             .setCapturedClientRequestMetadata(clientRequestMetadata)
             .setCapturedServerRequestMetadata(serverRequestMetadata)
             .build();
 
-    clientInterceptor = telemetry.createClientInterceptor();
-    serverInterceptor = telemetry.createServerInterceptor();
+    clientInterceptor = grpcTelemetry.createClientInterceptor();
   }
 
   public static ClientInterceptor clientInterceptor() {
     return clientInterceptor;
   }
 
-  public static ServerInterceptor serverInterceptor() {
-    return serverInterceptor;
-  }
-
   @Nullable
   public static Context.Storage storage() {
     return storageReference.get();
@@ -87,5 +81,9 @@ public static Context.Storage setStorage(Context.Storage storage) {
     return storage();
   }
 
+  public static void configureServerBuilder(ServerBuilder<?> serverBuilder) {
+    grpcTelemetry.configureServerBuilder(serverBuilder);
+  }
+
   private GrpcSingletons() {}
 }

instrumentation/grpc-1.6/library/README.md

@@ -37,9 +37,9 @@ void configureClientInterceptor(OpenTelemetry openTelemetry, NettyChannelBuilder
   nettyChannelBuilder.intercept(grpcTelemetry.createClientInterceptor());
 }
 
-// For server-side, attatch the interceptor to your service.
-ServerServiceDefinition configureServerInterceptor(OpenTelemetry openTelemetry, ServerServiceDefinition serviceDefinition) {
+// For server-side, configure the server builder.
+void configureServer(OpenTelemetry openTelemetry, ServerBuilder<?> serverBuilder) {
   GrpcTelemetry grpcTelemetry = GrpcTelemetry.create(openTelemetry);
-  return ServerInterceptors.intercept(serviceDefinition, grpcTelemetry.createServerInterceptor());
+  grpcTelemetry.configureServerBuilder(serverBuilder);
 }
 ```

instrumentation/grpc-1.6/library/src/main/java/io/opentelemetry/instrumentation/grpc/v1_6/GrpcRequest.java

@@ -12,7 +12,9 @@
 
 public final class GrpcRequest {
 
-  private final MethodDescriptor<?, ?> method;
+  @Nullable private final MethodDescriptor<?, ?> method;
+  private final String fullMethodName;
+  @Nullable private final String originalFullMethodName;
 
   @Nullable private volatile Metadata metadata;
 
@@ -29,11 +31,20 @@ public final class GrpcRequest {
       @Nullable SocketAddress peerSocketAddress,
       @Nullable String authority) {
     this.method = method;
+    this.fullMethodName = method.getFullMethodName();
+    this.originalFullMethodName = null;
     this.metadata = metadata;
     this.peerSocketAddress = peerSocketAddress;
     setLogicalAddress(authority);
   }
 
+  GrpcRequest(String fullMethodName, @Nullable String originalFullMethodName, Metadata metadata) {
+    this.method = null;
+    this.fullMethodName = fullMethodName;
+    this.originalFullMethodName = originalFullMethodName;
+    this.metadata = metadata;
+  }
+
   private void setLogicalAddress(@Nullable String authority) {
     if (authority == null) {
       return;
@@ -51,10 +62,20 @@ private void setLogicalAddress(@Nullable String authority) {
     }
   }
 
+  @Nullable
   public MethodDescriptor<?, ?> getMethod() {
     return method;
   }
 
+  String getFullMethodName() {
+    return fullMethodName;
+  }
+
+  @Nullable
+  String getOriginalFullMethodName() {
+    return originalFullMethodName;
+  }
+
   @Nullable
   public Metadata getMetadata() {
     return metadata;

instrumentation/grpc-1.6/library/src/main/java/io/opentelemetry/instrumentation/grpc/v1_6/GrpcRequestGetter.java

@@ -15,6 +15,8 @@
 
 final class GrpcRequestGetter implements TextMapGetter<GrpcRequest> {
 
+  static final GrpcRequestGetter INSTANCE = new GrpcRequestGetter();
+
   @Override
   public Iterable<String> keys(GrpcRequest request) {
     // Filter out HTTP/2 pseudo-headers (starting with ':') as they cannot be