Add OpenTelemetry tracing instrumentation to API layer (#13132)

Co-authored-by: tmckeon <tmckeon@apple.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: 3 people

pom.xml

@@ -189,6 +189,8 @@
         <cs.ini.version>0.5.4</cs.ini.version>
         <cs.caffeine.version>3.1.7</cs.caffeine.version>
         <cs.protobuf.version>3.25.5</cs.protobuf.version>
+        <cs.opentelemetry.version>1.51.0</cs.opentelemetry.version>
+        <cs.opentelemetry-instrumentation.version>2.16.0</cs.opentelemetry-instrumentation.version>
     </properties>
 
     <distributionManagement>
@@ -427,6 +429,16 @@
                 <artifactId>commons-validator</artifactId>
                 <version>${cs.commons-validator.version}</version>
             </dependency>
+            <dependency>
+                <groupId>io.opentelemetry</groupId>
+                <artifactId>opentelemetry-api</artifactId>
+                <version>${cs.opentelemetry.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>io.opentelemetry.instrumentation</groupId>
+                <artifactId>opentelemetry-instrumentation-annotations</artifactId>
+                <version>${cs.opentelemetry-instrumentation.version}</version>
+            </dependency>
             <dependency>
                 <groupId>javax.annotation</groupId>
                 <artifactId>javax.annotation-api</artifactId>

server/pom.xml

@@ -197,6 +197,14 @@
             <version>4.22.2.0-SNAPSHOT</version>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>io.opentelemetry.instrumentation</groupId>
+            <artifactId>opentelemetry-instrumentation-annotations</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-api</artifactId>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

server/src/main/java/com/cloud/api/ApiServer.java

@@ -58,6 +58,9 @@
 import javax.servlet.http.HttpServletResponse;
 import javax.servlet.http.HttpSession;
 
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.instrumentation.annotations.WithSpan;
+
 import com.cloud.cluster.ManagementServerHostVO;
 import com.cloud.cluster.dao.ManagementServerHostDao;
 import com.cloud.user.Account;
@@ -619,6 +622,7 @@ public void checkCharacterInkParams(final Map params) {
     }
 
     @Override
+    @WithSpan("ApiServer.handleRequest")
     @SuppressWarnings("rawtypes")
     public String handleRequest(final Map params, final String responseType, final StringBuilder auditTrailSb) throws ServerApiException {
         checkCharacterInkParams(params);
@@ -628,6 +632,10 @@ public String handleRequest(final Map params, final String responseType, final S
 
         try {
             command = (String[])params.get("command");
+            if (command != null && command.length > 0) {
+                Span.current().updateName("ApiServer.handleRequest " + command[0]);
+                Span.current().setAttribute("api.command", command[0]);
+            }
             if (command == null) {
                 logger.error("invalid request, no command sent");
                 if (logger.isTraceEnabled()) {

tools/docker/supervisord.conf

@@ -12,6 +12,7 @@ command=/bin/bash -c "mvn -pl client jetty:run -Dsimulator -Dorg.eclipse.jetty.a
 directory=/root
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
+redirect_stderr=true
 user=root
 
 [program:cloudstack-ui]