Add GitCommitSha to the client stats (#10833)

Add GitCommitSha to the client stats

changes

fix merge issues

Co-authored-by: andrea.marziali <andrea.marziali@datadoghq.com>

Author: amarziali

dd-trace-core/src/main/java/datadog/trace/common/metrics/SerializingMetricWriter.java

@@ -1,14 +1,20 @@
 package datadog.trace.common.metrics;
 
+import static datadog.trace.bootstrap.instrumentation.api.UTF8BytesString.EMPTY;
 import static java.nio.charset.StandardCharsets.ISO_8859_1;
 
 import datadog.communication.serialization.GrowableBuffer;
 import datadog.communication.serialization.WritableFormatter;
 import datadog.communication.serialization.msgpack.MsgPackWriter;
 import datadog.trace.api.ProcessTags;
 import datadog.trace.api.WellKnownTags;
+import datadog.trace.api.cache.DDCache;
+import datadog.trace.api.cache.DDCaches;
+import datadog.trace.api.git.GitInfo;
+import datadog.trace.api.git.GitInfoProvider;
 import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString;
 import java.util.List;
+import java.util.function.Function;
 
 public final class SerializingMetricWriter implements MetricWriter {
 
@@ -39,33 +45,57 @@ public final class SerializingMetricWriter implements MetricWriter {
   private static final byte[] HTTP_ENDPOINT = "HTTPEndpoint".getBytes(ISO_8859_1);
   private static final byte[] GRPC_STATUS_CODE = "GRPCStatusCode".getBytes(ISO_8859_1);
   private static final byte[] SERVICE_SOURCE = "srv_src".getBytes(ISO_8859_1);
+  private static final byte[] GIT_COMMIT_SHA = "GitCommitSha".getBytes(ISO_8859_1);
 
   // Constant declared here for compile-time folding
   public static final int TRISTATE_TRUE = TriState.TRUE.serialValue;
   public static final int TRISTATE_FALSE = TriState.FALSE.serialValue;
 
+  private static final Function<GitInfo, UTF8BytesString> SHA_COMMIT_GETTER =
+      gitInfo ->
+          gitInfo.getCommit() != null && gitInfo.getCommit().getSha() != null
+              ? UTF8BytesString.create(gitInfo.getCommit().getSha())
+              : EMPTY;
+
   private final WellKnownTags wellKnownTags;
   private final WritableFormatter writer;
   private final Sink sink;
   private final GrowableBuffer buffer;
+  private final DDCache<GitInfo, UTF8BytesString> gitInfoCache =
+      DDCaches.newFixedSizeWeakKeyCache(4);
   private long sequence = 0;
+  private final GitInfoProvider gitInfoProvider;
 
   public SerializingMetricWriter(WellKnownTags wellKnownTags, Sink sink) {
     this(wellKnownTags, sink, 512 * 1024);
   }
 
   public SerializingMetricWriter(WellKnownTags wellKnownTags, Sink sink, int initialCapacity) {
+    this(wellKnownTags, sink, initialCapacity, GitInfoProvider.INSTANCE);
+  }
+
+  public SerializingMetricWriter(
+      WellKnownTags wellKnownTags,
+      Sink sink,
+      int initialCapacity,
+      final GitInfoProvider gitInfoProvider) {
     this.wellKnownTags = wellKnownTags;
     this.buffer = new GrowableBuffer(initialCapacity);
     this.writer = new MsgPackWriter(buffer);
     this.sink = sink;
+    this.gitInfoProvider = new GitInfoProvider();
   }
 
   @Override
   public void startBucket(int metricCount, long start, long duration) {
     final UTF8BytesString processTags = ProcessTags.getTagsForSerialization();
     final boolean writeProcessTags = processTags != null;
-    writer.startMap(7 + (writeProcessTags ? 1 : 0));
+    // the gitinfo can be rebuild in time and initialized after this class is created. So that a
+    // cacheable is needed
+    final UTF8BytesString gitSha =
+        gitInfoCache.computeIfAbsent(gitInfoProvider.getGitInfo(), SHA_COMMIT_GETTER);
+    final boolean writeGitCommitSha = gitSha != EMPTY;
+    writer.startMap(7 + (writeProcessTags ? 1 : 0) + (writeGitCommitSha ? 1 : 0));
 
     writer.writeUTF8(RUNTIME_ID);
     writer.writeUTF8(wellKnownTags.getRuntimeId());
@@ -90,6 +120,11 @@ public void startBucket(int metricCount, long start, long duration) {
       writer.writeUTF8(processTags);
     }
 
+    if (writeGitCommitSha) {
+      writer.writeUTF8(GIT_COMMIT_SHA);
+      writer.writeUTF8(gitSha);
+    }
+
     writer.writeUTF8(STATS);
 
     writer.startArray(1);

dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SerializingMetricWriterTest.groovy

@@ -10,6 +10,9 @@ import datadog.trace.api.Config
 import datadog.trace.api.Pair
 import datadog.trace.api.ProcessTags
 import datadog.trace.api.WellKnownTags
+import datadog.trace.api.git.CommitInfo
+import datadog.trace.api.git.GitInfo
+import datadog.trace.api.git.GitInfoProvider
 import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString
 import datadog.trace.test.util.DDSpecification
 import java.nio.ByteBuffer
@@ -25,8 +28,8 @@ class SerializingMetricWriterTest extends DDSpecification {
 
   def "should produce correct message #iterationIndex with process tags enabled #withProcessTags" () {
     setup:
-    if (withProcessTags) {
-      injectSysConfig(EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED, "true")
+    if (!withProcessTags) {
+      injectSysConfig(EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED, "false")
     }
     ProcessTags.reset()
     long startTime = MILLISECONDS.toNanos(System.currentTimeMillis())
@@ -45,6 +48,9 @@ class SerializingMetricWriterTest extends DDSpecification {
     then:
     sink.validatedInput()
 
+    cleanup:
+    removeSysConfig(EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED)
+    ProcessTags.reset()
 
     where:
     content << [
@@ -136,8 +142,105 @@ class SerializingMetricWriterTest extends DDSpecification {
     withProcessTags << [true, false]
   }
 
+  def "ServiceSource optional in the payload"() {
+    setup:
+    long startTime = MILLISECONDS.toNanos(System.currentTimeMillis())
+    long duration = SECONDS.toNanos(10)
+    WellKnownTags wellKnownTags = new WellKnownTags("runtimeid", "hostname", "env", "service", "version", "language")
+
+    // Create keys with different combinations of HTTP fields
+    def keyWithNoSource = new MetricKey("resource", "service", "operation", null, "type", 200, false, false, "server", [], "GET", "/api/users", null)
+    def keyWithSource = new MetricKey("resource", "service", "operation", "source", "type", 200, false, false, "server", [], "POST", null, null)
+
+    def content = [
+      Pair.of(keyWithNoSource, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))),
+      Pair.of(keyWithSource, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))),
+    ]
+
+    ValidatingSink sink = new ValidatingSink(wellKnownTags, startTime, duration, content)
+    SerializingMetricWriter writer = new SerializingMetricWriter(wellKnownTags, sink, 128)
+
+    when:
+    writer.startBucket(content.size(), startTime, duration)
+    for (Pair<MetricKey, AggregateMetric> pair : content) {
+      writer.add(pair.getLeft(), pair.getRight())
+    }
+    writer.finishBucket()
+
+    then:
+    sink.validatedInput()
+  }
+
+  def "HTTPMethod and HTTPEndpoint fields are optional in payload"() {
+    setup:
+    long startTime = MILLISECONDS.toNanos(System.currentTimeMillis())
+    long duration = SECONDS.toNanos(10)
+    WellKnownTags wellKnownTags = new WellKnownTags("runtimeid", "hostname", "env", "service", "version", "language")
+
+    // Create keys with different combinations of HTTP fields
+    def keyWithBoth = new MetricKey("resource", "service", "operation", null, "type", 200, false, false, "server", [], "GET", "/api/users", null)
+    def keyWithMethodOnly = new MetricKey("resource", "service", "operation", null, "type", 200, false, false, "server", [], "POST", null,null)
+    def keyWithEndpointOnly = new MetricKey("resource", "service", "operation", null, "type", 200, false, false, "server", [], null, "/api/orders",null)
+    def keyWithNeither = new MetricKey("resource", "service", "operation", null, "type", 200, false, false, "client", [], null, null, null)
+
+    def content = [
+      Pair.of(keyWithBoth, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))),
+      Pair.of(keyWithMethodOnly, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))),
+      Pair.of(keyWithEndpointOnly, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))),
+      Pair.of(keyWithNeither, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L)))
+    ]
+
+    ValidatingSink sink = new ValidatingSink(wellKnownTags, startTime, duration, content)
+    SerializingMetricWriter writer = new SerializingMetricWriter(wellKnownTags, sink, 128)
+
+    when:
+    writer.startBucket(content.size(), startTime, duration)
+    for (Pair<MetricKey, AggregateMetric> pair : content) {
+      writer.add(pair.getLeft(), pair.getRight())
+    }
+    writer.finishBucket()
+
+    then:
+    sink.validatedInput()
+    // Test passes if validation in ValidatingSink succeeds
+    // ValidatingSink verifies that map size matches actual number of fields
+    // and that HTTPMethod/HTTPEndpoint are only present when non-empty
+  }
+
+  def "add git sha commit info when sha commit is #shaCommit"() {
+    setup:
+    GitInfoProvider gitInfoProvider = Mock(GitInfoProvider)
+    gitInfoProvider.getGitInfo() >> new GitInfo(null, null, null, new CommitInfo(shaCommit))
 
-  class ValidatingSink implements Sink {
+    long startTime = MILLISECONDS.toNanos(System.currentTimeMillis())
+    long duration = SECONDS.toNanos(10)
+    WellKnownTags wellKnownTags = new WellKnownTags("runtimeid", "hostname", "env", "service", "version", "language")
+
+    // Create keys with different combinations of HTTP fields
+    def key = new MetricKey("resource", "service", "operation", null, "type", 200, false, false, "server", [], "GET", "/api/users", null)
+
+    def content = [Pair.of(key, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))),]
+
+    ValidatingSink sink = new ValidatingSink(wellKnownTags, startTime, duration, content)
+    SerializingMetricWriter writer = new SerializingMetricWriter(wellKnownTags, sink, 128, gitInfoProvider)
+
+    when:
+
+    writer.startBucket(content.size(), startTime, duration)
+    for (Pair<MetricKey, AggregateMetric> pair : content) {
+      writer.add(pair.getLeft(), pair.getRight())
+    }
+    writer.finishBucket()
+
+    then:
+
+    sink.validatedInput()
+
+    where:
+    shaCommit << [null, "123456"]
+  }
+
+  static class ValidatingSink implements Sink {
 
     private final WellKnownTags wellKnownTags
     private final long startTimeNanos
@@ -161,7 +264,9 @@ class SerializingMetricWriterTest extends DDSpecification {
     void accept(int messageCount, ByteBuffer buffer) {
       MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(buffer)
       int mapSize = unpacker.unpackMapHeader()
-      assert mapSize == (7 + (Config.get().isExperimentalPropagateProcessTagsEnabled() ? 1 : 0))
+      String gitCommitSha = GitInfoProvider.INSTANCE.getGitInfo()?.getCommit()?.getSha()
+      assert mapSize == (7 + (Config.get().isExperimentalPropagateProcessTagsEnabled() ? 1 : 0)
+      + (gitCommitSha != null ? 1 : 0))
       assert unpacker.unpackString() == "RuntimeID"
       assert unpacker.unpackString() == wellKnownTags.getRuntimeId() as String
       assert unpacker.unpackString() == "Sequence"
@@ -178,6 +283,10 @@ class SerializingMetricWriterTest extends DDSpecification {
         assert unpacker.unpackString() == "ProcessTags"
         assert unpacker.unpackString() == ProcessTags.tagsForSerialization as String
       }
+      if (gitCommitSha != null) {
+        assert unpacker.unpackString() == "GitCommitSha"
+        assert unpacker.unpackString() == gitCommitSha
+      }
       assert unpacker.unpackString() == "Stats"
       int outerLength = unpacker.unpackArrayHeader()
       assert outerLength == 1