Reports loaded libraries in crashtracking reports (#11000)

chore(crashtracking): Better json assertions

feat(crashtracking): Reports dynamic libs in the error.files section

RFC: https://github.com/DataDog/libdatadog/blob/main/docs/RFCs/0011-crashtracker-structured-log-format-V1_X.md
Note that on BSD / macOs the loaded libs is not coming from procfs.

Merge branch 'master' into bdu/proc-memory-mappings

Co-authored-by: devflow.devflow-routing-intake <devflow.devflow-routing-intake@kubernetes.us1.ddbuild.io>

Author: bric3

dd-java-agent/agent-crashtracking/build.gradle

@@ -25,6 +25,8 @@ dependencies {
 
   testImplementation libs.bundles.junit5
   testImplementation libs.bundles.mockito
+  testImplementation libs.assertj.core
+  testImplementation libs.json.unit.assertj
   testImplementation libs.jackson.databind
   testImplementation libs.testcontainers
   testImplementation group: 'com.squareup.okhttp3', name: 'mockwebserver', version: libs.versions.okhttp.legacy.get()

dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java

@@ -586,6 +586,18 @@ private RequestBody makeErrorTrackingRequestBody(@Nonnull CrashLog payload, bool
           writer.endObject();
           writer.endObject();
         }
+        // files (e.g. /proc/self/maps or dynamic_libraries)
+        if (payload.files != null) {
+          writer.name("files");
+          writer.beginObject();
+          writer.name(payload.files.name);
+          writer.beginArray();
+          for (String fileLine : payload.files.lines) {
+            writer.value(fileLine);
+          }
+          writer.endArray();
+          writer.endObject();
+        }
         writer.endObject();
       }
       return RequestBody.create(APPLICATION_JSON, buf.readByteString());

dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/CrashLog.java

@@ -13,7 +13,7 @@ public final class CrashLog {
   public static final JsonAdapter<CrashLog> ADAPTER;
 
   static {
-    Moshi moshi = new Moshi.Builder().build();
+    Moshi moshi = new Moshi.Builder().add(new DynamicLibs.JsonAdapter()).build();
     ADAPTER = moshi.adapter(CrashLog.class);
   }
 
@@ -41,6 +41,12 @@ public final class CrashLog {
 
   public final Experimental experimental;
 
+  /**
+   * Useful files for triage and debugging (e.g. {@code /proc/self/maps}, {@code
+   * dynamic_libraries}).
+   */
+  public final DynamicLibs files;
+
   public CrashLog(
       String uuid,
       boolean incomplete,
@@ -61,6 +67,7 @@ public CrashLog(
         procInfo,
         sigInfo,
         dataSchemaVersion,
+        null,
         null);
   }
 
@@ -74,7 +81,8 @@ public CrashLog(
       ProcInfo procInfo,
       SigInfo sigInfo,
       String dataSchemaVersion,
-      Experimental experimental) {
+      Experimental experimental,
+      DynamicLibs files) {
     this.uuid = uuid != null ? uuid : RandomUtils.randomUUID().toString();
     this.incomplete = incomplete;
     this.timestamp = timestamp;
@@ -85,6 +93,7 @@ public CrashLog(
     this.sigInfo = sigInfo;
     this.dataSchemaVersion = dataSchemaVersion;
     this.experimental = experimental;
+    this.files = files;
   }
 
   public String toJson() {
@@ -113,7 +122,8 @@ public boolean equals(Object o) {
         && Objects.equals(procInfo, crashLog.procInfo)
         && Objects.equals(sigInfo, crashLog.sigInfo)
         && Objects.equals(dataSchemaVersion, crashLog.dataSchemaVersion)
-        && Objects.equals(experimental, crashLog.experimental);
+        && Objects.equals(experimental, crashLog.experimental)
+        && Objects.equals(files, crashLog.files);
   }
 
   @Override
@@ -129,7 +139,8 @@ public int hashCode() {
         sigInfo,
         version,
         dataSchemaVersion,
-        experimental);
+        experimental,
+        files);
   }
 
   public boolean equalsForTest(Object o) {
@@ -149,6 +160,7 @@ public boolean equalsForTest(Object o) {
         && Objects.equals(procInfo, crashLog.procInfo)
         && Objects.equals(sigInfo, crashLog.sigInfo)
         && Objects.equals(dataSchemaVersion, crashLog.dataSchemaVersion)
-        && Objects.equals(experimental, crashLog.experimental);
+        && Objects.equals(experimental, crashLog.experimental)
+        && Objects.equals(files, crashLog.files);
   }
 }

dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/DynamicLibs.java

@@ -0,0 +1,30 @@
+package datadog.crashtracking.dto;
+
+import com.squareup.moshi.FromJson;
+import com.squareup.moshi.ToJson;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class DynamicLibs {
+  public final String name;
+  public final List<String> lines;
+
+  public DynamicLibs(String name, List<String> lines) {
+    this.name = name;
+    this.lines = lines;
+  }
+
+  public static class JsonAdapter {
+    @ToJson
+    Map<String, List<String>> toJson(DynamicLibs dynamicLibs) {
+      return Collections.singletonMap(dynamicLibs.name, dynamicLibs.lines);
+    }
+
+    @FromJson
+    DynamicLibs fromJson(Map<String, List<String>> map) {
+      Map.Entry<String, List<String>> entry = map.entrySet().iterator().next();
+      return new DynamicLibs(entry.getKey(), entry.getValue());
+    }
+  }
+}

dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/parsers/HotspotCrashLogParser.java

@@ -6,6 +6,7 @@
 import datadog.crashtracking.buildid.BuildIdCollector;
 import datadog.crashtracking.buildid.BuildInfo;
 import datadog.crashtracking.dto.CrashLog;
+import datadog.crashtracking.dto.DynamicLibs;
 import datadog.crashtracking.dto.ErrorData;
 import datadog.crashtracking.dto.Experimental;
 import datadog.crashtracking.dto.Metadata;
@@ -341,6 +342,8 @@ public CrashLog parse(String uuid, String crashLog) {
     boolean incomplete = false;
     String oomMessage = null;
     Map<String, String> registers = null;
+    List<String> dynamicLibraryLines = null;
+    String dynamicLibraryKey = null;
 
     String[] lines = NEWLINE_SPLITTER.split(crashLog);
     outer:
@@ -449,15 +452,21 @@ public CrashLog parse(String uuid, String crashLog) {
         case DYNAMIC_LIBRARIES:
           if (line.isEmpty()) {
             state = State.SEEK_DYNAMIC_LIBRARIES;
-          }
-          final Matcher matcher = DYNAMIC_LIBS_PATH_PARSER.matcher(line);
-          if (matcher.matches()) {
-            final String pathString = matcher.group(1);
-            if (pathString != null && !pathString.isEmpty()) {
-              try {
-                final Path path = Paths.get(pathString);
-                buildIdCollector.resolveBuildId(path);
-              } catch (InvalidPathException ignored) {
+          } else {
+            if (dynamicLibraryKey == null) {
+              dynamicLibraryKey = detectDynamicLibrariesKey(line);
+              dynamicLibraryLines = new ArrayList<>();
+            }
+            final Matcher matcher = DYNAMIC_LIBS_PATH_PARSER.matcher(line);
+            if (matcher.matches()) {
+              final String pathString = matcher.group(1);
+              if (pathString != null && !pathString.isEmpty()) {
+                dynamicLibraryLines.add(line);
+                try {
+                  final Path path = Paths.get(pathString);
+                  buildIdCollector.resolveBuildId(path);
+                } catch (InvalidPathException ignored) {
+                }
               }
             }
           }
@@ -545,6 +554,10 @@ public CrashLog parse(String uuid, String crashLog) {
     ProcInfo procInfo = parsedPid != null ? new ProcInfo(parsedPid) : null;
     Experimental experimental =
         (registers != null && !registers.isEmpty()) ? new Experimental(registers) : null;
+    DynamicLibs files =
+        (dynamicLibraryLines != null && !dynamicLibraryLines.isEmpty())
+            ? new DynamicLibs(dynamicLibraryKey, dynamicLibraryLines)
+            : null;
     return new CrashLog(
         uuid,
         incomplete,
@@ -555,7 +568,8 @@ public CrashLog parse(String uuid, String crashLog) {
         procInfo,
         sigInfo,
         "1.0",
-        experimental);
+        experimental,
+        files);
   }
 
   static String dateTimeToISO(String datetime) {
@@ -572,6 +586,40 @@ static String dateTimeToISO(String datetime) {
     }
   }
 
+  /**
+   * Detects whether the Dynamic libraries section comes from Linux {@code /proc/self/maps} (address
+   * range format {@code addr-addr perms ...}) or from the BSD/macOS dyld callback (format {@code
+   * 0xaddr\tpath}). Returns the appropriate map key.
+   */
+  // The "Dynamic libraries:" section is written by os::print_dll_info(), whose implementation
+  // differs by platform:
+  //
+  // Linux
+  // -----
+  // This reads `/proc/{tid}/maps` verbatim via _print_ascii_file(), producing the usual
+  // `/proc/self/maps` format:
+  //
+  //   "addr-addr perms offset dev:inode [path]"
+  //
+  // Mainline:
+  // https://github.com/openjdk/jdk/blob/783f8f1adc4ea3ef7fd4c5ca5473aad76dfc7ed1/src/hotspot/os/linux/os_linux.cpp#L2086-L2099
+  //
+  // BSD/macOS
+  // ---------
+  // This relies on `_dyld_image_count()`/`_dyld_get_image_name()` (on macOS) or
+  // `dlinfo(RTLD_DI_LINKMAP)` (on FreeBSD/OpenBSD) via a callback, producing a simpler format:
+  //
+  //   "0xaddr\tpath"
+  //
+  // which lacks much of the information found in Linux's `/proc/self/maps`.
+  // Mainline:
+  // https://github.com/openjdk/jdk/blob/783f8f1adc4ea3ef7fd4c5ca5473aad76dfc7ed1/src/hotspot/os/bsd/os_bsd.cpp#L1382-L1387
+  static String detectDynamicLibrariesKey(String firstLine) {
+    int dash = firstLine.indexOf('-');
+    int space = firstLine.indexOf(' ');
+    return (dash > 0 && space > 0 && dash < space) ? "/proc/self/maps" : "dynamic_libraries";
+  }
+
   static Integer safelyParseInt(String value) {
     if (value == null) {
       return null;

dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/parsers/J9JavacoreParser.java

@@ -303,7 +303,8 @@ public CrashLog parse(String uuid, String javacoreContent) {
         procInfo,
         sigInfo,
         "1.0",
-        experimental);
+        experimental,
+        null);
   }
 
   private static Integer safelyParseInt(String value) {

dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/CrashUploaderTest.java

@@ -3,6 +3,7 @@
 import static datadog.crashtracking.CrashUploader.HEADER_DD_EVP_SUBDOMAIN;
 import static datadog.crashtracking.CrashUploader.HEADER_DD_TELEMETRY_API_VERSION;
 import static datadog.crashtracking.CrashUploader.TELEMETRY_API_VERSION;
+import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -292,9 +293,9 @@ public void testTelemetryHappyPath(String log) throws Exception {
     String message = event.get("payload").get(0).get("message").asText();
     CrashLog extracted = CrashLog.fromJson(message);
 
-    assertTrue(
-        expected.equalsForTest(extracted),
-        () -> "Expected: " + expected.toJson() + "\nbut got: " + extracted.toJson());
+    assertThatJson(extracted.toJson())
+        .whenIgnoringPaths("os_info", "metadata")
+        .isEqualTo(expected.toJson());
     assertEquals("severity:crash", event.get("payload").get(0).get("tags").asText());
     assertCommonPayload(event);
   }
@@ -356,19 +357,8 @@ public void testErrorTrackingHappyPath(String log) throws Exception {
     assertTrue(ddtags.contains("runtime_name:"));
 
     // assert platform independent equality
-    assertEquals(
-        expected,
-        extracted,
-        () -> {
-          try {
-            return "Expected: "
-                + mapper.writeValueAsString(expected)
-                + "\nbut got: "
-                + mapper.writeValueAsString(extracted);
-          } catch (JsonProcessingException e) {
-            throw new RuntimeException(e);
-          }
-        });
+    assertThatJson(mapper.writeValueAsString(extracted))
+        .isEqualTo(mapper.writeValueAsString(expected));
   }
 
   private void assertCommonHeader(JsonNode event) {