Skip to content

Commit 50402e3

Browse files
bric3devflow.devflow-routing-intake
bric3
and
devflow.devflow-routing-intake
authored
Adds crashtracking addresses and error thread name (#10984)
feat(crashtracking): Adds frame type fix(crashtracking): Incorrect parsing of some frames feat(crashtracking): Adds addresses on Java frames if available feat(crashtracking): Adds the missing `error.thread_name` The RFC 0011 states that https://github.com/DataDog/libdatadog/blob/main/docs/RFCs/0011-crashtracker-structured-log-format-V1_X.md fix(crashtracking): J9 wasn't properly reporting the frame type chore: make spotless happy Co-authored-by: devflow.devflow-routing-intake <devflow.devflow-routing-intake@kubernetes.us1.ddbuild.io>
1 parent 92b8b82 commit 50402e3

File tree

16 files changed

+944
-16
lines changed

16 files changed

+944
-16
lines changed

‎dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -525,6 +525,9 @@ private RequestBody makeErrorTrackingRequestBody(@Nonnull CrashLog payload, bool
525525
}
526526
writer.name("type").value(payload.error.kind);
527527
writer.name("message").value(payload.error.message);
528+
if (payload.error.threadName != null) {
529+
writer.name("thread_name").value(payload.error.threadName);
530+
}
528531
writer.name("source_type").value("Crashtracking");
529532
if (payload.error.stack != null) {
530533
writer.name("stack");

‎dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/ErrorData.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,22 @@ public final class ErrorData {
1010
public final String kind;
1111
public final String message;
1212

13+
@Json(name = "thread_name")
14+
public final String threadName;
15+
1316
@Json(name = "source_type")
1417
public final String sourceType = "Crashtracking";
1518

1619
public final StackTrace stack;
1720

1821
public ErrorData(String kind, String message, StackTrace stack) {
22+
this(kind, message, null, stack);
23+
}
24+
25+
public ErrorData(String kind, String message, String threadName, StackTrace stack) {
1926
this.kind = kind;
2027
this.message = message;
28+
this.threadName = threadName;
2129
this.stack = stack;
2230
}
2331

@@ -33,12 +41,13 @@ public boolean equals(Object o) {
3341
return isCrash == errorData.isCrash
3442
&& Objects.equals(kind, errorData.kind)
3543
&& Objects.equals(message, errorData.message)
44+
&& Objects.equals(threadName, errorData.threadName)
3645
&& Objects.equals(sourceType, errorData.sourceType)
3746
&& Objects.equals(stack, errorData.stack);
3847
}
3948

4049
@Override
4150
public int hashCode() {
42-
return Objects.hash(isCrash, kind, message, sourceType, stack);
51+
return Objects.hash(isCrash, kind, message, threadName, sourceType, stack);
4352
}
4453
}

‎dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/StackFrame.java‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ public final class StackFrame {
1010
public final Integer line;
1111
public final String function;
1212

13+
@Json(name = "type")
14+
public final String frameType;
15+
1316
@Json(name = "build_id")
1417
public final String buildId;
1518

@@ -19,23 +22,35 @@ public final class StackFrame {
1922
@Json(name = "file_type")
2023
public final BuildInfo.FileType fileType;
2124

25+
@Json(name = "ip")
26+
public final String ip;
27+
28+
@Json(name = "symbol_address")
29+
public final String symbolAddress;
30+
2231
@Json(name = "relative_address")
2332
public String relativeAddress;
2433

2534
public StackFrame(
2635
String path,
2736
Integer line,
2837
String function,
38+
String frameType,
2939
String buildId,
3040
BuildInfo.BuildIdType buildIdType,
3141
BuildInfo.FileType fileType,
42+
String ip,
43+
String symbolAddress,
3244
String relativeAddress) {
3345
this.path = path;
3446
this.line = line;
3547
this.function = function;
48+
this.frameType = frameType;
3649
this.buildId = buildId;
3750
this.buildIdType = buildIdType;
3851
this.fileType = fileType;
52+
this.ip = ip;
53+
this.symbolAddress = symbolAddress;
3954
this.relativeAddress = relativeAddress;
4055
}
4156

@@ -51,14 +66,27 @@ public boolean equals(Object o) {
5166
return Objects.equals(path, that.path)
5267
&& Objects.equals(line, that.line)
5368
&& Objects.equals(function, that.function)
69+
&& Objects.equals(frameType, that.frameType)
5470
&& Objects.equals(buildId, that.buildId)
5571
&& buildIdType == that.buildIdType
5672
&& fileType == that.fileType
73+
&& Objects.equals(ip, that.ip)
74+
&& Objects.equals(symbolAddress, that.symbolAddress)
5775
&& Objects.equals(relativeAddress, that.relativeAddress);
5876
}
5977

6078
@Override
6179
public int hashCode() {
62-
return Objects.hash(path, line, function, buildId, buildIdType, fileType, relativeAddress);
80+
return Objects.hash(
81+
path,
82+
line,
83+
function,
84+
frameType,
85+
buildId,
86+
buildIdType,
87+
fileType,
88+
ip,
89+
symbolAddress,
90+
relativeAddress);
6391
}
6492
}

‎dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/parsers/HotspotCrashLogParser.java‎

Lines changed: 97 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,39 @@ public HotspotCrashLogParser() {
9898
// find(), which would otherwise match the lowercase "sp"/"pc" tokens embedded in those lines.
9999
private static final Pattern REGISTER_LINE_START =
100100
Pattern.compile("^\\s*[A-Za-z][A-Za-z0-9]*\\s*=\\s*0x");
101+
private static final Pattern COMPILED_JAVA_ADDRESS_PARSER =
102+
Pattern.compile("@\\s+(0x[0-9a-fA-F]+)\\s+\\[(0x[0-9a-fA-F]+)\\+(0x[0-9a-fA-F]+)\\]");
103+
104+
// HotSpot crash logs encode the execution kind in the first column of each frame line.
105+
// Source references:
106+
// JDK 8:
107+
// https://github.com/openjdk/jdk8u/blob/73c9c6bcd062196cbebc4d9f22b13d2e20a14f98/hotspot/src/share/vm/runtime/frame.cpp#L710-L724
108+
// JDK 11:
109+
// https://github.com/openjdk/jdk11u/blob/970d6cf491a55fd6ab98ec3f449c13a58633078a/src/hotspot/share/runtime/frame.cpp#L647-L662
110+
// JDK 25:
111+
// https://github.com/openjdk/jdk25u/blob/2fe611a2a3386d097f636c15bd4d396a82dc695e/src/hotspot/share/runtime/frame.cpp#L652-L666
112+
// Mainline:
113+
// https://github.com/openjdk/jdk/blob/53c864a881d2183d3664a6a5a56480bd99fffe45/src/hotspot/share/runtime/frame.cpp#L647-L661
114+
// Note: the marker set changes across JDK lines. In particular, "A" appears in some HotSpot
115+
// versions but not all, so this mapping is best-effort rather than a stable cross-version enum.
116+
private static String hotspotFrameType(char marker) {
117+
switch (marker) {
118+
case 'J':
119+
return "compiled";
120+
case 'A': // exists in JDK 11
121+
return "aot_compiled";
122+
case 'j':
123+
return "interpreted";
124+
case 'V':
125+
return "vm";
126+
case 'v':
127+
return "stub";
128+
case 'C':
129+
return "native";
130+
default:
131+
return null;
132+
}
133+
}
101134

102135
private StackFrame parseLine(String line) {
103136
if (line == null || line.isEmpty()) {
@@ -107,24 +140,52 @@ private StackFrame parseLine(String line) {
107140
String functionName = null;
108141
Integer functionLine = null;
109142
String filename = null;
143+
String ip = null;
110144
String relAddress = null;
145+
String symbolAddress = null;
111146
char firstChar = line.charAt(0);
147+
String frameType = hotspotFrameType(firstChar);
112148
if (line.length() > 1 && !Character.isSpaceChar(line.charAt(1))) {
113149
// We can find entries like this in between the frames
114150
// Java frames: (J=compiled Java code, j=interpreted, Vv=VM code)
115151
return null;
116152
}
117153
switch (firstChar) {
118154
case 'J':
155+
case 'A':
119156
{
120157
// spotless:off
121158
// J 36572 c2 datadog.trace.util.AgentTaskScheduler$PeriodicTask.run()V (25 bytes) @ 0x00007f2fd0198488 [0x00007f2fd0198420+0x0000000000000068]
122159
// J 3896 c2 java.nio.ByteBuffer.allocate(I)Ljava/nio/ByteBuffer; java.base@21.0.1 (20 bytes) @ 0x0000000112ad51e8 [0x0000000112ad4fc0+0x0000000000000228]
160+
// J 302 java.util.zip.ZipFile.getEntry(J[BZ)J (0 bytes) @ 0x00007fa287303dce [0x00007fa287303d00+0xce]
123161
// spotless:on
124162
String[] parts = SPACE_SPLITTER.split(line);
125-
if (parts.length > 3) {
163+
int bytesToken = -1;
164+
for (int i = 0; i < parts.length - 1; i++) {
165+
if (parts[i].startsWith("(") && "bytes)".equals(parts[i + 1])) {
166+
bytesToken = i;
167+
break;
168+
}
169+
}
170+
if (bytesToken > 1) {
171+
String candidate = parts[bytesToken - 1];
172+
// Newer JVMs insert a module token before "(NN bytes)".
173+
if (candidate.contains("@")) {
174+
candidate = parts[bytesToken - 2];
175+
}
176+
if (!candidate.startsWith("(")) {
177+
functionName = candidate;
178+
}
179+
} else if (parts.length > 3 && !parts[3].startsWith("(")) {
126180
functionName = parts[3];
127181
}
182+
183+
Matcher matcher = COMPILED_JAVA_ADDRESS_PARSER.matcher(line);
184+
if (matcher.find()) {
185+
ip = matcher.group(1);
186+
symbolAddress = matcher.group(2);
187+
relAddress = matcher.group(3);
188+
}
128189
break;
129190
}
130191
case 'j':
@@ -200,9 +261,12 @@ private StackFrame parseLine(String line) {
200261
filename,
201262
functionLine,
202263
stripCompilerAnnotations(functionName),
264+
frameType,
203265
null,
204266
null,
205267
null,
268+
ip,
269+
symbolAddress,
206270
relAddress);
207271
}
208272
return null;
@@ -247,9 +311,30 @@ private static String normalizeFilename(String filename) {
247311
return filename.substring(0, prefixLen) + filename.substring(end);
248312
}
249313

314+
static String parseCurrentThreadName(String line) {
315+
if (line == null || !line.startsWith("Current thread ")) {
316+
return null;
317+
}
318+
final int separator = line.indexOf(':');
319+
if (separator < 0) {
320+
return null;
321+
}
322+
323+
String threadDescriptor = line.substring(separator + 1).trim();
324+
final int metadataStart = threadDescriptor.indexOf('[');
325+
if (metadataStart >= 0) {
326+
threadDescriptor = threadDescriptor.substring(0, metadataStart).trim();
327+
}
328+
if (threadDescriptor.isEmpty()) {
329+
return null;
330+
}
331+
return threadDescriptor;
332+
}
333+
250334
public CrashLog parse(String uuid, String crashLog) {
251335
SigInfo sigInfo = null;
252336
String pid = null;
337+
String threadName = null;
253338
List<StackFrame> frames = new ArrayList<>();
254339
String datetime = null;
255340
String datetimeRaw = null;
@@ -303,6 +388,9 @@ public CrashLog parse(String uuid, String crashLog) {
303388
}
304389
break;
305390
case THREAD:
391+
if (threadName == null) {
392+
threadName = parseCurrentThreadName(line);
393+
}
306394
// Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
307395
if (line.startsWith("Native frames: ")) {
308396
state = State.STACKTRACE;
@@ -424,25 +512,32 @@ public CrashLog parse(String uuid, String crashLog) {
424512
normalizeFilename(frame.path),
425513
frame.line,
426514
frame.function,
515+
frame.frameType,
427516
buildInfo.buildId,
428517
buildInfo.buildIdType,
429518
buildInfo.fileType,
519+
frame.ip,
520+
frame.symbolAddress,
430521
frame.relativeAddress));
431522
} else {
432523
enrichedFrames.add(
433524
new StackFrame(
434525
normalizeFilename(frame.path),
435526
frame.line,
436527
frame.function,
528+
frame.frameType,
437529
null,
438530
null,
439531
null,
532+
frame.ip,
533+
frame.symbolAddress,
440534
frame.relativeAddress));
441535
}
442536
}
443537

444538
ErrorData error =
445-
new ErrorData(kind, message, new StackTrace(enrichedFrames.toArray(new StackFrame[0])));
539+
new ErrorData(
540+
kind, message, threadName, new StackTrace(enrichedFrames.toArray(new StackFrame[0])));
446541
// We can not really extract the full metadata and os info from the crash log
447542
// This code assumes the parser is run on the same machine as the crash happened
448543
Metadata metadata = new Metadata("dd-trace-java", VersionInfo.VERSION, "java", null);

‎dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/parsers/J9JavacoreParser.java‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ public CrashLog parse(String uuid, String javacoreContent) {
111111
String exceptionDetail = null;
112112
String pid = null;
113113
String datetime = null;
114+
String currentThreadName = null;
114115
List<StackFrame> frames = new ArrayList<>();
115116
boolean incomplete = false;
116117
boolean foundThreadSection = false;
@@ -183,6 +184,7 @@ public CrashLog parse(String uuid, String javacoreContent) {
183184
if (inCurrentThread && line.startsWith("3XMTHREADINFO")) {
184185
Matcher threadMatcher = THREAD_INFO_PATTERN.matcher(line);
185186
if (threadMatcher.matches()) {
187+
currentThreadName = threadMatcher.group(1);
186188
collectingStack = true;
187189
}
188190
continue;
@@ -267,17 +269,24 @@ public CrashLog parse(String uuid, String javacoreContent) {
267269
frame.path,
268270
frame.line,
269271
frame.function,
272+
frame.frameType,
270273
buildInfo.buildId,
271274
buildInfo.buildIdType,
272275
buildInfo.fileType,
276+
frame.ip,
277+
frame.symbolAddress,
273278
frame.relativeAddress));
274279
} else {
275280
enrichedFrames.add(frame);
276281
}
277282
}
278283

279284
ErrorData error =
280-
new ErrorData(kind, message, new StackTrace(enrichedFrames.toArray(new StackFrame[0])));
285+
new ErrorData(
286+
kind,
287+
message,
288+
currentThreadName,
289+
new StackTrace(enrichedFrames.toArray(new StackFrame[0])));
281290
Metadata metadata = new Metadata("dd-trace-java", VersionInfo.VERSION, "java", null);
282291
Integer parsedPid = safelyParseInt(pid);
283292
ProcInfo procInfo = parsedPid != null ? new ProcInfo(parsedPid) : null;
@@ -381,11 +390,14 @@ private StackFrame parseJavaStackFrame(String frameText) {
381390
// Extract source file and line: method(File.java:123)
382391
int parenStart = function.lastIndexOf('(');
383392
int parenEnd = function.lastIndexOf(')');
393+
String frameType = "java";
384394
if (parenStart > 0 && parenEnd > parenStart) {
385395
String sourceInfo = function.substring(parenStart + 1, parenEnd);
386396
function = function.substring(0, parenStart);
387397

388-
if (!"Native Method".equals(sourceInfo)) {
398+
if ("Native Method".equals(sourceInfo)) {
399+
frameType = "native";
400+
} else {
389401
int colonIdx = sourceInfo.lastIndexOf(':');
390402
if (colonIdx > 0) {
391403
file = sourceInfo.substring(0, colonIdx);
@@ -400,7 +412,7 @@ private StackFrame parseJavaStackFrame(String frameText) {
400412
}
401413
}
402414

403-
return new StackFrame(file, line, function, null, null, null, null);
415+
return new StackFrame(file, line, function, frameType, null, null, null, null, null, null);
404416
}
405417

406418
/**
@@ -462,7 +474,7 @@ private StackFrame parseNativeStackFrame(String frameText) {
462474
}
463475
}
464476

465-
return new StackFrame(file, null, function, null, null, null, relAddress);
477+
return new StackFrame(file, null, function, "native", null, null, null, null, null, relAddress);
466478
}
467479

468480
private String parseDateTime(String datePart, String timePart) {

0 commit comments

Comments
 (0)