Skip to content

Add processor context for hotspot crash tracking #10867

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
bric3 wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Add processor context for hotspot crash tracking #10867

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -561,6 +561,18 @@ private RequestBody makeErrorTrackingRequestBody(@Nonnull CrashLog payload, bool
"os.version")); // this has been restructured under OsInfo so taking raw here
writer.endObject();
}
// experimental
if (payload.experimental != null && payload.experimental.ucontext != null) {
writer.name("experimental");
writer.beginObject();
writer.name("ucontext");
writer.beginObject();
for (Map.Entry<String, String> entry : payload.experimental.ucontext.entrySet()) {
writer.name(entry.getKey()).value(entry.getValue());
}
writer.endObject();
writer.endObject();
}
writer.endObject();
}
return RequestBody.create(APPLICATION_JSON, buf.readByteString());
Expand Down
36 changes: 33 additions & 3 deletions dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/CrashLog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ public final class CrashLog {
@Json(name = "sig_info")
public final SigInfo sigInfo;

public final Experimental experimental;

public CrashLog(
String uuid,
boolean incomplete,
Expand All @@ -49,6 +51,30 @@ public CrashLog(
ProcInfo procInfo,
SigInfo sigInfo,
String dataSchemaVersion) {
this(
uuid,
incomplete,
timestamp,
error,
metadata,
osInfo,
procInfo,
sigInfo,
dataSchemaVersion,
null);
}

public CrashLog(
String uuid,
boolean incomplete,
String timestamp,
ErrorData error,
Metadata metadata,
OSInfo osInfo,
ProcInfo procInfo,
SigInfo sigInfo,
String dataSchemaVersion,
Experimental experimental) {
this.uuid = uuid != null ? uuid : RandomUtils.randomUUID().toString();
this.incomplete = incomplete;
this.timestamp = timestamp;
Expand All @@ -58,6 +84,7 @@ public CrashLog(
this.procInfo = procInfo;
this.sigInfo = sigInfo;
this.dataSchemaVersion = dataSchemaVersion;
this.experimental = experimental;
}

public String toJson() {
Expand Down Expand Up @@ -85,7 +112,8 @@ public boolean equals(Object o) {
&& Objects.equals(osInfo, crashLog.osInfo)
&& Objects.equals(procInfo, crashLog.procInfo)
&& Objects.equals(sigInfo, crashLog.sigInfo)
&& Objects.equals(dataSchemaVersion, crashLog.dataSchemaVersion);
&& Objects.equals(dataSchemaVersion, crashLog.dataSchemaVersion)
&& Objects.equals(experimental, crashLog.experimental);
}

@Override
Expand All @@ -100,7 +128,8 @@ public int hashCode() {
procInfo,
sigInfo,
version,
dataSchemaVersion);
dataSchemaVersion,
experimental);
}

public boolean equalsForTest(Object o) {
Expand All @@ -119,6 +148,7 @@ public boolean equalsForTest(Object o) {
&& Objects.equals(error, crashLog.error)
&& Objects.equals(procInfo, crashLog.procInfo)
&& Objects.equals(sigInfo, crashLog.sigInfo)
&& Objects.equals(dataSchemaVersion, crashLog.dataSchemaVersion);
&& Objects.equals(dataSchemaVersion, crashLog.dataSchemaVersion)
&& Objects.equals(experimental, crashLog.experimental);
}
}
24 changes: 24 additions & 0 deletions dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/Experimental.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package datadog.crashtracking.dto;

import java.util.Map;
import java.util.Objects;

public final class Experimental {
public final Map<String, String> ucontext;

public Experimental(Map<String, String> ucontext) {
this.ucontext = ucontext;
}

@Override
public boolean equals(Object o) {
if (!(o instanceof Experimental)) return false;
Experimental that = (Experimental) o;
return Objects.equals(ucontext, that.ucontext);
}

@Override
public int hashCode() {
return Objects.hash(ucontext);
}
}
49 changes: 42 additions & 7 deletions ...gent-crashtracking/src/main/java/datadog/crashtracking/parsers/HotspotCrashLogParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import datadog.crashtracking.buildid.BuildInfo;
import datadog.crashtracking.dto.CrashLog;
import datadog.crashtracking.dto.ErrorData;
import datadog.crashtracking.dto.Experimental;
import datadog.crashtracking.dto.Metadata;
import datadog.crashtracking.dto.OSInfo;
import datadog.crashtracking.dto.ProcInfo;
Expand All @@ -21,8 +22,10 @@
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

Expand All @@ -45,6 +48,7 @@ enum State {
SUMMARY,
THREAD,
STACKTRACE,
REGISTERS,
SEEK_DYNAMIC_LIBRARIES,
DYNAMIC_LIBRARIES,
DONE
Expand All @@ -65,6 +69,9 @@ public HotspotCrashLogParser() {
"siginfo:\\s+si_signo:\\s+(\\d+)\\s+\\((\\w+)\\),\\s+si_code:\\s+(\\d+)\\s+\\(([^)]+)\\),\\s+si_addr:\\s+(0x[0-9a-fA-F]+)");
private static final Pattern DYNAMIC_LIBS_PATH_PARSER =
Pattern.compile("^(?:0x)?[0-9a-fA-F]+(?:-[0-9a-fA-F]+)?\\s+(?:[^\\s/\\[]+\\s+)*(.*)$");
// Matches register entries like: RAX=0x..., R8 =0x..., TRAPNO=0x...
private static final Pattern REGISTER_ENTRY_PARSER =
Pattern.compile("([A-Z0-9]+)\\s*=\\s*(0x[0-9a-fA-F]+)");

private StackFrame parseLine(String line) {
if (line == null || line.isEmpty()) {
Expand All @@ -84,10 +91,10 @@ private StackFrame parseLine(String line) {
switch (firstChar) {
case 'J':
{
// J 36572 c2 datadog.trace.util.AgentTaskScheduler$PeriodicTask.run()V (25 bytes) @
// 0x00007f2fd0198488 [0x00007f2fd0198420+0x0000000000000068]
// J 3896 c2 java.nio.ByteBuffer.allocate(I)Ljava/nio/ByteBuffer; java.base@21.0.1 (20
// bytes) @ 0x0000000112ad51e8 [0x0000000112ad4fc0+0x0000000000000228]
// spotless:off
// J 36572 c2 datadog.trace.util.AgentTaskScheduler$PeriodicTask.run()V (25 bytes) @ 0x00007f2fd0198488 [0x00007f2fd0198420+0x0000000000000068]
// J 3896 c2 java.nio.ByteBuffer.allocate(I)Ljava/nio/ByteBuffer; java.base@21.0.1 (20 bytes) @ 0x0000000112ad51e8 [0x0000000112ad4fc0+0x0000000000000228]
// spotless:on
String[] parts = SPACE_SPLITTER.split(line);
if (parts.length > 3) {
functionName = parts[3];
Expand Down Expand Up @@ -221,6 +228,7 @@ public CrashLog parse(String uuid, String crashLog) {
String datetime = null;
boolean incomplete = false;
String oomMessage = null;
Map<String, String> registers = null;

String[] lines = NEWLINE_SPLITTER.split(crashLog);
outer:
Expand Down Expand Up @@ -275,8 +283,9 @@ public CrashLog parse(String uuid, String crashLog) {
break;
case STACKTRACE:
if (line.startsWith("siginfo:")) {
// siginfo: si_signo: 11 (SIGSEGV), si_code: 1 (SEGV_MAPERR), si_addr:
// 0x0000000000000070
// spotless:off
// siginfo: si_signo: 11 (SIGSEGV), si_code: 1 (SEGV_MAPERR), si_addr: 0x0000000000000070
// spotless:on
final Matcher siginfoMatcher = SIGINFO_PARSER.matcher(line);
if (siginfoMatcher.matches()) {
Integer number = safelyParseInt(siginfoMatcher.group(1));
Expand All @@ -286,6 +295,9 @@ public CrashLog parse(String uuid, String crashLog) {
String address = siginfoMatcher.group(5);
sigInfo = new SigInfo(number, name, siCode, sigAction, address);
}
} else if (line.startsWith("Registers:")) {
registers = new LinkedHashMap<>();
state = State.REGISTERS;
} else if (line.contains("P R O C E S S")) {
state = State.SEEK_DYNAMIC_LIBRARIES;
} else {
Expand All @@ -296,6 +308,18 @@ public CrashLog parse(String uuid, String crashLog) {
}
}
break;
case REGISTERS:
if (!line.isEmpty() && !REGISTER_ENTRY_PARSER.matcher(line).find()) {
// non-empty line with no register entries signals end of section; reprocess in
// STACKTRACE
state = State.STACKTRACE;
} else {
final Matcher m = REGISTER_ENTRY_PARSER.matcher(line);
while (m.find()) {
registers.put(m.group(1), m.group(2));
}
}
break;
case SEEK_DYNAMIC_LIBRARIES:
if (line.startsWith("Dynamic libraries:")) {
state = State.DYNAMIC_LIBRARIES;
Expand Down Expand Up @@ -382,8 +406,19 @@ public CrashLog parse(String uuid, String crashLog) {
Metadata metadata = new Metadata("dd-trace-java", VersionInfo.VERSION, "java", null);
Integer parsedPid = safelyParseInt(pid);
ProcInfo procInfo = parsedPid != null ? new ProcInfo(parsedPid) : null;
Experimental experimental =
(registers != null && !registers.isEmpty()) ? new Experimental(registers) : null;
return new CrashLog(
uuid, incomplete, datetime, error, metadata, OSInfo.current(), procInfo, sigInfo, "1.0");
uuid,
incomplete,
datetime,
error,
metadata,
OSInfo.current(),
procInfo,
sigInfo,
"1.0",
experimental);
}

static String dateTimeToISO(String datetime) {
Expand Down
2 changes: 1 addition & 1 deletion ...nt-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry-2.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"timestamp":"2024-09-20T13:19:06Z","ddsource":"crashtracker","error":{"is_crash":true,"type":"UNKNOWN","message":"Process terminated by signal UNKNOWN","source_type":"Crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"function":"__pthread_clockjoin_ex+0x255","path":"libpthread.so.0","relative_address":"0x9cd5"}]}}}
{"timestamp":"2024-09-20T13:19:06Z","ddsource":"crashtracker","error":{"is_crash":true,"type":"UNKNOWN","message":"Process terminated by signal UNKNOWN","source_type":"Crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"function":"__pthread_clockjoin_ex+0x255","path":"libpthread.so.0","relative_address":"0x9cd5"}]}},"experimental":{"ucontext":{"RAX":"0x00000000000000ca","RBX":"0x00000000000000ca","RCX":"0x00007f011ab1ccd7","RDX":"0x000000000008ca23","RSP":"0x00007ffeabf89710","RBP":"0x00007ffeabf897b8","RSI":"0x0000000000000000","RDI":"0x00007f01192129d0","R8":"0x0000000000000000","R9":"0x00007f0119212700","R10":"0x0000000000000000","R11":"0x0000000000000246","R12":"0x000000000008ca23","R13":"0x00007f01192129d0","R14":"0x00007ffeabf89840","R15":"0x00007f0119212700","RIP":"0x00007f011ab1ccd5","EFLAGS":"0x0000000000000246","CSGSFS":"0x002b000000000033","ERR":"0x0000000000000000","TRAPNO":"0x0000000000000000"}}}
2 changes: 1 addition & 1 deletion ...gent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry.txt
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion .../agent-crashtracking/src/test/resources/golden/telemetry/sample-crash-for-telemetry-2.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"data_schema_version":"1.0","error":{"is_crash":true,"kind":"UNKNOWN","message":"Process terminated by signal UNKNOWN","source_type":"Crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"function":"__pthread_clockjoin_ex+0x255","path":"libpthread.so.0","relative_address":"0x9cd5"}]}},"incomplete":false,"metadata":{"family":"java","library_name":"dd-trace-java","library_version":"1.60.0-SNAPSHOT~cffe9c6085"},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":{"Semantic":[15,7,1]}},"proc_info":{"pid":576034},"timestamp":"2024-09-20T13:19:06Z","uuid":"a4194cd6-8cb3-45fd-9bd9-3af83e0a3ad3","version_id":0}
{"data_schema_version":"1.0","error":{"is_crash":true,"kind":"UNKNOWN","message":"Process terminated by signal UNKNOWN","source_type":"Crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"function":"__pthread_clockjoin_ex+0x255","path":"libpthread.so.0","relative_address":"0x9cd5"}]}},"incomplete":false,"metadata":{"family":"java","library_name":"dd-trace-java","library_version":"1.60.0-SNAPSHOT~cffe9c6085"},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":{"Semantic":[15,7,1]}},"experimental":{"ucontext":{"RAX":"0x00000000000000ca","RBX":"0x00000000000000ca","RCX":"0x00007f011ab1ccd7","RDX":"0x000000000008ca23","RSP":"0x00007ffeabf89710","RBP":"0x00007ffeabf897b8","RSI":"0x0000000000000000","RDI":"0x00007f01192129d0","R8":"0x0000000000000000","R9":"0x00007f0119212700","R10":"0x0000000000000000","R11":"0x0000000000000246","R12":"0x000000000008ca23","R13":"0x00007f01192129d0","R14":"0x00007ffeabf89840","R15":"0x00007f0119212700","RIP":"0x00007f011ab1ccd5","EFLAGS":"0x0000000000000246","CSGSFS":"0x002b000000000033","ERR":"0x0000000000000000","TRAPNO":"0x0000000000000000"}},"proc_info":{"pid":576034},"timestamp":"2024-09-20T13:19:06Z","uuid":"a4194cd6-8cb3-45fd-9bd9-3af83e0a3ad3","version_id":0}
Loading
Loading