Skip to content

Commit d597de3

Browse files
jbachorikjbachorik
authored
Add JFR scrubbing before profile upload (#10577)
Fix HelperScanner: treat field/method types as USES not REQUIRES The JVM only eagerly resolves superclass and interfaces during defineClass. Field types, method parameter/return types, and declared exceptions are resolved lazily. Marking them as REQUIRES created false dependency cycles that broke topological sort ordering when injecting large helper batches (2000+ classes). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> Add JFR scrubbing to profiling upload pipeline Scrub sensitive fields (system properties, JVM arguments, environment variables, process command lines) from JFR recordings before upload. - Add profiling-scrubber module wrapping jafar-tools Scrubber - Wire ScrubRecordingDataListener decorator into ProfilingAgent - Add RecordingData.getPath() to avoid stream materialization for file-backed recordings (ddprof) - Add config: dd.profiling.scrub.enabled, dd.profiling.scrub.fail-open, dd.profiling.scrub.event-type-excludes Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> Enable JFR scrubbing in native image smoke test - Guard ThrowableInstanceAdvice during native-image build to prevent JFR event class initialization errors - Enable profiling scrubber in native image build args - Add smoke test verifying JFR files with system property events are produced (scrubbing assertion deferred until jafar handles SubstrateVM JFR chunk format) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> Update jafar to 0.14.1 and assert JFR scrubbing in native smoke test Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> Use latest jafar parser Spotless! Merge branch 'master' into jb/jfr_redacting Co-authored-by: jaroslav.bachorik <jaroslav.bachorik@datadoghq.com>
1 parent d25685a commit d597de3

20 files changed

Lines changed: 797 additions & 28 deletions

File tree

‎dd-java-agent/agent-profiling/build.gradle‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@ excludedClassesCoverage += [
1313
'com.datadog.profiling.agent.ProfilingAgent',
1414
'com.datadog.profiling.agent.ProfilingAgent.ShutdownHook',
1515
'com.datadog.profiling.agent.ProfilingAgent.DataDumper',
16-
'com.datadog.profiling.agent.ProfilerFlare'
16+
'com.datadog.profiling.agent.ProfilerFlare',
17+
'com.datadog.profiling.agent.ScrubRecordingDataListener',
18+
'com.datadog.profiling.agent.ScrubRecordingDataListener.ScrubbedRecordingData'
1719
]
1820

1921
dependencies {
@@ -23,6 +25,7 @@ dependencies {
2325
api project(':dd-java-agent:agent-profiling:profiling-ddprof')
2426
api project(':dd-java-agent:agent-profiling:profiling-uploader')
2527
api project(':dd-java-agent:agent-profiling:profiling-controller')
28+
implementation project(':dd-java-agent:agent-profiling:profiling-scrubber')
2629
api project(':dd-java-agent:agent-profiling:profiling-controller-jfr')
2730
api project(':dd-java-agent:agent-profiling:profiling-controller-jfr:implementation')
2831
api project(':dd-java-agent:agent-profiling:profiling-controller-ddprof')
@@ -42,6 +45,11 @@ configurations {
4245

4346
tasks.named("shadowJar", ShadowJar) {
4447
dependencies deps.excludeShared
48+
49+
// Exclude multi-release versioned classes from jafar-parser.
50+
// These are duplicates of base classes for newer Java APIs and confuse
51+
// the GraalVM native-image builder when the profiling jar is embedded in the agent.
52+
exclude 'META-INF/versions/**'
4553
}
4654

4755
tasks.named("jar", Jar) {

‎dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerRecordingData.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import java.nio.file.Path;
88
import java.time.Instant;
99
import javax.annotation.Nonnull;
10+
import javax.annotation.Nullable;
1011

1112
final class DatadogProfilerRecordingData extends RecordingData {
1213
private final Path recordingFile;
@@ -36,4 +37,10 @@ public void release() {
3637
public String getName() {
3738
return "ddprof";
3839
}
40+
41+
@Nullable
42+
@Override
43+
public Path getPath() {
44+
return recordingFile;
45+
}
3946
}

‎dd-java-agent/agent-profiling/profiling-scrubber/build.gradle‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
apply from: "$rootDir/gradle/java.gradle"
2+
3+
minimumInstructionCoverage = 0.0
4+
minimumBranchCoverage = 0.0
5+
6+
dependencies {
7+
api libs.slf4j
8+
9+
implementation(libs.jafar.tools) {
10+
// Agent has its own slf4j binding
11+
exclude group: 'org.slf4j', module: 'slf4j-simple'
12+
}
13+
14+
testImplementation libs.bundles.junit5
15+
testImplementation libs.bundles.mockito
16+
testImplementation libs.bundles.jmc
17+
}

‎dd-java-agent/agent-profiling/profiling-scrubber/src/main/java/com/datadog/profiling/scrubber/DefaultScrubDefinition.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package com.datadog.profiling.scrubber;
2+
3+
import io.jafar.tools.Scrubber;
4+
import java.util.Collections;
5+
import java.util.HashMap;
6+
import java.util.HashSet;
7+
import java.util.List;
8+
import java.util.Map;
9+
import java.util.Set;
10+
11+
/** Provides the default scrub definition targeting sensitive JFR event fields. */
12+
public final class DefaultScrubDefinition {
13+
14+
private static final Map<String, Scrubber.ScrubField> DEFAULT_SCRUB_FIELDS;
15+
16+
static {
17+
Map<String, Scrubber.ScrubField> fields = new HashMap<>();
18+
// ScrubField(keyField, valueField, predicate): null keyField = scrub all values unconditionally
19+
// System properties may contain API keys, passwords
20+
fields.put("jdk.InitialSystemProperty", new Scrubber.ScrubField(null, "value", (k, v) -> true));
21+
// JVM args may contain credentials in -D flags
22+
fields.put("jdk.JVMInformation", new Scrubber.ScrubField(null, "jvmArguments", (k, v) -> true));
23+
// Env vars may contain secrets
24+
fields.put(
25+
"jdk.InitialEnvironmentVariable", new Scrubber.ScrubField(null, "value", (k, v) -> true));
26+
// Process command lines may reveal infrastructure
27+
fields.put("jdk.SystemProcess", new Scrubber.ScrubField(null, "commandLine", (k, v) -> true));
28+
DEFAULT_SCRUB_FIELDS = Collections.unmodifiableMap(fields);
29+
}
30+
31+
/**
32+
* Creates a scrubber with the default scrub definition.
33+
*
34+
* @param excludeEventTypes list of event type names to exclude from scrubbing, or null for none
35+
* @return a configured {@link JfrScrubber}
36+
*/
37+
public static JfrScrubber create(List<String> excludeEventTypes) {
38+
Set<String> excludeSet =
39+
excludeEventTypes != null
40+
? new HashSet<>(excludeEventTypes)
41+
: Collections.<String>emptySet();
42+
43+
return new JfrScrubber(
44+
eventTypeName -> {
45+
if (excludeSet.contains(eventTypeName)) {
46+
return null;
47+
}
48+
return DEFAULT_SCRUB_FIELDS.get(eventTypeName);
49+
});
50+
}
51+
52+
private DefaultScrubDefinition() {}
53+
}

‎dd-java-agent/agent-profiling/profiling-scrubber/src/main/java/com/datadog/profiling/scrubber/JfrScrubber.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package com.datadog.profiling.scrubber;
2+
3+
import io.jafar.tools.Scrubber;
4+
import java.nio.file.Path;
5+
import java.util.function.Function;
6+
7+
/**
8+
* Thin wrapper around {@link Scrubber} from jafar-tools, hiding jafar types from consumers outside
9+
* the profiling-scrubber module.
10+
*/
11+
public final class JfrScrubber {
12+
13+
private final Function<String, Scrubber.ScrubField> scrubDefinition;
14+
15+
/** Package-private: use {@link DefaultScrubDefinition#create} to obtain an instance. */
16+
JfrScrubber(Function<String, Scrubber.ScrubField> scrubDefinition) {
17+
this.scrubDefinition = scrubDefinition;
18+
}
19+
20+
/**
21+
* Scrub the given file by replacing targeted field values with 'x' bytes.
22+
*
23+
* @param input the input file to scrub
24+
* @param output the output file to write the scrubbed content to
25+
* @throws Exception if an error occurs during parsing or writing
26+
*/
27+
public void scrubFile(Path input, Path output) throws Exception {
28+
Scrubber.scrubFile(input, output, scrubDefinition);
29+
}
30+
}

‎dd-java-agent/agent-profiling/profiling-scrubber/src/test/java/com/datadog/profiling/scrubber/JfrScrubberTest.java‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
package com.datadog.profiling.scrubber;
2+
3+
import static org.junit.jupiter.api.Assertions.assertEquals;
4+
import static org.junit.jupiter.api.Assertions.assertTrue;
5+
import static org.openjdk.jmc.common.item.Attribute.attr;
6+
import static org.openjdk.jmc.common.unit.UnitLookup.PLAIN_TEXT;
7+
8+
import java.io.IOException;
9+
import java.io.InputStream;
10+
import java.nio.file.Files;
11+
import java.nio.file.Path;
12+
import java.nio.file.StandardCopyOption;
13+
import java.util.Collections;
14+
import org.junit.jupiter.api.BeforeEach;
15+
import org.junit.jupiter.api.Test;
16+
import org.junit.jupiter.api.io.TempDir;
17+
import org.openjdk.jmc.common.item.IAttribute;
18+
import org.openjdk.jmc.common.item.IItem;
19+
import org.openjdk.jmc.common.item.IItemCollection;
20+
import org.openjdk.jmc.common.item.IItemIterable;
21+
import org.openjdk.jmc.common.item.IMemberAccessor;
22+
import org.openjdk.jmc.common.item.ItemFilters;
23+
import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit;
24+
25+
class JfrScrubberTest {
26+
27+
@TempDir Path tempDir;
28+
29+
private Path inputFile;
30+
31+
@BeforeEach
32+
void setUp() throws IOException {
33+
inputFile = tempDir.resolve("input.jfr");
34+
try (InputStream is = getClass().getResourceAsStream("/test-recording.jfr")) {
35+
if (is == null) {
36+
throw new IllegalStateException("test-recording.jfr not found in test resources");
37+
}
38+
Files.copy(is, inputFile, StandardCopyOption.REPLACE_EXISTING);
39+
}
40+
}
41+
42+
@Test
43+
void scrubInitialSystemPropertyValues() throws Exception {
44+
JfrScrubber scrubber = DefaultScrubDefinition.create(null);
45+
Path outputFile = tempDir.resolve("output.jfr");
46+
scrubber.scrubFile(inputFile, outputFile);
47+
48+
assertTrue(Files.exists(outputFile));
49+
assertTrue(Files.size(outputFile) > 0, "Scrubbed file should not be empty");
50+
51+
// Verify scrubbed values contain only 'x' characters
52+
IItemCollection events = JfrLoaderToolkit.loadEvents(outputFile.toFile());
53+
IItemCollection systemPropertyEvents =
54+
events.apply(ItemFilters.type("jdk.InitialSystemProperty"));
55+
assertTrue(systemPropertyEvents.hasItems(), "Expected jdk.InitialSystemProperty events");
56+
57+
IAttribute<String> valueAttr = attr("value", "value", "value", PLAIN_TEXT);
58+
for (IItemIterable itemIterable : systemPropertyEvents) {
59+
IMemberAccessor<String, IItem> accessor = valueAttr.getAccessor(itemIterable.getType());
60+
for (IItem item : itemIterable) {
61+
String value = accessor.getMember(item);
62+
if (value != null && !value.isEmpty()) {
63+
assertTrue(
64+
value.chars().allMatch(c -> c == 'x'),
65+
"System property value should be scrubbed: " + value);
66+
}
67+
}
68+
}
69+
}
70+
71+
@Test
72+
void scrubWithNoMatchingEvents() throws Exception {
73+
// Scrubber with all default events excluded — nothing matches
74+
JfrScrubber scrubber = new JfrScrubber(name -> null);
75+
Path outputFile = tempDir.resolve("output.jfr");
76+
scrubber.scrubFile(inputFile, outputFile);
77+
78+
// Output should be identical to input when no events match
79+
assertEquals(Files.size(inputFile), Files.size(outputFile));
80+
}
81+
82+
@Test
83+
void scrubWithExcludedEventType() throws Exception {
84+
// Exclude jdk.InitialSystemProperty from scrubbing
85+
JfrScrubber scrubber =
86+
DefaultScrubDefinition.create(Collections.singletonList("jdk.InitialSystemProperty"));
87+
Path outputFile = tempDir.resolve("output.jfr");
88+
scrubber.scrubFile(inputFile, outputFile);
89+
90+
assertTrue(Files.exists(outputFile));
91+
assertTrue(Files.size(outputFile) > 0);
92+
93+
// Verify excluded event type values are preserved (not scrubbed to 'x')
94+
IItemCollection events = JfrLoaderToolkit.loadEvents(outputFile.toFile());
95+
IItemCollection systemPropertyEvents =
96+
events.apply(ItemFilters.type("jdk.InitialSystemProperty"));
97+
assertTrue(systemPropertyEvents.hasItems(), "Expected jdk.InitialSystemProperty events");
98+
99+
IAttribute<String> valueAttr = attr("value", "value", "value", PLAIN_TEXT);
100+
boolean foundNonTrivialValue = false;
101+
for (IItemIterable itemIterable : systemPropertyEvents) {
102+
IMemberAccessor<String, IItem> accessor = valueAttr.getAccessor(itemIterable.getType());
103+
for (IItem item : itemIterable) {
104+
String value = accessor.getMember(item);
105+
if (value != null && !value.isEmpty()) {
106+
// At least one value should NOT be all-x (proving exclusion worked)
107+
if (!value.chars().allMatch(c -> c == 'x')) {
108+
foundNonTrivialValue = true;
109+
}
110+
}
111+
}
112+
}
113+
assertTrue(
114+
foundNonTrivialValue, "Excluded event type values should be preserved, not scrubbed");
115+
}
116+
}

‎dd-java-agent/agent-profiling/profiling-scrubber/src/test/resources/test-recording.jfr‎

3.54 MB
Binary file not shown.

‎dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ProfilingAgent.java‎

Lines changed: 35 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22

33
import static datadog.environment.JavaVirtualMachine.isJavaVersion;
44
import static datadog.environment.JavaVirtualMachine.isJavaVersionAtLeast;
5+
import static datadog.trace.api.config.ProfilingConfig.PROFILING_SCRUB_ENABLED;
6+
import static datadog.trace.api.config.ProfilingConfig.PROFILING_SCRUB_ENABLED_DEFAULT;
7+
import static datadog.trace.api.config.ProfilingConfig.PROFILING_SCRUB_FAIL_OPEN;
8+
import static datadog.trace.api.config.ProfilingConfig.PROFILING_SCRUB_FAIL_OPEN_DEFAULT;
59
import static datadog.trace.api.config.ProfilingConfig.PROFILING_START_FORCE_FIRST;
610
import static datadog.trace.api.config.ProfilingConfig.PROFILING_START_FORCE_FIRST_DEFAULT;
711
import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY;
@@ -32,6 +36,7 @@
3236
import java.nio.file.Paths;
3337
import java.nio.file.StandardCopyOption;
3438
import java.time.Duration;
39+
import java.util.List;
3540
import java.util.concurrent.atomic.AtomicBoolean;
3641
import java.util.function.Predicate;
3742
import java.util.regex.Pattern;
@@ -137,6 +142,25 @@ public static synchronized boolean run(final boolean earlyStart, Instrumentation
137142

138143
uploader = new ProfileUploader(config, configProvider);
139144

145+
RecordingDataListener listener = uploader::upload;
146+
if (dumper != null) {
147+
RecordingDataListener upload = listener;
148+
listener =
149+
(type, data, sync) -> {
150+
dumper.onNewData(type, data, sync);
151+
upload.onNewData(type, data, sync);
152+
};
153+
}
154+
// Scrubber wraps the combined dumper+uploader so debug dumps also contain scrubbed data
155+
if (configProvider.getBoolean(PROFILING_SCRUB_ENABLED, PROFILING_SCRUB_ENABLED_DEFAULT)) {
156+
List<String> excludeEventTypes =
157+
configProvider.getList(ProfilingConfig.PROFILING_SCRUB_EXCLUDE_EVENTS);
158+
boolean failOpen =
159+
configProvider.getBoolean(
160+
PROFILING_SCRUB_FAIL_OPEN, PROFILING_SCRUB_FAIL_OPEN_DEFAULT);
161+
listener = wrapWithScrubber(listener, excludeEventTypes, failOpen);
162+
}
163+
140164
final Duration startupDelay = Duration.ofSeconds(config.getProfilingStartDelay());
141165
final Duration uploadPeriod = Duration.ofSeconds(config.getProfilingUploadPeriod());
142166

@@ -149,12 +173,7 @@ public static synchronized boolean run(final boolean earlyStart, Instrumentation
149173
configProvider,
150174
controller,
151175
context.snapshot(),
152-
dumper == null
153-
? uploader::upload
154-
: (type, data, sync) -> {
155-
dumper.onNewData(type, data, sync);
156-
uploader.upload(type, data, sync);
157-
},
176+
listener,
158177
startupDelay,
159178
startupDelayRandomRange,
160179
uploadPeriod,
@@ -181,6 +200,16 @@ public static synchronized boolean run(final boolean earlyStart, Instrumentation
181200
return false;
182201
}
183202

203+
private static RecordingDataListener wrapWithScrubber(
204+
RecordingDataListener listener, List<String> excludeEventTypes, boolean failOpen) {
205+
try {
206+
return ScrubRecordingDataListener.wrap(listener, excludeEventTypes, failOpen);
207+
} catch (Exception e) {
208+
log.warn(SEND_TELEMETRY, "Failed to initialize JFR scrubber", e);
209+
return listener;
210+
}
211+
}
212+
184213
private static boolean isStartForceFirstSafe() {
185214
return isJavaVersionAtLeast(14)
186215
|| (isJavaVersion(13) && isJavaVersionAtLeast(13, 0, 4))

0 commit comments

Comments
 (0)