Skip to content

Commit 1bb6683

Browse files
jbachorikclaude
jbachorik
and
claude
committed
Add JFR scrubbing to profiling upload pipeline
Scrub sensitive fields (system properties, JVM arguments, environment variables, process command lines) from JFR recordings before upload. - Add profiling-scrubber module wrapping jafar-tools Scrubber - Wire ScrubRecordingDataListener decorator into ProfilingAgent - Add RecordingData.getPath() to avoid stream materialization for file-backed recordings (ddprof) - Add config: dd.profiling.scrub.enabled, dd.profiling.scrub.fail-open, dd.profiling.scrub.event-type-excludes Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 0b609a5 commit 1bb6683

16 files changed

Lines changed: 707 additions & 8 deletions

File tree

‎dd-java-agent/agent-profiling/build.gradle‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@ excludedClassesCoverage += [
1313
'com.datadog.profiling.agent.ProfilingAgent',
1414
'com.datadog.profiling.agent.ProfilingAgent.ShutdownHook',
1515
'com.datadog.profiling.agent.ProfilingAgent.DataDumper',
16-
'com.datadog.profiling.agent.ProfilerFlare'
16+
'com.datadog.profiling.agent.ProfilerFlare',
17+
'com.datadog.profiling.agent.ScrubRecordingDataListener',
18+
'com.datadog.profiling.agent.ScrubRecordingDataListener.ScrubbedRecordingData'
1719
]
1820

1921
dependencies {
@@ -23,6 +25,7 @@ dependencies {
2325
api project(':dd-java-agent:agent-profiling:profiling-ddprof')
2426
api project(':dd-java-agent:agent-profiling:profiling-uploader')
2527
api project(':dd-java-agent:agent-profiling:profiling-controller')
28+
implementation project(':dd-java-agent:agent-profiling:profiling-scrubber')
2629
api project(':dd-java-agent:agent-profiling:profiling-controller-jfr')
2730
api project(':dd-java-agent:agent-profiling:profiling-controller-jfr:implementation')
2831
api project(':dd-java-agent:agent-profiling:profiling-controller-ddprof')
@@ -42,6 +45,11 @@ configurations {
4245

4346
tasks.named("shadowJar", ShadowJar) {
4447
dependencies deps.excludeShared
48+
49+
// Exclude multi-release versioned classes from jafar-parser.
50+
// These are duplicates of base classes for newer Java APIs and confuse
51+
// the GraalVM native-image builder when the profiling jar is embedded in the agent.
52+
exclude 'META-INF/versions/**'
4553
}
4654

4755
tasks.named("jar", Jar) {

‎dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerRecordingData.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import java.nio.file.Path;
88
import java.time.Instant;
99
import javax.annotation.Nonnull;
10+
import javax.annotation.Nullable;
1011

1112
final class DatadogProfilerRecordingData extends RecordingData {
1213
private final Path recordingFile;
@@ -36,4 +37,10 @@ public void release() {
3637
public String getName() {
3738
return "ddprof";
3839
}
40+
41+
@Nullable
42+
@Override
43+
public Path getPath() {
44+
return recordingFile;
45+
}
3946
}

‎dd-java-agent/agent-profiling/profiling-scrubber/build.gradle‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
apply from: "$rootDir/gradle/java.gradle"
2+
3+
minimumInstructionCoverage = 0.0
4+
minimumBranchCoverage = 0.0
5+
6+
dependencies {
7+
api libs.slf4j
8+
9+
implementation(libs.jafar.tools) {
10+
// Agent has its own slf4j binding
11+
exclude group: 'org.slf4j', module: 'slf4j-simple'
12+
}
13+
14+
testImplementation libs.bundles.junit5
15+
testImplementation libs.bundles.mockito
16+
testImplementation libs.bundles.jmc
17+
}

‎dd-java-agent/agent-profiling/profiling-scrubber/src/main/java/com/datadog/profiling/scrubber/DefaultScrubDefinition.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package com.datadog.profiling.scrubber;
2+
3+
import io.jafar.tools.Scrubber;
4+
import java.util.Collections;
5+
import java.util.HashMap;
6+
import java.util.HashSet;
7+
import java.util.List;
8+
import java.util.Map;
9+
import java.util.Set;
10+
11+
/** Provides the default scrub definition targeting sensitive JFR event fields. */
12+
public final class DefaultScrubDefinition {
13+
14+
private static final Map<String, Scrubber.ScrubField> DEFAULT_SCRUB_FIELDS;
15+
16+
static {
17+
Map<String, Scrubber.ScrubField> fields = new HashMap<>();
18+
// ScrubField(keyField, valueField, predicate): null keyField = scrub all values unconditionally
19+
// System properties may contain API keys, passwords
20+
fields.put("jdk.InitialSystemProperty", new Scrubber.ScrubField(null, "value", (k, v) -> true));
21+
// JVM args may contain credentials in -D flags
22+
fields.put("jdk.JVMInformation", new Scrubber.ScrubField(null, "jvmArguments", (k, v) -> true));
23+
// Env vars may contain secrets
24+
fields.put(
25+
"jdk.InitialEnvironmentVariable", new Scrubber.ScrubField(null, "value", (k, v) -> true));
26+
// Process command lines may reveal infrastructure
27+
fields.put("jdk.SystemProcess", new Scrubber.ScrubField(null, "commandLine", (k, v) -> true));
28+
DEFAULT_SCRUB_FIELDS = Collections.unmodifiableMap(fields);
29+
}
30+
31+
/**
32+
* Creates a scrubber with the default scrub definition.
33+
*
34+
* @param excludeEventTypes list of event type names to exclude from scrubbing, or null for none
35+
* @return a configured {@link JfrScrubber}
36+
*/
37+
public static JfrScrubber create(List<String> excludeEventTypes) {
38+
Set<String> excludeSet =
39+
excludeEventTypes != null
40+
? new HashSet<>(excludeEventTypes)
41+
: Collections.<String>emptySet();
42+
43+
return new JfrScrubber(
44+
eventTypeName -> {
45+
if (excludeSet.contains(eventTypeName)) {
46+
return null;
47+
}
48+
return DEFAULT_SCRUB_FIELDS.get(eventTypeName);
49+
});
50+
}
51+
52+
private DefaultScrubDefinition() {}
53+
}

‎dd-java-agent/agent-profiling/profiling-scrubber/src/main/java/com/datadog/profiling/scrubber/JfrScrubber.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package com.datadog.profiling.scrubber;
2+
3+
import io.jafar.tools.Scrubber;
4+
import java.nio.file.Path;
5+
import java.util.function.Function;
6+
7+
/**
8+
* Thin wrapper around {@link Scrubber} from jafar-tools, hiding jafar types from consumers outside
9+
* the profiling-scrubber module.
10+
*/
11+
public final class JfrScrubber {
12+
13+
private final Function<String, Scrubber.ScrubField> scrubDefinition;
14+
15+
/** Package-private: use {@link DefaultScrubDefinition#create} to obtain an instance. */
16+
JfrScrubber(Function<String, Scrubber.ScrubField> scrubDefinition) {
17+
this.scrubDefinition = scrubDefinition;
18+
}
19+
20+
/**
21+
* Scrub the given file by replacing targeted field values with 'x' bytes.
22+
*
23+
* @param input the input file to scrub
24+
* @param output the output file to write the scrubbed content to
25+
* @throws Exception if an error occurs during parsing or writing
26+
*/
27+
public void scrubFile(Path input, Path output) throws Exception {
28+
Scrubber.scrubFile(input, output, scrubDefinition);
29+
}
30+
}

‎dd-java-agent/agent-profiling/profiling-scrubber/src/test/java/com/datadog/profiling/scrubber/JfrScrubberTest.java‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
package com.datadog.profiling.scrubber;
2+
3+
import static org.junit.jupiter.api.Assertions.assertEquals;
4+
import static org.junit.jupiter.api.Assertions.assertTrue;
5+
import static org.openjdk.jmc.common.item.Attribute.attr;
6+
import static org.openjdk.jmc.common.unit.UnitLookup.PLAIN_TEXT;
7+
8+
import java.io.IOException;
9+
import java.io.InputStream;
10+
import java.nio.file.Files;
11+
import java.nio.file.Path;
12+
import java.nio.file.StandardCopyOption;
13+
import java.util.Collections;
14+
import org.junit.jupiter.api.BeforeEach;
15+
import org.junit.jupiter.api.Test;
16+
import org.junit.jupiter.api.io.TempDir;
17+
import org.openjdk.jmc.common.item.IAttribute;
18+
import org.openjdk.jmc.common.item.IItem;
19+
import org.openjdk.jmc.common.item.IItemCollection;
20+
import org.openjdk.jmc.common.item.IItemIterable;
21+
import org.openjdk.jmc.common.item.IMemberAccessor;
22+
import org.openjdk.jmc.common.item.ItemFilters;
23+
import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit;
24+
25+
class JfrScrubberTest {
26+
27+
@TempDir Path tempDir;
28+
29+
private Path inputFile;
30+
31+
@BeforeEach
32+
void setUp() throws IOException {
33+
inputFile = tempDir.resolve("input.jfr");
34+
try (InputStream is = getClass().getResourceAsStream("/test-recording.jfr")) {
35+
if (is == null) {
36+
throw new IllegalStateException("test-recording.jfr not found in test resources");
37+
}
38+
Files.copy(is, inputFile, StandardCopyOption.REPLACE_EXISTING);
39+
}
40+
}
41+
42+
@Test
43+
void scrubInitialSystemPropertyValues() throws Exception {
44+
JfrScrubber scrubber = DefaultScrubDefinition.create(null);
45+
Path outputFile = tempDir.resolve("output.jfr");
46+
scrubber.scrubFile(inputFile, outputFile);
47+
48+
assertTrue(Files.exists(outputFile));
49+
assertTrue(Files.size(outputFile) > 0, "Scrubbed file should not be empty");
50+
51+
// Verify scrubbed values contain only 'x' characters
52+
IItemCollection events = JfrLoaderToolkit.loadEvents(outputFile.toFile());
53+
IItemCollection systemPropertyEvents =
54+
events.apply(ItemFilters.type("jdk.InitialSystemProperty"));
55+
assertTrue(systemPropertyEvents.hasItems(), "Expected jdk.InitialSystemProperty events");
56+
57+
IAttribute<String> valueAttr = attr("value", "value", "value", PLAIN_TEXT);
58+
for (IItemIterable itemIterable : systemPropertyEvents) {
59+
IMemberAccessor<String, IItem> accessor = valueAttr.getAccessor(itemIterable.getType());
60+
for (IItem item : itemIterable) {
61+
String value = accessor.getMember(item);
62+
if (value != null && !value.isEmpty()) {
63+
assertTrue(
64+
value.chars().allMatch(c -> c == 'x'),
65+
"System property value should be scrubbed: " + value);
66+
}
67+
}
68+
}
69+
}
70+
71+
@Test
72+
void scrubWithNoMatchingEvents() throws Exception {
73+
// Scrubber with all default events excluded — nothing matches
74+
JfrScrubber scrubber = new JfrScrubber(name -> null);
75+
Path outputFile = tempDir.resolve("output.jfr");
76+
scrubber.scrubFile(inputFile, outputFile);
77+
78+
// Output should be identical to input when no events match
79+
assertEquals(Files.size(inputFile), Files.size(outputFile));
80+
}
81+
82+
@Test
83+
void scrubWithExcludedEventType() throws Exception {
84+
// Exclude jdk.InitialSystemProperty from scrubbing
85+
JfrScrubber scrubber =
86+
DefaultScrubDefinition.create(Collections.singletonList("jdk.InitialSystemProperty"));
87+
Path outputFile = tempDir.resolve("output.jfr");
88+
scrubber.scrubFile(inputFile, outputFile);
89+
90+
assertTrue(Files.exists(outputFile));
91+
assertTrue(Files.size(outputFile) > 0);
92+
93+
// Verify excluded event type values are preserved (not scrubbed to 'x')
94+
IItemCollection events = JfrLoaderToolkit.loadEvents(outputFile.toFile());
95+
IItemCollection systemPropertyEvents =
96+
events.apply(ItemFilters.type("jdk.InitialSystemProperty"));
97+
assertTrue(systemPropertyEvents.hasItems(), "Expected jdk.InitialSystemProperty events");
98+
99+
IAttribute<String> valueAttr = attr("value", "value", "value", PLAIN_TEXT);
100+
boolean foundNonTrivialValue = false;
101+
for (IItemIterable itemIterable : systemPropertyEvents) {
102+
IMemberAccessor<String, IItem> accessor = valueAttr.getAccessor(itemIterable.getType());
103+
for (IItem item : itemIterable) {
104+
String value = accessor.getMember(item);
105+
if (value != null && !value.isEmpty()) {
106+
// At least one value should NOT be all-x (proving exclusion worked)
107+
if (!value.chars().allMatch(c -> c == 'x')) {
108+
foundNonTrivialValue = true;
109+
}
110+
}
111+
}
112+
}
113+
assertTrue(
114+
foundNonTrivialValue, "Excluded event type values should be preserved, not scrubbed");
115+
}
116+
}

‎dd-java-agent/agent-profiling/profiling-scrubber/src/test/resources/test-recording.jfr‎

3.54 MB
Binary file not shown.

‎dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ProfilingAgent.java‎

Lines changed: 35 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22

33
import static datadog.environment.JavaVirtualMachine.isJavaVersion;
44
import static datadog.environment.JavaVirtualMachine.isJavaVersionAtLeast;
5+
import static datadog.trace.api.config.ProfilingConfig.PROFILING_SCRUB_ENABLED;
6+
import static datadog.trace.api.config.ProfilingConfig.PROFILING_SCRUB_ENABLED_DEFAULT;
7+
import static datadog.trace.api.config.ProfilingConfig.PROFILING_SCRUB_FAIL_OPEN;
8+
import static datadog.trace.api.config.ProfilingConfig.PROFILING_SCRUB_FAIL_OPEN_DEFAULT;
59
import static datadog.trace.api.config.ProfilingConfig.PROFILING_START_FORCE_FIRST;
610
import static datadog.trace.api.config.ProfilingConfig.PROFILING_START_FORCE_FIRST_DEFAULT;
711
import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY;
@@ -32,6 +36,7 @@
3236
import java.nio.file.Paths;
3337
import java.nio.file.StandardCopyOption;
3438
import java.time.Duration;
39+
import java.util.List;
3540
import java.util.concurrent.atomic.AtomicBoolean;
3641
import java.util.function.Predicate;
3742
import java.util.regex.Pattern;
@@ -137,6 +142,25 @@ public static synchronized boolean run(final boolean earlyStart, Instrumentation
137142

138143
uploader = new ProfileUploader(config, configProvider);
139144

145+
RecordingDataListener listener = uploader::upload;
146+
if (dumper != null) {
147+
RecordingDataListener upload = listener;
148+
listener =
149+
(type, data, sync) -> {
150+
dumper.onNewData(type, data, sync);
151+
upload.onNewData(type, data, sync);
152+
};
153+
}
154+
// Scrubber wraps the combined dumper+uploader so debug dumps also contain scrubbed data
155+
if (configProvider.getBoolean(PROFILING_SCRUB_ENABLED, PROFILING_SCRUB_ENABLED_DEFAULT)) {
156+
List<String> excludeEventTypes =
157+
configProvider.getList(ProfilingConfig.PROFILING_SCRUB_EXCLUDE_EVENTS);
158+
boolean failOpen =
159+
configProvider.getBoolean(
160+
PROFILING_SCRUB_FAIL_OPEN, PROFILING_SCRUB_FAIL_OPEN_DEFAULT);
161+
listener = wrapWithScrubber(listener, excludeEventTypes, failOpen);
162+
}
163+
140164
final Duration startupDelay = Duration.ofSeconds(config.getProfilingStartDelay());
141165
final Duration uploadPeriod = Duration.ofSeconds(config.getProfilingUploadPeriod());
142166

@@ -149,12 +173,7 @@ public static synchronized boolean run(final boolean earlyStart, Instrumentation
149173
configProvider,
150174
controller,
151175
context.snapshot(),
152-
dumper == null
153-
? uploader::upload
154-
: (type, data, sync) -> {
155-
dumper.onNewData(type, data, sync);
156-
uploader.upload(type, data, sync);
157-
},
176+
listener,
158177
startupDelay,
159178
startupDelayRandomRange,
160179
uploadPeriod,
@@ -181,6 +200,16 @@ public static synchronized boolean run(final boolean earlyStart, Instrumentation
181200
return false;
182201
}
183202

203+
private static RecordingDataListener wrapWithScrubber(
204+
RecordingDataListener listener, List<String> excludeEventTypes, boolean failOpen) {
205+
try {
206+
return ScrubRecordingDataListener.wrap(listener, excludeEventTypes, failOpen);
207+
} catch (Exception e) {
208+
log.warn(SEND_TELEMETRY, "Failed to initialize JFR scrubber", e);
209+
return listener;
210+
}
211+
}
212+
184213
private static boolean isStartForceFirstSafe() {
185214
return isJavaVersionAtLeast(14)
186215
|| (isJavaVersion(13) && isJavaVersionAtLeast(13, 0, 4))

0 commit comments

Comments
 (0)