Skip to content

Commit 93bae7d

Browse files
committed
Merge branch 'master' into paullgdc/agent_bootstrap/inject_when_log4j_patch
2 parents 5670cc9 + 19cd36d commit 93bae7d

7 files changed

Lines changed: 193 additions & 76 deletions

File tree

.circleci/config.continue.yml.j2

Lines changed: 4 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
3636
debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
3737
profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
3838

39-
default_system_tests_commit: &default_system_tests_commit 08276e905c5c81055412ea4c318856d879cf15e5
39+
default_system_tests_commit: &default_system_tests_commit 380d8deb4dd86369e30827ae3d908400c91d2d3e
4040

4141
parameters:
4242
nightly:
@@ -850,8 +850,6 @@ jobs:
850850
(
851851
echo "
852852
DEFAULT
853-
APM_TRACING_E2E
854-
APM_TRACING_E2E_SINGLE_SPAN
855853
TRACING_CONFIG_NONDEFAULT
856854
TRACING_CONFIG_NONDEFAULT_2
857855
TRACING_CONFIG_NONDEFAULT_3
@@ -875,14 +873,8 @@ jobs:
875873
"
876874
fi
877875
) | circleci tests split > scenarios.list
876+
export DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY
878877
for scenario in $(<scenarios.list); do
879-
if [[ $scenario =~ .*_E2E.* ]]; then
880-
export DD_SITE=datadoghq.com
881-
export DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY
882-
export DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY
883-
else
884-
export DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY
885-
fi
886878
echo "Running scenario $scenario"
887879
./run.sh $scenario
888880
done
@@ -922,27 +914,11 @@ jobs:
922914
923915
- run:
924916
name: Run APM Integrations tests
925-
environment:
926-
- AWS_ACCESS_KEY_ID: $SYSTEM_TESTS_IDM_AWS_ACCESS_KEY_ID
927-
- AWS_SECRET_ACCESS_KEY: $SYSTEM_TESTS_IDM_AWS_SECRET_ACCESS_KEY
928-
- AWS_REGION: us-east-1
929-
- AWS_DEFAULT_REGION: us-east-1 # AWS services should use `AWS_REGION`, but some still use the older `AWS_DEFAULT_REGION`
930917
# Stop the job after 5m to avoid excessive overhead. Will need adjustment as more tests are added.
931918
no_output_timeout: 5m
932919
command: |
933920
cd system-tests
934-
DD_SITE=datadoghq.com DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY ./run.sh INTEGRATIONS
935-
936-
- run:
937-
name: Run IDM Crossed Tracing Libraries propagation tests for messaging
938-
environment:
939-
- AWS_ACCESS_KEY_ID: $SYSTEM_TESTS_IDM_AWS_ACCESS_KEY_ID
940-
- AWS_SECRET_ACCESS_KEY: $SYSTEM_TESTS_IDM_AWS_SECRET_ACCESS_KEY
941-
- AWS_REGION: us-east-1
942-
- AWS_DEFAULT_REGION: us-east-1 # AWS services should use `AWS_REGION`, but some still use the older `AWS_DEFAULT_REGION`
943-
command: |
944-
cd system-tests
945-
DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY ./run.sh CROSSED_TRACING_LIBRARIES
921+
DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY ./run.sh INTEGRATIONS
946922
947923
- store_test_results:
948924
path: system-tests/logs_integrations
@@ -975,7 +951,7 @@ jobs:
975951
no_output_timeout: 5m
976952
command: |
977953
cd system-tests
978-
export DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY
954+
export DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY
979955
./run.sh DEBUGGER_SCENARIOS
980956
981957
- run:

.github/workflows/README.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,14 @@ _Action:_ Create a PR updating the Grade dependencies and their locking files.
129129

130130
_Recovery:_ Manually trigger the action again.
131131

132+
### run-system-tests [🔗](run-system-tests.yaml)
133+
134+
_Trigger:_ When pushing commits to `master` or manually.
135+
136+
_Action:_ Build the Java Client Library and runs [the system tests](https://github.com/DataDog/system-tests) against.
137+
138+
_Recovery:_ Manually trigger the action on the desired branch.
139+
132140

133141
## Maintenance
134142

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: system-tests
1+
name: Run system tests
22

33
on:
44
pull_request:
@@ -29,7 +29,7 @@ jobs:
2929
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
3030
restore-keys: |
3131
${{ runner.os }}-gradle-
32-
32+
3333
- name: Build dd-trace-java
3434
run: |
3535
GRADLE_OPTS="-Xms2g -Xmx4g -XX:+HeapDumpOnOutOfMemoryError -XX:+UseParallelGC" \
Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,15 @@
11
package datadog.yaml;
22

3-
import java.io.FileInputStream;
4-
import java.io.IOException;
53
import org.yaml.snakeyaml.Yaml;
64

75
public class YamlParser {
86
// Supports clazz == null for default yaml parsing
9-
public static <T> T parse(String filePath, Class<T> clazz) throws IOException {
7+
public static <T> T parse(String content, Class<T> clazz) {
108
Yaml yaml = new Yaml();
11-
try (FileInputStream fis = new FileInputStream(filePath)) {
12-
if (clazz == null) {
13-
return yaml.load(fis);
14-
} else {
15-
return yaml.loadAs(fis, clazz);
16-
}
9+
if (clazz == null) {
10+
return yaml.load(content);
11+
} else {
12+
return yaml.loadAs(content, clazz);
1713
}
1814
}
1915
}

internal-api/src/main/java/datadog/trace/bootstrap/config/provider/StableConfigParser.java

Lines changed: 100 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,27 @@
11
package datadog.trace.bootstrap.config.provider;
22

3-
import datadog.cli.CLIHelper;
43
import datadog.trace.bootstrap.config.provider.stableconfigyaml.ConfigurationMap;
54
import datadog.trace.bootstrap.config.provider.stableconfigyaml.Rule;
65
import datadog.trace.bootstrap.config.provider.stableconfigyaml.Selector;
76
import datadog.trace.bootstrap.config.provider.stableconfigyaml.StableConfigYaml;
87
import datadog.yaml.YamlParser;
98
import java.io.IOException;
9+
import java.nio.charset.StandardCharsets;
10+
import java.nio.file.Files;
11+
import java.nio.file.Paths;
1012
import java.util.Collections;
1113
import java.util.HashMap;
12-
import java.util.HashSet;
1314
import java.util.List;
14-
import java.util.Set;
1515
import java.util.function.BiPredicate;
1616
import org.slf4j.Logger;
1717
import org.slf4j.LoggerFactory;
1818

1919
public class StableConfigParser {
2020
private static final Logger log = LoggerFactory.getLogger(StableConfigParser.class);
2121

22-
private static final Set<String> VM_ARGS = new HashSet<>(CLIHelper.getVmArgs());
22+
private static final String ENVIRONMENT_VARIABLES_PREFIX = "environment_variables['";
23+
private static final String PROCESS_ARGUMENTS_PREFIX = "process_arguments['";
24+
private static final String UNDEFINED_VALUE = "UNDEFINED";
2325

2426
/**
2527
* Parses a configuration file and returns a stable configuration object.
@@ -37,7 +39,9 @@ public class StableConfigParser {
3739
*/
3840
public static StableConfigSource.StableConfig parse(String filePath) throws IOException {
3941
try {
40-
StableConfigYaml data = YamlParser.parse(filePath, StableConfigYaml.class);
42+
String content = new String(Files.readAllBytes(Paths.get(filePath)), StandardCharsets.UTF_8);
43+
String processedContent = processTemplate(content);
44+
StableConfigYaml data = YamlParser.parse(processedContent, StableConfigYaml.class);
4145

4246
String configId = data.getConfig_id();
4347
ConfigurationMap configMap = data.getApm_configuration_default();
@@ -66,7 +70,9 @@ public static StableConfigSource.StableConfig parse(String filePath) throws IOEx
6670

6771
} catch (IOException e) {
6872
log.debug(
69-
"Stable configuration file either not found or not readable at filepath {}", filePath);
73+
"Stable configuration file either not found or not readable at filepath {}. Error: {}",
74+
filePath,
75+
e.getMessage());
7076
}
7177
return StableConfigSource.StableConfig.EMPTY;
7278
}
@@ -166,14 +172,100 @@ static boolean selectorMatch(String origin, List<String> matches, String operato
166172
return false;
167173
}
168174
case "process_arguments":
169-
// For now, always return true if `key` exists in the JVM Args
170175
// TODO: flesh out the meaning of each operator for process_arguments
171-
return VM_ARGS.contains(key);
176+
if (!key.startsWith("-D")) {
177+
log.warn(
178+
"Ignoring unsupported process_arguments entry in selector match, '{}'. Only system properties specified with the '-D' prefix are supported.",
179+
key);
180+
return false;
181+
}
182+
// Cut the -D prefix
183+
return System.getProperty(key.substring(2)) != null;
172184
case "tags":
173185
// TODO: Support this down the line (Must define the source of "tags" first)
174186
return false;
175187
default:
176188
return false;
177189
}
178190
}
191+
192+
static String processTemplate(String content) throws IOException {
193+
// Do nothing if there are no variables to process
194+
int openIndex = content.indexOf("{{");
195+
if (openIndex == -1) {
196+
return content;
197+
}
198+
199+
StringBuilder result = new StringBuilder(content.length());
200+
201+
// Add everything before the opening braces
202+
result.append(content, 0, openIndex);
203+
204+
while (true) {
205+
206+
// Find the closing braces
207+
int closeIndex = content.indexOf("}}", openIndex);
208+
if (closeIndex == -1) {
209+
throw new IOException("Unterminated template in config");
210+
}
211+
212+
// Extract the template variable
213+
String templateVar = content.substring(openIndex + 2, closeIndex).trim();
214+
215+
// Process the template variable and get its value
216+
String value = processTemplateVar(templateVar);
217+
218+
// Add the processed value
219+
result.append(value);
220+
221+
// Continue with the next template variable
222+
openIndex = content.indexOf("{{", closeIndex);
223+
if (openIndex == -1) {
224+
// Stop and add everything left after the final closing braces
225+
result.append(content, closeIndex + 2, content.length());
226+
break;
227+
} else {
228+
// Add everything between the last braces and the next
229+
result.append(content, closeIndex + 2, openIndex);
230+
}
231+
}
232+
233+
return result.toString();
234+
}
235+
236+
private static String processTemplateVar(String templateVar) throws IOException {
237+
if (templateVar.startsWith(ENVIRONMENT_VARIABLES_PREFIX) && templateVar.endsWith("']")) {
238+
String envVar =
239+
templateVar
240+
.substring(ENVIRONMENT_VARIABLES_PREFIX.length(), templateVar.length() - 2)
241+
.trim();
242+
if (envVar.isEmpty()) {
243+
throw new IOException("Empty environment variable name in template");
244+
}
245+
String value = System.getenv(envVar.toUpperCase());
246+
if (value == null || value.isEmpty()) {
247+
return UNDEFINED_VALUE;
248+
}
249+
return value;
250+
} else if (templateVar.startsWith(PROCESS_ARGUMENTS_PREFIX) && templateVar.endsWith("']")) {
251+
String processArg =
252+
templateVar.substring(PROCESS_ARGUMENTS_PREFIX.length(), templateVar.length() - 2).trim();
253+
if (processArg.isEmpty()) {
254+
throw new IOException("Empty process argument in template");
255+
}
256+
if (!processArg.startsWith("-D")) {
257+
log.warn(
258+
"Ignoring unsupported process_arguments entry in template variable, '{}'. Only system properties specified with the '-D' prefix are supported.",
259+
processArg);
260+
return UNDEFINED_VALUE;
261+
}
262+
String value = System.getProperty(processArg.substring(2));
263+
if (value == null || value.isEmpty()) {
264+
return UNDEFINED_VALUE;
265+
}
266+
return value;
267+
} else {
268+
return UNDEFINED_VALUE;
269+
}
270+
}
179271
}

0 commit comments

Comments
 (0)