Skip to content

Commit 04daf16

Browse files
Voronenkozandrewitte
Voronenko
and
zandrewitte
authored
Config fallback (#51)
* improve(config) added fallback to process_sts_url or sts_url if specified * improve(config) added fallback to process_sts_url or sts_url if specified * improve(config) added fallback to process_sts_url or sts_url if specified * improve(config) added fallback to process_sts_url or sts_url if specified * addressing PR comments * addressed PR comments * addressed PR comments * chore(PR comments) removed duplication typo * go fmt * config fallback tests, give precedence to process specific config + env * add one more fallback test case, add logging on url set * fix logging * reconfigure logger * dev config * environment takes precedence over config Co-authored-by: zandrewitte <zandre.witte@gmail.com>
1 parent 689cc3a commit 04daf16

6 files changed

Lines changed: 284 additions & 36 deletions

File tree

‎checks/net_common.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,6 @@ func createRelationIdentifier(localEndpoint, remoteEndpoint *endpointID, directi
7474
}
7575
}
7676

77-
7877
// makeEndpointID returns a endpointID if the ip is valid and the hostname as the scope for local ips
7978
func makeEndpointID(namespace string, ipString string, isV6 bool, port int32) *endpointID {
8079
// We parse the ip here for normalization
@@ -99,8 +98,8 @@ func makeEndpointID(namespace string, ipString string, isV6 bool, port int32) *e
9998
// Represents the namespace part of connection identity. The connection namespace
10099
// determines its locality (e.g. the scope in which the network resides)
101100
type namespace struct {
102-
ClusterName string
103-
HostName string
101+
ClusterName string
102+
HostName string
104103
NetworkNamespace string
105104
}
106105

@@ -120,7 +119,7 @@ func (ns namespace) toString() string {
120119

121120
func makeNamespace(clusterName string, hostname string, connection common.ConnectionStats) namespace {
122121
// check if we're running in kubernetes, prepend the namespace with the kubernetes / openshift cluster name
123-
var ns = namespace { "", "", ""}
122+
var ns = namespace{"", "", ""}
124123
if clusterName != "" {
125124
ns.ClusterName = clusterName
126125
}
@@ -131,7 +130,7 @@ func makeNamespace(clusterName string, hostname string, connection common.Connec
131130
// https://github.com/weaveworks/scope/blob/7163f42170d72702fd55d2324d203c5b7be5c5cc/probe/endpoint/ebpf.go#L34
132131
// We disregard local ip addresses for now, those might be interesting when doing docker setups,
133132
// which are not the highest priority atm
134-
if (isLoopback(connection.Local) && isLoopback(connection.Remote)) {
133+
if isLoopback(connection.Local) && isLoopback(connection.Remote) {
135134
// For sure this is scoped to the host
136135
ns.HostName = hostname
137136
// Maybe even to a namespace on the host in case of k8s/docker containers

‎checks/net_test.go‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -80,18 +80,18 @@ func TestNetworkConnectionMax(t *testing.T) {
8080

8181
func makeConnectionStats(pid uint32, local, remote string, localPort, remotePort uint16) common.ConnectionStats {
8282
return common.ConnectionStats{
83-
Pid: pid,
84-
Type: common.TCP,
85-
Family: common.AF_INET,
86-
Direction: common.OUTGOING,
87-
Local: local,
88-
Remote: remote,
89-
LocalPort: localPort,
90-
RemotePort: remotePort,
83+
Pid: pid,
84+
Type: common.TCP,
85+
Family: common.AF_INET,
86+
Direction: common.OUTGOING,
87+
Local: local,
88+
Remote: remote,
89+
LocalPort: localPort,
90+
RemotePort: remotePort,
9191
NetworkNamespace: "ns",
92-
SendBytes: 0,
93-
RecvBytes: 0,
94-
State: common.ACTIVE,
92+
SendBytes: 0,
93+
RecvBytes: 0,
94+
State: common.ACTIVE,
9595
}
9696
}
9797

‎conf-dev.yaml‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,19 @@
11
log_level: TRACE
2+
log_to_console: true
3+
# Use this when running process agent locally
4+
#sts_url: http://localhost:7077/stsAgent
25

6+
# Use this when running process agent in the vagrant vms
7+
sts_url: http://192.168.56.1:7077/stsAgent
8+
9+
# Use this when running process agent with branch deploy
10+
#sts_url: https://<branch-name>.test.stackstate.io/stsAgent
311
api_key: API_KEY
412
process_agent_enabled: true
513
incremental_publishing_enabled: true
614
process_config:
715
enabled: 'true'
816

9-
# Use this when running process agent locally
10-
#process_dd_url: http://localhost:7077/stsAgent
11-
12-
# Use this when running process agent in the vagrant vms
13-
process_dd_url: http://192.168.56.1:7077/stsAgent
14-
15-
# Use this when running process agent with branch deploy
16-
#process_dd_url: https://<branch-name>.test.stackstate.io/stsAgent
17-
1817
queue_size: 10
1918
intervals:
2019
container: 8

‎config/config.go‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ func (a AgentConfig) CheckInterval(checkName string) time.Duration {
152152
}
153153

154154
const (
155-
defaultEndpoint = "https://process.datadoghq.com"
155+
defaultEndpoint = "http://localhost:7077/stsAgent"
156156
maxMessageBatch = 100
157157
)
158158

@@ -552,6 +552,7 @@ func mergeEnvironmentVariables(c *AgentConfig) *AgentConfig {
552552
c.proxy = nil
553553
}
554554

555+
// STS
555556
if v := os.Getenv("DD_PROCESS_AGENT_URL"); v != "" {
556557
u, err := url.Parse(v)
557558
if err != nil {
@@ -563,7 +564,19 @@ func mergeEnvironmentVariables(c *AgentConfig) *AgentConfig {
563564
if site := os.Getenv("DD_SITE"); site != "" {
564565
log.Infof("Using 'process_dd_url' (%s) and ignoring 'site' (%s)", v, site)
565566
}
567+
log.Infof("Overriding process api endpoint with environment variable `STS_PROCESS_AGENT_URL`: %s", u)
568+
} else if v := os.Getenv("STS_STS_URL"); v != "" {
569+
// check if we don't already have a api endpoint configured, specific process configuration takes precedence.
570+
u, err := url.Parse(v)
571+
if err != nil {
572+
log.Warnf("STS_STS_URL is invalid: %s", err)
573+
} else {
574+
log.Infof("overriding API endpoint from env STS_STS_URL")
575+
c.APIEndpoints[0].Endpoint = u
576+
}
577+
log.Infof("Overriding process api endpoint with environment variable `STS_STS_URL`: %s", u)
566578
}
579+
// /STS
567580

568581
// Process Arguments Scrubbing
569582
if enabled, err := isAffirmative(os.Getenv("DD_SCRUB_ARGS")); enabled {

‎config/config_test.go‎

Lines changed: 214 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,6 @@ func TestBlacklist(t *testing.T) {
7272
}
7373
}
7474

75-
7675
func TestBlacklistIncludeOnly(t *testing.T) {
7776
testBlacklist := []string{
7877
"^[^bla].*",
@@ -449,7 +448,9 @@ func TestDDAgentMultiAPIKeys(t *testing.T) {
449448
assert.NoError(err)
450449
assert.Equal(1, len(agentConfig.APIEndpoints))
451450
assert.Equal("foo", agentConfig.APIEndpoints[0].APIKey)
452-
assert.Equal("process.datadoghq.com", agentConfig.APIEndpoints[0].Endpoint.Hostname())
451+
de, err := url.Parse(defaultEndpoint)
452+
assert.NoError(err)
453+
assert.Equal(de.Hostname(), agentConfig.APIEndpoints[0].Endpoint.Hostname())
453454

454455
ddAgentConf, _ = ini.Load([]byte(strings.Join([]string{
455456
"[Main]",
@@ -1068,3 +1069,214 @@ func TestIsAffirmative(t *testing.T) {
10681069
assert.Nil(t, err)
10691070
assert.False(t, value)
10701071
}
1072+
1073+
//custom tests
1074+
1075+
func TestStackStateFallbackAgentConfigToProcessSTSUrl(t *testing.T) {
1076+
assert := assert.New(t)
1077+
os.Unsetenv("DD_PROCESS_AGENT_URL")
1078+
var ddy YamlAgentConfig
1079+
err := yaml.Unmarshal([]byte(strings.Join([]string{
1080+
"api_key: apikey_30",
1081+
"sts_url: http://default-endpoint.test.stackstate.com",
1082+
"process_agent_enabled: true",
1083+
"process_config:",
1084+
" enabled: 'true'",
1085+
" process_sts_url: http://process-endpoint.test.stackstate.com",
1086+
" queue_size: 10",
1087+
" intervals:",
1088+
" container: 8",
1089+
" process: 30",
1090+
"network_tracer_config:",
1091+
" network_tracing_enabled: 'true'",
1092+
" initial_connections_from_proc: 'true'",
1093+
}, "\n")), &ddy)
1094+
assert.NoError(err)
1095+
1096+
agentConfig, err := NewAgentConfig(nil, &ddy, nil)
1097+
assert.NoError(err)
1098+
1099+
ep := agentConfig.APIEndpoints[0]
1100+
assert.Equal("apikey_30", ep.APIKey)
1101+
assert.Equal("process-endpoint.test.stackstate.com", ep.Endpoint.Hostname())
1102+
}
1103+
1104+
func TestStackStateFallbackAgentConfigToSTSUrl(t *testing.T) {
1105+
assert := assert.New(t)
1106+
os.Unsetenv("DD_PROCESS_AGENT_URL")
1107+
var ddy YamlAgentConfig
1108+
err := yaml.Unmarshal([]byte(strings.Join([]string{
1109+
"api_key: apikey_30",
1110+
"sts_url: http://default-endpoint.test.stackstate.com",
1111+
"process_agent_enabled: true",
1112+
"process_config:",
1113+
" enabled: 'true'",
1114+
" queue_size: 10",
1115+
" intervals:",
1116+
" container: 8",
1117+
" process: 30",
1118+
"network_tracer_config:",
1119+
" network_tracing_enabled: 'true'",
1120+
" initial_connections_from_proc: 'true'",
1121+
}, "\n")), &ddy)
1122+
assert.NoError(err)
1123+
1124+
agentConfig, err := NewAgentConfig(nil, &ddy, nil)
1125+
assert.NoError(err)
1126+
1127+
ep := agentConfig.APIEndpoints[0]
1128+
assert.Equal("apikey_30", ep.APIKey)
1129+
assert.Equal("default-endpoint.test.stackstate.com", ep.Endpoint.Hostname())
1130+
}
1131+
1132+
func TestStackStateFallbackAgentConfigToEnvSTSUrl(t *testing.T) {
1133+
assert := assert.New(t)
1134+
os.Unsetenv("DD_PROCESS_AGENT_URL")
1135+
os.Unsetenv("STS_STS_URL")
1136+
os.Setenv("STS_STS_URL", "http://default-endpoint.test.stackstate.com")
1137+
var ddy YamlAgentConfig
1138+
err := yaml.Unmarshal([]byte(strings.Join([]string{
1139+
"api_key: apikey_30",
1140+
"process_agent_enabled: true",
1141+
"process_config:",
1142+
" enabled: 'true'",
1143+
" queue_size: 10",
1144+
" intervals:",
1145+
" container: 8",
1146+
" process: 30",
1147+
"network_tracer_config:",
1148+
" network_tracing_enabled: 'true'",
1149+
" initial_connections_from_proc: 'true'",
1150+
}, "\n")), &ddy)
1151+
assert.NoError(err)
1152+
1153+
agentConfig, err := NewAgentConfig(nil, &ddy, nil)
1154+
assert.NoError(err)
1155+
1156+
ep := agentConfig.APIEndpoints[0]
1157+
assert.Equal("apikey_30", ep.APIKey)
1158+
assert.Equal("default-endpoint.test.stackstate.com", ep.Endpoint.Hostname())
1159+
}
1160+
1161+
func TestStackStateFallbackAgentConfigEmptyUrlToEnvSTSUrl(t *testing.T) {
1162+
assert := assert.New(t)
1163+
os.Unsetenv("DD_PROCESS_AGENT_URL")
1164+
os.Unsetenv("STS_STS_URL")
1165+
os.Setenv("STS_STS_URL", "http://default-endpoint.test.stackstate.com")
1166+
var ddy YamlAgentConfig
1167+
err := yaml.Unmarshal([]byte(strings.Join([]string{
1168+
"api_key: apikey_30",
1169+
"process_agent_enabled: true",
1170+
"process_config:",
1171+
" process_sts_url: ",
1172+
" enabled: 'true'",
1173+
" queue_size: 10",
1174+
" intervals:",
1175+
" container: 8",
1176+
" process: 30",
1177+
"network_tracer_config:",
1178+
" network_tracing_enabled: 'true'",
1179+
" initial_connections_from_proc: 'true'",
1180+
}, "\n")), &ddy)
1181+
assert.NoError(err)
1182+
1183+
agentConfig, err := NewAgentConfig(nil, &ddy, nil)
1184+
assert.NoError(err)
1185+
1186+
ep := agentConfig.APIEndpoints[0]
1187+
assert.Equal("apikey_30", ep.APIKey)
1188+
assert.Equal("default-endpoint.test.stackstate.com", ep.Endpoint.Hostname())
1189+
}
1190+
1191+
//case 5: STS_URL as env PROCESS_AGENT_URL as env
1192+
func TestStackStatePreferAgentConfigToEnvPROCESS_AGENT_URL(t *testing.T) {
1193+
assert := assert.New(t)
1194+
os.Unsetenv("DD_PROCESS_AGENT_URL")
1195+
os.Unsetenv("STS_STS_URL")
1196+
os.Setenv("STS_STS_URL", "http://default-endpoint.test.stackstate.com")
1197+
os.Setenv("DD_PROCESS_AGENT_URL", "http://process-endpoint.test.stackstate.com")
1198+
var ddy YamlAgentConfig
1199+
err := yaml.Unmarshal([]byte(strings.Join([]string{
1200+
"api_key: apikey_30",
1201+
"process_agent_enabled: true",
1202+
"process_config:",
1203+
" enabled: 'true'",
1204+
" queue_size: 10",
1205+
" intervals:",
1206+
" container: 8",
1207+
" process: 30",
1208+
"network_tracer_config:",
1209+
" network_tracing_enabled: 'true'",
1210+
" initial_connections_from_proc: 'true'",
1211+
}, "\n")), &ddy)
1212+
assert.NoError(err)
1213+
1214+
agentConfig, err := NewAgentConfig(nil, &ddy, nil)
1215+
assert.NoError(err)
1216+
1217+
ep := agentConfig.APIEndpoints[0]
1218+
assert.Equal("apikey_30", ep.APIKey)
1219+
assert.Equal("process-endpoint.test.stackstate.com", ep.Endpoint.Hostname())
1220+
}
1221+
1222+
//case 7: STS_URL as env PROCESS_AGENT_URL as yaml - STS URL wins, more specific
1223+
func TestStackStatePreferSTS_STS_URLOverYamlProcessAgentConfig(t *testing.T) {
1224+
assert := assert.New(t)
1225+
os.Unsetenv("DD_PROCESS_AGENT_URL")
1226+
os.Unsetenv("STS_STS_URL")
1227+
os.Setenv("STS_STS_URL", "http://default-endpoint.test.stackstate.com")
1228+
var ddy YamlAgentConfig
1229+
err := yaml.Unmarshal([]byte(strings.Join([]string{
1230+
"api_key: apikey_30",
1231+
"process_agent_enabled: true",
1232+
"process_config:",
1233+
" enabled: 'true'",
1234+
" process_sts_url: http://process-endpoint.test.stackstate.com",
1235+
" queue_size: 10",
1236+
" intervals:",
1237+
" container: 8",
1238+
" process: 30",
1239+
"network_tracer_config:",
1240+
" network_tracing_enabled: 'true'",
1241+
" initial_connections_from_proc: 'true'",
1242+
}, "\n")), &ddy)
1243+
assert.NoError(err)
1244+
1245+
agentConfig, err := NewAgentConfig(nil, &ddy, nil)
1246+
assert.NoError(err)
1247+
1248+
ep := agentConfig.APIEndpoints[0]
1249+
assert.Equal("apikey_30", ep.APIKey)
1250+
assert.Equal("default-endpoint.test.stackstate.com", ep.Endpoint.Hostname())
1251+
}
1252+
1253+
//case 8: STS_URL as yaml, PROCESS_AGENT_URL as env - ENV wins
1254+
func TestStackStatePreferPROCESS_AGENT_URLOverYamlsts_sts_url(t *testing.T) {
1255+
assert := assert.New(t)
1256+
os.Unsetenv("STS_PROCESS_AGENT_URL")
1257+
os.Unsetenv("STS_STS_URL")
1258+
os.Setenv("DD_PROCESS_AGENT_URL", "http://process-endpoint.test.stackstate.com")
1259+
var ddy YamlAgentConfig
1260+
err := yaml.Unmarshal([]byte(strings.Join([]string{
1261+
"api_key: apikey_30",
1262+
"sts_url: http://default-endpoint.test.stackstate.com",
1263+
"process_agent_enabled: true",
1264+
"process_config:",
1265+
" enabled: 'true'",
1266+
" queue_size: 10",
1267+
" intervals:",
1268+
" container: 8",
1269+
" process: 30",
1270+
"network_tracer_config:",
1271+
" network_tracing_enabled: 'true'",
1272+
" initial_connections_from_proc: 'true'",
1273+
}, "\n")), &ddy)
1274+
assert.NoError(err)
1275+
1276+
agentConfig, err := NewAgentConfig(nil, &ddy, nil)
1277+
assert.NoError(err)
1278+
1279+
ep := agentConfig.APIEndpoints[0]
1280+
assert.Equal("apikey_30", ep.APIKey)
1281+
assert.Equal("process-endpoint.test.stackstate.com", ep.Endpoint.Hostname())
1282+
}

0 commit comments

Comments
 (0)