Skip to content

Commit 48c7394

Browse files
marpearsmarpears
committed
Updated sourceType examples in the docs, api and tests to use a pod label. Added an API validation test where sourceType is templated, and fixed single quote bug in API validation tests during oc apply.
1 parent f79124e commit 48c7394

13 files changed

Lines changed: 170 additions & 33 deletions

‎api/observability/v1/output_types.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1346,13 +1346,15 @@ type Splunk struct {
13461346
//
13471347
// Examples:
13481348
//
1349-
// 1. "log4j"
1349+
// 1. {.kubernetes.labels."splunk/sourcetype"||"generic_single_line"}
13501350
//
1351-
// 2. foo-{.bar||"none"}
1351+
// 2. "log4j"
13521352
//
1353-
// 3. {.foo||.bar||"missing"}
1353+
// 3. foo-{.bar||"none"}
13541354
//
1355-
// 4. foo.{.bar.baz||.qux.quux.corge||.grault||"nil"}-waldo.fred{.plugh||"none"}
1355+
// 4. {.foo||.bar||"missing"}
1356+
//
1357+
// 5. foo.{.bar.baz||.qux.quux.corge||.grault||"nil"}-waldo.fred{.plugh||"none"}
13561358
//
13571359
// +kubebuilder:validation:Optional
13581360
// +kubebuilder:validation:Pattern:=`^(([a-zA-Z0-9-_.\/])*(\{(\.[a-zA-Z0-9_]+|\."[^"]+")+((\|\|)(\.[a-zA-Z0-9_]+|\.?"[^"]+")+)*\|\|"[^"]*"\})*)*$`

‎bundle/manifests/cluster-logging.clusterserviceversion.yaml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1957,13 +1957,15 @@ spec:
19571957
19581958
Examples:
19591959
1960-
1. "log4j"
1960+
1. {.kubernetes.labels."splunk/sourcetype"||"generic_single_line"}
19611961
1962-
2. foo-{.bar||"none"}
1962+
2. "log4j"
19631963
1964-
3. {.foo||.bar||"missing"}
1964+
3. foo-{.bar||"none"}
19651965
1966-
4. foo.{.bar.baz||.qux.quux.corge||.grault||"nil"}-waldo.fred{.plugh||"none"}
1966+
4. {.foo||.bar||"missing"}
1967+
1968+
5. foo.{.bar.baz||.qux.quux.corge||.grault||"nil"}-waldo.fred{.plugh||"none"}
19671969
displayName: SourceType
19681970
path: outputs[0].splunk.sourceType
19691971
x-descriptors:

‎bundle/manifests/observability.openshift.io_clusterlogforwarders.yaml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3907,13 +3907,15 @@ spec:
39073907
39083908
Examples:
39093909
3910-
1. "log4j"
3910+
1. {.kubernetes.labels."splunk/sourcetype"||"generic_single_line"}
39113911
3912-
2. foo-{.bar||"none"}
3912+
2. "log4j"
39133913
3914-
3. {.foo||.bar||"missing"}
3914+
3. foo-{.bar||"none"}
39153915
3916-
4. foo.{.bar.baz||.qux.quux.corge||.grault||"nil"}-waldo.fred{.plugh||"none"}
3916+
4. {.foo||.bar||"missing"}
3917+
3918+
5. foo.{.bar.baz||.qux.quux.corge||.grault||"nil"}-waldo.fred{.plugh||"none"}
39173919
pattern: ^(([a-zA-Z0-9-_.\/])*(\{(\.[a-zA-Z0-9_]+|\."[^"]+")+((\|\|)(\.[a-zA-Z0-9_]+|\.?"[^"]+")+)*\|\|"[^"]*"\})*)*$
39183920
type: string
39193921
tuning:

‎config/crd/bases/observability.openshift.io_clusterlogforwarders.yaml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3907,13 +3907,15 @@ spec:
39073907
39083908
Examples:
39093909
3910-
1. "log4j"
3910+
1. {.kubernetes.labels."splunk/sourcetype"||"generic_single_line"}
39113911
3912-
2. foo-{.bar||"none"}
3912+
2. "log4j"
39133913
3914-
3. {.foo||.bar||"missing"}
3914+
3. foo-{.bar||"none"}
39153915
3916-
4. foo.{.bar.baz||.qux.quux.corge||.grault||"nil"}-waldo.fred{.plugh||"none"}
3916+
4. {.foo||.bar||"missing"}
3917+
3918+
5. foo.{.bar.baz||.qux.quux.corge||.grault||"nil"}-waldo.fred{.plugh||"none"}
39173919
pattern: ^(([a-zA-Z0-9-_.\/])*(\{(\.[a-zA-Z0-9_]+|\."[^"]+")+((\|\|)(\.[a-zA-Z0-9_]+|\.?"[^"]+")+)*\|\|"[^"]*"\})*)*$
39183920
type: string
39193921
tuning:

‎config/manifests/bases/cluster-logging.clusterserviceversion.yaml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1880,13 +1880,15 @@ spec:
18801880
18811881
Examples:
18821882
1883-
1. "log4j"
1883+
1. {.kubernetes.labels."splunk/sourcetype"||"generic_single_line"}
18841884
1885-
2. foo-{.bar||"none"}
1885+
2. "log4j"
18861886
1887-
3. {.foo||.bar||"missing"}
1887+
3. foo-{.bar||"none"}
18881888
1889-
4. foo.{.bar.baz||.qux.quux.corge||.grault||"nil"}-waldo.fred{.plugh||"none"}
1889+
4. {.foo||.bar||"missing"}
1890+
1891+
5. foo.{.bar.baz||.qux.quux.corge||.grault||"nil"}-waldo.fred{.plugh||"none"}
18901892
displayName: SourceType
18911893
path: outputs[0].splunk.sourceType
18921894
x-descriptors:

‎docs/features/logforwarding/outputs/splunk-forwarding.adoc‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,9 @@ spec:
3737
url: 'http://example-splunk-hec-service:8088' # <2>
3838
index: '{.log_type||"main"}' # <3>
3939
source: '{.log_source||"undefined"}' # <4>
40-
sourceType: '{.log_source||"undefined"}' # <5>
40+
sourceType: '{.kubernetes.labels."splunk/sourcetype"||"generic_single_line"}' # <5>
4141
indexedFields: ['.log_type', '.log_source'] # <6>
42-
payloadKey: '.kubernetes' # <7>
42+
payloadKey: '.message' # <7>
4343
tuning:
4444
compression: gzip # <8>
4545
pipelines:
@@ -85,7 +85,7 @@ metadata:
8585
name: myapp
8686
labels:
8787
app: myapp
88-
splunk_sourcetype: log4j
88+
splunk/sourcetype: log4j
8989
spec:
9090
containers:
9191
- name: myapp
@@ -125,7 +125,7 @@ spec:
125125
key: hecToken
126126
secretName: splunk-secret
127127
payloadKey: .message
128-
sourceType: '{.kubernetes.labels.splunk_sourcetype||"generic_single_line"}'
128+
sourceType: '{.kubernetes.labels."splunk/sourcetype"||"generic_single_line"}'
129129
url: http://splunk.customer.com:8088
130130
type: splunk
131131
pipelines:

‎internal/generator/vector/output/splunk/splunk_sink_with_payloadkey_and_sourcetype.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ if ._internal.log_type == "audit" {
2525
._internal.splunk.source = ._internal.log_source
2626
}
2727
payloadKey = ["message"]
28-
sourceType = to_string!(._internal.foo||"missing")
28+
sourceType = to_string!(._internal.kubernetes.labels."splunk/sourcetype"||"generic_single_line")
2929
if !is_null(payloadKey) {
3030
value = get!(., payloadKey)
3131
if !is_null(value) {

‎internal/generator/vector/output/splunk/splunk_sink_with_payloadkey_and_static_sourcetype.toml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
[transforms.splunk_hec_timestamp]
2+
type = "remap"
3+
inputs = ["pipelineName"]
4+
source = '''
5+
ts, err = parse_timestamp(._internal.timestamp,"%+")
6+
if err != null {
7+
log("could not parse timestamp. err=" + err, rate_limit_secs: 0)
8+
} else {
9+
._internal.timestamp = ts
10+
}
11+
'''
12+
13+
[transforms.splunk_hec_metadata]
14+
type = "remap"
15+
inputs = ["splunk_hec_timestamp"]
16+
source = '''
17+
# Splunk 'source' field detection
18+
if ._internal.log_type == "infrastructure" && ._internal.log_source == "node" {
19+
._internal.splunk.source = to_string!(._internal.systemd.u.SYSLOG_IDENTIFIER || "")
20+
}
21+
if ._internal.log_source == "container" {
22+
._internal.splunk.source = join!([._internal.kubernetes.namespace_name, ._internal.kubernetes.pod_name, ._internal.kubernetes.container_name], "_")
23+
}
24+
if ._internal.log_type == "audit" {
25+
._internal.splunk.source = ._internal.log_source
26+
}
27+
payloadKey = ["message"]
28+
sourceType = "custom-type"
29+
if !is_null(payloadKey) {
30+
value = get!(., payloadKey)
31+
if !is_null(value) {
32+
internal = ._internal
33+
. = {}
34+
. = set!(., payloadKey, value)
35+
._internal = internal
36+
._internal.splunk.sourcetype = sourceType
37+
} else {
38+
._internal.splunk.sourcetype = "_json"
39+
}
40+
}
41+
'''
42+
43+
[sinks.splunk_hec]
44+
type = "splunk_hec_logs"
45+
inputs = ["splunk_hec_metadata"]
46+
endpoint = "https://splunk-web:8088/endpoint"
47+
default_token = "SECRET[kubernetes_secret.vector-splunk-secret/hecToken]"
48+
timestamp_key = "._internal.timestamp"
49+
source = "{{ ._internal.splunk.source }}"
50+
sourcetype = "{{ ._internal.splunk.sourcetype }}"
51+
host_key = "._internal.hostname"
52+
[sinks.splunk_hec.encoding]
53+
codec = "json"
54+
except_fields = ["_internal"]

‎internal/generator/vector/output/splunk/splunk_test.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,8 +124,12 @@ var _ = Describe("Generating vector config for Splunk output", func() {
124124
Entry("with payloadKey", "splunk_sink_payloadkey.toml", framework.NoOptions, func(spec *obs.OutputSpec) {
125125
spec.Splunk.PayloadKey = ".openshift"
126126
}),
127-
Entry("with payloadKey and sourceType", "splunk_sink_with_payloadkey_and_sourcetype.toml", framework.NoOptions, func(spec *obs.OutputSpec) {
127+
Entry("with payloadKey and dynamic sourceType", "splunk_sink_with_payloadkey_and_sourcetype.toml", framework.NoOptions, func(spec *obs.OutputSpec) {
128128
spec.Splunk.PayloadKey = ".message"
129-
spec.Splunk.SourceType = `{.foo||"missing"}`
129+
spec.Splunk.SourceType = `{.kubernetes.labels."splunk/sourcetype"||"generic_single_line"}`
130+
}),
131+
Entry("with payloadKey and static sourceType", "splunk_sink_with_payloadkey_and_static_sourcetype.toml", framework.NoOptions, func(spec *obs.OutputSpec) {
132+
spec.Splunk.PayloadKey = ".message"
133+
spec.Splunk.SourceType = "custom-type"
130134
}))
131135
})

‎test/e2e/collection/apivalidations/api_validations_test.go‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@ var _ = Describe("", func() {
4040
Fail(err.Error())
4141
}
4242

43-
execCMD := exec.Command("sh", "-c", fmt.Sprintf("echo '%s' | oc -n %s create -f -", crYaml, deployNS))
43+
execCMD := exec.Command("oc", "-n", deployNS, "create", "-f", "-")
44+
execCMD.Stdin = bytes.NewReader(crYaml)
4445
reader, err := cmd.NewReader(execCMD)
4546
Expect(err).ToNot(HaveOccurred())
4647
defer func() {
@@ -129,10 +130,17 @@ var _ = Describe("", func() {
129130
Entry("should pass for Splunk with payloadKey", "splunk-payloadkey.yaml", func(out string, err error) {
130131
Expect(err).ToNot(HaveOccurred())
131132
}),
132-
Entry("should pass for Splunk if sourceType is used with payloadKey", "splunk-payloadkey-and-sourcetype.yaml", func(out string, err error) {
133+
Entry("should pass for Splunk if static sourceType is used with payloadKey", "splunk-payloadkey-and-sourcetype.yaml", func(out string, err error) {
133134
Expect(err).ToNot(HaveOccurred())
134135
}),
135-
Entry("should fail for Splunk if sourceType is not used with payloadKey", "splunk-sourcetype.yaml", func(out string, err error) {
136+
Entry("should pass for Splunk if templated sourceType is used with payloadKey", "splunk-payloadkey-and-templated-sourcetype.yaml", func(out string, err error) {
137+
Expect(err).ToNot(HaveOccurred())
138+
}),
139+
Entry("should fail for Splunk if static sourceType is not used with payloadKey", "splunk-sourcetype.yaml", func(out string, err error) {
140+
Expect(err).To(HaveOccurred())
141+
Expect(err.Error()).To(MatchRegexp("sourceType can only be set when payloadKey is defined"))
142+
}),
143+
Entry("should fail for Splunk if templated sourceType is not used with payloadKey", "splunk-templated-sourcetype.yaml", func(out string, err error) {
136144
Expect(err).To(HaveOccurred())
137145
Expect(err.Error()).To(MatchRegexp("sourceType can only be set when payloadKey is defined"))
138146
}),

0 commit comments

Comments
 (0)