Skip to content

Commit aba33a6

Browse files
ndakkounendakkoune
authored
[AWSINTS-3493] fix(go-forwarder): add ddtags extraction from message (#1106)
1 parent 5349a82 commit aba33a6

3 files changed

Lines changed: 222 additions & 6 deletions

File tree

‎aws/logs_monitoring_go/internal/parsing/cloudwatchlogs.go‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,14 +59,21 @@ func parseCloudwatchLogs(ctx context.Context, event json.RawMessage, cfg *config
5959

6060
var entries []model.CloudwatchLogEntry
6161
for _, le := range data.LogEvents {
62+
ddtags, ddtagsService, message := extractFromMessage(le.Message)
63+
entryService := service
64+
if ddtagsService != "" {
65+
entryService = ddtagsService
66+
}
67+
ddtags = append(ddtags, "service:"+entryService)
68+
6269
entry := model.CloudwatchLogEntry{
6370
ID: le.ID,
6471
Timestamp: le.Timestamp,
65-
Message: le.Message,
72+
Message: message,
6673
Source: source,
67-
Service: service,
74+
Service: entryService,
6875
Host: host,
69-
Tags: tags,
76+
Tags: append(ddtags, tags...),
7077
AWS: metadata,
7178
}
7279
entries = append(entries, entry)

‎aws/logs_monitoring_go/internal/parsing/tags.go‎

Lines changed: 48 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,24 +6,27 @@
66
package parsing
77

88
import (
9+
"encoding/json"
910
"strings"
1011

1112
"github.com/DataDog/datadog-serverless-functions/aws/logs_monitoring_go/internal/config"
1213
"github.com/DataDog/datadog-serverless-functions/aws/logs_monitoring_go/internal/model"
1314
"github.com/aws/aws-lambda-go/lambdacontext"
1415
)
1516

17+
const DdtagsKey = "ddtags"
18+
1619
func getTagsAndService(cfg config.Config) (model.Tags, string) {
1720
var tags model.Tags
1821
var service string
1922

2023
if cfg.CustomTags != "" {
2124
for tag := range strings.SplitSeq(cfg.CustomTags, ",") {
2225
if strings.HasPrefix(tag, "service:") {
23-
if service != "" {
24-
continue
26+
if service == "" {
27+
service = tag[8:]
2528
}
26-
service = tag[8:]
29+
continue
2730
}
2831

2932
tags = append(tags, tag)
@@ -41,3 +44,45 @@ func getTagsAndService(cfg config.Config) (model.Tags, string) {
4144

4245
return tags, service
4346
}
47+
48+
func extractFromMessage(message string) (model.Tags, string, string) {
49+
var tags model.Tags
50+
var service string
51+
52+
var jsonMessage map[string]any
53+
if err := json.Unmarshal([]byte(message), &jsonMessage); err != nil {
54+
return nil, service, message
55+
}
56+
57+
ddtagsRaw, ok := jsonMessage[DdtagsKey]
58+
if !ok {
59+
return nil, service, message
60+
}
61+
62+
ddtagsStr, ok := ddtagsRaw.(string)
63+
if !ok {
64+
return nil, service, message
65+
}
66+
67+
ddtagsStr = strings.ReplaceAll(ddtagsStr, " ", "")
68+
69+
for tag := range strings.SplitSeq(ddtagsStr, ",") {
70+
if strings.HasPrefix(tag, "service:") {
71+
if service == "" {
72+
service = tag[8:]
73+
}
74+
continue
75+
}
76+
77+
tags = append(tags, tag)
78+
}
79+
80+
delete(jsonMessage, DdtagsKey)
81+
82+
newMessage, err := json.Marshal(jsonMessage)
83+
if err != nil {
84+
return nil, service, message
85+
}
86+
87+
return tags, service, string(newMessage)
88+
}

‎aws/logs_monitoring_go/internal/parsing/tags_test.go‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
// Unless explicitly stated otherwise all files in this repository are licensed
2+
// under the Apache License Version 2.0.
3+
// This product includes software developed at Datadog (https://www.datadoghq.com/).
4+
// Copyright 2026-Present Datadog, Inc.
5+
6+
package parsing
7+
8+
import (
9+
"encoding/json"
10+
"slices"
11+
"strings"
12+
"testing"
13+
14+
"github.com/DataDog/datadog-serverless-functions/aws/logs_monitoring_go/internal/model"
15+
)
16+
17+
func TestExtractFromMessage(t *testing.T) {
18+
t.Parallel()
19+
20+
tests := map[string]struct {
21+
message string
22+
wantTags model.Tags
23+
wantService string
24+
wantMessage string
25+
}{
26+
"empty string": {
27+
message: "",
28+
wantTags: nil,
29+
wantService: "",
30+
wantMessage: "",
31+
},
32+
"plain text": {
33+
message: "ERROR something went wrong",
34+
wantTags: nil,
35+
wantService: "",
36+
wantMessage: "ERROR something went wrong",
37+
},
38+
"invalid json": {
39+
message: `{not valid json}`,
40+
wantTags: nil,
41+
wantService: "",
42+
wantMessage: `{not valid json}`,
43+
},
44+
"json without ddtags": {
45+
message: `{"level":"INFO","msg":"hello"}`,
46+
wantTags: nil,
47+
wantService: "",
48+
wantMessage: `{"level":"INFO","msg":"hello"}`,
49+
},
50+
"ddtags is not a string": {
51+
message: `{"ddtags":["tag1","tag2"]}`,
52+
wantTags: nil,
53+
wantService: "",
54+
wantMessage: `{"ddtags":["tag1","tag2"]}`,
55+
},
56+
"single tag": {
57+
message: `{"msg":"hello","ddtags":"env:prod"}`,
58+
wantTags: model.Tags{"env:prod"},
59+
wantService: "",
60+
wantMessage: `{"msg":"hello"}`,
61+
},
62+
"multiple tags": {
63+
message: `{"msg":"hello","ddtags":"env:prod,team:backend"}`,
64+
wantTags: model.Tags{"env:prod", "team:backend"},
65+
wantService: "",
66+
wantMessage: `{"msg":"hello"}`,
67+
},
68+
"tags with spaces are cleaned": {
69+
message: `{"msg":"hello","ddtags":"env:prod, team:backend, version:1.0"}`,
70+
wantTags: model.Tags{"env:prod", "team:backend", "version:1.0"},
71+
wantService: "",
72+
wantMessage: `{"msg":"hello"}`,
73+
},
74+
"service tag extracted": {
75+
message: `{"msg":"hello","ddtags":"service:my-app,env:prod"}`,
76+
wantTags: model.Tags{"env:prod"},
77+
wantService: "my-app",
78+
wantMessage: `{"msg":"hello"}`,
79+
},
80+
"service only": {
81+
message: `{"msg":"hello","ddtags":"service:my-app"}`,
82+
wantTags: nil,
83+
wantService: "my-app",
84+
wantMessage: `{"msg":"hello"}`,
85+
},
86+
"first service wins": {
87+
message: `{"msg":"hello","ddtags":"service:first,service:second,env:prod"}`,
88+
wantTags: model.Tags{"env:prod"},
89+
wantService: "first",
90+
wantMessage: `{"msg":"hello"}`,
91+
},
92+
"ddtags is empty string": {
93+
message: `{"msg":"hello","ddtags":""}`,
94+
wantTags: model.Tags{""},
95+
wantService: "",
96+
wantMessage: `{"msg":"hello"}`,
97+
},
98+
"ddtags only field in json": {
99+
message: `{"ddtags":"env:prod"}`,
100+
wantTags: model.Tags{"env:prod"},
101+
wantService: "",
102+
wantMessage: `{}`,
103+
},
104+
}
105+
106+
for name, tc := range tests {
107+
t.Run(name, func(t *testing.T) {
108+
t.Parallel()
109+
110+
gotTags, gotService, gotMessage := extractFromMessage(tc.message)
111+
112+
if !slices.Equal(gotTags, tc.wantTags) {
113+
t.Errorf("tags: got %v, want %v", gotTags, tc.wantTags)
114+
}
115+
if gotService != tc.wantService {
116+
t.Errorf("service: got %q, want %q", gotService, tc.wantService)
117+
}
118+
if gotMessage != tc.wantMessage {
119+
t.Errorf("message: got %q, want %q", gotMessage, tc.wantMessage)
120+
}
121+
})
122+
}
123+
}
124+
125+
func FuzzExtractFromMessage(f *testing.F) {
126+
seeds := []string{
127+
"",
128+
"plain text, not json",
129+
`{not valid json}`,
130+
`{"msg":"hello"}`,
131+
`{"ddtags":["tag1","tag2"]}`,
132+
`{"ddtags":42}`,
133+
`{"msg":"hello","ddtags":"env:prod"}`,
134+
`{"msg":"hello","ddtags":"env:prod, team:backend, version:1.0"}`,
135+
`{"msg":"hello","ddtags":"service:my-app,env:prod"}`,
136+
`{"msg":"hello","ddtags":"service:my-app"}`,
137+
`{"msg":"hello","ddtags":"service:first,service:second,env:prod"}`,
138+
`{"msg":"hello","ddtags":""}`,
139+
`{"ddtags":"env:prod"}`,
140+
}
141+
for _, seed := range seeds {
142+
f.Add(seed)
143+
}
144+
145+
f.Fuzz(func(t *testing.T, message string) {
146+
tags, _, outMessage := extractFromMessage(message)
147+
148+
if outMessage != message {
149+
var parsed map[string]any
150+
if err := json.Unmarshal([]byte(outMessage), &parsed); err != nil {
151+
t.Errorf("output message is not valid JSON: %v", err)
152+
}
153+
if _, ok := parsed[DdtagsKey]; ok {
154+
t.Errorf("output message still contains %q key", DdtagsKey)
155+
}
156+
}
157+
158+
for _, tag := range tags {
159+
if strings.HasPrefix(tag, "service:") {
160+
t.Errorf("tag %q should have been extracted as service, not returned in tags", tag)
161+
}
162+
}
163+
})
164+
}

0 commit comments

Comments
 (0)