Skip to content

Commit 52d93e1

Browse files
committed
refactor codebase, add tooling, add client mocking and tdd tests
1 parent 8f5f869 commit 52d93e1

15 files changed

Lines changed: 516 additions & 444 deletions

File tree

aws/logs_monitoring_go/Makefile

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,12 @@
1-
.PHONY: build package test lint clean sam-build sam-invoke sam-deploy build-ForwarderFunction
2-
31
BINARY_NAME := bootstrap
42
ZIP_NAME := forwarder.zip
53

4+
# Go
5+
.PHONY: build package test lint clean audit
6+
67
build:
78
GOOS=linux GOARCH=arm64 go build -o $(BINARY_NAME) ./cmd/forwarder/
89

9-
# Used by `sam build`
10-
build-ForwarderFunction:
11-
GOOS=linux GOARCH=arm64 go build -o $(ARTIFACTS_DIR)/bootstrap ./cmd/forwarder/
12-
1310
package: build
1411
zip $(ZIP_NAME) $(BINARY_NAME)
1512

@@ -19,15 +16,26 @@ test:
1916
lint:
2017
golangci-lint run ./...
2118

19+
audit:
20+
go vet ./...
21+
go tool staticcheck ./...
22+
go tool govulncheck
23+
2224
clean:
2325
rm -f $(BINARY_NAME) $(ZIP_NAME)
2426

27+
# SAM
28+
.PHONY: build-ForwarderFunction sam-build sam-invoke sam-deploy
29+
30+
build-ForwarderFunction:
31+
GOOS=linux GOARCH=arm64 go build -o $(ARTIFACTS_DIR)/bootstrap ./cmd/forwarder/
32+
2533
sam-build:
2634
sam build
2735

36+
sam-deploy: sam-build
37+
sam deploy
38+
2839
EVENT ?= events/cloudwatch_logs.json
2940
sam-invoke: sam-build
3041
sam local invoke ForwarderFunction -e $(EVENT)
31-
32-
sam-deploy: sam-build
33-
sam deploy

aws/logs_monitoring_go/cmd/forwarder/main.go

Lines changed: 10 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -8,49 +8,28 @@ package main
88
import (
99
"context"
1010
"encoding/json"
11-
"log"
1211
"log/slog"
13-
"os"
14-
"strings"
1512

1613
"github.com/DataDog/datadog-serverless-functions/aws/logs_monitoring_go/internal/config"
1714

1815
"github.com/aws/aws-lambda-go/lambda"
1916
)
2017

21-
func initLogger(level string) {
22-
var slogLevel slog.Level
23-
switch strings.ToUpper(level) {
24-
case "DEBUG":
25-
slogLevel = slog.LevelDebug
26-
case "INFO":
27-
slogLevel = slog.LevelInfo
28-
case "WARNING", "WARN":
29-
slogLevel = slog.LevelWarn
30-
case "ERROR":
31-
slogLevel = slog.LevelError
32-
default:
33-
slogLevel = slog.LevelInfo
18+
func main() {
19+
ctx := context.Background()
20+
cfg, err := config.Load(ctx)
21+
if err != nil {
22+
slog.Error("config load failed", slog.String("error", err.Error()))
23+
return
3424
}
35-
slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{
36-
Level: slogLevel,
37-
})))
25+
26+
lambda.Start(handleRequest(cfg))
3827
}
3928

29+
// cfg not used for now, will be when forwarding logic added
4030
func handleRequest(cfg *config.Config) func(context.Context, json.RawMessage) error {
4131
return func(ctx context.Context, event json.RawMessage) error {
42-
slog.Info("received event", "event", string(event))
32+
slog.Info("received event", slog.String("event", string(event)))
4333
return nil
4434
}
4535
}
46-
47-
func main() {
48-
ctx := context.Background()
49-
cfg, err := config.Load(ctx)
50-
if err != nil {
51-
log.Fatalf("config: %v", err)
52-
}
53-
// TODO: exit if forwading disabled ?
54-
initLogger(cfg.LogLevel)
55-
lambda.Start(handleRequest(cfg))
56-
}

aws/logs_monitoring_go/go.mod

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@ require (
66
github.com/aws/aws-lambda-go v1.53.0
77
github.com/aws/aws-sdk-go-v2 v1.41.4
88
github.com/aws/aws-sdk-go-v2/config v1.32.12
9-
github.com/stretchr/testify v1.11.1
9+
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.41.4
10+
github.com/google/go-cmp v0.7.0
1011
)
1112

1213
require (
@@ -22,7 +23,18 @@ require (
2223
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
2324
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect
2425
github.com/aws/smithy-go v1.24.2 // indirect
25-
github.com/davecgh/go-spew v1.1.1 // indirect
26-
github.com/pmezard/go-difflib v1.0.0 // indirect
27-
gopkg.in/yaml.v3 v3.0.1 // indirect
26+
github.com/stretchr/testify v1.11.1 // indirect
27+
go.uber.org/mock v0.6.0 // indirect
28+
golang.org/x/mod v0.34.0 // indirect
29+
golang.org/x/sync v0.20.0 // indirect
30+
golang.org/x/sys v0.42.0 // indirect
31+
golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c // indirect
32+
golang.org/x/tools v0.43.0 // indirect
33+
golang.org/x/vuln v1.1.4 // indirect
34+
)
35+
36+
tool (
37+
go.uber.org/mock/mockgen
38+
golang.org/x/tools/cmd/goimports
39+
golang.org/x/vuln/cmd/govulncheck
2840
)

aws/logs_monitoring_go/go.sum

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhL
1818
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
1919
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y=
2020
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk=
21+
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.41.4 h1:9aZbO86sraeCIHHCpZhxwN9tnVy9POkSKzi4/TpT54A=
22+
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.41.4/go.mod h1:cxiXDhEzIq7Xx1BtmC4lGBK3SwAZ79+EUWiKawYHo14=
2123
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow=
2224
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
2325
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o=
@@ -30,11 +32,31 @@ github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
3032
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
3133
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
3234
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
35+
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
36+
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
3337
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
3438
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
3539
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
3640
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
37-
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
38-
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
41+
go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y=
42+
go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU=
43+
golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
44+
golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc=
45+
golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
46+
golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
47+
golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
48+
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
49+
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
50+
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
51+
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
52+
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
53+
golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c h1:6a8FdnNk6bTXBjR4AGKFgUKuo+7GnR3FX5L7CbveeZc=
54+
golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c/go.mod h1:TpUTTEp9frx7rTdLpC9gFG9kdI7zVLFTFFlqaH2Cncw=
55+
golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
56+
golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
57+
golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
58+
golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
59+
golang.org/x/vuln v1.1.4 h1:Ju8QsuyhX3Hk8ma3CesTbO8vfJD9EvUBgHvkxHBzj0I=
60+
golang.org/x/vuln v1.1.4/go.mod h1:F+45wmU18ym/ca5PLTPLsSzr2KppzswxPP603ldA67s=
3961
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
4062
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

aws/logs_monitoring_go/internal/config/apikey.go

Lines changed: 43 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -7,161 +7,82 @@ package config
77

88
import (
99
"context"
10-
"crypto/tls"
1110
"errors"
1211
"fmt"
12+
"io"
1313
"log/slog"
1414
"net/http"
1515
"os"
16-
"strings"
1716
"time"
1817

19-
"github.com/aws/aws-sdk-go-v2/aws"
2018
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
2119
awsconfig "github.com/aws/aws-sdk-go-v2/config"
2220
)
2321

24-
const (
25-
httpClientTimeout = 10 * time.Second
26-
maxRetries = 5
27-
retryBackoffFactor = 1 * time.Second
22+
var (
23+
ErrMissingAPIKey = errors.New("missing Datadog API key")
24+
ErrInvalidAPIKey = errors.New("invalid Datadog API key format")
2825
)
2926

30-
type resolveOptions struct {
31-
AWSCfg aws.Config
32-
Value string
33-
}
34-
35-
type apiKeyResolver func(ctx context.Context, opts resolveOptions) (string, error)
36-
37-
var retryableStatusCodes = map[int]bool{
38-
429: true,
39-
500: true,
40-
502: true,
41-
503: true,
42-
504: true,
43-
}
44-
45-
var resolvers = []struct {
46-
envVar string
47-
resolve apiKeyResolver
48-
}{
49-
{"DD_API_KEY_SECRET_ARN", resolveFromSecretsManager},
50-
{"DD_API_KEY_SSM_NAME", resolveFromSSM},
51-
{"DD_KMS_API_KEY", resolveFromKMS},
52-
{"DD_API_KEY", resolveFromEnv},
53-
}
27+
const (
28+
httpClientTimeout = 3 * time.Second
29+
)
5430

5531
func (c *Config) resolveAPIKey(ctx context.Context) error {
56-
awsCfg, err := awsconfig.LoadDefaultConfig(ctx,
57-
awsconfig.WithHTTPClient(awshttp.NewBuildableClient().WithTimeout(httpClientTimeout)),
58-
)
59-
32+
awsCfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithHTTPClient(awshttp.NewBuildableClient().WithTimeout(httpClientTimeout)))
6033
if err != nil {
6134
return fmt.Errorf("loading AWS config: %w", err)
6235
}
6336

64-
for _, resolver := range resolvers {
65-
if v, ok := os.LookupEnv(resolver.envVar); ok {
66-
slog.Debug("resolving API key", "source", resolver.envVar)
67-
key, err := resolver.resolve(ctx, resolveOptions{
68-
AWSCfg: awsCfg,
69-
Value: v,
70-
})
71-
if err != nil {
72-
return fmt.Errorf("resolving API key from %s: %w", resolver.envVar, err)
73-
}
74-
c.APIKey = strings.TrimSpace(key)
75-
slog.Debug("API key resolved", "source", resolver.envVar)
76-
return nil
77-
}
37+
if v, ok := os.LookupEnv("DD_API_KEY_SECRET_ARN"); ok {
38+
return c.resolveFromSecretsManager(ctx, awsCfg, v)
39+
}
40+
41+
if v, ok := os.LookupEnv("DD_API_KEY_SSM_NAME"); ok {
42+
return c.resolveFromSSM(ctx, awsCfg, v)
43+
}
44+
45+
if v, ok := os.LookupEnv("DD_KMS_API_KEY"); ok {
46+
return c.resolveFromKMS(ctx, awsCfg, v)
7847
}
79-
return errors.New("no API key configured: set DD_API_KEY_SECRET_ARN, DD_API_KEY_SSM_NAME, DD_KMS_API_KEY, or DD_API_KEY. See: https://docs.datadoghq.com/serverless/forwarder/")
48+
49+
return errors.New("no API key configured: set DD_API_KEY_SECRET_ARN, DD_API_KEY_SSM_NAME or DD_KMS_API_KEY. See: https://docs.datadoghq.com/serverless/forwarder/")
8050
}
8151

82-
// Note: the API key verification could fail (e.g. Datadog verification endpoint or network problem)
83-
// Instead of failing the whole lambda at startup, it should run up to the log sending part and verify the
84-
// key at this moment, adding the run to the future retry logic in such case.
85-
// The method may disappear in the future.
86-
func (c *Config) validateAPIKey() error {
87-
if c.APIKey == "" || c.APIKey == "<YOUR_DATADOG_API_KEY>" {
88-
return errors.New("missing Datadog API key. Set DD_API_KEY environment variable. See: https://docs.datadoghq.com/serverless/forwarder/")
52+
func (c *Config) validateAPIKey(ctx context.Context) error {
53+
if c.APIKey == "" {
54+
return fmt.Errorf("%w: set DD_API_KEY_SECRET_ARN, DD_API_KEY_SSM_NAME or DD_KMS_API_KEY. See: https://docs.datadoghq.com/serverless/forwarder/", ErrMissingAPIKey)
8955
}
9056

9157
if len(c.APIKey) != 32 {
92-
return fmt.Errorf("invalid Datadog API key format: expected 32 characters, got %d. Verify your API key at https://app.%s/organization-settings/api-keys", len(c.APIKey), c.Site)
58+
return fmt.Errorf("%w: expected 32 characters, got %d. Verify your API key at https://app.%s/organization-settings/api-keys", ErrInvalidAPIKey, len(c.APIKey), c.Site)
9359
}
9460

9561
slog.Debug("validating Datadog API key")
9662

97-
client := &http.Client{
98-
Timeout: httpClientTimeout,
99-
Transport: &http.Transport{
100-
TLSClientConfig: &tls.Config{
101-
InsecureSkipVerify: c.SkipSSLValidation,
102-
},
103-
},
63+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.APIURL+"/api/v1/validate", nil)
64+
if err != nil {
65+
slog.Warn("failed to build API key validation request", slog.String("error", err.Error()))
66+
return nil
10467
}
68+
req.Header.Set("DD-API-KEY", c.APIKey)
10569

106-
url := fmt.Sprintf("%s/api/v1/validate?api_key=%s", c.APIURL, c.APIKey)
107-
108-
var lastErr error
109-
var lastStatus int
110-
for attempt := range maxRetries {
111-
resp, err := client.Get(url)
112-
if err != nil {
113-
lastErr = err
114-
slog.Debug("API key validation request failed, retrying", "attempt", attempt+1, "error", err)
115-
time.Sleep(retryBackoffFactor * time.Duration(1<<attempt))
116-
continue
117-
}
118-
if err := resp.Body.Close(); err != nil {
119-
slog.Debug("failed to close response body", "error", err)
120-
}
121-
122-
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
123-
slog.Debug("API key validated successfully")
124-
return nil
125-
}
126-
127-
if !retryableStatusCodes[resp.StatusCode] {
128-
slog.Warn("API key validation failed. Verify your API key is correct and DD_SITE matches your Datadog account region. See: https://docs.datadoghq.com/getting_started/site/",
129-
"status", resp.StatusCode,
130-
"site", c.Site,
131-
)
132-
return nil
133-
}
134-
135-
lastStatus = resp.StatusCode
136-
slog.Debug("API key validation returned retryable status, retrying", "attempt", attempt+1, "status", resp.StatusCode)
137-
time.Sleep(retryBackoffFactor * time.Duration(1<<attempt))
70+
client := &http.Client{Timeout: httpClientTimeout}
71+
resp, err := client.Do(req)
72+
if err != nil {
73+
slog.Warn("failed to validate API key", slog.String("error", err.Error()))
74+
return nil
13875
}
139-
140-
if lastErr != nil {
141-
slog.Warn("API key validation failed after retries, continuing anyway", "attempts", maxRetries, "error", lastErr)
142-
} else {
143-
slog.Warn("API key validation failed after retries. Verify your API key is correct and DD_SITE matches your Datadog account region. See: https://docs.datadoghq.com/getting_started/site/",
144-
"attempts", maxRetries,
145-
"lastStatus", lastStatus,
146-
"site", c.Site,
147-
)
76+
defer func() {
77+
io.Copy(io.Discard, resp.Body)
78+
resp.Body.Close()
79+
}()
80+
81+
if resp.StatusCode == http.StatusForbidden {
82+
slog.Warn("invalid Datadog API key", slog.String("url", "https://app."+c.Site+"/organization-settings/api-keys"))
83+
} else if resp.StatusCode != http.StatusOK {
84+
slog.Warn("unexpected response from validation endpoint", slog.String("status", resp.Status))
14885
}
14986

15087
return nil
15188
}
152-
153-
func resolveFromSecretsManager(ctx context.Context, opts resolveOptions) (string, error) {
154-
return "", nil
155-
}
156-
157-
func resolveFromSSM(ctx context.Context, opts resolveOptions) (string, error) {
158-
return "", nil
159-
}
160-
161-
func resolveFromKMS(ctx context.Context, opts resolveOptions) (string, error) {
162-
return "", nil
163-
}
164-
165-
func resolveFromEnv(ctx context.Context, opts resolveOptions) (string, error) {
166-
return opts.Value, nil
167-
}

0 commit comments

Comments
 (0)