From 9c4e3c2e9c0460ffa97b37c7731d35ddeb685d91 Mon Sep 17 00:00:00 2001 From: Nabil Dakkoune Date: Fri, 20 Mar 2026 09:53:59 +0100 Subject: [PATCH 01/16] [AWSX-2144] feat(go-forwarder): add basic api key and config logic --- aws/logs_monitoring_go/cmd/forwarder/main.go | 41 +++++- aws/logs_monitoring_go/go.mod | 20 +++ aws/logs_monitoring_go/go.sum | 34 +++++ .../internal/config/apikey.go | 101 +++++++++++++++ .../internal/config/config.go | 119 ++++++++++++++++++ 5 files changed, 311 insertions(+), 4 deletions(-) create mode 100644 aws/logs_monitoring_go/internal/config/apikey.go create mode 100644 aws/logs_monitoring_go/internal/config/config.go diff --git a/aws/logs_monitoring_go/cmd/forwarder/main.go b/aws/logs_monitoring_go/cmd/forwarder/main.go index 347a2a716..6550520cb 100644 --- a/aws/logs_monitoring_go/cmd/forwarder/main.go +++ b/aws/logs_monitoring_go/cmd/forwarder/main.go @@ -9,15 +9,48 @@ import ( "context" "encoding/json" "log" + "log/slog" + "os" + "strings" + + "github.com/DataDog/datadog-serverless-functions/aws/logs_monitoring_go/internal/config" "github.com/aws/aws-lambda-go/lambda" ) -func handleRequest(ctx context.Context, event json.RawMessage) error { - log.Printf("Received event: %s", string(event)) - return nil +func initLogger(level string) { + var slogLevel slog.Level + switch strings.ToUpper(level) { + case "DEBUG": + slogLevel = slog.LevelDebug + case "INFO": + slogLevel = slog.LevelInfo + case "WARNING", "WARN": + slogLevel = slog.LevelWarn + case "ERROR": + slogLevel = slog.LevelError + default: + slogLevel = slog.LevelInfo + } + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + Level: slogLevel, + }))) +} + +func handleRequest(cfg *config.Config) func(context.Context, json.RawMessage) error { + return func(ctx context.Context, event json.RawMessage) error { + slog.Info("received event", "event", string(event)) + return nil + } } func main() { - lambda.Start(handleRequest) + ctx := context.Background() + cfg, err := config.Load(ctx) + if err != nil { + log.Fatalf("config: %v", err) + } + // TODO: exit if forwading disabled ? + initLogger(cfg.LogLevel) + lambda.Start(handleRequest(cfg)) } diff --git a/aws/logs_monitoring_go/go.mod b/aws/logs_monitoring_go/go.mod index 99422bf68..839db3ace 100644 --- a/aws/logs_monitoring_go/go.mod +++ b/aws/logs_monitoring_go/go.mod @@ -3,3 +3,23 @@ module github.com/DataDog/datadog-serverless-functions/aws/logs_monitoring_go go 1.26 require github.com/aws/aws-lambda-go v1.53.0 + +require ( + github.com/aws/aws-sdk-go-v2 v1.41.4 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.12 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.12 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect + github.com/aws/aws-sdk-go-v2/service/kms v1.50.3 // indirect + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.41.4 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect + github.com/aws/aws-sdk-go-v2/service/ssm v1.68.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect + github.com/aws/smithy-go v1.24.2 // indirect +) diff --git a/aws/logs_monitoring_go/go.sum b/aws/logs_monitoring_go/go.sum index 0d766f423..51a3fd7c0 100644 --- a/aws/logs_monitoring_go/go.sum +++ b/aws/logs_monitoring_go/go.sum @@ -1,5 +1,39 @@ github.com/aws/aws-lambda-go v1.53.0 h1:uAMv6W/vCP/L494BAUSxe+8KVBIPK+SGPyapFt3FuMk= github.com/aws/aws-lambda-go v1.53.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A= +github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k= +github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2/config v1.32.12 h1:O3csC7HUGn2895eNrLytOJQdoL2xyJy0iYXhoZ1OmP0= +github.com/aws/aws-sdk-go-v2/config v1.32.12/go.mod h1:96zTvoOFR4FURjI+/5wY1vc1ABceROO4lWgWJuxgy0g= +github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8= +github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk= +github.com/aws/aws-sdk-go-v2/service/kms v1.50.3 h1:s/zDSG/a/Su9aX+v0Ld9cimUCdkr5FWPmBV8owaEbZY= +github.com/aws/aws-sdk-go-v2/service/kms v1.50.3/go.mod h1:/iSgiUor15ZuxFGQSTf3lA2FmKxFsQoc2tADOarQBSw= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.41.4 h1:9aZbO86sraeCIHHCpZhxwN9tnVy9POkSKzi4/TpT54A= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.41.4/go.mod h1:cxiXDhEzIq7Xx1BtmC4lGBK3SwAZ79+EUWiKawYHo14= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE= +github.com/aws/aws-sdk-go-v2/service/ssm v1.68.3 h1:bBoWhx8lsFLTXintRX64ZBXcmFZbGqUmaPUrjXECqIc= +github.com/aws/aws-sdk-go-v2/service/ssm v1.68.3/go.mod h1:rcRkKbUJ2437WuXdq9fbj+MjTudYWzY9Ct8kiBbN8a8= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk= +github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= +github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/aws/logs_monitoring_go/internal/config/apikey.go b/aws/logs_monitoring_go/internal/config/apikey.go new file mode 100644 index 000000000..3ec526a2d --- /dev/null +++ b/aws/logs_monitoring_go/internal/config/apikey.go @@ -0,0 +1,101 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-Present Datadog, Inc. + +package config + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + awsconfig "github.com/aws/aws-sdk-go-v2/config" +) + +const ( + TIMEOUT = 5 +) + +type resolveOptions struct { + AWSCfg aws.Config + Value string + UseFIPS bool +} + +type apiKeyResolver func(ctx context.Context, opts resolveOptions) (string, error) + +var resolvers = []struct { + envVar string + resolve apiKeyResolver +}{ + {"DD_API_KEY_SECRET_ARN", resolveFromSecretsManager}, + {"DD_API_KEY_SSM_NAME", resolveFromSSM}, + {"DD_KMS_API_KEY", resolveFromKMS}, + {"DD_API_KEY", resolveFromEnv}, +} + +func resolveAPIKey(ctx context.Context, useFIPS bool) (string, error) { + awsCfg, err := awsconfig.LoadDefaultConfig(ctx, + awsconfig.WithHTTPClient(awshttp.NewBuildableClient().WithTimeout(time.Second*TIMEOUT)), + ) + + if err != nil { + return "", fmt.Errorf("loading AWS config: %w", err) + } + + for _, resolver := range resolvers { + if v, ok := os.LookupEnv(resolver.envVar); ok { + slog.Debug("resolving API key", "source", resolver.envVar) + resolver.resolve(ctx, resolveOptions{ + AWSCfg: awsCfg, + Value: v, + UseFIPS: useFIPS, + }) + } + } + return "", errors.New("no API key configured: set DD_API_KEY, DD_API_KEY_SECRET_ARN, DD_API_KEY_SSM_NAME, or DD_KMS_API_KEY. See: https://docs.datadoghq.com/serverless/forwarder/") +} + +func resolveFromSecretsManager(ctx context.Context, opts resolveOptions) (string, error) { + return "", nil +} + +func resolveFromSSM(ctx context.Context, opts resolveOptions) (string, error) { + return "", nil +} + +func resolveFromKMS(ctx context.Context, opts resolveOptions) (string, error) { + return "", nil +} + +func resolveFromEnv(ctx context.Context, opts resolveOptions) (string, error) { + // if len(opts.Value) != 32 { + // return "", fmt.Errorf("invalid datadog api key format") + // } + + // client := &http.Client{ + // Timeout: TIMEOUT * time.Second, + // Transport: &http.Transport{ + // TLSClientConfig: &tls.Config{ + // InsecureSkipVerify: skipSSLValidation, + // }, + // }, + // } + + // res, err := http.Get() + // if err != nil { + + // } + + return opts.Value, nil +} + +func validateAPIKey(cfg Config) { + +} diff --git a/aws/logs_monitoring_go/internal/config/config.go b/aws/logs_monitoring_go/internal/config/config.go new file mode 100644 index 000000000..1a72c2794 --- /dev/null +++ b/aws/logs_monitoring_go/internal/config/config.go @@ -0,0 +1,119 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026-Present Datadog, Inc. + +package config + +import ( + "context" + "fmt" + "log/slog" + "os" + "strconv" + "strings" +) + +var deprecatedEnvironmentVariables = []string{ + "DD_ADDITIONAL_TARGET_LAMBDAS", + "DD_ENRICH_CLOUDWATCH_TAGS", + "DD_ENRICH_S3_TAGS", + "DD_FETCH_LAMBDA_TAGS", + "DD_FETCH_LOG_GROUP_TAGS", + "DD_FETCH_S3_TAGS", + "DD_FETCH_STEP_FUNCTIONS_TAGS", + "DD_TAGS_CACHE_TTL_SECONDS", + "DD_TRACE_INTAKE_URL", + "DD_USE_VPC", +} + +type Config struct { + APIKey string + Site string + URL string + Port int + APIURL string + ForwardLog bool + UseCompression bool + CompressionLevel int + NoSSL bool + SkipSSLValidation bool + Tags string + Source string + LogLevel string + UseFIPS bool +} + +func Load(ctx context.Context) (*Config, error) { + cfg := &Config{ + Site: envOrDefault("DD_SITE", "datadoghq.com"), + Port: envOrDefaultInt("DD_PORT", 443), + ForwardLog: envOrDefaultBool("DD_FORWARD_LOG", true), + UseCompression: envOrDefaultBool("DD_USE_COMPRESSION", true), + CompressionLevel: envOrDefaultInt("DD_COMPRESSION_LEVEL", 6), + NoSSL: envOrDefaultBool("DD_NO_SSL", false), + SkipSSLValidation: envOrDefaultBool("DD_SKIP_SSL_VALIDATION", false), + Tags: envOrDefault("DD_TAGS", ""), + Source: envOrDefault("DD_SOURCE", ""), + LogLevel: envOrDefault("DD_LOG_LEVEL", "INFO"), + } + + scheme := "https" + if cfg.NoSSL { + scheme = "http" + } + cfg.URL = envOrDefault("DD_URL", "http-intake.logs."+cfg.Site) + cfg.APIURL = envOrDefault("DD_API_URL", fmt.Sprintf("%s://api.%s", scheme, cfg.Site)) + + logDroppedEnvVars() + + useFIPS := envOrDefaultBool("DD_USE_FIPS", false) + cfg.UseFIPS = useFIPS + apiKey, err := resolveAPIKey(ctx, useFIPS) + if err != nil { + return nil, fmt.Errorf("resolving API key: %w", err) + } + cfg.APIKey = apiKey + + if err := validateAPIKey(cfg); err != nil { + return nil, fmt.Errorf("validating API key: %w", err) + } + + return cfg, nil +} + +func logDroppedEnvVars() { + for _, name := range deprecatedEnvironmentVariables { + if _, ok := os.LookupEnv(name); ok { + slog.Warn("deprecated env var set, will be ignored", "name", name) + } + } +} + +func envOrDefault(key, fallback string) string { + if v, ok := os.LookupEnv(key); ok { + return v + } + return fallback +} + +func envOrDefaultBool(key string, fallback bool) bool { + v, ok := os.LookupEnv(key) + if !ok { + return fallback + } + return strings.EqualFold(v, "true") +} + +func envOrDefaultInt(key string, fallback int) int { + v, ok := os.LookupEnv(key) + if !ok { + return fallback + } + n, err := strconv.Atoi(v) + if err != nil { + slog.Warn("invalid integer for env var, using default", "key", key, "value", v, "default", fallback) + return fallback + } + return n +} From 887c3b99d5ad4aaaa13899e6b8104e986d4bc2d6 Mon Sep 17 00:00:00 2001 From: Nabil Dakkoune Date: Fri, 20 Mar 2026 11:41:44 +0100 Subject: [PATCH 02/16] fix: change logger from unstructured to JSON --- aws/logs_monitoring_go/cmd/forwarder/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/logs_monitoring_go/cmd/forwarder/main.go b/aws/logs_monitoring_go/cmd/forwarder/main.go index 6550520cb..da4d9e964 100644 --- a/aws/logs_monitoring_go/cmd/forwarder/main.go +++ b/aws/logs_monitoring_go/cmd/forwarder/main.go @@ -32,7 +32,7 @@ func initLogger(level string) { default: slogLevel = slog.LevelInfo } - slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{ Level: slogLevel, }))) } From 8568a56fc9b17dfe1989b81c90d49220540489c4 Mon Sep 17 00:00:00 2001 From: Nabil Dakkoune Date: Fri, 20 Mar 2026 11:43:08 +0100 Subject: [PATCH 03/16] fix: tranform func to methods, delete unused env vars, refactor code --- .../internal/config/apikey.go | 36 +++++++++------ .../internal/config/config.go | 45 ++++++++----------- 2 files changed, 40 insertions(+), 41 deletions(-) diff --git a/aws/logs_monitoring_go/internal/config/apikey.go b/aws/logs_monitoring_go/internal/config/apikey.go index 3ec526a2d..8f82a83d4 100644 --- a/aws/logs_monitoring_go/internal/config/apikey.go +++ b/aws/logs_monitoring_go/internal/config/apikey.go @@ -19,13 +19,12 @@ import ( ) const ( - TIMEOUT = 5 + awsClientTimeout = 5 * time.Second ) type resolveOptions struct { - AWSCfg aws.Config - Value string - UseFIPS bool + AWSCfg aws.Config + Value string } type apiKeyResolver func(ctx context.Context, opts resolveOptions) (string, error) @@ -40,26 +39,30 @@ var resolvers = []struct { {"DD_API_KEY", resolveFromEnv}, } -func resolveAPIKey(ctx context.Context, useFIPS bool) (string, error) { +func (c *Config) resolveAPIKey(ctx context.Context) error { awsCfg, err := awsconfig.LoadDefaultConfig(ctx, - awsconfig.WithHTTPClient(awshttp.NewBuildableClient().WithTimeout(time.Second*TIMEOUT)), + awsconfig.WithHTTPClient(awshttp.NewBuildableClient().WithTimeout(awsClientTimeout)), ) if err != nil { - return "", fmt.Errorf("loading AWS config: %w", err) + return fmt.Errorf("loading AWS config: %w", err) } for _, resolver := range resolvers { if v, ok := os.LookupEnv(resolver.envVar); ok { slog.Debug("resolving API key", "source", resolver.envVar) - resolver.resolve(ctx, resolveOptions{ - AWSCfg: awsCfg, - Value: v, - UseFIPS: useFIPS, + key, err := resolver.resolve(ctx, resolveOptions{ + AWSCfg: awsCfg, + Value: v, }) + if err != nil { + return fmt.Errorf("resolving API key from %s: %w", resolver.envVar, err) + } + c.APIKey = key + return nil } } - return "", errors.New("no API key configured: set DD_API_KEY, DD_API_KEY_SECRET_ARN, DD_API_KEY_SSM_NAME, or DD_KMS_API_KEY. See: https://docs.datadoghq.com/serverless/forwarder/") + return errors.New("no API key configured: set DD_API_KEY_SECRET_ARN, DD_API_KEY_SSM_NAME, DD_KMS_API_KEY, or DD_API_KEY. See: https://docs.datadoghq.com/serverless/forwarder/") } func resolveFromSecretsManager(ctx context.Context, opts resolveOptions) (string, error) { @@ -96,6 +99,11 @@ func resolveFromEnv(ctx context.Context, opts resolveOptions) (string, error) { return opts.Value, nil } -func validateAPIKey(cfg Config) { - +// Note: the API key verification could fail (e.g. Datadog verification endpoint or network problem) +// Instead of failing the whole lambda at startup, it should run up to the log sending part and verify the +// key at this moment, adding the run to the future retry logic in such case. +// The method may disappear in the future. +func (c *Config) validateAPIKey() error { + // TODO: implement validation against Datadog API + return nil } diff --git a/aws/logs_monitoring_go/internal/config/config.go b/aws/logs_monitoring_go/internal/config/config.go index 1a72c2794..36f71a671 100644 --- a/aws/logs_monitoring_go/internal/config/config.go +++ b/aws/logs_monitoring_go/internal/config/config.go @@ -33,13 +33,8 @@ type Config struct { URL string Port int APIURL string - ForwardLog bool - UseCompression bool - CompressionLevel int NoSSL bool SkipSSLValidation bool - Tags string - Source string LogLevel string UseFIPS bool } @@ -48,46 +43,34 @@ func Load(ctx context.Context) (*Config, error) { cfg := &Config{ Site: envOrDefault("DD_SITE", "datadoghq.com"), Port: envOrDefaultInt("DD_PORT", 443), - ForwardLog: envOrDefaultBool("DD_FORWARD_LOG", true), - UseCompression: envOrDefaultBool("DD_USE_COMPRESSION", true), - CompressionLevel: envOrDefaultInt("DD_COMPRESSION_LEVEL", 6), NoSSL: envOrDefaultBool("DD_NO_SSL", false), SkipSSLValidation: envOrDefaultBool("DD_SKIP_SSL_VALIDATION", false), - Tags: envOrDefault("DD_TAGS", ""), - Source: envOrDefault("DD_SOURCE", ""), LogLevel: envOrDefault("DD_LOG_LEVEL", "INFO"), + UseFIPS: envOrDefaultBool("DD_USE_FIPS", false), } - scheme := "https" - if cfg.NoSSL { - scheme = "http" - } - cfg.URL = envOrDefault("DD_URL", "http-intake.logs."+cfg.Site) - cfg.APIURL = envOrDefault("DD_API_URL", fmt.Sprintf("%s://api.%s", scheme, cfg.Site)) + cfg.deriveURLs() logDroppedEnvVars() - useFIPS := envOrDefaultBool("DD_USE_FIPS", false) - cfg.UseFIPS = useFIPS - apiKey, err := resolveAPIKey(ctx, useFIPS) - if err != nil { + if err := cfg.resolveAPIKey(ctx); err != nil { return nil, fmt.Errorf("resolving API key: %w", err) } - cfg.APIKey = apiKey - if err := validateAPIKey(cfg); err != nil { + if err := cfg.validateAPIKey(); err != nil { return nil, fmt.Errorf("validating API key: %w", err) } return cfg, nil } -func logDroppedEnvVars() { - for _, name := range deprecatedEnvironmentVariables { - if _, ok := os.LookupEnv(name); ok { - slog.Warn("deprecated env var set, will be ignored", "name", name) - } +func (c *Config) deriveURLs() { + scheme := "https" + if c.NoSSL { + scheme = "http" } + c.URL = envOrDefault("DD_URL", "http-intake.logs."+c.Site) + c.APIURL = envOrDefault("DD_API_URL", fmt.Sprintf("%s://api.%s", scheme, c.Site)) } func envOrDefault(key, fallback string) string { @@ -117,3 +100,11 @@ func envOrDefaultInt(key string, fallback int) int { } return n } + +func logDroppedEnvVars() { + for _, name := range deprecatedEnvironmentVariables { + if _, ok := os.LookupEnv(name); ok { + slog.Warn("deprecated env var set, will be ignored", "name", name) + } + } +} From 417d354aa5e9eff5a034ee846ca9d42a1d4c9bae Mon Sep 17 00:00:00 2001 From: Nabil Dakkoune Date: Fri, 20 Mar 2026 14:31:12 +0100 Subject: [PATCH 04/16] feat(go-forwarder): add tests --- aws/logs_monitoring_go/go.mod | 15 ++- aws/logs_monitoring_go/go.sum | 12 +- .../internal/config/apikey.go | 111 +++++++++++----- .../internal/config/apikey_test.go | 120 ++++++++++++++++++ .../internal/config/config.go | 4 +- .../internal/config/config_test.go | 85 +++++++++++++ 6 files changed, 302 insertions(+), 45 deletions(-) create mode 100644 aws/logs_monitoring_go/internal/config/apikey_test.go create mode 100644 aws/logs_monitoring_go/internal/config/config_test.go diff --git a/aws/logs_monitoring_go/go.mod b/aws/logs_monitoring_go/go.mod index 839db3ace..ed0cb1d58 100644 --- a/aws/logs_monitoring_go/go.mod +++ b/aws/logs_monitoring_go/go.mod @@ -2,11 +2,14 @@ module github.com/DataDog/datadog-serverless-functions/aws/logs_monitoring_go go 1.26 -require github.com/aws/aws-lambda-go v1.53.0 +require ( + github.com/aws/aws-lambda-go v1.53.0 + github.com/aws/aws-sdk-go-v2 v1.41.4 + github.com/aws/aws-sdk-go-v2/config v1.32.12 + github.com/stretchr/testify v1.11.1 +) require ( - github.com/aws/aws-sdk-go-v2 v1.41.4 // indirect - github.com/aws/aws-sdk-go-v2/config v1.32.12 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.12 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect @@ -14,12 +17,12 @@ require ( github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect - github.com/aws/aws-sdk-go-v2/service/kms v1.50.3 // indirect - github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.41.4 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect - github.com/aws/aws-sdk-go-v2/service/ssm v1.68.3 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect github.com/aws/smithy-go v1.24.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/aws/logs_monitoring_go/go.sum b/aws/logs_monitoring_go/go.sum index 51a3fd7c0..42874c9c2 100644 --- a/aws/logs_monitoring_go/go.sum +++ b/aws/logs_monitoring_go/go.sum @@ -18,14 +18,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhL github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk= -github.com/aws/aws-sdk-go-v2/service/kms v1.50.3 h1:s/zDSG/a/Su9aX+v0Ld9cimUCdkr5FWPmBV8owaEbZY= -github.com/aws/aws-sdk-go-v2/service/kms v1.50.3/go.mod h1:/iSgiUor15ZuxFGQSTf3lA2FmKxFsQoc2tADOarQBSw= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.41.4 h1:9aZbO86sraeCIHHCpZhxwN9tnVy9POkSKzi4/TpT54A= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.41.4/go.mod h1:cxiXDhEzIq7Xx1BtmC4lGBK3SwAZ79+EUWiKawYHo14= github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow= github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE= -github.com/aws/aws-sdk-go-v2/service/ssm v1.68.3 h1:bBoWhx8lsFLTXintRX64ZBXcmFZbGqUmaPUrjXECqIc= -github.com/aws/aws-sdk-go-v2/service/ssm v1.68.3/go.mod h1:rcRkKbUJ2437WuXdq9fbj+MjTudYWzY9Ct8kiBbN8a8= github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o= github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU= @@ -38,7 +32,9 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/aws/logs_monitoring_go/internal/config/apikey.go b/aws/logs_monitoring_go/internal/config/apikey.go index 8f82a83d4..9f7e9db32 100644 --- a/aws/logs_monitoring_go/internal/config/apikey.go +++ b/aws/logs_monitoring_go/internal/config/apikey.go @@ -7,9 +7,11 @@ package config import ( "context" + "crypto/tls" "errors" "fmt" "log/slog" + "net/http" "os" "time" @@ -19,7 +21,9 @@ import ( ) const ( - awsClientTimeout = 5 * time.Second + httpClientTimeout = 10 * time.Second + maxRetries = 5 + retryBackoffFactor = 1 * time.Second ) type resolveOptions struct { @@ -29,6 +33,14 @@ type resolveOptions struct { type apiKeyResolver func(ctx context.Context, opts resolveOptions) (string, error) +var retryableStatusCodes = map[int]bool{ + 429: true, + 500: true, + 502: true, + 503: true, + 504: true, +} + var resolvers = []struct { envVar string resolve apiKeyResolver @@ -41,7 +53,7 @@ var resolvers = []struct { func (c *Config) resolveAPIKey(ctx context.Context) error { awsCfg, err := awsconfig.LoadDefaultConfig(ctx, - awsconfig.WithHTTPClient(awshttp.NewBuildableClient().WithTimeout(awsClientTimeout)), + awsconfig.WithHTTPClient(awshttp.NewBuildableClient().WithTimeout(httpClientTimeout)), ) if err != nil { @@ -65,6 +77,74 @@ func (c *Config) resolveAPIKey(ctx context.Context) error { return errors.New("no API key configured: set DD_API_KEY_SECRET_ARN, DD_API_KEY_SSM_NAME, DD_KMS_API_KEY, or DD_API_KEY. See: https://docs.datadoghq.com/serverless/forwarder/") } +// Note: the API key verification could fail (e.g. Datadog verification endpoint or network problem) +// Instead of failing the whole lambda at startup, it should run up to the log sending part and verify the +// key at this moment, adding the run to the future retry logic in such case. +// The method may disappear in the future. +func (c *Config) validateAPIKey() error { + if c.APIKey == "" || c.APIKey == "" { + return errors.New("missing Datadog API key. Set DD_API_KEY environment variable. See: https://docs.datadoghq.com/serverless/forwarder/") + } + + if len(c.APIKey) != 32 { + return fmt.Errorf("invalid Datadog API key format: expected 32 characters, got %d. Verify your API key at https://app.%s/organization-settings/api-keys", len(c.APIKey), c.Site) + } + + slog.Debug("validating Datadog API key") + + client := &http.Client{ + Timeout: httpClientTimeout, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: c.SkipSSLValidation, + }, + }, + } + + url := fmt.Sprintf("%s/api/v1/validate?api_key=%s", c.APIURL, c.APIKey) + + var lastErr error + var lastStatus int + for attempt := range maxRetries { + resp, err := client.Get(url) + if err != nil { + lastErr = err + slog.Debug("API key validation request failed, retrying", "attempt", attempt+1, "error", err) + time.Sleep(retryBackoffFactor * time.Duration(1<= 200 && resp.StatusCode < 300 { + return nil + } + + if !retryableStatusCodes[resp.StatusCode] { + slog.Warn("API key validation failed. Verify your API key is correct and DD_SITE matches your Datadog account region. See: https://docs.datadoghq.com/getting_started/site/", + "status", resp.StatusCode, + "site", c.Site, + ) + return nil + } + + lastStatus = resp.StatusCode + slog.Debug("API key validation returned retryable status, retrying", "attempt", attempt+1, "status", resp.StatusCode) + time.Sleep(retryBackoffFactor * time.Duration(1< Date: Fri, 20 Mar 2026 16:00:54 +0100 Subject: [PATCH 05/16] fix --- aws/logs_monitoring_go/internal/config/apikey.go | 5 ++++- aws/logs_monitoring_go/internal/config/config.go | 3 ++- aws/logs_monitoring_go/internal/config/config_test.go | 7 ++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/aws/logs_monitoring_go/internal/config/apikey.go b/aws/logs_monitoring_go/internal/config/apikey.go index 9f7e9db32..5eb252102 100644 --- a/aws/logs_monitoring_go/internal/config/apikey.go +++ b/aws/logs_monitoring_go/internal/config/apikey.go @@ -13,6 +13,7 @@ import ( "log/slog" "net/http" "os" + "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -70,7 +71,8 @@ func (c *Config) resolveAPIKey(ctx context.Context) error { if err != nil { return fmt.Errorf("resolving API key from %s: %w", resolver.envVar, err) } - c.APIKey = key + c.APIKey = strings.TrimSpace(key) + slog.Debug("API key resolved", "source", resolver.envVar) return nil } } @@ -116,6 +118,7 @@ func (c *Config) validateAPIKey() error { resp.Body.Close() if resp.StatusCode >= 200 && resp.StatusCode < 300 { + slog.Debug("API key validated successfully") return nil } diff --git a/aws/logs_monitoring_go/internal/config/config.go b/aws/logs_monitoring_go/internal/config/config.go index fe9748a86..ca29c2e52 100644 --- a/aws/logs_monitoring_go/internal/config/config.go +++ b/aws/logs_monitoring_go/internal/config/config.go @@ -50,6 +50,7 @@ func Load(ctx context.Context) (*Config, error) { } cfg.deriveURLs() + slog.Debug("config loaded", "site", cfg.Site, "intakeURL", cfg.IntakeURL, "apiURL", cfg.APIURL, "port", cfg.Port, "noSSL", cfg.NoSSL, "useFIPS", cfg.UseFIPS) logDroppedEnvVars() @@ -69,7 +70,7 @@ func (c *Config) deriveURLs() { if c.NoSSL { scheme = "http" } - c.IntakeURL = envOrDefault("DD_URL", "http-intake.logs."+c.Site) + c.IntakeURL = envOrDefault("DD_URL", fmt.Sprintf("%s://http-intake.logs.%s", scheme, c.Site)) c.APIURL = envOrDefault("DD_API_URL", fmt.Sprintf("%s://api.%s", scheme, c.Site)) } diff --git a/aws/logs_monitoring_go/internal/config/config_test.go b/aws/logs_monitoring_go/internal/config/config_test.go index e407a8f7a..96283a313 100644 --- a/aws/logs_monitoring_go/internal/config/config_test.go +++ b/aws/logs_monitoring_go/internal/config/config_test.go @@ -11,20 +11,21 @@ func TestDeriveURLs(t *testing.T) { t.Run("defaults", func(t *testing.T) { cfg := &Config{Site: "datadoghq.com", NoSSL: false} cfg.deriveURLs() - assert.Equal(t, "http-intake.logs.datadoghq.com", cfg.IntakeURL) + assert.Equal(t, "https://http-intake.logs.datadoghq.com", cfg.IntakeURL) assert.Equal(t, "https://api.datadoghq.com", cfg.APIURL) }) t.Run("NoSSL switches scheme to http", func(t *testing.T) { cfg := &Config{Site: "datadoghq.com", NoSSL: true} cfg.deriveURLs() + assert.Equal(t, "http://http-intake.logs.datadoghq.com", cfg.IntakeURL) assert.Equal(t, "http://api.datadoghq.com", cfg.APIURL) }) t.Run("custom site", func(t *testing.T) { cfg := &Config{Site: "datadoghq.eu", NoSSL: false} cfg.deriveURLs() - assert.Equal(t, "http-intake.logs.datadoghq.eu", cfg.IntakeURL) + assert.Equal(t, "https://http-intake.logs.datadoghq.eu", cfg.IntakeURL) assert.Equal(t, "https://api.datadoghq.eu", cfg.APIURL) }) @@ -57,7 +58,7 @@ func TestLoad(t *testing.T) { assert.Equal(t, false, cfg.SkipSSLValidation) assert.Equal(t, "INFO", cfg.LogLevel) assert.Equal(t, false, cfg.UseFIPS) - assert.Equal(t, "http-intake.logs.datadoghq.com", cfg.IntakeURL) + assert.Equal(t, "https://http-intake.logs.datadoghq.com", cfg.IntakeURL) assert.Equal(t, "abcdef1234567890abcdef1234567890", cfg.APIKey) }) From 8f5f86930fef46da8b9aa9e49ba7e7de9c6d969a Mon Sep 17 00:00:00 2001 From: Nabil Dakkoune Date: Fri, 20 Mar 2026 16:06:47 +0100 Subject: [PATCH 06/16] fix --- aws/logs_monitoring_go/internal/config/apikey.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aws/logs_monitoring_go/internal/config/apikey.go b/aws/logs_monitoring_go/internal/config/apikey.go index 5eb252102..dd8ffb908 100644 --- a/aws/logs_monitoring_go/internal/config/apikey.go +++ b/aws/logs_monitoring_go/internal/config/apikey.go @@ -115,7 +115,9 @@ func (c *Config) validateAPIKey() error { time.Sleep(retryBackoffFactor * time.Duration(1<= 200 && resp.StatusCode < 300 { slog.Debug("API key validated successfully") From 52d93e1c89a62550aed44577c5c822ba36773e3f Mon Sep 17 00:00:00 2001 From: Nabil Dakkoune Date: Tue, 24 Mar 2026 17:59:22 +0100 Subject: [PATCH 07/16] refactor codebase, add tooling, add client mocking and tdd tests --- aws/logs_monitoring_go/Makefile | 26 ++- aws/logs_monitoring_go/cmd/forwarder/main.go | 41 ++--- aws/logs_monitoring_go/go.mod | 20 ++- aws/logs_monitoring_go/go.sum | 26 ++- .../internal/config/apikey.go | 165 +++++------------- .../internal/config/apikey_test.go | 124 +------------ .../internal/config/config.go | 99 ++--------- .../internal/config/config_test.go | 132 ++++++-------- .../internal/config/environment.go | 55 ++++++ .../internal/config/environment_test.go | 64 +++++++ aws/logs_monitoring_go/internal/config/kms.go | 16 ++ .../internal/config/logger.go | 19 ++ .../internal/config/secretsmanager.go | 62 +++++++ .../internal/config/secretsmanager_test.go | 95 ++++++++++ aws/logs_monitoring_go/internal/config/ssm.go | 16 ++ 15 files changed, 516 insertions(+), 444 deletions(-) create mode 100644 aws/logs_monitoring_go/internal/config/environment.go create mode 100644 aws/logs_monitoring_go/internal/config/environment_test.go create mode 100644 aws/logs_monitoring_go/internal/config/kms.go create mode 100644 aws/logs_monitoring_go/internal/config/logger.go create mode 100644 aws/logs_monitoring_go/internal/config/secretsmanager.go create mode 100644 aws/logs_monitoring_go/internal/config/secretsmanager_test.go create mode 100644 aws/logs_monitoring_go/internal/config/ssm.go diff --git a/aws/logs_monitoring_go/Makefile b/aws/logs_monitoring_go/Makefile index a3878f199..2c5cb9c3d 100644 --- a/aws/logs_monitoring_go/Makefile +++ b/aws/logs_monitoring_go/Makefile @@ -1,15 +1,12 @@ -.PHONY: build package test lint clean sam-build sam-invoke sam-deploy build-ForwarderFunction - BINARY_NAME := bootstrap ZIP_NAME := forwarder.zip +# Go +.PHONY: build package test lint clean audit + build: GOOS=linux GOARCH=arm64 go build -o $(BINARY_NAME) ./cmd/forwarder/ -# Used by `sam build` -build-ForwarderFunction: - GOOS=linux GOARCH=arm64 go build -o $(ARTIFACTS_DIR)/bootstrap ./cmd/forwarder/ - package: build zip $(ZIP_NAME) $(BINARY_NAME) @@ -19,15 +16,26 @@ test: lint: golangci-lint run ./... +audit: + go vet ./... + go tool staticcheck ./... + go tool govulncheck + clean: rm -f $(BINARY_NAME) $(ZIP_NAME) +# SAM +.PHONY: build-ForwarderFunction sam-build sam-invoke sam-deploy + +build-ForwarderFunction: + GOOS=linux GOARCH=arm64 go build -o $(ARTIFACTS_DIR)/bootstrap ./cmd/forwarder/ + sam-build: sam build +sam-deploy: sam-build + sam deploy + EVENT ?= events/cloudwatch_logs.json sam-invoke: sam-build sam local invoke ForwarderFunction -e $(EVENT) - -sam-deploy: sam-build - sam deploy diff --git a/aws/logs_monitoring_go/cmd/forwarder/main.go b/aws/logs_monitoring_go/cmd/forwarder/main.go index da4d9e964..0bcf3fa57 100644 --- a/aws/logs_monitoring_go/cmd/forwarder/main.go +++ b/aws/logs_monitoring_go/cmd/forwarder/main.go @@ -8,49 +8,28 @@ package main import ( "context" "encoding/json" - "log" "log/slog" - "os" - "strings" "github.com/DataDog/datadog-serverless-functions/aws/logs_monitoring_go/internal/config" "github.com/aws/aws-lambda-go/lambda" ) -func initLogger(level string) { - var slogLevel slog.Level - switch strings.ToUpper(level) { - case "DEBUG": - slogLevel = slog.LevelDebug - case "INFO": - slogLevel = slog.LevelInfo - case "WARNING", "WARN": - slogLevel = slog.LevelWarn - case "ERROR": - slogLevel = slog.LevelError - default: - slogLevel = slog.LevelInfo +func main() { + ctx := context.Background() + cfg, err := config.Load(ctx) + if err != nil { + slog.Error("config load failed", slog.String("error", err.Error())) + return } - slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{ - Level: slogLevel, - }))) + + lambda.Start(handleRequest(cfg)) } +// cfg not used for now, will be when forwarding logic added func handleRequest(cfg *config.Config) func(context.Context, json.RawMessage) error { return func(ctx context.Context, event json.RawMessage) error { - slog.Info("received event", "event", string(event)) + slog.Info("received event", slog.String("event", string(event))) return nil } } - -func main() { - ctx := context.Background() - cfg, err := config.Load(ctx) - if err != nil { - log.Fatalf("config: %v", err) - } - // TODO: exit if forwading disabled ? - initLogger(cfg.LogLevel) - lambda.Start(handleRequest(cfg)) -} diff --git a/aws/logs_monitoring_go/go.mod b/aws/logs_monitoring_go/go.mod index ed0cb1d58..9b5d63d3e 100644 --- a/aws/logs_monitoring_go/go.mod +++ b/aws/logs_monitoring_go/go.mod @@ -6,7 +6,8 @@ require ( github.com/aws/aws-lambda-go v1.53.0 github.com/aws/aws-sdk-go-v2 v1.41.4 github.com/aws/aws-sdk-go-v2/config v1.32.12 - github.com/stretchr/testify v1.11.1 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.41.4 + github.com/google/go-cmp v0.7.0 ) require ( @@ -22,7 +23,18 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect github.com/aws/smithy-go v1.24.2 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + github.com/stretchr/testify v1.11.1 // indirect + go.uber.org/mock v0.6.0 // indirect + golang.org/x/mod v0.34.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.42.0 // indirect + golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c // indirect + golang.org/x/tools v0.43.0 // indirect + golang.org/x/vuln v1.1.4 // indirect +) + +tool ( + go.uber.org/mock/mockgen + golang.org/x/tools/cmd/goimports + golang.org/x/vuln/cmd/govulncheck ) diff --git a/aws/logs_monitoring_go/go.sum b/aws/logs_monitoring_go/go.sum index 42874c9c2..994d196a5 100644 --- a/aws/logs_monitoring_go/go.sum +++ b/aws/logs_monitoring_go/go.sum @@ -18,6 +18,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhL github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.41.4 h1:9aZbO86sraeCIHHCpZhxwN9tnVy9POkSKzi4/TpT54A= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.41.4/go.mod h1:cxiXDhEzIq7Xx1BtmC4lGBK3SwAZ79+EUWiKawYHo14= github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow= github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE= github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o= @@ -30,11 +32,31 @@ github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c h1:6a8FdnNk6bTXBjR4AGKFgUKuo+7GnR3FX5L7CbveeZc= +golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c/go.mod h1:TpUTTEp9frx7rTdLpC9gFG9kdI7zVLFTFFlqaH2Cncw= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= +golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= +golang.org/x/vuln v1.1.4 h1:Ju8QsuyhX3Hk8ma3CesTbO8vfJD9EvUBgHvkxHBzj0I= +golang.org/x/vuln v1.1.4/go.mod h1:F+45wmU18ym/ca5PLTPLsSzr2KppzswxPP603ldA67s= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/aws/logs_monitoring_go/internal/config/apikey.go b/aws/logs_monitoring_go/internal/config/apikey.go index dd8ffb908..6bce047e2 100644 --- a/aws/logs_monitoring_go/internal/config/apikey.go +++ b/aws/logs_monitoring_go/internal/config/apikey.go @@ -7,161 +7,82 @@ package config import ( "context" - "crypto/tls" "errors" "fmt" + "io" "log/slog" "net/http" "os" - "strings" "time" - "github.com/aws/aws-sdk-go-v2/aws" awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" awsconfig "github.com/aws/aws-sdk-go-v2/config" ) -const ( - httpClientTimeout = 10 * time.Second - maxRetries = 5 - retryBackoffFactor = 1 * time.Second +var ( + ErrMissingAPIKey = errors.New("missing Datadog API key") + ErrInvalidAPIKey = errors.New("invalid Datadog API key format") ) -type resolveOptions struct { - AWSCfg aws.Config - Value string -} - -type apiKeyResolver func(ctx context.Context, opts resolveOptions) (string, error) - -var retryableStatusCodes = map[int]bool{ - 429: true, - 500: true, - 502: true, - 503: true, - 504: true, -} - -var resolvers = []struct { - envVar string - resolve apiKeyResolver -}{ - {"DD_API_KEY_SECRET_ARN", resolveFromSecretsManager}, - {"DD_API_KEY_SSM_NAME", resolveFromSSM}, - {"DD_KMS_API_KEY", resolveFromKMS}, - {"DD_API_KEY", resolveFromEnv}, -} +const ( + httpClientTimeout = 3 * time.Second +) func (c *Config) resolveAPIKey(ctx context.Context) error { - awsCfg, err := awsconfig.LoadDefaultConfig(ctx, - awsconfig.WithHTTPClient(awshttp.NewBuildableClient().WithTimeout(httpClientTimeout)), - ) - + awsCfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithHTTPClient(awshttp.NewBuildableClient().WithTimeout(httpClientTimeout))) if err != nil { return fmt.Errorf("loading AWS config: %w", err) } - for _, resolver := range resolvers { - if v, ok := os.LookupEnv(resolver.envVar); ok { - slog.Debug("resolving API key", "source", resolver.envVar) - key, err := resolver.resolve(ctx, resolveOptions{ - AWSCfg: awsCfg, - Value: v, - }) - if err != nil { - return fmt.Errorf("resolving API key from %s: %w", resolver.envVar, err) - } - c.APIKey = strings.TrimSpace(key) - slog.Debug("API key resolved", "source", resolver.envVar) - return nil - } + if v, ok := os.LookupEnv("DD_API_KEY_SECRET_ARN"); ok { + return c.resolveFromSecretsManager(ctx, awsCfg, v) + } + + if v, ok := os.LookupEnv("DD_API_KEY_SSM_NAME"); ok { + return c.resolveFromSSM(ctx, awsCfg, v) + } + + if v, ok := os.LookupEnv("DD_KMS_API_KEY"); ok { + return c.resolveFromKMS(ctx, awsCfg, v) } - return errors.New("no API key configured: set DD_API_KEY_SECRET_ARN, DD_API_KEY_SSM_NAME, DD_KMS_API_KEY, or DD_API_KEY. See: https://docs.datadoghq.com/serverless/forwarder/") + + return errors.New("no API key configured: set DD_API_KEY_SECRET_ARN, DD_API_KEY_SSM_NAME or DD_KMS_API_KEY. See: https://docs.datadoghq.com/serverless/forwarder/") } -// Note: the API key verification could fail (e.g. Datadog verification endpoint or network problem) -// Instead of failing the whole lambda at startup, it should run up to the log sending part and verify the -// key at this moment, adding the run to the future retry logic in such case. -// The method may disappear in the future. -func (c *Config) validateAPIKey() error { - if c.APIKey == "" || c.APIKey == "" { - return errors.New("missing Datadog API key. Set DD_API_KEY environment variable. See: https://docs.datadoghq.com/serverless/forwarder/") +func (c *Config) validateAPIKey(ctx context.Context) error { + if c.APIKey == "" { + return fmt.Errorf("%w: set DD_API_KEY_SECRET_ARN, DD_API_KEY_SSM_NAME or DD_KMS_API_KEY. See: https://docs.datadoghq.com/serverless/forwarder/", ErrMissingAPIKey) } if len(c.APIKey) != 32 { - return fmt.Errorf("invalid Datadog API key format: expected 32 characters, got %d. Verify your API key at https://app.%s/organization-settings/api-keys", len(c.APIKey), c.Site) + return fmt.Errorf("%w: expected 32 characters, got %d. Verify your API key at https://app.%s/organization-settings/api-keys", ErrInvalidAPIKey, len(c.APIKey), c.Site) } slog.Debug("validating Datadog API key") - client := &http.Client{ - Timeout: httpClientTimeout, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: c.SkipSSLValidation, - }, - }, + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.APIURL+"/api/v1/validate", nil) + if err != nil { + slog.Warn("failed to build API key validation request", slog.String("error", err.Error())) + return nil } + req.Header.Set("DD-API-KEY", c.APIKey) - url := fmt.Sprintf("%s/api/v1/validate?api_key=%s", c.APIURL, c.APIKey) - - var lastErr error - var lastStatus int - for attempt := range maxRetries { - resp, err := client.Get(url) - if err != nil { - lastErr = err - slog.Debug("API key validation request failed, retrying", "attempt", attempt+1, "error", err) - time.Sleep(retryBackoffFactor * time.Duration(1<= 200 && resp.StatusCode < 300 { - slog.Debug("API key validated successfully") - return nil - } - - if !retryableStatusCodes[resp.StatusCode] { - slog.Warn("API key validation failed. Verify your API key is correct and DD_SITE matches your Datadog account region. See: https://docs.datadoghq.com/getting_started/site/", - "status", resp.StatusCode, - "site", c.Site, - ) - return nil - } - - lastStatus = resp.StatusCode - slog.Debug("API key validation returned retryable status, retrying", "attempt", attempt+1, "status", resp.StatusCode) - time.Sleep(retryBackoffFactor * time.Duration(1< Date: Wed, 25 Mar 2026 15:53:44 +0100 Subject: [PATCH 08/16] using mockgen + take comment suggestions --- aws/logs_monitoring_go/Makefile | 21 +++- aws/logs_monitoring_go/cmd/forwarder/main.go | 2 +- .../internal/config/apikey.go | 27 +++-- .../internal/config/apikey_test.go | 6 - .../internal/config/secretsmanager.go | 30 +++-- .../internal/config/secretsmanager_mockgen.go | 62 +++++++++++ .../internal/config/secretsmanager_test.go | 105 +++++++++--------- 7 files changed, 163 insertions(+), 90 deletions(-) delete mode 100644 aws/logs_monitoring_go/internal/config/apikey_test.go create mode 100644 aws/logs_monitoring_go/internal/config/secretsmanager_mockgen.go diff --git a/aws/logs_monitoring_go/Makefile b/aws/logs_monitoring_go/Makefile index 2c5cb9c3d..4643223a6 100644 --- a/aws/logs_monitoring_go/Makefile +++ b/aws/logs_monitoring_go/Makefile @@ -2,40 +2,53 @@ BINARY_NAME := bootstrap ZIP_NAME := forwarder.zip # Go -.PHONY: build package test lint clean audit +.PHONY: build build: - GOOS=linux GOARCH=arm64 go build -o $(BINARY_NAME) ./cmd/forwarder/ + GOOS=linux GOARCH=arm64 go build -ldflags="-s" -installsuffix nocgo -o $(BINARY_NAME) ./cmd/forwarder/ +.PHONY: package package: build zip $(ZIP_NAME) $(BINARY_NAME) +.PHONY: test test: go test -race ./... +.PHONY: lint lint: golangci-lint run ./... +.PHONY: audit audit: go vet ./... go tool staticcheck ./... go tool govulncheck +.PHONY: generate +generate: + go generate ./... + +.PHONY: clean clean: rm -f $(BINARY_NAME) $(ZIP_NAME) # SAM -.PHONY: build-ForwarderFunction sam-build sam-invoke sam-deploy +.PHONY: build-ForwarderFunction build-ForwarderFunction: - GOOS=linux GOARCH=arm64 go build -o $(ARTIFACTS_DIR)/bootstrap ./cmd/forwarder/ + GOOS=linux GOARCH=arm64 go build -ldflags="-s" -installsuffix nocgo -o $(ARTIFACTS_DIR)/bootstrap ./cmd/forwarder/ +.PHONY: sam-build sam-build: sam build +.PHONY: sam-deploy sam-deploy: sam-build sam deploy EVENT ?= events/cloudwatch_logs.json + +.PHONY: sam-invoke sam-invoke: sam-build sam local invoke ForwarderFunction -e $(EVENT) diff --git a/aws/logs_monitoring_go/cmd/forwarder/main.go b/aws/logs_monitoring_go/cmd/forwarder/main.go index 0bcf3fa57..5b8f8649b 100644 --- a/aws/logs_monitoring_go/cmd/forwarder/main.go +++ b/aws/logs_monitoring_go/cmd/forwarder/main.go @@ -19,7 +19,7 @@ func main() { ctx := context.Background() cfg, err := config.Load(ctx) if err != nil { - slog.Error("config load failed", slog.String("error", err.Error())) + slog.Error("config load failed", slog.Any("error", err)) return } diff --git a/aws/logs_monitoring_go/internal/config/apikey.go b/aws/logs_monitoring_go/internal/config/apikey.go index 6bce047e2..eacb84921 100644 --- a/aws/logs_monitoring_go/internal/config/apikey.go +++ b/aws/logs_monitoring_go/internal/config/apikey.go @@ -25,7 +25,7 @@ var ( ) const ( - httpClientTimeout = 3 * time.Second + httpClientTimeout = 5 * time.Second ) func (c *Config) resolveAPIKey(ctx context.Context) error { @@ -35,7 +35,16 @@ func (c *Config) resolveAPIKey(ctx context.Context) error { } if v, ok := os.LookupEnv("DD_API_KEY_SECRET_ARN"); ok { - return c.resolveFromSecretsManager(ctx, awsCfg, v) + client, err := c.createSecretsManagerAPIClient(ctx, awsCfg) + if err != nil { + return fmt.Errorf("creating Secrets Manager client: %w", err) + } + apiKey, err := resolveFromSecretsManager(ctx, client, v) + if err != nil { + return fmt.Errorf("resolving from secrets manager: %w", err) + } + c.APIKey = apiKey + return nil } if v, ok := os.LookupEnv("DD_API_KEY_SSM_NAME"); ok { @@ -55,14 +64,14 @@ func (c *Config) validateAPIKey(ctx context.Context) error { } if len(c.APIKey) != 32 { - return fmt.Errorf("%w: expected 32 characters, got %d. Verify your API key at https://app.%s/organization-settings/api-keys", ErrInvalidAPIKey, len(c.APIKey), c.Site) + return fmt.Errorf("expected 32 characters, got %d. Verify your API key at https://app.%s/organization-settings/api-keys: %w", len(c.APIKey), c.Site, ErrInvalidAPIKey) } slog.Debug("validating Datadog API key") req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.APIURL+"/api/v1/validate", nil) if err != nil { - slog.Warn("failed to build API key validation request", slog.String("error", err.Error())) + slog.Warn("failed to build API key validation request", slog.Any("error", err)) return nil } req.Header.Set("DD-API-KEY", c.APIKey) @@ -70,12 +79,16 @@ func (c *Config) validateAPIKey(ctx context.Context) error { client := &http.Client{Timeout: httpClientTimeout} resp, err := client.Do(req) if err != nil { - slog.Warn("failed to validate API key", slog.String("error", err.Error())) + slog.Warn("failed to validate API key", slog.Any("error", err)) return nil } defer func() { - io.Copy(io.Discard, resp.Body) - resp.Body.Close() + if _, err := io.Copy(io.Discard, resp.Body); err != nil { + slog.Warn("failed to drain response body", slog.Any("error", err)) + } + if err := resp.Body.Close(); err != nil { + slog.Warn("failed to close response body", slog.Any("error", err)) + } }() if resp.StatusCode == http.StatusForbidden { diff --git a/aws/logs_monitoring_go/internal/config/apikey_test.go b/aws/logs_monitoring_go/internal/config/apikey_test.go deleted file mode 100644 index a55623328..000000000 --- a/aws/logs_monitoring_go/internal/config/apikey_test.go +++ /dev/null @@ -1,6 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2026-Present Datadog, Inc. - -package config diff --git a/aws/logs_monitoring_go/internal/config/secretsmanager.go b/aws/logs_monitoring_go/internal/config/secretsmanager.go index 85fd4ce17..bb00e398a 100644 --- a/aws/logs_monitoring_go/internal/config/secretsmanager.go +++ b/aws/logs_monitoring_go/internal/config/secretsmanager.go @@ -5,6 +5,8 @@ package config +//go:generate go tool mockgen -source=secretsmanager.go -package=config -destination=secretsmanager_mockgen.go + import ( "context" "fmt" @@ -15,18 +17,11 @@ import ( "github.com/aws/aws-sdk-go-v2/service/secretsmanager" ) -type SecretsManager interface { +type SecretsManagerAPIClient interface { GetSecretValue(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) } -// used for mocking purpose -var getSecretsManagerClient = func(cfg aws.Config, optFns ...func(*secretsmanager.Options)) SecretsManager { - return secretsmanager.NewFromConfig(cfg, optFns...) -} - -func (c *Config) resolveFromSecretsManager(ctx context.Context, awsCfg aws.Config, arn string) error { - slog.Debug("resolving API key from Secrets Manager") - +func (c *Config) createSecretsManagerAPIClient(ctx context.Context, awsCfg aws.Config) (SecretsManagerAPIClient, error) { resolver := secretsmanager.NewDefaultEndpointResolverV2() params := secretsmanager.EndpointParameters{ Region: aws.String(awsCfg.Region), @@ -40,23 +35,24 @@ func (c *Config) resolveFromSecretsManager(ctx context.Context, awsCfg aws.Confi endpoint, err = resolver.ResolveEndpoint(ctx, params) } if err != nil { - return fmt.Errorf("resolve endpoint: %w", err) + return nil, fmt.Errorf("resolve endpoint: %w", err) } - client := getSecretsManagerClient(awsCfg, func(o *secretsmanager.Options) { + return secretsmanager.NewFromConfig(awsCfg, func(o *secretsmanager.Options) { o.BaseEndpoint = aws.String(endpoint.URI.String()) - }) + }), nil +} +func resolveFromSecretsManager(ctx context.Context, client SecretsManagerAPIClient, arn string) (string, error) { result, err := client.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{ SecretId: aws.String(arn), }) if err != nil { - return fmt.Errorf("fetching secret %s: %w", arn, err) + return "", fmt.Errorf("fetching secret %s: %w", arn, err) } - if result.SecretString == nil { - return fmt.Errorf("secret %s has no string value", arn) + return "", fmt.Errorf("secret %s has no string value", arn) } - c.APIKey = strings.TrimSpace(*result.SecretString) - return nil + + return strings.TrimSpace(*result.SecretString), nil } diff --git a/aws/logs_monitoring_go/internal/config/secretsmanager_mockgen.go b/aws/logs_monitoring_go/internal/config/secretsmanager_mockgen.go new file mode 100644 index 000000000..e5f05e3ee --- /dev/null +++ b/aws/logs_monitoring_go/internal/config/secretsmanager_mockgen.go @@ -0,0 +1,62 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: secretsmanager.go +// +// Generated by this command: +// +// mockgen -source=secretsmanager.go -package=config -destination=secretsmanager_mockgen.go +// + +// Package config is a generated GoMock package. +package config + +import ( + context "context" + reflect "reflect" + + secretsmanager "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + gomock "go.uber.org/mock/gomock" +) + +// MockSecretsManagerAPIClient is a mock of SecretsManagerAPIClient interface. +type MockSecretsManagerAPIClient struct { + ctrl *gomock.Controller + recorder *MockSecretsManagerAPIClientMockRecorder + isgomock struct{} +} + +// MockSecretsManagerAPIClientMockRecorder is the mock recorder for MockSecretsManagerAPIClient. +type MockSecretsManagerAPIClientMockRecorder struct { + mock *MockSecretsManagerAPIClient +} + +// NewMockSecretsManagerAPIClient creates a new mock instance. +func NewMockSecretsManagerAPIClient(ctrl *gomock.Controller) *MockSecretsManagerAPIClient { + mock := &MockSecretsManagerAPIClient{ctrl: ctrl} + mock.recorder = &MockSecretsManagerAPIClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSecretsManagerAPIClient) EXPECT() *MockSecretsManagerAPIClientMockRecorder { + return m.recorder +} + +// GetSecretValue mocks base method. +func (m *MockSecretsManagerAPIClient) GetSecretValue(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, params} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetSecretValue", varargs...) + ret0, _ := ret[0].(*secretsmanager.GetSecretValueOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSecretValue indicates an expected call of GetSecretValue. +func (mr *MockSecretsManagerAPIClientMockRecorder) GetSecretValue(ctx, params any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, params}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSecretValue", reflect.TypeOf((*MockSecretsManagerAPIClient)(nil).GetSecretValue), varargs...) +} diff --git a/aws/logs_monitoring_go/internal/config/secretsmanager_test.go b/aws/logs_monitoring_go/internal/config/secretsmanager_test.go index 2591a8e04..22a9430e5 100644 --- a/aws/logs_monitoring_go/internal/config/secretsmanager_test.go +++ b/aws/logs_monitoring_go/internal/config/secretsmanager_test.go @@ -12,72 +12,67 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "go.uber.org/mock/gomock" ) -func GetSecretValueFromSecretsManager(ctx context.Context, api SecretsManagerGetSecretValueAPI, secretId string) (string, error) { - secret, err := api.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{ - SecretId: &secretId, - }) - if err != nil { - return "", err - } - if secret == nil { - return "", errors.New("got nil secret") - } - if secret.SecretString == nil { - return "", errors.New("secret has no string value") - } - return *secret.SecretString, nil -} - -type mockGetSecretValueAPI func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) - -func (m mockGetSecretValueAPI) GetSecretValue(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { - return m(ctx, params, optFns...) -} - -func TestGetSecretFromSecretsManager(t *testing.T) { +func TestResolveFromSecretsManager(t *testing.T) { tests := map[string]struct { - client func(t *testing.T) SecretsManagerGetSecretValueAPI - secretId string - expect string - wantErr bool + mockSetup func(m *MockSecretsManagerAPIClient) + arn string + wantKey string + wantErr bool }{ - "default": { - client: func(t *testing.T) SecretsManagerGetSecretValueAPI { - return mockGetSecretValueAPI(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { - t.Helper() - if params.SecretId == nil { - t.Fatal("expect SecretId not to be nil") - } - return &secretsmanager.GetSecretValueOutput{ - SecretString: aws.String("my-32-characters-datadog-api-key"), - }, nil - }) + "success": { + mockSetup: func(m *MockSecretsManagerAPIClient) { + m.EXPECT(). + GetSecretValue(gomock.Any(), gomock.Any()). + Return(&secretsmanager.GetSecretValueOutput{ + SecretString: aws.String("abcdef1234567890abcdef1234567890"), + }, nil) + }, + arn: "arn:aws:secretsmanager:us-east-1:012345678901:secret:my-secret", + wantKey: "abcdef1234567890abcdef1234567890", + }, + "whitespace_trimmed": { + mockSetup: func(m *MockSecretsManagerAPIClient) { + m.EXPECT(). + GetSecretValue(gomock.Any(), gomock.Any()). + Return(&secretsmanager.GetSecretValueOutput{ + SecretString: aws.String(" abcdef1234567890abcdef1234567890 \n"), + }, nil) }, - secretId: "arn:aws:secretsmanager:us-east-1:012345678901:secret:my-secret-abcdef", - expect: "my-32-characters-datadog-api-key", + arn: "arn:aws:secretsmanager:us-east-1:012345678901:secret:my-secret", + wantKey: "abcdef1234567890abcdef1234567890", + }, + "aws_error": { + mockSetup: func(m *MockSecretsManagerAPIClient) { + m.EXPECT(). + GetSecretValue(gomock.Any(), gomock.Any()). + Return(nil, errors.New("AccessDeniedException: access denied")) + }, + arn: "arn:aws:secretsmanager:us-east-1:012345678901:secret:my-secret", + wantErr: true, }, "nil_secret_string": { - client: func(t *testing.T) SecretsManagerGetSecretValueAPI { - return mockGetSecretValueAPI(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { - t.Helper() - if params.SecretId == nil { - t.Fatal("expect SecretId not to be nil") - } - return &secretsmanager.GetSecretValueOutput{ + mockSetup: func(m *MockSecretsManagerAPIClient) { + m.EXPECT(). + GetSecretValue(gomock.Any(), gomock.Any()). + Return(&secretsmanager.GetSecretValueOutput{ SecretString: nil, - }, nil - }) + }, nil) }, - secretId: "arn:aws:secretsmanager:us-east-1:012345678901:secret:my-secret-abcdef", - wantErr: true, + arn: "arn:aws:secretsmanager:us-east-1:012345678901:secret:my-secret", + wantErr: true, }, } for name, tc := range tests { t.Run(name, func(t *testing.T) { - secretId, err := GetSecretValueFromSecretsManager(context.Background(), tc.client(t), tc.secretId) + ctrl := gomock.NewController(t) + mock := NewMockSecretsManagerAPIClient(ctrl) + tc.mockSetup(mock) + + got, err := resolveFromSecretsManager(context.Background(), mock, tc.arn) if tc.wantErr { if err == nil { t.Fatal("expected error, got nil") @@ -85,10 +80,10 @@ func TestGetSecretFromSecretsManager(t *testing.T) { return } if err != nil { - t.Errorf("expect no error, got %v", err) + t.Fatalf("unexpected error: %v", err) } - if secretId != tc.expect { - t.Errorf("expect %v, got %v", tc.expect, secretId) + if got != tc.wantKey { + t.Errorf("got %q, want %q", got, tc.wantKey) } }) } From 553920870ba97201533ed62ead3f1edc72f39164 Mon Sep 17 00:00:00 2001 From: Nabil Dakkoune Date: Wed, 25 Mar 2026 15:54:59 +0100 Subject: [PATCH 09/16] update makefile --- aws/logs_monitoring_go/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/logs_monitoring_go/Makefile b/aws/logs_monitoring_go/Makefile index 4643223a6..682f0cafc 100644 --- a/aws/logs_monitoring_go/Makefile +++ b/aws/logs_monitoring_go/Makefile @@ -35,8 +35,8 @@ clean: # SAM -.PHONY: build-ForwarderFunction -build-ForwarderFunction: +.PHONY: build-forwarder-function +build-forwarder-function: GOOS=linux GOARCH=arm64 go build -ldflags="-s" -installsuffix nocgo -o $(ARTIFACTS_DIR)/bootstrap ./cmd/forwarder/ .PHONY: sam-build From 42ae7a04a2d7819660fd8a13ee9b5c62c8dba637 Mon Sep 17 00:00:00 2001 From: Nabil Dakkoune Date: Wed, 25 Mar 2026 16:26:40 +0100 Subject: [PATCH 10/16] update makefile --- aws/logs_monitoring_go/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aws/logs_monitoring_go/Makefile b/aws/logs_monitoring_go/Makefile index 682f0cafc..af8ab7429 100644 --- a/aws/logs_monitoring_go/Makefile +++ b/aws/logs_monitoring_go/Makefile @@ -22,7 +22,6 @@ lint: .PHONY: audit audit: go vet ./... - go tool staticcheck ./... go tool govulncheck .PHONY: generate @@ -35,8 +34,9 @@ clean: # SAM -.PHONY: build-forwarder-function -build-forwarder-function: +# Used only by SAM +.PHONY: build-ForwarderFunction +build-ForwarderFunction: GOOS=linux GOARCH=arm64 go build -ldflags="-s" -installsuffix nocgo -o $(ARTIFACTS_DIR)/bootstrap ./cmd/forwarder/ .PHONY: sam-build From 30281d7b68c14708e92a7d6818f5068183cb2e96 Mon Sep 17 00:00:00 2001 From: Nabil Dakkoune Date: Thu, 26 Mar 2026 09:16:12 +0100 Subject: [PATCH 11/16] Update aws/logs_monitoring_go/internal/config/apikey.go Co-authored-by: Vincent Boutour --- aws/logs_monitoring_go/internal/config/apikey.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aws/logs_monitoring_go/internal/config/apikey.go b/aws/logs_monitoring_go/internal/config/apikey.go index eacb84921..f1f9f7beb 100644 --- a/aws/logs_monitoring_go/internal/config/apikey.go +++ b/aws/logs_monitoring_go/internal/config/apikey.go @@ -39,10 +39,12 @@ func (c *Config) resolveAPIKey(ctx context.Context) error { if err != nil { return fmt.Errorf("creating Secrets Manager client: %w", err) } + apiKey, err := resolveFromSecretsManager(ctx, client, v) if err != nil { return fmt.Errorf("resolving from secrets manager: %w", err) } + c.APIKey = apiKey return nil } From 1c88c2d797cd0d1f1d21a6196703db6d70528b7b Mon Sep 17 00:00:00 2001 From: Nabil Dakkoune Date: Thu, 26 Mar 2026 09:16:30 +0100 Subject: [PATCH 12/16] Update aws/logs_monitoring_go/internal/config/apikey.go Co-authored-by: Vincent Boutour --- aws/logs_monitoring_go/internal/config/apikey.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/logs_monitoring_go/internal/config/apikey.go b/aws/logs_monitoring_go/internal/config/apikey.go index f1f9f7beb..f146f9f36 100644 --- a/aws/logs_monitoring_go/internal/config/apikey.go +++ b/aws/logs_monitoring_go/internal/config/apikey.go @@ -62,7 +62,7 @@ func (c *Config) resolveAPIKey(ctx context.Context) error { func (c *Config) validateAPIKey(ctx context.Context) error { if c.APIKey == "" { - return fmt.Errorf("%w: set DD_API_KEY_SECRET_ARN, DD_API_KEY_SSM_NAME or DD_KMS_API_KEY. See: https://docs.datadoghq.com/serverless/forwarder/", ErrMissingAPIKey) + return fmt.Errorf("set DD_API_KEY_SECRET_ARN, DD_API_KEY_SSM_NAME or DD_KMS_API_KEY. See: https://docs.datadoghq.com/serverless/forwarder/: %w", ErrMissingAPIKey) } if len(c.APIKey) != 32 { From 85eac1d98ba73523ff7e7129bdc80b198b17c000 Mon Sep 17 00:00:00 2001 From: Nabil Dakkoune Date: Thu, 26 Mar 2026 09:38:30 +0100 Subject: [PATCH 13/16] Update aws/logs_monitoring_go/internal/config/config_test.go Co-authored-by: Vincent Boutour --- aws/logs_monitoring_go/internal/config/config_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/aws/logs_monitoring_go/internal/config/config_test.go b/aws/logs_monitoring_go/internal/config/config_test.go index d77b2077e..efddcac9f 100644 --- a/aws/logs_monitoring_go/internal/config/config_test.go +++ b/aws/logs_monitoring_go/internal/config/config_test.go @@ -18,7 +18,6 @@ func TestLoadConfig(t *testing.T) { want Config }{ "default": { - env: map[string]string{}, want: Config{ Site: "datadoghq.com", IntakeURL: "https://http-intake.logs.datadoghq.com", From 6a653003547d525b15891e81a111c65bc4f43d54 Mon Sep 17 00:00:00 2001 From: Nabil Dakkoune Date: Thu, 26 Mar 2026 09:44:05 +0100 Subject: [PATCH 14/16] alpha sort --- aws/logs_monitoring_go/internal/config/environment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/logs_monitoring_go/internal/config/environment.go b/aws/logs_monitoring_go/internal/config/environment.go index 9383efaa9..ec75dd092 100644 --- a/aws/logs_monitoring_go/internal/config/environment.go +++ b/aws/logs_monitoring_go/internal/config/environment.go @@ -14,13 +14,13 @@ import ( var deprecatedEnvironmentVariables = []string{ "DD_ADDITIONAL_TARGET_LAMBDAS", "DD_API_KEY", + "DD_COMPRESSION_LEVEL", "DD_ENRICH_CLOUDWATCH_TAGS", "DD_ENRICH_S3_TAGS", "DD_FETCH_LAMBDA_TAGS", "DD_FETCH_LOG_GROUP_TAGS", "DD_FETCH_S3_TAGS", "DD_FETCH_STEP_FUNCTIONS_TAGS", - "DD_COMPRESSION_LEVEL", "DD_FORWARD_LOG", "DD_NO_SSL", "DD_PORT", From ceec656a37fda77d6c48e9d19baf0830acec74d1 Mon Sep 17 00:00:00 2001 From: Nabil Dakkoune Date: Thu, 26 Mar 2026 09:53:45 +0100 Subject: [PATCH 15/16] refactor envOrDefaultBool to use strconv.ParseBool --- aws/logs_monitoring_go/internal/config/environment.go | 9 +++++++-- .../internal/config/environment_test.go | 6 ++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/aws/logs_monitoring_go/internal/config/environment.go b/aws/logs_monitoring_go/internal/config/environment.go index ec75dd092..d7d183699 100644 --- a/aws/logs_monitoring_go/internal/config/environment.go +++ b/aws/logs_monitoring_go/internal/config/environment.go @@ -8,7 +8,7 @@ package config import ( "log/slog" "os" - "strings" + "strconv" ) var deprecatedEnvironmentVariables = []string{ @@ -43,7 +43,12 @@ func envOrDefaultBool(key string, fallback bool) bool { if !ok { return fallback } - return strings.EqualFold(v, "true") + b, err := strconv.ParseBool(v) + if err != nil { + slog.Warn("invalid boolean env var, using default", slog.String("key", key), slog.String("value", v), slog.Bool("default", fallback)) + return fallback + } + return b } func logDroppedEnvVars() { diff --git a/aws/logs_monitoring_go/internal/config/environment_test.go b/aws/logs_monitoring_go/internal/config/environment_test.go index acb0bee5a..d2cf37896 100644 --- a/aws/logs_monitoring_go/internal/config/environment_test.go +++ b/aws/logs_monitoring_go/internal/config/environment_test.go @@ -46,8 +46,10 @@ func TestEnvOrDefaultBool(t *testing.T) { "true_uppercase": {value: "TRUE", set: true, fallback: false, want: true}, "true_mixed_case": {value: "True", set: true, fallback: false, want: true}, "false_value": {value: "false", set: true, fallback: true, want: false}, - "invalid_value": {value: "yes", set: true, fallback: true, want: false}, - "empty_value": {value: "", set: true, fallback: true, want: false}, + "one_is_true": {value: "1", set: true, fallback: false, want: true}, + "zero_is_false": {value: "0", set: true, fallback: true, want: false}, + "invalid_uses_fallback": {value: "yes", set: true, fallback: true, want: true}, + "empty_uses_fallback": {value: "", set: true, fallback: true, want: true}, } for name, tc := range tests { From c388ec3fc1707d8e1323fc720f08808e2451127a Mon Sep 17 00:00:00 2001 From: Nabil Dakkoune Date: Thu, 26 Mar 2026 09:55:55 +0100 Subject: [PATCH 16/16] Update aws/logs_monitoring_go/internal/config/secretsmanager.go Co-authored-by: Vincent Boutour --- aws/logs_monitoring_go/internal/config/secretsmanager.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/aws/logs_monitoring_go/internal/config/secretsmanager.go b/aws/logs_monitoring_go/internal/config/secretsmanager.go index bb00e398a..c96af9415 100644 --- a/aws/logs_monitoring_go/internal/config/secretsmanager.go +++ b/aws/logs_monitoring_go/internal/config/secretsmanager.go @@ -48,10 +48,11 @@ func resolveFromSecretsManager(ctx context.Context, client SecretsManagerAPIClie SecretId: aws.String(arn), }) if err != nil { - return "", fmt.Errorf("fetching secret %s: %w", arn, err) + return "", fmt.Errorf("fetching secret `%s`: %w", arn, err) } + if result.SecretString == nil { - return "", fmt.Errorf("secret %s has no string value", arn) + return "", fmt.Errorf("secret `%s` has no string value", arn) } return strings.TrimSpace(*result.SecretString), nil