add decode function to avoid base64 encoding

Author: ndakkoune

aws/logs_monitoring_go/internal/parsing/cloudwatchlogs.go

@@ -45,6 +45,10 @@ func parseCloudwatchLogs(ctx context.Context, event json.RawMessage, cfg *config
 		return nil, fmt.Errorf("parse: %w", err)
 	}
 
+	return buildCloudwatchLogEntries(ctx, data, cfg)
+}
+
+func buildCloudwatchLogEntries(ctx context.Context, data events.CloudwatchLogsData, cfg *config.Config) ([]model.CloudwatchLogEntry, error) {
 	if data.MessageType == "CONTROL_MESSAGE" {
 		return nil, nil
 	}

aws/logs_monitoring_go/internal/parsing/kinesis.go

@@ -6,8 +6,9 @@
 package parsing
 
 import (
+	"bytes"
+	"compress/gzip"
 	"context"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"log/slog"
@@ -25,18 +26,13 @@ func HandleKinesis(ctx context.Context, event json.RawMessage, cfg *config.Confi
 	}
 
 	for _, record := range kinesisEvent.Records {
-		cwEvent := events.CloudwatchLogsEvent{
-			AWSLogs: events.CloudwatchLogsRawData{
-				Data: base64.StdEncoding.EncodeToString(record.Kinesis.Data),
-			},
-		}
-
-		cwRaw, err := json.Marshal(cwEvent)
+		data, err := decodeCloudwatchLogsData(record.Kinesis.Data)
 		if err != nil {
-			return fmt.Errorf("marshal cloudwatch event from kinesis: %w", err)
+			slog.WarnContext(ctx, "skipping kinesis record", "error", err)
+			continue
 		}
 
-		entries, err := parseCloudwatchLogs(ctx, cwRaw, cfg)
+		entries, err := buildCloudwatchLogEntries(ctx, data, cfg)
 		if err != nil {
 			slog.WarnContext(ctx, "skipping kinesis record", "error", err)
 			continue
@@ -50,3 +46,17 @@ func HandleKinesis(ctx context.Context, event json.RawMessage, cfg *config.Confi
 	}
 	return nil
 }
+
+func decodeCloudwatchLogsData(data []byte) (events.CloudwatchLogsData, error) {
+	zr, err := gzip.NewReader(bytes.NewReader(data))
+	if err != nil {
+		return events.CloudwatchLogsData{}, fmt.Errorf("gzip: %w", err)
+	}
+	defer zr.Close()
+
+	var d events.CloudwatchLogsData
+	if err := json.NewDecoder(zr).Decode(&d); err != nil {
+		return events.CloudwatchLogsData{}, fmt.Errorf("decode: %w", err)
+	}
+	return d, nil
+}