feat(logging): enable file-backed request/response sources for enhanced API logging

- Introduced support for file-backed logging of API requests and responses to handle large payloads efficiently.
- Refactored `attachWebsocketLogSources` to `attachRequestLogSources` for broader request and response handling.
- Added new methods for appending request/response data to file-backed sources and updated existing logging workflows for compatibility.
- Improved cleanup and merge logic for file-backed sources during request processing.
- Updated tests to cover newly introduced file-backed logging functionality.

Author: luispater

config.example.yaml

@@ -49,7 +49,7 @@ pprof:
   enable: false
   addr: "127.0.0.1:8316"
 
-# When true, disable high-overhead HTTP middleware features to reduce per-request memory usage under high concurrency.
+# When true, disable high-overhead request logging and HTTP middleware features to reduce per-request memory usage under high concurrency.
 commercial-mode: false
 
 # When true, write application logs to rotating files instead of stdout

internal/api/middleware/request_logging.go

@@ -58,7 +58,7 @@ func RequestLoggingMiddleware(logger logging.RequestLogger) gin.HandlerFunc {
 			wrapper.logOnErrorOnly = true
 		}
 		c.Writer = wrapper
-		attachWebsocketLogSources(c, logger, loggerEnabled)
+		attachRequestLogSources(c, logger, loggerEnabled)
 
 		// Process the request
 		c.Next()
@@ -75,14 +75,23 @@ type fileBodySourceFactory interface {
 	NewFileBodySource(prefix string) (*logging.FileBodySource, error)
 }
 
-func attachWebsocketLogSources(c *gin.Context, logger logging.RequestLogger, loggerEnabled bool) {
-	if c == nil || !loggerEnabled || !isResponsesWebsocketUpgrade(c.Request) {
+func attachRequestLogSources(c *gin.Context, logger logging.RequestLogger, loggerEnabled bool) {
+	if c == nil || !loggerEnabled {
 		return
 	}
 	factory, ok := logger.(fileBodySourceFactory)
 	if !ok || factory == nil {
 		return
 	}
+	if source, errSource := factory.NewFileBodySource("api-request"); errSource == nil {
+		c.Set(logging.APIRequestSourceContextKey, source)
+	}
+	if source, errSource := factory.NewFileBodySource("api-response"); errSource == nil {
+		c.Set(logging.APIResponseSourceContextKey, source)
+	}
+	if !isResponsesWebsocketUpgrade(c.Request) {
+		return
+	}
 	if source, errSource := factory.NewFileBodySource("websocket-timeline"); errSource == nil {
 		c.Set(logging.WebsocketTimelineSourceContextKey, source)
 	}

internal/api/middleware/request_logging_test.go

@@ -144,7 +144,7 @@ func TestShouldCaptureRequestBody(t *testing.T) {
 	}
 }
 
-func TestAttachWebsocketLogSourcesUsesLoggerLogsDir(t *testing.T) {
+func TestAttachRequestLogSourcesUsesLoggerLogsDir(t *testing.T) {
 	gin.SetMode(gin.TestMode)
 
 	logsDir := t.TempDir()
@@ -154,7 +154,7 @@ func TestAttachWebsocketLogSourcesUsesLoggerLogsDir(t *testing.T) {
 	c.Request = httptest.NewRequest(http.MethodGet, "/v1/responses", nil)
 	c.Request.Header.Set("Upgrade", "websocket")
 
-	attachWebsocketLogSources(c, logger, true)
+	attachRequestLogSources(c, logger, true)
 	defer cleanupFileBodySourcesFromContext(c)
 
 	for _, key := range []string{

internal/api/middleware/response_writer.go

@@ -282,9 +282,11 @@ func (w *ResponseWriterWrapper) Finalize(c *gin.Context) error {
 	hasAPIError := len(slicesAPIResponseError) > 0 || finalStatusCode >= http.StatusBadRequest
 	forceLog := w.logOnErrorOnly && hasAPIError && !w.logger.IsEnabled()
 	websocketTimelineSource := w.extractWebsocketTimelineSource(c)
+	apiRequestSource := w.extractAPIRequestSource(c)
+	apiResponseSource := w.extractAPIResponseSource(c)
 	apiWebsocketTimelineSource := w.extractAPIWebsocketTimelineSource(c)
 	if !w.logger.IsEnabled() && !forceLog {
-		cleanupFileBodySources(websocketTimelineSource, apiWebsocketTimelineSource)
+		cleanupFileBodySources(websocketTimelineSource, apiRequestSource, apiResponseSource, apiWebsocketTimelineSource)
 		return nil
 	}
 
@@ -303,33 +305,63 @@ func (w *ResponseWriterWrapper) Finalize(c *gin.Context) error {
 
 		// Write API Request and Response to the streaming log before closing
 		apiRequest := w.extractAPIRequest(c)
-		if len(apiRequest) > 0 {
-			_ = w.streamWriter.WriteAPIRequest(apiRequest)
-		}
 		apiResponse := w.extractAPIResponse(c)
-		if len(apiResponse) > 0 {
-			_ = w.streamWriter.WriteAPIResponse(apiResponse)
+		if sourceWriter, ok := w.streamWriter.(interface {
+			WriteAPIRequestSource(*logging.FileBodySource) error
+			WriteAPIResponseSource(*logging.FileBodySource) error
+		}); ok {
+			if len(apiRequest) > 0 {
+				_ = w.streamWriter.WriteAPIRequest(apiRequest)
+			}
+			if apiRequestSource != nil && apiRequestSource.HasPayload() {
+				_ = sourceWriter.WriteAPIRequestSource(apiRequestSource)
+			}
+			if len(apiResponse) > 0 {
+				_ = w.streamWriter.WriteAPIResponse(apiResponse)
+			}
+			if apiResponseSource != nil && apiResponseSource.HasPayload() {
+				_ = sourceWriter.WriteAPIResponseSource(apiResponseSource)
+			}
+		} else {
+			var errMerge error
+			apiRequest, errMerge = mergeFileBodySource(apiRequest, apiRequestSource)
+			if errMerge != nil {
+				cleanupFileBodySources(websocketTimelineSource, apiResponseSource, apiWebsocketTimelineSource)
+				return errMerge
+			}
+			apiResponse, errMerge = mergeFileBodySource(apiResponse, apiResponseSource)
+			if errMerge != nil {
+				cleanupFileBodySources(websocketTimelineSource, apiWebsocketTimelineSource)
+				return errMerge
+			}
+			if len(apiRequest) > 0 {
+				_ = w.streamWriter.WriteAPIRequest(apiRequest)
+			}
+			if len(apiResponse) > 0 {
+				_ = w.streamWriter.WriteAPIResponse(apiResponse)
+			}
 		}
 		apiWebsocketTimeline := w.extractAPIWebsocketTimeline(c)
 		var errMerge error
 		apiWebsocketTimeline, errMerge = mergeFileBodySource(apiWebsocketTimeline, apiWebsocketTimelineSource)
 		if errMerge != nil {
-			cleanupFileBodySources(websocketTimelineSource)
+			cleanupFileBodySources(websocketTimelineSource, apiRequestSource, apiResponseSource)
 			return errMerge
 		}
-		cleanupFileBodySources(websocketTimelineSource)
 		if len(apiWebsocketTimeline) > 0 {
 			_ = w.streamWriter.WriteAPIWebsocketTimeline(apiWebsocketTimeline)
 		}
 		if err := w.streamWriter.Close(); err != nil {
 			w.streamWriter = nil
+			cleanupFileBodySources(websocketTimelineSource, apiRequestSource, apiResponseSource)
 			return err
 		}
 		w.streamWriter = nil
+		cleanupFileBodySources(websocketTimelineSource, apiRequestSource, apiResponseSource)
 		return nil
 	}
 
-	return w.logRequest(w.extractRequestBody(c), finalStatusCode, w.cloneHeaders(), w.extractResponseBody(c), w.extractWebsocketTimeline(c), websocketTimelineSource, w.extractAPIRequest(c), w.extractAPIResponse(c), w.extractAPIWebsocketTimeline(c), apiWebsocketTimelineSource, w.extractAPIResponseTimestamp(c), slicesAPIResponseError, forceLog)
+	return w.logRequest(w.extractRequestBody(c), finalStatusCode, w.cloneHeaders(), w.extractResponseBody(c), w.extractWebsocketTimeline(c), websocketTimelineSource, w.extractAPIRequest(c), apiRequestSource, w.extractAPIResponse(c), apiResponseSource, w.extractAPIWebsocketTimeline(c), apiWebsocketTimelineSource, w.extractAPIResponseTimestamp(c), slicesAPIResponseError, forceLog)
 }
 
 func (w *ResponseWriterWrapper) cloneHeaders() map[string][]string {
@@ -369,6 +401,14 @@ func (w *ResponseWriterWrapper) extractAPIResponse(c *gin.Context) []byte {
 	return data
 }
 
+func (w *ResponseWriterWrapper) extractAPIRequestSource(c *gin.Context) *logging.FileBodySource {
+	return extractFileBodySource(c, logging.APIRequestSourceContextKey)
+}
+
+func (w *ResponseWriterWrapper) extractAPIResponseSource(c *gin.Context) *logging.FileBodySource {
+	return extractFileBodySource(c, logging.APIResponseSourceContextKey)
+}
+
 func (w *ResponseWriterWrapper) extractAPIWebsocketTimeline(c *gin.Context) []byte {
 	apiTimeline, isExist := c.Get("API_WEBSOCKET_TIMELINE")
 	if !isExist {
@@ -460,15 +500,53 @@ func extractBodyOverride(c *gin.Context, key string) []byte {
 	return nil
 }
 
-func (w *ResponseWriterWrapper) logRequest(requestBody []byte, statusCode int, headers map[string][]string, body, websocketTimeline []byte, websocketTimelineSource *logging.FileBodySource, apiRequestBody, apiResponseBody, apiWebsocketTimeline []byte, apiWebsocketTimelineSource *logging.FileBodySource, apiResponseTimestamp time.Time, apiResponseErrors []*interfaces.ErrorMessage, forceLog bool) error {
+func (w *ResponseWriterWrapper) logRequest(requestBody []byte, statusCode int, headers map[string][]string, body, websocketTimeline []byte, websocketTimelineSource *logging.FileBodySource, apiRequestBody []byte, apiRequestSource *logging.FileBodySource, apiResponseBody []byte, apiResponseSource *logging.FileBodySource, apiWebsocketTimeline []byte, apiWebsocketTimelineSource *logging.FileBodySource, apiResponseTimestamp time.Time, apiResponseErrors []*interfaces.ErrorMessage, forceLog bool) error {
 	if w.requestInfo == nil {
-		cleanupFileBodySources(websocketTimelineSource, apiWebsocketTimelineSource)
+		cleanupFileBodySources(websocketTimelineSource, apiRequestSource, apiResponseSource, apiWebsocketTimelineSource)
 		return nil
 	}
 
+	if loggerWithAllSources, ok := w.logger.(interface {
+		LogRequestWithOptionsAndAllSources(string, string, map[string][]string, []byte, int, map[string][]string, []byte, []byte, *logging.FileBodySource, []byte, *logging.FileBodySource, []byte, *logging.FileBodySource, []byte, *logging.FileBodySource, []*interfaces.ErrorMessage, bool, string, time.Time, time.Time) error
+	}); ok {
+		return loggerWithAllSources.LogRequestWithOptionsAndAllSources(
+			w.requestInfo.URL,
+			w.requestInfo.Method,
+			w.requestInfo.Headers,
+			requestBody,
+			statusCode,
+			headers,
+			body,
+			websocketTimeline,
+			websocketTimelineSource,
+			apiRequestBody,
+			apiRequestSource,
+			apiResponseBody,
+			apiResponseSource,
+			apiWebsocketTimeline,
+			apiWebsocketTimelineSource,
+			apiResponseErrors,
+			forceLog,
+			w.requestInfo.RequestID,
+			w.requestInfo.Timestamp,
+			apiResponseTimestamp,
+		)
+	}
+
 	if loggerWithSources, ok := w.logger.(interface {
 		LogRequestWithOptionsAndSources(string, string, map[string][]string, []byte, int, map[string][]string, []byte, []byte, *logging.FileBodySource, []byte, []byte, []byte, *logging.FileBodySource, []*interfaces.ErrorMessage, bool, string, time.Time, time.Time) error
 	}); ok {
+		var errMerge error
+		apiRequestBody, errMerge = mergeFileBodySource(apiRequestBody, apiRequestSource)
+		if errMerge != nil {
+			cleanupFileBodySources(websocketTimelineSource, apiResponseSource, apiWebsocketTimelineSource)
+			return errMerge
+		}
+		apiResponseBody, errMerge = mergeFileBodySource(apiResponseBody, apiResponseSource)
+		if errMerge != nil {
+			cleanupFileBodySources(websocketTimelineSource, apiWebsocketTimelineSource)
+			return errMerge
+		}
 		return loggerWithSources.LogRequestWithOptionsAndSources(
 			w.requestInfo.URL,
 			w.requestInfo.Method,
@@ -493,6 +571,16 @@ func (w *ResponseWriterWrapper) logRequest(requestBody []byte, statusCode int, h
 
 	var errMerge error
 	websocketTimeline, errMerge = mergeFileBodySource(websocketTimeline, websocketTimelineSource)
+	if errMerge != nil {
+		cleanupFileBodySources(apiRequestSource, apiResponseSource, apiWebsocketTimelineSource)
+		return errMerge
+	}
+	apiRequestBody, errMerge = mergeFileBodySource(apiRequestBody, apiRequestSource)
+	if errMerge != nil {
+		cleanupFileBodySources(apiResponseSource, apiWebsocketTimelineSource)
+		return errMerge
+	}
+	apiResponseBody, errMerge = mergeFileBodySource(apiResponseBody, apiResponseSource)
 	if errMerge != nil {
 		cleanupFileBodySources(apiWebsocketTimelineSource)
 		return errMerge

internal/api/server.go

@@ -72,6 +72,17 @@ func defaultRequestLoggerFactory(cfg *config.Config, configPath string) logging.
 	return logger
 }
 
+func effectiveSDKConfig(cfg *config.Config) *config.SDKConfig {
+	if cfg == nil {
+		return nil
+	}
+	sdkCfg := cfg.SDKConfig
+	if cfg.CommercialMode {
+		sdkCfg.RequestLog = false
+	}
+	return &sdkCfg
+}
+
 // WithMiddleware appends additional Gin middleware during server construction.
 func WithMiddleware(mw ...gin.HandlerFunc) ServerOption {
 	return func(cfg *serverOptionConfig) {
@@ -257,7 +268,7 @@ func NewServer(cfg *config.Config, authManager *auth.Manager, accessManager *sdk
 	// Create server instance
 	s := &Server{
 		engine:              engine,
-		handlers:            handlers.NewBaseAPIHandlers(&cfg.SDKConfig, authManager),
+		handlers:            handlers.NewBaseAPIHandlers(effectiveSDKConfig(cfg), authManager),
 		cfg:                 cfg,
 		accessManager:       accessManager,
 		requestLogger:       requestLogger,
@@ -1453,7 +1464,7 @@ func (s *Server) UpdateClients(cfg *config.Config) {
 	// Save YAML snapshot for next comparison
 	s.oldConfigYaml, _ = yaml.Marshal(cfg)
 
-	s.handlers.UpdateClients(&cfg.SDKConfig)
+	s.handlers.UpdateClients(effectiveSDKConfig(cfg))
 
 	if s.mgmt != nil {
 		s.mgmt.SetConfig(cfg)

internal/config/config.go

@@ -52,7 +52,7 @@ type Config struct {
 	// Pprof config controls the optional pprof HTTP debug server.
 	Pprof PprofConfig `yaml:"pprof" json:"pprof"`
 
-	// CommercialMode disables high-overhead HTTP middleware features to minimize per-request memory usage.
+	// CommercialMode disables high-overhead request logging and HTTP middleware features to minimize per-request memory usage.
 	CommercialMode bool `yaml:"commercial-mode" json:"commercial-mode"`
 
 	// LoggingToFile controls whether application logs are written to rotating files or stdout.