feat(logging): add file-backed sources for request logging

- Introduced `FileBodySource` to support large request log sections stored in temp files.
- Added file-backed support for WebSocket timeline and API WebSocket timeline logging.
- Updated `LogRequest` and middleware to integrate optional file-backed sources.
- Implemented clean-up mechanisms to manage temporary log files after processing.

Author: luispater

internal/api/middleware/request_logging.go

@@ -58,6 +58,7 @@ func RequestLoggingMiddleware(logger logging.RequestLogger) gin.HandlerFunc {
 			wrapper.logOnErrorOnly = true
 		}
 		c.Writer = wrapper
+		attachWebsocketLogSources(c, logger, loggerEnabled)
 
 		// Process the request
 		c.Next()
@@ -70,6 +71,26 @@ func RequestLoggingMiddleware(logger logging.RequestLogger) gin.HandlerFunc {
 	}
 }
 
+type fileBodySourceFactory interface {
+	NewFileBodySource(prefix string) (*logging.FileBodySource, error)
+}
+
+func attachWebsocketLogSources(c *gin.Context, logger logging.RequestLogger, loggerEnabled bool) {
+	if c == nil || !loggerEnabled || !isResponsesWebsocketUpgrade(c.Request) {
+		return
+	}
+	factory, ok := logger.(fileBodySourceFactory)
+	if !ok || factory == nil {
+		return
+	}
+	if source, errSource := factory.NewFileBodySource("websocket-timeline"); errSource == nil {
+		c.Set(logging.WebsocketTimelineSourceContextKey, source)
+	}
+	if source, errSource := factory.NewFileBodySource("api-websocket-timeline"); errSource == nil {
+		c.Set(logging.APIWebsocketTimelineSourceContextKey, source)
+	}
+}
+
 func shouldSkipMethodForRequestLogging(req *http.Request) bool {
 	if req == nil {
 		return true

internal/api/middleware/request_logging_test.go

@@ -6,11 +6,13 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"os"
 	"strings"
 	"testing"
 
 	"github.com/gin-gonic/gin"
 	"github.com/klauspost/compress/zstd"
+	"github.com/router-for-me/CLIProxyAPI/v7/internal/logging"
 )
 
 func TestShouldSkipMethodForRequestLogging(t *testing.T) {
@@ -142,6 +144,63 @@ func TestShouldCaptureRequestBody(t *testing.T) {
 	}
 }
 
+func TestAttachWebsocketLogSourcesUsesLoggerLogsDir(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+
+	logsDir := t.TempDir()
+	logger := logging.NewFileRequestLogger(true, logsDir, "", 0)
+	recorder := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(recorder)
+	c.Request = httptest.NewRequest(http.MethodGet, "/v1/responses", nil)
+	c.Request.Header.Set("Upgrade", "websocket")
+
+	attachWebsocketLogSources(c, logger, true)
+	defer cleanupFileBodySourcesFromContext(c)
+
+	for _, key := range []string{
+		logging.WebsocketTimelineSourceContextKey,
+		logging.APIWebsocketTimelineSourceContextKey,
+	} {
+		value, exists := c.Get(key)
+		if !exists {
+			t.Fatalf("expected %s source to be attached", key)
+		}
+		source, ok := value.(*logging.FileBodySource)
+		if !ok || source == nil {
+			t.Fatalf("%s source type = %T", key, value)
+		}
+		file, errPart := source.CreatePart("probe")
+		if errPart != nil {
+			t.Fatalf("CreatePart(%s): %v", key, errPart)
+		}
+		path := file.Name()
+		if errClose := file.Close(); errClose != nil {
+			t.Fatalf("close part: %v", errClose)
+		}
+		if !strings.HasPrefix(path, logsDir+string(os.PathSeparator)) {
+			t.Fatalf("%s part path %s is not under logs dir %s", key, path, logsDir)
+		}
+	}
+}
+
+func cleanupFileBodySourcesFromContext(c *gin.Context) {
+	if c == nil {
+		return
+	}
+	for _, key := range []string{
+		logging.WebsocketTimelineSourceContextKey,
+		logging.APIWebsocketTimelineSourceContextKey,
+	} {
+		value, exists := c.Get(key)
+		if !exists {
+			continue
+		}
+		if source, ok := value.(*logging.FileBodySource); ok && source != nil {
+			_ = source.Cleanup()
+		}
+	}
+}
+
 func TestCaptureRequestInfoDecodesZstdRequestBodyForLog(t *testing.T) {
 	gin.SetMode(gin.TestMode)
 

internal/api/middleware/response_writer.go

@@ -280,7 +280,10 @@ func (w *ResponseWriterWrapper) Finalize(c *gin.Context) error {
 
 	hasAPIError := len(slicesAPIResponseError) > 0 || finalStatusCode >= http.StatusBadRequest
 	forceLog := w.logOnErrorOnly && hasAPIError && !w.logger.IsEnabled()
+	websocketTimelineSource := w.extractWebsocketTimelineSource(c)
+	apiWebsocketTimelineSource := w.extractAPIWebsocketTimelineSource(c)
 	if !w.logger.IsEnabled() && !forceLog {
+		cleanupFileBodySources(websocketTimelineSource, apiWebsocketTimelineSource)
 		return nil
 	}
 
@@ -307,6 +310,13 @@ func (w *ResponseWriterWrapper) Finalize(c *gin.Context) error {
 			_ = w.streamWriter.WriteAPIResponse(apiResponse)
 		}
 		apiWebsocketTimeline := w.extractAPIWebsocketTimeline(c)
+		var errMerge error
+		apiWebsocketTimeline, errMerge = mergeFileBodySource(apiWebsocketTimeline, apiWebsocketTimelineSource)
+		if errMerge != nil {
+			cleanupFileBodySources(websocketTimelineSource)
+			return errMerge
+		}
+		cleanupFileBodySources(websocketTimelineSource)
 		if len(apiWebsocketTimeline) > 0 {
 			_ = w.streamWriter.WriteAPIWebsocketTimeline(apiWebsocketTimeline)
 		}
@@ -318,7 +328,7 @@ func (w *ResponseWriterWrapper) Finalize(c *gin.Context) error {
 		return nil
 	}
 
-	return w.logRequest(w.extractRequestBody(c), finalStatusCode, w.cloneHeaders(), w.extractResponseBody(c), w.extractWebsocketTimeline(c), w.extractAPIRequest(c), w.extractAPIResponse(c), w.extractAPIWebsocketTimeline(c), w.extractAPIResponseTimestamp(c), slicesAPIResponseError, forceLog)
+	return w.logRequest(w.extractRequestBody(c), finalStatusCode, w.cloneHeaders(), w.extractResponseBody(c), w.extractWebsocketTimeline(c), websocketTimelineSource, w.extractAPIRequest(c), w.extractAPIResponse(c), w.extractAPIWebsocketTimeline(c), apiWebsocketTimelineSource, w.extractAPIResponseTimestamp(c), slicesAPIResponseError, forceLog)
 }
 
 func (w *ResponseWriterWrapper) cloneHeaders() map[string][]string {
@@ -370,6 +380,10 @@ func (w *ResponseWriterWrapper) extractAPIWebsocketTimeline(c *gin.Context) []by
 	return bytes.Clone(data)
 }
 
+func (w *ResponseWriterWrapper) extractAPIWebsocketTimelineSource(c *gin.Context) *logging.FileBodySource {
+	return extractFileBodySource(c, logging.APIWebsocketTimelineSourceContextKey)
+}
+
 func (w *ResponseWriterWrapper) extractAPIResponseTimestamp(c *gin.Context) time.Time {
 	ts, isExist := c.Get("API_RESPONSE_TIMESTAMP")
 	if !isExist {
@@ -405,6 +419,25 @@ func (w *ResponseWriterWrapper) extractWebsocketTimeline(c *gin.Context) []byte
 	return extractBodyOverride(c, websocketTimelineOverrideContextKey)
 }
 
+func (w *ResponseWriterWrapper) extractWebsocketTimelineSource(c *gin.Context) *logging.FileBodySource {
+	return extractFileBodySource(c, logging.WebsocketTimelineSourceContextKey)
+}
+
+func extractFileBodySource(c *gin.Context, key string) *logging.FileBodySource {
+	if c == nil {
+		return nil
+	}
+	value, exists := c.Get(key)
+	if !exists {
+		return nil
+	}
+	source, ok := value.(*logging.FileBodySource)
+	if !ok || source == nil {
+		return nil
+	}
+	return source
+}
+
 func extractBodyOverride(c *gin.Context, key string) []byte {
 	if c == nil {
 		return nil
@@ -426,11 +459,48 @@ func extractBodyOverride(c *gin.Context, key string) []byte {
 	return nil
 }
 
-func (w *ResponseWriterWrapper) logRequest(requestBody []byte, statusCode int, headers map[string][]string, body, websocketTimeline, apiRequestBody, apiResponseBody, apiWebsocketTimeline []byte, apiResponseTimestamp time.Time, apiResponseErrors []*interfaces.ErrorMessage, forceLog bool) error {
+func (w *ResponseWriterWrapper) logRequest(requestBody []byte, statusCode int, headers map[string][]string, body, websocketTimeline []byte, websocketTimelineSource *logging.FileBodySource, apiRequestBody, apiResponseBody, apiWebsocketTimeline []byte, apiWebsocketTimelineSource *logging.FileBodySource, apiResponseTimestamp time.Time, apiResponseErrors []*interfaces.ErrorMessage, forceLog bool) error {
 	if w.requestInfo == nil {
+		cleanupFileBodySources(websocketTimelineSource, apiWebsocketTimelineSource)
 		return nil
 	}
 
+	if loggerWithSources, ok := w.logger.(interface {
+		LogRequestWithOptionsAndSources(string, string, map[string][]string, []byte, int, map[string][]string, []byte, []byte, *logging.FileBodySource, []byte, []byte, []byte, *logging.FileBodySource, []*interfaces.ErrorMessage, bool, string, time.Time, time.Time) error
+	}); ok {
+		return loggerWithSources.LogRequestWithOptionsAndSources(
+			w.requestInfo.URL,
+			w.requestInfo.Method,
+			w.requestInfo.Headers,
+			requestBody,
+			statusCode,
+			headers,
+			body,
+			websocketTimeline,
+			websocketTimelineSource,
+			apiRequestBody,
+			apiResponseBody,
+			apiWebsocketTimeline,
+			apiWebsocketTimelineSource,
+			apiResponseErrors,
+			forceLog,
+			w.requestInfo.RequestID,
+			w.requestInfo.Timestamp,
+			apiResponseTimestamp,
+		)
+	}
+
+	var errMerge error
+	websocketTimeline, errMerge = mergeFileBodySource(websocketTimeline, websocketTimelineSource)
+	if errMerge != nil {
+		cleanupFileBodySources(apiWebsocketTimelineSource)
+		return errMerge
+	}
+	apiWebsocketTimeline, errMerge = mergeFileBodySource(apiWebsocketTimeline, apiWebsocketTimelineSource)
+	if errMerge != nil {
+		return errMerge
+	}
+
 	if loggerWithOptions, ok := w.logger.(interface {
 		LogRequestWithOptions(string, string, map[string][]string, []byte, int, map[string][]string, []byte, []byte, []byte, []byte, []byte, []*interfaces.ErrorMessage, bool, string, time.Time, time.Time) error
 	}); ok {
@@ -472,3 +542,34 @@ func (w *ResponseWriterWrapper) logRequest(requestBody []byte, statusCode int, h
 		apiResponseTimestamp,
 	)
 }
+
+func mergeFileBodySource(payload []byte, source *logging.FileBodySource) ([]byte, error) {
+	if source == nil {
+		return payload, nil
+	}
+	defer cleanupFileBodySources(source)
+	if !source.HasPayload() {
+		return payload, nil
+	}
+	var buf bytes.Buffer
+	if len(payload) > 0 {
+		buf.Write(payload)
+		if !bytes.HasSuffix(payload, []byte("\n")) {
+			buf.WriteByte('\n')
+		}
+		buf.WriteByte('\n')
+	}
+	if errWrite := source.WriteTo(&buf); errWrite != nil {
+		return nil, errWrite
+	}
+	return buf.Bytes(), nil
+}
+
+func cleanupFileBodySources(sources ...*logging.FileBodySource) {
+	for _, source := range sources {
+		if source == nil {
+			continue
+		}
+		_ = source.Cleanup()
+	}
+}