Capture application logs (#52)

* Capture application logs

* Tweaks

* Fix

* Extend test matrix

* Simplify

* Fix

Author: itssimon

.github/workflows/tests.yaml

@@ -16,7 +16,7 @@ jobs:
       - uses: actions/checkout@v6
       - uses: actions/setup-python@v6
         with:
-          python-version: "3.13"
+          python-version: "3.14"
       - uses: pre-commit/action@v3.0.1
 
   test-coverage:
@@ -40,7 +40,7 @@ jobs:
       fail-fast: false
       matrix:
         java: [17, 21]
-        spring-boot: [3.0.13, 3.1.12, 3.2.12, 3.3.8, 3.4.3]
+        spring-boot: [3.0.13, 3.1.12, 3.2.12, 3.3.8, 3.4.3, 3.5.9]
     steps:
       - uses: actions/checkout@v6
       - uses: actions/setup-java@v5

src/main/java/io/apitally/common/ApitallyAppender.java

@@ -0,0 +1,74 @@
+package io.apitally.common;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.slf4j.LoggerFactory;
+
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.AppenderBase;
+
+import io.apitally.common.dto.LogRecord;
+
+public class ApitallyAppender extends AppenderBase<ILoggingEvent> {
+    private static final String NAME = "ApitallyAppender";
+    private static final int MAX_BUFFER_SIZE = 1000;
+    private static final int MAX_MESSAGE_LENGTH = 2048;
+
+    private static final ThreadLocal<List<LogRecord>> logBuffer = new ThreadLocal<>();
+
+    public static synchronized void register() {
+        if (!(LoggerFactory.getILoggerFactory() instanceof LoggerContext loggerContext)) {
+            return;
+        }
+        Logger rootLogger = loggerContext.getLogger(Logger.ROOT_LOGGER_NAME);
+
+        if (rootLogger.getAppender(NAME) != null) {
+            return;
+        }
+
+        ApitallyAppender appender = new ApitallyAppender();
+        appender.setContext(loggerContext);
+        appender.setName(NAME);
+        appender.start();
+        rootLogger.addAppender(appender);
+    }
+
+    public static void startCapture() {
+        logBuffer.set(new ArrayList<>());
+    }
+
+    public static List<LogRecord> endCapture() {
+        List<LogRecord> logs = logBuffer.get();
+        logBuffer.remove();
+        return logs;
+    }
+
+    @Override
+    protected void append(ILoggingEvent event) {
+        List<LogRecord> buffer = logBuffer.get();
+        if (buffer == null || buffer.size() >= MAX_BUFFER_SIZE) {
+            return;
+        }
+
+        double timestamp = event.getTimeStamp() / 1000.0;
+        String loggerName = event.getLoggerName();
+        String level = event.getLevel().toString();
+        String message = truncateMessage(event.getFormattedMessage());
+
+        buffer.add(new LogRecord(timestamp, loggerName, level, message));
+    }
+
+    private static String truncateMessage(String message) {
+        if (message == null) {
+            return null;
+        }
+        if (message.length() <= MAX_MESSAGE_LENGTH) {
+            return message;
+        }
+        String suffix = "... (truncated)";
+        return message.substring(0, MAX_MESSAGE_LENGTH - suffix.length()) + suffix;
+    }
+}

src/main/java/io/apitally/common/ConsumerRegistry.java

@@ -75,4 +75,9 @@ public List<Consumer> getAndResetConsumers() {
         updated.clear();
         return data;
     }
+
+    public void reset() {
+        consumers.clear();
+        updated.clear();
+    }
 }

src/main/java/io/apitally/common/RequestLogger.java

@@ -26,6 +26,7 @@
 
 import io.apitally.common.dto.ExceptionDto;
 import io.apitally.common.dto.Header;
+import io.apitally.common.dto.LogRecord;
 import io.apitally.common.dto.Request;
 import io.apitally.common.dto.RequestLogItem;
 import io.apitally.common.dto.Response;
@@ -147,7 +148,7 @@ public void setSuspendUntil(long timestamp) {
         this.suspendUntil = timestamp;
     }
 
-    public void logRequest(Request request, Response response, Exception exception) {
+    public void logRequest(Request request, Response response, Exception exception, List<LogRecord> logs) {
         if (!enabled || suspendUntil != null && suspendUntil > System.currentTimeMillis()) {
             return;
         }
@@ -181,7 +182,11 @@ public void logRequest(Request request, Response response, Exception exception)
                 exceptionDto = new ExceptionDto(exception);
             }
 
-            RequestLogItem item = new RequestLogItem(request, response, exceptionDto);
+            if (!config.isLogCaptureEnabled() && logs != null) {
+                logs = null;
+            }
+
+            RequestLogItem item = new RequestLogItem(request, response, exceptionDto, logs);
             pendingWrites.add(item);
 
             if (pendingWrites.size() > MAX_PENDING_WRITES) {
@@ -280,6 +285,9 @@ public void writeToFile() throws IOException {
                 if (item.getException() != null) {
                     itemNode.set("exception", objectMapper.valueToTree(item.getException()));
                 }
+                if (item.getLogs() != null && !item.getLogs().isEmpty()) {
+                    itemNode.set("logs", objectMapper.valueToTree(item.getLogs()));
+                }
 
                 String serializedItem = objectMapper.writeValueAsString(itemNode);
                 currentFile.writeLine(serializedItem.getBytes(StandardCharsets.UTF_8));

src/main/java/io/apitally/common/RequestLoggingConfig.java

@@ -11,6 +11,7 @@ public class RequestLoggingConfig {
     private boolean responseHeadersIncluded = true;
     private boolean responseBodyIncluded = false;
     private boolean exceptionIncluded = true;
+    private boolean logCaptureEnabled = false;
     private List<String> queryParamMaskPatterns = new ArrayList<>();
     private List<String> headerMaskPatterns = new ArrayList<>();
     private List<String> bodyFieldMaskPatterns = new ArrayList<>();
@@ -73,6 +74,14 @@ public void setExceptionIncluded(boolean exceptionIncluded) {
         this.exceptionIncluded = exceptionIncluded;
     }
 
+    public boolean isLogCaptureEnabled() {
+        return logCaptureEnabled;
+    }
+
+    public void setLogCaptureEnabled(boolean logCaptureEnabled) {
+        this.logCaptureEnabled = logCaptureEnabled;
+    }
+
     public List<String> getQueryParamMaskPatterns() {
         return queryParamMaskPatterns;
     }

src/main/java/io/apitally/common/dto/LogRecord.java

@@ -0,0 +1,31 @@
+package io.apitally.common.dto;
+
+public class LogRecord {
+    private final double timestamp;
+    private final String logger;
+    private final String level;
+    private final String message;
+
+    public LogRecord(double timestamp, String logger, String level, String message) {
+        this.timestamp = timestamp;
+        this.logger = logger;
+        this.level = level;
+        this.message = message;
+    }
+
+    public double getTimestamp() {
+        return timestamp;
+    }
+
+    public String getLogger() {
+        return logger;
+    }
+
+    public String getLevel() {
+        return level;
+    }
+
+    public String getMessage() {
+        return message;
+    }
+}

src/main/java/io/apitally/common/dto/RequestLogItem.java

@@ -1,5 +1,6 @@
 package io.apitally.common.dto;
 
+import java.util.List;
 import java.util.UUID;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -9,12 +10,14 @@ public class RequestLogItem extends BaseDto {
     private final Request request;
     private final Response response;
     private final ExceptionDto exception;
+    private final List<LogRecord> logs;
 
-    public RequestLogItem(Request request, Response response, ExceptionDto exception) {
+    public RequestLogItem(Request request, Response response, ExceptionDto exception, List<LogRecord> logs) {
         this.uuid = UUID.randomUUID().toString();
         this.request = request;
         this.response = response;
         this.exception = exception;
+        this.logs = logs;
     }
 
     @JsonProperty("uuid")
@@ -36,4 +39,9 @@ public Response getResponse() {
     public ExceptionDto getException() {
         return exception;
     }
+
+    @JsonProperty("logs")
+    public List<LogRecord> getLogs() {
+        return logs;
+    }
 }

src/main/java/io/apitally/spring/ApitallyAutoConfiguration.java

@@ -10,6 +10,7 @@
 import org.springframework.core.Ordered;
 import org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping;
 
+import io.apitally.common.ApitallyAppender;
 import io.apitally.common.ApitallyClient;
 import io.apitally.common.dto.Path;
 
@@ -25,6 +26,11 @@ public ApitallyClient apitallyClient(ApitallyProperties properties,
         Map<String, String> versions = ApitallyUtils.getVersions();
         client.setStartupData(paths, versions, "java:spring");
         client.startSync();
+
+        if (properties.getRequestLogging().isEnabled() && properties.getRequestLogging().isLogCaptureEnabled()) {
+            ApitallyAppender.register();
+        }
+
         return client;
     }
 

src/main/java/io/apitally/spring/ApitallyFilter.java

@@ -2,6 +2,7 @@
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -13,13 +14,16 @@
 import org.springframework.web.util.ContentCachingRequestWrapper;
 import org.springframework.web.util.ContentCachingResponseWrapper;
 
+import io.apitally.common.ApitallyAppender;
 import io.apitally.common.ApitallyClient;
 import io.apitally.common.ConsumerRegistry;
 import io.apitally.common.RequestLogger;
 import io.apitally.common.dto.Consumer;
 import io.apitally.common.dto.Header;
+import io.apitally.common.dto.LogRecord;
 import io.apitally.common.dto.Request;
 import io.apitally.common.dto.Response;
+
 import jakarta.servlet.FilterChain;
 import jakarta.servlet.ServletException;
 import jakarta.servlet.ServletOutputStream;
@@ -65,9 +69,16 @@ protected void doFilterInternal(@NonNull HttpServletRequest request, @NonNull Ht
                 ? new CountingResponseWrapper(response)
                 : null;
 
+        final boolean shouldCaptureLogs = client.requestLogger.getConfig().isEnabled()
+                && client.requestLogger.getConfig().isLogCaptureEnabled();
+
         Exception exception = null;
         final long startTime = System.currentTimeMillis();
 
+        if (shouldCaptureLogs) {
+            ApitallyAppender.startCapture();
+        }
+
         try {
             filterChain.doFilter(
                     cachingRequest != null ? cachingRequest : request,
@@ -76,6 +87,7 @@ protected void doFilterInternal(@NonNull HttpServletRequest request, @NonNull Ht
             exception = e;
             throw e;
         } finally {
+            final List<LogRecord> capturedLogs = shouldCaptureLogs ? ApitallyAppender.endCapture() : null;
             final long responseTimeInMillis = System.currentTimeMillis() - startTime;
             final String path = (String) request.getAttribute(HandlerMapping.BEST_MATCHING_PATTERN_ATTRIBUTE);
 
@@ -126,7 +138,8 @@ protected void doFilterInternal(@NonNull HttpServletRequest request, @NonNull Ht
                                     request.getRequestURL().toString(), requestHeaders, requestSize, requestBody),
                             new Response(response.getStatus(), responseTimeInMillis / 1000.0, responseHeaders,
                                     responseSize, responseBody),
-                            exception);
+                            exception,
+                            capturedLogs);
                 }
 
                 // Add validation error to counter