From a8c85d1184f1c3d35755303f2d1c6196c64aed9c Mon Sep 17 00:00:00 2001 From: Mitchell Williams Date: Fri, 23 May 2025 14:49:00 -0600 Subject: [PATCH] feat!: restore PathRequestLoggingFilter Any service implementing this change needs to remove their own logback-spring.xml or add the appenders that will be added to java-mdx-web/src/main/resources/logback-spring.xml to their own instance --- mdx-gateway-generator/gradle.lockfile | 14 ++-- mdx-gateways/gradle.lockfile | 14 ++-- mdx-models/gradle.lockfile | 14 ++-- mdx-web/gradle.lockfile | 14 ++-- .../web/filter/PathRequestLoggingFilter.java | 66 ++++++++++++++++--- realtime/gradle.lockfile | 16 ++--- 6 files changed, 92 insertions(+), 46 deletions(-) diff --git a/mdx-gateway-generator/gradle.lockfile b/mdx-gateway-generator/gradle.lockfile index 96e0915d..10af29a7 100644 --- a/mdx-gateway-generator/gradle.lockfile +++ b/mdx-gateway-generator/gradle.lockfile @@ -28,13 +28,13 @@ com.google.guava:guava:31.0.1-jre=runtimeClasspath,testRuntimeClasspath com.google.guava:guava:31.1-jre=compileClasspath,testCompileClasspath com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.google.j2objc:j2objc-annotations:1.3=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:common:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:context:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:gateway-generator:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:gateway:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:messaging:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:platform:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:utilities:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:common:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:context:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:gateway-generator:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:gateway:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:messaging:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:platform:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:utilities:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.squareup:javapoet:1.13.0=runtimeClasspath,testRuntimeClasspath com.sun.activation:jakarta.activation:1.2.2=runtimeClasspath,testRuntimeClasspath com.sun.istack:istack-commons-runtime:3.0.12=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath diff --git a/mdx-gateways/gradle.lockfile b/mdx-gateways/gradle.lockfile index ff83f693..5b668306 100644 --- a/mdx-gateways/gradle.lockfile +++ b/mdx-gateways/gradle.lockfile @@ -33,13 +33,13 @@ com.google.guava:guava:31.0.1-jre=annotationProcessor com.google.guava:guava:31.1-jre=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.google.j2objc:j2objc-annotations:1.3=annotationProcessor,checkstyle,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:common:4.5.1=annotationProcessor,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:context:4.5.1=annotationProcessor,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:gateway-generator:4.5.1=annotationProcessor -com.mx.path-core:gateway:4.5.1=annotationProcessor,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:messaging:4.5.1=annotationProcessor,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:platform:4.5.1=annotationProcessor,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:utilities:4.5.1=annotationProcessor,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:common:4.5.2=annotationProcessor,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:context:4.5.2=annotationProcessor,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:gateway-generator:4.5.2=annotationProcessor +com.mx.path-core:gateway:4.5.2=annotationProcessor,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:messaging:4.5.2=annotationProcessor,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:platform:4.5.2=annotationProcessor,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:utilities:4.5.2=annotationProcessor,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.puppycrawl.tools:checkstyle:8.29=checkstyle com.squareup:javapoet:1.13.0=annotationProcessor com.sun.activation:jakarta.activation:1.2.2=annotationProcessor,runtimeClasspath,testRuntimeClasspath diff --git a/mdx-models/gradle.lockfile b/mdx-models/gradle.lockfile index cf76fca6..6b303179 100644 --- a/mdx-models/gradle.lockfile +++ b/mdx-models/gradle.lockfile @@ -28,13 +28,13 @@ com.google.guava:guava:28.2-jre=checkstyle com.google.guava:guava:31.1-jre=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=checkstyle,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.google.j2objc:j2objc-annotations:1.3=checkstyle,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:common:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:context:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:gateway:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:messaging:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:platform:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:testing:4.5.1=testCompileClasspath,testRuntimeClasspath -com.mx.path-core:utilities:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:common:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:context:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:gateway:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:messaging:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:platform:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:testing:4.5.2=testCompileClasspath,testRuntimeClasspath +com.mx.path-core:utilities:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.puppycrawl.tools:checkstyle:8.29=checkstyle com.sun.activation:jakarta.activation:1.2.2=runtimeClasspath,testRuntimeClasspath com.sun.istack:istack-commons-runtime:3.0.12=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath diff --git a/mdx-web/gradle.lockfile b/mdx-web/gradle.lockfile index 246aca70..46af8460 100644 --- a/mdx-web/gradle.lockfile +++ b/mdx-web/gradle.lockfile @@ -33,13 +33,13 @@ com.google.guava:guava:31.1-jre=compileClasspath,runtimeClasspath,testCompileCla com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=checkstyle,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.google.j2objc:j2objc-annotations:1.3=checkstyle,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.jayway.jsonpath:json-path:2.7.0=testCompileClasspath,testRuntimeClasspath -com.mx.path-core:common:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:context:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:gateway:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:messaging:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:platform:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:testing:4.5.1=testCompileClasspath,testRuntimeClasspath -com.mx.path-core:utilities:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:common:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:context:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:gateway:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:messaging:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:platform:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:testing:4.5.2=testCompileClasspath,testRuntimeClasspath +com.mx.path-core:utilities:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.puppycrawl.tools:checkstyle:8.29=checkstyle com.sun.activation:jakarta.activation:1.2.2=runtimeClasspath,testRuntimeClasspath com.sun.istack:istack-commons-runtime:3.0.12=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath diff --git a/mdx-web/src/main/java/com/mx/path/model/mdx/web/filter/PathRequestLoggingFilter.java b/mdx-web/src/main/java/com/mx/path/model/mdx/web/filter/PathRequestLoggingFilter.java index 4ebc193a..7282c62f 100644 --- a/mdx-web/src/main/java/com/mx/path/model/mdx/web/filter/PathRequestLoggingFilter.java +++ b/mdx-web/src/main/java/com/mx/path/model/mdx/web/filter/PathRequestLoggingFilter.java @@ -2,11 +2,9 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.Collection; import java.util.Enumeration; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.UUID; @@ -25,6 +23,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; import org.springframework.web.filter.OncePerRequestFilter; import org.springframework.web.util.ContentCachingRequestWrapper; import org.springframework.web.util.ContentCachingResponseWrapper; @@ -35,9 +35,8 @@ * This filter logs incoming requests. * These values are inserted into the MDC. */ -// TODO Needs more testing before enabling. This is causing opensearch issues -//@Component -//@Order(FilterOrderSequence.REQUEST_LOGGING_FILTER) +@Component +@Order(FilterOrderSequence.REQUEST_LOGGING_FILTER) public class PathRequestLoggingFilter extends OncePerRequestFilter { // Statics @@ -156,6 +155,7 @@ private void logRequest(ContentCachingRequestWrapper request, ContentCachingResp MDC.remove("request_body"); } + MDC.put("api_call_payload", buildApiPayload(response, request)); MDC.put("request_duration", String.valueOf(elapsedTime)); MDC.put("status", String.valueOf(response.getStatus())); @@ -190,6 +190,7 @@ private void logRequest(ContentCachingRequestWrapper request, ContentCachingResp * Resets the MDC by just cleaning up what we place there. */ protected void resetMDC() { + MDC.remove("api_call_payload"); MDC.remove("client_guid"); MDC.remove("client_id"); MDC.remove("device_trace_id"); @@ -212,6 +213,55 @@ protected void resetMDC() { MDC.remove("user_guid"); } + private String buildApiPayload(ContentCachingResponseWrapper response, ContentCachingRequestWrapper request) { + StringBuilder b = new StringBuilder(); + b.append("\n= Request\n\n"); + b.append(request.getMethod()); + b.append(" "); + b.append(request.getRequestURL()); + b.append("\n\n"); + if (request.getHeaderNames() != null) { + Map requestHeaders = this.buildRequestHeadersMap(request); + Map maskedRequestHeaders = this.maskHeaders(requestHeaders); + b.append(this.buildHeaderString(maskedRequestHeaders)); + } + + String queryString = request.getQueryString(); + if (queryString != null) { + Map queryParams = this.buildQueryStringMap(queryString); + b.append(this.buildHeaderString(this.maskHeaders(queryParams))); + } + + String requestBody = new String(request.getContentAsByteArray(), StandardCharsets.UTF_8); + if (!requestBody.isEmpty()) { + b.append("\n"); + b.append(MdxLogMasker.maskPayload(requestBody)); + b.append("\n"); + } + + b.append("\n= Response\n\n"); + b.append(response.getStatus()); + b.append("\n"); + if (response.getHeaderNames() != null) { + Map responseHeaders = this.buildResponseHeadersMap(response); + Map maskedResponseHeaders = this.maskHeaders(responseHeaders); + + if (response.getContentType() != null) { + maskedResponseHeaders.put("Content-Type", response.getContentType()); + } + + if (!maskedResponseHeaders.isEmpty()) { + b.append(buildHeaderString(maskedResponseHeaders)); + } + } + + String responseBody = new String(response.getContentAsByteArray(), StandardCharsets.UTF_8); + if (!responseBody.isEmpty()) { + b.append(MdxLogMasker.maskPayload(responseBody)); + } + return b.toString(); + } + private Map buildRequestHeadersMap(HttpServletRequest request) { final Enumeration headerNames = request.getHeaderNames(); final Map headerMap = new HashMap<>(); @@ -239,14 +289,10 @@ private Map buildResponseHeadersMap(HttpServletResponse response for (String headerName : headerNames) { final Collection headerValues = response.getHeaders(headerName); - String headerValuesFlattened = ""; if (headerValues != null) { - final List headerValuesList = new ArrayList<>(headerValues); - headerValuesFlattened = String.join(", ", headerValuesList); + headerMap.put(headerName, (String) headerValues.toArray()[0]); } - - headerMap.put(headerName, headerValuesFlattened); } return headerMap; diff --git a/realtime/gradle.lockfile b/realtime/gradle.lockfile index 8973cd3b..621a5055 100644 --- a/realtime/gradle.lockfile +++ b/realtime/gradle.lockfile @@ -28,14 +28,14 @@ com.google.guava:guava:28.2-jre=checkstyle com.google.guava:guava:31.1-jre=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=checkstyle,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.google.j2objc:j2objc-annotations:1.3=checkstyle,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:common:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:context:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:gateway:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:http:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:messaging:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:platform:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.mx.path-core:testing:4.5.1=testCompileClasspath,testRuntimeClasspath -com.mx.path-core:utilities:4.5.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:common:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:context:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:gateway:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:http:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:messaging:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:platform:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.mx.path-core:testing:4.5.2=testCompileClasspath,testRuntimeClasspath +com.mx.path-core:utilities:4.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.puppycrawl.tools:checkstyle:8.29=checkstyle com.sun.activation:jakarta.activation:1.2.2=runtimeClasspath,testRuntimeClasspath com.sun.istack:istack-commons-runtime:3.0.12=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath