diff --git a/common/utils-java/src/main/java/org/apache/spark/internal/LogKeys.java b/common/utils-java/src/main/java/org/apache/spark/internal/LogKeys.java index 32d34026e5a8a..50d47d2da5ab9 100644 --- a/common/utils-java/src/main/java/org/apache/spark/internal/LogKeys.java +++ b/common/utils-java/src/main/java/org/apache/spark/internal/LogKeys.java @@ -288,6 +288,11 @@ public enum LogKeys implements LogKey { HOST_LOCAL_BLOCKS_SIZE, HOST_PORT, HOST_PORT2, + HTTP_METHOD, + HTTP_QUERY_STRING, + HTTP_REFERER, + HTTP_STATUS_CODE, + HTTP_USER_AGENT, HUGE_METHOD_LIMIT, HYBRID_STORE_DISK_BACKEND, IDENTIFIER, diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala index a4e047f7683ac..de3330260a364 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala @@ -22,6 +22,7 @@ import java.util.zip.ZipOutputStream import scala.util.control.NonFatal import scala.xml.Node +import jakarta.servlet.Filter import jakarta.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse} import org.eclipse.jetty.ee10.servlet.{ServletContextHandler, ServletHolder} @@ -74,6 +75,14 @@ class HistoryServer( // and its metrics, for testing as well as monitoring val cacheMetrics = appCache.metrics + override protected def getInternalFilters: Seq[() => Filter] = { + if (conf.get(History.HISTORY_SERVER_UI_ACCESS_LOG_ENABLED)) { + Seq(() => new HistoryServerAccessLogFilter(conf)) + } else { + Nil + } + } + private val loaderServlet = new HttpServlet { protected override def doGet(req: HttpServletRequest, res: HttpServletResponse): Unit = { diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerAccessLogFilter.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerAccessLogFilter.scala new file mode 100644 index 0000000000000..de541f2dfb718 --- /dev/null +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerAccessLogFilter.scala @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.deploy.history + +import java.net.URLDecoder +import java.nio.charset.StandardCharsets.UTF_8 +import java.util.concurrent.TimeUnit + +import scala.util.control.NonFatal + +import jakarta.servlet.{Filter, FilterChain, ServletRequest, ServletResponse} +import jakarta.servlet.http.{HttpServletRequest, HttpServletResponse, HttpServletResponseWrapper} + +import org.apache.spark.SparkConf +import org.apache.spark.internal.Logging +import org.apache.spark.internal.LogKeys._ +import org.apache.spark.internal.config.{History, SECRET_REDACTION_PATTERN} +import org.apache.spark.util.Utils + +private[history] class HistoryServerAccessLogFilter(conf: SparkConf) + extends Filter with Logging { + + import HistoryServerAccessLogFilter._ + + private val redactionPattern = Some(conf.get(SECRET_REDACTION_PATTERN)) + + private val excludedPathPrefixes = conf.get(History.HISTORY_SERVER_UI_ACCESS_LOG_EXCLUDE_PATHS) + .map(normalizePathPrefix) + .filter(_.nonEmpty) + + override def doFilter( + request: ServletRequest, + response: ServletResponse, + chain: FilterChain): Unit = { + (request, response) match { + case (httpRequest: HttpServletRequest, httpResponse: HttpServletResponse) + if !shouldSkip(httpRequest) => + doFilterWithAccessLog(httpRequest, httpResponse, chain) + + case _ => + chain.doFilter(request, response) + } + } + + private def doFilterWithAccessLog( + request: HttpServletRequest, + response: HttpServletResponse, + chain: FilterChain): Unit = { + val responseWrapper = new StatusCaptureResponse(response) + val startNs = System.nanoTime() + var error: Throwable = null + try { + chain.doFilter(request, responseWrapper) + } catch { + case NonFatal(e) => + error = e + throw e + } finally { + logAccess(request, responseWrapper, startNs, Option(error)) + } + } + + private def shouldSkip(request: HttpServletRequest): Boolean = { + val requestPath = Option(request.getRequestURI).getOrElse("") + excludedPathPrefixes.exists(matchesPathPrefix(requestPath, _)) + } + + private def logAccess( + request: HttpServletRequest, + response: StatusCaptureResponse, + startNs: Long, + error: Option[Throwable]): Unit = { + val durationMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNs) + val status = if (error.isDefined && response.status == HttpServletResponse.SC_OK) { + HttpServletResponse.SC_INTERNAL_SERVER_ERROR + } else { + response.status + } + val errorClass = error.map(_.getClass.getName).getOrElse(MissingField) + + logInfo(log"Spark History Server access" + + log" method=${MDC(HTTP_METHOD, field(request.getMethod))}" + + log" uri=${MDC(URI, field(redact(request.getRequestURI)))}" + + log" query=${MDC(HTTP_QUERY_STRING, redactQueryString(request.getQueryString))}" + + log" status=${MDC(HTTP_STATUS_CODE, status)}" + + log" durationMs=${MDC(DURATION, durationMs)}" + + log" remoteAddress=${MDC(REMOTE_ADDRESS, field(request.getRemoteAddr))}" + + log" user=${MDC(USER_NAME, field(remoteUser(request)))}" + + log" userAgent=${MDC(HTTP_USER_AGENT, field(redact(request.getHeader("User-Agent"))))}" + + log" referer=${MDC(HTTP_REFERER, field(redact(request.getHeader("Referer"))))}" + + log" error=${MDC(ERROR, errorClass)}") + } + + private def remoteUser(request: HttpServletRequest): String = { + Option(request.getRemoteUser) + .orElse(Option(request.getUserPrincipal).map(_.getName)) + .orNull + } + + private def redact(value: String): String = { + Option(value).map(Utils.redact(redactionPattern, _)).orNull + } + + private def redactQueryString(queryString: String): String = { + Option(queryString) + .filter(_.nonEmpty) + .map(_.split("&", -1).map(redactQueryParam).mkString("&")) + .getOrElse(MissingField) + } + + private def redactQueryParam(param: String): String = { + val separator = param.indexOf('=') + if (separator < 0) { + val decodedParam = decodeQueryComponent(param) + val redactedParam = Utils.redact(redactionPattern, decodedParam) + if (redactedParam != decodedParam) { + Utils.REDACTION_REPLACEMENT_TEXT + } else { + field(param) + } + } else { + val rawKey = param.substring(0, separator) + val rawValue = param.substring(separator + 1) + val decodedKey = decodeQueryComponent(rawKey) + val decodedValue = decodeQueryComponent(rawValue) + val redactedValue = Utils.redact(redactionPattern, Seq(decodedKey -> decodedValue)).head._2 + val valueToLog = if (redactedValue == Utils.REDACTION_REPLACEMENT_TEXT) { + Utils.REDACTION_REPLACEMENT_TEXT + } else { + rawValue + } + s"${field(rawKey)}=${field(valueToLog)}" + } + } +} + +private[history] object HistoryServerAccessLogFilter { + + private val MissingField = "-" + + private def normalizePathPrefix(path: String): String = { + val trimmed = path.trim + if (trimmed.isEmpty) { + "" + } else { + val withLeadingSlash = if (trimmed.startsWith("/")) trimmed else s"/$trimmed" + if (withLeadingSlash == "/") withLeadingSlash else withLeadingSlash.stripSuffix("/") + } + } + + private def matchesPathPrefix(requestPath: String, prefix: String): Boolean = { + prefix == "/" || requestPath == prefix || requestPath.startsWith(s"$prefix/") + } + + private def decodeQueryComponent(value: String): String = { + try { + URLDecoder.decode(value, UTF_8.name()) + } catch { + case NonFatal(_) => value + } + } + + private def field(value: String): String = { + Option(value) + .filter(_.nonEmpty) + .map { v => + v.map { + case c if Character.isWhitespace(c) || Character.isISOControl(c) => '_' + case c => c + }.mkString + } + .getOrElse(MissingField) + } + + private class StatusCaptureResponse(response: HttpServletResponse) + extends HttpServletResponseWrapper(response) { + + private var _status = HttpServletResponse.SC_OK + + def status: Int = _status + + override def setStatus(sc: Int): Unit = { + _status = sc + super.setStatus(sc) + } + + override def sendError(sc: Int): Unit = { + _status = sc + super.sendError(sc) + } + + override def sendError(sc: Int, msg: String): Unit = { + _status = sc + super.sendError(sc, msg) + } + + override def sendRedirect(location: String): Unit = { + _status = HttpServletResponse.SC_FOUND + super.sendRedirect(location) + } + } +} diff --git a/core/src/main/scala/org/apache/spark/internal/config/History.scala b/core/src/main/scala/org/apache/spark/internal/config/History.scala index 1936ecf68103d..9b3756cff6e68 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/History.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/History.scala @@ -148,6 +148,27 @@ private[spark] object History { .intConf .createWithDefault(18080) + val HISTORY_SERVER_UI_ACCESS_LOG_ENABLED = + ConfigBuilder("spark.history.ui.accessLog.enabled") + .doc("Whether the History Server should log HTTP access records for its web UI and REST " + + "API. When enabled, each non-excluded request is logged at INFO level after it " + + "completes. Query string values are redacted using spark.redaction.regex.") + .version("4.3.0") + .withBindingPolicy(ConfigBindingPolicy.NOT_APPLICABLE) + .booleanConf + .createWithDefault(false) + + val HISTORY_SERVER_UI_ACCESS_LOG_EXCLUDE_PATHS = + ConfigBuilder("spark.history.ui.accessLog.excludePaths") + .doc("Comma-separated list of request path prefixes to exclude from History Server " + + "HTTP access logs. This can be used to avoid logging high-volume low-value requests " + + "such as static resources.") + .version("4.3.0") + .withBindingPolicy(ConfigBindingPolicy.NOT_APPLICABLE) + .stringConf + .toSequence + .createWithDefault(Seq("/static", "/favicon.ico")) + val FAST_IN_PROGRESS_PARSING = ConfigBuilder("spark.history.fs.inProgressOptimization.enabled") .doc("Enable optimized handling of in-progress logs. This option may leave finished " + diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala index 3c21fd6d2a003..67d5c6af7fb6d 100644 --- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala @@ -254,6 +254,17 @@ private[spark] object JettyUtils extends Logging { conf: SparkConf, serverName: String = "", poolSize: Int = 200): ServerInfo = { + startJettyServer(hostName, port, sslOptions, conf, serverName, poolSize, Nil) + } + + def startJettyServer( + hostName: String, + port: Int, + sslOptions: SSLOptions, + conf: SparkConf, + serverName: String, + poolSize: Int, + internalFilters: Seq[() => Filter]): ServerInfo = { val stopTimeout = conf.get(UI_JETTY_STOP_TIMEOUT) logInfo(log"Start Jetty ${MDC(HOST, hostName)}:${MDC(PORT, port)}" + @@ -381,7 +392,7 @@ private[spark] object JettyUtils extends Logging { server.addConnector(httpConnector) pool.setMaxThreads(math.max(pool.getMaxThreads, minThreads)) - ServerInfo(server, httpPort, securePort, conf, collection) + ServerInfo(server, httpPort, securePort, conf, collection, internalFilters) } catch { case e: Exception => server.stop() @@ -469,7 +480,8 @@ private[spark] case class ServerInfo( boundPort: Int, securePort: Option[Int], private val conf: SparkConf, - private val rootHandler: ContextHandlerCollection) extends Logging { + private val rootHandler: ContextHandlerCollection, + private val internalFilters: Seq[() => Filter] = Nil) extends Logging { def addHandler( handler: ServletContextHandler, @@ -547,6 +559,13 @@ private[spark] case class ServerInfo( JettyUtils.addFilter(handler, filter, oldParams ++ newParams) } + // Internal filters run after user-installed filters so authentication wrappers are visible, + // and before the security filter so denied requests can still be observed. + internalFilters.foreach { filter => + val holder = new FilterHolder(filter()) + handler.addFilter(holder, "/*", EnumSet.of(DispatcherType.REQUEST)) + } + // This filter must come after user-installed filters, since that's where authentication // filters are installed. This means that custom filters will see the request before it's // been validated by the security filter. diff --git a/core/src/main/scala/org/apache/spark/ui/WebUI.scala b/core/src/main/scala/org/apache/spark/ui/WebUI.scala index 7f8f2556dd088..324781f9219c2 100644 --- a/core/src/main/scala/org/apache/spark/ui/WebUI.scala +++ b/core/src/main/scala/org/apache/spark/ui/WebUI.scala @@ -23,7 +23,7 @@ import scala.collection.mutable.ArrayBuffer import scala.collection.mutable.HashMap import scala.xml.Node -import jakarta.servlet.DispatcherType +import jakarta.servlet.{DispatcherType, Filter} import jakarta.servlet.http.{HttpServlet, HttpServletRequest} import org.eclipse.jetty.ee10.servlet.{FilterHolder, FilterMapping, ServletContextHandler, ServletHolder} import org.json4s.JsonAST.{JNothing, JValue} @@ -137,13 +137,16 @@ private[spark] abstract class WebUI( attachHandler(JettyUtils.createStaticHandler(resourceBase, path)) } + protected def getInternalFilters: Seq[() => Filter] = Nil + /** A hook to initialize components of the UI */ def initialize(): Unit def initServer(): ServerInfo = { val hostName = Option(conf.getenv("SPARK_LOCAL_IP")) .getOrElse(if (Utils.preferIPv6) "[::]" else "0.0.0.0") - val server = startJettyServer(hostName, port, sslOptions, conf, name, poolSize) + val server = startJettyServer( + hostName, port, sslOptions, conf, name, poolSize, getInternalFilters) server } diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerAccessLogFilterSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerAccessLogFilterSuite.scala new file mode 100644 index 0000000000000..170a78c001411 --- /dev/null +++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerAccessLogFilterSuite.scala @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.deploy.history + +import jakarta.servlet.{FilterChain, ServletRequest, ServletResponse} +import jakarta.servlet.http.{HttpServletRequest, HttpServletResponse} +import org.apache.logging.log4j.Level +import org.mockito.Mockito.when +import org.scalatestplus.mockito.MockitoSugar + +import org.apache.spark.{SparkConf, SparkFunSuite} +import org.apache.spark.internal.config.History._ +import org.apache.spark.util.Utils + +class HistoryServerAccessLogFilterSuite extends SparkFunSuite with MockitoSugar { + + test("logs History Server access records") { + val filter = new HistoryServerAccessLogFilter(new SparkConf(false)) + val request = mockRequest( + uri = "/api/v1/applications", + queryString = "token=visible&session=secret-value&status=complete", + userAgent = "curl 8.0", + referer = "http://localhost/history") + when(request.getRemoteUser).thenReturn("alice") + val response = mock[HttpServletResponse] + val chain = new FilterChain { + override def doFilter(req: ServletRequest, res: ServletResponse): Unit = { + res.asInstanceOf[HttpServletResponse].setStatus(HttpServletResponse.SC_CREATED) + } + } + + val message = withAccessLogAppender { + filter.doFilter(request, response, chain) + }.head + + assert(message.contains("Spark History Server access")) + assert(message.contains("method=GET")) + assert(message.contains("uri=/api/v1/applications")) + assert(message.contains(s"token=${Utils.REDACTION_REPLACEMENT_TEXT}")) + assert(message.contains(s"session=${Utils.REDACTION_REPLACEMENT_TEXT}")) + assert(message.contains("status=complete")) + assert(message.contains("status=201")) + assert(message.contains("remoteAddress=192.0.2.10")) + assert(message.contains("user=alice")) + assert(message.contains("userAgent=curl_8.0")) + assert(message.contains("referer=http://localhost/history")) + assert(message.contains("error=-")) + } + + test("does not log excluded paths") { + val filter = new HistoryServerAccessLogFilter(new SparkConf(false)) + val response = mock[HttpServletResponse] + var calls = 0 + val chain = new FilterChain { + override def doFilter(req: ServletRequest, res: ServletResponse): Unit = { + calls += 1 + } + } + + val messages = withAccessLogAppender { + filter.doFilter(mockRequest(uri = "/favicon.ico"), response, chain) + filter.doFilter(mockRequest(uri = "/static/bootstrap.css"), response, chain) + } + + assert(messages.isEmpty) + assert(calls === 2) + } + + test("redacts query parameters without values") { + val filter = new HistoryServerAccessLogFilter(new SparkConf(false)) + val request = mockRequest( + uri = "/api/v1/applications", + queryString = "password&plain") + val response = mock[HttpServletResponse] + val chain = new FilterChain { + override def doFilter(req: ServletRequest, res: ServletResponse): Unit = {} + } + + val message = withAccessLogAppender { + filter.doFilter(request, response, chain) + }.head + + assert(message.contains(s"query=${Utils.REDACTION_REPLACEMENT_TEXT}&plain")) + } + + + test("logs exception class and server error status") { + val filter = new HistoryServerAccessLogFilter(new SparkConf(false)) + val request = mockRequest(uri = "/history/app-1/jobs") + val response = mock[HttpServletResponse] + val chain = new FilterChain { + override def doFilter(req: ServletRequest, res: ServletResponse): Unit = { + throw new IllegalStateException("failed") + } + } + + val messages = withAccessLogAppender { + intercept[IllegalStateException] { + filter.doFilter(request, response, chain) + } + } + + assert(messages.size === 1) + assert(messages.head.contains("status=500")) + assert(messages.head.contains("error=java.lang.IllegalStateException")) + } + + test("uses configured excluded path prefixes") { + val conf = new SparkConf(false) + .set(HISTORY_SERVER_UI_ACCESS_LOG_EXCLUDE_PATHS, Seq("/api/private")) + val filter = new HistoryServerAccessLogFilter(conf) + val response = mock[HttpServletResponse] + val chain = new FilterChain { + override def doFilter(req: ServletRequest, res: ServletResponse): Unit = {} + } + + val messages = withAccessLogAppender { + filter.doFilter(mockRequest(uri = "/api/private/health"), response, chain) + filter.doFilter(mockRequest(uri = "/favicon.ico"), response, chain) + } + + assert(messages.size === 1) + assert(messages.head.contains("uri=/favicon.ico")) + } + + private def mockRequest( + uri: String, + queryString: String = null, + userAgent: String = "JUnit", + referer: String = null): HttpServletRequest = { + val request = mock[HttpServletRequest] + when(request.getMethod).thenReturn("GET") + when(request.getRequestURI).thenReturn(uri) + when(request.getQueryString).thenReturn(queryString) + when(request.getRemoteAddr).thenReturn("192.0.2.10") + when(request.getRemoteUser).thenReturn(null) + when(request.getUserPrincipal).thenReturn(null) + when(request.getHeader("User-Agent")).thenReturn(userAgent) + when(request.getHeader("Referer")).thenReturn(referer) + request + } + + private def withAccessLogAppender(f: => Unit): Seq[String] = { + val logAppender = new LogAppender + withLogAppender( + logAppender, + loggerNames = Seq(classOf[HistoryServerAccessLogFilter].getName), + level = Some(Level.INFO)) { + f + } + logAppender.loggingEvents.map(_.getMessage.getFormattedMessage) + .filter(_.contains("Spark History Server access")) + .toSeq + } +} diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala index 79e0001fa22fb..c9a8e8d05d4b2 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala @@ -27,6 +27,7 @@ import scala.jdk.CollectionConverters._ import jakarta.servlet._ import jakarta.servlet.http.{HttpServletRequest, HttpServletRequestWrapper, HttpServletResponse} import org.apache.hadoop.fs.{FileStatus, FileSystem, Path} +import org.apache.logging.log4j.Level import org.json4s.JsonAST._ import org.json4s.jackson.JsonMethods import org.json4s.jackson.JsonMethods._ @@ -653,6 +654,34 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with } } + test("history server access log") { + stop() + init() + + val defaultMessages = withHistoryServerAccessLogAppender { + HistoryServerSuite.getContentAndCode(historyServerUrl("/api/v1/applications")) + } + assert(defaultMessages.isEmpty) + + stop() + init(HISTORY_SERVER_UI_ACCESS_LOG_ENABLED.key -> "true") + + val appId = "local-1430917381535" + val messages = withHistoryServerAccessLogAppender { + HistoryServerSuite.getContentAndCode( + historyServerUrl("/api/v1/applications?token=secret-value")) + HistoryServerSuite.getContentAndCode(historyServerUrl(s"/history/$appId/1/jobs/")) + HistoryServerSuite.getContentAndCode(historyServerUrl("/favicon.ico")) + } + + assert(messages.exists { msg => + msg.contains("uri=/api/v1/applications") && + msg.contains(s"query=token=${Utils.REDACTION_REPLACEMENT_TEXT}") + }) + assert(messages.exists(_.contains(s"uri=/history/$appId/1/jobs"))) + assert(!messages.exists(_.contains("uri=/favicon.ico"))) + } + test("SPARK-33215: speed up event log download by skipping UI rebuild") { val appId = "local-1430917381535" @@ -741,6 +770,23 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with HistoryServerSuite.getUrl(generateURL(path)) } + private def withHistoryServerAccessLogAppender(f: => Unit): Seq[String] = { + val logAppender = new LogAppender + withLogAppender( + logAppender, + loggerNames = Seq(classOf[HistoryServerAccessLogFilter].getName), + level = Some(Level.INFO)) { + f + } + logAppender.loggingEvents.map(_.getMessage.getFormattedMessage) + .filter(_.contains("Spark History Server access")) + .toSeq + } + + private def historyServerUrl(path: String): URL = { + new URI(s"http://127.0.0.1:${server.boundPort}$path").toURL + } + def generateURL(path: String): URL = { new URI(s"http://$localhost:$port/api/v1/$path").toURL } diff --git a/docs/monitoring.md b/docs/monitoring.md index 4f90cbb5645c5..f0101bdb704f3 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -249,6 +249,34 @@ Security options for the Spark History Server are covered more detail in the 1.0.0 + + spark.history.ui.accessLog.enabled + false + + Whether the History Server should log HTTP access records for its web UI and REST API. + When enabled, each non-excluded request is logged at INFO level through + the History Server process logs after the request completes by the + org.apache.spark.deploy.history.HistoryServerAccessLogFilter logger. + Access records include the request + method, URI, redacted query string, status code, duration, remote address, remote user + if available, user agent, referer, and request failure class if the request chain throws + an exception. Query string values are redacted using + spark.redaction.regex. Requests rejected by user-installed UI filters before + Spark's internal access log filter may not be recorded. + + 4.3.0 + + + spark.history.ui.accessLog.excludePaths + /static,/favicon.ico + + Comma-separated list of request path prefixes to exclude from History Server HTTP + access logs. The default excludes static resources and browser favicon requests to + reduce low-value log volume. Set this to an empty value to log all History Server + requests when spark.history.ui.accessLog.enabled is enabled. + + 4.3.0 + spark.history.kerberos.enabled false