Merge pull request #25 from riptano/DSP-19490-dse-2.2

Dsp 19490 dse 2.2

Author: alexliu68

bin/server_package.sh

@@ -35,12 +35,16 @@ else
   echo "Please specify SCALA_VERSION in ${configFile}"
   exit 1
 fi
+
+if [ -z "${SBT_BIN}" ]; then
+  export SBT_BIN="sbt"
+fi
 set -u
 
 echo "Packaging job-server for environment ${ENV}..."
 
 pushd "${bin}/.." > /dev/null
-  if ! sbt ++"${SCALA_VERSION}" job-server-extras/assembly; then
+  if ! "$SBT_BIN" ++"${SCALA_VERSION}" job-server-extras/assembly; then
     echo "Assembly failed"
     exit 1
   fi

job-server/src/main/scala/spark/jobserver/JobServer.scala

@@ -12,6 +12,7 @@ import org.slf4j.LoggerFactory
 import scala.collection.JavaConverters._
 import scala.concurrent.{Await, ExecutionContext}
 import scala.concurrent.duration._
+import scala.util.matching.Regex
 import scala.util.Try
 
 /**
@@ -32,6 +33,9 @@ import scala.util.Try
  */
 object JobServer {
   val logger = LoggerFactory.getLogger(getClass)
+  final val DEFAULT_CREDENTIAL_PATTERN = "(?i)credentials|secret|password|token"
+  final val EMPTY_VALUE_PATTERN = "\"\",?".r
+  final val REDACTION_REPLACEMENT_TEXT = " \"*********(redacted)\""
 
   class InvalidConfiguration(error: String) extends RuntimeException(error)
 
@@ -49,8 +53,12 @@ object JobServer {
     } else {
       defaultConfig
     }
-    logger.info("Starting JobServer with config {}", config.getConfig("spark").root.render())
-    logger.info("Spray config: {}", config.getConfig("spray.can.server").root.render())
+
+    val credentialPattern = credentialRegex(config)
+    val sparkConfig = config.getConfig("spark").root.render()
+    logger.info("Starting JobServer with config {}", maskCredentials(sparkConfig, credentialPattern))
+    val sprayConfig = config.getConfig("spray.can.server").root.render()
+    logger.info("Spray config: {}", maskCredentials(sprayConfig, credentialPattern))
 
     // TODO: Hardcode for now to get going. Make it configurable later.
     val system = makeSystem(config)
@@ -122,6 +130,35 @@ object JobServer {
     new WebApi(system, config, port, binManager, dataManager, supervisor, jobInfo).start()
   }
 
+  private def maskCredentials(lines: String, credentialRegex: Regex): String = {
+    lines
+      .split("\n")
+      .toSeq
+      .map {
+        line => line.split(":") match {
+          // if key matches credential keys pattern and value is not empty, mask credentials
+          case Array(key, value) if (credentialRegex.findFirstIn(key).nonEmpty
+            && EMPTY_VALUE_PATTERN.findFirstIn(value.stripMargin).isEmpty) =>
+            Array(key, REDACTION_REPLACEMENT_TEXT).mkString(":")
+          case _ => line
+        }
+      }.mkString("\n")
+  }
+
+  private def credentialRegex(config: Config): Regex = {
+    // Use default credential keys if spark.ui.confidentialKeys is not set
+    val credentialPattern = try {
+      config.getString("spark.redaction.regex")
+    } catch {
+      case _: Exception =>
+        logger.info(s"spark.redaction.regex is not set, " +
+          s"use default credential pattern $DEFAULT_CREDENTIAL_PATTERN")
+        DEFAULT_CREDENTIAL_PATTERN
+    }
+
+    credentialPattern.r
+  }
+
   private def parseInitialBinaryConfig(key: String, config: Config): Map[String, String] = {
     if (config.hasPath(key)) {
       val initialJarsConfig = config.getConfig(key).root

project/Dependencies.scala

@@ -89,7 +89,7 @@ object Dependencies {
   lazy val apiDeps = sparkDeps ++ miscDeps :+ typeSafeConfigDeps :+ scalaTestDep
 
   val repos = Seq(
-    "datastax-release" at "http://datastax.artifactoryonline.com/datastax/datastax-releases-local",
+    "datastax-release" at "https://repo.sjc.dsinternal.org/artifactory/datastax-releases-local",
     "Typesafe Repo" at "http://repo.typesafe.com/typesafe/releases/",
     "sonatype snapshots" at "https://oss.sonatype.org/content/repositories/snapshots/",
     "spray repo" at "http://repo.spray.io"

project/Versions.scala

@@ -1,7 +1,7 @@
 import scala.util.Properties.isJavaAtLeast
 
 object Versions {
-  lazy val spark = sys.env.getOrElse("SPARK_VERSION", "2.2.0.6")
+  lazy val spark = sys.env.getOrElse("SPARK_VERSION", "2.2.3.4")
 
   lazy val akka = "2.4.9"
   lazy val dseDriver = "1.6.2"
@@ -27,5 +27,5 @@ object Versions {
   lazy val spray = "1.3.3"
   lazy val sprayJson = "1.3.2"
   lazy val typeSafeConfig = if (isJavaAtLeast("1.8")) "1.3.0" else "1.2.1"
-  lazy val cassandraConnector = "6.0.0-6aa37a9"
+  lazy val cassandraConnector = "6.0.9"
 }