[SNAPPYDATA] updated dependencies to address major CVEs

- also exclude log4j2 from yarn shuffle shadow jar
- fixed occasional failures in kafka suite

Author: Sumedh Wale

build.gradle

@@ -60,30 +60,29 @@ allprojects {
     scalaBinaryVersion = '2.11'
     scalaVersion = scalaBinaryVersion + '.8'
     hadoopVersion = '3.2.0'
-    protobufVersion = '3.6.1'
-    jerseyVersion = '2.22.2'
+    protobufVersion = '3.16.1'
+    jerseyVersion = '2.35'
     sunJerseyVersion = '1.19.4'
     jettyVersion = '9.4.43.v20210629'
     yarnTestJettyVersion = '9.3.24.v20180605' // only used by yarn tests
     jettyOldVersion = '6.1.26'
     log4j2Version = '2.17.2'
     slf4jVersion = '1.7.32'
-    junitVersion = '4.12'
+    junitVersion = '4.13.2'
     mockitoVersion = '1.10.19'
     javaxServletVersion = '3.1.0'
     guavaVersion = '14.0.1'
     hiveVersion = '1.21.2.7.0.3.2-3'
     chillVersion = '0.8.5'
     kryoVersion = '4.0.2'
-    nettyVersion = '3.10.6.Final'
-    nettyAllVersion = '4.1.73.Final'
+    nettyVersion = '4.1.77.Final'
     derbyVersion = '10.14.2.0'
-    httpClientVersion = '4.5.6'
+    httpClientVersion = '4.5.13'
     httpCoreVersion = '4.4.10'
     levelDbJniVersion = '1.8'
     jackson1Version = '1.9.14-atlassian-6'
-    jacksonVersion = '2.13.1'
-    jacksonDatabindVersion = '2.13.1'
+    jacksonVersion = '2.13.3'
+    jacksonDatabindVersion = '2.13.3'
     snappyJavaVersion = '1.1.8.4'
     lz4Version = '1.8.0'
     lzfVersion = '1.1'
@@ -92,11 +91,13 @@ allprojects {
     metricsVersion = '3.2.6'
     janinoVersion = '3.0.8'
     thriftVersion = '0.9.3'
+    fb303Version = '0.9.3'
     antlrVersion = '4.5.3'
     jpamVersion = '1.1'
     seleniumVersion = '2.52.0'
     curatorVersion = '2.7.1'
-    commonsCodecVersion = '1.11'
+    commonsCodecVersion = '1.15'
+    commonsCompressVersion = '1.21'
     commonsCryptoVersion = '1.0.0'
     commonsLang3Version = '3.8.1'
     commonsMath3Version = '3.6.1'
@@ -120,7 +121,7 @@ allprojects {
     dockerClientVersion = '8.14.5'
     mysqlVersion = '8.0.13'
     postgresqlVersion = '42.2.5'
-    ojdbc6Version = '11.2.0.4'
+    ojdbc8Version = '12.2.0.1'
     zookeeperVersion = '3.6.2'
     activationVersion = '1.1.1'
     roaringBitmapVersion = '0.6.66'
@@ -181,7 +182,9 @@ allprojects {
         exclude(group: 'org.apache.ant', module: 'ant')
         exclude(group: 'com.esotericsoftware.kryo', module: 'kryo')
         exclude(group: 'commons-codec', module: 'commons-codec')
+        exclude(group: 'org.apache.commons', module: 'commons-compress')
         exclude(group: 'commons-httpclient', module: 'commons-httpclient')
+        exclude(group: 'org.apache.avro', module: 'avro')
         exclude(group: 'org.apache.avro', module: 'avro-mapred')
         exclude(group: 'org.apache.calcite', module: 'calcite-core')
         exclude(group: 'org.apache.curator', module: 'apache-curator')
@@ -232,7 +235,7 @@ allprojects {
         exclude(group: 'org.apache.velocity', module: 'velocity')
         exclude(group: 'org.apache.avro', module: 'avro-ipc')
       },
-      netty_all: dependencies.create("io.netty:netty-all:${nettyAllVersion}") {
+      netty_all: dependencies.create("io.netty:netty-all:${nettyVersion}") {
         exclude(group: 'io.netty', module: 'netty-codec-dns')
         exclude(group: 'io.netty', module: 'netty-codec-haproxy')
         exclude(group: 'io.netty', module: 'netty-codec-http')

common/network-yarn/build.gradle

@@ -41,7 +41,7 @@ shadowJar {
     exclude(dependency('org.scala-lang:.*'))
     exclude(dependency('org.scala-lang.modules:.*'))
     exclude(dependency('org.slf4j:.*'))
-    exclude(dependency('log4j:.*'))
+    exclude(dependency('org.apache.logging.log4j:.*'))
     exclude(dependency('org.scalatest:.*'))
   }
   //configurations = [ project.configurations.runtimeJar ]

core/build.gradle

@@ -24,6 +24,7 @@ dependencies {
   compile project(subprojectBase + 'snappy-spark-unsafe_' + scalaBinaryVersion)
   compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion)
 
+  compile group: 'org.apache.commons', name: 'commons-compress', version: commonsCompressVersion
   compile libraries.avro_ipc
   compile libraries.avro_mapred
   compile group: 'com.google.guava', name: 'guava', version: guavaVersion
@@ -35,14 +36,11 @@ dependencies {
     exclude(group: 'com.esotericsoftware', module: 'kryo-shaded')
   }
   compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: xbeanAsm5Version
-  // explicitly include netty from akka-remote to not let zookeeper override it
-  compile(group: 'io.netty', name: 'netty', version: nettyVersion) {
-    exclude(group: 'log4j', module: 'log4j')
-  }
   // explicitly exclude old netty from zookeeper
   compile(group: 'org.apache.zookeeper', name: 'zookeeper', version: zookeeperVersion) {
     exclude(group: 'org.jboss.netty', module: 'netty')
     exclude(group: 'jline', module: 'jline')
+    exclude(group: 'io.netty', module: 'netty')
     exclude(group: 'io.netty', module: 'netty-handler')
     exclude(group: 'io.netty', module: 'netty-transport-native-epoll')
     exclude(group: 'com.github.spotbugs', module: 'spotbugs-annotations')
@@ -93,10 +91,15 @@ dependencies {
   compile group: 'commons-net', name: 'commons-net', version: commonsNetVersion
   compile group: 'org.json4s', name: 'json4s-jackson_' + scalaBinaryVersion, version: json4sVersion
   compile group: 'org.glassfish.jersey.core', name: 'jersey-client', version: jerseyVersion
-  compile group: 'org.glassfish.jersey.core', name: 'jersey-common', version: jerseyVersion
-  compile group: 'org.glassfish.jersey.core', name: 'jersey-server', version: jerseyVersion
+  compile(group: 'org.glassfish.jersey.core', name: 'jersey-common', version: jerseyVersion) {
+    exclude(group: 'com.sun.activation', module: 'jakarta.activation')
+  }
+  compile(group: 'org.glassfish.jersey.core', name: 'jersey-server', version: jerseyVersion) {
+    exclude(group: 'jakarta.xml.bind', module: 'jakarta.xml.bind-api')
+  }
   compile group: 'org.glassfish.jersey.containers', name: 'jersey-container-servlet', version: jerseyVersion
   compile group: 'org.glassfish.jersey.containers', name: 'jersey-container-servlet-core', version: jerseyVersion
+  compile group: 'org.glassfish.jersey.inject', name: 'jersey-hk2', version: jerseyVersion
   compile(group: 'com.clearspring.analytics', name: 'stream', version: streamVersion) {
     exclude(group: 'it.unimi.dsi', module: 'fastutil')
   }
@@ -137,6 +140,7 @@ dependencies {
     exclude(group: 'com.google.guava', module: 'guava')
     exclude(group: 'io.netty', module: 'netty')
   }
+  testCompile group: 'org.glassfish.jersey.test-framework.providers', name: 'jersey-test-framework-provider-simple', version: jerseyVersion
   testCompile(group: 'org.seleniumhq.selenium', name: 'selenium-htmlunit-driver', version: seleniumVersion) {
     exclude(group: 'com.google.guava', module: 'guava')
   }

external/docker-integration-tests/build.gradle

@@ -37,7 +37,7 @@ dependencies {
   testCompile group: 'org.apache.httpcomponents', name: 'httpcore', version: httpCoreVersion
   testCompile group: 'mysql', name: 'mysql-connector-java', version: mysqlVersion
   testCompile group: 'org.postgresql', name: 'postgresql', version: postgresqlVersion
-  testCompile group: 'com.oracle.database.jdbc', name: 'ojdbc6', version: ojdbc6Version
+  testCompile group: 'com.oracle.database.jdbc', name: 'ojdbc8', version: ojdbc8Version
   testCompile group: 'com.sun.jersey', name: 'jersey-server', version: sunJerseyVersion
   testCompile group: 'com.sun.jersey', name: 'jersey-core', version: sunJerseyVersion
   testCompile group: 'com.sun.jersey', name: 'jersey-servlet', version: sunJerseyVersion

external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceSuite.scala

@@ -432,7 +432,7 @@ class KafkaSourceSuite extends KafkaSourceTest {
       .format("kafka")
       .option("kafka.bootstrap.servers", testUtils.brokerAddress)
       .option("kafka.metadata.max.age.ms", "1")
-      .option("kafka.default.api.timeout.ms", "3000")
+      .option("kafka.default.api.timeout.ms", "10000")
       .option("subscribePattern", s"$topicPrefix-.*")
       .option("failOnDataLoss", "false")
 
@@ -577,16 +577,16 @@ class KafkaSourceSuite extends KafkaSourceTest {
       .readStream
       .format("kafka")
       .option("kafka.bootstrap.servers", testUtils.brokerAddress)
-      .option("kafka.default.api.timeout.ms", "3000")
+      .option("kafka.default.api.timeout.ms", "10000")
       .option("kafka.metadata.max.age.ms", "1")
       .option("subscribe", topic)
       // If a topic is deleted and we try to poll data starting from offset 0,
       // the Kafka consumer will just block until timeout and return an empty result.
       // So set the timeout to 1 second to make this test fast.
-      .option("kafkaConsumer.pollTimeoutMs", "1000")
+      .option("kafkaConsumer.pollTimeoutMs", "3000")
       .option("startingOffsets", "earliest")
       .option("failOnDataLoss", "false")
-      .option("kafka.default.api.timeout.ms", "3000")
+      .option("kafka.default.api.timeout.ms", "10000")
     val kafka = reader.load()
       .selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)")
       .as[(String, String)]
@@ -888,7 +888,7 @@ class KafkaSourceStressSuite extends KafkaSourceTest {
         .option("kafka.metadata.max.age.ms", "1")
         .option("subscribePattern", "stress.*")
         .option("failOnDataLoss", "false")
-        .option("kafka.default.api.timeout.ms", "3000")
+        .option("kafka.default.api.timeout.ms", "10000")
         .load()
         .selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)")
         .as[(String, String)]
@@ -985,11 +985,11 @@ class KafkaSourceStressForDontFailOnDataLossSuite extends StreamTest with Shared
       .format("kafka")
       .option("kafka.bootstrap.servers", testUtils.brokerAddress)
       .option("kafka.metadata.max.age.ms", "1")
-      .option("kafka.default.api.timeout.ms", "3000")
+      .option("kafka.default.api.timeout.ms", "10000")
       .option("subscribePattern", "failOnDataLoss.*")
       .option("startingOffsets", "earliest")
       .option("failOnDataLoss", "false")
-      .option("fetchOffset.retryIntervalMs", "3000")
+      .option("fetchOffset.retryIntervalMs", "10000")
     val kafka = reader.load()
       .selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)")
       .as[(String, String)]

sql/hive/build.gradle

@@ -57,7 +57,7 @@ dependencies {
   compile(group: 'org.apache.thrift', name: 'libthrift', version: thriftVersion) {
     exclude(group: 'org.slf4j', module: 'slf4j-api')
   }
-  compile(group: 'org.apache.thrift', name: 'libfb303', version: thriftVersion) {
+  compile(group: 'org.apache.thrift', name: 'libfb303', version: fb303Version) {
     exclude(group: 'org.slf4j', module: 'slf4j-api')
   }
 

yarn/build.gradle

@@ -35,6 +35,7 @@ dependencies {
     exclude(group: 'org.ow2.asm', module: 'asm')
     exclude(group: 'org.jboss.netty', module: 'netty')
     exclude(group: 'javax.servlet', module: 'servlet-api')
+    exclude(group: 'org.apache.commons', module: 'commons-compress')
     exclude(group: 'commons-logging', module: 'commons-logging')
     exclude(group: 'com.sun.jersey')
     exclude(group: 'com.sun.jersey.jersey-test-framework')
@@ -78,7 +79,7 @@ dependencies {
   compile(group: 'org.apache.thrift', name: 'libthrift', version: thriftVersion) {
     exclude(group: 'org.slf4j', module: 'slf4j-api')
   }
-  compile(group: 'org.apache.thrift', name: 'libfb303', version: thriftVersion) {
+  compile(group: 'org.apache.thrift', name: 'libfb303', version: fb303Version) {
     exclude(group: 'org.slf4j', module: 'slf4j-api')
   }