Merge branch 'main' into gh129-lockfile

Author: colindean

.github/CODEOWNERS

@@ -1 +1 @@
-* @colindean @c-horn @samratmitra-0812
+* @target/data-validator-members @c-horn

.github/dependabot.yml

@@ -5,3 +5,7 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
+
+# Until Dependabot provides Scala updates, they are are handed by the Scala Steward action.
+# https://github.com/target/data-validator/blob/master/.github/workflows/scala-steward.yaml
+# Check periodically for updates: https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem

.github/workflows/ci.yaml

@@ -16,15 +16,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout current branch (full)
-        uses: actions/checkout@v3
-
-      - name: Setup Java and Scala
-        uses: olafurpg/setup-scala@v14
+        uses: actions/checkout@v4
+      - uses: coursier/cache-action@v6
         with:
-          java-version: adopt@1.8
-
-      - name: Cache sbt
-        uses: actions/cache@v3
+          extraKey: ${{ runner.os }}
+      - uses: coursier/setup-action@v1
         with:
           path: |
             ~/.sbt
@@ -34,6 +30,8 @@ jobs:
             ~/AppData/Local/Coursier/Cache/v1
             ~/Library/Caches/Coursier/v1
           key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }}
-
-      - name: Build, test, and package project
-        run: bin/sbt clean update dependencyLockCheck compile test package makePom
+          jvm: adopt:1.8
+      - name: Build, test, and package project on Spark 3.5
+        run: bin/sbt clean update dependencyLockCheck compile test package makePom -DsparkVersion=3.5.1
+      - name: Build and package project on "legacy" Spark
+        run: bin/sbt clean update dependencyLockCheck compile package makePom

.github/workflows/release.yaml

@@ -15,15 +15,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout current branch (full)
-        uses: actions/checkout@v3
-
-      - name: Setup Java and Scala
-        uses: olafurpg/setup-scala@v14
+        uses: actions/checkout@v4
+      - uses: coursier/cache-action@v6
         with:
-          java-version: adopt@1.8
-
-      - name: Cache sbt
-        uses: actions/cache@v3
+          extraKey: ${{ runner.os }}
+      - uses: coursier/setup-action@v1
         with:
           path: |
             ~/.sbt
@@ -33,7 +29,9 @@ jobs:
             ~/AppData/Local/Coursier/Cache/v1
             ~/Library/Caches/Coursier/v1
           key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }}
-
+          jvm: adopt:1.8
       # uses sbt-github-packages, see build.sbt
       - name: Publish with SBT
         run: bin/sbt dependencyLockCheck publish
+      - name: Publish with SBT
+        run: bin/sbt dependencyLockCheck publish -DsparkVersion=3.5.1

.github/workflows/scala-steward.yaml

@@ -21,7 +21,7 @@ jobs:
     name: Launch Scala Steward
     steps:
       - name: Install JDK for Scala Steward use
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '11'

.scalafmt.conf

@@ -1,4 +1,4 @@
-version = 3.7.3
+version = 3.8.1
 runner.dialect = scala211
 project.git = true
 align.preset = none

README.md

@@ -65,6 +65,9 @@ Usage: data-validator [options]
   --help                   Show this help message and exit.
 ```
 
+If you want to build with Java 11 or newer, set the "MODERN_JAVA" environment variable.
+This may become the default in the future.
+
 ## Example Run
 
 With the JAR directly:

bin/sbt

@@ -34,11 +34,11 @@
 
 set -o pipefail
 
-declare -r sbt_release_version="1.6.2"
-declare -r sbt_unreleased_version="1.7.0-M2"
+declare -r sbt_release_version="1.9.9"
+declare -r sbt_unreleased_version="1.9.9"
 
-declare -r latest_213="2.13.8"
-declare -r latest_212="2.12.15"
+declare -r latest_213="2.13.13"
+declare -r latest_212="2.12.19"
 declare -r latest_211="2.11.12"
 declare -r latest_210="2.10.7"
 declare -r latest_29="2.9.3"

build.sbt

@@ -1,11 +1,42 @@
 name := "data-validator"
 organization := "com.target"
 
-scalaVersion := "2.11.12"
+val sparkVersion = settingKey[String]("Spark version")
 
-val sparkVersion = "2.3.4"
+sparkVersion := System.getProperty("sparkVersion", "2.3.4")
 
-val circeVersion = "0.11.2"
+scalaVersion := {
+  if (sparkVersion.value > "3.0") {
+    "2.12.19"
+  } else {
+    "2.11.12"
+  }
+}
+
+val sparkValidationVersion = settingKey[String]("Version of package")
+
+sparkValidationVersion := "0.15.0"
+
+version := sparkVersion.value + "_" + sparkValidationVersion.value
+
+val circeVersion = settingKey[String]("Circe version")
+val circeYamlVersion = settingKey[String]("Circe YAML version")
+
+circeVersion := {
+  if (sparkVersion.value > "3.0") {
+    "0.14.6"
+  } else {
+    "0.11.2"
+  }
+}
+
+circeYamlVersion := {
+  if (sparkVersion.value > "3.0") {
+    "0.15.1"
+  } else {
+    "0.10.1"
+  }
+}
 
 //addDependencyTreePlugin
 enablePlugins(GitVersioning)
@@ -35,18 +66,28 @@ libraryDependencies ++= Seq(
   "com.github.scopt" %% "scopt" % "4.1.0",
   "com.sun.mail" % "javax.mail" % "1.6.2",
   "com.lihaoyi" %% "scalatags" % "0.12.0",
-  "io.circe" %% "circe-yaml" % "0.10.1",
-  "io.circe" %% "circe-core" % circeVersion,
-  "io.circe" %% "circe-generic" % circeVersion,
-  "io.circe" %% "circe-parser" % circeVersion,
-  "org.apache.spark" %% "spark-sql" % sparkVersion % Provided,
+  "io.circe" %% "circe-yaml" % circeYamlVersion.value,
+  "io.circe" %% "circe-core" % circeVersion.value,
+  "io.circe" %% "circe-generic" % circeVersion.value,
+  "io.circe" %% "circe-parser" % circeVersion.value,
+  "org.apache.spark" %% "spark-sql" % sparkVersion.value % Provided,
   "junit" % "junit" % "4.13.2" % Test,
-  "org.scalatest" %% "scalatest" % "3.2.15" % Test,
+  "org.scalatest" %% "scalatest" % "3.2.18" % Test,
   "com.github.sbt" % "junit-interface" % "0.13.3" % Test exclude ("junit", "junit-dep")
 )
 
 Test / fork := true
-javaOptions ++= Seq("-Xms512M", "-Xmx2048M", "-XX:+CMSClassUnloadingEnabled")
+javaOptions ++= (if (sparkVersion.value > "3.0" && System.getenv("MODERN_JAVA") == "TRUE") {
+  // For modern Java we need to open up a lot of config options.
+  Seq("-Xms4048M", "-Xmx4048M",
+    // these were added in JDK 11 and newer, apparently.
+    "-Dio.netty.tryReflectionSetAccessible=true",
+    "--add-opens=java.base/java.lang=ALL-UNNAMED",
+    "--add-opens=java.base/java.io=ALL-UNNAMED",
+    "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED")
+} else {
+    Seq("-Xms4048M", "-Xmx4048M")
+})
 Test / parallelExecution := false
 // required for unit tests, but not set in some environments
 Test / envVars ++= Map(
@@ -57,6 +98,11 @@ Test / envVars ++= Map(
 
 assembly / mainClass := Some("com.target.data_validator.Main")
 
+assembly / assemblyShadeRules := Seq(
+        ShadeRule.rename("shapeless.**" -> "new_shapeless.@1").inAll,
+        ShadeRule.rename("cats.kernel.**" -> s"new_cats.kernel.@1").inAll
+      )
+
 // Enforces scalastyle checks
 val compileScalastyle = TaskKey[Unit]("compileScalastyle")
 scalastyleFailOnWarning := true
@@ -75,6 +121,6 @@ compileScalastyle := (Compile / scalastyle).toTask("").value
 
 (Compile / runMain) := Defaults.runMainTask(Compile / fullClasspath, Compile / run / runner).evaluated
 TaskKey[Unit]("generateTestData") := {
-  libraryDependencies += "org.apache.spark" %% "spark-sql" % sparkVersion
+  libraryDependencies += "org.apache.spark" %% "spark-sql" % sparkVersion.value
   (Compile / runMain).toTask(" com.target.data_validator.GenTestData").value
 }

project/assembly.sbt

@@ -1 +1 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.1")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.2.0")