Merge pull request #436 from codacy/duplication-tools-api

feature: Use duplication tools from API

Author: lolgab

.circleci/config.yml

@@ -89,6 +89,8 @@ references:
           webhook: https://hooks.slack.com/services/$SLACK_TOKEN
           failure_message: ":red_circle: A $CIRCLE_JOB job has failed! See more on: $RP_LAUNCH_URL, login with the default user."
 
+  cache_prefix: &cache_prefix sbt-cache-31012022
+
 jobs:
   maven_dependencies:
     docker:
@@ -144,35 +146,39 @@ workflows:
           write_sbt_version: true
       - codacy/sbt:
           name: compile
-          install_sbt_version: 1.3.12
+          install_sbt_version: 1.6.1
           cmd: sbt coverage +test:compile
+          cache_prefix: *cache_prefix
           persist_to_workspace: true
           requires:
             - codacy/checkout_and_version
       - codacy/sbt:
           name: lint
-          install_sbt_version: 1.3.12
+          install_sbt_version: 1.6.1
           cmd: |
             sbt "scalafmtCheckAll;scalafmtSbtCheck;scapegoat;scalafixEnable;scalafix --test"
+          cache_prefix: *cache_prefix
           requires:
             - compile
       - codacy/sbt:
           name: test
-          install_sbt_version: 1.3.12
+          install_sbt_version: 1.6.1
           cmd: |
             git config --global user.email "team@codacy.com"
             git config --global user.name "Codacy Team"
             sbt coverage +test coverageReport
             bash <(curl -Ls https://coverage.codacy.com/get.sh) report --skip
+          cache_prefix: *cache_prefix
           requires:
             - compile
       - codacy/sbt:
           name: publish_docker_locally
-          install_sbt_version: 1.3.12
+          install_sbt_version: 1.6.1
           cmd: |
             sbt "set codacyAnalysisCli / version := \"dev-snapshot\";
                  codacyAnalysisCli/docker:publishLocal"
             docker save --output docker-image.tar $CIRCLE_PROJECT_REPONAME:dev-snapshot
+          cache_prefix: *cache_prefix
           persist_to_workspace: true
           requires:
             - compile
@@ -191,13 +197,14 @@ workflows:
       #       - maven_dependencies
       - codacy/sbt:
           name: publish_lib
-          install_sbt_version: 1.3.12
+          install_sbt_version: 1.6.1
           context: CodacyAWS
           cmd: |
             sbt "retrieveGPGKeys;
                  codacyAnalysisCore/publishSigned;
                  +codacyAnalysisModels/publishSigned;
                  sonatypeBundleRelease"
+          cache_prefix: *cache_prefix
           requires:
             - publish_docker_locally
           filters:
@@ -216,14 +223,15 @@ workflows:
                 - master
       - codacy/sbt:
           name: build_cli_jar
-          install_sbt_version: 1.3.12
+          install_sbt_version: 1.6.1
           persist_to_workspace: true
           cmd: |
             sbt "set codacyAnalysisCli / version := \"$(cat .version)\";
-                 set test in assembly := {};
+                 set assembly / test := {};
                  codacyAnalysisCli/assembly"
             mkdir -p ~/workdir/artifacts/
             mv ./cli/target/scala-2.12/codacy-analysis-cli-assembly-$(cat .version).jar ~/workdir/artifacts/
+          cache_prefix: *cache_prefix
           requires:
             - publish_docker_locally
           filters:

.gitignore

@@ -85,6 +85,7 @@ project/plugins/project/
 .DS_Store
 
 .bloop
+.bsp
 .metals
 project/metals.sbt
 project/project/

build.sbt

@@ -4,19 +4,17 @@ import sbt._
 import codacy.libs._
 import sbt._
 
-Universal / javaOptions ++= Seq("-Xms1g", "-Xmx2g", "-Xss512m", "-XX:+UseG1GC", "-XX:+UseStringDeduplication")
-
 val assemblyCommon = Seq(
-  test in assembly := {},
+  assembly / test := {},
   // Without this assembly merge strategy, gives the following error:
   // (codacyAnalysisCli / assembly) deduplicate: different file contents found in the following:
   // [error] org/bouncycastle/bcpg-jdk15on/1.64/bcpg-jdk15on-1.64.jar:META-INF/versions/9/module-info.class
   // Workaround:
   // https://stackoverflow.com/questions/54834125/sbt-assembly-deduplicate-module-info-class
-  assemblyMergeStrategy in assembly := {
+  assembly / assemblyMergeStrategy := {
     case "META-INF/versions/9/module-info.class" => MergeStrategy.discard
     case x =>
-      val oldStrategy = (assemblyMergeStrategy in assembly).value
+      val oldStrategy = (assembly / assemblyMergeStrategy).value
       oldStrategy(x)
   })
 
@@ -60,8 +58,6 @@ lazy val codacyAnalysisCore = project
     sonatypeInformation,
     description := "Library to analyze projects")
   .settings(assemblyCommon: _*)
-  // Disable legacy Scalafmt plugin imported by codacy-sbt-plugin
-  .disablePlugins(com.lucidchart.sbt.scalafmt.ScalafmtCorePlugin)
   .dependsOn(codacyAnalysisModels)
 
 lazy val codacyAnalysisCli = project
@@ -79,18 +75,14 @@ lazy val codacyAnalysisCli = project
   .settings(assemblyCommon: _*)
   .enablePlugins(JavaAppPackaging)
   .enablePlugins(DockerPlugin)
-  // Disable legacy Scalafmt plugin imported by codacy-sbt-plugin
-  .disablePlugins(com.lucidchart.sbt.scalafmt.ScalafmtCorePlugin)
   .dependsOn(codacyAnalysisCore % "compile->compile;test->test", toolRepositoryRemote)
   .aggregate(codacyAnalysisCore, toolRepositoryRemote)
 
 lazy val toolRepositoryRemote = project
   .in(file("toolRepository-remote"))
   .settings(Common.genericSettings, libraryDependencies ++= Dependencies.specs2)
   .settings(assemblyCommon: _*)
-  // Disable legacy Scalafmt plugin imported by codacy-sbt-plugin
-  .disablePlugins(com.lucidchart.sbt.scalafmt.ScalafmtCorePlugin)
-  .dependsOn(codacyAnalysisCore, codacyAnalysisModels, codacyApiClient)
+  .dependsOn(codacyAnalysisCore % "compile->compile;test->test", codacyAnalysisModels, codacyApiClient)
 
 lazy val codacyAnalysisModels = project
   .in(file("model"))
@@ -104,8 +96,6 @@ lazy val codacyAnalysisModels = project
     description := "Library with analysis models")
   .settings(assemblyCommon: _*)
   .settings(sonatypeInformation)
-  // Disable legacy Scalafmt plugin imported by codacy-sbt-plugin
-  .disablePlugins(com.lucidchart.sbt.scalafmt.ScalafmtCorePlugin)
   .enablePlugins(JavaAppPackaging)
 
 lazy val apiSwaggerFile: File =

cli/src/main/scala/com/codacy/analysis/cli/analysis/ToolSelector.scala

@@ -10,25 +10,29 @@ import com.codacy.plugins.api.languages.Language
 class ToolSelector(toolRepository: ToolRepository) {
 
   private val toolCollector = new ToolCollector(toolRepository)
+  private val duplicationToolCollector = new DuplicationToolCollector(toolRepository)
 
   def allTools(toolInput: Option[String],
                configuration: CLIConfiguration.Tool,
                languages: Set[Language]): Either[CLIError, Set[ITool]] = {
 
+    def toolsEither = tools(toolInput, configuration, languages)
     def metricsTools = MetricsToolCollector.fromLanguages(languages)
-    def duplicationTools = DuplicationToolCollector.fromLanguages(languages)
+    def duplicationToolsEither =
+      duplicationToolCollector.fromLanguages(languages).left.map(e => CLIError.CouldNotGetTools(e.message))
 
     toolInput match {
       case None =>
-        val toolsEither = tools(toolInput, configuration, languages)
-
-        toolsEither.map(_ ++ metricsTools ++ duplicationTools)
+        for {
+          tools <- toolsEither
+          duplicationTools <- duplicationToolsEither
+        } yield tools ++ metricsTools ++ duplicationTools
 
       case Some("metrics") =>
         Right(metricsTools.map(_.to[ITool]))
 
       case Some("duplication") =>
-        Right(duplicationTools.map(_.to[ITool]))
+        duplicationToolsEither.map(_.map(_.to[ITool]))
 
       case Some(_) =>
         val toolsEither = tools(toolInput, configuration, languages)

cli/src/main/scala/com/codacy/analysis/cli/command/AnalyseCommand.scala

@@ -25,11 +25,12 @@ import com.codacy.analysis.core.configuration.CodacyConfigurationFileLoader
 import com.codacy.analysis.core.files.FileCollector
 import com.codacy.analysis.core.git.{Commit, Git, Repository}
 import com.codacy.analysis.core.model._
+import com.codacy.analysis.core.storage.FileDataStorage
 import com.codacy.analysis.core.upload.ResultsUploader
 import com.codacy.analysis.core.utils.Logger
 import com.codacy.analysis.core.utils.SeqOps._
 import com.codacy.toolRepository.remote.ToolRepositoryRemote
-import com.codacy.toolRepository.remote.storage.{PatternSpecDataStorage, ToolSpecDataStorage}
+import com.codacy.toolRepository.remote.storage.ToolPatternsSpecsEncoders._
 import org.log4s.getLogger
 
 import scala.concurrent.ExecutionContext.Implicits.global
@@ -81,7 +82,9 @@ object AnalyseCommand {
 
     val toolsClient =
       ToolsClient(codacyApiUrl)(httpClient = httpClient, ec = actorSystem.dispatcher, mat = materializer)
-    new ToolRepositoryRemote(toolsClient, ToolSpecDataStorage.apply, PatternSpecDataStorage.apply)(
+    val toolsDataStorage = FileDataStorage[ToolSpec]("tools")
+    val duplicationToolsDataStorage = FileDataStorage[DuplicationToolSpec]("duplicationTools")
+    new ToolRepositoryRemote(toolsClient, toolsDataStorage, duplicationToolsDataStorage, FileDataStorage(_))(
       ec = actorSystem.dispatcher,
       mat = materializer)
   }

cli/src/test/scala/com/codacy/analysis/cli/AnalyseExecutorSpec.scala

@@ -166,7 +166,7 @@ class AnalyseExecutorSpec extends Specification with NoLanguageFeatures with Moc
     val analyser: Analyser[Try] = Analyser(analyserName)
     val fileCollector: FileCollector[Try] = FileCollector.defaultCollector()
 
-    val toolRepository = new ToolRepositoryMock
+    val toolRepository = ToolRepositoryMock
     val toolSelector = new ToolSelector(toolRepository)
     new AnalyseExecutor(formatter, analyser, fileCollector, configuration, toolSelector).run() must beRight
   }

cli/src/test/scala/com/codacy/analysis/cli/CLISpec.scala

@@ -8,9 +8,8 @@ import com.codacy.analysis.core.git.Commit
 import com.codacy.analysis.core.model.{DuplicationClone, FileError, Result, ToolResult}
 import com.codacy.analysis.core.utils.TestUtils._
 import com.codacy.plugins.utils.CommandRunner
-import io.circe
 import io.circe.generic.auto._
-import io.circe.{Json, parser}
+import io.circe.parser
 import org.specs2.control.NoLanguageFeatures
 import org.specs2.matcher.FileMatchers
 import org.specs2.mutable.Specification

cli/src/test/scala/com/codacy/analysis/cli/ToolSelectorSpec.scala

@@ -5,19 +5,20 @@ import com.codacy.analysis.cli.analysis.ToolSelector
 import com.codacy.analysis.cli.configuration.CLIConfiguration
 import com.codacy.analysis.core.files.FilesTarget
 import com.codacy.analysis.core.model.{AnalyserError, PatternSpec, ToolSpec}
-import com.codacy.analysis.core.tools.{Tool, ToolRepository}
+import com.codacy.analysis.core.tools.{Tool, ToolRepositoryStub}
 import com.codacy.analysis.core.utils.LanguagesHelper
 import com.codacy.plugins.api.languages.Languages.{Javascript, Python}
 import com.codacy.plugins.api.languages.{Language, Languages}
 import org.specs2.control.NoLanguageFeatures
 import org.specs2.mutable.Specification
+import com.codacy.analysis.core.model.DuplicationToolSpec
 
 class ToolSelectorSpec extends Specification with NoLanguageFeatures {
 
   val emptyFilesTarget = FilesTarget(File(""), Set.empty, Set.empty)
   val noLocalConfiguration = Left("no config")
 
-  val toolRepository = new ToolRepository {
+  val toolRepository = new ToolRepositoryStub {
 
     private def getToolSpec(uuid: String, name: String, languages: Set[Language]) =
       ToolSpec(
@@ -37,7 +38,7 @@ class ToolSelectorSpec extends Specification with NoLanguageFeatures {
         isClientSide = false,
         hasUIConfiguration = true)
 
-    override def list(): Either[AnalyserError, Seq[ToolSpec]] =
+    override def listTools(): Either[AnalyserError, Seq[ToolSpec]] =
       Right(
         Seq(
           getToolSpec("34225275-f79e-4b85-8126-c7512c987c0d", "PyLint", Set(Python)),
@@ -46,9 +47,10 @@ class ToolSelectorSpec extends Specification with NoLanguageFeatures {
           getToolSpec("38794ba2-94d8-4178-ab99-1f5c1d12760c", "BundlerAudit", Set(Languages.Ruby)),
           getToolSpec("cf05f3aa-fd23-4586-8cce-5368917ec3e5", "ESLint", Set(Languages.Javascript))))
 
-    override def get(uuid: String): Either[AnalyserError, ToolSpec] = ???
-
     override def listPatterns(tool: ToolSpec): Either[AnalyserError, Seq[PatternSpec]] = Right(Seq.empty)
+
+    override def listDuplicationTools(): Either[AnalyserError, Seq[DuplicationToolSpec]] =
+      Right(Seq(DuplicationToolSpec("", Set(Languages.Ruby))))
   }
   val toolSelector: ToolSelector = new ToolSelector(toolRepository)
 

…lysis/core/storage/FileDataStorage.scala …/analysis/core/storage/DataStorage.scalacore/src/main/scala/com/codacy/analysis/core/storage/FileDataStorage.scala renamed to core/src/main/scala/com/codacy/analysis/core/storage/DataStorage.scala core/src/main/scala/com/codacy/analysis/core/storage/FileDataStorage.scala renamed to core/src/main/scala/com/codacy/analysis/core/storage/DataStorage.scala

@@ -8,36 +8,46 @@ import org.log4s.{Logger, getLogger}
 
 import scala.util.{Failure, Success, Try}
 
-abstract class FileDataStorage[T](val storageFilename: String) {
+trait DataStorage[T] {
+  def get(): Option[Seq[T]]
+  def save(values: Seq[T]): Boolean
+  def invalidate(): Try[Unit]
+}
 
-  implicit val encoder: Encoder[T]
-  implicit val decoder: Decoder[T]
+object FileDataStorage {
 
-  private val logger: Logger = getLogger
+  def apply[T](storageFilename: String)(implicit encoder: Encoder[T], decoder: Decoder[T]): DataStorage[T] = {
+    val cacheFolder: File = {
+      val defaultFolder = File.currentWorkingDirectory / ".codacy" / "codacy-analysis-cli"
+      val osNameOpt = sys.props.get("os.name").map(_.toLowerCase)
+      val result = osNameOpt match {
+        case Some(sysName) if sysName.contains("mac") || sysName == "darwin" =>
+          home / "Library" / "Caches" / "Codacy" / "codacy-analysis-cli"
 
-  private val cacheFolder: File = {
-    val defaultFolder = File.currentWorkingDirectory / ".codacy" / "codacy-analysis-cli"
-    val osNameOpt = sys.props.get("os.name").map(_.toLowerCase)
-    val result = osNameOpt match {
-      case Some(sysName) if sysName.contains("mac") || sysName == "darwin" =>
-        home / "Library" / "Caches" / "Codacy" / "codacy-analysis-cli"
+        case Some(sysName) if sysName.contains("nix") || sysName.contains("nux") =>
+          home / ".cache" / "codacy" / "codacy-analysis-cli"
 
-      case Some(sysName) if sysName.contains("nix") || sysName.contains("nux") =>
-        home / ".cache" / "codacy" / "codacy-analysis-cli"
+        case Some(sysName) if sysName.contains("windows") =>
+          sys.env.get("APPDATA").fold(defaultFolder) { windowsCacheDir =>
+            File(windowsCacheDir) / "Codacy" / "codacy-analysis-cli"
+          }
 
-      case Some(sysName) if sysName.contains("windows") =>
-        sys.env.get("APPDATA").fold(defaultFolder) { windowsCacheDir =>
-          File(windowsCacheDir) / "Codacy" / "codacy-analysis-cli"
-        }
+        case _ => defaultFolder
+      }
 
-      case _ => defaultFolder
+      result.createIfNotExists(asDirectory = true, createParents = true)
+      result
     }
-
-    result.createIfNotExists(asDirectory = true, createParents = true)
-    result
+    apply[T](cacheFolder / storageFilename)
   }
 
-  val storageFile: File = cacheFolder / storageFilename
+  def apply[T](storageFile: File)(implicit encoder: Encoder[T], decoder: Decoder[T]): DataStorage[T] =
+    new FileDataStorage(storageFile)
+}
+
+private class FileDataStorage[T] private[storage] (storageFile: File)(implicit encoder: Encoder[T], decoder: Decoder[T])
+    extends DataStorage[T] {
+  private val logger: Logger = getLogger
 
   private def writeToFile(content: String): Try[File] =
     Try {