Merge pull request #440 from codacy/feature/upload-batch-size-param-CY-5727

feature: add upload batch size flag CY-5727

Author: pedrocodacy

README.md

@@ -205,6 +205,7 @@ Found [Clone] 7 duplicated lines with 10 tokens:
     * `--skip-commit-uuid-validation` [default: false] - Force using a commit UUID even if it doesn't belong to the current git branch.
     * `--skip-uncommitted-files-check` [default: false] - Skip check for uncommitted files in the analysis directory
     * `--upload` [default: false] - Request to push results to Codacy
+    * `--upload-batch-size` [default: 50000] - Batch size for upload of results
     * `--skip-ssl-verification` [default: false] - Skip the SSL certificate verification when communicating with the Codacy API
     * `--parallel` [default: 2] - Number of tools to run in parallel
     * `--max-allowed-issues` [default: 0] - Maximum number of issues allowed for the analysis to succeed

cli/src/main/scala/com/codacy/analysis/cli/command/AnalyseCommand.scala

@@ -69,7 +69,11 @@ object AnalyseCommand {
         configuration.analysis,
         toolSelector)
     val uploaderOpt: Either[String, Option[ResultsUploader]] =
-      ResultsUploader(codacyClientOpt, configuration.upload.upload, configuration.upload.commitUuid)
+      ResultsUploader(
+        codacyClientOpt,
+        configuration.upload.upload,
+        configuration.upload.commitUuid,
+        configuration.upload.batchSize)
 
     new AnalyseCommand(analyze, configuration, analyseExecutor, uploaderOpt)
   }

cli/src/main/scala/com/codacy/analysis/cli/command/CLIApp.scala

@@ -7,7 +7,8 @@ import com.codacy.analysis.cli.analysis.ExitStatus
 import com.codacy.analysis.cli.command.ArgumentParsers._
 import com.codacy.analysis.cli.formatter.Formatter
 import com.codacy.analysis.core.analysis.Analyser
-import com.codacy.analysis.core.clients.{ProjectName, UserName, OrganizationProvider}
+import com.codacy.analysis.core.clients.{OrganizationProvider, ProjectName, UserName}
+import com.codacy.analysis.core.configuration.AppConfiguration
 import com.codacy.analysis.core.git.Commit
 
 import scala.concurrent.duration.Duration
@@ -148,6 +149,8 @@ final case class Analyze(
   skipUncommittedFilesCheck: Int @@ Counter = Tag.of(0),
   @ExtraName("u") @ValueDescription("If the results should be uploaded to the API")
   upload: Int @@ Counter = Tag.of(0),
+  @ValueDescription(s"Batch size for upload of results. If not set defaults to ${AppConfiguration.batchSize}")
+  uploadBatchSize: Int = AppConfiguration.batchSize,
   @ExtraName("i") @ValueDescription(
     "[default: false] - Skip the SSL certificate verification when communicating with the Codacy API")
   skipSslVerification: Int @@ Counter = Tag.of(0),

cli/src/main/scala/com/codacy/analysis/cli/configuration/CLIConfiguration.scala

@@ -66,7 +66,7 @@ object CLIConfiguration {
     }
   }
 
-  final case class Upload(commitUuid: Option[Commit.Uuid], upload: Boolean)
+  final case class Upload(commitUuid: Option[Commit.Uuid], upload: Boolean, batchSize: Int)
   final case class Result(maxAllowedIssues: Int, failIfIncomplete: Boolean)
 
   final case class Output(format: String, file: Option[File], ghCodeScanningCompat: Boolean)
@@ -215,7 +215,7 @@ object CLIConfiguration {
         remoteProjectConfiguration,
         analyze.tmpDirectory,
         analyze.maxToolMemory)
-    val uploadConfiguration = Upload(commitUuid, analyze.uploadValue)
+    val uploadConfiguration = Upload(commitUuid, analyze.uploadValue, analyze.uploadBatchSize)
     val resultConfiguration = Result(analyze.maxAllowedIssues, analyze.failIfIncompleteValue)
 
     CLIConfiguration(analysisConfiguration, uploadConfiguration, resultConfiguration)

cli/src/test/scala/com/codacy/analysis/cli/configuration/CLIConfigurationSpec.scala

@@ -40,6 +40,7 @@ class CLIConfigurationSpec extends Specification with NoLanguageFeatures {
   }
   private val toolInput = Option("hey! i'm a tool!")
   private val commitUuid = Option(Commit.Uuid("uuid"))
+  private val batchSize = 5
 
   private val defaultAnalyse = Analyze(
     options = CommonOptions(),
@@ -49,7 +50,8 @@ class CLIConfigurationSpec extends Specification with NoLanguageFeatures {
     format = Json.name,
     toolTimeout = Option.empty,
     commitUuid = commitUuid,
-    extras = ExtraOptions(analyser = Analyser.defaultAnalyser.name))
+    extras = ExtraOptions(analyser = Analyser.defaultAnalyser.name),
+    uploadBatchSize = batchSize)
   private val defaultEnvironment = new Environment(Map.empty)
   private val httpHelper = new HttpHelper(remoteUrl, Map.empty, false)
 
@@ -85,7 +87,8 @@ class CLIConfigurationSpec extends Specification with NoLanguageFeatures {
           allowNetwork = Tag.of(1),
           upload = Tag.of(1),
           maxAllowedIssues = 5,
-          failIfIncomplete = Tag.of(1))
+          failIfIncomplete = Tag.of(1),
+          uploadBatchSize = batchSize)
 
         val expectedConfiguration = CLIConfiguration(
           analysis = CLIConfiguration.Analysis(
@@ -106,7 +109,7 @@ class CLIConfigurationSpec extends Specification with NoLanguageFeatures {
               extraToolConfigurations = Option.empty,
               extensionsByLanguage = Map.empty),
             maxToolMemory = Some("3000000000")),
-          upload = CLIConfiguration.Upload(commitUuid = commitUuid, upload = true),
+          upload = CLIConfiguration.Upload(commitUuid = commitUuid, upload = true, batchSize = batchSize),
           result = CLIConfiguration.Result(maxAllowedIssues = 5, failIfIncomplete = true))
 
         val actualConfiguration =
@@ -169,7 +172,7 @@ class CLIConfigurationSpec extends Specification with NoLanguageFeatures {
             extraToolConfigurations = Option.empty,
             extensionsByLanguage = Map.empty),
           maxToolMemory = Some("3000000000")),
-        upload = CLIConfiguration.Upload(commitUuid = commitUuid, upload = false),
+        upload = CLIConfiguration.Upload(commitUuid = commitUuid, upload = false, batchSize = batchSize),
         result = CLIConfiguration.Result(maxAllowedIssues = 0, failIfIncomplete = false))
 
       val actualConfiguration =
@@ -221,7 +224,7 @@ class CLIConfigurationSpec extends Specification with NoLanguageFeatures {
                 .Extra(baseSubDir = engineConfig.baseSubDir, extraValues = engineConfig.extraValues))),
             extensionsByLanguage = Map(Languages.Scala -> Set(".scala", ".alacs"))),
           maxToolMemory = Some("3000000000")),
-        upload = CLIConfiguration.Upload(commitUuid = commitUuid, upload = false),
+        upload = CLIConfiguration.Upload(commitUuid = commitUuid, upload = false, batchSize = batchSize),
         result = CLIConfiguration.Result(maxAllowedIssues = 0, failIfIncomplete = false))
 
       val actualConfiguration =
@@ -254,7 +257,7 @@ class CLIConfigurationSpec extends Specification with NoLanguageFeatures {
             extraToolConfigurations = Option.empty,
             extensionsByLanguage = Map.empty),
           maxToolMemory = Some("3000000000")),
-        upload = CLIConfiguration.Upload(commitUuid = commitUuid, upload = false),
+        upload = CLIConfiguration.Upload(commitUuid = commitUuid, upload = false, batchSize = batchSize),
         result = CLIConfiguration.Result(maxAllowedIssues = 0, failIfIncomplete = false))
 
       val actualConfiguration =
@@ -334,7 +337,7 @@ class CLIConfigurationSpec extends Specification with NoLanguageFeatures {
                 .Extra(baseSubDir = engineConfig.baseSubDir, extraValues = engineConfig.extraValues))),
             extensionsByLanguage = Map(Languages.Scala -> Set(".sc"))),
           maxToolMemory = Some("3000000000")),
-        upload = CLIConfiguration.Upload(commitUuid = commitUuid, upload = false),
+        upload = CLIConfiguration.Upload(commitUuid = commitUuid, upload = false, batchSize = batchSize),
         result = CLIConfiguration.Result(maxAllowedIssues = 0, failIfIncomplete = false))
 
       val actualConfiguration =

core/src/main/scala/com/codacy/analysis/core/upload/ResultsUploader.scala

@@ -23,11 +23,8 @@ object ResultsUploader {
 
   private val logger: Logger = getLogger
 
-  def apply(codacyClientOpt: Option[CodacyClient],
-            upload: Boolean,
-            commitUuidOpt: Option[Commit.Uuid],
-            batchSize: Option[Int] = Option.empty[Int])(implicit
-    context: ExecutionContext): Either[String, Option[ResultsUploader]] = {
+  def apply(codacyClientOpt: Option[CodacyClient], upload: Boolean, commitUuidOpt: Option[Commit.Uuid], batchSize: Int)(
+    implicit context: ExecutionContext): Either[String, Option[ResultsUploader]] = {
     if (upload) {
       for {
         codacyClient <- codacyClientOpt.toRight("No credentials found.")
@@ -40,17 +37,17 @@ object ResultsUploader {
   }
 }
 
-class ResultsUploader private (commitUuid: Commit.Uuid, codacyClient: CodacyClient, batchSizeOpt: Option[Int])(implicit
+class ResultsUploader private (commitUuid: Commit.Uuid, codacyClient: CodacyClient, uploadBatchSize: Int)(implicit
   context: ExecutionContext) {
 
   private val logger: Logger = getLogger
 
-  private val batchSize: Int = batchSizeOpt.map {
+  private val batchSize: Int = uploadBatchSize match {
     case size if size > 0 => size
     case size =>
       logger.warn(s"Illegal value for upload batch size ($size), using default batch size")
       ResultsUploader.defaultBatchSize
-  }.getOrElse(ResultsUploader.defaultBatchSize)
+  }
 
   def sendResults(toolResults: Seq[ResultsUploader.ToolResults],
                   metricsResults: Seq[MetricsResult],

core/src/test/scala/com.codacy.analysis.core/upload/ResultsUploaderSpec.scala

@@ -32,23 +32,24 @@ class ResultsUploaderSpec extends Specification with NoLanguageFeatures with Moc
   private val otherTool = "rubocop"
   private val otherFilenames = Set(Paths.get("src/things/Clazz.scala"))
   private val batchSize = 10
+  private val defaultBatchSize = 50000
 
   "ResultsUploader" should {
 
     "not create the uploader if upload is not requested" in {
       val codacyClient = mock[CodacyClient]
-      ResultsUploader(Option(codacyClient), upload = false, Option(commitUuid), Option(batchSize)) must beRight(
+      ResultsUploader(Option(codacyClient), upload = false, Option(commitUuid), batchSize) must beRight(
         Option.empty[ResultsUploader])
     }
 
     "fail to create the uploader if the client is not passed" in {
-      ResultsUploader(Option.empty[CodacyClient], upload = true, Option(commitUuid), Option(batchSize)) must beLeft(
+      ResultsUploader(Option.empty[CodacyClient], upload = true, Option(commitUuid), batchSize) must beLeft(
         "No credentials found.")
     }
 
     "fail to create the uploader if the commit is not passed" in {
       val codacyClient = mock[CodacyClient]
-      ResultsUploader(Option(codacyClient), upload = true, Option.empty[Commit.Uuid], Option(batchSize)) must beLeft(
+      ResultsUploader(Option(codacyClient), upload = true, Option.empty[Commit.Uuid], batchSize) must beLeft(
         "No commit found.")
     }
 
@@ -87,7 +88,7 @@ class ResultsUploaderSpec extends Specification with NoLanguageFeatures with Moc
         .thenReturn(Future.successful(().asRight[String]))
 
       val uploader: ResultsUploader =
-        ResultsUploader(Option(codacyClient), upload = true, Some(commitUuid), None).right.get.get
+        ResultsUploader(Option(codacyClient), upload = true, Some(commitUuid), defaultBatchSize).right.get.get
 
       def testFileMetrics(i: Int) = {
         MetricsAnalysis.FileResults(
@@ -145,7 +146,7 @@ class ResultsUploaderSpec extends Specification with NoLanguageFeatures with Moc
         .thenReturn(Future.successful(().asRight[String]))
 
       val uploader: ResultsUploader =
-        ResultsUploader(Option(codacyClient), upload = true, Some(commitUuid), None).right.get.get
+        ResultsUploader(Option(codacyClient), upload = true, Some(commitUuid), defaultBatchSize).right.get.get
 
       def testClone(i: Int) = {
         DuplicationClone("", i, i, Set.empty)
@@ -184,7 +185,7 @@ class ResultsUploaderSpec extends Specification with NoLanguageFeatures with Moc
       }
       val codacyClient = mock[CodacyClient]
       val uploader: ResultsUploader =
-        ResultsUploader(Option(codacyClient), upload = true, Option(commitUuid), Option(batchSize)).right.get.get
+        ResultsUploader(Option(codacyClient), upload = true, Option(commitUuid), batchSize).right.get.get
 
       when(
         codacyClient.sendRemoteIssues(