Added automatically added tags about affected ecosystems (#2853)

DrAlexD · web-flow · commit ebad7c8b3fd3 · 2023-11-13T16:30:38.000+03:00
### What's done:
- added logic for automatically added tags about affected ecosystems during vulnerabilities uploading.
- changed tag length range from [3, 15] to [2, 15].
diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/BackendForCosvService.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/BackendForCosvService.kt
@@ -5,6 +5,7 @@ import com.saveourtool.save.backend.security.OrganizationPermissionEvaluator
 import com.saveourtool.save.backend.security.UserPermissionEvaluator
 import com.saveourtool.save.entities.Organization
 import com.saveourtool.save.entities.User
+import com.saveourtool.save.entities.cosv.LnkVulnerabilityMetadataTag
 import com.saveourtool.save.permission.Permission
 import org.springframework.security.core.Authentication
 import org.springframework.stereotype.Service
@@ -19,6 +20,7 @@ class BackendForCosvService(
     private val userDetailsService: UserDetailsService,
     private val userPermissionEvaluator: UserPermissionEvaluator,
     private val organizationPermissionEvaluator: OrganizationPermissionEvaluator,
+    private val tagService: TagService,
     configProperties: ConfigProperties,
 ) : IBackendService {
     override val workingDir: Path = configProperties.workingDir
@@ -41,4 +43,9 @@ class BackendForCosvService(
     override fun saveUser(user: User): User = userDetailsService.saveUser(user)
 
     override fun saveOrganization(organization: Organization) = organizationService.updateOrganization(organization)
+
+    override fun addVulnerabilityTags(
+        identifier: String,
+        tagName: Set<String>
+    ): List<LnkVulnerabilityMetadataTag>? = tagService.addVulnerabilityTags(identifier, tagName)
 }
diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/TagService.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/TagService.kt
@@ -5,8 +5,8 @@ import com.saveourtool.save.cosv.repository.LnkVulnerabilityMetadataTagRepositor
 import com.saveourtool.save.cosv.repository.VulnerabilityMetadataRepository
 import com.saveourtool.save.entities.Tag
 import com.saveourtool.save.entities.cosv.LnkVulnerabilityMetadataTag
+import com.saveourtool.save.utils.error
 import com.saveourtool.save.utils.getLogger
-import com.saveourtool.save.utils.info
 import com.saveourtool.save.utils.orNotFound
 import com.saveourtool.save.validation.TAG_ERROR_MESSAGE
 import com.saveourtool.save.validation.isValidTag
@@ -37,8 +37,6 @@ class TagService(
      */
     @Transactional
     fun addVulnerabilityTag(identifier: String, tagName: String): LnkVulnerabilityMetadataTag {
-        log.info { "Trying to add $tagName to $identifier vulnerability" }
-
         if (!tagName.isValidTag()) {
             throw ResponseStatusException(HttpStatus.CONFLICT, TAG_ERROR_MESSAGE)
         }
@@ -52,6 +50,32 @@ class TagService(
         )
     }
 
+    /**
+     * @param identifier [com.saveourtool.save.entities.cosv.VulnerabilityMetadata.identifier]
+     * @param tagNames tags to add
+     * @return new [LnkVulnerabilityMetadataTag]
+     */
+    @Transactional
+    fun addVulnerabilityTags(identifier: String, tagNames: Set<String>): List<LnkVulnerabilityMetadataTag>? {
+        if (tagNames.any { !it.isValidTag() }) {
+            log.error { TAG_ERROR_MESSAGE }
+            return null
+        }
+
+        val metadata = vulnerabilityMetadataRepository.findByIdentifier(identifier) ?: run {
+            log.error { "Could not find metadata for vulnerability $identifier" }
+            return null
+        }
+
+        val links = tagNames.map {
+            tagRepository.findByName(it) ?: tagRepository.save(Tag(it))
+        }.map {
+            LnkVulnerabilityMetadataTag(metadata, it)
+        }
+
+        return lnkVulnerabilityMetadataTagRepository.saveAll(links)
+    }
+
     /**
      * @param identifier [com.saveourtool.save.entities.cosv.VulnerabilityMetadata.identifier]
      * @param tagName tag to delete
diff --git a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/validation/ValidationErrorMessages.kt b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/validation/ValidationErrorMessages.kt
@@ -43,7 +43,7 @@ const val CVE_NAME_ERROR_MESSAGE = "CVE identifier is invalid"
 /**
  * Error message that is shown when tag is invalid.
  */
-const val TAG_ERROR_MESSAGE = "Tag length should be in [3, 15] range, no commas are allowed."
+const val TAG_ERROR_MESSAGE = "Tag length should be in [2, 15] range, no commas are allowed."
 
 /**
  * Error message that is shown when severity score vector is invalid.
diff --git a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/validation/ValidationUtils.kt b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/validation/ValidationUtils.kt
@@ -13,7 +13,7 @@ const val NAMING_MAX_LENGTH = 22
 private val namingAllowedSpecialSymbols = setOf('-', '_', '.')
 
 @Suppress("MagicNumber")
-private val tagLengthRange = 3..15
+private val tagLengthRange = 2..15
 
 /**
  * Check if name is valid.
diff --git a/save-cosv/src/main/kotlin/com/saveourtool/save/backend/service/IBackendService.kt b/save-cosv/src/main/kotlin/com/saveourtool/save/backend/service/IBackendService.kt
@@ -4,6 +4,7 @@ package com.saveourtool.save.backend.service
 
 import com.saveourtool.save.entities.Organization
 import com.saveourtool.save.entities.User
+import com.saveourtool.save.entities.cosv.LnkVulnerabilityMetadataTag
 import com.saveourtool.save.info.UserPermissions
 import com.saveourtool.save.permission.Permission
 import org.jetbrains.annotations.Blocking
@@ -63,4 +64,14 @@ interface IBackendService {
         organizationName: String,
         permission: Permission,
     ): Boolean
+
+    /**
+     * @param identifier [com.saveourtool.save.entities.cosv.VulnerabilityMetadata.identifier]
+     * @param tagName tag to add
+     * @return new [LnkVulnerabilityMetadataTag]
+     */
+    fun addVulnerabilityTags(
+        identifier: String,
+        tagName: Set<String>
+    ): List<LnkVulnerabilityMetadataTag>?
 }
diff --git a/save-cosv/src/main/kotlin/com/saveourtool/save/cosv/controllers/RawCosvFileController.kt b/save-cosv/src/main/kotlin/com/saveourtool/save/cosv/controllers/RawCosvFileController.kt
@@ -278,7 +278,7 @@ class RawCosvFileController(
                 backendService.getOrganizationByName(organizationName) to backendService.getUserByName(authentication.name)
             }
                 .flatMap { (organization, user) ->
-                    cosvService.process(ids, user, organization)
+                    cosvService.processAndAddTagsAndUpdateRating(ids, user, organization)
                 }
                 .subscribeOn(Schedulers.boundedElastic())
                 .subscribe()
@@ -287,7 +287,7 @@ class RawCosvFileController(
     /**
      * @param organizationName
      * @param authentication
-     * @return statistics [RawCosvFileStatisticDto] with counts of all, uploaded, processing, failed raw cosv files in [organizationName]
+     * @return statistics [RawCosvFileStatisticsDto] with counts of all, uploaded, processing, failed raw cosv files in [organizationName]
      */
     @RequiresAuthorizationSourceHeader
     @GetMapping("/statistics")
diff --git a/save-cosv/src/main/kotlin/com/saveourtool/save/cosv/service/CosvService.kt b/save-cosv/src/main/kotlin/com/saveourtool/save/cosv/service/CosvService.kt
@@ -22,7 +22,6 @@ import reactor.core.publisher.Mono
 import reactor.core.scheduler.Schedulers
 import reactor.kotlin.core.publisher.switchIfEmpty
 import reactor.kotlin.core.publisher.toFlux
-import reactor.kotlin.extra.math.sumAll
 
 import java.nio.ByteBuffer
 import javax.annotation.PostConstruct
@@ -31,6 +30,8 @@ import kotlin.time.Duration.Companion.minutes
 import kotlin.time.Duration.Companion.seconds
 import kotlinx.serialization.serializer
 
+typealias VulnerabilityMetadataDtoList = List<VulnerabilityMetadataDto>
+
 /**
  * Service for vulnerabilities
  */
@@ -46,6 +47,8 @@ class CosvService(
     private val lnkVulnerabilityMetadataTagRepository: LnkVulnerabilityMetadataTagRepository,
     private val cosvGeneratedIdRepository: CosvGeneratedIdRepository,
 ) {
+    private val scheduler = Schedulers.boundedElastic()
+
     /**
      * Init method to restore all raw cosv files with `in progress` state to process
      */
@@ -71,7 +74,7 @@ class CosvService(
                     "Storage ${RawCosvFileStorage::class.simpleName} and repository ${CosvRepository::class.simpleName} are not initialized in $initMaxTime"
                 }
             }
-            .subscribeOn(Schedulers.boundedElastic())
+            .subscribeOn(scheduler)
             .subscribe()
     }
 
@@ -90,7 +93,11 @@ class CosvService(
                         .flatMap {
                             doProcess(it.requiredId(), user, organization)
                         }
-                        .updateRating(user, organization)
+                        .collectList()
+                        .map { it.flatten() }
+                        .blockingMap {
+                            vulnerabilityRatingService.addRatingForBulkUpload(user, organization, it.size)
+                        }
                 }
         }
         .thenJust(Unit)
@@ -102,12 +109,14 @@ class CosvService(
     fun generateIdentifier(): String = cosvGeneratedIdRepository.saveAndFlush(CosvGeneratedId()).getIdentifier()
 
     /**
+     * Method to process all raw cosv files, add ecosystem tags to new vulnerabilities, update user rating
+     *
      * @param rawCosvFileIds
      * @param user
      * @param organization
      * @return empty [Mono]
      */
-    fun process(
+    fun processAndAddTagsAndUpdateRating(
         rawCosvFileIds: Collection<Long>,
         user: User,
         organization: Organization,
@@ -118,7 +127,30 @@ class CosvService(
                     doProcess(rawCosvFileId, user, organization)
                 }
         }
-        .updateRating(user, organization)
+        .collectList()
+        .map { it.flatten() }
+        .flatMap { metadataList ->
+            Mono.just(metadataList.size).doOnSuccess {
+                metadataList.toFlux().flatMap { vulnerabilityMetadataDto ->
+                    getVulnerabilityExt(vulnerabilityMetadataDto.identifier).mapNotNull { vulnerabilityExt ->
+                        vulnerabilityExt.cosv.affected?.mapNotNull { it.`package`?.ecosystem }?.toSet()?.let {
+                            vulnerabilityExt.metadataDto.identifier to it
+                        }
+                    }
+                }
+                    .collectList()
+                    .blockingMap { identifierToTagsList ->
+                        identifierToTagsList.forEach { (identifier, tags) ->
+                            backendService.addVulnerabilityTags(identifier, tags)
+                        }
+                    }
+                    .subscribeOn(scheduler)
+                    .subscribe()
+            }
+        }
+        .blockingMap {
+            vulnerabilityRatingService.addRatingForBulkUpload(user, organization, it)
+        }
         .map {
             log.debug {
                 "Finished processing raw COSV files $rawCosvFileIds"
@@ -144,7 +176,7 @@ class CosvService(
         rawCosvFileId: Long,
         user: User,
         organization: Organization,
-    ): Mono<Int> = rawCosvFileStorage.downloadById(rawCosvFileId)
+    ): Mono<VulnerabilityMetadataDtoList> = rawCosvFileStorage.downloadById(rawCosvFileId)
         .collectToInputStream()
         .flatMap { inputStream ->
             val errorMessage by lazy {
@@ -170,22 +202,14 @@ class CosvService(
                         .flatMap {
                             rawCosvFileStorage.deleteById(rawCosvFileId)
                         }
-                        .thenReturn(metadataList.size)
+                        .thenReturn(metadataList)
                 }
                 .onErrorResume { error ->
                     log.error(error) { errorMessage }
-                    Mono.just(0)
+                    Mono.just(emptyList())
                 }
         }
 
-    private fun Flux<Int>.updateRating(
-        user: User,
-        organization: Organization,
-    ): Mono<Unit> = sumAll()
-        .blockingMap {
-            vulnerabilityRatingService.addRatingForBulkUpload(user, organization, it)
-        }
-
     /**
      * @param cosvId
      * @param updater
diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/vuln/VulnerabilityTagsComponent.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/vuln/VulnerabilityTagsComponent.kt
@@ -46,7 +46,7 @@ val vulnerabilityTagsComponent: FC<VulnerabilityTagsComponentProps> = FC { props
                 className = ClassName("form-control custom-input $validityClassName")
                 value = newTag
                 placeholder = "Add a new tag...".t()
-                title = "Tag should not have commas, length should be more than 2 and less than 16.".t()
+                title = "Tag should not have commas, length should be more than 1 and less than 16.".t()
                 asDynamic()["data-toggle"] = "tooltip"
                 asDynamic()["data-placement"] = "top"
                 onChange = { event -> setNewTag(event.target.value) }
