Support upload vulnerability in COSV -- backend only (#2401)

CONTRIBUTING.md

@@ -31,3 +31,15 @@ In order to run `save-orchestrator` on Mac with M1 in order to make it run execu
 1. manually put all the files from `save-agent-*-distribution.jar` into `save-orchestrator/build/resources/main` as well as `save-*-linuxX64.kexe` (temporary workaround) 
 2. run `docker-mac-settings.sh` script (from `save-deploy` folder) in order to let docker be available via TCP 
 Also check `save-deploy/README.md` for extra information
+
+## Generating OpenAPI specification
+
+We generate [OpenAPI specification](https://swagger.io/specification/) for backend's endpoints.
+The job [`backend-api-spec-update`](.github/workflows/backend-api-spec-update.yml) does it:
+it runs each week and creates a PR with changes.
+
+By default, it checks all endpoints from controllers in packages:
+  - `com.saveourtool.save.backend.controllers`
+  - `com.saveourtool.save.cosv.controllers`
+
+If you need to support a new package, need to update `com.saveourtool.save.backend.configs.ApiGroupsConfiguration`.

gradle/libs.versions.toml

@@ -43,6 +43,7 @@ cpg = "7.0.0"
 cpg-neo4j-ogm = { strictly = "4.0.4" }
 cpg-neo4j-java-driver = { strictly = "5.6.0" }
 aws-sdk = "2.20.84"
+cosv4k = "0.0.11"
 
 [plugins]
 kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" }
@@ -203,3 +204,6 @@ arrow-kt-core = { module = "io.arrow-kt:arrow-core", version.ref = "arrow-kt"}
 aws-sdk-bom = { module = "software.amazon.awssdk:bom", version.ref = "aws-sdk" }
 aws-sdk-s3 = { module = "software.amazon.awssdk:s3" }
 aws-sdk-netty-nio = { module = "software.amazon.awssdk:netty-nio-client" }
+
+# vulnerability
+cosv4k = { module = "com.saveourtool.cosv4k:cosv4k", version.ref = "cosv4k" }

save-backend/build.gradle.kts

@@ -48,6 +48,7 @@ dependencies {
     implementation(projects.saveCloudCommon)
     implementation(projects.authenticationService)
     implementation(projects.testAnalysisCore)
+    implementation(projects.saveCosv)
     implementation(libs.save.common.jvm)
     implementation(libs.spring.boot.starter.quartz)
     implementation(libs.spring.boot.starter.security)

save-backend/src/main/kotlin/com/saveourtool/save/backend/SaveApplication.kt

@@ -1,14 +1,17 @@
 package com.saveourtool.save.backend
 
 import com.saveourtool.save.backend.configs.ConfigProperties
+import com.saveourtool.save.cosv.CosvConfiguration
 import com.saveourtool.save.s3.DefaultS3Configuration
+
 import org.springframework.boot.SpringApplication
 import org.springframework.boot.autoconfigure.SpringBootApplication
 import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.context.annotation.Import
 import org.springframework.http.ResponseEntity
 import reactor.core.publisher.Flux
 import reactor.core.publisher.ParallelFlux
+
 import java.nio.ByteBuffer
 
 internal typealias FluxResponse<T> = ResponseEntity<Flux<T>>
@@ -20,7 +23,10 @@ internal typealias ByteBufferFluxResponse = FluxResponse<ByteBuffer>
  */
 @SpringBootApplication
 @EnableConfigurationProperties(ConfigProperties::class)
-@Import(DefaultS3Configuration::class)
+@Import(
+    DefaultS3Configuration::class,
+    CosvConfiguration::class,
+)
 class SaveApplication
 
 fun main(args: Array<String>) {

save-backend/src/main/kotlin/com/saveourtool/save/backend/utils/ApiGroupsConfiguration.kt → save-backend/src/main/kotlin/com/saveourtool/save/backend/configs/ApiGroupsConfiguration.kt

@@ -1,4 +1,4 @@
-package com.saveourtool.save.backend.utils
+package com.saveourtool.save.backend.configs
 
 import com.saveourtool.save.latestVersion
 import com.saveourtool.save.v1
@@ -21,7 +21,10 @@ class ApiGroupsConfiguration {
     fun openApiAll(): GroupedOpenApi? = GroupedOpenApi.builder()
         .group("all")
         .pathsToMatch("/api/**")
-        .packagesToScan("com.saveourtool.save.backend.controllers")
+        .packagesToScan(
+            "com.saveourtool.save.backend.controllers",
+            "com.saveourtool.save.cosv.controllers"
+        )
         .build()
 
     @Bean
@@ -43,6 +46,9 @@ class ApiGroupsConfiguration {
         .group(groupName)
         .pathsToMatch("/api/$version/**")
         .pathsToExclude("?!(/api/$version).+")
-        .packagesToScan("com.saveourtool.save.backend.controllers")
+        .packagesToScan(
+            "com.saveourtool.save.backend.controllers",
+            "com.saveourtool.save.cosv.controllers"
+        )
         .build()
 }

save-backend/src/main/kotlin/com/saveourtool/save/backend/controllers/ContainerLogController.kt

@@ -3,6 +3,7 @@ package com.saveourtool.save.backend.controllers
 import com.saveourtool.save.backend.service.AgentService
 import com.saveourtool.save.configs.ApiSwaggerSupport
 import com.saveourtool.save.service.LogService
+import com.saveourtool.save.utils.StringListResponse
 import com.saveourtool.save.utils.blockingToMono
 import com.saveourtool.save.utils.toInstantAtDefaultZone
 import com.saveourtool.save.v1

save-backend/src/main/kotlin/com/saveourtool/save/backend/controllers/TestSuitesSourceController.kt

@@ -32,7 +32,6 @@ import reactor.kotlin.core.util.function.component1
 import reactor.kotlin.core.util.function.component2
 
 typealias EntitySaveStatusResponse = ResponseEntity<EntitySaveStatus>
-typealias StringListResponse = ResponseEntity<List<String>>
 
 /**
  * Controller for [TestSuitesSource]

save-backend/src/main/kotlin/com/saveourtool/save/backend/service/vulnerability/VulnerabilityService.kt

@@ -5,6 +5,7 @@ import com.saveourtool.save.backend.repository.OrganizationRepository
 import com.saveourtool.save.backend.repository.TagRepository
 import com.saveourtool.save.backend.repository.UserRepository
 import com.saveourtool.save.backend.repository.vulnerability.*
+import com.saveourtool.save.backend.service.IVulnerabilityService
 import com.saveourtool.save.backend.utils.hasRole
 import com.saveourtool.save.domain.Role
 import com.saveourtool.save.entities.Organization
@@ -44,7 +45,7 @@ class VulnerabilityService(
     private val organizationRepository: OrganizationRepository,
     private val lnkVulnerabilityTagRepository: LnkVulnerabilityTagRepository,
     private val tagRepository: TagRepository,
-) {
+) : IVulnerabilityService {
     @Suppress("TYPE_ALIAS")
     private val getTagMap: List<Vulnerability>.() -> Map<Vulnerability, Set<Tag>> = {
         lnkVulnerabilityTagRepository.findByVulnerabilityIdIn(this.map { it.requiredId() })
@@ -77,7 +78,7 @@ class VulnerabilityService(
      * @param name name of vulnerability
      * @return vulnerability by name
      */
-    fun findByName(name: String) = vulnerabilityRepository.findByName(name)
+    override fun findByName(name: String) = vulnerabilityRepository.findByName(name)
 
     /**
      * @param userName creator of vulnerability
@@ -266,13 +267,14 @@ class VulnerabilityService(
     /**
      * @param vulnerabilityDto dto of new vulnerability
      * @param authentication auth info of a current user
+     * @return saved [Vulnerability]
      */
     @Suppress("TOO_LONG_FUNCTION")
     @Transactional
-    fun save(
+    override fun save(
         vulnerabilityDto: VulnerabilityDto,
         authentication: Authentication,
-    ) {
+    ): Vulnerability {
         val userId = authentication.userId()
         val user = userRepository.getByIdOrNotFound(userId)
         val organizationNew = vulnerabilityDto.organization?.name?.let { organizationRepository.findByName(it) }
@@ -290,7 +292,7 @@ class VulnerabilityService(
             organization = organizationNew,
         )
         val vulnerabilityNew = vulnerabilityRepository.saveAndFlush(vulnerability)
-        val newName = "SOTV-${LocalDateTime.now().year}-${vulnerabilityNew.id}"
+        val newName = vulnerabilityDto.name.takeIf { it.isNotBlank() } ?: "SOTV-${LocalDateTime.now().year}-${vulnerabilityNew.id}"
 
         val existedTags = tagRepository.findByNameIn(vulnerabilityDto.tags)
         val existedTagNames = existedTags.map { tag -> tag.name }
@@ -300,27 +302,12 @@ class VulnerabilityService(
         vulnerabilityRepository.save(
             vulnerabilityNew.apply {
                 name = newName
-                projects = vulnerabilityDto.projects.map { dto ->
-                    VulnerabilityProject(
-                        name = dto.name,
-                        url = dto.url,
-                        versions = dto.versions,
-                        type = dto.type,
-                        vulnerability = vulnerabilityNew,
-                    )
-                }
+                projects = vulnerabilityDto.projects.map { dto -> dto.toEntity(vulnerabilityNew) }
             }
         )
 
         vulnerabilityDateRepository.saveAll(
-            vulnerabilityDto.dates.map { dto ->
-                VulnerabilityDate(
-                    date = dto.date.toJavaLocalDateTime(),
-                    type = dto.type,
-                    vulnerability = vulnerabilityNew,
-                    userId = user.requiredId(),
-                )
-            }
+            vulnerabilityDto.dates.map { dto -> dto.toEntity(vulnerabilityNew, user) }
         )
 
         lnkVulnerabilityUserRepository.saveAll(
@@ -337,6 +324,8 @@ class VulnerabilityService(
 
         val vulnTagLinks = existedTags.plus(newTags).map { LnkVulnerabilityTag(vulnerability, it) }
         lnkVulnerabilityTagRepository.saveAll(vulnTagLinks)
+
+        return vulnerabilityNew
     }
 
     /**
@@ -364,7 +353,7 @@ class VulnerabilityService(
         if (!authentication.hasRole(Role.SUPER_ADMIN) && (userId != vulnerability.userId || vulnerability.status == VulnerabilityStatus.APPROVED)) {
             throw ResponseStatusException(HttpStatus.FORBIDDEN)
         }
-        
+
         val vulnerabilityUpdate = vulnerability.apply {
             progress = vulnerabilityDto.progress
             description = vulnerabilityDto.description.orEmpty()
@@ -561,5 +550,25 @@ class VulnerabilityService(
     companion object {
         private const val VULNERABILITY_ORGANIZATION_RATING = 10
         private const val VULNERABILITY_OWNER_RATING = 10
+
+        private fun VulnerabilityProjectDto.toEntity(
+            vulnerability: Vulnerability,
+        ) = VulnerabilityProject(
+            name = name,
+            url = url,
+            versions = versions,
+            type = type,
+            vulnerability = vulnerability,
+        )
+
+        private fun VulnerabilityDateDto.toEntity(
+            vulnerability: Vulnerability,
+            user: User,
+        ) = VulnerabilityDate(
+            date = date.toJavaLocalDateTime(),
+            type = type,
+            vulnerability = vulnerability,
+            userId = user.requiredId(),
+        )
     }
 }

save-backend/src/test/kotlin/com/saveourtool/save/backend/DatabaseTest.kt

@@ -5,6 +5,7 @@ import com.saveourtool.save.backend.configs.ApplicationConfiguration
 import com.saveourtool.save.backend.repository.AgentStatusRepository
 import com.saveourtool.save.backend.repository.ProjectRepository
 import com.saveourtool.save.backend.repository.TestExecutionRepository
+import com.saveourtool.save.backend.service.IVulnerabilityService
 import com.saveourtool.save.backend.utils.InfraExtension
 import com.saveourtool.save.domain.TestResultStatus
 import org.junit.jupiter.api.Assertions.assertTrue
@@ -13,12 +14,17 @@ import org.junit.jupiter.api.extension.ExtendWith
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase
 import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest
+import org.springframework.boot.test.mock.mockito.MockBean
+import org.springframework.boot.test.mock.mockito.MockBeans
 import org.springframework.context.annotation.Import
 
 @Import(ApplicationConfiguration::class)
 @DataJpaTest
 @AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE)
 @ExtendWith(InfraExtension::class)
+@MockBeans(
+    MockBean(IVulnerabilityService::class),
+)
 class DatabaseTest {
     @Autowired
     private lateinit var projectRepository: ProjectRepository

save-backend/src/test/kotlin/com/saveourtool/save/backend/DownloadFilesTest.kt

@@ -70,6 +70,7 @@ import kotlin.io.path.*
     MockBean(ProjectPermissionEvaluator::class),
     MockBean(DebugInfoStorage::class),
     MockBean(ExecutionInfoStorage::class),
+    MockBean(IVulnerabilityService::class),
 )
 class DownloadFilesTest {
     private val organization = Organization.stub(2).apply {

save-backend/src/test/kotlin/com/saveourtool/save/backend/controller/JpaSpecificationTest.kt

@@ -3,19 +3,25 @@ package com.saveourtool.save.backend.controller
 import com.saveourtool.save.agent.AgentState
 import com.saveourtool.save.backend.configs.ApplicationConfiguration
 import com.saveourtool.save.backend.repository.AgentStatusRepository
+import com.saveourtool.save.backend.service.IVulnerabilityService
 import com.saveourtool.save.backend.utils.InfraExtension
 import org.junit.jupiter.api.Assertions
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.extension.ExtendWith
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase
 import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest
+import org.springframework.boot.test.mock.mockito.MockBean
+import org.springframework.boot.test.mock.mockito.MockBeans
 import org.springframework.context.annotation.Import
 
 @Import(ApplicationConfiguration::class)
 @DataJpaTest
 @AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE)
 @ExtendWith(InfraExtension::class)
+@MockBeans(
+    MockBean(IVulnerabilityService::class),
+)
 class JpaSpecificationTest {
     @Autowired
     lateinit var agentStatusRepository: AgentStatusRepository

save-backend/src/test/kotlin/com/saveourtool/save/backend/controller/LnkUserOrganizationControllerTest.kt

@@ -33,6 +33,7 @@ import org.springframework.test.web.reactive.server.WebTestClient
 @MockBeans(
     MockBean(OriginalLoginRepository::class),
     MockBean(NamedParameterJdbcTemplate::class),
+    MockBean(IVulnerabilityService::class),
 )
 @AutoConfigureWebTestClient
 class LnkUserOrganizationControllerTest {

save-backend/src/test/kotlin/com/saveourtool/save/backend/controller/OrganizationControllerTest.kt

@@ -86,6 +86,7 @@ import java.util.concurrent.TimeUnit
     MockBean(LnkOrganizationTestSuiteService::class),
     MockBean(LnkExecutionTestSuiteService::class),
     MockBean(AvatarStorage::class),
+    MockBean(IVulnerabilityService::class),
 )
 @AutoConfigureWebTestClient
 @Suppress("UnsafeCallOnNullableType")

save-backend/src/test/kotlin/com/saveourtool/save/backend/controller/PermissionControllerTest.kt

@@ -42,6 +42,7 @@ import reactor.util.function.Tuples
     MockBean(LnkUserOrganizationService::class),
     MockBean(OriginalLoginRepository::class),
     MockBean(NamedParameterJdbcTemplate::class),
+    MockBean(IVulnerabilityService::class),
 )
 @AutoConfigureWebTestClient
 class PermissionControllerTest {

save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/storage/AbstractMigrationReactiveStorage.kt

@@ -85,8 +85,11 @@ abstract class AbstractMigrationReactiveStorage<O : Any, N : Any>(
      */
     protected abstract fun N.toOldKey(): O
 
+    @Suppress("WRONG_OVERLOADING_FUNCTION_ARGUMENTS")
     override fun list(): Flux<O> = initializer.validateAndRun { newStorageProjectReactor.list().map { it.toOldKey() } }
 
+    override fun list(prefix: String): Flux<O> = initializer.validateAndRun { newStorageProjectReactor.list(prefix).map { it.toOldKey() } }
+
     override fun download(key: O): Flux<ByteBuffer> = initializer.validateAndRun { newStorageProjectReactor.download(key.toNewKey()) }
 
     override fun upload(key: O, content: Flux<ByteBuffer>): Mono<O> = initializer.validateAndRun { newStorageProjectReactor.upload(key.toNewKey(), content).map { it.toOldKey() } }

save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/storage/AbstractReactiveStorage.kt

@@ -44,8 +44,11 @@ abstract class AbstractReactiveStorage<K : Any>(
      */
     protected open fun doInit(underlying: DefaultStorageProjectReactor<K>): Mono<Unit> = Mono.empty()
 
+    @Suppress("WRONG_OVERLOADING_FUNCTION_ARGUMENTS")
     override fun list(): Flux<K> = initializer.validateAndRun { storageProjectReactor.list() }
 
+    override fun list(prefix: String): Flux<K> = initializer.validateAndRun { storageProjectReactor.list(prefix) }
+
     override fun download(key: K): Flux<ByteBuffer> = initializer.validateAndRun { storageProjectReactor.download(key) }
 
     override fun upload(key: K, content: Flux<ByteBuffer>): Mono<K> = initializer.validateAndRun { storageProjectReactor.upload(key, content) }

save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/storage/DefaultStorageProjectReactor.kt

@@ -32,11 +32,16 @@ class DefaultStorageProjectReactor<K : Any>(
 ) : StorageProjectReactor<K> {
     private val log: Logger = getLogger(this::class)
 
-    override fun list(): Flux<K> = s3Operations.listObjectsV2(s3KeyManager.commonPrefix)
+    @Suppress("WRONG_OVERLOADING_FUNCTION_ARGUMENTS")
+    override fun list(): Flux<K> = doList(s3KeyManager.commonPrefix)
+
+    override fun list(prefix: String): Flux<K> = doList(s3KeyManager.commonPrefix + prefix.removePrefix(PATH_DELIMITER))
+
+    private fun doList(prefix: String): Flux<K> = s3Operations.listObjectsV2(prefix)
         .toMonoAndPublishOn()
         .expand { lastResponse ->
             if (lastResponse.isTruncated) {
-                s3Operations.listObjectsV2(s3KeyManager.commonPrefix, lastResponse.nextContinuationToken())
+                s3Operations.listObjectsV2(prefix, lastResponse.nextContinuationToken())
                     .toMonoAndPublishOn()
             } else {
                 Mono.empty()

save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/storage/StorageProjectReactor.kt

@@ -14,8 +14,15 @@ interface StorageProjectReactor<K> {
     /**
      * @return list of keys in storage
      */
+    @Suppress("WRONG_OVERLOADING_FUNCTION_ARGUMENTS")
     fun list(): Flux<K>
 
+    /**
+     * @param prefix a common prefix for all keys
+     * @return list of keys in storage
+     */
+    fun list(prefix: String): Flux<K>
+
     /**
      * @param key a key to be checked
      * @return true if the key exists, otherwise false

save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/utils/KotlinUtilsJvm.kt

@@ -9,5 +9,6 @@ import java.util.concurrent.CompletableFuture
 
 typealias EmptyResponse = ResponseEntity<Void>
 typealias StringResponse = ResponseEntity<String>
+typealias StringListResponse = ResponseEntity<List<String>>
 
 typealias ListCompletableFuture<T> = CompletableFuture<List<T>>