Supported pagination for raw cosv files (#2704)

- supported listing of raw cosv files by pages
- added a button to show more
- removed immediately listing during unzip and upload

save-backend/src/main/kotlin/com/saveourtool/save/backend/service/vulnerability/VulnerabilityService.kt

@@ -122,9 +122,9 @@ class VulnerabilityService(
         page: Int,
         size: Int,
     ): List<VulnerabilityMetadataDto> {
-        val metadataList = vulnerabilityMetadataRepository.findAll({ root, cq, cb -> getFilterPredicate(root, cq, cb, filter, authentication) },
-            PageRequest.of(page, size)
-        )
+        val metadataList = vulnerabilityMetadataRepository.kFindAll(PageRequest.of(page, size)) { root, cq, cb ->
+            getFilterPredicate(root, cq, cb, filter, authentication)
+        }
 
         val tagMap = metadataList.content.toTagMap()
         return metadataList.content.map { metadata -> metadata to tagMap[metadata].orEmpty() }

save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/utils/SpringUtils.kt

@@ -8,6 +8,10 @@ import com.saveourtool.save.spring.entity.BaseEntity
 import com.saveourtool.save.spring.repository.BaseEntityRepository
 import com.saveourtool.save.storage.StorageProjectReactor
 import io.ktor.http.*
+import org.springframework.data.domain.Page
+import org.springframework.data.domain.Pageable
+import org.springframework.data.jpa.domain.Specification
+import org.springframework.data.jpa.repository.JpaSpecificationExecutor
 import org.springframework.data.repository.findByIdOrNull
 import org.springframework.http.ResponseEntity
 import org.springframework.http.codec.multipart.Part
@@ -70,6 +74,15 @@ inline fun <reified T : BaseEntity, R : BaseEntityRepository<T>> R.getByIdOrNotF
     "Not found ${T::class.simpleName} by id = $id"
 }
 
+/**
+ * Returns a [Page] of entities matching the given [Specification].
+ *
+ * @param spec can be null.
+ * @param pageable must not be null.
+ * @return never null.
+ */
+fun <T : Any> JpaSpecificationExecutor<T>.kFindAll(pageable: Pageable, spec: Specification<T>?): Page<T> = findAll(spec, pageable)
+
 private fun <K : Any> StorageProjectReactor<K>.doUpload(key: K, contentBytes: ByteArray) = contentBytes.size.toLong()
     .let { contentLength ->
         upload(key, contentLength, Flux.just(ByteBuffer.wrap(contentBytes)))

save-cosv/src/main/kotlin/com/saveourtool/save/cosv/controllers/CosvController.kt

@@ -15,6 +15,7 @@ import com.saveourtool.save.utils.*
 import com.saveourtool.save.v1
 
 import org.reactivestreams.Publisher
+import org.springframework.data.domain.PageRequest
 import org.springframework.http.*
 import org.springframework.http.codec.multipart.FilePart
 import org.springframework.security.core.Authentication
@@ -268,16 +269,44 @@ class CosvController(
     /**
      * @param organizationName
      * @param authentication
+     * @return count of uploaded raw cosv files in [organizationName]
+     */
+    @RequiresAuthorizationSourceHeader
+    @GetMapping("/{organizationName}/count")
+    fun count(
+        @PathVariable organizationName: String,
+        authentication: Authentication,
+    ): Mono<Long> = hasPermission(authentication, organizationName, Permission.READ, "read")
+        .flatMap {
+            rawCosvFileStorage.countByOrganization(organizationName)
+        }
+
+    /**
+     * @param organizationName
+     * @param authentication
+     * @param page
+     * @param size
      * @return list of uploaded raw cosv files in [organizationName]
      */
     @RequiresAuthorizationSourceHeader
-    @GetMapping("/{organizationName}/list")
+    @GetMapping(
+        "/{organizationName}/list",
+        consumes = [MediaType.APPLICATION_JSON_VALUE],
+        produces = [MediaType.APPLICATION_NDJSON_VALUE],
+    )
     fun list(
         @PathVariable organizationName: String,
+        @RequestParam page: Int,
+        @RequestParam size: Int,
         authentication: Authentication,
-    ): Flux<RawCosvFileDto> = hasPermission(authentication, organizationName, Permission.READ, "read")
+    ): ResponseEntity<RawCosvFileDtoFlux> = hasPermission(authentication, organizationName, Permission.READ, "read")
         .flatMapMany {
-            rawCosvFileStorage.listByOrganization(organizationName)
+            rawCosvFileStorage.listByOrganization(organizationName, PageRequest.of(page, size))
+        }
+        .let {
+            ResponseEntity.ok()
+                .cacheControlForNdjson()
+                .body(it)
         }
 
     /**

save-cosv/src/main/kotlin/com/saveourtool/save/cosv/repository/RawCosvFileRepository.kt

@@ -2,6 +2,7 @@ package com.saveourtool.save.cosv.repository
 
 import com.saveourtool.save.entities.cosv.RawCosvFile
 import com.saveourtool.save.spring.repository.BaseEntityRepository
+import org.springframework.data.domain.PageRequest
 import org.springframework.stereotype.Repository
 
 /**
@@ -17,9 +18,22 @@ interface RawCosvFileRepository : BaseEntityRepository<RawCosvFile> {
      */
     fun findByOrganizationNameAndUserNameAndFileName(organizationName: String, userName: String, fileName: String): RawCosvFile?
 
+    /**
+     * @param organizationName name from [RawCosvFile.organization]
+     * @return count of all [RawCosvFile]s which has provided [RawCosvFile.organization]
+     */
+    fun countAllByOrganizationName(organizationName: String): Long
+
     /**
      * @param organizationName name from [RawCosvFile.organization]
      * @return all [RawCosvFile]s which has provided [RawCosvFile.organization]
      */
     fun findAllByOrganizationName(organizationName: String): Collection<RawCosvFile>
+
+    /**
+     * @param organizationName name from [RawCosvFile.organization]
+     * @param pageRequest
+     * @return all [RawCosvFile]s which has provided [RawCosvFile.organization]
+     */
+    fun findAllByOrganizationName(organizationName: String, pageRequest: PageRequest): Collection<RawCosvFile>
 }

save-cosv/src/main/kotlin/com/saveourtool/save/cosv/storage/RawCosvFileS3KeyManager.kt

@@ -13,6 +13,7 @@ import com.saveourtool.save.storage.concatS3Key
 import com.saveourtool.save.storage.key.AbstractS3KeyDtoManager
 import com.saveourtool.save.utils.BlockingBridge
 import com.saveourtool.save.utils.getByIdOrNotFound
+import org.springframework.data.domain.PageRequest
 import org.springframework.stereotype.Component
 import org.springframework.transaction.annotation.Transactional
 
@@ -41,11 +42,23 @@ class RawCosvFileS3KeyManager(
 
     /**
      * @param organizationName
+     * @return count of all [RawCosvFileDto]s which has provided [RawCosvFileDto.organizationName]
+     */
+    fun countByOrganization(
+        organizationName: String,
+    ): Long = repository.countAllByOrganizationName(organizationName)
+
+    /**
+     * @param organizationName
+     * @param pageRequest
      * @return all [RawCosvFileDto]s which has provided [RawCosvFileDto.organizationName]
      */
     fun listByOrganization(
         organizationName: String,
-    ): Collection<RawCosvFileDto> = repository.findAllByOrganizationName(organizationName).map { it.toDto() }
+        pageRequest: PageRequest? = null,
+    ): Collection<RawCosvFileDto> = run {
+        pageRequest?.let { repository.findAllByOrganizationName(organizationName, it) } ?: repository.findAllByOrganizationName(organizationName)
+    }.map { it.toDto() }
 
     /**
      * @param ids

save-cosv/src/main/kotlin/com/saveourtool/save/cosv/storage/RawCosvFileStorage.kt

@@ -12,6 +12,7 @@ import com.saveourtool.save.storage.deleteUnexpectedKeys
 import com.saveourtool.save.utils.blockingToFlux
 import com.saveourtool.save.utils.blockingToMono
 import com.saveourtool.save.utils.switchIfEmptyToNotFound
+import org.springframework.data.domain.PageRequest
 import org.springframework.stereotype.Component
 import reactor.core.publisher.Flux
 import reactor.core.publisher.Mono
@@ -42,12 +43,24 @@ class RawCosvFileStorage(
 
     /**
      * @param organizationName
-     * @return all [RawCosvFileDto]s which has provided [RawCosvFile.organization]
+     * @return count of all [RawCosvFileDto]s which fits to [filter]
+     */
+    fun countByOrganization(
+        organizationName: String,
+    ): Mono<Long> = blockingToMono {
+        s3KeyManager.countByOrganization(organizationName)
+    }
+
+    /**
+     * @param organizationName
+     * @param pageRequest
+     * @return all [RawCosvFileDto]s which fits to [filter]
      */
     fun listByOrganization(
         organizationName: String,
+        pageRequest: PageRequest? = null,
     ): Flux<RawCosvFileDto> = blockingToFlux {
-        s3KeyManager.listByOrganization(organizationName)
+        s3KeyManager.listByOrganization(organizationName, pageRequest)
     }
 
     /**

save-demo/src/main/kotlin/com/saveourtool/save/demo/service/DemoService.kt

@@ -12,6 +12,7 @@ import com.saveourtool.save.demo.storage.DependencyStorage
 import com.saveourtool.save.filters.DemoFilter
 import com.saveourtool.save.utils.StringResponse
 import com.saveourtool.save.utils.blockingToMono
+import com.saveourtool.save.utils.kFindAll
 import com.saveourtool.save.utils.switchIfEmptyToNotFound
 
 import org.springframework.data.domain.PageRequest
@@ -94,7 +95,7 @@ class DemoService(
      * @param pageSize amount of [Demo]s that should be fetched
      * @return list of [Demo]s that match [DemoFilter]
      */
-    fun getFiltered(demoFilter: DemoFilter, pageSize: Int): List<Demo> = demoRepository.findAll({ root, _, cb ->
+    fun getFiltered(demoFilter: DemoFilter, pageSize: Int): List<Demo> = demoRepository.kFindAll(PageRequest.ofSize(pageSize)) { root, _, cb ->
         with(demoFilter) {
             val organizationNamePredicate = if (organizationName.isBlank()) {
                 cb.and()
@@ -112,7 +113,7 @@ class DemoService(
                 projectNamePredicate,
             )
         }
-    }, PageRequest.ofSize(pageSize))
+    }
         .filter { demoFilter.statuses.isEmpty() || getStatus(it).block() in demoFilter.statuses }
         .toList()
 

save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/basic/fileuploader/CosvFileManagerComponent.kt

@@ -44,17 +44,23 @@ import kotlinx.coroutines.flow.filter
 import kotlinx.coroutines.flow.onCompletion
 import kotlinx.serialization.json.Json
 
+private const val DEFAULT_SIZE = 10
+
 val cosvFileManagerComponent: FC<Props> = FC { _ ->
     useTooltip()
     val (t) = useTranslation("vulnerability-upload")
 
     @Suppress("GENERIC_VARIABLE_WRONG_DECLARATION")
     val organizationSelectForm = selectFormRequired<String>()
 
+    val (allAvailableFilesCount, setAllAvailableFilesCount) = useState(0L)
+    val (lastPage, setLastPage) = useState(0)
     val (availableFiles, setAvailableFiles) = useState<List<RawCosvFileDto>>(emptyList())
     val (selectedFiles, setSelectedFiles) = useState<List<RawCosvFileDto>>(emptyList())
     val (filesForUploading, setFilesForUploading) = useState<List<File>>(emptyList())
 
+    val leftAvailableFilesCount = allAvailableFilesCount - lastPage * DEFAULT_SIZE
+
     val (userOrganizations, setUserOrganizations) = useState(emptyList<OrganizationDto>())
     val (selectedOrganization, setSelectedOrganization) = useState<String>()
 
@@ -112,15 +118,50 @@ val cosvFileManagerComponent: FC<Props> = FC { _ ->
         setUserOrganizations(organizations)
     }
 
-    val fetchFiles = useDeferredRequest {
+    val fetchMoreFiles = useDeferredRequest {
         selectedOrganization?.let {
-            val result: List<RawCosvFileDto> = get(
+            val newPage = lastPage.inc()
+            val response = get(
                 url = "$apiUrl/cosv/$selectedOrganization/list",
+                params = jso<dynamic> {
+                    page = newPage - 1
+                    size = DEFAULT_SIZE
+                },
+                headers = Headers().withAcceptNdjson().withContentTypeJson(),
+                loadingHandler = ::loadingHandler,
+                responseHandler = ::noopResponseHandler
+            )
+            when {
+                response.ok -> {
+                    setStreamingOperationActive(true)
+                    response
+                        .readLines()
+                        .filter(String::isNotEmpty)
+                        .onCompletion {
+                            setStreamingOperationActive(false)
+                            setLastPage(newPage)
+                        }
+                        .collect { message ->
+                            val uploadedFile: RawCosvFileDto = Json.decodeFromString(message)
+                            setAvailableFiles { it.plus(uploadedFile) }
+                        }
+                }
+                else -> window.alert("Failed to fetch next page: ${response.unpackMessageOrNull().orEmpty()}")
+            }
+        }
+    }
+    val reFetchFiles = useDeferredRequest {
+        selectedOrganization?.let {
+            val count: Long = get(
+                url = "$apiUrl/cosv/$selectedOrganization/count",
                 jsonHeaders,
                 loadingHandler = ::loadingHandler,
                 responseHandler = ::noopResponseHandler
             ).decodeFromJsonString()
-            setAvailableFiles(result)
+            setAvailableFiles(emptyList())
+            setAllAvailableFilesCount(count)
+            setLastPage(0)
+            fetchMoreFiles()
         }
     }
 
@@ -139,11 +180,11 @@ val cosvFileManagerComponent: FC<Props> = FC { _ ->
                 .filter(String::isNotEmpty)
                 .onCompletion {
                     setStreamingOperationActive(false)
+                    reFetchFiles()
                 }
                 .collect { message ->
                     val uploadedFile: RawCosvFileDto = Json.decodeFromString(message)
                     setProcessedBytes { it.plus(uploadedFile.requiredContentLength()) }
-                    setAvailableFiles { it.plus(uploadedFile) }
                 }
             else -> window.alert(response.unpackMessageOrNull().orEmpty())
         }
@@ -173,11 +214,6 @@ val cosvFileManagerComponent: FC<Props> = FC { _ ->
                         }
                         .collect { message ->
                             val entryResponse: UnzipRawCosvFileResponse = Json.decodeFromString(message)
-                            entryResponse.result?.let { result ->
-                                setAvailableFiles {
-                                    it.plus(result)
-                                }
-                            }
                             if (entryResponse.updateCounters) {
                                 setTotalBytes(entryResponse.fullSize)
                                 setProcessedBytes(entryResponse.processedSize)
@@ -186,10 +222,11 @@ val cosvFileManagerComponent: FC<Props> = FC { _ ->
                             }
                         }
                 }
-                else -> window.alert(response.unpackMessageOrNull().orEmpty())
+                else -> window.alert("Failed to unzip ${file.fileName}: ${response.unpackMessageOrNull().orEmpty()}")
             }
             if (response.ok) {
                 setFileToUnzip(null)
+                reFetchFiles()
             }
         }
     }
@@ -244,7 +281,7 @@ val cosvFileManagerComponent: FC<Props> = FC { _ ->
             disabled = false
             onChangeFun = { value ->
                 setSelectedOrganization(value)
-                fetchFiles()
+                reFetchFiles()
             }
         }
 
@@ -264,7 +301,7 @@ val cosvFileManagerComponent: FC<Props> = FC { _ ->
                     submitAllUploadedCosvFiles()
                 }
                 buttonBuilder(faReload) {
-                    fetchFiles()
+                    reFetchFiles()
                 }
             }
 
@@ -360,6 +397,15 @@ val cosvFileManagerComponent: FC<Props> = FC { _ ->
                     }
                 }
             }
+
+            if (leftAvailableFilesCount > 0) {
+                li {
+                    className = ClassName("list-group-item p-0 d-flex bg-light justify-content-center")
+                    buttonBuilder("Load more (left $leftAvailableFilesCount)", isDisabled = isStreamingOperationActive) {
+                        fetchMoreFiles()
+                    }
+                }
+            }
         }
     }
 }