Removed old storages (#1804)

save-backend/build.gradle.kts

@@ -24,7 +24,6 @@ openApi {
     waitTimeInSeconds.set(120)
 
     customBootRun {
-        jvmArgs.add("-Dbackend.fileStorage.location=\${user.home}/cnb/files")
         jvmArgs.add("-Dbackend.test-analysis-settings.replay-on-startup=false")
         args.add("--debug")
     }

save-backend/src/main/kotlin/com/saveourtool/save/backend/configs/ConfigProperties.kt

@@ -18,13 +18,11 @@ import kotlin.time.toJavaDuration
  * @property orchestratorUrl url of save-orchestrator
  * @property demoUrl url of save-demo
  * @property initialBatchSize initial size of tests batch (for further scaling)
- * @property fileStorage configuration of file storage
- * @property fileStorage configuration of S3 storage
+ * @property s3Storage configuration of S3 storage
  * @property scheduling configuration for scheduled tasks
  * @property agentSettings properties for save-agents
  * @property testAnalysisSettings properties of the flaky test detector.
  * @property loki config of loki service for logging
- * @property s3Storage
  */
 @ConstructorBinding
 @ConfigurationProperties(prefix = "backend")
@@ -33,20 +31,12 @@ data class ConfigProperties(
     val orchestratorUrl: String,
     val demoUrl: String,
     val initialBatchSize: Int,
-    val fileStorage: FileStorageConfig,
     val s3Storage: S3StorageConfig,
     val scheduling: Scheduling = Scheduling(),
     val agentSettings: AgentSettings = AgentSettings(),
     val testAnalysisSettings: TestAnalysisSettings = TestAnalysisSettings(),
     val loki: LokiConfig? = null,
 ) {
-    /**
-     * @property location location of file storage
-     */
-    data class FileStorageConfig(
-        val location: String,
-    )
-
     /**
      * @property endpoint S3 endpoint (URI)
      * @property bucketName bucket name for all S3 storages

save-backend/src/main/kotlin/com/saveourtool/save/backend/controllers/AvatarController.kt

@@ -4,7 +4,7 @@ import com.saveourtool.save.backend.ByteBufferFluxResponse
 import com.saveourtool.save.backend.service.OrganizationService
 import com.saveourtool.save.backend.service.UserDetailsService
 import com.saveourtool.save.backend.storage.AvatarKey
-import com.saveourtool.save.backend.storage.MigrationAvatarStorage
+import com.saveourtool.save.backend.storage.AvatarStorage
 import com.saveourtool.save.configs.ApiSwaggerSupport
 import com.saveourtool.save.utils.*
 import com.saveourtool.save.v1
@@ -37,7 +37,7 @@ import kotlin.time.toJavaDuration
 @RestController
 @RequestMapping(path = ["/api/$v1/avatar"])
 internal class AvatarController(
-    private val avatarStorage: MigrationAvatarStorage,
+    private val avatarStorage: AvatarStorage,
     private val organizationService: OrganizationService,
     private val userDetailsService: UserDetailsService,
 ) {

save-backend/src/main/kotlin/com/saveourtool/save/backend/controllers/DemoManagerController.kt

@@ -4,7 +4,6 @@ import com.saveourtool.save.backend.configs.ConfigProperties
 import com.saveourtool.save.backend.security.ProjectPermissionEvaluator
 import com.saveourtool.save.backend.service.LnkProjectGithubService
 import com.saveourtool.save.backend.service.ProjectService
-import com.saveourtool.save.backend.storage.FileStorage
 import com.saveourtool.save.configs.RequiresAuthorizationSourceHeader
 import com.saveourtool.save.demo.DemoCreationRequest
 import com.saveourtool.save.demo.DemoDto
@@ -43,7 +42,6 @@ class DemoManagerController(
     private val projectService: ProjectService,
     private val lnkProjectGithubService: LnkProjectGithubService,
     private val projectPermissionEvaluator: ProjectPermissionEvaluator,
-    private val fileStorage: FileStorage,
     configProperties: ConfigProperties,
     customizers: List<WebClientCustomizer>,
 ) {

save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/AvatarStorage.kt

@@ -1,58 +1,52 @@
-/**
- * Storage for avatars + key for this storage
- */
-
 package com.saveourtool.save.backend.storage
 
 import com.saveourtool.save.backend.configs.ConfigProperties
-import com.saveourtool.save.storage.AbstractFileBasedStorage
+import com.saveourtool.save.storage.AbstractS3Storage
+import com.saveourtool.save.storage.concatS3Key
+import com.saveourtool.save.storage.s3KeyToParts
 import com.saveourtool.save.utils.AvatarType
 import com.saveourtool.save.utils.orNotFound
 import org.springframework.stereotype.Service
 import reactor.core.publisher.Flux
 import reactor.core.publisher.Mono
+import software.amazon.awssdk.services.s3.S3AsyncClient
 import java.nio.ByteBuffer
-import java.nio.file.Path
-import kotlin.io.path.div
-import kotlin.io.path.name
 
 /**
  * Storage for Avatars
  * Currently, key is (AvatarType, ObjectName)
  */
 @Service
-class AvatarStorage(configProperties: ConfigProperties) :
-    AbstractFileBasedStorage<AvatarKey>(Path.of(configProperties.fileStorage.location) / "images" / "avatars") {
-    /**
-     * @param rootDir
-     * @param pathToContent
-     * @return [AvatarKey] object is built by [Path]
-     */
-    override fun buildKey(rootDir: Path, pathToContent: Path): AvatarKey = AvatarKey(
-        type = AvatarType.findByUrlPath(pathToContent.parent.name)
-            .orNotFound {
-                "Not supported type for path: ${pathToContent.parent.name}"
-            },
-        objectName = pathToContent.name,
-    )
+class AvatarStorage(
+    configProperties: ConfigProperties,
+    s3Client: S3AsyncClient,
+) : AbstractS3Storage<AvatarKey>(
+    s3Client,
+    configProperties.s3Storage.bucketName,
+    concatS3Key(configProperties.s3Storage.prefix, "images", "avatars")
+) {
+    override fun buildKey(s3KeySuffix: String): AvatarKey {
+        val (typeStr, objectName) = s3KeySuffix.s3KeyToParts()
+        return AvatarKey(
+            type = AvatarType.findByUrlPath(typeStr)
+                .orNotFound {
+                    "Not supported type for path: $typeStr"
+                },
+            objectName = objectName,
+        )
+    }
+
+    override fun buildS3KeySuffix(key: AvatarKey): String = concatS3Key(key.type.urlPath, key.objectName)
 
     /**
      * @param key
      * @param content
      * @return `Mono` with file size
      */
     fun upsert(key: AvatarKey, content: Flux<ByteBuffer>): Mono<Long> = list()
-        .filter { it.objectName == key.objectName }
+        .filter { it == key }
         .singleOrEmpty()
         .flatMap { delete(it) }
         .switchIfEmpty(Mono.just(true))
         .flatMap { upload(key, content) }
-
-    /**
-     * @param rootDir
-     * @param key
-     * @return [Path] is built by [AvatarKey] object
-     */
-    override fun buildPathToContent(rootDir: Path, key: AvatarKey): Path = rootDir.resolve(key.type.urlPath)
-        .resolve(key.objectName)
 }

save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/FileStorage.kt

@@ -15,9 +15,6 @@ import reactor.core.publisher.Flux
 import reactor.core.publisher.Mono
 import software.amazon.awssdk.services.s3.S3AsyncClient
 
-import java.nio.file.Path
-
-import kotlin.io.path.div
 import kotlinx.datetime.toJavaLocalDateTime
 
 /**
@@ -31,7 +28,6 @@ class FileStorage(
     private val projectService: ProjectService,
     private val executionService: ExecutionService,
 ) : AbstractStorageWithDatabase<FileDto, File, FileRepository>(
-    Path.of(configProperties.fileStorage.location) / "storage",
     s3Client,
     configProperties.s3Storage.bucketName,
     concatS3Key(configProperties.s3Storage.prefix, "storage"),

save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/TestsSourceSnapshotStorage.kt

@@ -21,8 +21,6 @@ import org.springframework.stereotype.Component
 import reactor.core.publisher.Mono
 import software.amazon.awssdk.services.s3.S3AsyncClient
 
-import java.nio.file.Path
-
 import kotlin.io.path.*
 
 /**
@@ -37,14 +35,11 @@ class TestsSourceSnapshotStorage(
     private val testSuitesService: TestSuitesService,
     private val executionService: ExecutionService,
 ) : AbstractStorageWithDatabase<TestsSourceSnapshotDto, TestsSourceSnapshot, TestsSourceSnapshotRepository>(
-    Path.of(configProperties.fileStorage.location) / "testSuites",
     s3Client,
     configProperties.s3Storage.bucketName,
     concatS3Key(configProperties.s3Storage.prefix, "tests-source-snapshot"),
     testsSourceSnapshotRepository
 ) {
-    private val tmpDir = (Path.of(configProperties.fileStorage.location) / "tmp").createDirectories()
-
     override fun createNewEntityFromDto(dto: TestsSourceSnapshotDto): TestsSourceSnapshot = dto.toEntity { testSuitesSourceRepository.getByIdOrNotFound(it) }
 
     override fun findByDto(
@@ -63,7 +58,7 @@ class TestsSourceSnapshotStorage(
      * @return [TestFilesContent] filled with test files
      */
     fun getTestContent(request: TestFilesRequest): Mono<TestFilesContent> {
-        val tmpSourceDir = createTempDirectory(tmpDir, "source-")
+        val tmpSourceDir = createTempDirectory("source-")
         val tmpArchive = createTempFile(tmpSourceDir, "archive-", ARCHIVE_EXTENSION)
         val sourceContent = download(request.testsSourceSnapshot)
             .map { DefaultDataBufferFactory.sharedInstance.wrap(it) }

save-backend/src/main/resources/application-dev.properties

@@ -4,7 +4,6 @@ spring.datasource.password=123
 backend.preprocessorUrl=http://localhost:5200
 backend.orchestratorUrl=http://localhost:5100
 backend.demoUrl=http://localhost:5421
-backend.fileStorage.location=${user.home}/.save-cloud/cnb/files
 backend.s3-storage.endpoint=http://localhost:9000
 backend.s3-storage.bucketName=cnb
 backend.s3-storage.prefix=

save-backend/src/main/resources/application.properties

@@ -6,7 +6,6 @@ backend.preprocessorUrl=http://preprocessor:5200
 backend.orchestratorUrl=http://orchestrator:5100
 backend.demoUrl=http://demo:5421
 backend.initialBatchSize=100
-backend.file-storage.location=/home/cnb/files
 backend.test-analysis-settings.sliding-window-size=2048
 backend.test-analysis-settings.parallel-startup=true
 server.port=5800

save-backend/src/test/kotlin/com/saveourtool/save/backend/DownloadFilesTest.kt

@@ -23,7 +23,6 @@ import org.jetbrains.annotations.Blocking
 
 import org.junit.jupiter.api.Assertions
 import org.junit.jupiter.api.Test
-import org.junit.jupiter.api.io.TempDir
 import org.mockito.kotlin.*
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.boot.context.properties.EnableConfigurationProperties
@@ -38,8 +37,6 @@ import org.springframework.http.MediaType
 import org.springframework.http.client.MultipartBodyBuilder
 import org.springframework.security.test.context.support.WithMockUser
 import org.springframework.test.context.ActiveProfiles
-import org.springframework.test.context.DynamicPropertyRegistry
-import org.springframework.test.context.DynamicPropertySource
 import org.springframework.test.web.reactive.server.WebTestClient
 import org.springframework.test.web.reactive.server.expectBody
 import org.springframework.test.web.reactive.server.expectBodyList
@@ -50,7 +47,6 @@ import reactor.core.scheduler.Schedulers
 import reactor.kotlin.core.publisher.toMono
 import java.nio.ByteBuffer
 
-import java.nio.file.Path
 import java.time.LocalDateTime
 import java.util.*
 import java.util.concurrent.Future
@@ -61,7 +57,6 @@ import kotlin.io.path.*
 @Import(
     WebConfig::class,
     NoopWebSecurityConfig::class,
-    AvatarStorage::class,
     S11nTestConfig::class,
 )
 @AutoConfigureWebTestClient
@@ -289,16 +284,6 @@ class DownloadFilesTest {
     }
 
     companion object {
-        @TempDir internal lateinit var tmpDir: Path
-
-        @DynamicPropertySource
-        @JvmStatic
-        fun properties(registry: DynamicPropertyRegistry) {
-            registry.add("backend.fileStorage.location") {
-                tmpDir.absolutePathString()
-            }
-        }
-
         private fun FileDto.candidateTo(file: File) = name == file.name && projectCoordinates == file.project.toProjectCoordinates()
 
         /**