feat: Batch operations (#1743)

.github/workflows/test.yml

@@ -56,7 +56,7 @@ jobs:
             ~/backend-development.tgz
 
   backend-test:
-    name: Backend testing ‍🔎
+    name: BT ‍🔎
     needs: [backend-build]
     runs-on: ubuntu-latest
     strategy:
@@ -66,6 +66,7 @@ jobs:
           [
             "server-app:runContextRecreatingTests",
             "server-app:runStandardTests",
+            "server-app:runWebsocketTests",
             "ee-test:test",
             "data:test",
           ]

backend/api/build.gradle

@@ -44,6 +44,7 @@ dependencies {
     implementation "org.springframework.boot:spring-boot-starter-hateoas"
     implementation 'org.springframework.boot:spring-boot-starter-web'
     implementation("org.springframework.boot:spring-boot-starter-security")
+    implementation "org.springframework.boot:spring-boot-starter-websocket"
 
     implementation(project(':data'))
     implementation(project(':misc'))

backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/AutoTranslationController.kt

@@ -73,7 +73,7 @@ When no languages provided, it translates only untranslated languages."""
 
     autoTranslationService.autoTranslate(
       key = key,
-      languageTags = languages,
+      languageTags = languages?.toList(),
       useTranslationMemory = useTranslationMemory ?: false,
       useMachineTranslation = useMachineTranslation ?: false
     )

backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobManagementController.kt

@@ -0,0 +1,100 @@
+package io.tolgee.api.v2.controllers.batch
+
+import io.swagger.v3.oas.annotations.Operation
+import io.swagger.v3.oas.annotations.tags.Tag
+import io.tolgee.batch.BatchJobCancellationManager
+import io.tolgee.batch.BatchJobDto
+import io.tolgee.batch.BatchJobService
+import io.tolgee.hateoas.batch.BatchJobModel
+import io.tolgee.hateoas.batch.BatchJobModelAssembler
+import io.tolgee.model.batch.BatchJob
+import io.tolgee.model.enums.Scope
+import io.tolgee.model.views.BatchJobView
+import io.tolgee.security.AuthenticationFacade
+import io.tolgee.security.apiKeyAuth.AccessWithApiKey
+import io.tolgee.security.project_auth.AccessWithAnyProjectPermission
+import io.tolgee.security.project_auth.AccessWithProjectPermission
+import io.tolgee.security.project_auth.ProjectHolder
+import io.tolgee.service.security.SecurityService
+import org.springdoc.api.annotations.ParameterObject
+import org.springframework.data.domain.Pageable
+import org.springframework.data.web.PagedResourcesAssembler
+import org.springframework.data.web.SortDefault
+import org.springframework.hateoas.PagedModel
+import org.springframework.web.bind.annotation.CrossOrigin
+import org.springframework.web.bind.annotation.GetMapping
+import org.springframework.web.bind.annotation.PathVariable
+import org.springframework.web.bind.annotation.PutMapping
+import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.RestController
+import javax.validation.Valid
+
+@RestController
+@CrossOrigin(origins = ["*"])
+@RequestMapping(value = ["/v2/projects/{projectId:\\d+}/", "/v2/projects/"])
+@Tag(name = "Batch job management")
+@Suppress("SpringJavaInjectionPointsAutowiringInspection", "MVCPathVariableInspection")
+class BatchJobManagementController(
+  private val batchJobCancellationManager: BatchJobCancellationManager,
+  private val batchJobService: BatchJobService,
+  private val projectHolder: ProjectHolder,
+  private val batchJobModelAssembler: BatchJobModelAssembler,
+  private val pagedResourcesAssembler: PagedResourcesAssembler<BatchJobView>,
+  private val authenticationFacade: AuthenticationFacade,
+  private val securityService: SecurityService
+) {
+  @GetMapping(value = ["batch-jobs"])
+  @AccessWithApiKey()
+  @AccessWithProjectPermission(Scope.BATCH_JOBS_VIEW)
+  @Operation(summary = "Lists all batch jobs in project")
+  fun list(@Valid @ParameterObject @SortDefault("id") pageable: Pageable): PagedModel<BatchJobModel> {
+    val views = batchJobService.getViews(projectHolder.project.id, null, pageable)
+    return pagedResourcesAssembler.toModel(views, batchJobModelAssembler)
+  }
+
+  @GetMapping(value = ["my-batch-jobs"])
+  @AccessWithApiKey()
+  @AccessWithAnyProjectPermission()
+  @Operation(summary = "Lists all batch jobs in project started by current user")
+  fun myList(@Valid @ParameterObject @SortDefault("id") pageable: Pageable): PagedModel<BatchJobModel> {
+    val views = batchJobService.getViews(
+      projectId = projectHolder.project.id,
+      userAccount = authenticationFacade.userAccount,
+      pageable = pageable
+    )
+    return pagedResourcesAssembler.toModel(views, batchJobModelAssembler)
+  }
+
+  @GetMapping(value = ["batch-jobs/{id}"])
+  @AccessWithApiKey()
+  @AccessWithAnyProjectPermission()
+  @Operation(summary = "Returns the batch job")
+  fun get(@PathVariable id: Long): BatchJobModel {
+    val view = batchJobService.getView(id)
+    checkViewPermission(view.batchJob)
+    return batchJobModelAssembler.toModel(view)
+  }
+
+  @PutMapping(value = ["batch-jobs/{id}/cancel"])
+  @AccessWithApiKey()
+  @AccessWithAnyProjectPermission()
+  @Operation(summary = "Stops batch job (if possible)")
+  fun cancel(@PathVariable id: Long) {
+    checkCancelPermission(batchJobService.getJobDto(id))
+    batchJobCancellationManager.cancel(id)
+  }
+
+  private fun checkViewPermission(job: BatchJob) {
+    if (job.author?.id == authenticationFacade.userAccount.id) {
+      return
+    }
+    securityService.checkProjectPermission(projectHolder.project.id, Scope.BATCH_JOBS_VIEW)
+  }
+
+  private fun checkCancelPermission(job: BatchJobDto) {
+    if (job.authorId == authenticationFacade.userAccount.id) {
+      return
+    }
+    securityService.checkProjectPermission(projectHolder.project.id, Scope.BATCH_JOBS_CANCEL)
+  }
+}

backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobController.kt

@@ -0,0 +1,68 @@
+package io.tolgee.api.v2.controllers.batch
+
+import io.swagger.v3.oas.annotations.Operation
+import io.swagger.v3.oas.annotations.tags.Tag
+import io.tolgee.batch.BatchJobService
+import io.tolgee.batch.BatchJobType
+import io.tolgee.batch.request.BatchTranslateRequest
+import io.tolgee.batch.request.DeleteKeysRequest
+import io.tolgee.hateoas.batch.BatchJobModel
+import io.tolgee.hateoas.batch.BatchJobModelAssembler
+import io.tolgee.model.batch.BatchJob
+import io.tolgee.model.enums.Scope
+import io.tolgee.security.AuthenticationFacade
+import io.tolgee.security.apiKeyAuth.AccessWithApiKey
+import io.tolgee.security.project_auth.AccessWithProjectPermission
+import io.tolgee.security.project_auth.ProjectHolder
+import io.tolgee.service.security.SecurityService
+import org.springframework.web.bind.annotation.CrossOrigin
+import org.springframework.web.bind.annotation.PutMapping
+import org.springframework.web.bind.annotation.RequestBody
+import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.RestController
+import javax.validation.Valid
+
+@RestController
+@CrossOrigin(origins = ["*"])
+@RequestMapping(value = ["/v2/projects/{projectId:\\d+}/start-batch-job", "/v2/projects/start-batch-job"])
+@Tag(name = "Start batch jobs")
+@Suppress("SpringJavaInjectionPointsAutowiringInspection", "MVCPathVariableInspection")
+class StartBatchJobController(
+  private val securityService: SecurityService,
+  private val projectHolder: ProjectHolder,
+  private val batchJobService: BatchJobService,
+  private val authenticationFacade: AuthenticationFacade,
+  private val batchJobModelAssembler: BatchJobModelAssembler
+) {
+  @PutMapping(value = ["/translate"])
+  @AccessWithApiKey()
+  @AccessWithProjectPermission(Scope.BATCH_AUTO_TRANSLATE)
+  @Operation(summary = "Translates provided keys to provided languages")
+  fun translate(@Valid @RequestBody data: BatchTranslateRequest): BatchJobModel {
+    securityService.checkLanguageTranslatePermission(projectHolder.project.id, data.targetLanguageIds)
+    securityService.checkKeyIdsExistAndIsFromProject(data.keyIds, projectHolder.project.id)
+    return batchJobService.startJob(
+      data,
+      projectHolder.projectEntity,
+      authenticationFacade.userAccountEntity,
+      BatchJobType.TRANSLATION
+    ).model
+  }
+
+  @PutMapping(value = ["/delete-keys"])
+  @AccessWithApiKey()
+  @AccessWithProjectPermission(Scope.KEYS_DELETE)
+  @Operation(summary = "Translates provided keys to provided languages")
+  fun deleteKeys(@Valid @RequestBody data: DeleteKeysRequest): BatchJobModel {
+    securityService.checkKeyIdsExistAndIsFromProject(data.keyIds, projectHolder.project.id)
+    return batchJobService.startJob(
+      data,
+      projectHolder.projectEntity,
+      authenticationFacade.userAccountEntity,
+      BatchJobType.DELETE_KEYS
+    ).model
+  }
+
+  val BatchJob.model
+    get() = batchJobModelAssembler.toModel(batchJobService.getView(this))
+}

backend/api/src/main/kotlin/io/tolgee/component/ProjectTranslationLastModifiedManager.kt

@@ -26,7 +26,7 @@ class ProjectTranslationLastModifiedManager(
 
   @EventListener
   fun onActivity(event: OnProjectActivityEvent) {
-    event.activityHolder.activityRevision?.projectId?.let { projectId ->
+    event.activityRevision.projectId?.let { projectId ->
       getCache()?.put(projectId, currentDateProvider.date.time)
     }
   }

backend/api/src/main/kotlin/io/tolgee/component/lockingProvider/RedissonLockingProvider.kt

@@ -1,11 +1,23 @@
 package io.tolgee.component.lockingProvider
 
 import io.tolgee.component.LockingProvider
+import org.redisson.api.RLock
 import org.redisson.api.RedissonClient
-import java.util.concurrent.locks.Lock
 
 class RedissonLockingProvider(private val redissonClient: RedissonClient) : LockingProvider {
-  override fun getLock(name: String): Lock {
+  override fun getLock(name: String): RLock {
     return redissonClient.getLock(name)
   }
+
+  override fun <T> withLocking(name: String, fn: () -> T): T {
+    val lock = this.getLock(name)
+    lock.lock()
+    try {
+      return fn()
+    } finally {
+      if (lock.isHeldByCurrentThread) {
+        lock.unlock()
+      }
+    }
+  }
 }

backend/api/src/main/kotlin/io/tolgee/component/lockingProvider/SimpleLockingProvider.kt

@@ -15,4 +15,14 @@ class SimpleLockingProvider : LockingProvider {
   override fun getLock(name: String): Lock {
     return map.getOrPut(name) { ReentrantLock() }
   }
+
+  override fun <T> withLocking(name: String, fn: () -> T): T {
+    val lock = this.getLock(name)
+    lock.lock()
+    try {
+      return fn()
+    } finally {
+      lock.unlock()
+    }
+  }
 }

backend/api/src/main/kotlin/io/tolgee/configuration/RedisLockingConfiguration.kt

@@ -6,15 +6,15 @@ import org.redisson.Redisson
 import org.redisson.api.RedissonClient
 import org.springframework.boot.autoconfigure.AutoConfigureAfter
 import org.springframework.boot.autoconfigure.condition.ConditionalOnClass
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
+import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
 import org.springframework.data.redis.core.RedisOperations
 
 @Configuration
 @ConditionalOnClass(Redisson::class, RedisOperations::class)
 @AutoConfigureAfter(ConditionalRedissonAutoConfiguration::class)
-@ConditionalOnProperty(name = ["tolgee.cache.use-redis"], havingValue = "true")
+@ConditionalOnExpression("\${tolgee.cache.use-redis:false} and \${tolgee.cache.enabled:false}")
 class RedisLockingConfiguration(
   val redissonClient: RedissonClient
 ) {

backend/api/src/main/kotlin/io/tolgee/configuration/RedissonCacheConfiguration.kt

@@ -8,7 +8,7 @@ import org.redisson.spring.cache.CacheConfig
 import org.redisson.spring.cache.RedissonSpringCacheManager
 import org.springframework.boot.autoconfigure.AutoConfigureAfter
 import org.springframework.boot.autoconfigure.condition.ConditionalOnClass
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
+import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression
 import org.springframework.cache.CacheManager
 import org.springframework.cache.annotation.EnableCaching
 import org.springframework.context.annotation.Bean
@@ -19,7 +19,7 @@ import org.springframework.data.redis.core.RedisOperations
 @EnableCaching
 @ConditionalOnClass(Redisson::class, RedisOperations::class)
 @AutoConfigureAfter(ConditionalRedissonAutoConfiguration::class)
-@ConditionalOnProperty(name = ["tolgee.cache.use-redis"], havingValue = "true")
+@ConditionalOnExpression("\${tolgee.cache.use-redis:false} and \${tolgee.cache.enabled:false}")
 class RedissonCacheConfiguration(private val tolgeeProperties: TolgeeProperties) {
   @Bean
   fun cacheManager(redissonClient: RedissonClient): CacheManager? {

backend/api/src/main/kotlin/io/tolgee/hateoas/batch/BatchJobModel.kt

@@ -0,0 +1,40 @@
+package io.tolgee.hateoas.batch
+
+import io.swagger.v3.oas.annotations.media.Schema
+import io.tolgee.batch.BatchJobType
+import io.tolgee.hateoas.user_account.SimpleUserAccountModel
+import io.tolgee.model.batch.BatchJobStatus
+import org.springframework.hateoas.RepresentationModel
+import org.springframework.hateoas.server.core.Relation
+import java.io.Serializable
+
+@Suppress("unused")
+@Relation(collectionRelation = "batchJobs", itemRelation = "batchJob")
+open class BatchJobModel(
+  @Schema(description = "Batch job id")
+  val id: Long,
+
+  @Schema(description = "Status of the batch job")
+  val status: BatchJobStatus,
+
+  @Schema(description = "Type of the batch job")
+  val type: BatchJobType,
+
+  @Schema(description = "Total items, that have been processed so far")
+  val progress: Int,
+
+  @Schema(description = "Total items")
+  val totalItems: Int,
+
+  @Schema(description = "The user who started the job")
+  val author: SimpleUserAccountModel?,
+
+  @Schema(description = "The time when the job created")
+  val createdAt: String,
+
+  @Schema(description = "The activity revision id, that stores the activity details of the job")
+  val activityRevisionId: Long?,
+
+  @Schema(description = "If the job failed, this is the error message")
+  val errorMessage: String?,
+) : RepresentationModel<BatchJobModel>(), Serializable

backend/api/src/main/kotlin/io/tolgee/hateoas/batch/BatchJobModelAssembler.kt

@@ -0,0 +1,28 @@
+package io.tolgee.hateoas.batch
+
+import io.tolgee.api.v2.controllers.batch.BatchJobManagementController
+import io.tolgee.hateoas.user_account.SimpleUserAccountModelAssembler
+import io.tolgee.model.views.BatchJobView
+import org.springframework.hateoas.server.mvc.RepresentationModelAssemblerSupport
+import org.springframework.stereotype.Component
+
+@Component
+class BatchJobModelAssembler(
+  val simpleUserAccountModelAssembler: SimpleUserAccountModelAssembler
+) : RepresentationModelAssemblerSupport<BatchJobView, BatchJobModel>(
+  BatchJobManagementController::class.java, BatchJobModel::class.java
+) {
+  override fun toModel(view: BatchJobView): BatchJobModel {
+    return BatchJobModel(
+      id = view.batchJob.id,
+      type = view.batchJob.type,
+      status = view.batchJob.status,
+      progress = view.progress,
+      totalItems = view.batchJob.totalItems,
+      author = view.batchJob.author?.let { simpleUserAccountModelAssembler.toModel(it) },
+      createdAt = view.batchJob.createdAt.toString(),
+      activityRevisionId = view.batchJob.activityRevision?.id,
+      errorMessage = view.errorMessage?.code
+    )
+  }
+}

backend/api/src/main/kotlin/io/tolgee/hateoas/user_account/SimpleUserAccountModelAssembler.kt

@@ -1,6 +1,7 @@
 package io.tolgee.hateoas.user_account
 
 import io.tolgee.api.v2.controllers.V2UserController
+import io.tolgee.dtos.cacheable.UserAccountDto
 import io.tolgee.model.UserAccount
 import io.tolgee.service.AvatarService
 import org.springframework.hateoas.server.mvc.RepresentationModelAssemblerSupport
@@ -23,4 +24,16 @@ class SimpleUserAccountModelAssembler(
       deleted = entity.deletedAt != null
     )
   }
+
+  fun toModel(dto: UserAccountDto): SimpleUserAccountModel {
+    val avatar = avatarService.getAvatarLinks(dto.avatarHash)
+
+    return SimpleUserAccountModel(
+      id = dto.id,
+      username = dto.username,
+      name = dto.name,
+      avatar = avatar,
+      deleted = dto.deleted
+    )
+  }
 }