feat: Batch operations (#1793)

Co-authored-by: Jan Cizmar <[email protected]>
Co-authored-by: Martin Chrástek <[email protected]>

.github/workflows/test.yml

@@ -56,7 +56,7 @@ jobs:
             ~/backend-development.tgz
 
   backend-test:
-    name: Backend testing ‍🔎
+    name: BT ‍🔎
     needs: [backend-build]
     runs-on: ubuntu-latest
     strategy:
@@ -66,6 +66,7 @@ jobs:
           [
             "server-app:runContextRecreatingTests",
             "server-app:runStandardTests",
+            "server-app:runWebsocketTests",
             "ee-test:test",
             "data:test",
           ]

backend/api/build.gradle

@@ -44,6 +44,7 @@ dependencies {
     implementation "org.springframework.boot:spring-boot-starter-hateoas"
     implementation 'org.springframework.boot:spring-boot-starter-web'
     implementation("org.springframework.boot:spring-boot-starter-security")
+    implementation "org.springframework.boot:spring-boot-starter-websocket"
 
     implementation(project(':data'))
     implementation(project(':misc'))
@@ -57,8 +58,9 @@ dependencies {
     implementation libs.springDocOpenApiKotlin
     implementation libs.springDocOpenApiDataRest
     implementation libs.springDocOpenApiHateoas
-
-    implementation libs.redissonSpringBootStarter
+    implementation dependencies.create(libs.redissonSpringBootStarter.get()) {
+        exclude group: 'org.redisson', module: 'redisson-spring-data-31'
+    }
     implementation libs.redissonSpringData
 
     /**

backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/AutoTranslationController.kt

@@ -71,11 +71,12 @@ When no languages provided, it translates only untranslated languages."""
       throw BadRequestException(Message.CANNOT_TRANSLATE_BASE_LANGUAGE)
     }
 
-    autoTranslationService.autoTranslate(
+    autoTranslationService.autoTranslateSync(
       key = key,
-      languageTags = languages,
+      languageTags = languages?.toList(),
       useTranslationMemory = useTranslationMemory ?: false,
-      useMachineTranslation = useMachineTranslation ?: false
+      useMachineTranslation = useMachineTranslation ?: false,
+      isBatch = true
     )
   }
 

backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/KeyController.kt

@@ -90,8 +90,10 @@ class KeyController(
       projectHolder.projectEntity.checkScreenshotsUploadPermission()
     }
 
-    dto.translations?.keys?.let { languageTags ->
-      securityService.checkLanguageTranslatePermissionByTag(projectHolder.project.id, languageTags)
+    dto.translations?.filterValues { !it.isNullOrEmpty() }?.keys?.let { languageTags ->
+      if (languageTags.isNotEmpty()) {
+        securityService.checkLanguageTranslatePermissionByTag(projectHolder.project.id, languageTags)
+      }
     }
 
     val key = keyService.create(projectHolder.projectEntity, dto)

backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/V2ProjectsController.kt

@@ -320,7 +320,8 @@ V2ProjectsController(
     val config = autoTranslateService.getConfig(projectHolder.projectEntity)
     return AutoTranslationSettingsDto(
       usingTranslationMemory = config.usingTm,
-      usingMachineTranslation = config.usingPrimaryMtService
+      usingMachineTranslation = config.usingPrimaryMtService,
+      enableForImport = config.enableForImport,
     )
   }
 }

backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobManagementController.kt

@@ -0,0 +1,116 @@
+package io.tolgee.api.v2.controllers.batch
+
+import io.swagger.v3.oas.annotations.Operation
+import io.swagger.v3.oas.annotations.tags.Tag
+import io.tolgee.batch.BatchJobCancellationManager
+import io.tolgee.batch.BatchJobService
+import io.tolgee.batch.data.BatchJobDto
+import io.tolgee.hateoas.batch.BatchJobModel
+import io.tolgee.hateoas.batch.BatchJobModelAssembler
+import io.tolgee.model.batch.BatchJob
+import io.tolgee.model.enums.Scope
+import io.tolgee.model.views.BatchJobView
+import io.tolgee.security.AuthenticationFacade
+import io.tolgee.security.apiKeyAuth.AccessWithApiKey
+import io.tolgee.security.project_auth.AccessWithAnyProjectPermission
+import io.tolgee.security.project_auth.AccessWithProjectPermission
+import io.tolgee.security.project_auth.ProjectHolder
+import io.tolgee.service.security.SecurityService
+import org.springdoc.api.annotations.ParameterObject
+import org.springframework.data.domain.Pageable
+import org.springframework.data.web.PagedResourcesAssembler
+import org.springframework.data.web.SortDefault
+import org.springframework.hateoas.CollectionModel
+import org.springframework.hateoas.PagedModel
+import org.springframework.web.bind.annotation.CrossOrigin
+import org.springframework.web.bind.annotation.GetMapping
+import org.springframework.web.bind.annotation.PathVariable
+import org.springframework.web.bind.annotation.PutMapping
+import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.RestController
+import javax.validation.Valid
+
+@RestController
+@CrossOrigin(origins = ["*"])
+@RequestMapping(value = ["/v2/projects/{projectId:\\d+}/", "/v2/projects/"])
+@Tag(name = "Batch Operations management")
+@Suppress("SpringJavaInjectionPointsAutowiringInspection", "MVCPathVariableInspection")
+class BatchJobManagementController(
+  private val batchJobCancellationManager: BatchJobCancellationManager,
+  private val batchJobService: BatchJobService,
+  private val projectHolder: ProjectHolder,
+  private val batchJobModelAssembler: BatchJobModelAssembler,
+  private val pagedResourcesAssembler: PagedResourcesAssembler<BatchJobView>,
+  private val authenticationFacade: AuthenticationFacade,
+  private val securityService: SecurityService
+) {
+  @GetMapping(value = ["batch-jobs"])
+  @AccessWithApiKey()
+  @AccessWithProjectPermission(Scope.BATCH_JOBS_VIEW)
+  @Operation(summary = "Lists all batch operations in project")
+  fun list(@Valid @ParameterObject @SortDefault("id") pageable: Pageable): PagedModel<BatchJobModel> {
+    val views = batchJobService.getViews(projectHolder.project.id, null, pageable)
+    return pagedResourcesAssembler.toModel(views, batchJobModelAssembler)
+  }
+
+  @GetMapping(value = ["my-batch-jobs"])
+  @AccessWithApiKey()
+  @AccessWithAnyProjectPermission()
+  @Operation(summary = "Lists all batch operations in project started by current user")
+  fun myList(@Valid @ParameterObject @SortDefault("id") pageable: Pageable): PagedModel<BatchJobModel> {
+    val views = batchJobService.getViews(
+      projectId = projectHolder.project.id,
+      userAccount = authenticationFacade.userAccount,
+      pageable = pageable
+    )
+    return pagedResourcesAssembler.toModel(views, batchJobModelAssembler)
+  }
+
+  @GetMapping(value = ["current-batch-jobs"])
+  @AccessWithApiKey()
+  @AccessWithAnyProjectPermission()
+  @Operation(
+    summary = "Returns all running and pending batch operations",
+    description = "Completed batch operations are returned only if they are not older than 1 hour. " +
+      "If user doesn't have permission to view all batch operations, only their operations are returned."
+  )
+  fun currentJobs(): CollectionModel<BatchJobModel> {
+    val views = batchJobService.getCurrentJobViews(
+      projectId = projectHolder.project.id,
+    )
+    return batchJobModelAssembler.toCollectionModel(views)
+  }
+
+  @GetMapping(value = ["batch-jobs/{id}"])
+  @AccessWithApiKey()
+  @AccessWithAnyProjectPermission()
+  @Operation(summary = "Returns batch operation")
+  fun get(@PathVariable id: Long): BatchJobModel {
+    val view = batchJobService.getView(id)
+    checkViewPermission(view.batchJob)
+    return batchJobModelAssembler.toModel(view)
+  }
+
+  @PutMapping(value = ["batch-jobs/{id}/cancel"])
+  @AccessWithApiKey()
+  @AccessWithAnyProjectPermission()
+  @Operation(summary = "Stops batch operation (if possible)")
+  fun cancel(@PathVariable id: Long) {
+    checkCancelPermission(batchJobService.getJobDto(id))
+    batchJobCancellationManager.cancel(id)
+  }
+
+  private fun checkViewPermission(job: BatchJob) {
+    if (job.author?.id == authenticationFacade.userAccount.id) {
+      return
+    }
+    securityService.checkProjectPermission(projectHolder.project.id, Scope.BATCH_JOBS_VIEW)
+  }
+
+  private fun checkCancelPermission(job: BatchJobDto) {
+    if (job.authorId == authenticationFacade.userAccount.id) {
+      return
+    }
+    securityService.checkProjectPermission(projectHolder.project.id, Scope.BATCH_JOBS_CANCEL)
+  }
+}

backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobController.kt

@@ -0,0 +1,198 @@
+package io.tolgee.api.v2.controllers.batch
+
+import io.swagger.v3.oas.annotations.Operation
+import io.swagger.v3.oas.annotations.tags.Tag
+import io.tolgee.batch.BatchJobService
+import io.tolgee.batch.data.BatchJobType
+import io.tolgee.batch.request.ClearTranslationsRequest
+import io.tolgee.batch.request.CopyTranslationRequest
+import io.tolgee.batch.request.DeleteKeysRequest
+import io.tolgee.batch.request.MachineTranslationRequest
+import io.tolgee.batch.request.PreTranslationByTmRequest
+import io.tolgee.batch.request.SetKeysNamespaceRequest
+import io.tolgee.batch.request.SetTranslationsStateStateRequest
+import io.tolgee.batch.request.TagKeysRequest
+import io.tolgee.batch.request.UntagKeysRequest
+import io.tolgee.constants.Message
+import io.tolgee.exceptions.BadRequestException
+import io.tolgee.hateoas.batch.BatchJobModel
+import io.tolgee.hateoas.batch.BatchJobModelAssembler
+import io.tolgee.model.batch.BatchJob
+import io.tolgee.model.enums.Scope
+import io.tolgee.security.AuthenticationFacade
+import io.tolgee.security.apiKeyAuth.AccessWithApiKey
+import io.tolgee.security.project_auth.AccessWithProjectPermission
+import io.tolgee.security.project_auth.ProjectHolder
+import io.tolgee.service.security.SecurityService
+import org.springframework.web.bind.annotation.CrossOrigin
+import org.springframework.web.bind.annotation.PostMapping
+import org.springframework.web.bind.annotation.RequestBody
+import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.RestController
+import javax.validation.Valid
+
+@RestController
+@CrossOrigin(origins = ["*"])
+@RequestMapping(value = ["/v2/projects/{projectId:\\d+}/start-batch-job", "/v2/projects/start-batch-job"])
+@Tag(name = "Batch Operations")
+@Suppress("SpringJavaInjectionPointsAutowiringInspection", "MVCPathVariableInspection")
+class StartBatchJobController(
+  private val securityService: SecurityService,
+  private val projectHolder: ProjectHolder,
+  private val batchJobService: BatchJobService,
+  private val authenticationFacade: AuthenticationFacade,
+  private val batchJobModelAssembler: BatchJobModelAssembler
+) {
+  @PostMapping(value = ["/pre-translate-by-tm"])
+  @AccessWithApiKey()
+  @AccessWithProjectPermission(Scope.BATCH_PRE_TRANSLATE_BY_TM)
+  @Operation(
+    summary = "Pre-translate by TM",
+    description = "Pre-translate provided keys to provided languages by TM."
+  )
+  fun translate(@Valid @RequestBody data: PreTranslationByTmRequest): BatchJobModel {
+    securityService.checkLanguageTranslatePermission(projectHolder.project.id, data.targetLanguageIds)
+    securityService.checkKeyIdsExistAndIsFromProject(data.keyIds, projectHolder.project.id)
+    return batchJobService.startJob(
+      data,
+      projectHolder.projectEntity,
+      authenticationFacade.userAccountEntity,
+      BatchJobType.PRE_TRANSLATE_BT_TM,
+    ).model
+  }
+
+  @PostMapping(value = ["/machine-translate"])
+  @AccessWithApiKey()
+  @AccessWithProjectPermission(Scope.BATCH_MACHINE_TRANSLATE)
+  @Operation(
+    summary = "Machine Translation",
+    description = "Translate provided keys to provided languages through primary MT provider."
+  )
+  fun machineTranslation(@Valid @RequestBody data: MachineTranslationRequest): BatchJobModel {
+    securityService.checkLanguageTranslatePermission(projectHolder.project.id, data.targetLanguageIds)
+    securityService.checkKeyIdsExistAndIsFromProject(data.keyIds, projectHolder.project.id)
+    return batchJobService.startJob(
+      data,
+      projectHolder.projectEntity,
+      authenticationFacade.userAccountEntity,
+      BatchJobType.MACHINE_TRANSLATE,
+    ).model
+  }
+
+  @PostMapping(value = ["/delete-keys"])
+  @AccessWithApiKey()
+  @AccessWithProjectPermission(Scope.KEYS_DELETE)
+  @Operation(summary = "Delete keys")
+  fun deleteKeys(@Valid @RequestBody data: DeleteKeysRequest): BatchJobModel {
+    securityService.checkKeyIdsExistAndIsFromProject(data.keyIds, projectHolder.project.id)
+    return batchJobService.startJob(
+      data,
+      projectHolder.projectEntity,
+      authenticationFacade.userAccountEntity,
+      BatchJobType.DELETE_KEYS,
+    ).model
+  }
+
+  @PostMapping(value = ["/set-translation-state"])
+  @AccessWithApiKey()
+  @AccessWithProjectPermission(Scope.TRANSLATIONS_STATE_EDIT)
+  @Operation(summary = "Set translation state")
+  fun setTranslationState(@Valid @RequestBody data: SetTranslationsStateStateRequest): BatchJobModel {
+    securityService.checkKeyIdsExistAndIsFromProject(data.keyIds, projectHolder.project.id)
+    securityService.checkLanguageStateChangePermission(projectHolder.project.id, data.languageIds)
+    return batchJobService.startJob(
+      data,
+      projectHolder.projectEntity,
+      authenticationFacade.userAccountEntity,
+      BatchJobType.SET_TRANSLATIONS_STATE,
+    ).model
+  }
+
+  @PostMapping(value = ["/clear-translations"])
+  @AccessWithApiKey()
+  @AccessWithProjectPermission(Scope.TRANSLATIONS_EDIT)
+  @Operation(
+    summary = "Clear translation values",
+    description = "Clear translation values for provided keys in selected languages."
+  )
+  fun clearTranslations(@Valid @RequestBody data: ClearTranslationsRequest): BatchJobModel {
+    securityService.checkKeyIdsExistAndIsFromProject(data.keyIds, projectHolder.project.id)
+    securityService.checkLanguageTranslatePermission(projectHolder.project.id, data.languageIds)
+    return batchJobService.startJob(
+      data,
+      projectHolder.projectEntity,
+      authenticationFacade.userAccountEntity,
+      BatchJobType.CLEAR_TRANSLATIONS,
+    ).model
+  }
+
+  @PostMapping(value = ["/copy-translations"])
+  @AccessWithApiKey()
+  @AccessWithProjectPermission(Scope.TRANSLATIONS_EDIT)
+  @Operation(
+    summary = "Copy translation values",
+    description = "Copy translation values from one language to other languages."
+  )
+  fun copyTranslations(@Valid @RequestBody data: CopyTranslationRequest): BatchJobModel {
+    securityService.checkKeyIdsExistAndIsFromProject(data.keyIds, projectHolder.project.id)
+    securityService.checkLanguageTranslatePermission(projectHolder.project.id, data.targetLanguageIds)
+    securityService.checkLanguageViewPermission(projectHolder.project.id, listOf(data.sourceLanguageId))
+    return batchJobService.startJob(
+      data,
+      projectHolder.projectEntity,
+      authenticationFacade.userAccountEntity,
+      BatchJobType.COPY_TRANSLATIONS,
+    ).model
+  }
+
+  @PostMapping(value = ["/tag-keys"])
+  @AccessWithApiKey()
+  @AccessWithProjectPermission(Scope.KEYS_EDIT)
+  @Operation(summary = "Add tags")
+  fun tagKeys(@Valid @RequestBody data: TagKeysRequest): BatchJobModel {
+    data.tags.validate()
+    securityService.checkKeyIdsExistAndIsFromProject(data.keyIds, projectHolder.project.id)
+    return batchJobService.startJob(
+      data,
+      projectHolder.projectEntity,
+      authenticationFacade.userAccountEntity,
+      BatchJobType.TAG_KEYS,
+    ).model
+  }
+
+  @PostMapping(value = ["/untag-keys"])
+  @AccessWithApiKey()
+  @AccessWithProjectPermission(Scope.KEYS_EDIT)
+  @Operation(summary = "Remove tags")
+  fun untagKeys(@Valid @RequestBody data: UntagKeysRequest): BatchJobModel {
+    securityService.checkKeyIdsExistAndIsFromProject(data.keyIds, projectHolder.project.id)
+    return batchJobService.startJob(
+      data,
+      projectHolder.projectEntity,
+      authenticationFacade.userAccountEntity,
+      BatchJobType.UNTAG_KEYS,
+    ).model
+  }
+
+  @PostMapping(value = ["/set-keys-namespace"])
+  @AccessWithApiKey()
+  @AccessWithProjectPermission(Scope.KEYS_EDIT)
+  @Operation(summary = "Set keys namespace")
+  fun setKeysNamespace(@Valid @RequestBody data: SetKeysNamespaceRequest): BatchJobModel {
+    securityService.checkKeyIdsExistAndIsFromProject(data.keyIds, projectHolder.project.id)
+    return batchJobService.startJob(
+      data,
+      projectHolder.projectEntity,
+      authenticationFacade.userAccountEntity,
+      BatchJobType.SET_KEYS_NAMESPACE,
+    ).model
+  }
+
+  val BatchJob.model
+    get() = batchJobModelAssembler.toModel(batchJobService.getView(this))
+
+  private fun List<String>.validate() {
+    if (this.any { it.isBlank() }) throw BadRequestException(Message.TAG_IS_BLANK)
+    if (this.any { it.length > 100 }) throw BadRequestException(Message.TAG_TOO_LOG)
+  }
+}