From 0ac95d1a4f65c827a7b037e93591c08e17431283 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Tue, 11 Jul 2023 15:07:49 +0200 Subject: [PATCH] feat: Batch operations (#1743) --- .github/workflows/test.yml | 3 +- backend/api/build.gradle | 1 + .../controllers/AutoTranslationController.kt | 2 +- .../api/v2/controllers/BigMetaController.kt | 0 .../batch/BatchJobManagementController.kt | 100 +++++ .../batch/StartBatchJobController.kt | 68 ++++ .../ConfigurationDocumentationProvider.kt | 0 .../ConfigurationPropsController.kt | 0 .../v2/controllers/configurationProps/data.kt | 0 .../ProjectTranslationLastModifiedManager.kt | 2 +- .../RedissonLockingProvider.kt | 16 +- .../lockingProvider/SimpleLockingProvider.kt | 10 + .../RedisLockingConfiguration.kt | 4 +- .../RedissonCacheConfiguration.kt | 4 +- .../io/tolgee/hateoas/batch/BatchJobModel.kt | 40 ++ .../hateoas/batch/BatchJobModelAssembler.kt | 28 ++ .../SimpleUserAccountModelAssembler.kt | 13 + .../websocket/ActivityWebsocketListener.kt | 60 ++- .../websocket/RedisWebsocketEventPublisher.kt | 0 .../SimpleWebsocketEventPublisher.kt | 0 .../io/tolgee/websocket/WebSocketConfig.kt | 0 .../WebsocketPublisherConfiguration.kt | 0 backend/app/build.gradle | 16 +- .../src/main/kotlin/io/tolgee/Application.kt | 30 +- .../kotlin/io/tolgee/ExceptionHandlers.kt | 19 + .../PostgresAutoStartConfiguration.kt | 6 +- .../SimpleLockingConfiguration.kt | 4 +- .../PostgresDockerRunner.kt | 2 +- .../PostgresEmbeddedRunner.kt | 2 +- .../PostgresRunner.kt | 2 +- .../PostgresRunnerFactory.kt | 2 +- .../kotlin/io/tolgee/websocket/ModifiedKey.kt | 9 - .../tolgee/websocket/RedisPubSubReceiver.kt | 17 - ...bsocketRedisPubSubReceiverConfiguration.kt | 35 -- .../app/src/main/resources/application.yaml | 2 + .../test/kotlin/io/tolgee/HealthCheckTest.kt | 7 +- .../src/test/kotlin/io/tolgee/PatAuthTest.kt | 2 - .../controllers/ProjectStatsControllerTest.kt | 1 + .../batch/BatchJobManagementControllerTest.kt | 265 ++++++++++++ .../batch/StartBatchJobControllerTest.kt | 132 ++++++ .../batch/AbstractBatchJobsGeneralTest.kt | 384 ++++++++++++++++++ .../batch/BatchJobsGeneralWithRedisTest.kt | 88 ++++ .../batch/BatchJobsGeneralWithoutRedisTest.kt | 11 + .../io/tolgee/cache/CacheWithRedisTest.kt | 2 + .../io/tolgee/cache/CacheWithoutRedisTest.kt | 2 + .../tolgee/websocket/AbstractWebsocketTest.kt | 44 +- .../tolgee/websocket/WebsocketTestHelper.kt | 61 ++- .../websocket/WebsocketWithRedisTest.kt | 2 - .../websocket/WebsocketWithoutRedisTest.kt | 2 - .../app/src/test/resources/application.yaml | 17 +- backend/data/build.gradle | 7 + .../io/tolgee/activity/ActivityHolder.kt | 29 +- .../io/tolgee/activity/ActivityService.kt | 8 +- .../io/tolgee/activity/data/ActivityType.kt | 3 +- .../iterceptor/ActivityInterceptor.kt | 17 + .../iterceptor/InterceptedEventsManager.kt | 10 +- .../io/tolgee/batch/AtomicProgressState.kt | 87 ++++ .../io/tolgee/batch/BatchJobActionService.kt | 149 +++++++ .../tolgee/batch/BatchJobActivityFinalizer.kt | 153 +++++++ .../batch/BatchJobCancellationManager.kt | 79 ++++ .../batch/BatchJobConcurrentLauncher.kt | 153 +++++++ .../kotlin/io/tolgee/batch/BatchJobDto.kt | 35 ++ .../kotlin/io/tolgee/batch/BatchJobService.kt | 157 +++++++ .../kotlin/io/tolgee/batch/BatchJobType.kt | 30 ++ .../io/tolgee/batch/CachingBatchJobService.kt | 59 +++ .../io/tolgee/batch/ChunkProcessingUtil.kt | 163 ++++++++ .../kotlin/io/tolgee/batch/ChunkProcessor.kt | 11 + .../io/tolgee/batch/ExecutionQueueItem.kt | 7 + .../kotlin/io/tolgee/batch/JobCancelEvent.kt | 5 + .../io/tolgee/batch/JobChunkExecutionQueue.kt | 118 ++++++ .../io/tolgee/batch/JobQueueItemsEvent.kt | 6 + .../io/tolgee/batch/OnBatchJobCompleted.kt | 5 + .../kotlin/io/tolgee/batch/ProgressManager.kt | 177 ++++++++ .../kotlin/io/tolgee/batch/QueueEventType.kt | 5 + .../io/tolgee/batch/WebsocketProgressInfo.kt | 11 + .../batch/events/OnBatchJobCancelled.kt | 8 + .../tolgee/batch/events/OnBatchJobFailed.kt | 10 + .../tolgee/batch/events/OnBatchJobProgress.kt | 9 + .../batch/events/OnBatchJobStatusUpdated.kt | 5 + .../batch/events/OnBatchJobSucceeded.kt | 8 + .../main/kotlin/io/tolgee/batch/exceptions.kt | 26 ++ .../processors/DeleteKeysChunkProcessor.kt | 43 ++ .../processors/TranslationChunkProcessor.kt | 76 ++++ .../batch/request/BatchTranslateRequest.kt | 23 ++ .../tolgee/batch/request/DeleteKeysRequest.kt | 8 + .../batch/state/BatchJobStateProvider.kt | 98 +++++ .../io/tolgee/batch/state/ExecutionState.kt | 11 + .../component/AutoTranslationListener.kt | 19 +- .../io/tolgee/component/LockingProvider.kt | 10 +- .../io/tolgee/component/UsingRedisProvider.kt | 20 + .../atomicLong/AtomicLongProvider.kt | 42 ++ .../atomicLong/MemoryTolgeeAtomicLong.kt | 28 ++ .../atomicLong/RedisTolgeeAtomicLong.kt | 20 + .../eventListeners/LanguageStatsListener.kt | 8 +- .../StoreProjectActivityListener.kt | 2 +- .../configuration/ActivityHolderConfig.kt | 4 +- .../io/tolgee/configuration/DateConverter.kt | 17 - .../configuration/tolgee/BatchProperties.kt | 10 + .../tolgee/RabbitmqAutostartProperties.kt | 27 ++ .../configuration/tolgee/TolgeeProperties.kt | 6 + .../main/kotlin/io/tolgee/constants/Caches.kt | 1 + .../kotlin/io/tolgee/constants/Message.kt | 4 +- .../testDataBuilder/data/BatchJobsTestData.kt | 32 ++ .../tolgee/dtos/cacheable/UserAccountDto.kt | 8 +- .../tolgee/events/OnProjectActivityEvent.kt | 10 +- .../io/tolgee/exceptions/ErrorException.kt | 18 +- .../tolgee/exceptions/ExceptionWithMessage.kt | 23 ++ .../main/kotlin/io/tolgee/model/Permission.kt | 6 + .../tolgee/model/activity/ActivityRevision.kt | 30 ++ .../kotlin/io/tolgee/model/batch/BatchJob.kt | 55 +++ .../model/batch/BatchJobChunkExecution.kt | 56 +++ .../batch/BatchJobChunkExecutionStatus.kt | 10 + .../io/tolgee/model/batch/BatchJobStatus.kt | 9 + .../kotlin/io/tolgee/model/batch/IBatchJob.kt | 6 + .../tolgee/model/batch/TranslateJobParams.kt | 28 ++ .../kotlin/io/tolgee/model/enums/Scope.kt | 17 +- .../model/translation/TranslationComment.kt | 3 +- .../io/tolgee/model/views/BatchJobView.kt | 10 + .../model/views/JobErrorMessagesView.kt | 10 + .../io/tolgee/pubSub/RedisPubSubReceiver.kt | 32 ++ .../RedisPubSubReceiverConfiguration.kt | 63 +++ .../tolgee/repository/BatchJobRepository.kt | 49 +++ .../io/tolgee/repository/KeyRepository.kt | 18 + .../KeyScreenshotReferenceRepository.kt | 1 + .../repository/TranslationRepository.kt | 8 +- .../activity/ActivityRevisionRepository.kt | 2 +- .../TranslationCommentRepository.kt | 2 + .../tolgee/service/bigMeta/BigMetaService.kt | 4 +- .../io/tolgee/service/key/KeyService.kt | 9 +- .../service/project/LanguageStatsService.kt | 10 +- .../service/security/SecurityService.kt | 22 +- .../translation/AutoTranslationService.kt | 43 +- .../translation/TranslationCommentService.kt | 4 + .../service/translation/TranslationService.kt | 4 +- .../kotlin/io/tolgee/util/TolgeeAtomicLong.kt | 9 + .../kotlin/io/tolgee/util/transactionUtil.kt | 47 ++- .../kotlin/io/tolgee/websocket/ActorInfo.kt | 0 .../kotlin/io/tolgee/websocket/ActorType.kt | 0 .../websocket/RedisWebsocketEventWrapper.kt | 0 .../io/tolgee/websocket/WebsocketEvent.kt | 2 +- .../websocket/WebsocketEventPublisher.kt | 0 .../tolgee/websocket/WebsocketEventType.kt} | 5 +- .../main/resources/db/changelog/schema.xml | 128 ++++++ backend/testing/build.gradle | 1 + .../kotlin/io/tolgee/AbstractSpringTest.kt | 23 +- .../kotlin/io/tolgee/CleanDbTestListener.kt | 33 +- .../tolgee/testing/AbstractControllerTest.kt | 2 + .../testing/AbstractTransactionalTest.kt | 4 +- .../kotlin/io/tolgee/testing/WebsocketTest.kt | 6 + .../tests/src/test/resources/application.yaml | 4 + settings.gradle | 2 +- 151 files changed, 4099 insertions(+), 313 deletions(-) rename backend/{app => api}/src/main/kotlin/io/tolgee/api/v2/controllers/BigMetaController.kt (100%) create mode 100644 backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobManagementController.kt create mode 100644 backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobController.kt rename backend/{app => api}/src/main/kotlin/io/tolgee/api/v2/controllers/configurationProps/ConfigurationDocumentationProvider.kt (100%) rename backend/{app => api}/src/main/kotlin/io/tolgee/api/v2/controllers/configurationProps/ConfigurationPropsController.kt (100%) rename backend/{app => api}/src/main/kotlin/io/tolgee/api/v2/controllers/configurationProps/data.kt (100%) create mode 100644 backend/api/src/main/kotlin/io/tolgee/hateoas/batch/BatchJobModel.kt create mode 100644 backend/api/src/main/kotlin/io/tolgee/hateoas/batch/BatchJobModelAssembler.kt rename backend/{app => api}/src/main/kotlin/io/tolgee/websocket/ActivityWebsocketListener.kt (58%) rename backend/{app => api}/src/main/kotlin/io/tolgee/websocket/RedisWebsocketEventPublisher.kt (100%) rename backend/{app => api}/src/main/kotlin/io/tolgee/websocket/SimpleWebsocketEventPublisher.kt (100%) rename backend/{app => api}/src/main/kotlin/io/tolgee/websocket/WebSocketConfig.kt (100%) rename backend/{app => api}/src/main/kotlin/io/tolgee/websocket/WebsocketPublisherConfiguration.kt (100%) rename backend/app/src/main/kotlin/io/tolgee/{postgresStarters => postgresRunners}/PostgresDockerRunner.kt (98%) rename backend/app/src/main/kotlin/io/tolgee/{postgresStarters => postgresRunners}/PostgresEmbeddedRunner.kt (99%) rename backend/app/src/main/kotlin/io/tolgee/{postgresStarters => postgresRunners}/PostgresRunner.kt (66%) rename backend/app/src/main/kotlin/io/tolgee/{postgresStarters => postgresRunners}/PostgresRunnerFactory.kt (96%) delete mode 100644 backend/app/src/main/kotlin/io/tolgee/websocket/ModifiedKey.kt delete mode 100644 backend/app/src/main/kotlin/io/tolgee/websocket/RedisPubSubReceiver.kt delete mode 100644 backend/app/src/main/kotlin/io/tolgee/websocket/WebsocketRedisPubSubReceiverConfiguration.kt create mode 100644 backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobManagementControllerTest.kt create mode 100644 backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt create mode 100644 backend/app/src/test/kotlin/io/tolgee/batch/AbstractBatchJobsGeneralTest.kt create mode 100644 backend/app/src/test/kotlin/io/tolgee/batch/BatchJobsGeneralWithRedisTest.kt create mode 100644 backend/app/src/test/kotlin/io/tolgee/batch/BatchJobsGeneralWithoutRedisTest.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/AtomicProgressState.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActivityFinalizer.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/BatchJobCancellationManager.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/BatchJobConcurrentLauncher.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/BatchJobDto.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/BatchJobType.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/CachingBatchJobService.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/ChunkProcessingUtil.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/ChunkProcessor.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/ExecutionQueueItem.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/JobCancelEvent.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/JobChunkExecutionQueue.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/JobQueueItemsEvent.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/OnBatchJobCompleted.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/ProgressManager.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/QueueEventType.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/WebsocketProgressInfo.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/events/OnBatchJobCancelled.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/events/OnBatchJobFailed.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/events/OnBatchJobProgress.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/events/OnBatchJobStatusUpdated.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/events/OnBatchJobSucceeded.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/exceptions.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/processors/DeleteKeysChunkProcessor.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/processors/TranslationChunkProcessor.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/request/BatchTranslateRequest.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/request/DeleteKeysRequest.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/state/BatchJobStateProvider.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/state/ExecutionState.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/component/UsingRedisProvider.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/component/atomicLong/AtomicLongProvider.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/component/atomicLong/MemoryTolgeeAtomicLong.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/component/atomicLong/RedisTolgeeAtomicLong.kt delete mode 100644 backend/data/src/main/kotlin/io/tolgee/configuration/DateConverter.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/BatchProperties.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/RabbitmqAutostartProperties.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BatchJobsTestData.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/exceptions/ExceptionWithMessage.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJob.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobChunkExecution.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobChunkExecutionStatus.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobStatus.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/model/batch/IBatchJob.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/model/batch/TranslateJobParams.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/model/views/BatchJobView.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/model/views/JobErrorMessagesView.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/pubSub/RedisPubSubReceiver.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/pubSub/RedisPubSubReceiverConfiguration.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/repository/BatchJobRepository.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/util/TolgeeAtomicLong.kt rename backend/{app => data}/src/main/kotlin/io/tolgee/websocket/ActorInfo.kt (100%) rename backend/{app => data}/src/main/kotlin/io/tolgee/websocket/ActorType.kt (100%) rename backend/{app => data}/src/main/kotlin/io/tolgee/websocket/RedisWebsocketEventWrapper.kt (100%) rename backend/{app => data}/src/main/kotlin/io/tolgee/websocket/WebsocketEvent.kt (89%) rename backend/{app => data}/src/main/kotlin/io/tolgee/websocket/WebsocketEventPublisher.kt (100%) rename backend/{app/src/main/kotlin/io/tolgee/websocket/Types.kt => data/src/main/kotlin/io/tolgee/websocket/WebsocketEventType.kt} (51%) create mode 100644 backend/testing/src/main/kotlin/io/tolgee/testing/WebsocketTest.kt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bf4c0ba2f7..cc2e5b82fa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,7 +56,7 @@ jobs: ~/backend-development.tgz backend-test: - name: Backend testing ‍🔎 + name: BT ‍🔎 needs: [backend-build] runs-on: ubuntu-latest strategy: @@ -66,6 +66,7 @@ jobs: [ "server-app:runContextRecreatingTests", "server-app:runStandardTests", + "server-app:runWebsocketTests", "ee-test:test", "data:test", ] diff --git a/backend/api/build.gradle b/backend/api/build.gradle index 119628f00b..788f8d2cc8 100644 --- a/backend/api/build.gradle +++ b/backend/api/build.gradle @@ -44,6 +44,7 @@ dependencies { implementation "org.springframework.boot:spring-boot-starter-hateoas" implementation 'org.springframework.boot:spring-boot-starter-web' implementation("org.springframework.boot:spring-boot-starter-security") + implementation "org.springframework.boot:spring-boot-starter-websocket" implementation(project(':data')) implementation(project(':misc')) diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/AutoTranslationController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/AutoTranslationController.kt index 1da0addbc1..2df8652660 100644 --- a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/AutoTranslationController.kt +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/AutoTranslationController.kt @@ -73,7 +73,7 @@ When no languages provided, it translates only untranslated languages.""" autoTranslationService.autoTranslate( key = key, - languageTags = languages, + languageTags = languages?.toList(), useTranslationMemory = useTranslationMemory ?: false, useMachineTranslation = useMachineTranslation ?: false ) diff --git a/backend/app/src/main/kotlin/io/tolgee/api/v2/controllers/BigMetaController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/BigMetaController.kt similarity index 100% rename from backend/app/src/main/kotlin/io/tolgee/api/v2/controllers/BigMetaController.kt rename to backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/BigMetaController.kt diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobManagementController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobManagementController.kt new file mode 100644 index 0000000000..779b335e31 --- /dev/null +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobManagementController.kt @@ -0,0 +1,100 @@ +package io.tolgee.api.v2.controllers.batch + +import io.swagger.v3.oas.annotations.Operation +import io.swagger.v3.oas.annotations.tags.Tag +import io.tolgee.batch.BatchJobCancellationManager +import io.tolgee.batch.BatchJobDto +import io.tolgee.batch.BatchJobService +import io.tolgee.hateoas.batch.BatchJobModel +import io.tolgee.hateoas.batch.BatchJobModelAssembler +import io.tolgee.model.batch.BatchJob +import io.tolgee.model.enums.Scope +import io.tolgee.model.views.BatchJobView +import io.tolgee.security.AuthenticationFacade +import io.tolgee.security.apiKeyAuth.AccessWithApiKey +import io.tolgee.security.project_auth.AccessWithAnyProjectPermission +import io.tolgee.security.project_auth.AccessWithProjectPermission +import io.tolgee.security.project_auth.ProjectHolder +import io.tolgee.service.security.SecurityService +import org.springdoc.api.annotations.ParameterObject +import org.springframework.data.domain.Pageable +import org.springframework.data.web.PagedResourcesAssembler +import org.springframework.data.web.SortDefault +import org.springframework.hateoas.PagedModel +import org.springframework.web.bind.annotation.CrossOrigin +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.PutMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController +import javax.validation.Valid + +@RestController +@CrossOrigin(origins = ["*"]) +@RequestMapping(value = ["/v2/projects/{projectId:\\d+}/", "/v2/projects/"]) +@Tag(name = "Batch job management") +@Suppress("SpringJavaInjectionPointsAutowiringInspection", "MVCPathVariableInspection") +class BatchJobManagementController( + private val batchJobCancellationManager: BatchJobCancellationManager, + private val batchJobService: BatchJobService, + private val projectHolder: ProjectHolder, + private val batchJobModelAssembler: BatchJobModelAssembler, + private val pagedResourcesAssembler: PagedResourcesAssembler, + private val authenticationFacade: AuthenticationFacade, + private val securityService: SecurityService +) { + @GetMapping(value = ["batch-jobs"]) + @AccessWithApiKey() + @AccessWithProjectPermission(Scope.BATCH_JOBS_VIEW) + @Operation(summary = "Lists all batch jobs in project") + fun list(@Valid @ParameterObject @SortDefault("id") pageable: Pageable): PagedModel { + val views = batchJobService.getViews(projectHolder.project.id, null, pageable) + return pagedResourcesAssembler.toModel(views, batchJobModelAssembler) + } + + @GetMapping(value = ["my-batch-jobs"]) + @AccessWithApiKey() + @AccessWithAnyProjectPermission() + @Operation(summary = "Lists all batch jobs in project started by current user") + fun myList(@Valid @ParameterObject @SortDefault("id") pageable: Pageable): PagedModel { + val views = batchJobService.getViews( + projectId = projectHolder.project.id, + userAccount = authenticationFacade.userAccount, + pageable = pageable + ) + return pagedResourcesAssembler.toModel(views, batchJobModelAssembler) + } + + @GetMapping(value = ["batch-jobs/{id}"]) + @AccessWithApiKey() + @AccessWithAnyProjectPermission() + @Operation(summary = "Returns the batch job") + fun get(@PathVariable id: Long): BatchJobModel { + val view = batchJobService.getView(id) + checkViewPermission(view.batchJob) + return batchJobModelAssembler.toModel(view) + } + + @PutMapping(value = ["batch-jobs/{id}/cancel"]) + @AccessWithApiKey() + @AccessWithAnyProjectPermission() + @Operation(summary = "Stops batch job (if possible)") + fun cancel(@PathVariable id: Long) { + checkCancelPermission(batchJobService.getJobDto(id)) + batchJobCancellationManager.cancel(id) + } + + private fun checkViewPermission(job: BatchJob) { + if (job.author?.id == authenticationFacade.userAccount.id) { + return + } + securityService.checkProjectPermission(projectHolder.project.id, Scope.BATCH_JOBS_VIEW) + } + + private fun checkCancelPermission(job: BatchJobDto) { + if (job.authorId == authenticationFacade.userAccount.id) { + return + } + securityService.checkProjectPermission(projectHolder.project.id, Scope.BATCH_JOBS_CANCEL) + } +} diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobController.kt new file mode 100644 index 0000000000..48b97982f8 --- /dev/null +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobController.kt @@ -0,0 +1,68 @@ +package io.tolgee.api.v2.controllers.batch + +import io.swagger.v3.oas.annotations.Operation +import io.swagger.v3.oas.annotations.tags.Tag +import io.tolgee.batch.BatchJobService +import io.tolgee.batch.BatchJobType +import io.tolgee.batch.request.BatchTranslateRequest +import io.tolgee.batch.request.DeleteKeysRequest +import io.tolgee.hateoas.batch.BatchJobModel +import io.tolgee.hateoas.batch.BatchJobModelAssembler +import io.tolgee.model.batch.BatchJob +import io.tolgee.model.enums.Scope +import io.tolgee.security.AuthenticationFacade +import io.tolgee.security.apiKeyAuth.AccessWithApiKey +import io.tolgee.security.project_auth.AccessWithProjectPermission +import io.tolgee.security.project_auth.ProjectHolder +import io.tolgee.service.security.SecurityService +import org.springframework.web.bind.annotation.CrossOrigin +import org.springframework.web.bind.annotation.PutMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController +import javax.validation.Valid + +@RestController +@CrossOrigin(origins = ["*"]) +@RequestMapping(value = ["/v2/projects/{projectId:\\d+}/start-batch-job", "/v2/projects/start-batch-job"]) +@Tag(name = "Start batch jobs") +@Suppress("SpringJavaInjectionPointsAutowiringInspection", "MVCPathVariableInspection") +class StartBatchJobController( + private val securityService: SecurityService, + private val projectHolder: ProjectHolder, + private val batchJobService: BatchJobService, + private val authenticationFacade: AuthenticationFacade, + private val batchJobModelAssembler: BatchJobModelAssembler +) { + @PutMapping(value = ["/translate"]) + @AccessWithApiKey() + @AccessWithProjectPermission(Scope.BATCH_AUTO_TRANSLATE) + @Operation(summary = "Translates provided keys to provided languages") + fun translate(@Valid @RequestBody data: BatchTranslateRequest): BatchJobModel { + securityService.checkLanguageTranslatePermission(projectHolder.project.id, data.targetLanguageIds) + securityService.checkKeyIdsExistAndIsFromProject(data.keyIds, projectHolder.project.id) + return batchJobService.startJob( + data, + projectHolder.projectEntity, + authenticationFacade.userAccountEntity, + BatchJobType.TRANSLATION + ).model + } + + @PutMapping(value = ["/delete-keys"]) + @AccessWithApiKey() + @AccessWithProjectPermission(Scope.KEYS_DELETE) + @Operation(summary = "Translates provided keys to provided languages") + fun deleteKeys(@Valid @RequestBody data: DeleteKeysRequest): BatchJobModel { + securityService.checkKeyIdsExistAndIsFromProject(data.keyIds, projectHolder.project.id) + return batchJobService.startJob( + data, + projectHolder.projectEntity, + authenticationFacade.userAccountEntity, + BatchJobType.DELETE_KEYS + ).model + } + + val BatchJob.model + get() = batchJobModelAssembler.toModel(batchJobService.getView(this)) +} diff --git a/backend/app/src/main/kotlin/io/tolgee/api/v2/controllers/configurationProps/ConfigurationDocumentationProvider.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/configurationProps/ConfigurationDocumentationProvider.kt similarity index 100% rename from backend/app/src/main/kotlin/io/tolgee/api/v2/controllers/configurationProps/ConfigurationDocumentationProvider.kt rename to backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/configurationProps/ConfigurationDocumentationProvider.kt diff --git a/backend/app/src/main/kotlin/io/tolgee/api/v2/controllers/configurationProps/ConfigurationPropsController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/configurationProps/ConfigurationPropsController.kt similarity index 100% rename from backend/app/src/main/kotlin/io/tolgee/api/v2/controllers/configurationProps/ConfigurationPropsController.kt rename to backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/configurationProps/ConfigurationPropsController.kt diff --git a/backend/app/src/main/kotlin/io/tolgee/api/v2/controllers/configurationProps/data.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/configurationProps/data.kt similarity index 100% rename from backend/app/src/main/kotlin/io/tolgee/api/v2/controllers/configurationProps/data.kt rename to backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/configurationProps/data.kt diff --git a/backend/api/src/main/kotlin/io/tolgee/component/ProjectTranslationLastModifiedManager.kt b/backend/api/src/main/kotlin/io/tolgee/component/ProjectTranslationLastModifiedManager.kt index 29d31aaa9d..02457a4bbd 100644 --- a/backend/api/src/main/kotlin/io/tolgee/component/ProjectTranslationLastModifiedManager.kt +++ b/backend/api/src/main/kotlin/io/tolgee/component/ProjectTranslationLastModifiedManager.kt @@ -26,7 +26,7 @@ class ProjectTranslationLastModifiedManager( @EventListener fun onActivity(event: OnProjectActivityEvent) { - event.activityHolder.activityRevision?.projectId?.let { projectId -> + event.activityRevision.projectId?.let { projectId -> getCache()?.put(projectId, currentDateProvider.date.time) } } diff --git a/backend/api/src/main/kotlin/io/tolgee/component/lockingProvider/RedissonLockingProvider.kt b/backend/api/src/main/kotlin/io/tolgee/component/lockingProvider/RedissonLockingProvider.kt index e41e20f76d..d0331d235e 100644 --- a/backend/api/src/main/kotlin/io/tolgee/component/lockingProvider/RedissonLockingProvider.kt +++ b/backend/api/src/main/kotlin/io/tolgee/component/lockingProvider/RedissonLockingProvider.kt @@ -1,11 +1,23 @@ package io.tolgee.component.lockingProvider import io.tolgee.component.LockingProvider +import org.redisson.api.RLock import org.redisson.api.RedissonClient -import java.util.concurrent.locks.Lock class RedissonLockingProvider(private val redissonClient: RedissonClient) : LockingProvider { - override fun getLock(name: String): Lock { + override fun getLock(name: String): RLock { return redissonClient.getLock(name) } + + override fun withLocking(name: String, fn: () -> T): T { + val lock = this.getLock(name) + lock.lock() + try { + return fn() + } finally { + if (lock.isHeldByCurrentThread) { + lock.unlock() + } + } + } } diff --git a/backend/api/src/main/kotlin/io/tolgee/component/lockingProvider/SimpleLockingProvider.kt b/backend/api/src/main/kotlin/io/tolgee/component/lockingProvider/SimpleLockingProvider.kt index 3e2b91714b..ce8f4067fe 100644 --- a/backend/api/src/main/kotlin/io/tolgee/component/lockingProvider/SimpleLockingProvider.kt +++ b/backend/api/src/main/kotlin/io/tolgee/component/lockingProvider/SimpleLockingProvider.kt @@ -15,4 +15,14 @@ class SimpleLockingProvider : LockingProvider { override fun getLock(name: String): Lock { return map.getOrPut(name) { ReentrantLock() } } + + override fun withLocking(name: String, fn: () -> T): T { + val lock = this.getLock(name) + lock.lock() + try { + return fn() + } finally { + lock.unlock() + } + } } diff --git a/backend/api/src/main/kotlin/io/tolgee/configuration/RedisLockingConfiguration.kt b/backend/api/src/main/kotlin/io/tolgee/configuration/RedisLockingConfiguration.kt index 4a857d1e97..d72a475ffd 100644 --- a/backend/api/src/main/kotlin/io/tolgee/configuration/RedisLockingConfiguration.kt +++ b/backend/api/src/main/kotlin/io/tolgee/configuration/RedisLockingConfiguration.kt @@ -6,7 +6,7 @@ import org.redisson.Redisson import org.redisson.api.RedissonClient import org.springframework.boot.autoconfigure.AutoConfigureAfter import org.springframework.boot.autoconfigure.condition.ConditionalOnClass -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.data.redis.core.RedisOperations @@ -14,7 +14,7 @@ import org.springframework.data.redis.core.RedisOperations @Configuration @ConditionalOnClass(Redisson::class, RedisOperations::class) @AutoConfigureAfter(ConditionalRedissonAutoConfiguration::class) -@ConditionalOnProperty(name = ["tolgee.cache.use-redis"], havingValue = "true") +@ConditionalOnExpression("\${tolgee.cache.use-redis:false} and \${tolgee.cache.enabled:false}") class RedisLockingConfiguration( val redissonClient: RedissonClient ) { diff --git a/backend/api/src/main/kotlin/io/tolgee/configuration/RedissonCacheConfiguration.kt b/backend/api/src/main/kotlin/io/tolgee/configuration/RedissonCacheConfiguration.kt index 14450a2743..c702b7b574 100644 --- a/backend/api/src/main/kotlin/io/tolgee/configuration/RedissonCacheConfiguration.kt +++ b/backend/api/src/main/kotlin/io/tolgee/configuration/RedissonCacheConfiguration.kt @@ -8,7 +8,7 @@ import org.redisson.spring.cache.CacheConfig import org.redisson.spring.cache.RedissonSpringCacheManager import org.springframework.boot.autoconfigure.AutoConfigureAfter import org.springframework.boot.autoconfigure.condition.ConditionalOnClass -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.cache.CacheManager import org.springframework.cache.annotation.EnableCaching import org.springframework.context.annotation.Bean @@ -19,7 +19,7 @@ import org.springframework.data.redis.core.RedisOperations @EnableCaching @ConditionalOnClass(Redisson::class, RedisOperations::class) @AutoConfigureAfter(ConditionalRedissonAutoConfiguration::class) -@ConditionalOnProperty(name = ["tolgee.cache.use-redis"], havingValue = "true") +@ConditionalOnExpression("\${tolgee.cache.use-redis:false} and \${tolgee.cache.enabled:false}") class RedissonCacheConfiguration(private val tolgeeProperties: TolgeeProperties) { @Bean fun cacheManager(redissonClient: RedissonClient): CacheManager? { diff --git a/backend/api/src/main/kotlin/io/tolgee/hateoas/batch/BatchJobModel.kt b/backend/api/src/main/kotlin/io/tolgee/hateoas/batch/BatchJobModel.kt new file mode 100644 index 0000000000..cb4e4bce30 --- /dev/null +++ b/backend/api/src/main/kotlin/io/tolgee/hateoas/batch/BatchJobModel.kt @@ -0,0 +1,40 @@ +package io.tolgee.hateoas.batch + +import io.swagger.v3.oas.annotations.media.Schema +import io.tolgee.batch.BatchJobType +import io.tolgee.hateoas.user_account.SimpleUserAccountModel +import io.tolgee.model.batch.BatchJobStatus +import org.springframework.hateoas.RepresentationModel +import org.springframework.hateoas.server.core.Relation +import java.io.Serializable + +@Suppress("unused") +@Relation(collectionRelation = "batchJobs", itemRelation = "batchJob") +open class BatchJobModel( + @Schema(description = "Batch job id") + val id: Long, + + @Schema(description = "Status of the batch job") + val status: BatchJobStatus, + + @Schema(description = "Type of the batch job") + val type: BatchJobType, + + @Schema(description = "Total items, that have been processed so far") + val progress: Int, + + @Schema(description = "Total items") + val totalItems: Int, + + @Schema(description = "The user who started the job") + val author: SimpleUserAccountModel?, + + @Schema(description = "The time when the job created") + val createdAt: String, + + @Schema(description = "The activity revision id, that stores the activity details of the job") + val activityRevisionId: Long?, + + @Schema(description = "If the job failed, this is the error message") + val errorMessage: String?, +) : RepresentationModel(), Serializable diff --git a/backend/api/src/main/kotlin/io/tolgee/hateoas/batch/BatchJobModelAssembler.kt b/backend/api/src/main/kotlin/io/tolgee/hateoas/batch/BatchJobModelAssembler.kt new file mode 100644 index 0000000000..7f5ecf4a47 --- /dev/null +++ b/backend/api/src/main/kotlin/io/tolgee/hateoas/batch/BatchJobModelAssembler.kt @@ -0,0 +1,28 @@ +package io.tolgee.hateoas.batch + +import io.tolgee.api.v2.controllers.batch.BatchJobManagementController +import io.tolgee.hateoas.user_account.SimpleUserAccountModelAssembler +import io.tolgee.model.views.BatchJobView +import org.springframework.hateoas.server.mvc.RepresentationModelAssemblerSupport +import org.springframework.stereotype.Component + +@Component +class BatchJobModelAssembler( + val simpleUserAccountModelAssembler: SimpleUserAccountModelAssembler +) : RepresentationModelAssemblerSupport( + BatchJobManagementController::class.java, BatchJobModel::class.java +) { + override fun toModel(view: BatchJobView): BatchJobModel { + return BatchJobModel( + id = view.batchJob.id, + type = view.batchJob.type, + status = view.batchJob.status, + progress = view.progress, + totalItems = view.batchJob.totalItems, + author = view.batchJob.author?.let { simpleUserAccountModelAssembler.toModel(it) }, + createdAt = view.batchJob.createdAt.toString(), + activityRevisionId = view.batchJob.activityRevision?.id, + errorMessage = view.errorMessage?.code + ) + } +} diff --git a/backend/api/src/main/kotlin/io/tolgee/hateoas/user_account/SimpleUserAccountModelAssembler.kt b/backend/api/src/main/kotlin/io/tolgee/hateoas/user_account/SimpleUserAccountModelAssembler.kt index 5b4c668a6c..97a04929c4 100644 --- a/backend/api/src/main/kotlin/io/tolgee/hateoas/user_account/SimpleUserAccountModelAssembler.kt +++ b/backend/api/src/main/kotlin/io/tolgee/hateoas/user_account/SimpleUserAccountModelAssembler.kt @@ -1,6 +1,7 @@ package io.tolgee.hateoas.user_account import io.tolgee.api.v2.controllers.V2UserController +import io.tolgee.dtos.cacheable.UserAccountDto import io.tolgee.model.UserAccount import io.tolgee.service.AvatarService import org.springframework.hateoas.server.mvc.RepresentationModelAssemblerSupport @@ -23,4 +24,16 @@ class SimpleUserAccountModelAssembler( deleted = entity.deletedAt != null ) } + + fun toModel(dto: UserAccountDto): SimpleUserAccountModel { + val avatar = avatarService.getAvatarLinks(dto.avatarHash) + + return SimpleUserAccountModel( + id = dto.id, + username = dto.username, + name = dto.name, + avatar = avatar, + deleted = dto.deleted + ) + } } diff --git a/backend/app/src/main/kotlin/io/tolgee/websocket/ActivityWebsocketListener.kt b/backend/api/src/main/kotlin/io/tolgee/websocket/ActivityWebsocketListener.kt similarity index 58% rename from backend/app/src/main/kotlin/io/tolgee/websocket/ActivityWebsocketListener.kt rename to backend/api/src/main/kotlin/io/tolgee/websocket/ActivityWebsocketListener.kt index 7b01218637..52a8ef5835 100644 --- a/backend/app/src/main/kotlin/io/tolgee/websocket/ActivityWebsocketListener.kt +++ b/backend/api/src/main/kotlin/io/tolgee/websocket/ActivityWebsocketListener.kt @@ -1,16 +1,25 @@ package io.tolgee.websocket import io.tolgee.activity.projectActivityView.RelationDescriptionExtractor +import io.tolgee.batch.OnBatchJobCompleted +import io.tolgee.batch.WebsocketProgressInfo +import io.tolgee.batch.events.OnBatchJobCancelled +import io.tolgee.batch.events.OnBatchJobFailed +import io.tolgee.batch.events.OnBatchJobProgress +import io.tolgee.batch.events.OnBatchJobSucceeded +import io.tolgee.constants.Message import io.tolgee.events.OnProjectActivityStoredEvent import io.tolgee.hateoas.user_account.SimpleUserAccountModelAssembler import io.tolgee.model.activity.ActivityModifiedEntity import io.tolgee.model.activity.ActivityRevision +import io.tolgee.model.batch.BatchJobStatus import io.tolgee.model.key.Key import io.tolgee.model.translation.Translation import io.tolgee.service.security.UserAccountService import org.springframework.context.event.EventListener import org.springframework.scheduling.annotation.Async import org.springframework.stereotype.Component +import org.springframework.transaction.event.TransactionalEventListener @Component class ActivityWebsocketListener( @@ -39,7 +48,7 @@ class ActivityWebsocketListener( fun getActorInfo(userId: Long?): ActorInfo { return userId?.let { - val user = userAccountService.findActive(userId) ?: return@let null + val user = userAccountService.findDto(userId) ?: return@let null ActorInfo( type = ActorType.USER, data = simpleUserAccountModelAssembler.toModel(user) @@ -63,7 +72,7 @@ class ActivityWebsocketListener( } else null websocketEventPublisher( - "/projects/${activityRevision.projectId!!}/${Types.TRANSLATION_DATA_MODIFIED.typeName}", + "/projects/${activityRevision.projectId!!}/${WebsocketEventType.TRANSLATION_DATA_MODIFIED.typeName}", WebsocketEvent( actor = getActorInfo(activityRevision.authorId), data = data, @@ -74,6 +83,53 @@ class ActivityWebsocketListener( ) } + @EventListener(OnBatchJobProgress::class) + fun onBatchJobProgress(event: OnBatchJobProgress) { + val realStatus = if (event.job.status == BatchJobStatus.PENDING) + BatchJobStatus.RUNNING + else + event.job.status + + websocketEventPublisher( + "/projects/${event.job.projectId}/${WebsocketEventType.BATCH_JOB_PROGRESS.typeName}", + WebsocketEvent( + actor = getActorInfo(event.job.authorId), + data = WebsocketProgressInfo(event.job.id, event.processed, event.total, realStatus), + sourceActivity = null, + activityId = null, + dataCollapsed = false + ) + ) + } + + @TransactionalEventListener(OnBatchJobSucceeded::class) + fun onBatchJobSucceeded(event: OnBatchJobSucceeded) { + onBatchJobCompleted(event) + } + + @TransactionalEventListener(OnBatchJobFailed::class) + fun onBatchJobFailed(event: OnBatchJobFailed) { + onBatchJobCompleted(event, event.errorMessage) + } + + @TransactionalEventListener(OnBatchJobCancelled::class) + fun onBatchJobCancelled(event: OnBatchJobCancelled) { + onBatchJobCompleted(event) + } + + fun onBatchJobCompleted(event: OnBatchJobCompleted, errorMessage: Message? = null) { + websocketEventPublisher( + "/projects/${event.job.projectId}/${WebsocketEventType.BATCH_JOB_PROGRESS.typeName}", + WebsocketEvent( + actor = getActorInfo(event.job.authorId), + data = WebsocketProgressInfo(event.job.id, null, null, event.job.status, errorMessage?.code), + sourceActivity = null, + activityId = null, + dataCollapsed = false + ) + ) + } + private fun getModifiedEntityView(it: ActivityModifiedEntity): MutableMap { val data = mutableMapOf("id" to it.entityId) it.describingData?.let { describingData -> data.putAll(describingData) } diff --git a/backend/app/src/main/kotlin/io/tolgee/websocket/RedisWebsocketEventPublisher.kt b/backend/api/src/main/kotlin/io/tolgee/websocket/RedisWebsocketEventPublisher.kt similarity index 100% rename from backend/app/src/main/kotlin/io/tolgee/websocket/RedisWebsocketEventPublisher.kt rename to backend/api/src/main/kotlin/io/tolgee/websocket/RedisWebsocketEventPublisher.kt diff --git a/backend/app/src/main/kotlin/io/tolgee/websocket/SimpleWebsocketEventPublisher.kt b/backend/api/src/main/kotlin/io/tolgee/websocket/SimpleWebsocketEventPublisher.kt similarity index 100% rename from backend/app/src/main/kotlin/io/tolgee/websocket/SimpleWebsocketEventPublisher.kt rename to backend/api/src/main/kotlin/io/tolgee/websocket/SimpleWebsocketEventPublisher.kt diff --git a/backend/app/src/main/kotlin/io/tolgee/websocket/WebSocketConfig.kt b/backend/api/src/main/kotlin/io/tolgee/websocket/WebSocketConfig.kt similarity index 100% rename from backend/app/src/main/kotlin/io/tolgee/websocket/WebSocketConfig.kt rename to backend/api/src/main/kotlin/io/tolgee/websocket/WebSocketConfig.kt diff --git a/backend/app/src/main/kotlin/io/tolgee/websocket/WebsocketPublisherConfiguration.kt b/backend/api/src/main/kotlin/io/tolgee/websocket/WebsocketPublisherConfiguration.kt similarity index 100% rename from backend/app/src/main/kotlin/io/tolgee/websocket/WebsocketPublisherConfiguration.kt rename to backend/api/src/main/kotlin/io/tolgee/websocket/WebsocketPublisherConfiguration.kt diff --git a/backend/app/build.gradle b/backend/app/build.gradle index 09c4b466b5..4896f41fdd 100644 --- a/backend/app/build.gradle +++ b/backend/app/build.gradle @@ -79,7 +79,7 @@ dependencies { implementation("org.springframework.ldap:spring-ldap-core") implementation("org.springframework.security:spring-security-ldap") implementation "org.springframework.boot:spring-boot-starter-batch" - implementation "org.springframework.boot:spring-boot-starter-websocket" + implementation "org.springframework.boot:spring-boot-starter-actuator" /** * TESTING @@ -95,6 +95,7 @@ dependencies { testImplementation("io.socket:socket.io-client:1.0.1") testImplementation group: 'org.springframework.batch', name: 'spring-batch-test', version: '4.3.5' testImplementation libs.sendInBlue + testImplementation "org.springframework.boot:spring-boot-starter-websocket" /** * MISC @@ -105,8 +106,9 @@ dependencies { implementation libs.amazonSTS implementation libs.icu4j implementation libs.jacksonModuleKotlin - implementation libs.redissonSpringBootStarter - implementation libs.redissonSpringData + + testApi libs.redissonSpringBootStarter + testApi libs.redissonSpringData /** * KOTLIN @@ -179,7 +181,13 @@ task runContextRecreatingTests(type: Test, group: 'verification') { task runStandardTests(type: Test, group: 'verification') { useJUnitPlatform { - excludeTags "contextRecreating" + excludeTags "contextRecreating", "websocket" + } +} + +task runWebsocketTests(type: Test, group: 'verification') { + useJUnitPlatform { + includeTags "websocket" } maxHeapSize = "8000m" } diff --git a/backend/app/src/main/kotlin/io/tolgee/Application.kt b/backend/app/src/main/kotlin/io/tolgee/Application.kt index 53f90bbeb3..d7c1577690 100644 --- a/backend/app/src/main/kotlin/io/tolgee/Application.kt +++ b/backend/app/src/main/kotlin/io/tolgee/Application.kt @@ -1,43 +1,17 @@ package io.tolgee import io.tolgee.configuration.Banner -import org.redisson.spring.starter.RedissonAutoConfiguration import org.springframework.boot.SpringApplication -import org.springframework.boot.actuate.autoconfigure.endpoint.jmx.JmxEndpointAutoConfiguration -import org.springframework.boot.actuate.autoconfigure.info.InfoContributorAutoConfiguration -import org.springframework.boot.actuate.autoconfigure.liquibase.LiquibaseEndpointAutoConfiguration -import org.springframework.boot.actuate.autoconfigure.metrics.CompositeMeterRegistryAutoConfiguration -import org.springframework.boot.actuate.autoconfigure.metrics.JvmMetricsAutoConfiguration -import org.springframework.boot.actuate.autoconfigure.metrics.MetricsEndpointAutoConfiguration -import org.springframework.boot.actuate.autoconfigure.metrics.jdbc.DataSourcePoolMetricsAutoConfiguration -import org.springframework.boot.actuate.autoconfigure.metrics.startup.StartupTimeMetricsListenerAutoConfiguration -import org.springframework.boot.actuate.autoconfigure.metrics.web.tomcat.TomcatMetricsAutoConfiguration -import org.springframework.boot.actuate.autoconfigure.system.DiskSpaceHealthContributorAutoConfiguration import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.boot.autoconfigure.domain.EntityScan -import org.springframework.boot.autoconfigure.jmx.JmxAutoConfiguration import org.springframework.boot.autoconfigure.ldap.LdapAutoConfiguration import org.springframework.boot.context.properties.ConfigurationPropertiesScan import org.springframework.data.jpa.repository.config.EnableJpaAuditing import org.springframework.data.jpa.repository.config.EnableJpaRepositories @SpringBootApplication( - exclude = [ - RedissonAutoConfiguration::class, - CompositeMeterRegistryAutoConfiguration::class, - DataSourcePoolMetricsAutoConfiguration::class, - DiskSpaceHealthContributorAutoConfiguration::class, - InfoContributorAutoConfiguration::class, - JmxAutoConfiguration::class, - JvmMetricsAutoConfiguration::class, - JmxEndpointAutoConfiguration::class, - LdapAutoConfiguration::class, - LiquibaseEndpointAutoConfiguration::class, - MetricsEndpointAutoConfiguration::class, - StartupTimeMetricsListenerAutoConfiguration::class, - TomcatMetricsAutoConfiguration::class, - ], - scanBasePackages = ["io.tolgee"] + scanBasePackages = ["io.tolgee"], + exclude = [LdapAutoConfiguration::class] ) @EnableJpaAuditing @EntityScan("io.tolgee.model") diff --git a/backend/app/src/main/kotlin/io/tolgee/ExceptionHandlers.kt b/backend/app/src/main/kotlin/io/tolgee/ExceptionHandlers.kt index aed0185e21..4f2c8c5c18 100644 --- a/backend/app/src/main/kotlin/io/tolgee/ExceptionHandlers.kt +++ b/backend/app/src/main/kotlin/io/tolgee/ExceptionHandlers.kt @@ -17,6 +17,7 @@ import org.springframework.dao.InvalidDataAccessApiUsageException import org.springframework.http.HttpStatus import org.springframework.http.ResponseEntity import org.springframework.http.converter.HttpMessageNotReadableException +import org.springframework.validation.BindException import org.springframework.validation.FieldError import org.springframework.validation.ObjectError import org.springframework.web.HttpRequestMethodNotSupportedException @@ -78,6 +79,24 @@ class ExceptionHandlers { ) } + @ExceptionHandler(BindException::class) + fun handleBindExceptions( + ex: BindException + ): ResponseEntity>> { + val errors: MutableMap = HashMap() + + ex.bindingResult.allErrors.forEach { error: ObjectError -> + val fieldName = (error as FieldError).field + val errorMessage = error.getDefaultMessage() + errors[fieldName] = errorMessage ?: "" + } + + return ResponseEntity( + Collections.singletonMap(ValidationErrorType.STANDARD_VALIDATION.name, errors), + HttpStatus.BAD_REQUEST + ) + } + @ExceptionHandler(MissingServletRequestParameterException::class) fun handleMissingServletRequestParameterException( ex: MissingServletRequestParameterException diff --git a/backend/app/src/main/kotlin/io/tolgee/configuration/PostgresAutoStartConfiguration.kt b/backend/app/src/main/kotlin/io/tolgee/configuration/PostgresAutoStartConfiguration.kt index 1e9e8120d3..ec89907efe 100644 --- a/backend/app/src/main/kotlin/io/tolgee/configuration/PostgresAutoStartConfiguration.kt +++ b/backend/app/src/main/kotlin/io/tolgee/configuration/PostgresAutoStartConfiguration.kt @@ -1,9 +1,10 @@ package io.tolgee.configuration import io.tolgee.configuration.tolgee.PostgresAutostartProperties -import io.tolgee.postgresStarters.PostgresRunner -import io.tolgee.postgresStarters.PostgresRunnerFactory +import io.tolgee.postgresRunners.PostgresRunner +import io.tolgee.postgresRunners.PostgresRunnerFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.boot.context.properties.ConfigurationProperties import org.springframework.boot.jdbc.DataSourceBuilder import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @@ -19,6 +20,7 @@ class PostgresAutoStartConfiguration( private var _dataSource: DataSource? = null @Bean + @ConfigurationProperties(prefix = "spring.datasource") fun getDataSource(): DataSource { _dataSource?.let { return it } postgresRunner.run() diff --git a/backend/app/src/main/kotlin/io/tolgee/configuration/SimpleLockingConfiguration.kt b/backend/app/src/main/kotlin/io/tolgee/configuration/SimpleLockingConfiguration.kt index f6e994e7eb..aea0ca0d9e 100644 --- a/backend/app/src/main/kotlin/io/tolgee/configuration/SimpleLockingConfiguration.kt +++ b/backend/app/src/main/kotlin/io/tolgee/configuration/SimpleLockingConfiguration.kt @@ -2,12 +2,12 @@ package io.tolgee.configuration import io.tolgee.component.LockingProvider import io.tolgee.component.lockingProvider.SimpleLockingProvider -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @Configuration -@ConditionalOnExpression("\${tolgee.cache.use-redis:false} == false") +@ConditionalOnMissingBean(LockingProvider::class) class SimpleLockingConfiguration { @Bean fun getLockingProvider(): LockingProvider { diff --git a/backend/app/src/main/kotlin/io/tolgee/postgresStarters/PostgresDockerRunner.kt b/backend/app/src/main/kotlin/io/tolgee/postgresRunners/PostgresDockerRunner.kt similarity index 98% rename from backend/app/src/main/kotlin/io/tolgee/postgresStarters/PostgresDockerRunner.kt rename to backend/app/src/main/kotlin/io/tolgee/postgresRunners/PostgresDockerRunner.kt index c0c813fbea..725dcedf48 100644 --- a/backend/app/src/main/kotlin/io/tolgee/postgresStarters/PostgresDockerRunner.kt +++ b/backend/app/src/main/kotlin/io/tolgee/postgresRunners/PostgresDockerRunner.kt @@ -1,4 +1,4 @@ -package io.tolgee.postgresStarters +package io.tolgee.postgresRunners import io.tolgee.configuration.tolgee.PostgresAutostartProperties import io.tolgee.misc.dockerRunner.DockerContainerRunner diff --git a/backend/app/src/main/kotlin/io/tolgee/postgresStarters/PostgresEmbeddedRunner.kt b/backend/app/src/main/kotlin/io/tolgee/postgresRunners/PostgresEmbeddedRunner.kt similarity index 99% rename from backend/app/src/main/kotlin/io/tolgee/postgresStarters/PostgresEmbeddedRunner.kt rename to backend/app/src/main/kotlin/io/tolgee/postgresRunners/PostgresEmbeddedRunner.kt index e26860d3a2..dccc52c8f4 100644 --- a/backend/app/src/main/kotlin/io/tolgee/postgresStarters/PostgresEmbeddedRunner.kt +++ b/backend/app/src/main/kotlin/io/tolgee/postgresRunners/PostgresEmbeddedRunner.kt @@ -1,4 +1,4 @@ -package io.tolgee.postgresStarters +package io.tolgee.postgresRunners import io.tolgee.configuration.tolgee.FileStorageProperties import io.tolgee.configuration.tolgee.PostgresAutostartProperties diff --git a/backend/app/src/main/kotlin/io/tolgee/postgresStarters/PostgresRunner.kt b/backend/app/src/main/kotlin/io/tolgee/postgresRunners/PostgresRunner.kt similarity index 66% rename from backend/app/src/main/kotlin/io/tolgee/postgresStarters/PostgresRunner.kt rename to backend/app/src/main/kotlin/io/tolgee/postgresRunners/PostgresRunner.kt index d259aba4f5..92ddc98f3c 100644 --- a/backend/app/src/main/kotlin/io/tolgee/postgresStarters/PostgresRunner.kt +++ b/backend/app/src/main/kotlin/io/tolgee/postgresRunners/PostgresRunner.kt @@ -1,4 +1,4 @@ -package io.tolgee.postgresStarters +package io.tolgee.postgresRunners interface PostgresRunner { fun run() diff --git a/backend/app/src/main/kotlin/io/tolgee/postgresStarters/PostgresRunnerFactory.kt b/backend/app/src/main/kotlin/io/tolgee/postgresRunners/PostgresRunnerFactory.kt similarity index 96% rename from backend/app/src/main/kotlin/io/tolgee/postgresStarters/PostgresRunnerFactory.kt rename to backend/app/src/main/kotlin/io/tolgee/postgresRunners/PostgresRunnerFactory.kt index f53918c461..aae0296d4a 100644 --- a/backend/app/src/main/kotlin/io/tolgee/postgresStarters/PostgresRunnerFactory.kt +++ b/backend/app/src/main/kotlin/io/tolgee/postgresRunners/PostgresRunnerFactory.kt @@ -1,4 +1,4 @@ -package io.tolgee.postgresStarters +package io.tolgee.postgresRunners import io.tolgee.configuration.tolgee.PostgresAutostartProperties import org.springframework.context.ApplicationContext diff --git a/backend/app/src/main/kotlin/io/tolgee/websocket/ModifiedKey.kt b/backend/app/src/main/kotlin/io/tolgee/websocket/ModifiedKey.kt deleted file mode 100644 index 5df7c98cae..0000000000 --- a/backend/app/src/main/kotlin/io/tolgee/websocket/ModifiedKey.kt +++ /dev/null @@ -1,9 +0,0 @@ -package io.tolgee.websocket - -import io.tolgee.activity.data.PropertyModification - -data class ModifiedKey( - val id: Long, - val name: String?, - val modifications: Map -) diff --git a/backend/app/src/main/kotlin/io/tolgee/websocket/RedisPubSubReceiver.kt b/backend/app/src/main/kotlin/io/tolgee/websocket/RedisPubSubReceiver.kt deleted file mode 100644 index be2022eb43..0000000000 --- a/backend/app/src/main/kotlin/io/tolgee/websocket/RedisPubSubReceiver.kt +++ /dev/null @@ -1,17 +0,0 @@ -package io.tolgee.websocket - -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import io.tolgee.util.Logging -import io.tolgee.util.logger -import org.springframework.messaging.simp.SimpMessagingTemplate - -class RedisPubSubReceiver( - private val template: SimpMessagingTemplate -) : Logging { - - fun receiveMessage(message: String) { - val data = jacksonObjectMapper().readValue(message, RedisWebsocketEventWrapper::class.java) - template.convertAndSend(data.destination, data.message) - logger.info("Sending message to ${data.destination}") - } -} diff --git a/backend/app/src/main/kotlin/io/tolgee/websocket/WebsocketRedisPubSubReceiverConfiguration.kt b/backend/app/src/main/kotlin/io/tolgee/websocket/WebsocketRedisPubSubReceiverConfiguration.kt deleted file mode 100644 index e8f26f4a3c..0000000000 --- a/backend/app/src/main/kotlin/io/tolgee/websocket/WebsocketRedisPubSubReceiverConfiguration.kt +++ /dev/null @@ -1,35 +0,0 @@ -package io.tolgee.websocket - -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty -import org.springframework.context.annotation.Bean -import org.springframework.context.annotation.Configuration -import org.springframework.data.redis.connection.RedisConnectionFactory -import org.springframework.data.redis.listener.PatternTopic -import org.springframework.data.redis.listener.RedisMessageListenerContainer -import org.springframework.data.redis.listener.adapter.MessageListenerAdapter -import org.springframework.messaging.simp.SimpMessagingTemplate - -@Configuration -@ConditionalOnProperty(name = ["tolgee.websocket.use-redis"], havingValue = "true") -class WebsocketRedisPubSubReceiverConfiguration( - private val template: SimpMessagingTemplate, - private val connectionFactory: RedisConnectionFactory -) { - @Bean - fun redisPubsubReceiver(): RedisPubSubReceiver { - return RedisPubSubReceiver(template) - } - - @Bean - fun redisPubsubListenerAdapter(): MessageListenerAdapter { - return MessageListenerAdapter(redisPubsubReceiver(), RedisPubSubReceiver::receiveMessage.name) - } - - @Bean - fun redisPubsubContainer(): RedisMessageListenerContainer { - val container = RedisMessageListenerContainer() - container.connectionFactory = connectionFactory - container.addMessageListener(redisPubsubListenerAdapter(), PatternTopic("websocket")) - return container - } -} diff --git a/backend/app/src/main/resources/application.yaml b/backend/app/src/main/resources/application.yaml index cacef47e64..a981904d04 100644 --- a/backend/app/src/main/resources/application.yaml +++ b/backend/app/src/main/resources/application.yaml @@ -1,4 +1,6 @@ spring: + autoconfigure: + exclude: org.redisson.spring.starter.RedissonAutoConfiguration, org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration data: redis: repositories: diff --git a/backend/app/src/test/kotlin/io/tolgee/HealthCheckTest.kt b/backend/app/src/test/kotlin/io/tolgee/HealthCheckTest.kt index 1662559912..c1c298a566 100644 --- a/backend/app/src/test/kotlin/io/tolgee/HealthCheckTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/HealthCheckTest.kt @@ -5,17 +5,14 @@ package io.tolgee import io.tolgee.fixtures.andIsOk +import io.tolgee.fixtures.andPrettyPrint import io.tolgee.testing.AbstractControllerTest -import io.tolgee.testing.ContextRecreatingTest import org.junit.jupiter.api.Test -import org.springframework.transaction.annotation.Transactional -@Transactional -@ContextRecreatingTest class HealthCheckTest : AbstractControllerTest() { @Test fun `health check works`() { - performGet("/actuator/health").andIsOk + performGet("/actuator/health").andPrettyPrint.andIsOk } } diff --git a/backend/app/src/test/kotlin/io/tolgee/PatAuthTest.kt b/backend/app/src/test/kotlin/io/tolgee/PatAuthTest.kt index d6549f9684..05527e3576 100644 --- a/backend/app/src/test/kotlin/io/tolgee/PatAuthTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/PatAuthTest.kt @@ -8,13 +8,11 @@ import io.tolgee.fixtures.andIsForbidden import io.tolgee.fixtures.andIsOk import io.tolgee.model.Pat import io.tolgee.testing.AbstractControllerTest -import io.tolgee.testing.ContextRecreatingTest import io.tolgee.testing.assertions.Assertions.assertThat import org.junit.jupiter.api.Test import org.springframework.http.HttpHeaders import java.util.* -@ContextRecreatingTest class PatAuthTest : AbstractControllerTest() { @Test fun `user authorizes with PAT`() { diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/ProjectStatsControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/ProjectStatsControllerTest.kt index c37da3ebc6..b72497e664 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/ProjectStatsControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/ProjectStatsControllerTest.kt @@ -105,6 +105,7 @@ class ProjectStatsControllerTest : ProjectAuthControllerTest("/v2/projects/") { isEqualTo( """ { + "2022-03-20" : 1, "2022-04-01" : 1, "2022-04-05" : 5, "2022-04-20" : 2 diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobManagementControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobManagementControllerTest.kt new file mode 100644 index 0000000000..33af835f9c --- /dev/null +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobManagementControllerTest.kt @@ -0,0 +1,265 @@ +package io.tolgee.api.v2.controllers.batch + +import io.tolgee.ProjectAuthControllerTest +import io.tolgee.batch.BatchJobActionService +import io.tolgee.batch.BatchJobConcurrentLauncher +import io.tolgee.batch.BatchJobDto +import io.tolgee.batch.BatchJobService +import io.tolgee.batch.BatchJobType +import io.tolgee.batch.JobChunkExecutionQueue +import io.tolgee.batch.processors.TranslationChunkProcessor +import io.tolgee.batch.request.BatchTranslateRequest +import io.tolgee.batch.state.BatchJobStateProvider +import io.tolgee.development.testDataBuilder.data.BatchJobsTestData +import io.tolgee.fixtures.andAssertThatJson +import io.tolgee.fixtures.andIsForbidden +import io.tolgee.fixtures.andIsOk +import io.tolgee.fixtures.isValidId +import io.tolgee.fixtures.node +import io.tolgee.fixtures.waitForNotThrowing +import io.tolgee.model.batch.BatchJob +import io.tolgee.model.batch.BatchJobStatus +import io.tolgee.testing.ContextRecreatingTest +import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod +import io.tolgee.testing.assert +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.whenever +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc +import org.springframework.boot.test.mock.mockito.MockBean +import org.springframework.test.annotation.DirtiesContext +import java.util.concurrent.ConcurrentHashMap + +@AutoConfigureMockMvc +@ContextRecreatingTest +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class BatchJobManagementControllerTest : ProjectAuthControllerTest("/v2/projects/") { + + lateinit var testData: BatchJobsTestData + + @Autowired + lateinit var batchJobActionService: BatchJobActionService + + @Autowired + lateinit var batchJobService: BatchJobService + + @Autowired + @MockBean + lateinit var translationChunkProcessor: TranslationChunkProcessor + + @Autowired + lateinit var batchJobStateProvider: BatchJobStateProvider + + @Autowired + lateinit var jobChunkExecutionQueue: JobChunkExecutionQueue + + @Autowired + lateinit var batchJobConcurrentLauncher: BatchJobConcurrentLauncher + + @BeforeEach + fun setup() { + testData = BatchJobsTestData() + jobChunkExecutionQueue.populateQueue() + whenever(translationChunkProcessor.getParams(any(), any())).thenCallRealMethod() + whenever(translationChunkProcessor.getTarget(any())).thenCallRealMethod() + } + + @AfterEach + fun after() { + batchJobConcurrentLauncher.pause = false + } + + @Test + @ProjectJWTAuthTestMethod + fun `cancels a job`() { + val keys = testData.addTranslationOperationData(10) + saveAndPrepare() + + val keyIds = keys.map { it.id }.toList() + + batchJobConcurrentLauncher.pause = true + + performProjectAuthPut( + "start-batch-job/translate", + mapOf( + "keyIds" to keyIds, + "targetLanguageIds" to listOf( + testData.projectBuilder.getLanguageByTag("cs")!!.self.id + ) + ) + ).andIsOk + + val job = getSingleJob() + + performProjectAuthPut("batch-jobs/${job.id}/cancel") + .andIsOk + + waitForNotThrowing { + getSingleJob().status.assert.isEqualTo(BatchJobStatus.CANCELLED) + } + } + + @Test + @ProjectJWTAuthTestMethod + fun `cannot cancel other's job`() { + saveAndPrepare() + + batchJobConcurrentLauncher.pause = true + + val job = runChunkedJob(10) + + userAccount = testData.anotherUser + + performProjectAuthPut("batch-jobs/${job.id}/cancel") + .andIsForbidden + } + + @Test + @ProjectJWTAuthTestMethod + fun `returns list of jobs`() { + saveAndPrepare() + + val jobIds = ConcurrentHashMap.newKeySet() + var wait = true + whenever( + translationChunkProcessor.process(any(), any(), any(), any()) + ).then { + val id = it.getArgument(0).id + if (jobIds.size == 2 && !jobIds.contains(id)) { + while (wait) { + Thread.sleep(100) + } + } else { + jobIds.add(id) + } + } + + val jobs = (1..3).map { runChunkedJob(50) } + + waitForNotThrowing(pollTime = 1000, timeout = 10000) { + val dtos = jobs.map { batchJobService.getJobDto(it.id) } + dtos.forEach { + val state = batchJobStateProvider.getCached(it.id) + println( + "Job ${it.id} status ${it.status} progress: ${state?.values?.sumOf { it.successTargets.size }}" + ) + } + dtos.count { it.status == BatchJobStatus.SUCCESS }.assert.isEqualTo(2) + dtos.count { it.status == BatchJobStatus.RUNNING }.assert.isEqualTo(1) + } + + performProjectAuthGet("batch-jobs?sort=status&sort=id") + .andIsOk.andAssertThatJson { + node("_embedded.batchJobs") { + isArray.hasSize(3) + node("[0].status").isEqualTo("RUNNING") + node("[0].progress").isEqualTo(0) + node("[1].id").isValidId + node("[1].status").isEqualTo("SUCCESS") + node("[1].progress").isEqualTo(50) + } + } + + wait = false + + waitForNotThrowing(pollTime = 1000, timeout = 10000) { + val dtos = jobs.map { batchJobService.getJobDto(it.id) } + dtos.count { it.status == BatchJobStatus.SUCCESS }.assert.isEqualTo(3) + } + + performProjectAuthGet("batch-jobs?sort=status&sort=id") + .andIsOk.andAssertThatJson { + node("_embedded.batchJobs") { + isArray.hasSize(3) + node("[0].status").isEqualTo("SUCCESS") + } + } + } + + @Test + @ProjectJWTAuthTestMethod + fun `returns list of my jobs`() { + saveAndPrepare() + + val jobs = (1..3).map { runChunkedJob(50) } + + waitForNotThrowing(pollTime = 1000, timeout = 10000) { + val dtos = jobs.map { batchJobService.getJobDto(it.id) } + dtos.count { it.status == BatchJobStatus.SUCCESS }.assert.isEqualTo(3) + } + + performProjectAuthGet("my-batch-jobs?sort=status&sort=id") + .andIsOk.andAssertThatJson { + node("_embedded.batchJobs") { + isArray.hasSize(3) + node("[0].status").isEqualTo("SUCCESS") + } + } + + userAccount = testData.anotherUser + + performProjectAuthGet("my-batch-jobs?sort=status&sort=id") + .andIsOk.andAssertThatJson { + node("_embedded.batchJobs").isAbsent() + } + } + + @Test + @ProjectJWTAuthTestMethod + fun `returns single job`() { + saveAndPrepare() + + val job = runChunkedJob(50) + + waitForNotThrowing(pollTime = 1000, timeout = 10000) { + getSingleJob().status.assert.isEqualTo(BatchJobStatus.SUCCESS) + } + + performProjectAuthGet("batch-jobs/${job.id}") + .andIsOk.andAssertThatJson { + node("status").isEqualTo("SUCCESS") + } + } + + @Test + @ProjectJWTAuthTestMethod + fun `cannot get other's job`() { + saveAndPrepare() + + val job = runChunkedJob(10) + + waitForNotThrowing(pollTime = 100, timeout = 10000) { + getSingleJob().status.assert.isEqualTo(BatchJobStatus.SUCCESS) + } + + userAccount = testData.anotherUser + + performProjectAuthGet("batch-jobs/${job.id}") + .andIsForbidden + } + + private fun getSingleJob(): BatchJob = + entityManager.createQuery("""from BatchJob""", BatchJob::class.java).singleResult + + private fun saveAndPrepare() { + testDataService.saveTestData(testData.root) + userAccount = testData.user + this.projectSupplier = { testData.projectBuilder.self } + } + + protected fun runChunkedJob(keyCount: Int): BatchJob { + return executeInNewTransaction { + batchJobService.startJob( + request = BatchTranslateRequest().apply { + keyIds = (1L..keyCount).map { it } + }, + project = testData.projectBuilder.self, + author = testData.user, + type = BatchJobType.TRANSLATION + ) + } + } +} diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt new file mode 100644 index 0000000000..c745ccbdbf --- /dev/null +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt @@ -0,0 +1,132 @@ +package io.tolgee.api.v2.controllers.batch + +import io.tolgee.ProjectAuthControllerTest +import io.tolgee.development.testDataBuilder.data.BatchJobsTestData +import io.tolgee.fixtures.andAssertThatJson +import io.tolgee.fixtures.andIsOk +import io.tolgee.fixtures.isValidId +import io.tolgee.fixtures.waitForNotThrowing +import io.tolgee.model.batch.BatchJob +import io.tolgee.model.batch.BatchJobStatus +import io.tolgee.model.translation.Translation +import io.tolgee.testing.ContextRecreatingTest +import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod +import io.tolgee.testing.assert +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc + +@AutoConfigureMockMvc +@ContextRecreatingTest +class StartBatchJobControllerTest : ProjectAuthControllerTest("/v2/projects/") { + lateinit var testData: BatchJobsTestData + var fakeBefore = false + + @BeforeEach + fun setup() { + testData = BatchJobsTestData() + fakeBefore = internalProperties.fakeMtProviders + internalProperties.fakeMtProviders = true + machineTranslationProperties.google.apiKey = "mock" + machineTranslationProperties.google.defaultEnabled = true + machineTranslationProperties.google.defaultPrimary = true + machineTranslationProperties.aws.defaultEnabled = false + machineTranslationProperties.aws.accessKey = "mock" + machineTranslationProperties.aws.secretKey = "mock" + } + + @AfterEach + fun after() { + internalProperties.fakeMtProviders = fakeBefore + } + + fun saveAndPrepare() { + testDataService.saveTestData(testData.root) + userAccount = testData.user + this.projectSupplier = { testData.projectBuilder.self } + } + + @Test + @ProjectJWTAuthTestMethod + fun `it batch translates`() { + val keyCount = 100 + val keys = testData.addTranslationOperationData(keyCount) + saveAndPrepare() + + val keyIds = keys.map { it.id }.toList() + + performProjectAuthPut( + "start-batch-job/translate", + mapOf( + "keyIds" to keyIds, + "targetLanguageIds" to listOf( + testData.projectBuilder.getLanguageByTag("cs")!!.self.id, + testData.projectBuilder.getLanguageByTag("de")!!.self.id + ) + ) + ) + .andIsOk + .andAssertThatJson { + node("id").isValidId + } + + waitForAllTranslated(keyIds, keyCount) + executeInNewTransaction { + val jobs = entityManager.createQuery("""from BatchJob""", BatchJob::class.java) + .resultList + jobs.assert.hasSize(1) + val job = jobs[0] + job.status.assert.isEqualTo(BatchJobStatus.SUCCESS) + job.activityRevision.assert.isNotNull + job.activityRevision!!.modifiedEntities.assert.hasSize(200) + } + } + + private fun waitForAllTranslated(keyIds: List, keyCount: Int) { + waitForNotThrowing(pollTime = 1000) { + @Suppress("UNCHECKED_CAST") val czechTranslations = entityManager.createQuery( + """ + from Translation t where t.key.id in :keyIds and t.language.tag = 'cs' + """.trimIndent() + ).setParameter("keyIds", keyIds).resultList as List + czechTranslations.assert.hasSize(keyCount) + czechTranslations.forEach { + it.text.assert.contains("translated with GOOGLE from en to cs") + } + } + } + + @Test + @ProjectJWTAuthTestMethod + fun `it deletes keys`() { + val keyCount = 100 + val keys = testData.addTranslationOperationData(keyCount) + saveAndPrepare() + + val keyIds = keys.map { it.id }.toList() + + performProjectAuthPut( + "start-batch-job/delete-keys", + mapOf( + "keyIds" to keyIds, + ) + ).andIsOk + + waitForNotThrowing(pollTime = 1000, timeout = 10000) { + val all = keyService.getAll(testData.projectBuilder.self.id) + all.assert.isEmpty() + } + + waitForNotThrowing(pollTime = 1000, timeout = 10000) { + executeInNewTransaction { + val data = entityManager + .createQuery("""from BatchJob""", BatchJob::class.java) + .resultList + + data.assert.hasSize(1) + data[0].activityRevision.assert.isNotNull + } + } + } +} diff --git a/backend/app/src/test/kotlin/io/tolgee/batch/AbstractBatchJobsGeneralTest.kt b/backend/app/src/test/kotlin/io/tolgee/batch/AbstractBatchJobsGeneralTest.kt new file mode 100644 index 0000000000..eec697f1b3 --- /dev/null +++ b/backend/app/src/test/kotlin/io/tolgee/batch/AbstractBatchJobsGeneralTest.kt @@ -0,0 +1,384 @@ +package io.tolgee.batch + +import io.tolgee.AbstractSpringTest +import io.tolgee.batch.processors.DeleteKeysChunkProcessor +import io.tolgee.batch.processors.TranslationChunkProcessor +import io.tolgee.batch.request.BatchTranslateRequest +import io.tolgee.batch.request.DeleteKeysRequest +import io.tolgee.component.CurrentDateProvider +import io.tolgee.constants.Message +import io.tolgee.development.testDataBuilder.data.BatchJobsTestData +import io.tolgee.exceptions.OutOfCreditsException +import io.tolgee.fixtures.waitFor +import io.tolgee.fixtures.waitForNotThrowing +import io.tolgee.model.batch.BatchJob +import io.tolgee.model.batch.BatchJobStatus +import io.tolgee.security.JwtTokenProvider +import io.tolgee.testing.WebsocketTest +import io.tolgee.testing.assert +import io.tolgee.websocket.WebsocketTestHelper +import kotlinx.coroutines.ensureActive +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.Mockito +import org.mockito.kotlin.any +import org.mockito.kotlin.argThat +import org.mockito.kotlin.times +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.mock.mockito.MockBean +import org.springframework.boot.test.web.server.LocalServerPort +import org.springframework.test.annotation.DirtiesContext +import java.util.* +import kotlin.coroutines.CoroutineContext +import kotlin.math.ceil + +@WebsocketTest +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) +abstract class AbstractBatchJobsGeneralTest : AbstractSpringTest() { + + private lateinit var testData: BatchJobsTestData + + @Autowired + lateinit var batchJobService: BatchJobService + + @MockBean + @Autowired + lateinit var translationChunkProcessor: TranslationChunkProcessor + + @MockBean + @Autowired + lateinit var deleteKeysChunkProcessor: DeleteKeysChunkProcessor + + @Autowired + lateinit var batchJobActionService: BatchJobActionService + + @Autowired + lateinit var jwtTokenProvider: JwtTokenProvider + + @LocalServerPort + private val port: Int? = null + + lateinit var websocketHelper: WebsocketTestHelper + + @Autowired + lateinit var currentDateProvider: CurrentDateProvider + + @Autowired + lateinit var batchJobCancellationManager: BatchJobCancellationManager + + @Autowired + lateinit var jobChunkExecutionQueue: JobChunkExecutionQueue + + @BeforeEach + fun setup() { + jobChunkExecutionQueue.clear() + Mockito.reset(translationChunkProcessor) + Mockito.clearInvocations(translationChunkProcessor) + whenever(translationChunkProcessor.getParams(any(), any())).thenCallRealMethod() + whenever(translationChunkProcessor.getTarget(any())).thenCallRealMethod() + whenever(deleteKeysChunkProcessor.getParams(any(), any())).thenCallRealMethod() + whenever(deleteKeysChunkProcessor.getTarget(any())).thenCallRealMethod() + jobChunkExecutionQueue.populateQueue() + testData = BatchJobsTestData() + testDataService.saveTestData(testData.root) + currentDateProvider.forcedDate = Date(1687237928000) + websocketHelper = WebsocketTestHelper( + port, + jwtTokenProvider.generateToken(testData.user.id).toString(), + testData.projectBuilder.self.id + ) + websocketHelper.listenForBatchJobProgress() + } + + @AfterEach() + fun teardown() { + jobChunkExecutionQueue.clear() + currentDateProvider.forcedDate = null + websocketHelper.stop() + } + + @Test + fun `executes operation`() { + websocketHelper = WebsocketTestHelper( + port, + jwtTokenProvider.generateToken(testData.user.id).toString(), + testData.projectBuilder.self.id + ) + websocketHelper.listenForBatchJobProgress() + + val job = runChunkedJob(1000) + + job.totalItems.assert.isEqualTo(1000) + + waitForNotThrowing(pollTime = 1000) { + verify( + translationChunkProcessor, + times(ceil(job.totalItems.toDouble() / BatchJobType.TRANSLATION.chunkSize).toInt()) + ).process(any(), any(), any(), any()) + } + + waitForNotThrowing(pollTime = 1000) { + executeInNewTransaction { + val finishedJob = batchJobService.getJobDto(job.id) + finishedJob.status.assert.isEqualTo(BatchJobStatus.SUCCESS) + } + } + + // 100 progress messages + 1 finish message + websocketHelper.receivedMessages.assert.hasSize(101) + } + + @Test + fun `correctly reports failed test when FailedDontRequeueException thrown`() { + websocketHelper = WebsocketTestHelper( + port, + jwtTokenProvider.generateToken(testData.user.id).toString(), + testData.projectBuilder.self.id + ) + websocketHelper.listenForBatchJobProgress() + + val job = runChunkedJob(1000) + + val exceptions = (1..50).map { _ -> + FailedDontRequeueException( + Message.OUT_OF_CREDITS, + cause = OutOfCreditsException(OutOfCreditsException.Reason.OUT_OF_CREDITS), + successfulTargets = listOf() + ) + } + + whenever( + translationChunkProcessor.process( + any(), any(), any(), any() + ) + ) + .thenThrow(*exceptions.toTypedArray()) + .then {} + + waitForNotThrowing(pollTime = 1000) { + executeInNewTransaction { + val finishedJob = batchJobService.getJobDto(job.id) + finishedJob.status.assert.isEqualTo(BatchJobStatus.FAILED) + } + } + + // 100 progress messages + 1 finish message + websocketHelper.receivedMessages.assert.hasSize(51) + websocketHelper.receivedMessages.last.contains("FAILED") + websocketHelper.receivedMessages.last.contains("out_of_credits") + + waitForNotThrowing { + executeInNewTransaction { + batchJobService.getView(job.id).errorMessage.assert.isEqualTo(Message.OUT_OF_CREDITS) + } + } + } + + @Test + fun `retries failed with generic exception`() { + websocketHelper = WebsocketTestHelper( + port, + jwtTokenProvider.generateToken(testData.user.id).toString(), + testData.projectBuilder.self.id + ) + websocketHelper.listenForBatchJobProgress() + + whenever( + translationChunkProcessor.process( + any(), + argThat { this.containsAll((1L..10).toList()) }, + any(), + any() + ) + ).thenThrow(RuntimeException("OMG! It failed")) + + val job = runChunkedJob(1000) + + (1..3).forEach { + waitForNotThrowing { + jobChunkExecutionQueue.find { it.executeAfter == currentDateProvider.date.time + 2000 }.assert.isNotNull + } + currentDateProvider.forcedDate = Date(currentDateProvider.date.time + 2000) + } + + waitForNotThrowing(pollTime = 1000) { + executeInNewTransaction { + val finishedJob = batchJobService.getJobDto(job.id) + val finishedJobEntity = batchJobService.getJobEntity(job.id) + finishedJobEntity.status.assert.isEqualTo(BatchJobStatus.FAILED) + finishedJob.status.assert.isEqualTo(BatchJobStatus.FAILED) + } + } + + entityManager.createQuery("""from BatchJobChunkExecution b where b.batchJob.id = :id""") + .setParameter("id", job.id).resultList.assert.hasSize(103) + + // 100 progress messages + 1 finish message + websocketHelper.receivedMessages.assert.hasSize(100) + websocketHelper.receivedMessages.last.contains("FAILED") + } + + @Test + fun `retries failed with RequeueWithTimeoutException`() { + websocketHelper = WebsocketTestHelper( + port, + jwtTokenProvider.generateToken(testData.user.id).toString(), + testData.projectBuilder.self.id + ) + websocketHelper.listenForBatchJobProgress() + + val throwingChunk = (1L..10).toList() + + whenever( + translationChunkProcessor.process( + any(), + argThat { this.containsAll(throwingChunk) }, + any(), + any() + ) + ).thenThrow( + RequeueWithTimeoutException( + message = Message.OUT_OF_CREDITS, + successfulTargets = listOf(), + cause = OutOfCreditsException(OutOfCreditsException.Reason.OUT_OF_CREDITS), + timeoutInMs = 100, + increaseFactor = 10, + maxRetries = 3 + ) + ) + + val job = runChunkedJob(1000) + + waitForNotThrowing { + jobChunkExecutionQueue.find { it.executeAfter == currentDateProvider.date.time + 100 }.assert.isNotNull + } + + currentDateProvider.forcedDate = Date(currentDateProvider.date.time + 100) + + waitForNotThrowing { + jobChunkExecutionQueue.find { it.executeAfter == currentDateProvider.date.time + 1000 }.assert.isNotNull + } + + currentDateProvider.forcedDate = Date(currentDateProvider.date.time + 1000) + + waitForNotThrowing { + jobChunkExecutionQueue.find { it.executeAfter == currentDateProvider.date.time + 10000 }.assert.isNotNull + } + + currentDateProvider.forcedDate = Date(currentDateProvider.date.time + 10000) + + waitForNotThrowing(pollTime = 1000) { + executeInNewTransaction { + val finishedJob = batchJobService.getJobDto(job.id) + finishedJob.status.assert.isEqualTo(BatchJobStatus.FAILED) + } + } + + entityManager.createQuery("""from BatchJobChunkExecution b where b.batchJob.id = :id""") + .setParameter("id", job.id).resultList.assert.hasSize(103) + + // 100 progress messages + 1 finish message + websocketHelper.receivedMessages.assert.hasSize(100) + websocketHelper.receivedMessages.last.contains("FAILED") + } + + @Test + fun `publishes progress of single chunk job`() { + whenever( + deleteKeysChunkProcessor.process( + any(), + any(), + any(), + any() + ) + ).thenAnswer { + @Suppress("UNCHECKED_CAST") + val chunk = it.arguments[1] as List + + @Suppress("UNCHECKED_CAST") + val onProgress = it.arguments[3] as ((progress: Int) -> Unit) + + chunk.forEachIndexed { index, _ -> + onProgress(index + 1) + } + } + + val job = runSingleChunkJob(100) + + waitForNotThrowing(pollTime = 1000) { + executeInNewTransaction { + val finishedJob = batchJobService.getJobDto(job.id) + finishedJob.status.assert.isEqualTo(BatchJobStatus.SUCCESS) + } + + entityManager.createQuery("""from BatchJobChunkExecution b where b.batchJob.id = :id""") + .setParameter("id", job.id).resultList.assert.hasSize(1) + + // 100 progress messages + 1 finish message + websocketHelper.receivedMessages.assert.hasSize(101) + websocketHelper.receivedMessages.last.contains("SUCCESS") + } + } + + @Test + fun `cancels the job`() { + var count = 0 + + whenever(translationChunkProcessor.process(any(), any(), any(), any())).then { + if (count++ > 50) { + while (true) { + val context = it.arguments[2] as CoroutineContext + context.ensureActive() + Thread.sleep(100) + } + } + } + + val job = runChunkedJob(1000) + + waitFor { + count > 50 + } + + batchJobCancellationManager.cancel(job.id) + + waitForNotThrowing(pollTime = 1000) { + executeInNewTransaction { + val finishedJob = batchJobService.getJobDto(job.id) + finishedJob.status.assert.isEqualTo(BatchJobStatus.CANCELLED) + } + } + + websocketHelper.receivedMessages.assert.hasSizeGreaterThan(49) + websocketHelper.receivedMessages.last.contains("CANCELLED") + } + + protected fun runChunkedJob(keyCount: Int): BatchJob { + return executeInNewTransaction { + batchJobService.startJob( + request = BatchTranslateRequest().apply { + keyIds = (1L..keyCount).map { it } + }, + project = testData.projectBuilder.self, + author = testData.user, + type = BatchJobType.TRANSLATION + ) + } + } + + protected fun runSingleChunkJob(keyCount: Int): BatchJob { + return executeInNewTransaction { + batchJobService.startJob( + request = DeleteKeysRequest().apply { + keyIds = (1L..keyCount).map { it } + }, + project = testData.projectBuilder.self, + author = testData.user, + type = BatchJobType.DELETE_KEYS + ) + } + } +} diff --git a/backend/app/src/test/kotlin/io/tolgee/batch/BatchJobsGeneralWithRedisTest.kt b/backend/app/src/test/kotlin/io/tolgee/batch/BatchJobsGeneralWithRedisTest.kt new file mode 100644 index 0000000000..b3ac514cbe --- /dev/null +++ b/backend/app/src/test/kotlin/io/tolgee/batch/BatchJobsGeneralWithRedisTest.kt @@ -0,0 +1,88 @@ +package io.tolgee.batch + +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.tolgee.fixtures.RedisRunner +import io.tolgee.fixtures.waitForNotThrowing +import io.tolgee.pubSub.RedisPubSubReceiverConfiguration.Companion.JOB_QUEUE_TOPIC +import io.tolgee.testing.assert +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.Test +import org.mockito.Mockito +import org.mockito.kotlin.eq +import org.mockito.kotlin.times +import org.mockito.kotlin.verify +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.test.mock.mockito.SpyBean +import org.springframework.context.ApplicationContextInitializer +import org.springframework.context.ConfigurableApplicationContext +import org.springframework.data.redis.core.StringRedisTemplate +import org.springframework.test.annotation.DirtiesContext +import org.springframework.test.context.ContextConfiguration + +@SpringBootTest( + properties = [ + "tolgee.cache.use-redis=true", + "tolgee.cache.enabled=true", + "tolgee.websocket.use-redis=true", + "spring.redis.port=56379", + ], + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT +) +@ContextConfiguration(initializers = [BatchJobsGeneralWithRedisTest.Companion.Initializer::class]) +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class BatchJobsGeneralWithRedisTest : AbstractBatchJobsGeneralTest() { + companion object { + val redisRunner = RedisRunner() + + class Initializer : ApplicationContextInitializer { + override fun initialize(configurableApplicationContext: ConfigurableApplicationContext) { + redisRunner.run() + } + } + } + + @Autowired + lateinit var jobConcurrentLauncher: BatchJobConcurrentLauncher + + @SpyBean + @Autowired + lateinit var redisTemplate: StringRedisTemplate + + @AfterAll + fun cleanup() { + Mockito.reset(redisTemplate) + Mockito.clearInvocations(redisTemplate) + redisRunner.stop() + jobConcurrentLauncher.pause = false + } + + @Test + fun `removes from queue using event`() { + jobConcurrentLauncher.pause = true + + runChunkedJob(keyCount = 200) + + waitForNotThrowing { + val peek = jobChunkExecutionQueue.peek() + peek.assert.isNotNull + } + + val peek = jobChunkExecutionQueue.peek() + jobChunkExecutionQueue.contains(peek).assert.isTrue() + Mockito.clearInvocations(redisTemplate) + batchJobActionService.publishRemoveConsuming(peek) + verify(redisTemplate, times(1)) + .convertAndSend( + eq(JOB_QUEUE_TOPIC), + eq( + jacksonObjectMapper().writeValueAsString( + JobQueueItemsEvent(listOf(peek), QueueEventType.REMOVE) + ) + ) + ) + waitForNotThrowing(timeout = 2000) { + jobChunkExecutionQueue.contains(peek).assert.isFalse() + } + } +} diff --git a/backend/app/src/test/kotlin/io/tolgee/batch/BatchJobsGeneralWithoutRedisTest.kt b/backend/app/src/test/kotlin/io/tolgee/batch/BatchJobsGeneralWithoutRedisTest.kt new file mode 100644 index 0000000000..f926debf45 --- /dev/null +++ b/backend/app/src/test/kotlin/io/tolgee/batch/BatchJobsGeneralWithoutRedisTest.kt @@ -0,0 +1,11 @@ +package io.tolgee.batch + +import org.springframework.boot.test.context.SpringBootTest + +@SpringBootTest( + properties = [ + "tolgee.cache.enabled=true", + ], + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT +) +class BatchJobsGeneralWithoutRedisTest : AbstractBatchJobsGeneralTest() diff --git a/backend/app/src/test/kotlin/io/tolgee/cache/CacheWithRedisTest.kt b/backend/app/src/test/kotlin/io/tolgee/cache/CacheWithRedisTest.kt index 2615d5a41e..cd14555c5a 100644 --- a/backend/app/src/test/kotlin/io/tolgee/cache/CacheWithRedisTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/cache/CacheWithRedisTest.kt @@ -9,6 +9,7 @@ import org.redisson.spring.cache.RedissonSpringCacheManager import org.springframework.boot.test.context.SpringBootTest import org.springframework.context.ApplicationContextInitializer import org.springframework.context.ConfigurableApplicationContext +import org.springframework.test.annotation.DirtiesContext import org.springframework.test.context.ContextConfiguration @ContextRecreatingTest @@ -22,6 +23,7 @@ import org.springframework.test.context.ContextConfiguration ] ) @ContextConfiguration(initializers = [CacheWithRedisTest.Companion.Initializer::class]) +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) class CacheWithRedisTest : AbstractCacheTest() { companion object { val redisRunner = RedisRunner() diff --git a/backend/app/src/test/kotlin/io/tolgee/cache/CacheWithoutRedisTest.kt b/backend/app/src/test/kotlin/io/tolgee/cache/CacheWithoutRedisTest.kt index c3c43f7dc3..ff3745a9d5 100644 --- a/backend/app/src/test/kotlin/io/tolgee/cache/CacheWithoutRedisTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/cache/CacheWithoutRedisTest.kt @@ -5,6 +5,7 @@ import io.tolgee.testing.assertions.Assertions.assertThat import org.junit.jupiter.api.Test import org.springframework.boot.test.context.SpringBootTest import org.springframework.cache.caffeine.CaffeineCacheManager +import org.springframework.test.annotation.DirtiesContext @ContextRecreatingTest @SpringBootTest( @@ -13,6 +14,7 @@ import org.springframework.cache.caffeine.CaffeineCacheManager "tolgee.internal.fake-mt-providers=false", ] ) +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) class CacheWithoutRedisTest : AbstractCacheTest() { @Test fun `it has proper cache manager`() { diff --git a/backend/app/src/test/kotlin/io/tolgee/websocket/AbstractWebsocketTest.kt b/backend/app/src/test/kotlin/io/tolgee/websocket/AbstractWebsocketTest.kt index 0a818c471e..5f826803fd 100644 --- a/backend/app/src/test/kotlin/io/tolgee/websocket/AbstractWebsocketTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/websocket/AbstractWebsocketTest.kt @@ -5,61 +5,51 @@ import io.tolgee.development.testDataBuilder.data.BaseTestData import io.tolgee.fixtures.andIsOk import io.tolgee.fixtures.isValidId import io.tolgee.fixtures.node -import io.tolgee.fixtures.waitFor import io.tolgee.model.UserAccount import io.tolgee.model.key.Key import io.tolgee.model.translation.Translation +import io.tolgee.testing.WebsocketTest import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod import io.tolgee.testing.assert import net.javacrumbs.jsonunit.assertj.assertThatJson +import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance import org.springframework.boot.test.web.server.LocalServerPort -import java.util.concurrent.LinkedBlockingDeque @TestInstance(TestInstance.Lifecycle.PER_CLASS) +@WebsocketTest abstract class AbstractWebsocketTest : ProjectAuthControllerTest("/v2/projects/") { lateinit var testData: BaseTestData lateinit var translation: Translation lateinit var key: Key - lateinit var receivedMessages: LinkedBlockingDeque lateinit var notPermittedUser: UserAccount + lateinit var helper: WebsocketTestHelper @LocalServerPort private val port: Int? = null - /** - * Asserts that event with provided name was triggered by runnable provided in "dispatch" function - */ - fun assertNotified( - dispatchCallback: () -> Unit, - assertCallback: ((value: LinkedBlockingDeque) -> Unit) - ) { - Thread.sleep(200) - dispatchCallback() - waitFor(3000) { - receivedMessages.isNotEmpty() - } - assertCallback(receivedMessages) - } - @BeforeEach fun beforeEach() { prepareTestData() - val helper = WebsocketTestHelper( + helper = WebsocketTestHelper( port, jwtTokenProvider.generateToken(testData.user.id).toString(), testData.projectBuilder.self.id ) - helper.listen() - receivedMessages = helper.receivedMessages + helper.listenForTranslationDataModified() + } + + @AfterEach + fun after() { + helper.stop() } @Test @ProjectJWTAuthTestMethod fun `notifies on key modification`() { - assertNotified( + helper.assertNotified( { performProjectAuthPut("keys/${key.id}", mapOf("name" to "name edited")) } @@ -94,7 +84,7 @@ abstract class AbstractWebsocketTest : ProjectAuthControllerTest("/v2/projects/" @Test @ProjectJWTAuthTestMethod fun `notifies on key deletion`() { - assertNotified( + helper.assertNotified( { performProjectAuthDelete("keys/${key.id}") } @@ -122,7 +112,7 @@ abstract class AbstractWebsocketTest : ProjectAuthControllerTest("/v2/projects/" @Test @ProjectJWTAuthTestMethod fun `notifies on key creation`() { - assertNotified( + helper.assertNotified( { performProjectAuthPost("keys", mapOf("name" to "new key")) } @@ -150,7 +140,7 @@ abstract class AbstractWebsocketTest : ProjectAuthControllerTest("/v2/projects/" @Test @ProjectJWTAuthTestMethod fun `notifies on translation modification`() { - assertNotified( + helper.assertNotified( { performProjectAuthPut( "translations", @@ -205,7 +195,7 @@ abstract class AbstractWebsocketTest : ProjectAuthControllerTest("/v2/projects/" jwtTokenProvider.generateToken(notPermittedUser.id).toString(), testData.projectBuilder.self.id ) - notPermittedSubscriptionHelper.listen() + notPermittedSubscriptionHelper.listenForTranslationDataModified() performProjectAuthPut( "translations", mapOf( @@ -217,7 +207,7 @@ abstract class AbstractWebsocketTest : ProjectAuthControllerTest("/v2/projects/" notPermittedSubscriptionHelper.receivedMessages.assert.isEmpty() // but authorized user received the message - receivedMessages.assert.isNotEmpty + helper.receivedMessages.assert.isNotEmpty } private fun prepareTestData() { diff --git a/backend/app/src/test/kotlin/io/tolgee/websocket/WebsocketTestHelper.kt b/backend/app/src/test/kotlin/io/tolgee/websocket/WebsocketTestHelper.kt index 87b558e908..fe210b2a34 100644 --- a/backend/app/src/test/kotlin/io/tolgee/websocket/WebsocketTestHelper.kt +++ b/backend/app/src/test/kotlin/io/tolgee/websocket/WebsocketTestHelper.kt @@ -1,5 +1,6 @@ package io.tolgee.websocket +import io.tolgee.fixtures.waitFor import io.tolgee.util.Logging import io.tolgee.util.logger import org.springframework.lang.Nullable @@ -17,31 +18,59 @@ import java.lang.reflect.Type import java.util.concurrent.LinkedBlockingDeque import java.util.concurrent.TimeUnit -class WebsocketTestHelper(val port: Int?, val jwtToken: String, val projectId: Long) { +class WebsocketTestHelper(val port: Int?, val jwtToken: String, val projectId: Long) : Logging { + private var sessionHandler: MySessionHandler? = null lateinit var receivedMessages: LinkedBlockingDeque - fun listen() { - receivedMessages = LinkedBlockingDeque() + fun listenForTranslationDataModified() { + listen("/projects/$projectId/${WebsocketEventType.TRANSLATION_DATA_MODIFIED.typeName}") + } - val webSocketStompClient = WebSocketStompClient( + fun listenForBatchJobProgress() { + listen("/projects/$projectId/${WebsocketEventType.BATCH_JOB_PROGRESS.typeName}") + } + + private val webSocketStompClient by lazy { + WebSocketStompClient( SockJsClient(listOf(WebSocketTransport(StandardWebSocketClient()))) ) + } - webSocketStompClient.messageConverter = SimpleMessageConverter() + private var connection: StompSession? = null + + fun listen(path: String) { + receivedMessages = LinkedBlockingDeque() - webSocketStompClient.connect( + webSocketStompClient.messageConverter = SimpleMessageConverter() + sessionHandler = MySessionHandler(path, receivedMessages) + connection = webSocketStompClient.connect( "http://localhost:$port/websocket", WebSocketHttpHeaders(), StompHeaders().apply { add("jwtToken", jwtToken) }, - MySessionHandler("/projects/$projectId/translation-data-modified", receivedMessages) + sessionHandler!! ).get(10, TimeUnit.SECONDS) } + fun stop() { + logger.info("Stopping websocket listener") + try { + sessionHandler?.subscription?.unsubscribe() + connection?.disconnect() + } catch (e: IllegalStateException) { + logger.warn("Could not unsubscribe from websocket", e) + } + webSocketStompClient.stop() + logger.info("Stopped websocket listener") + } + private class MySessionHandler( val dest: String, val receivedMessages: LinkedBlockingDeque ) : StompSessionHandlerAdapter(), Logging { + + var subscription: StompSession.Subscription? = null + override fun afterConnected(session: StompSession, connectedHeaders: StompHeaders) { - session.subscribe(dest, this) + subscription = session.subscribe(dest, this) } override fun handleException( @@ -72,4 +101,20 @@ class WebsocketTestHelper(val port: Int?, val jwtToken: String, val projectId: L } } } + + /** + * Asserts that event with provided name was triggered by runnable provided in "dispatch" function + */ + fun assertNotified( + dispatchCallback: () -> Unit, + assertCallback: ((value: LinkedBlockingDeque) -> Unit) + ) { + Thread.sleep(200) + dispatchCallback() + waitFor(3000) { + receivedMessages.isNotEmpty() + } + assertCallback(receivedMessages) + stop() + } } diff --git a/backend/app/src/test/kotlin/io/tolgee/websocket/WebsocketWithRedisTest.kt b/backend/app/src/test/kotlin/io/tolgee/websocket/WebsocketWithRedisTest.kt index d5a61229ee..9bd0d6f019 100644 --- a/backend/app/src/test/kotlin/io/tolgee/websocket/WebsocketWithRedisTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/websocket/WebsocketWithRedisTest.kt @@ -7,7 +7,6 @@ import org.springframework.boot.test.context.SpringBootTest import org.springframework.context.ApplicationContextInitializer import org.springframework.context.ConfigurableApplicationContext import org.springframework.test.context.ContextConfiguration -import org.springframework.test.context.junit.jupiter.DisabledIf @ContextRecreatingTest @SpringBootTest( @@ -19,7 +18,6 @@ import org.springframework.test.context.junit.jupiter.DisabledIf ) @ContextConfiguration(initializers = [WebsocketWithRedisTest.Companion.Initializer::class]) @TestInstance(TestInstance.Lifecycle.PER_CLASS) -@DisabledIf("${'$'}{tolgee.test.disableWebsocketTests:false}") class WebsocketWithRedisTest : AbstractWebsocketTest() { companion object { val redisRunner = RedisRunner() diff --git a/backend/app/src/test/kotlin/io/tolgee/websocket/WebsocketWithoutRedisTest.kt b/backend/app/src/test/kotlin/io/tolgee/websocket/WebsocketWithoutRedisTest.kt index bfe4a09da2..efe1794fd5 100644 --- a/backend/app/src/test/kotlin/io/tolgee/websocket/WebsocketWithoutRedisTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/websocket/WebsocketWithoutRedisTest.kt @@ -3,7 +3,6 @@ package io.tolgee.websocket import io.tolgee.testing.ContextRecreatingTest import org.junit.jupiter.api.TestInstance import org.springframework.boot.test.context.SpringBootTest -import org.springframework.test.context.junit.jupiter.DisabledIf @ContextRecreatingTest @SpringBootTest( @@ -13,5 +12,4 @@ import org.springframework.test.context.junit.jupiter.DisabledIf webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT ) @TestInstance(TestInstance.Lifecycle.PER_CLASS) -@DisabledIf("${'$'}{tolgee.test.disableWebsocketTests:false}") class WebsocketWithoutRedisTest : AbstractWebsocketTest() diff --git a/backend/app/src/test/resources/application.yaml b/backend/app/src/test/resources/application.yaml index 356d012ad5..780ae54f1c 100644 --- a/backend/app/src/test/resources/application.yaml +++ b/backend/app/src/test/resources/application.yaml @@ -1,4 +1,8 @@ spring: + autoconfigure: + exclude: + - org.redisson.spring.starter.RedissonAutoConfiguration + - org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration jpa: show-sql: true properties: @@ -27,6 +31,10 @@ spring: initialize-schema: always jmx: enabled: false + datasource: + hikari: + maximum-pool-size: 100 + maximum-pool-size: 15 tolgee: postgres-autostart: enabled: true @@ -67,9 +75,12 @@ tolgee: telemetry: enabled: false server: http://localhost:8080 -logging: - level: - io.tolgee.billing.api.v2.OrganizationInvoicesController: DEBUG +#logging: +# level: +# io.tolgee.billing.api.v2.OrganizationInvoicesController: DEBUG +# io.tolgee.batch.BatchJobActionService: DEBUG +# io.tolgee.component.atomicLong.AtomicLongProvider: DEBUG +# io.tolgee.batch: TRACE # org.springframework.orm.jpa: DEBUG # org.springframework.transaction: DEBUG # org.hibernate.type: TRACE diff --git a/backend/data/build.gradle b/backend/data/build.gradle index 8e01457225..05ea7e335b 100644 --- a/backend/data/build.gradle +++ b/backend/data/build.gradle @@ -106,6 +106,7 @@ dependencies { kapt "org.springframework.boot:spring-boot-configuration-processor" implementation "org.springframework.boot:spring-boot-configuration-processor" implementation "org.springframework.boot:spring-boot-starter-batch" + implementation "org.springframework.boot:spring-boot-starter-websocket" /** @@ -115,6 +116,12 @@ dependencies { implementation 'org.hibernate:hibernate-jpamodelgen' kapt "org.hibernate:hibernate-jpamodelgen" + /** + * Redisson + */ + implementation libs.redissonSpringBootStarter + implementation libs.redissonSpringData + /** * Liquibase */ diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt index 6ddff51701..1d282416b5 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt @@ -1,23 +1,23 @@ package io.tolgee.activity import io.tolgee.activity.data.ActivityType -import io.tolgee.events.OnProjectActivityEvent import io.tolgee.model.EntityWithId import io.tolgee.model.activity.ActivityModifiedEntity import io.tolgee.model.activity.ActivityRevision -import io.tolgee.util.Logging -import org.springframework.context.ApplicationContext -import javax.annotation.PreDestroy import kotlin.reflect.KClass -open class ActivityHolder( - private val applicationContext: ApplicationContext -) : Logging { +open class ActivityHolder { open var activity: ActivityType? = null + set(value) { + field = value + activityRevision?.type = value + } open var meta: MutableMap = mutableMapOf() - open var activityRevision: ActivityRevision? = null + open val activityRevision: ActivityRevision by lazy { + ActivityRevision() + } open var modifiedCollections: MutableMap, List?> = mutableMapOf() @@ -28,18 +28,11 @@ open class ActivityHolder( open var utmData: UtmData = null open var sdkInfo: Map? = null - @PreDestroy - open fun preDestroy() { - if (!transactionRollbackOnly) { - applicationContext.publishEvent(OnProjectActivityEvent(this)) - } - } - /** * This field stores all modified entities, it's stored before the transaction is committed */ open var modifiedEntities: - MutableMap< - KClass, MutableMap - > = mutableMapOf() + ModifiedEntitiesType = mutableMapOf() } + +typealias ModifiedEntitiesType = MutableMap, MutableMap> diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt index 482b785489..a49c47c9b4 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt @@ -4,6 +4,7 @@ import io.tolgee.activity.data.ActivityType import io.tolgee.activity.projectActivityView.ProjectActivityViewDataProvider import io.tolgee.dtos.query_results.TranslationHistoryView import io.tolgee.events.OnProjectActivityStoredEvent +import io.tolgee.model.activity.ActivityRevision import io.tolgee.model.views.activity.ProjectActivityView import io.tolgee.repository.activity.ActivityModifiedEntityRepository import org.springframework.context.ApplicationContext @@ -20,9 +21,8 @@ class ActivityService( private val activityModifiedEntityRepository: ActivityModifiedEntityRepository ) { @Transactional - fun storeActivityData(activityHolder: ActivityHolder) { - val activityRevision = activityHolder.activityRevision ?: return - activityRevision.modifiedEntities = activityHolder.modifiedEntities.values.flatMap { it.values }.toMutableList() + fun storeActivityData(activityRevision: ActivityRevision, modifiedEntities: ModifiedEntitiesType) { + activityRevision.modifiedEntities = modifiedEntities.values.flatMap { it.values }.toMutableList() entityManager.persist(activityRevision) activityRevision.describingRelations.forEach { @@ -31,6 +31,8 @@ class ActivityService( activityRevision.modifiedEntities.forEach { activityModifiedEntity -> entityManager.persist(activityModifiedEntity) } + entityManager.flush() + activityRevision.afterFlush?.invoke() applicationContext.publishEvent(OnProjectActivityStoredEvent(this, activityRevision)) } diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/data/ActivityType.kt b/backend/data/src/main/kotlin/io/tolgee/activity/data/ActivityType.kt index 5a165f94ba..896e5a1c0d 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/data/ActivityType.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/data/ActivityType.kt @@ -30,5 +30,6 @@ enum class ActivityType( DELETE_LANGUAGE(restrictEntitiesInList = arrayOf(Language::class)), CREATE_PROJECT, EDIT_PROJECT, - NAMESPACE_EDIT + NAMESPACE_EDIT, + BATCH_AUTO_TRANSLATE } diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityInterceptor.kt b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityInterceptor.kt index 7a2c729ca4..64c4e75034 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityInterceptor.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityInterceptor.kt @@ -1,6 +1,8 @@ package io.tolgee.activity.iterceptor +import io.tolgee.activity.ActivityHolder import io.tolgee.activity.data.RevisionType +import io.tolgee.events.OnProjectActivityEvent import io.tolgee.util.Logging import org.hibernate.EmptyInterceptor import org.hibernate.Transaction @@ -20,6 +22,21 @@ class ActivityInterceptor : EmptyInterceptor(), Logging { interceptedEventsManager.onAfterTransactionCompleted(tx) } + override fun beforeTransactionCompletion(tx: Transaction) { + if (tx.isActive) { + val holder = this.applicationContext.getBean(ActivityHolder::class.java) + val activityRevision = holder.activityRevision + if (!activityRevision.isInitializedByInterceptor) return + applicationContext.publishEvent( + OnProjectActivityEvent( + activityRevision, + holder.modifiedEntities, + holder.organizationId + ) + ) + } + } + override fun onSave( entity: Any?, id: Serializable?, diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt index cac448a8d6..2ad300469c 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/InterceptedEventsManager.kt @@ -193,10 +193,9 @@ class InterceptedEventsManager( private val activityRevision: ActivityRevision get() { - var activityRevision = activityHolder.activityRevision - - if (activityRevision == null) { - activityRevision = ActivityRevision().also { revision -> + if (!activityHolder.activityRevision.isInitializedByInterceptor) { + activityHolder.activityRevision.isInitializedByInterceptor = true + activityHolder.activityRevision.also { revision -> revision.authorId = userAccount?.id try { revision.projectId = projectHolder.project.id @@ -206,10 +205,9 @@ class InterceptedEventsManager( } revision.type = activityHolder.activity } - activityHolder.activityRevision = activityRevision } - return activityRevision + return activityHolder.activityRevision } private val userAccount: UserAccountDto? diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/AtomicProgressState.kt b/backend/data/src/main/kotlin/io/tolgee/batch/AtomicProgressState.kt new file mode 100644 index 0000000000..0b4eb5277c --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/AtomicProgressState.kt @@ -0,0 +1,87 @@ +package io.tolgee.batch + +import io.tolgee.component.LockingProvider +import io.tolgee.component.atomicLong.AtomicLongProvider +import io.tolgee.util.Logging +import io.tolgee.util.TolgeeAtomicLong +import io.tolgee.util.logger +import org.springframework.stereotype.Component +import java.math.BigInteger +import javax.persistence.EntityManager + +@Component +class AtomicProgressState( + private val atomicLongProvider: AtomicLongProvider, + private val entityManager: EntityManager, + private val lockingProvider: LockingProvider +) : Logging { + fun withAtomicState( + jobId: Long, + currentExecutionId: Long? = null, + fn: (totalProgress: TolgeeAtomicLong, chunkProgress: TolgeeAtomicLong) -> T + ): T { + val lock = lockingProvider.getLock("batch_job_atomic_state_$jobId") + lock.lock() + try { + return fn( + getProgressAtomicLong(jobId, currentExecutionId), + getCompletedChunksAtomicLong(jobId, currentExecutionId) + ) + } finally { + lock.unlock() + } + } + + fun getAtomicState(jobId: Long): Pair { + return this.withAtomicState(jobId) { progress, chunks -> + progress.get() to chunks.get() + } + } + + private fun getProgressAtomicLong(jobId: Long, currentExecutionId: Long? = null): TolgeeAtomicLong { + return atomicLongProvider.get("batch_job_progress_$jobId") { + getInitialProgress(jobId, currentExecutionId) + } + } + + private fun getCompletedChunksAtomicLong(jobId: Long, currentExecutionId: Long? = null): TolgeeAtomicLong { + return atomicLongProvider.get("batch_job_completed_chunks_$jobId") { + val initial = getInitialCompletedChunks(jobId, currentExecutionId) + logger.debug("Initial completed chunks: $initial") + initial + } + } + + fun getCompletedChunksCommittedAtomicLong(jobId: Long, currentExecutionId: Long? = null): TolgeeAtomicLong { + return atomicLongProvider.get("batch_job_completed_chunks_committed_$jobId") { + val initial = getInitialCompletedChunks(jobId, currentExecutionId) + logger.debug("Initial completed chunks: $initial") + initial + } + } + + private fun getInitialCompletedChunks(jobId: Long, currentExecutionId: Long?): Long { + return entityManager.createQuery( + """ + select count(id) from BatchJobChunkExecution bjce + where bjce.batchJob.id = :jobId and bjce.retry = false and bjce.status != 'PENDING' and bjce.id <> :currentId + """.trimIndent() + ) + .setParameter("jobId", jobId) + .setParameter("currentId", currentExecutionId) + .singleResult?.let { it as Long } ?: 0 + } + + private fun getInitialProgress(jobId: Long, currentExecutionId: Long?): Long { + return entityManager.createNativeQuery( + """ + select sum(jsonb_array_length(success_targets)) + from batch_job_chunk_execution + where batch_job_id = :jobId and id <> :currentId + """ + ) + .setParameter("jobId", jobId) + .setParameter("currentId", currentExecutionId ?: 0) + .singleResult?.let { (it as BigInteger).toLong() } ?: 0 + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt new file mode 100644 index 0000000000..c90650c5a9 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt @@ -0,0 +1,149 @@ +package io.tolgee.batch + +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.sentry.Sentry +import io.tolgee.component.UsingRedisProvider +import io.tolgee.model.batch.BatchJobChunkExecution +import io.tolgee.model.batch.BatchJobChunkExecutionStatus +import io.tolgee.pubSub.RedisPubSubReceiverConfiguration +import io.tolgee.util.Logging +import io.tolgee.util.executeInNewTransaction +import io.tolgee.util.logger +import org.hibernate.LockOptions +import org.springframework.boot.context.event.ApplicationReadyEvent +import org.springframework.context.ApplicationContext +import org.springframework.context.annotation.Lazy +import org.springframework.context.event.EventListener +import org.springframework.data.redis.core.StringRedisTemplate +import org.springframework.stereotype.Service +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.TransactionDefinition +import javax.persistence.EntityManager +import javax.persistence.LockModeType + +@Service +class BatchJobActionService( + private val entityManager: EntityManager, + private val transactionManager: PlatformTransactionManager, + private val applicationContext: ApplicationContext, + private val usingRedisProvider: UsingRedisProvider, + @Lazy + private val progressManager: ProgressManager, + @Lazy + private val batchJobService: BatchJobService, + private val jobChunkExecutionQueue: JobChunkExecutionQueue, + @Lazy + private val redisTemplate: StringRedisTemplate, + private val concurrentExecutionLauncher: BatchJobConcurrentLauncher +) : Logging { + companion object { + const val MIN_TIME_BETWEEN_OPERATIONS = 10 + } + + @EventListener(ApplicationReadyEvent::class) + fun run() { + println("Application ready") + executeInNewTransaction(transactionManager) { + jobChunkExecutionQueue.populateQueue() + } + + concurrentExecutionLauncher.run { executionItem, coroutineContext -> + var retryExecution: BatchJobChunkExecution? = null + val execution = executeInNewTransaction( + transactionManager, + isolationLevel = TransactionDefinition.ISOLATION_DEFAULT + ) { + catchingExceptions(executionItem) { + + val lockedExecution = getPendingUnlockedExecutionItem(executionItem) + ?: return@executeInNewTransaction null + + publishRemoveConsuming(executionItem) + + progressManager.handleJobRunning(lockedExecution.batchJob.id) + val batchJobDto = batchJobService.getJobDto(lockedExecution.batchJob.id) + + logger.debug("Job ${batchJobDto.id}: 🟡 Processing chunk ${lockedExecution.id}") + val util = ChunkProcessingUtil(lockedExecution, applicationContext, coroutineContext) + util.processChunk() + + progressManager.handleProgress(lockedExecution) + entityManager.persist(lockedExecution) + + if (lockedExecution.retry) { + retryExecution = util.retryExecution + entityManager.persist(util.retryExecution) + } + + logger.debug("Job ${batchJobDto.id}: ✅ Processed chunk ${lockedExecution.id}") + return@executeInNewTransaction lockedExecution + } + } + execution?.let { progressManager.handleChunkCompletedCommitted(it) } + addRetryExecutionToQueue(retryExecution) + } + } + + private fun getPendingUnlockedExecutionItem(executionItem: ExecutionQueueItem): BatchJobChunkExecution? { + val lockedExecution = getExecutionIfCanAcquireLock(executionItem.chunkExecutionId) + if (lockedExecution == null) { + logger.debug("⚠️ Chunk ${executionItem.chunkExecutionId} is locked, skipping") + return null + } + if (lockedExecution.status != BatchJobChunkExecutionStatus.PENDING) { + logger.debug("⚠️ Chunk ${executionItem.chunkExecutionId} is not pending, skipping") + return null + } + return lockedExecution + } + + private fun addRetryExecutionToQueue(retryExecution: BatchJobChunkExecution?) { + retryExecution?.let { + jobChunkExecutionQueue.addToQueue(listOf(it)) + logger.debug("Job ${it.batchJob.id}: Added chunk ${it.id} for re-trial") + } + } + + private inline fun catchingExceptions(executionItem: ExecutionQueueItem, fn: () -> T): T? { + return try { + fn() + } catch (e: Throwable) { + logger.error("Error processing chunk ${executionItem.chunkExecutionId}", e) + Sentry.captureException(e) + jobChunkExecutionQueue.addItemsToLocalQueue(listOf(executionItem)) + null + } + } + + fun publishRemoveConsuming(item: ExecutionQueueItem) { + if (usingRedisProvider.areWeUsingRedis) { + val message = jacksonObjectMapper() + .writeValueAsString(JobQueueItemsEvent(listOf(item), QueueEventType.REMOVE)) + redisTemplate.convertAndSend(RedisPubSubReceiverConfiguration.JOB_QUEUE_TOPIC, message) + } + } + + fun getExecutionIfCanAcquireLock(id: Long): BatchJobChunkExecution? { + entityManager.createNativeQuery("""SET enable_seqscan=off""") + return entityManager.createQuery( + """ + from BatchJobChunkExecution bjce + where bjce.id = :id + """.trimIndent(), + BatchJobChunkExecution::class.java + ) + .setParameter("id", id) + .setLockMode(LockModeType.PESSIMISTIC_WRITE) + .setHint( + "javax.persistence.lock.timeout", + LockOptions.SKIP_LOCKED + ).resultList.singleOrNull() + } + + fun cancelLocalJob(jobId: Long) { + jobChunkExecutionQueue.cancelJob(jobId) + concurrentExecutionLauncher.runningJobs.filter { it.value.first == jobId }.forEach { + it.value.second.cancel() + } + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActivityFinalizer.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActivityFinalizer.kt new file mode 100644 index 0000000000..465ab4e3a2 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActivityFinalizer.kt @@ -0,0 +1,153 @@ +package io.tolgee.batch + +import io.tolgee.activity.ActivityHolder +import io.tolgee.batch.events.OnBatchJobCancelled +import io.tolgee.batch.events.OnBatchJobFailed +import io.tolgee.batch.events.OnBatchJobSucceeded +import io.tolgee.batch.state.BatchJobStateProvider +import io.tolgee.fixtures.waitFor +import org.springframework.context.event.EventListener +import org.springframework.stereotype.Component +import javax.persistence.EntityManager + +@Component +class BatchJobActivityFinalizer( + private val entityManager: EntityManager, + private val activityHolder: ActivityHolder, + private val batchJobStateProvider: BatchJobStateProvider, +) { + @EventListener(OnBatchJobSucceeded::class) + fun finalizeActivityWhenJobSucceeded(event: OnBatchJobSucceeded) { + finalizeActivityWhenJobCompleted(event.job) + } + + @EventListener(OnBatchJobFailed::class) + fun finalizeActivityWhenJobFailed(event: OnBatchJobFailed) { + finalizeActivityWhenJobCompleted(event.job) + } + + @EventListener(OnBatchJobCancelled::class) + fun finalizeActivityWhenJobCancelled(event: OnBatchJobCancelled) { + finalizeActivityWhenJobCompleted(event.job) + } + + fun finalizeActivityWhenJobCompleted(job: BatchJobDto) { + val activityRevision = + activityHolder.activityRevision ?: throw IllegalStateException("Activity revision is not set") + + activityRevision.afterFlush = afterFlush@{ + waitForOtherChunksToComplete(job) + val revisionIds = getRevisionIds(job.id) + + val activityRevisionIdToMergeInto = revisionIds.firstOrNull() ?: return@afterFlush + revisionIds.remove(activityRevisionIdToMergeInto) + + mergeDescribingEntities(activityRevisionIdToMergeInto, revisionIds) + mergeModifiedEntities(activityRevisionIdToMergeInto, revisionIds) + deleteUnusedRevisions(revisionIds) + setJobIdToRevision(activityRevisionIdToMergeInto, job.id) + } + } + + private fun waitForOtherChunksToComplete(job: BatchJobDto) { + waitFor(20000) { + val committedChunks = batchJobStateProvider.get(job.id).values + .count { !it.retry && it.transactionCommitted && it.status.completed } + committedChunks == job.totalChunks - 1 + } + } + + private fun setJobIdToRevision(activityRevisionIdToMergeInto: Long, jobId: Long) { + entityManager.createNativeQuery( + """ + update activity_revision set batch_job_chunk_execution_id = null, batch_job_id = :jobId + where id = :activityRevisionIdToMergeInto + """ + ) + .setParameter("activityRevisionIdToMergeInto", activityRevisionIdToMergeInto) + .setParameter("jobId", jobId) + .executeUpdate() + } + + private fun deleteUnusedRevisions(revisionIds: MutableList) { + entityManager.createNativeQuery( + """ + delete from activity_revision where id in (:revisionIds) + """ + ) + .setParameter("revisionIds", revisionIds) + .executeUpdate() + } + + private fun mergeModifiedEntities( + activityRevisionIdToMergeInto: Long, + revisionIds: MutableList + ) { + entityManager.createNativeQuery( + """ + update activity_modified_entity set activity_revision_id = :activityRevisionIdToMergeInto + where activity_revision_id in (:revisionIds) + """ + ) + .setParameter("activityRevisionIdToMergeInto", activityRevisionIdToMergeInto) + .setParameter("revisionIds", revisionIds) + .executeUpdate() + } + + private fun mergeDescribingEntities( + activityRevisionIdToMergeInto: Long, + revisionIds: MutableList + ) { + removeDuplicityDescribingEntities(activityRevisionIdToMergeInto, revisionIds) + + entityManager.createNativeQuery( + """ + update activity_describing_entity set activity_revision_id = :activityRevisionIdToMergeInto + where activity_revision_id in (:revisionIds) + """ + ) + .setParameter("activityRevisionIdToMergeInto", activityRevisionIdToMergeInto) + .setParameter("revisionIds", revisionIds) + .executeUpdate() + } + + private fun removeDuplicityDescribingEntities( + activityRevisionIdToMergeInto: Long, + revisionIds: MutableList + ) { + entityManager.createNativeQuery( + """ + delete from activity_describing_entity + where (entity_class, entity_id) in + (select entity_class, entity_id + from activity_describing_entity + where activity_revision_id in (:revisionIds) + or activity_revision_id = :activityRevisionIdToMergeInto + group by entity_class, entity_id + having count(*) > 1) + and + activity_revision_id not in (select min(activity_revision_id) + from activity_describing_entity + where activity_revision_id in (:revisionIds) + or activity_revision_id = :activityRevisionIdToMergeInto + group by entity_class, entity_id + having count(*) > 1) + """.trimIndent() + ) + .setParameter("activityRevisionIdToMergeInto", activityRevisionIdToMergeInto) + .setParameter("revisionIds", revisionIds) + .executeUpdate() + } + + private fun getRevisionIds(jobId: Long): MutableList = entityManager.createQuery( + """ + select ar.id + from ActivityRevision ar + join ar.batchJobChunkExecution b + where b.batchJob.id = :jobId + """, + Long::class.javaObjectType + ) + .setParameter("jobId", jobId) + .resultList +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobCancellationManager.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobCancellationManager.kt new file mode 100644 index 0000000000..708bd4ca05 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobCancellationManager.kt @@ -0,0 +1,79 @@ +package io.tolgee.batch + +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.tolgee.component.UsingRedisProvider +import io.tolgee.model.batch.BatchJobChunkExecution +import io.tolgee.model.batch.BatchJobChunkExecutionStatus +import io.tolgee.pubSub.RedisPubSubReceiverConfiguration +import io.tolgee.util.executeInNewTransaction +import org.hibernate.LockOptions +import org.springframework.context.annotation.Lazy +import org.springframework.context.event.EventListener +import org.springframework.data.redis.core.StringRedisTemplate +import org.springframework.stereotype.Component +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.TransactionDefinition +import org.springframework.transaction.annotation.Transactional +import javax.persistence.EntityManager +import javax.persistence.LockModeType + +@Component +class BatchJobCancellationManager( + private val usingRedisProvider: UsingRedisProvider, + @Lazy + private val redisTemplate: StringRedisTemplate, + private val entityManager: EntityManager, + private val transactionManager: PlatformTransactionManager, + private val batchJobActionService: BatchJobActionService, + private val progressManager: ProgressManager +) { + @Transactional + fun cancel(id: Long) { + cancelJob(id) + if (usingRedisProvider.areWeUsingRedis) { + redisTemplate.convertAndSend( + RedisPubSubReceiverConfiguration.JOB_CANCEL_TOPIC, + jacksonObjectMapper().writeValueAsString(id) + ) + return + } + batchJobActionService.cancelLocalJob(id) + } + + @EventListener(JobCancelEvent::class) + fun cancelJobListener(event: JobCancelEvent) { + batchJobActionService.cancelLocalJob(event.jobId) + } + + fun cancelJob(jobId: Long) { + executeInNewTransaction( + transactionManager = transactionManager, + isolationLevel = TransactionDefinition.ISOLATION_DEFAULT + ) { + entityManager.createNativeQuery("""SET enable_seqscan=off""") + val executions = entityManager.createQuery( + """ + from BatchJobChunkExecution bjce + where bjce.batchJob.id = :id + and status = :status + """, + BatchJobChunkExecution::class.java + ) + .setLockMode(LockModeType.PESSIMISTIC_WRITE) + .setHint( + "javax.persistence.lock.timeout", + LockOptions.SKIP_LOCKED + ) + .setParameter("id", jobId) + .setParameter("status", BatchJobChunkExecutionStatus.PENDING) + .resultList + + executions.forEach { execution -> + execution.status = BatchJobChunkExecutionStatus.CANCELLED + entityManager.persist(execution) + } + + executions.forEach { progressManager.handleProgress(it) } + } + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobConcurrentLauncher.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobConcurrentLauncher.kt new file mode 100644 index 0000000000..38d4216a89 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobConcurrentLauncher.kt @@ -0,0 +1,153 @@ +package io.tolgee.batch + +import io.sentry.Sentry +import io.tolgee.component.CurrentDateProvider +import io.tolgee.configuration.tolgee.BatchProperties +import io.tolgee.util.Logging +import io.tolgee.util.logger +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import org.springframework.stereotype.Component +import java.util.concurrent.ConcurrentHashMap +import javax.annotation.PreDestroy +import kotlin.coroutines.CoroutineContext + +@Component +class BatchJobConcurrentLauncher( + private val batchProperties: BatchProperties, + private val jobChunkExecutionQueue: JobChunkExecutionQueue, + private val currentDateProvider: CurrentDateProvider +) : Logging { + companion object { + val runningInstances: ConcurrentHashMap.KeySetView = + ConcurrentHashMap.newKeySet() + } + + val runningJobs: ConcurrentHashMap> = ConcurrentHashMap() + var pause = false + var masterRunJob: Job? = null + var run = true + + fun stop() { + logger.trace("Stopping batch job launcher ${System.identityHashCode(this)}}") + run = false + runBlocking(Dispatchers.IO) { + masterRunJob?.join() + } + logger.trace("Batch job launcher stopped ${System.identityHashCode(this)}") + runningInstances.remove(this) + } + + @PreDestroy + fun preDestroy() { + this.stop() + } + + fun repeatForever(fn: () -> Unit) { + runningInstances.forEach { it.stop() } + runningInstances.add(this) + + logger.trace("Started batch job action service ${System.identityHashCode(this)}") + while (run) { + try { + val startTime = System.currentTimeMillis() + fn() + val sleepTime = getSleepTime(startTime) + if (sleepTime > 0) { + Thread.sleep(sleepTime) + } + } catch (e: Throwable) { + Sentry.captureException(e) + logger.error("Error in batch job action service", e) + } + } + } + + private fun getSleepTime(startTime: Long): Long { + if (!jobChunkExecutionQueue.isEmpty() && jobsToLaunch > 0) { + return 0 + } + return BatchJobActionService.MIN_TIME_BETWEEN_OPERATIONS - (System.currentTimeMillis() - startTime) + } + + fun run(processExecution: (executionItem: ExecutionQueueItem, coroutineContext: CoroutineContext) -> Unit) { + @Suppress("OPT_IN_USAGE") + masterRunJob = GlobalScope.launch(Dispatchers.IO) { + repeatForever { + if (pause) { + return@repeatForever + } + + val jobsToLaunch = jobsToLaunch + if (jobsToLaunch <= 0) { + return@repeatForever + } + + logger.trace("Jobs to launch: $jobsToLaunch") + val items = (1..jobsToLaunch) + .mapNotNull { jobChunkExecutionQueue.poll() } + + logItemsPulled(items) + + items.forEach { executionItem -> + handleItem(executionItem, processExecution) + } + } + } + } + + private fun logItemsPulled(items: List) { + if (items.isNotEmpty()) { + logger.debug( + "Pulled ${items.size} items from queue: " + + items.joinToString(", ") { it.chunkExecutionId.toString() } + ) + } + logger.debug( + "${jobChunkExecutionQueue.size} is left in the queue (${System.identityHashCode(jobChunkExecutionQueue)}): " + + jobChunkExecutionQueue.joinToString(", ") { it.chunkExecutionId.toString() } + ) + } + + private fun CoroutineScope.handleItem( + executionItem: ExecutionQueueItem, + processExecution: (executionItem: ExecutionQueueItem, coroutineContext: CoroutineContext) -> Unit + ) { + if (!executionItem.isTimeToExecute()) { + logger.debug( + """Execution ${executionItem.chunkExecutionId} not ready to execute, adding back to queue: + | Difference ${executionItem.executeAfter!! - currentDateProvider.date.time}""".trimMargin() + ) + jobChunkExecutionQueue.addItemsToLocalQueue(listOf(executionItem)) + return + } + + val job = launch { + processExecution(executionItem, this.coroutineContext) + } + + runningJobs[executionItem.chunkExecutionId] = executionItem.jobId to job + + job.invokeOnCompletion { + onJobCompleted(executionItem) + } + logger.debug("Execution ${executionItem.chunkExecutionId} launched. Running jobs: ${runningJobs.size}") + } + + private fun onJobCompleted(executionItem: ExecutionQueueItem) { + runningJobs.remove(executionItem.chunkExecutionId) + logger.debug("Chunk ${executionItem.chunkExecutionId}: Completed") + logger.debug("Running jobs: ${runningJobs.size}") + } + + private val jobsToLaunch get() = batchProperties.concurrency - runningJobs.size + + fun ExecutionQueueItem.isTimeToExecute(): Boolean { + val executeAfter = this.executeAfter ?: return true + return executeAfter <= currentDateProvider.date.time + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobDto.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobDto.kt new file mode 100644 index 0000000000..1266c7e243 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobDto.kt @@ -0,0 +1,35 @@ +package io.tolgee.batch + +import io.tolgee.model.batch.BatchJob +import io.tolgee.model.batch.BatchJobStatus +import io.tolgee.model.batch.IBatchJob + +class BatchJobDto( + override var id: Long, + val projectId: Long, + val authorId: Long?, + val target: List, + val totalItems: Int, + val totalChunks: Int, + val chunkSize: Int, + override var status: BatchJobStatus, + val type: BatchJobType +) : IBatchJob { + val chunkedTarget get() = BatchJob.chunkTarget(chunkSize, target) + + companion object { + fun fromEntity(entity: BatchJob): BatchJobDto { + return BatchJobDto( + id = entity.id, + projectId = entity.project.id, + authorId = entity.author?.id, + target = entity.target, + totalItems = entity.totalItems, + totalChunks = entity.totalChunks, + chunkSize = entity.chunkSize, + status = entity.status, + type = entity.type, + ) + } + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt new file mode 100644 index 0000000000..6eae82a287 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt @@ -0,0 +1,157 @@ +package io.tolgee.batch + +import io.tolgee.constants.Message +import io.tolgee.dtos.cacheable.UserAccountDto +import io.tolgee.exceptions.NotFoundException +import io.tolgee.model.Project +import io.tolgee.model.UserAccount +import io.tolgee.model.batch.BatchJob +import io.tolgee.model.batch.BatchJobChunkExecution +import io.tolgee.model.batch.BatchJobStatus +import io.tolgee.model.batch.IBatchJob +import io.tolgee.model.views.BatchJobView +import io.tolgee.repository.BatchJobRepository +import io.tolgee.util.Logging +import io.tolgee.util.executeInNewTransaction +import io.tolgee.util.logger +import org.springframework.context.ApplicationContext +import org.springframework.context.annotation.Lazy +import org.springframework.data.domain.Page +import org.springframework.data.domain.Pageable +import org.springframework.stereotype.Service +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.annotation.Transactional +import java.math.BigInteger +import javax.persistence.EntityManager + +@Service +class BatchJobService( + private val batchJobRepository: BatchJobRepository, + private val entityManager: EntityManager, + private val applicationContext: ApplicationContext, + private val transactionManager: PlatformTransactionManager, + private val cachingBatchJobService: CachingBatchJobService, + @Lazy + private val progressManager: ProgressManager, + private val jobChunkExecutionQueue: JobChunkExecutionQueue +) : Logging { + + @Transactional + fun startJob( + request: RequestType, + project: Project, + author: UserAccount?, + type: BatchJobType + ): BatchJob { + var executions: List? = null + val job = executeInNewTransaction(transactionManager) { + val processor = getProcessor(type) + val target = processor.getTarget(request) + + val job = BatchJob().apply { + this.project = project + this.author = author + this.target = target + this.totalItems = target.size + this.chunkSize = type.chunkSize + this.type = type + } + val chunked = job.chunkedTarget + job.totalChunks = chunked.size + cachingBatchJobService.saveJob(job) + + val params = processor.getParams(request, job) + + params?.let { + entityManager.persist(params) + } + + executions = chunked.mapIndexed { chunkNumber, _ -> + BatchJobChunkExecution().apply { + batchJob = job + this.chunkNumber = chunkNumber + entityManager.persist(this) + } + } + job + } + + executions?.let { jobChunkExecutionQueue.addToQueue(it) } + logger.debug( + "Starting job ${job.id}, aadded ${executions?.size} executions to queue ${ + System.identityHashCode( + jobChunkExecutionQueue + ) + }" + ) + + return job + } + + fun findJobEntity(id: Long): BatchJob? { + return batchJobRepository.findById(id).orElse(null) + } + + fun getJobEntity(id: Long): BatchJob { + return findJobEntity(id) ?: throw NotFoundException(io.tolgee.constants.Message.BATCH_JOB_NOT_FOUND) + } + + fun findJobDto(id: Long): BatchJobDto? { + return cachingBatchJobService.findJobDto(id) + } + + fun getJobDto(id: Long): BatchJobDto { + return this.findJobDto(id) ?: throw NotFoundException(io.tolgee.constants.Message.BATCH_JOB_NOT_FOUND) + } + + fun getViews(projectId: Long, userAccount: UserAccountDto?, pageable: Pageable): Page { + val jobs = batchJobRepository.getJobs(projectId, userAccount?.id, pageable) + + val progresses = getProgresses(jobs) + + val errorMessages = getErrorMessages(jobs) + + return jobs.map { + BatchJobView(it, progresses[it.id] ?: 0, errorMessages[it.id]) + } + } + + fun getErrorMessages(jobs: Iterable): Map { + val needsErrorMessage = jobs.filter { it.status == BatchJobStatus.FAILED }.map { it.id }.toList() + + return batchJobRepository.getErrorMessages(needsErrorMessage) + .groupBy { it.batchJobId } + .mapValues { it.value.minBy { value -> value.updatedAt }.errorMessage } + } + + private fun getProgresses(jobs: Iterable): Map { + val cachedProgresses = + jobs.associate { + it.id to + if (it.status == BatchJobStatus.RUNNING) + progressManager.getJobCachedProgress(jobId = it.id) + else + null + } + val needsProgress = cachedProgresses.filter { it.value == null }.map { it.key }.toList() + val progresses = batchJobRepository.getProgresses(needsProgress) + .associate { (it[0] as BigInteger).toLong() to it[1] as BigInteger } + + return jobs.associate { it.id to (cachedProgresses[it.id] ?: progresses[it.id]?.toLong() ?: 0).toInt() } + } + + fun getView(jobId: Long): BatchJobView { + val job = batchJobRepository.findById(jobId).orElseThrow { NotFoundException() } + return getView(job) + } + + fun getView(job: BatchJob): BatchJobView { + val progress = getProgresses(listOf(job))[job.id] ?: 0 + val errorMessage = getErrorMessages(listOf(job))[job.id] + return BatchJobView(job, progress, errorMessage) + } + + @Suppress("UNCHECKED_CAST") + fun getProcessor(type: BatchJobType): ChunkProcessor = + applicationContext.getBean(type.processor.java) as ChunkProcessor +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobType.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobType.kt new file mode 100644 index 0000000000..e58b728af0 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobType.kt @@ -0,0 +1,30 @@ +package io.tolgee.batch + +import io.tolgee.activity.data.ActivityType +import io.tolgee.batch.processors.DeleteKeysChunkProcessor +import io.tolgee.batch.processors.TranslationChunkProcessor +import kotlin.reflect.KClass + +enum class BatchJobType( + val activityType: ActivityType, + /** + * 0 means no chunking + */ + val chunkSize: Int, + val maxRetries: Int, + val processor: KClass>, + val defaultRetryWaitTimeInMs: Int = 2000, +) { + TRANSLATION( + activityType = ActivityType.BATCH_AUTO_TRANSLATE, + chunkSize = 10, + maxRetries = 3, + processor = TranslationChunkProcessor::class, + ), + DELETE_KEYS( + activityType = ActivityType.KEY_DELETE, + chunkSize = 0, + maxRetries = 3, + processor = DeleteKeysChunkProcessor::class, + ); +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/CachingBatchJobService.kt b/backend/data/src/main/kotlin/io/tolgee/batch/CachingBatchJobService.kt new file mode 100644 index 0000000000..a5d1aecd16 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/CachingBatchJobService.kt @@ -0,0 +1,59 @@ +package io.tolgee.batch + +import io.tolgee.constants.Caches +import io.tolgee.model.batch.BatchJob +import io.tolgee.model.batch.BatchJobStatus +import io.tolgee.repository.BatchJobRepository +import org.springframework.cache.annotation.CacheEvict +import org.springframework.cache.annotation.Cacheable +import org.springframework.context.annotation.Lazy +import org.springframework.stereotype.Service +import org.springframework.transaction.annotation.Transactional +import javax.persistence.EntityManager + +@Service +class CachingBatchJobService( + private val batchJobRepository: BatchJobRepository, + @Lazy + private val batchJobService: BatchJobService, + private val entityManager: EntityManager +) { + + @Transactional + @CacheEvict( + cacheNames = [Caches.BATCH_JOBS], + key = "#result.id" + ) + fun saveJob(batchJob: BatchJob): BatchJob { + return batchJobRepository.save(batchJob) + } + + @Transactional + @CacheEvict( + cacheNames = [Caches.BATCH_JOBS], + key = "#jobId" + ) + fun setRunningState(jobId: Long) { + entityManager.createQuery("""update BatchJob set status = :status where id = :id and status = :pendingStatus""") + .setParameter("status", BatchJobStatus.RUNNING) + .setParameter("id", jobId) + .setParameter("pendingStatus", BatchJobStatus.PENDING) + .executeUpdate() + } + + @Cacheable( + cacheNames = [Caches.BATCH_JOBS], + key = "#id" + ) + fun findJobDto(id: Long): BatchJobDto? { + val entity = batchJobService.findJobEntity(id) ?: return null + return BatchJobDto.fromEntity(entity) + } + + @CacheEvict( + cacheNames = [Caches.BATCH_JOBS], + key = "#jobId" + ) + fun evictJobCache(jobId: Long) { + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/ChunkProcessingUtil.kt b/backend/data/src/main/kotlin/io/tolgee/batch/ChunkProcessingUtil.kt new file mode 100644 index 0000000000..14ddbf0091 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/ChunkProcessingUtil.kt @@ -0,0 +1,163 @@ +package io.tolgee.batch + +import io.sentry.Sentry +import io.tolgee.activity.ActivityHolder +import io.tolgee.component.CurrentDateProvider +import io.tolgee.exceptions.ExceptionWithMessage +import io.tolgee.model.batch.BatchJobChunkExecution +import io.tolgee.model.batch.BatchJobChunkExecutionStatus +import io.tolgee.util.Logging +import io.tolgee.util.logger +import org.hibernate.LockOptions +import org.springframework.context.ApplicationContext +import java.util.* +import javax.persistence.EntityManager +import kotlin.coroutines.CoroutineContext +import kotlin.math.pow + +open class ChunkProcessingUtil( + val execution: BatchJobChunkExecution, + private val applicationContext: ApplicationContext, + private val coroutineContext: CoroutineContext, +) : Logging { + open fun processChunk() { + try { + val processor = batchJobService.getProcessor(job.type) + processor.process(job, toProcess, coroutineContext) { + if (it != toProcess.size) { + progressManager.publishSingleChunkProgress(job.id, it) + } + } + successfulTargets = toProcess + execution.status = BatchJobChunkExecutionStatus.SUCCESS + handleActivity() + } catch (e: Throwable) { + handleException(e) + } finally { + successfulTargets?.let { + execution.successTargets = it + } + } + } + + private fun handleActivity() { + val activityRevision = activityHolder.activityRevision + activityRevision.batchJobChunkExecution = execution + val batchJobDto = batchJobService.getJobDto(job.id) + activityRevision.projectId = batchJobDto.projectId + activityHolder.activity = batchJobDto.type.activityType + } + + private fun handleException(exception: Throwable) { + if (exception is kotlinx.coroutines.CancellationException) { + execution.status = BatchJobChunkExecutionStatus.CANCELLED + return + } + + execution.exception = exception.stackTraceToString() + execution.status = BatchJobChunkExecutionStatus.FAILED + execution.errorMessage = (exception as? ExceptionWithMessage)?.tolgeeMessage + + Sentry.captureException(exception) + logger.error(exception.message, exception) + + if (exception is ChunkFailedException) { + successfulTargets = exception.successfulTargets + successfulTargets?.let { execution.successTargets = it } + } + + if (exception is FailedDontRequeueException) { + return + } + + retryFailedExecution(exception) + } + + private fun retryFailedExecution(exception: Throwable) { + var maxRetries = job.type.maxRetries + var waitTime = job.type.defaultRetryWaitTimeInMs + + if (exception is RequeueWithTimeoutException) { + maxRetries = exception.maxRetries + waitTime = getWaitTime(exception) + } + + if (retries >= maxRetries) { + logger.debug("Max retries reached for job execution $execution") + Sentry.captureException(exception) + return + } + + logger.debug("Retrying job execution $execution in ${waitTime}ms") + retryExecution.executeAfter = Date(waitTime + currentDateProvider.date.time) + execution.retry = true + } + + private fun getWaitTime(exception: RequeueWithTimeoutException) = + exception.timeoutInMs * (exception.increaseFactor.toDouble().pow(retries.toDouble())).toInt() + + private val job by lazy { batchJobService.getJobDto(execution.batchJob.id) } + + private val activityHolder by lazy { + applicationContext.getBean(ActivityHolder::class.java) + } + + private val entityManager by lazy { + applicationContext.getBean(EntityManager::class.java) + } + + private val currentDateProvider by lazy { + applicationContext.getBean(CurrentDateProvider::class.java) + } + + private val batchJobService by lazy { + applicationContext.getBean(BatchJobService::class.java) + } + + private val progressManager by lazy { + applicationContext.getBean(ProgressManager::class.java) + } + + private var successfulTargets: List? = null + + private val toProcess by lazy { + val chunked = job.chunkedTarget + val chunk = chunked[execution.chunkNumber] + val previousSuccessfulTargets = previousExecutions.flatMap { it.successTargets }.toSet() + val toProcess = chunk.toMutableSet() + toProcess.removeAll(previousSuccessfulTargets) + toProcess.toList() + } + + val retryExecution: BatchJobChunkExecution by lazy { + BatchJobChunkExecution().apply { + batchJob = entityManager.getReference(execution.batchJob::class.java, job.id) + chunkNumber = execution.chunkNumber + status = BatchJobChunkExecutionStatus.PENDING + } + } + + private val retries: Int by lazy { + previousExecutions.size + } + + @Suppress("UNCHECKED_CAST") + private val previousExecutions: List by lazy { + entityManager.createQuery( + """ + from BatchJobChunkExecution + where chunkNumber = :chunkNumber + and batchJob.id = :batchJobId + and status = :status + """.trimIndent() + ) + .setParameter("chunkNumber", execution.chunkNumber) + .setParameter("batchJobId", job.id) + .setParameter("status", BatchJobChunkExecutionStatus.FAILED) + .setHint( + "javax.persistence.lock.timeout", + LockOptions.NO_WAIT + ) + .resultList as List + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/ChunkProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/batch/ChunkProcessor.kt new file mode 100644 index 0000000000..4bb94a4a24 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/ChunkProcessor.kt @@ -0,0 +1,11 @@ +package io.tolgee.batch + +import io.tolgee.model.EntityWithId +import io.tolgee.model.batch.BatchJob +import kotlin.coroutines.CoroutineContext + +interface ChunkProcessor { + fun process(job: BatchJobDto, chunk: List, coroutineContext: CoroutineContext, onProgress: ((Int) -> Unit)) + fun getTarget(data: RequestType): List + fun getParams(data: RequestType, job: BatchJob): EntityWithId? +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/ExecutionQueueItem.kt b/backend/data/src/main/kotlin/io/tolgee/batch/ExecutionQueueItem.kt new file mode 100644 index 0000000000..628c6d6ab8 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/ExecutionQueueItem.kt @@ -0,0 +1,7 @@ +package io.tolgee.batch + +data class ExecutionQueueItem( + val chunkExecutionId: Long, + val jobId: Long, + val executeAfter: Long? +) diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/JobCancelEvent.kt b/backend/data/src/main/kotlin/io/tolgee/batch/JobCancelEvent.kt new file mode 100644 index 0000000000..2dba05ee50 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/JobCancelEvent.kt @@ -0,0 +1,5 @@ +package io.tolgee.batch + +class JobCancelEvent( + val jobId: Long +) diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/JobChunkExecutionQueue.kt b/backend/data/src/main/kotlin/io/tolgee/batch/JobChunkExecutionQueue.kt new file mode 100644 index 0000000000..6efc677595 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/JobChunkExecutionQueue.kt @@ -0,0 +1,118 @@ +package io.tolgee.batch + +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.tolgee.component.UsingRedisProvider +import io.tolgee.model.batch.BatchJobChunkExecution +import io.tolgee.model.batch.BatchJobChunkExecutionStatus +import io.tolgee.pubSub.RedisPubSubReceiverConfiguration +import io.tolgee.util.Logging +import org.hibernate.LockOptions +import org.springframework.context.annotation.Lazy +import org.springframework.context.event.EventListener +import org.springframework.data.redis.core.StringRedisTemplate +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Component +import java.util.* +import java.util.concurrent.ConcurrentLinkedQueue +import javax.persistence.EntityManager + +@Component +class JobChunkExecutionQueue( + private val entityManager: EntityManager, + private val usingRedisProvider: UsingRedisProvider, + @Lazy + private val redisTemplate: StringRedisTemplate + +) : Logging { + companion object { + /** + * It's static + */ + private val queue = ConcurrentLinkedQueue() + } + + @EventListener(JobQueueItemsEvent::class) + fun onJobItemEvent(event: JobQueueItemsEvent) { + when (event.type) { + QueueEventType.ADD -> this.addItemsToLocalQueue(event.items) + QueueEventType.REMOVE -> queue.removeAll(event.items.toSet()) + } + } + + @Scheduled(fixedDelay = 60000) + fun populateQueue() { + val data = entityManager.createQuery( + """ + from BatchJobChunkExecution bjce + join fetch bjce.batchJob bk + where bjce.status = :executionStatus + order by bjce.createdAt asc, bjce.executeAfter asc, bjce.id asc + """.trimIndent(), + BatchJobChunkExecution::class.java + ).setParameter("executionStatus", BatchJobChunkExecutionStatus.PENDING) + .setHint( + "javax.persistence.lock.timeout", + LockOptions.SKIP_LOCKED + ).resultList + addExecutionsToLocalQueue(data) + } + + fun addExecutionsToLocalQueue(data: List) { + val ids = queue.map { it.chunkExecutionId }.toSet() + data.forEach { + if (!ids.contains(it.id)) { + queue.add(it.toItem()) + } + } + } + + fun addItemsToLocalQueue(data: List) { + data.forEach { + if (!queue.contains(it)) { + queue.add(it) + } + } + } + + fun addToQueue(executions: List) { + if (usingRedisProvider.areWeUsingRedis) { + val items = executions.map { it.toItem() } + val event = JobQueueItemsEvent(items, QueueEventType.ADD) + redisTemplate.convertAndSend( + RedisPubSubReceiverConfiguration.JOB_QUEUE_TOPIC, + jacksonObjectMapper().writeValueAsString(event) + ) + return + } + this.addExecutionsToLocalQueue(executions) + } + + fun cancelJob(jobId: Long) { + queue.removeIf { it.jobId == jobId } + } + + private fun BatchJobChunkExecution.toItem() = + ExecutionQueueItem(id, batchJob.id, executeAfter?.time) + + val size get() = queue.size + + fun joinToString(separator: String = ", ", transform: (item: ExecutionQueueItem) -> String) = + queue.joinToString(separator, transform = transform) + + fun poll(): ExecutionQueueItem? { + return queue.poll() + } + + fun clear() { + queue.clear() + } + + fun find(function: (ExecutionQueueItem) -> Boolean): ExecutionQueueItem? { + return queue.find(function) + } + + fun peek(): ExecutionQueueItem = queue.peek() + fun contains(item: ExecutionQueueItem?): Boolean = queue.contains(item) + + fun isEmpty(): Boolean = queue.isEmpty() +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/JobQueueItemsEvent.kt b/backend/data/src/main/kotlin/io/tolgee/batch/JobQueueItemsEvent.kt new file mode 100644 index 0000000000..34d368f9cd --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/JobQueueItemsEvent.kt @@ -0,0 +1,6 @@ +package io.tolgee.batch + +data class JobQueueItemsEvent( + val items: List, + val type: QueueEventType +) diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/OnBatchJobCompleted.kt b/backend/data/src/main/kotlin/io/tolgee/batch/OnBatchJobCompleted.kt new file mode 100644 index 0000000000..fee8fadea7 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/OnBatchJobCompleted.kt @@ -0,0 +1,5 @@ +package io.tolgee.batch + +interface OnBatchJobCompleted { + val job: BatchJobDto +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/ProgressManager.kt b/backend/data/src/main/kotlin/io/tolgee/batch/ProgressManager.kt new file mode 100644 index 0000000000..517df09d8d --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/ProgressManager.kt @@ -0,0 +1,177 @@ +package io.tolgee.batch + +import io.tolgee.batch.events.OnBatchJobCancelled +import io.tolgee.batch.events.OnBatchJobFailed +import io.tolgee.batch.events.OnBatchJobProgress +import io.tolgee.batch.events.OnBatchJobStatusUpdated +import io.tolgee.batch.events.OnBatchJobSucceeded +import io.tolgee.batch.state.BatchJobStateProvider +import io.tolgee.batch.state.ExecutionState +import io.tolgee.constants.Message +import io.tolgee.model.batch.BatchJob +import io.tolgee.model.batch.BatchJobChunkExecution +import io.tolgee.model.batch.BatchJobChunkExecutionStatus +import io.tolgee.model.batch.BatchJobStatus +import io.tolgee.util.Logging +import io.tolgee.util.executeInNewTransaction +import io.tolgee.util.logger +import org.springframework.context.ApplicationEventPublisher +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Component +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.TransactionDefinition +import org.springframework.transaction.event.TransactionalEventListener +import javax.persistence.EntityManager + +@Component +class ProgressManager( + private val entityManager: EntityManager, + private val eventPublisher: ApplicationEventPublisher, + private val transactionManager: PlatformTransactionManager, + private val batchJobService: BatchJobService, + private val batchJobStateProvider: BatchJobStateProvider, + private val cachingBatchJobService: CachingBatchJobService +) : Logging { + + fun handleProgress(execution: BatchJobChunkExecution) { + val job = batchJobService.getJobDto(execution.batchJob.id) + + val info = batchJobStateProvider.updateState(job.id) { + it[execution.id] = + ExecutionState( + successTargets = execution.successTargets, + status = execution.status, + chunkNumber = execution.chunkNumber, + retry = execution.retry, + transactionCommitted = false + ) + + it.getInfoForJobResult() + } + + if (execution.successTargets.isNotEmpty()) { + eventPublisher.publishEvent(OnBatchJobProgress(job, info.progress, job.totalItems.toLong())) + } + + handleJobStatus( + job, + progress = info.progress, + isAnyCancelled = info.isAnyCancelled, + completedChunks = info.completedChunks + ) + } + + fun handleChunkCompletedCommitted(execution: BatchJobChunkExecution) { + batchJobStateProvider.updateState(execution.batchJob.id) { + it.compute(execution.id) { _, v -> + v?.copy(transactionCommitted = true) + } + } + } + + fun handleJobStatus( + job: BatchJobDto, + progress: Long, + completedChunks: Long, + isAnyCancelled: Boolean, + errorMessage: Message? = null + ) { + logger.debug("Job ${job.id} completed chunks: $completedChunks of ${job.totalChunks}") + logger.debug("Job ${job.id} progress: $progress of ${job.totalItems}") + + if (job.totalChunks.toLong() != completedChunks) { + return + } + + val jobEntity = batchJobService.getJobEntity(job.id) + + try { + if (isAnyCancelled) { + jobEntity.status = BatchJobStatus.CANCELLED + cachingBatchJobService.saveJob(jobEntity) + eventPublisher.publishEvent(OnBatchJobCancelled(jobEntity.dto)) + return + } + + if (job.totalItems.toLong() != progress) { + jobEntity.status = BatchJobStatus.FAILED + cachingBatchJobService.saveJob(jobEntity) + val errorMessage = errorMessage ?: batchJobService.getErrorMessages(listOf(job))[job.id] + eventPublisher.publishEvent(OnBatchJobFailed(jobEntity.dto, errorMessage)) + return + } + + jobEntity.status = BatchJobStatus.SUCCESS + logger.debug("Publishing success event for job ${job.id}") + eventPublisher.publishEvent(OnBatchJobSucceeded(jobEntity.dto)) + cachingBatchJobService.saveJob(jobEntity) + } finally { + eventPublisher.publishEvent(OnBatchJobStatusUpdated(job.id)) + } + } + + @TransactionalEventListener(OnBatchJobStatusUpdated::class) + fun handleJobStatusUpdated(event: OnBatchJobStatusUpdated) { + cachingBatchJobService.evictJobCache(event.jobId) + } + + fun Map.getInfoForJobResult(): JobResultInfo { + var completedChunks = 0L + var progress = 0L + this.values.forEach { + if (it.status.completed && !it.retry) completedChunks++ + progress += it.successTargets.size + } + val isAnyCancelled = this.values.any { it.status == BatchJobChunkExecutionStatus.CANCELLED } + return JobResultInfo(completedChunks, progress, isAnyCancelled) + } + + @Scheduled(fixedRate = 60 * 1000) + fun updateProgress() { + executeInNewTransaction(transactionManager) { + val jobs = entityManager.createQuery( + """ + select bj from BatchJob bj + where bj.status = :pendingStatus or bj.status = :runningStatus + """, + BatchJob::class.java + ).setParameter("pendingStatus", BatchJobStatus.PENDING) + .setParameter("runningStatus", BatchJobStatus.RUNNING) + .resultList + + jobs.forEach { job -> + val state = batchJobStateProvider.get(job.id) + val info = state.getInfoForJobResult() + // let's not keep the locked when we handle the status + handleJobStatus( + BatchJobDto.fromEntity(job), + progress = info.progress, + completedChunks = info.completedChunks, + info.isAnyCancelled + ) + } + } + } + + fun getJobCachedProgress(jobId: Long): Long? { + return batchJobStateProvider.getCached(jobId)?.getInfoForJobResult()?.progress + } + + fun publishSingleChunkProgress(jobId: Long, progress: Int) { + val job = batchJobService.getJobDto(jobId) + eventPublisher.publishEvent(OnBatchJobProgress(job, progress.toLong(), job.totalItems.toLong())) + } + + fun handleJobRunning(id: Long) { + executeInNewTransaction(transactionManager, isolationLevel = TransactionDefinition.ISOLATION_DEFAULT) { + logger.trace("""Fetching job $id""") + val job = batchJobService.getJobDto(id) + if (job.status == BatchJobStatus.PENDING) { + logger.debug("""Updating job state to running ${job.id}""") + cachingBatchJobService.setRunningState(job.id) + } + } + } + + data class JobResultInfo(val completedChunks: Long, val progress: Long, val isAnyCancelled: Boolean) +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/QueueEventType.kt b/backend/data/src/main/kotlin/io/tolgee/batch/QueueEventType.kt new file mode 100644 index 0000000000..17ceb26b25 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/QueueEventType.kt @@ -0,0 +1,5 @@ +package io.tolgee.batch + +enum class QueueEventType { + ADD, REMOVE +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/WebsocketProgressInfo.kt b/backend/data/src/main/kotlin/io/tolgee/batch/WebsocketProgressInfo.kt new file mode 100644 index 0000000000..3ddf34c767 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/WebsocketProgressInfo.kt @@ -0,0 +1,11 @@ +package io.tolgee.batch + +import io.tolgee.model.batch.BatchJobStatus + +data class WebsocketProgressInfo( + val jobId: Long, + val processed: Long?, + val total: Long?, + val status: BatchJobStatus, + val errorMessage: String? = null, +) diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/events/OnBatchJobCancelled.kt b/backend/data/src/main/kotlin/io/tolgee/batch/events/OnBatchJobCancelled.kt new file mode 100644 index 0000000000..31d475a9f5 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/events/OnBatchJobCancelled.kt @@ -0,0 +1,8 @@ +package io.tolgee.batch.events + +import io.tolgee.batch.BatchJobDto +import io.tolgee.batch.OnBatchJobCompleted + +data class OnBatchJobCancelled( + override val job: BatchJobDto, +) : OnBatchJobCompleted diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/events/OnBatchJobFailed.kt b/backend/data/src/main/kotlin/io/tolgee/batch/events/OnBatchJobFailed.kt new file mode 100644 index 0000000000..373e6bcfed --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/events/OnBatchJobFailed.kt @@ -0,0 +1,10 @@ +package io.tolgee.batch.events + +import io.tolgee.batch.BatchJobDto +import io.tolgee.batch.OnBatchJobCompleted +import io.tolgee.constants.Message + +data class OnBatchJobFailed( + override val job: BatchJobDto, + val errorMessage: Message?, +) : OnBatchJobCompleted diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/events/OnBatchJobProgress.kt b/backend/data/src/main/kotlin/io/tolgee/batch/events/OnBatchJobProgress.kt new file mode 100644 index 0000000000..ed5c914a5a --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/events/OnBatchJobProgress.kt @@ -0,0 +1,9 @@ +package io.tolgee.batch.events + +import io.tolgee.batch.BatchJobDto + +data class OnBatchJobProgress( + val job: BatchJobDto, + val processed: Long, + val total: Long, +) diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/events/OnBatchJobStatusUpdated.kt b/backend/data/src/main/kotlin/io/tolgee/batch/events/OnBatchJobStatusUpdated.kt new file mode 100644 index 0000000000..88b827c66a --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/events/OnBatchJobStatusUpdated.kt @@ -0,0 +1,5 @@ +package io.tolgee.batch.events + +class OnBatchJobStatusUpdated( + val jobId: Long +) diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/events/OnBatchJobSucceeded.kt b/backend/data/src/main/kotlin/io/tolgee/batch/events/OnBatchJobSucceeded.kt new file mode 100644 index 0000000000..7b0daf9688 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/events/OnBatchJobSucceeded.kt @@ -0,0 +1,8 @@ +package io.tolgee.batch.events + +import io.tolgee.batch.BatchJobDto +import io.tolgee.batch.OnBatchJobCompleted + +data class OnBatchJobSucceeded( + override val job: BatchJobDto, +) : OnBatchJobCompleted diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/exceptions.kt b/backend/data/src/main/kotlin/io/tolgee/batch/exceptions.kt new file mode 100644 index 0000000000..746836be2b --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/exceptions.kt @@ -0,0 +1,26 @@ +package io.tolgee.batch + +import io.tolgee.constants.Message +import io.tolgee.exceptions.ExceptionWithMessage + +open class ChunkFailedException( + message: Message, + val successfulTargets: List, + override val cause: Throwable +) : + ExceptionWithMessage(message) + +open class FailedDontRequeueException( + message: Message, + successfulTargets: List, + cause: Throwable +) : ChunkFailedException(message, successfulTargets, cause) + +open class RequeueWithTimeoutException( + message: Message, + successfulTargets: List, + cause: Throwable, + val timeoutInMs: Int = 100, + val increaseFactor: Int = 10, + val maxRetries: Int = 3 +) : ChunkFailedException(message, successfulTargets, cause) diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/processors/DeleteKeysChunkProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/batch/processors/DeleteKeysChunkProcessor.kt new file mode 100644 index 0000000000..370070311b --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/processors/DeleteKeysChunkProcessor.kt @@ -0,0 +1,43 @@ +package io.tolgee.batch.processors + +import io.tolgee.batch.BatchJobDto +import io.tolgee.batch.ChunkProcessor +import io.tolgee.batch.request.DeleteKeysRequest +import io.tolgee.model.EntityWithId +import io.tolgee.model.batch.BatchJob +import io.tolgee.service.key.KeyService +import kotlinx.coroutines.ensureActive +import org.springframework.stereotype.Component +import javax.persistence.EntityManager +import kotlin.coroutines.CoroutineContext + +@Component +class DeleteKeysChunkProcessor( + private val keyService: KeyService, + private val entityManager: EntityManager +) : ChunkProcessor { + override fun process( + job: BatchJobDto, + chunk: List, + coroutineContext: CoroutineContext, + onProgress: ((Int) -> Unit) + ) { + coroutineContext.ensureActive() + val subChunked = chunk.chunked(100) + var progress: Int = 0 + subChunked.forEach { subChunk -> + keyService.deleteMultiple(subChunk) + entityManager.flush() + progress += subChunk.size + onProgress.invoke(progress) + } + } + + override fun getTarget(data: DeleteKeysRequest): List { + return data.keyIds + } + + override fun getParams(data: DeleteKeysRequest, job: BatchJob): EntityWithId? { + return null + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/processors/TranslationChunkProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/batch/processors/TranslationChunkProcessor.kt new file mode 100644 index 0000000000..918f625780 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/processors/TranslationChunkProcessor.kt @@ -0,0 +1,76 @@ +package io.tolgee.batch.processors + +import io.tolgee.batch.BatchJobDto +import io.tolgee.batch.ChunkProcessor +import io.tolgee.batch.FailedDontRequeueException +import io.tolgee.batch.RequeueWithTimeoutException +import io.tolgee.batch.request.BatchTranslateRequest +import io.tolgee.constants.Message +import io.tolgee.exceptions.OutOfCreditsException +import io.tolgee.model.EntityWithId +import io.tolgee.model.batch.BatchJob +import io.tolgee.model.batch.TranslateJobParams +import io.tolgee.service.LanguageService +import io.tolgee.service.key.KeyService +import io.tolgee.service.translation.AutoTranslationService +import kotlinx.coroutines.ensureActive +import org.springframework.stereotype.Component +import javax.persistence.EntityManager +import kotlin.coroutines.CoroutineContext + +@Component +class TranslationChunkProcessor( + private val autoTranslationService: AutoTranslationService, + private val keyService: KeyService, + private val languageService: LanguageService, + private val entityManager: EntityManager +) : ChunkProcessor { + override fun process( + job: BatchJobDto, + chunk: List, + coroutineContext: CoroutineContext, + onProgress: (Int) -> Unit + ) { + val keys = keyService.find(chunk) + val parameters = getParams(job) + val languages = languageService.findByIdIn(parameters.targetLanguageIds) + + val successfulTargets = mutableListOf() + keys.forEach { key -> + coroutineContext.ensureActive() + try { + autoTranslationService.autoTranslate( + key = key, + languageTags = languages.map { it.tag }, + useTranslationMemory = parameters.useTranslationMemory, + useMachineTranslation = parameters.useMachineTranslation + ) + successfulTargets.add(key.id) + } catch (e: OutOfCreditsException) { + throw FailedDontRequeueException(Message.OUT_OF_CREDITS, successfulTargets, e) + } catch (e: Throwable) { + throw RequeueWithTimeoutException(Message.TRANSLATION_FAILED, successfulTargets, e) + } + } + } + + private fun getParams(job: BatchJobDto): TranslateJobParams { + return entityManager.createQuery("""from TranslateJobParams tjp where tjp.batchJob.id = :batchJobId""") + .setParameter("batchJobId", job.id).singleResult as? TranslateJobParams + ?: throw IllegalStateException("No params found") + } + + override fun getTarget(data: BatchTranslateRequest): List { + return data.keyIds + } + + override fun getParams(data: BatchTranslateRequest, job: BatchJob): EntityWithId { + return TranslateJobParams().apply { + this.batchJob = job + this.targetLanguageIds = data.targetLanguageIds + this.useMachineTranslation = data.useMachineTranslation + this.useTranslationMemory = data.useTranslationMemory + this.service = data.service + } + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/request/BatchTranslateRequest.kt b/backend/data/src/main/kotlin/io/tolgee/batch/request/BatchTranslateRequest.kt new file mode 100644 index 0000000000..2bf5f89a84 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/request/BatchTranslateRequest.kt @@ -0,0 +1,23 @@ +package io.tolgee.batch.request + +import io.swagger.v3.oas.annotations.media.Schema +import io.tolgee.constants.MtServiceType +import javax.validation.constraints.NotEmpty +import javax.validation.constraints.Size + +class BatchTranslateRequest { + @NotEmpty + var keyIds: List = listOf() + + @Size(min = 1) + var targetLanguageIds: List = listOf() + + var useMachineTranslation: Boolean = true + + var useTranslationMemory: Boolean = true + + @field:Schema( + description = "Translation service provider to use for translation. When null, Tolgee will use the primary service." + ) + var service: MtServiceType? = null +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/request/DeleteKeysRequest.kt b/backend/data/src/main/kotlin/io/tolgee/batch/request/DeleteKeysRequest.kt new file mode 100644 index 0000000000..1126079c2e --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/request/DeleteKeysRequest.kt @@ -0,0 +1,8 @@ +package io.tolgee.batch.request + +import javax.validation.constraints.NotEmpty + +class DeleteKeysRequest { + @NotEmpty + var keyIds: List = listOf() +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/state/BatchJobStateProvider.kt b/backend/data/src/main/kotlin/io/tolgee/batch/state/BatchJobStateProvider.kt new file mode 100644 index 0000000000..cf7755d59a --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/state/BatchJobStateProvider.kt @@ -0,0 +1,98 @@ +package io.tolgee.batch.state + +import io.tolgee.component.LockingProvider +import io.tolgee.component.UsingRedisProvider +import io.tolgee.model.batch.BatchJobChunkExecution +import org.redisson.api.RMap +import org.redisson.api.RedissonClient +import org.springframework.context.annotation.Lazy +import org.springframework.stereotype.Component +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentMap +import javax.persistence.EntityManager + +@Component +class BatchJobStateProvider( + val usingRedisProvider: UsingRedisProvider, + @Lazy + val redissonClient: RedissonClient, + val entityManager: EntityManager, + val lockingProvider: LockingProvider +) { + companion object { + private val localJobStatesMap by lazy { + ConcurrentHashMap>() + } + } + + fun updateState(jobId: Long, block: (MutableMap) -> T): T { + return lockingProvider.withLocking("batch_job_state_lock_$jobId") { + val map = get(jobId) + val result = block(map) + getStatesMap()[jobId] = map + result + } + } + + fun get(jobId: Long): MutableMap { + return if (usingRedisProvider.areWeUsingRedis) { + getRedissonMap(jobId) + } else { + getLocalMap(jobId) + } + } + + /** + * Doesn't init map if not exists + */ + fun getCached(jobId: Long): MutableMap? { + return getStatesMap()[jobId] + } + + fun getStatesMap(): ConcurrentMap> { + return if (usingRedisProvider.areWeUsingRedis) { + getRedissonStatesMap() + } else { + localJobStatesMap + } + } + + private fun getLocalMap(jobId: Long): MutableMap { + return localJobStatesMap.getOrPut(jobId) { + getInitialState(jobId) + } + } + + private fun getRedissonMap(jobId: Long): MutableMap { + val statesMap = getRedissonStatesMap() + + return statesMap.getOrPut(jobId) { + getInitialState(jobId) + } + } + + private fun getRedissonStatesMap(): RMap> { + return redissonClient.getMap("batch_job_state") + } + + fun getInitialState(jobId: Long): MutableMap { + val executions = entityManager.createQuery( + """ + from BatchJobChunkExecution bjce + where bjce.batchJob.id = :jobId + """, + BatchJobChunkExecution::class.java + ) + .setParameter("jobId", jobId).resultList + + return executions.associate { + it.id to ExecutionState( + it.successTargets, + it.status, + it.chunkNumber, + it.retry, + true + ) + }.toMutableMap() + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/state/ExecutionState.kt b/backend/data/src/main/kotlin/io/tolgee/batch/state/ExecutionState.kt new file mode 100644 index 0000000000..75a106a309 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/state/ExecutionState.kt @@ -0,0 +1,11 @@ +package io.tolgee.batch.state + +import io.tolgee.model.batch.BatchJobChunkExecutionStatus + +data class ExecutionState( + var successTargets: List, + var status: BatchJobChunkExecutionStatus, + var chunkNumber: Int, + var retry: Boolean, + var transactionCommitted: Boolean, +) diff --git a/backend/data/src/main/kotlin/io/tolgee/component/AutoTranslationListener.kt b/backend/data/src/main/kotlin/io/tolgee/component/AutoTranslationListener.kt index 17ed67af8a..d1cfb24d29 100644 --- a/backend/data/src/main/kotlin/io/tolgee/component/AutoTranslationListener.kt +++ b/backend/data/src/main/kotlin/io/tolgee/component/AutoTranslationListener.kt @@ -1,15 +1,18 @@ package io.tolgee.component import io.tolgee.events.OnTranslationsSet +import io.tolgee.exceptions.OutOfCreditsException import io.tolgee.service.translation.AutoTranslationService +import io.tolgee.util.Logging +import io.tolgee.util.logger import org.springframework.context.event.EventListener import org.springframework.core.annotation.Order import org.springframework.stereotype.Component @Component class AutoTranslationListener( - private val autoTranslationService: AutoTranslationService, -) { + private val autoTranslationService: AutoTranslationService +) : Logging { @Order(2) @EventListener @@ -17,10 +20,14 @@ class AutoTranslationListener( val baseLanguage = event.key.project.baseLanguage ?: return val wasUntranslatedBefore = event.oldValues[baseLanguage.tag].isNullOrEmpty() val isTranslatedAfter = !event.translations.find { it.language == baseLanguage }?.text.isNullOrEmpty() - if (wasUntranslatedBefore && isTranslatedAfter) { - autoTranslationService.autoTranslate( - key = event.key, - ) + try { + if (wasUntranslatedBefore && isTranslatedAfter) { + autoTranslationService.autoTranslate( + key = event.key, + ) + } + } catch (e: OutOfCreditsException) { + logger.debug("Auto translation failed because of out of credits") } } } diff --git a/backend/data/src/main/kotlin/io/tolgee/component/LockingProvider.kt b/backend/data/src/main/kotlin/io/tolgee/component/LockingProvider.kt index f59d56348e..0e41c48484 100644 --- a/backend/data/src/main/kotlin/io/tolgee/component/LockingProvider.kt +++ b/backend/data/src/main/kotlin/io/tolgee/component/LockingProvider.kt @@ -5,13 +5,5 @@ import java.util.concurrent.locks.Lock interface LockingProvider { fun getLock(name: String): Lock - fun withLocking(name: String, fn: () -> Unit) { - val lock = this.getLock(name) - lock.lock() - try { - fn() - } finally { - lock.unlock() - } - } + fun withLocking(name: String, fn: () -> T): T } diff --git a/backend/data/src/main/kotlin/io/tolgee/component/UsingRedisProvider.kt b/backend/data/src/main/kotlin/io/tolgee/component/UsingRedisProvider.kt new file mode 100644 index 0000000000..d0c7c7b594 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/component/UsingRedisProvider.kt @@ -0,0 +1,20 @@ +package io.tolgee.component + +import org.springframework.beans.factory.NoSuchBeanDefinitionException +import org.springframework.context.ApplicationContext +import org.springframework.data.redis.core.StringRedisTemplate +import org.springframework.stereotype.Component + +@Component +class UsingRedisProvider( + private val applicationContext: ApplicationContext +) { + val areWeUsingRedis: Boolean by lazy { + try { + applicationContext.getBean(StringRedisTemplate::class.java) + true + } catch (e: NoSuchBeanDefinitionException) { + false + } + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/component/atomicLong/AtomicLongProvider.kt b/backend/data/src/main/kotlin/io/tolgee/component/atomicLong/AtomicLongProvider.kt new file mode 100644 index 0000000000..3f45cf2180 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/component/atomicLong/AtomicLongProvider.kt @@ -0,0 +1,42 @@ +package io.tolgee.component.atomicLong + +import io.tolgee.component.UsingRedisProvider +import io.tolgee.util.Logging +import io.tolgee.util.TolgeeAtomicLong +import io.tolgee.util.logger +import org.redisson.api.RedissonClient +import org.springframework.context.ApplicationContext +import org.springframework.stereotype.Component +import java.util.concurrent.TimeUnit + +@Component +class AtomicLongProvider( + val isUsingRedisProvider: UsingRedisProvider, + val applicationContext: ApplicationContext +) : Logging { + fun get(name: String, defaultProvider: () -> Long): TolgeeAtomicLong { + return if (isUsingRedisProvider.areWeUsingRedis) { + // we need to lock it, because we don't want to set the default multiple times + val lock = redissonClient.getLock("lock_$name") + try { + lock.lock(10, TimeUnit.SECONDS) + logger.debug("Acquired lock for $name") + val atomicLong = redissonClient.getAtomicLong(name) + if (!atomicLong.isExists) { + atomicLong.set(defaultProvider()) + } + RedisTolgeeAtomicLong(atomicLong) + } finally { + if (lock.isHeldByCurrentThread) { + lock.unlock() + } + } + } else { + MemoryTolgeeAtomicLong(name, defaultProvider) + } + } + + val redissonClient: RedissonClient by lazy { + applicationContext.getBean(RedissonClient::class.java) + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/component/atomicLong/MemoryTolgeeAtomicLong.kt b/backend/data/src/main/kotlin/io/tolgee/component/atomicLong/MemoryTolgeeAtomicLong.kt new file mode 100644 index 0000000000..23c5759197 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/component/atomicLong/MemoryTolgeeAtomicLong.kt @@ -0,0 +1,28 @@ +package io.tolgee.component.atomicLong + +import io.tolgee.util.TolgeeAtomicLong +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicLong + +class MemoryTolgeeAtomicLong( + private val name: String, + private val defaultProvider: () -> Long +) : TolgeeAtomicLong { + companion object { + private val map = ConcurrentHashMap() + } + + private val it: AtomicLong by lazy { map.getOrPut(name) { AtomicLong(defaultProvider()) } } + + override fun addAndGet(delta: Long): Long { + return it.addAndGet(delta) + } + + override fun delete() { + map.remove(name) + } + + override fun get(): Long { + return it.get() + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/component/atomicLong/RedisTolgeeAtomicLong.kt b/backend/data/src/main/kotlin/io/tolgee/component/atomicLong/RedisTolgeeAtomicLong.kt new file mode 100644 index 0000000000..f2512dd88a --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/component/atomicLong/RedisTolgeeAtomicLong.kt @@ -0,0 +1,20 @@ +package io.tolgee.component.atomicLong + +import io.tolgee.util.TolgeeAtomicLong +import org.redisson.api.RAtomicLong + +class RedisTolgeeAtomicLong( + private val it: RAtomicLong +) : TolgeeAtomicLong { + override fun addAndGet(delta: Long): Long { + return it.addAndGet(delta) + } + + override fun delete() { + it.delete() + } + + override fun get(): Long { + return it.get() + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/component/eventListeners/LanguageStatsListener.kt b/backend/data/src/main/kotlin/io/tolgee/component/eventListeners/LanguageStatsListener.kt index 2b13ef1afe..dd84130a1c 100644 --- a/backend/data/src/main/kotlin/io/tolgee/component/eventListeners/LanguageStatsListener.kt +++ b/backend/data/src/main/kotlin/io/tolgee/component/eventListeners/LanguageStatsListener.kt @@ -7,21 +7,21 @@ import io.tolgee.model.key.Key import io.tolgee.model.translation.Translation import io.tolgee.service.project.LanguageStatsService import io.tolgee.util.runSentryCatching -import org.springframework.context.event.EventListener import org.springframework.scheduling.annotation.Async import org.springframework.stereotype.Component +import org.springframework.transaction.event.TransactionalEventListener @Component class LanguageStatsListener( private var languageStatsService: LanguageStatsService ) { - @EventListener + @TransactionalEventListener @Async fun onActivity(event: OnProjectActivityEvent) { runSentryCatching { - val projectId = event.activityHolder.activityRevision?.projectId ?: return + val projectId = event.activityRevision.projectId ?: return - val modifiedEntityClasses = event.activityHolder.modifiedEntities.keys.toSet() + val modifiedEntityClasses = event.modifiedEntities.keys.toSet() val isStatsModified = modifiedEntityClasses.contains(Language::class) || modifiedEntityClasses.contains(Translation::class) || modifiedEntityClasses.contains(Key::class) || diff --git a/backend/data/src/main/kotlin/io/tolgee/component/eventListeners/StoreProjectActivityListener.kt b/backend/data/src/main/kotlin/io/tolgee/component/eventListeners/StoreProjectActivityListener.kt index 710a44935b..99616ec89a 100644 --- a/backend/data/src/main/kotlin/io/tolgee/component/eventListeners/StoreProjectActivityListener.kt +++ b/backend/data/src/main/kotlin/io/tolgee/component/eventListeners/StoreProjectActivityListener.kt @@ -11,6 +11,6 @@ class StoreProjectActivityListener( ) { @EventListener fun onActivity(event: OnProjectActivityEvent) { - activityService.storeActivityData(event.activityHolder) + activityService.storeActivityData(event.activityRevision, event.modifiedEntities) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/ActivityHolderConfig.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/ActivityHolderConfig.kt index 645a146a36..88eb08ac63 100644 --- a/backend/data/src/main/kotlin/io/tolgee/configuration/ActivityHolderConfig.kt +++ b/backend/data/src/main/kotlin/io/tolgee/configuration/ActivityHolderConfig.kt @@ -21,14 +21,14 @@ class ActivityHolderConfig { @ConditionalOnMissingBean @Qualifier("transactionActivityHolder") fun transactionActivityHolder(applicationContext: ApplicationContext): ActivityHolder { - return ActivityHolder(applicationContext) + return ActivityHolder() } @Bean @RequestScope @Qualifier("requestActivityHolder") fun requestActivityHolder(applicationContext: ApplicationContext): ActivityHolder { - return ActivityHolder(applicationContext) + return ActivityHolder() } @Bean diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/DateConverter.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/DateConverter.kt deleted file mode 100644 index e7120b09fb..0000000000 --- a/backend/data/src/main/kotlin/io/tolgee/configuration/DateConverter.kt +++ /dev/null @@ -1,17 +0,0 @@ -package io.tolgee.configuration - -import org.springframework.boot.context.properties.ConfigurationPropertiesBinding -import org.springframework.core.convert.converter.Converter -import org.springframework.stereotype.Component -import java.text.SimpleDateFormat -import java.util.* - -@Component -@ConfigurationPropertiesBinding -class DateConverter : Converter { - override fun convert(source: String?): Date? { - return if (source == null) { - null - } else SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse(source) - } -} diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/BatchProperties.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/BatchProperties.kt new file mode 100644 index 0000000000..b5186627b2 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/BatchProperties.kt @@ -0,0 +1,10 @@ +package io.tolgee.configuration.tolgee + +import io.tolgee.configuration.annotations.DocProperty +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties(prefix = "tolgee.batch") +class BatchProperties { + @DocProperty(description = "How many parallel jobs can be run at once on single Tolgee instance") + var concurrency: Int = 10 +} diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/RabbitmqAutostartProperties.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/RabbitmqAutostartProperties.kt new file mode 100644 index 0000000000..6c296559f4 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/RabbitmqAutostartProperties.kt @@ -0,0 +1,27 @@ +package io.tolgee.configuration.tolgee + +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties(prefix = "tolgee.rabbitmq-autostart") +class RabbitmqAutostartProperties { + var enabled: Boolean = true + var mode: RabbitMqAutostartMode = RabbitMqAutostartMode.DOCKER + var port: Int = 25672 + var managementPort: Int = 25673 + var containerName: String = "tolgee_rabbitmq" + var defaultUser: String = "rabbit" + var defaultPassword: String = "rabbit" + + enum class RabbitMqAutostartMode { + /** + * Starts docker container with postgres + */ + DOCKER, + + /** + * Expects that postgres is installed in the same container. + * So the Postgres is started with Tolgee. + */ + EMBEDDED + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/TolgeeProperties.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/TolgeeProperties.kt index c0ee0ebaf9..be9f65476e 100644 --- a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/TolgeeProperties.kt +++ b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/TolgeeProperties.kt @@ -110,6 +110,12 @@ open class TolgeeProperties( @DocProperty(description = "Maximum length of translations.") open var maxTranslationTextLength: Long = 10000, + @DocProperty( + description = "Properties related to batch jobs", + displayName = "Batch jobs" + ) + open var batch: BatchProperties = BatchProperties(), + var cache: CacheProperties = CacheProperties(), var recaptcha: ReCaptchaProperties = ReCaptchaProperties(), var machineTranslation: MachineTranslationProperties = MachineTranslationProperties(), diff --git a/backend/data/src/main/kotlin/io/tolgee/constants/Caches.kt b/backend/data/src/main/kotlin/io/tolgee/constants/Caches.kt index 470b4a7e46..95de6d7279 100644 --- a/backend/data/src/main/kotlin/io/tolgee/constants/Caches.kt +++ b/backend/data/src/main/kotlin/io/tolgee/constants/Caches.kt @@ -6,6 +6,7 @@ interface Caches { const val USER_ACCOUNTS = "userAccounts" const val PROJECTS = "projects" const val PERMISSIONS = "permissions" + const val BATCH_JOBS = "batchJobs" const val RATE_LIMITS = "rateLimits" const val MACHINE_TRANSLATIONS = "machineTranslations" const val PROJECT_TRANSLATIONS_MODIFIED = "projectTranslationsModified" diff --git a/backend/data/src/main/kotlin/io/tolgee/constants/Message.kt b/backend/data/src/main/kotlin/io/tolgee/constants/Message.kt index fd82348cb5..4d0bf358a0 100644 --- a/backend/data/src/main/kotlin/io/tolgee/constants/Message.kt +++ b/backend/data/src/main/kotlin/io/tolgee/constants/Message.kt @@ -148,7 +148,9 @@ enum class Message { BIG_META_NOT_FROM_PROJECT, MT_SERVICE_NOT_ENABLED, PROJECT_NOT_SELECTED, - PLAN_HAS_SUBSCRIBERS + PLAN_HAS_SUBSCRIBERS, + TRANSLATION_FAILED, + BATCH_JOB_NOT_FOUND ; val code: String diff --git a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BatchJobsTestData.kt b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BatchJobsTestData.kt new file mode 100644 index 0000000000..3d9a17677c --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BatchJobsTestData.kt @@ -0,0 +1,32 @@ +package io.tolgee.development.testDataBuilder.data + +import io.tolgee.model.enums.Scope +import io.tolgee.model.key.Key + +class BatchJobsTestData : BaseTestData() { + + val anotherUser = root.addUserAccount { username = "anotherUser" }.self + + init { + this.projectBuilder.addPermission { + user = anotherUser + scopes = arrayOf(Scope.KEYS_VIEW) + } + } + + fun addTranslationOperationData(keyCount: Int = 100): List { + this.projectBuilder.addCzech() + this.projectBuilder.addGerman() + + return (1..keyCount).map { + this.projectBuilder.addKey { + name = "key$it" + }.build { + addTranslation { + language = englishLanguage + text = "en$it" + } + }.self + } + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/dtos/cacheable/UserAccountDto.kt b/backend/data/src/main/kotlin/io/tolgee/dtos/cacheable/UserAccountDto.kt index 02d0171f23..03378fde89 100644 --- a/backend/data/src/main/kotlin/io/tolgee/dtos/cacheable/UserAccountDto.kt +++ b/backend/data/src/main/kotlin/io/tolgee/dtos/cacheable/UserAccountDto.kt @@ -8,7 +8,9 @@ data class UserAccountDto( val username: String, val role: UserAccount.Role?, val id: Long, - val needsSuperJwt: Boolean + val needsSuperJwt: Boolean, + val avatarHash: String?, + val deleted: Boolean ) : Serializable { companion object { fun fromEntity(entity: UserAccount) = UserAccountDto( @@ -16,7 +18,9 @@ data class UserAccountDto( username = entity.username, role = entity.role, id = entity.id, - needsSuperJwt = entity.needsSuperJwt + needsSuperJwt = entity.needsSuperJwt, + avatarHash = entity.avatarHash, + deleted = entity.deletedAt != null ) } diff --git a/backend/data/src/main/kotlin/io/tolgee/events/OnProjectActivityEvent.kt b/backend/data/src/main/kotlin/io/tolgee/events/OnProjectActivityEvent.kt index 3dff4cc6e8..edebb2bf89 100644 --- a/backend/data/src/main/kotlin/io/tolgee/events/OnProjectActivityEvent.kt +++ b/backend/data/src/main/kotlin/io/tolgee/events/OnProjectActivityEvent.kt @@ -1,8 +1,10 @@ package io.tolgee.events -import io.tolgee.activity.ActivityHolder -import org.springframework.context.ApplicationEvent +import io.tolgee.activity.ModifiedEntitiesType +import io.tolgee.model.activity.ActivityRevision class OnProjectActivityEvent( - val activityHolder: ActivityHolder, -) : ApplicationEvent(activityHolder) + val activityRevision: ActivityRevision, + val modifiedEntities: ModifiedEntitiesType, + val organizationId: Long? +) diff --git a/backend/data/src/main/kotlin/io/tolgee/exceptions/ErrorException.kt b/backend/data/src/main/kotlin/io/tolgee/exceptions/ErrorException.kt index c8c7fff772..6b461bc644 100644 --- a/backend/data/src/main/kotlin/io/tolgee/exceptions/ErrorException.kt +++ b/backend/data/src/main/kotlin/io/tolgee/exceptions/ErrorException.kt @@ -4,23 +4,13 @@ import io.tolgee.constants.Message import org.springframework.http.HttpStatus import java.io.Serializable -abstract class ErrorException : RuntimeException { - val params: List? - val code: String +abstract class ErrorException : ExceptionWithMessage { + constructor(message: Message, params: List? = null) : super(message, params) - constructor(message: Message, params: List?) : super(message.code) { - this.params = params - this.code = message.code - } - - constructor(message: Message) : this(message, null) - - constructor(code: String, params: List? = null) : super(code + params.toString()) { - this.code = code - this.params = params - } + constructor(code: String, params: List? = null) : super(code, params) val errorResponseBody: ErrorResponseBody get() = ErrorResponseBody(this.code, params) + abstract val httpStatus: HttpStatus? } diff --git a/backend/data/src/main/kotlin/io/tolgee/exceptions/ExceptionWithMessage.kt b/backend/data/src/main/kotlin/io/tolgee/exceptions/ExceptionWithMessage.kt new file mode 100644 index 0000000000..872df06312 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/exceptions/ExceptionWithMessage.kt @@ -0,0 +1,23 @@ +package io.tolgee.exceptions + +import io.tolgee.constants.Message +import java.io.Serializable + +abstract class ExceptionWithMessage( + private val _code: String? = null, + val params: List? +) : RuntimeException("$_code $params") { + + var tolgeeMessage: Message? = null + + val code: String + get() = _code ?: tolgeeMessage?.code ?: throw IllegalStateException("Exception code or message not set") + + constructor(message: Message, params: List?) : this(message.code, params) { + this.tolgeeMessage = message + } + + constructor(message: Message) : this(null, null) { + this.tolgeeMessage = message + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/model/Permission.kt b/backend/data/src/main/kotlin/io/tolgee/model/Permission.kt index c4d0bc04a0..7e0aa3cacd 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/Permission.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/Permission.kt @@ -58,6 +58,12 @@ class Permission( @Type(type = "enum-array") @Column(name = "scopes", columnDefinition = "varchar[]") private var _scopes: Array? = null + set(value) { + field = value + if (!value.isNullOrEmpty()) { + this.type = null + } + } override var scopes: Array get() = _scopes ?: type?.availableScopes ?: throw IllegalStateException() diff --git a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityRevision.kt b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityRevision.kt index e686e8dbff..3e1b4c17c4 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityRevision.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityRevision.kt @@ -3,6 +3,8 @@ package io.tolgee.model.activity import com.vladmihalcea.hibernate.type.json.JsonBinaryType import io.tolgee.activity.data.ActivityType import io.tolgee.component.CurrentDateProvider +import io.tolgee.model.batch.BatchJob +import io.tolgee.model.batch.BatchJobChunkExecution import org.hibernate.annotations.NotFound import org.hibernate.annotations.NotFoundAction import org.hibernate.annotations.Type @@ -17,11 +19,13 @@ import javax.persistence.Entity import javax.persistence.EntityListeners import javax.persistence.EnumType import javax.persistence.Enumerated +import javax.persistence.FetchType import javax.persistence.GeneratedValue import javax.persistence.GenerationType import javax.persistence.Id import javax.persistence.Index import javax.persistence.OneToMany +import javax.persistence.OneToOne import javax.persistence.PrePersist import javax.persistence.SequenceGenerator import javax.persistence.Table @@ -82,6 +86,32 @@ class ActivityRevision : java.io.Serializable { @OneToMany(mappedBy = "activityRevision") var modifiedEntities: MutableList = mutableListOf() + /** + * For chunked jobs, this field is set for every chunk. + * When job is running, each chunk has it's own activity revision. + * When job is finished, all the chunks revisions are merged into one revision and + * this field is set to null. + * + * Instead, [batchJob] is set. + */ + @OneToOne(fetch = FetchType.LAZY) + var batchJobChunkExecution: BatchJobChunkExecution? = null + + @OneToOne(fetch = FetchType.LAZY) + var batchJob: BatchJob? = null + + @Transient + var afterFlush: (() -> Unit)? = null + + /** + * The instance is created in the Holder by default, but it is not initialized by the interceptor, + * so projectId and authorId might be null. + * + * This flag is set to true when the instance is initialized by the interceptor. + */ + @Transient + var isInitializedByInterceptor: Boolean = false + companion object { @Configurable class ActivityRevisionListener { diff --git a/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJob.kt b/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJob.kt new file mode 100644 index 0000000000..12c59e3e52 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJob.kt @@ -0,0 +1,55 @@ +package io.tolgee.model.batch + +import com.vladmihalcea.hibernate.type.json.JsonBinaryType +import io.tolgee.batch.BatchJobDto +import io.tolgee.batch.BatchJobType +import io.tolgee.model.Project +import io.tolgee.model.StandardAuditModel +import io.tolgee.model.UserAccount +import io.tolgee.model.activity.ActivityRevision +import org.hibernate.annotations.Type +import org.hibernate.annotations.TypeDef +import org.hibernate.annotations.TypeDefs +import javax.persistence.Entity +import javax.persistence.Enumerated +import javax.persistence.FetchType +import javax.persistence.ManyToOne +import javax.persistence.OneToOne + +@Entity +@TypeDefs( + value = [TypeDef(name = "jsonb", typeClass = JsonBinaryType::class)] +) +class BatchJob : StandardAuditModel(), IBatchJob { + @ManyToOne(fetch = FetchType.LAZY) + lateinit var project: Project + + @ManyToOne(fetch = FetchType.LAZY) + var author: UserAccount? = null + + @Type(type = "jsonb") + var target: List = listOf() + + var totalItems: Int = 0 + + var totalChunks: Int = 0 + + var chunkSize: Int = 0 + + override var status: BatchJobStatus = BatchJobStatus.PENDING + + @Enumerated + var type: BatchJobType = BatchJobType.TRANSLATION + + @OneToOne(mappedBy = "batchJob", fetch = FetchType.LAZY) + var activityRevision: ActivityRevision? = null + + val chunkedTarget get() = chunkTarget(chunkSize, target) + + val dto get() = BatchJobDto.fromEntity(this) + + companion object { + fun chunkTarget(chunkSize: Int, target: List): List> = + if (chunkSize == 0) listOf(target) else target.chunked(chunkSize) + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobChunkExecution.kt b/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobChunkExecution.kt new file mode 100644 index 0000000000..edcc9b8bae --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobChunkExecution.kt @@ -0,0 +1,56 @@ +package io.tolgee.model.batch + +import com.vladmihalcea.hibernate.type.json.JsonBinaryType +import io.tolgee.constants.Message +import io.tolgee.model.StandardAuditModel +import io.tolgee.model.activity.ActivityRevision +import org.hibernate.annotations.ColumnDefault +import org.hibernate.annotations.Type +import org.hibernate.annotations.TypeDef +import org.hibernate.annotations.TypeDefs +import java.util.* +import javax.persistence.Column +import javax.persistence.Entity +import javax.persistence.EnumType +import javax.persistence.Enumerated +import javax.persistence.FetchType +import javax.persistence.ManyToOne +import javax.persistence.OneToOne +import javax.persistence.Table + +@Entity +@Table( + indexes = [ + javax.persistence.Index(columnList = "chunkNumber"), + javax.persistence.Index(columnList = "status"), + ] +) +@TypeDefs( + value = [TypeDef(name = "jsonb", typeClass = JsonBinaryType::class)] +) +class BatchJobChunkExecution : StandardAuditModel() { + @ManyToOne(fetch = FetchType.LAZY, optional = false) + lateinit var batchJob: BatchJob + + @Enumerated(EnumType.STRING) + var status: BatchJobChunkExecutionStatus = BatchJobChunkExecutionStatus.PENDING + + var chunkNumber: Int = 0 + + @Type(type = "jsonb") + var successTargets: List = listOf() + + @Column(columnDefinition = "text") + var exception: String? = null + + @Enumerated(EnumType.STRING) + var errorMessage: Message? = null + + var executeAfter: Date? = null + + @ColumnDefault("false") + var retry: Boolean = false + + @OneToOne(fetch = FetchType.LAZY, mappedBy = "batchJobChunkExecution") + var activityRevision: ActivityRevision? = null +} diff --git a/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobChunkExecutionStatus.kt b/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobChunkExecutionStatus.kt new file mode 100644 index 0000000000..0a334cd7bd --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobChunkExecutionStatus.kt @@ -0,0 +1,10 @@ +package io.tolgee.model.batch + +enum class BatchJobChunkExecutionStatus( + val completed: Boolean +) { + PENDING(false), + SUCCESS(true), + FAILED(true), + CANCELLED(true), +} diff --git a/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobStatus.kt b/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobStatus.kt new file mode 100644 index 0000000000..b12366ffd9 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobStatus.kt @@ -0,0 +1,9 @@ +package io.tolgee.model.batch + +enum class BatchJobStatus { + PENDING, + RUNNING, + SUCCESS, + FAILED, + CANCELLED, +} diff --git a/backend/data/src/main/kotlin/io/tolgee/model/batch/IBatchJob.kt b/backend/data/src/main/kotlin/io/tolgee/model/batch/IBatchJob.kt new file mode 100644 index 0000000000..83d6206e6f --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/model/batch/IBatchJob.kt @@ -0,0 +1,6 @@ +package io.tolgee.model.batch + +interface IBatchJob { + var id: Long + var status: BatchJobStatus +} diff --git a/backend/data/src/main/kotlin/io/tolgee/model/batch/TranslateJobParams.kt b/backend/data/src/main/kotlin/io/tolgee/model/batch/TranslateJobParams.kt new file mode 100644 index 0000000000..8aa126c932 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/model/batch/TranslateJobParams.kt @@ -0,0 +1,28 @@ +package io.tolgee.model.batch + +import com.vladmihalcea.hibernate.type.json.JsonBinaryType +import io.tolgee.constants.MtServiceType +import io.tolgee.model.StandardAuditModel +import org.hibernate.annotations.Type +import org.hibernate.annotations.TypeDef +import org.hibernate.annotations.TypeDefs +import javax.persistence.Entity +import javax.persistence.OneToOne + +@Entity +@TypeDefs( + value = [TypeDef(name = "jsonb", typeClass = JsonBinaryType::class)] +) +class TranslateJobParams : StandardAuditModel() { + @OneToOne(optional = false) + lateinit var batchJob: BatchJob + + @Type(type = "jsonb") + var targetLanguageIds: List = mutableListOf() + + var useMachineTranslation: Boolean = true + + var useTranslationMemory: Boolean = true + + var service: MtServiceType? = null +} diff --git a/backend/data/src/main/kotlin/io/tolgee/model/enums/Scope.kt b/backend/data/src/main/kotlin/io/tolgee/model/enums/Scope.kt index 2a6dcbb670..7b7ee20793 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/enums/Scope.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/enums/Scope.kt @@ -28,6 +28,9 @@ enum class Scope( KEYS_VIEW("keys.view"), KEYS_DELETE("keys.delete"), KEYS_CREATE("keys.create"), + BATCH_JOBS_VIEW("batch-jobs.view"), + BATCH_JOBS_CANCEL("batch-jobs.cancel"), + BATCH_AUTO_TRANSLATE("batch-auto-translate"), ; fun expand() = Scope.expand(this) @@ -36,14 +39,15 @@ enum class Scope( private val keysView = HierarchyItem(KEYS_VIEW) private val translationsView = HierarchyItem(TRANSLATIONS_VIEW, listOf(keysView)) private val screenshotsView = HierarchyItem(SCREENSHOTS_VIEW, listOf(keysView)) + private val translationsEdit = HierarchyItem( + TRANSLATIONS_EDIT, + listOf(translationsView) + ) val hierarchy = HierarchyItem( ADMIN, listOf( - HierarchyItem( - TRANSLATIONS_EDIT, - listOf(translationsView) - ), + translationsEdit, HierarchyItem( KEYS_EDIT, listOf( @@ -94,7 +98,10 @@ enum class Scope( HierarchyItem( TRANSLATIONS_STATE_EDIT, listOf(HierarchyItem(TRANSLATIONS_VIEW)) - ) + ), + HierarchyItem(BATCH_JOBS_VIEW), + HierarchyItem(BATCH_JOBS_CANCEL), + HierarchyItem(BATCH_AUTO_TRANSLATE, listOf(translationsEdit)) ) ) diff --git a/backend/data/src/main/kotlin/io/tolgee/model/translation/TranslationComment.kt b/backend/data/src/main/kotlin/io/tolgee/model/translation/TranslationComment.kt index 6c77b6ef59..72b59bc77c 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/translation/TranslationComment.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/translation/TranslationComment.kt @@ -10,6 +10,7 @@ import io.tolgee.model.enums.TranslationCommentState import org.hibernate.validator.constraints.Length import javax.persistence.Column import javax.persistence.Entity +import javax.persistence.FetchType.LAZY import javax.persistence.Index import javax.persistence.ManyToOne import javax.persistence.Table @@ -35,6 +36,6 @@ class TranslationComment( @ManyToOne var translation: Translation ) : StandardAuditModel() { - @ManyToOne(optional = false) + @ManyToOne(optional = false, fetch = LAZY) lateinit var author: UserAccount } diff --git a/backend/data/src/main/kotlin/io/tolgee/model/views/BatchJobView.kt b/backend/data/src/main/kotlin/io/tolgee/model/views/BatchJobView.kt new file mode 100644 index 0000000000..ea4e84503b --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/model/views/BatchJobView.kt @@ -0,0 +1,10 @@ +package io.tolgee.model.views + +import io.tolgee.constants.Message +import io.tolgee.model.batch.BatchJob + +class BatchJobView( + val batchJob: BatchJob, + val progress: Int, + val errorMessage: Message? +) diff --git a/backend/data/src/main/kotlin/io/tolgee/model/views/JobErrorMessagesView.kt b/backend/data/src/main/kotlin/io/tolgee/model/views/JobErrorMessagesView.kt new file mode 100644 index 0000000000..0f15995b5b --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/model/views/JobErrorMessagesView.kt @@ -0,0 +1,10 @@ +package io.tolgee.model.views + +import io.tolgee.constants.Message +import java.util.* + +interface JobErrorMessagesView { + val batchJobId: Long + val errorMessage: Message + val updatedAt: Date +} diff --git a/backend/data/src/main/kotlin/io/tolgee/pubSub/RedisPubSubReceiver.kt b/backend/data/src/main/kotlin/io/tolgee/pubSub/RedisPubSubReceiver.kt new file mode 100644 index 0000000000..2d8ae8e091 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/pubSub/RedisPubSubReceiver.kt @@ -0,0 +1,32 @@ +package io.tolgee.pubSub + +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.tolgee.batch.JobCancelEvent +import io.tolgee.batch.JobQueueItemsEvent +import io.tolgee.util.Logging +import io.tolgee.util.logger +import io.tolgee.websocket.RedisWebsocketEventWrapper +import org.springframework.context.ApplicationEventPublisher +import org.springframework.messaging.simp.SimpMessagingTemplate + +class RedisPubSubReceiver( + private val template: SimpMessagingTemplate, + private val applicationEventPublisher: ApplicationEventPublisher +) : Logging { + + fun receiveWebsocketMessage(message: String) { + val data = jacksonObjectMapper().readValue(message, RedisWebsocketEventWrapper::class.java) + template.convertAndSend(data.destination, data.message) + logger.debug("Sending message to ${data.destination}") + } + + fun receiveJobQueueMessage(message: String) { + val data = jacksonObjectMapper().readValue(message, JobQueueItemsEvent::class.java) + applicationEventPublisher.publishEvent(data) + } + + fun receiveJobCancel(message: String) { + val data = jacksonObjectMapper().readValue(message, Long::class.java) + applicationEventPublisher.publishEvent(JobCancelEvent(data)) + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/pubSub/RedisPubSubReceiverConfiguration.kt b/backend/data/src/main/kotlin/io/tolgee/pubSub/RedisPubSubReceiverConfiguration.kt new file mode 100644 index 0000000000..4c8262ff48 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/pubSub/RedisPubSubReceiverConfiguration.kt @@ -0,0 +1,63 @@ +package io.tolgee.pubSub + +import io.tolgee.configuration.tolgee.TolgeeProperties +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression +import org.springframework.context.ApplicationEventPublisher +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.data.redis.connection.RedisConnectionFactory +import org.springframework.data.redis.listener.PatternTopic +import org.springframework.data.redis.listener.RedisMessageListenerContainer +import org.springframework.data.redis.listener.adapter.MessageListenerAdapter +import org.springframework.messaging.simp.SimpMessagingTemplate + +@Configuration +@ConditionalOnExpression( + "\${tolgee.websocket.use-redis:false} or " + + "(\${tolgee.cache.use-redis:false} and \${tolgee.cache.enabled:false})" +) +class RedisPubSubReceiverConfiguration( + private val template: SimpMessagingTemplate, + private val connectionFactory: RedisConnectionFactory, + private val applicationEventPublisher: ApplicationEventPublisher, + private val tolgeeProperties: TolgeeProperties +) { + + companion object { + const val WEBSOCKET_TOPIC = "websocket" + const val JOB_QUEUE_TOPIC = "job_queue" + const val JOB_CANCEL_TOPIC = "job_cancel" + } + + @Bean + fun redisPubsubReceiver(): RedisPubSubReceiver { + return RedisPubSubReceiver(template, applicationEventPublisher) + } + + @Bean + fun redisWebsocketPubsubListenerAdapter(): MessageListenerAdapter { + return MessageListenerAdapter(redisPubsubReceiver(), RedisPubSubReceiver::receiveWebsocketMessage.name) + } + + @Bean + fun redisJobQueuePubsubListenerAdapter(): MessageListenerAdapter { + return MessageListenerAdapter(redisPubsubReceiver(), RedisPubSubReceiver::receiveJobQueueMessage.name) + } + + @Bean + fun redisJobCancelPubsubListenerAdapter(): MessageListenerAdapter { + return MessageListenerAdapter(redisPubsubReceiver(), RedisPubSubReceiver::receiveJobCancel.name) + } + + @Bean + fun redisPubsubContainer(): RedisMessageListenerContainer { + val container = RedisMessageListenerContainer() + container.connectionFactory = connectionFactory + if (tolgeeProperties.websocket.useRedis) { + container.addMessageListener(redisWebsocketPubsubListenerAdapter(), PatternTopic(WEBSOCKET_TOPIC)) + } + container.addMessageListener(redisJobQueuePubsubListenerAdapter(), PatternTopic(JOB_QUEUE_TOPIC)) + container.addMessageListener(redisJobCancelPubsubListenerAdapter(), PatternTopic(JOB_CANCEL_TOPIC)) + return container + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/BatchJobRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/BatchJobRepository.kt new file mode 100644 index 0000000000..22a165dd91 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/repository/BatchJobRepository.kt @@ -0,0 +1,49 @@ +package io.tolgee.repository + +import io.tolgee.model.batch.BatchJob +import io.tolgee.model.views.JobErrorMessagesView +import org.springframework.data.domain.Page +import org.springframework.data.domain.Pageable +import org.springframework.data.jpa.repository.JpaRepository +import org.springframework.data.jpa.repository.Query +import org.springframework.stereotype.Repository + +@Repository +interface BatchJobRepository : JpaRepository { + @Query( + value = """ + select j from BatchJob j + left join fetch j.author + left join fetch j.activityRevision + where j.project.id = :projectId + and (:userAccountId is null or j.author.id = :userAccountId) + """, + countQuery = """ + select count(j) from BatchJob j + where j.project.id = :projectId + and (:userAccountId is null or j.author.id = :userAccountId) + """ + ) + fun getJobs(projectId: Long, userAccountId: Long?, pageable: Pageable): Page + + @Query( + nativeQuery = true, + value = """ + select batch_job_chunk_execution.batch_job_id, sum(jsonb_array_length(success_targets)) + from batch_job_chunk_execution + where batch_job_id in :jobIds + group by batch_job_chunk_execution.batch_job_id + """ + ) + fun getProgresses(jobIds: List): List> + + @Query( + value = """ + select bjce.batchJob.id as batchJobId, bjce.id as executionId, bjce.errorMessage as errorMessage, bjce.updatedAt as updatedAt + from BatchJobChunkExecution bjce + where bjce.batchJob.id in :jobIds + and bjce.errorMessage is not null + """ + ) + fun getErrorMessages(jobIds: List): List +} diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/KeyRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/KeyRepository.kt index d20fee711e..bf5c77eec6 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/KeyRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/KeyRepository.kt @@ -43,6 +43,17 @@ interface KeyRepository : JpaRepository { fun deleteAllByIdIn(ids: Collection) fun findAllByIdIn(ids: Collection): List + @Query( + """ + from Key k + left join fetch k.namespace + left join fetch k.keyMeta + left join fetch k.keyScreenshotReferences + where k.id in :ids + """ + ) + fun findAllByIdInForDelete(ids: Collection): List + @Query("from Key k join fetch k.project left join fetch k.keyMeta where k.id in :ids") fun findWithProjectsAndMetas(ids: Set): List @@ -113,4 +124,11 @@ interface KeyRepository : JpaRepository { """ ) fun getWithTags(keys: Set): List + + @Query( + """ + select k.project.id from Key k where k.id in :keysIds + """ + ) + fun getProjectIdsForKeyIds(keysIds: List): List } diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/KeyScreenshotReferenceRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/KeyScreenshotReferenceRepository.kt index 5eb6405a3b..19a0bc127b 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/KeyScreenshotReferenceRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/KeyScreenshotReferenceRepository.kt @@ -32,6 +32,7 @@ interface KeyScreenshotReferenceRepository : JpaRepository { ) fun getAllByLanguageId(languageId: Long): List - @Query("from Translation t join fetch t.key k left join fetch k.keyMeta where t.key.id in :keyIds") + @Query( + """from Translation t + join fetch t.key k + left join fetch k.keyMeta + left join fetch t.comments + where t.key.id in :keyIds""" + ) fun getAllByKeyIdIn(keyIds: Collection): Collection @Query( diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/activity/ActivityRevisionRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/activity/ActivityRevisionRepository.kt index dc725e4f86..41d7c9f878 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/activity/ActivityRevisionRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/activity/ActivityRevisionRepository.kt @@ -14,7 +14,7 @@ interface ActivityRevisionRepository : JpaRepository { @Query( """ from ActivityRevision ar - where ar.projectId = :projectId and ar.type is not null + where ar.projectId = :projectId and ar.type is not null and ar.batchJobChunkExecution is null """ ) fun getForProject(projectId: Long, pageable: Pageable): Page diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/translation/TranslationCommentRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/translation/TranslationCommentRepository.kt index 3418576ff4..a464a71e3d 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/translation/TranslationCommentRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/translation/TranslationCommentRepository.kt @@ -16,4 +16,6 @@ interface TranslationCommentRepository : JpaRepository fun getPagedByTranslation(translation: Translation, pageable: Pageable): Page fun deleteAllByTranslationIdIn(translationIds: Collection) + + fun deleteByTranslationIdIn(ids: Collection) } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/BigMetaService.kt b/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/BigMetaService.kt index 4daaeb3857..2a054b08a0 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/BigMetaService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/bigMeta/BigMetaService.kt @@ -40,6 +40,7 @@ class BigMetaService( return keysDistanceRepository.save(keysDistance) } + @Transactional fun store(data: BigMetaDto, project: Project) { storeRelatedKeysInOrder(data.relatedKeysInOrder, project) } @@ -78,6 +79,7 @@ class BigMetaService( return entityManager.createQuery(query).resultList } + @Transactional fun findExistingKeyDistances(keys: List, project: Project): List { val keyIds = keys.map { it.id } return findExistingKeysDistancesByIds(keyIds) @@ -114,7 +116,7 @@ class BigMetaService( fun onKeyDeleted(event: OnProjectActivityEvent) { runSentryCatching { val ids = - event.activityHolder.modifiedEntities[Key::class]?.values?.filter { it.revisionType.isDel() } + event.modifiedEntities[Key::class]?.values?.filter { it.revisionType.isDel() } ?.map { it.entityId } if (ids.isNullOrEmpty()) { diff --git a/backend/data/src/main/kotlin/io/tolgee/service/key/KeyService.kt b/backend/data/src/main/kotlin/io/tolgee/service/key/KeyService.kt index a451ba9d73..01e8fb9e5f 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/key/KeyService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/key/KeyService.kt @@ -184,16 +184,17 @@ class KeyService( translationService.deleteAllByKeys(ids) keyMetaService.deleteAllByKeyIdIn(ids) screenshotService.deleteAllByKeyId(ids) - val keys = keyRepository.findAllByIdIn(ids) + val keys = keyRepository.findAllByIdInForDelete(ids) val namespaces = keys.map { it.namespace } keyRepository.deleteAllByIdIn(keys.map { it.id }) namespaceService.deleteUnusedNamespaces(namespaces) } + @Transactional fun deleteAllByProject(projectId: Long) { val ids = keyRepository.getIdsByProjectId(projectId) keyMetaService.deleteAllByKeyIdIn(ids) - keyRepository.deleteAllByIdIn(ids) + this.deleteMultiple(ids) } @Autowired @@ -242,4 +243,8 @@ class KeyService( fun getPaged(projectId: Long, pageable: Pageable): Page = keyRepository.getAllByProjectId(projectId, pageable) fun getKeysWithTags(keys: Set): List = keyRepository.getWithTags(keys) + + fun find(id: List): List { + return keyRepository.findAllByIdIn(id) + } } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/project/LanguageStatsService.kt b/backend/data/src/main/kotlin/io/tolgee/service/project/LanguageStatsService.kt index a25a18a420..6656062527 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/project/LanguageStatsService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/project/LanguageStatsService.kt @@ -8,7 +8,7 @@ import io.tolgee.repository.LanguageStatsRepository import io.tolgee.service.LanguageService import io.tolgee.service.query_builders.LanguageStatsProvider import io.tolgee.util.Logging -import io.tolgee.util.executeInNewTransaction +import io.tolgee.util.executeInNewRepeatableTransaction import io.tolgee.util.logger import org.springframework.stereotype.Service import org.springframework.transaction.PlatformTransactionManager @@ -28,14 +28,14 @@ class LanguageStatsService( ) : Logging { fun refreshLanguageStats(projectId: Long) { lockingProvider.withLocking("refresh-lang-stats-$projectId") { - executeInNewTransaction(platformTransactionManager) { + executeInNewRepeatableTransaction(platformTransactionManager) tx@{ val languages = languageService.findAll(projectId) val allRawLanguageStats = getLanguageStatsRaw(projectId) try { val baseLanguage = projectService.getOrCreateBaseLanguage(projectId) val rawBaseLanguageStats = allRawLanguageStats.find { it.languageId == baseLanguage?.id } - ?: return@executeInNewTransaction + ?: return@tx val projectStats = projectStatsService.getProjectStats(projectId) val languageStats = languageStatsRepository.getAllByProjectId(projectId) .associateBy { it.language.id } @@ -49,7 +49,7 @@ class LanguageStatsService( val translatedOrReviewedKeys = rawLanguageStats.translatedKeys + rawLanguageStats.reviewedKeys val translatedOrReviewedWords = rawLanguageStats.translatedWords + rawLanguageStats.reviewedWords val untranslatedWords = baseWords - translatedOrReviewedWords - val language = languages.find { it.id == rawLanguageStats.languageId } ?: return@executeInNewTransaction + val language = languages.find { it.id == rawLanguageStats.languageId } ?: return@tx val stats = languageStats.computeIfAbsent(language.id) { LanguageStats(language) } @@ -71,7 +71,7 @@ class LanguageStatsService( languageStatsRepository.save(it) } } catch (e: NotFoundException) { - logger.warn("Cannot save Language Stats due to NotFoundException. Project deleted too fast?") + logger.warn("Cannot save Language Stats due to NotFoundException. Project deleted too fast?", e) } } } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/security/SecurityService.kt b/backend/data/src/main/kotlin/io/tolgee/service/security/SecurityService.kt index 4e65020649..0441269a40 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/security/SecurityService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/security/SecurityService.kt @@ -11,6 +11,7 @@ import io.tolgee.model.Project import io.tolgee.model.UserAccount import io.tolgee.model.enums.Scope import io.tolgee.model.translation.Translation +import io.tolgee.repository.KeyRepository import io.tolgee.security.AuthenticationFacade import io.tolgee.service.LanguageService import org.springframework.beans.factory.annotation.Autowired @@ -20,7 +21,8 @@ import java.io.Serializable @Service class SecurityService @Autowired constructor( private val authenticationFacade: AuthenticationFacade, - private val languageService: LanguageService + private val languageService: LanguageService, + private val keyRepository: KeyRepository ) { @set:Autowired @@ -248,6 +250,24 @@ class SecurityService @Autowired constructor( return permissionService.getProjectPermissionScopes(projectId, userId) } + fun checkKeyIdsExistAndIsFromProject(keyIds: List, projectId: Long) { + val projectIds = keyRepository.getProjectIdsForKeyIds(keyIds) + + if (projectIds.size != keyIds.size) { + throw NotFoundException(Message.KEY_NOT_FOUND) + } + + val firstProjectId = projectIds[0] + + if (projectIds.any { it != firstProjectId }) { + throw PermissionException(Message.MULTIPLE_PROJECTS_NOT_SUPPORTED) + } + + if (firstProjectId != projectId) { + throw PermissionException(Message.KEY_NOT_FROM_PROJECT) + } + } + private fun isCurrentUserServerAdmin(): Boolean { return isUserAdmin(activeUser) } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/AutoTranslationService.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/AutoTranslationService.kt index 1cf0235b1b..6c949f0b06 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/translation/AutoTranslationService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/AutoTranslationService.kt @@ -2,7 +2,6 @@ package io.tolgee.service.translation import io.tolgee.constants.MtServiceType import io.tolgee.dtos.request.AutoTranslationSettingsDto -import io.tolgee.exceptions.OutOfCreditsException import io.tolgee.model.AutoTranslationConfig import io.tolgee.model.Project import io.tolgee.model.enums.TranslationState @@ -27,25 +26,17 @@ class AutoTranslationService( fun autoTranslate( key: Key, - languageTags: Set? = null, + languageTags: List? = null, + useTranslationMemory: Boolean? = null, + useMachineTranslation: Boolean? = null ) { val config = getConfig(key.project) - if (config.usingPrimaryMtService || config.usingTm) { - autoTranslate(key, languageTags, config.usingTm, config.usingPrimaryMtService) - } - } - fun autoTranslate( - key: Key, - languageTags: Set? = null, - useTranslationMemory: Boolean, - useMachineTranslation: Boolean - ) { - if (useTranslationMemory) { - autoTranslateUsingTm(key, languageTags) + if (useTranslationMemory ?: config.usingTm) { + autoTranslateUsingTm(key, languageTags?.toSet()) } - if (useMachineTranslation) { - autoTranslateUsingMachineTranslation(key, languageTags) + if (useMachineTranslation ?: config.usingPrimaryMtService) { + autoTranslateUsingMachineTranslation(key, languageTags?.toSet()) } } @@ -60,20 +51,16 @@ class AutoTranslationService( ) { val languages = translations.map { it.language } - try { - mtService.getPrimaryMachineTranslations(key, languages) - .zip(translations) - .asSequence() - .forEach { (translateResult, translation) -> - translateResult?.let { - it.translatedText?.let { text -> - translation.setValueAndState(text, it.usedService) - } + mtService.getPrimaryMachineTranslations(key, languages) + .zip(translations) + .asSequence() + .forEach { (translateResult, translation) -> + translateResult?.let { + it.translatedText?.let { text -> + translation.setValueAndState(text, it.usedService) } } - } catch (e: OutOfCreditsException) { - logger.debug("Out of credits for primary MT service") - } + } } private fun autoTranslateUsingTm(key: Key, languageTags: Set? = null) { diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationCommentService.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationCommentService.kt index fe61342c5f..c3b806e239 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationCommentService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationCommentService.kt @@ -84,4 +84,8 @@ class TranslationCommentService( ): TranslationComment { return translationCommentRepository.save(entity) } + + fun deleteByTranslationIdIn(ids: Collection) { + return translationCommentRepository.deleteByTranslationIdIn(ids) + } } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt index 059b0fbfdd..0ca50ac2e2 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt @@ -41,7 +41,8 @@ class TranslationService( private val tolgeeProperties: TolgeeProperties, private val applicationEventPublisher: ApplicationEventPublisher, private val translationViewDataProvider: TranslationViewDataProvider, - private val entityManager: EntityManager + private val entityManager: EntityManager, + private val translationCommentService: TranslationCommentService ) { @set:Autowired @set:Lazy @@ -218,6 +219,7 @@ class TranslationService( fun deleteByIdIn(ids: Collection) { importService.onExistingTranslationsRemoved(ids) + translationCommentService.deleteByTranslationIdIn(ids) translationRepository.deleteByIdIn(ids) } diff --git a/backend/data/src/main/kotlin/io/tolgee/util/TolgeeAtomicLong.kt b/backend/data/src/main/kotlin/io/tolgee/util/TolgeeAtomicLong.kt new file mode 100644 index 0000000000..ba02515766 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/util/TolgeeAtomicLong.kt @@ -0,0 +1,9 @@ +package io.tolgee.util + +interface TolgeeAtomicLong { + fun addAndGet(delta: Long): Long + + fun delete() + + fun get(): Long +} diff --git a/backend/data/src/main/kotlin/io/tolgee/util/transactionUtil.kt b/backend/data/src/main/kotlin/io/tolgee/util/transactionUtil.kt index e7013eb2d6..2919ab7f54 100644 --- a/backend/data/src/main/kotlin/io/tolgee/util/transactionUtil.kt +++ b/backend/data/src/main/kotlin/io/tolgee/util/transactionUtil.kt @@ -1,17 +1,21 @@ package io.tolgee.util +import org.springframework.dao.CannotAcquireLockException import org.springframework.transaction.PlatformTransactionManager import org.springframework.transaction.TransactionDefinition import org.springframework.transaction.support.TransactionTemplate +import javax.persistence.OptimisticLockException fun executeInNewTransaction( transactionManager: PlatformTransactionManager, - isolationLevel: Int = TransactionDefinition.ISOLATION_SERIALIZABLE, + isolationLevel: Int = TransactionDefinition.ISOLATION_DEFAULT, + propagationBehavior: Int = TransactionDefinition.PROPAGATION_REQUIRES_NEW, fn: () -> T ): T { val tt = TransactionTemplate(transactionManager) - tt.propagationBehavior = TransactionDefinition.PROPAGATION_REQUIRES_NEW + tt.propagationBehavior = propagationBehavior tt.isolationLevel = isolationLevel + return tt.execute { fn() } as T @@ -21,5 +25,42 @@ fun executeInNewTransaction( transactionManager: PlatformTransactionManager, fn: () -> T ): T { - return executeInNewTransaction(transactionManager, TransactionDefinition.ISOLATION_DEFAULT, fn) + return executeInNewTransaction( + transactionManager = transactionManager, + fn = fn, + propagationBehavior = TransactionDefinition.PROPAGATION_REQUIRES_NEW + ) +} + +fun executeInNewRepeatableTransaction( + transactionManager: PlatformTransactionManager, + propagationBehavior: Int = TransactionDefinition.PROPAGATION_REQUIRES_NEW, + fn: () -> T +): T { + var exception: Exception? = null + var repeats = 0 + for (it in 1..100) { + try { + return executeInNewTransaction( + transactionManager, + propagationBehavior = propagationBehavior, + isolationLevel = TransactionDefinition.ISOLATION_SERIALIZABLE + ) { + fn() + } + } catch (e: Exception) { + when (e) { + is OptimisticLockException, is CannotAcquireLockException -> { + exception = e + repeats++ + } + + else -> throw e + } + } + } + throw RepeatedlyCannotSerializeTransactionException(exception!!, repeats) } + +class RepeatedlyCannotSerializeTransactionException(cause: Throwable, repeats: Int) : + RuntimeException("Retry failed $repeats times.", cause) diff --git a/backend/app/src/main/kotlin/io/tolgee/websocket/ActorInfo.kt b/backend/data/src/main/kotlin/io/tolgee/websocket/ActorInfo.kt similarity index 100% rename from backend/app/src/main/kotlin/io/tolgee/websocket/ActorInfo.kt rename to backend/data/src/main/kotlin/io/tolgee/websocket/ActorInfo.kt diff --git a/backend/app/src/main/kotlin/io/tolgee/websocket/ActorType.kt b/backend/data/src/main/kotlin/io/tolgee/websocket/ActorType.kt similarity index 100% rename from backend/app/src/main/kotlin/io/tolgee/websocket/ActorType.kt rename to backend/data/src/main/kotlin/io/tolgee/websocket/ActorType.kt diff --git a/backend/app/src/main/kotlin/io/tolgee/websocket/RedisWebsocketEventWrapper.kt b/backend/data/src/main/kotlin/io/tolgee/websocket/RedisWebsocketEventWrapper.kt similarity index 100% rename from backend/app/src/main/kotlin/io/tolgee/websocket/RedisWebsocketEventWrapper.kt rename to backend/data/src/main/kotlin/io/tolgee/websocket/RedisWebsocketEventWrapper.kt diff --git a/backend/app/src/main/kotlin/io/tolgee/websocket/WebsocketEvent.kt b/backend/data/src/main/kotlin/io/tolgee/websocket/WebsocketEvent.kt similarity index 89% rename from backend/app/src/main/kotlin/io/tolgee/websocket/WebsocketEvent.kt rename to backend/data/src/main/kotlin/io/tolgee/websocket/WebsocketEvent.kt index 6982b8cf0f..8c92da85fb 100644 --- a/backend/app/src/main/kotlin/io/tolgee/websocket/WebsocketEvent.kt +++ b/backend/data/src/main/kotlin/io/tolgee/websocket/WebsocketEvent.kt @@ -6,6 +6,6 @@ data class WebsocketEvent( val actor: ActorInfo, val data: Any? = null, val sourceActivity: ActivityType?, - val activityId: Long, + val activityId: Long?, val dataCollapsed: Boolean ) diff --git a/backend/app/src/main/kotlin/io/tolgee/websocket/WebsocketEventPublisher.kt b/backend/data/src/main/kotlin/io/tolgee/websocket/WebsocketEventPublisher.kt similarity index 100% rename from backend/app/src/main/kotlin/io/tolgee/websocket/WebsocketEventPublisher.kt rename to backend/data/src/main/kotlin/io/tolgee/websocket/WebsocketEventPublisher.kt diff --git a/backend/app/src/main/kotlin/io/tolgee/websocket/Types.kt b/backend/data/src/main/kotlin/io/tolgee/websocket/WebsocketEventType.kt similarity index 51% rename from backend/app/src/main/kotlin/io/tolgee/websocket/Types.kt rename to backend/data/src/main/kotlin/io/tolgee/websocket/WebsocketEventType.kt index de372f70a4..020d0b52d7 100644 --- a/backend/app/src/main/kotlin/io/tolgee/websocket/Types.kt +++ b/backend/data/src/main/kotlin/io/tolgee/websocket/WebsocketEventType.kt @@ -1,7 +1,8 @@ package io.tolgee.websocket -enum class Types() { - TRANSLATION_DATA_MODIFIED; +enum class WebsocketEventType() { + TRANSLATION_DATA_MODIFIED, + BATCH_JOB_PROGRESS; val typeName get() = name.lowercase().replace("_", "-") } diff --git a/backend/data/src/main/resources/db/changelog/schema.xml b/backend/data/src/main/resources/db/changelog/schema.xml index b528902e75..29e7211909 100644 --- a/backend/data/src/main/resources/db/changelog/schema.xml +++ b/backend/data/src/main/resources/db/changelog/schema.xml @@ -2521,6 +2521,134 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/backend/testing/build.gradle b/backend/testing/build.gradle index 004a5821a9..5081f578dd 100644 --- a/backend/testing/build.gradle +++ b/backend/testing/build.gradle @@ -81,6 +81,7 @@ dependencies { } kapt "org.springframework.boot:spring-boot-configuration-processor" implementation "org.springframework.boot:spring-boot-configuration-processor" + api "org.springframework.boot:spring-boot-starter-actuator" /** * Testing diff --git a/backend/testing/src/main/kotlin/io/tolgee/AbstractSpringTest.kt b/backend/testing/src/main/kotlin/io/tolgee/AbstractSpringTest.kt index dee904a578..bca4be4dcb 100644 --- a/backend/testing/src/main/kotlin/io/tolgee/AbstractSpringTest.kt +++ b/backend/testing/src/main/kotlin/io/tolgee/AbstractSpringTest.kt @@ -51,7 +51,22 @@ import org.springframework.transaction.TransactionDefinition import org.springframework.transaction.support.TransactionTemplate @TestInstance(TestInstance.Lifecycle.PER_CLASS) -@SpringBootTest +@SpringBootTest( +// exclude = [ +// CompositeMeterRegistryAutoConfiguration::class, +// DataSourcePoolMetricsAutoConfiguration::class, +// DiskSpaceHealthContributorAutoConfiguration::class, +// InfoContributorAutoConfiguration::class, +// JmxAutoConfiguration::class, +// JvmMetricsAutoConfiguration::class, +// JmxEndpointAutoConfiguration::class, +// LdapAutoConfiguration::class, +// LiquibaseEndpointAutoConfiguration::class, +// MetricsEndpointAutoConfiguration::class, +// StartupTimeMetricsListenerAutoConfiguration::class, +// TomcatMetricsAutoConfiguration::class, +// ] +) abstract class AbstractSpringTest : AbstractTransactionalTest() { @Autowired protected lateinit var dbPopulator: DbPopulatorReal @@ -238,9 +253,9 @@ abstract class AbstractSpringTest : AbstractTransactionalTest() { fun executeInNewTransaction(fn: () -> T): T { return io.tolgee.util.executeInNewTransaction( - platformTransactionManager, - TransactionDefinition.ISOLATION_DEFAULT, - fn + transactionManager = platformTransactionManager, + fn = fn, + isolationLevel = TransactionDefinition.ISOLATION_DEFAULT ) } } diff --git a/backend/testing/src/main/kotlin/io/tolgee/CleanDbTestListener.kt b/backend/testing/src/main/kotlin/io/tolgee/CleanDbTestListener.kt index 4fcd76deb6..b954d61dea 100644 --- a/backend/testing/src/main/kotlin/io/tolgee/CleanDbTestListener.kt +++ b/backend/testing/src/main/kotlin/io/tolgee/CleanDbTestListener.kt @@ -1,14 +1,17 @@ package io.tolgee +import io.tolgee.batch.BatchJobConcurrentLauncher import kotlinx.coroutines.TimeoutCancellationException -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.withTimeout import org.postgresql.util.PSQLException import org.slf4j.LoggerFactory import org.springframework.context.ApplicationContext import org.springframework.test.context.TestContext import org.springframework.test.context.TestExecutionListener import java.sql.ResultSet +import java.util.concurrent.Executors +import java.util.concurrent.Future +import java.util.concurrent.TimeUnit +import java.util.concurrent.TimeoutException import javax.sql.DataSource import kotlin.system.measureTimeMillis @@ -21,9 +24,15 @@ class CleanDbTestListener : TestExecutionListener { ) override fun beforeTestMethod(testContext: TestContext) { + val appContext: ApplicationContext = testContext.applicationContext + val jobChunkExecutionQueue = appContext.getBean(BatchJobConcurrentLauncher::class.java) + jobChunkExecutionQueue.pause = true + if (!shouldClenBeforeClass(testContext)) { cleanWithRetries(testContext) } + + jobChunkExecutionQueue.pause = false } private fun cleanWithRetries(testContext: TestContext) { @@ -32,10 +41,8 @@ class CleanDbTestListener : TestExecutionListener { var i = 0 while (true) { try { - runBlocking { - withTimeout(3000) { - doClean(testContext) - } + withTimeout(3000) { + doClean(testContext) } break } catch (e: Exception) { @@ -104,4 +111,18 @@ class CleanDbTestListener : TestExecutionListener { @Throws(Exception::class) override fun prepareTestInstance(testContext: TestContext) { } + + private fun withTimeout(timeout: Long, block: () -> Unit) { + val executor = Executors.newSingleThreadExecutor() + val future: Future = executor.submit(block) + + try { + println(future[timeout, TimeUnit.MILLISECONDS]) + } catch (e: TimeoutException) { + future.cancel(true) + throw e + } + + executor.shutdownNow() + } } diff --git a/backend/testing/src/main/kotlin/io/tolgee/testing/AbstractControllerTest.kt b/backend/testing/src/main/kotlin/io/tolgee/testing/AbstractControllerTest.kt index c63f02d66d..c34919af46 100644 --- a/backend/testing/src/main/kotlin/io/tolgee/testing/AbstractControllerTest.kt +++ b/backend/testing/src/main/kotlin/io/tolgee/testing/AbstractControllerTest.kt @@ -11,6 +11,7 @@ import io.tolgee.security.LoginRequest import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Qualifier import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc +import org.springframework.boot.test.context.SpringBootTest import org.springframework.http.HttpHeaders import org.springframework.http.MediaType import org.springframework.test.web.servlet.MockMvc @@ -21,6 +22,7 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders import java.io.UnsupportedEncodingException @AutoConfigureMockMvc +@SpringBootTest abstract class AbstractControllerTest : AbstractSpringTest(), RequestPerformer { diff --git a/backend/testing/src/main/kotlin/io/tolgee/testing/AbstractTransactionalTest.kt b/backend/testing/src/main/kotlin/io/tolgee/testing/AbstractTransactionalTest.kt index 75cd912aa6..187734da92 100644 --- a/backend/testing/src/main/kotlin/io/tolgee/testing/AbstractTransactionalTest.kt +++ b/backend/testing/src/main/kotlin/io/tolgee/testing/AbstractTransactionalTest.kt @@ -5,6 +5,7 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.TestExecutionListeners import org.springframework.test.context.support.DependencyInjectionTestExecutionListener +import org.springframework.test.context.support.DirtiesContextTestExecutionListener import org.springframework.test.context.transaction.TestTransaction import org.springframework.test.context.transaction.TransactionalTestExecutionListener import javax.persistence.EntityManager @@ -13,7 +14,8 @@ import javax.persistence.EntityManager value = [ TransactionalTestExecutionListener::class, DependencyInjectionTestExecutionListener::class, - CleanDbTestListener::class + CleanDbTestListener::class, + DirtiesContextTestExecutionListener::class ] ) @ActiveProfiles(profiles = ["local"]) diff --git a/backend/testing/src/main/kotlin/io/tolgee/testing/WebsocketTest.kt b/backend/testing/src/main/kotlin/io/tolgee/testing/WebsocketTest.kt new file mode 100644 index 0000000000..2c1671d89a --- /dev/null +++ b/backend/testing/src/main/kotlin/io/tolgee/testing/WebsocketTest.kt @@ -0,0 +1,6 @@ +package io.tolgee.testing + +import org.junit.jupiter.api.Tag + +@Tag("websocket") +annotation class WebsocketTest diff --git a/ee/backend/tests/src/test/resources/application.yaml b/ee/backend/tests/src/test/resources/application.yaml index 9d275955cb..2d06804c1f 100644 --- a/ee/backend/tests/src/test/resources/application.yaml +++ b/ee/backend/tests/src/test/resources/application.yaml @@ -1,4 +1,8 @@ spring: + autoconfigure: + exclude: + - org.redisson.spring.starter.RedissonAutoConfiguration + - org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration jpa: show-sql: true properties: diff --git a/settings.gradle b/settings.gradle index 7ad009bd10..1489bcbc92 100644 --- a/settings.gradle +++ b/settings.gradle @@ -43,7 +43,7 @@ dependencyResolutionManagement { library('jjwtJackson', 'io.jsonwebtoken', 'jjwt-jackson').version(jjwtVersion) library('assertJCore', 'org.assertj:assertj-core:3.19.0') library('springmockk', 'com.ninja-squad:springmockk:3.0.1') - library('mockito', 'org.mockito.kotlin:mockito-kotlin:3.2.0') + library('mockito', 'org.mockito.kotlin:mockito-kotlin:5.0.0') library('commonsCodec', 'commons-codec:commons-codec:1.15') library('icu4j', 'com.ibm.icu:icu4j:69.1') library('jsonUnitAssert', 'net.javacrumbs.json-unit:json-unit-assertj:2.28.0')