diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobController.kt index 2c29520b71..dad9e56be8 100644 --- a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobController.kt +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobController.kt @@ -8,7 +8,12 @@ import io.tolgee.batch.request.BatchTranslateRequest import io.tolgee.batch.request.ClearTranslationsRequest import io.tolgee.batch.request.CopyTranslationRequest import io.tolgee.batch.request.DeleteKeysRequest +import io.tolgee.batch.request.SetKeysNamespaceRequest import io.tolgee.batch.request.SetTranslationsStateStateRequest +import io.tolgee.batch.request.TagKeysRequest +import io.tolgee.batch.request.UntagKeysRequest +import io.tolgee.constants.Message +import io.tolgee.exceptions.BadRequestException import io.tolgee.hateoas.batch.BatchJobModel import io.tolgee.hateoas.batch.BatchJobModelAssembler import io.tolgee.model.batch.BatchJob @@ -112,6 +117,54 @@ class StartBatchJobController( ).model } + @PostMapping(value = ["/tag-keys"]) + @AccessWithApiKey() + @AccessWithProjectPermission(Scope.KEYS_EDIT) + @Operation(summary = "Tag keys") + fun tagKeys(@Valid @RequestBody data: TagKeysRequest): BatchJobModel { + data.tags.validate() + securityService.checkKeyIdsExistAndIsFromProject(data.keyIds, projectHolder.project.id) + return batchJobService.startJob( + data, + projectHolder.projectEntity, + authenticationFacade.userAccountEntity, + BatchJobType.TAG_KEYS + ).model + } + + @PostMapping(value = ["/untag-keys"]) + @AccessWithApiKey() + @AccessWithProjectPermission(Scope.KEYS_EDIT) + @Operation(summary = "Tag keys") + fun untagKeys(@Valid @RequestBody data: UntagKeysRequest): BatchJobModel { + securityService.checkKeyIdsExistAndIsFromProject(data.keyIds, projectHolder.project.id) + return batchJobService.startJob( + data, + projectHolder.projectEntity, + authenticationFacade.userAccountEntity, + BatchJobType.UNTAG_KEYS + ).model + } + + @PostMapping(value = ["/set-keys-namespace"]) + @AccessWithApiKey() + @AccessWithProjectPermission(Scope.KEYS_EDIT) + @Operation(summary = "Tag keys") + fun setKeysNamespace(@Valid @RequestBody data: SetKeysNamespaceRequest): BatchJobModel { + securityService.checkKeyIdsExistAndIsFromProject(data.keyIds, projectHolder.project.id) + return batchJobService.startJob( + data, + projectHolder.projectEntity, + authenticationFacade.userAccountEntity, + BatchJobType.SET_KEYS_NAMESPACE + ).model + } + val BatchJob.model get() = batchJobModelAssembler.toModel(batchJobService.getView(this)) + + private fun List.validate() { + if (this.any { it.isBlank() }) throw BadRequestException(Message.TAG_IS_BLANK) + if (this.any { it.length > 100 }) throw BadRequestException(Message.TAG_TOO_LOG) + } } diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt index 7aee489ba6..9c94b115d7 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt @@ -2,10 +2,14 @@ package io.tolgee.api.v2.controllers.batch import io.tolgee.ProjectAuthControllerTest import io.tolgee.batch.BatchJobChunkExecutionQueue +import io.tolgee.batch.BatchJobService import io.tolgee.development.testDataBuilder.data.BatchJobsTestData import io.tolgee.fixtures.andAssertThatJson +import io.tolgee.fixtures.andIsBadRequest import io.tolgee.fixtures.andIsOk +import io.tolgee.fixtures.andPrettyPrint import io.tolgee.fixtures.isValidId +import io.tolgee.fixtures.waitFor import io.tolgee.fixtures.waitForNotThrowing import io.tolgee.model.batch.BatchJob import io.tolgee.model.batch.BatchJobStatus @@ -19,6 +23,8 @@ import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc +import org.springframework.test.web.servlet.ResultActions +import java.util.function.Consumer @AutoConfigureMockMvc @ContextRecreatingTest @@ -29,6 +35,9 @@ class StartBatchJobControllerTest : ProjectAuthControllerTest("/v2/projects/") { @Autowired lateinit var batchJobOperationQueue: BatchJobChunkExecutionQueue + @Autowired + lateinit var batchJobService: BatchJobService + @BeforeEach fun setup() { batchJobOperationQueue.clear() @@ -222,4 +231,136 @@ class StartBatchJobControllerTest : ProjectAuthControllerTest("/v2/projects/") { all.count { it.text?.startsWith("en") == true }.assert.isEqualTo(allKeyIds.size + keyIds.size * 2) } } + + @Test + @ProjectJWTAuthTestMethod + fun `it validates tag length`() { + performProjectAuthPost( + "start-batch-job/tag-keys", + mapOf( + "keyIds" to listOf(1), + "tags" to listOf("a".repeat(101)), + ) + ).andIsBadRequest.andPrettyPrint + } + + @Test + @ProjectJWTAuthTestMethod + fun `it tags keys`() { + val keyCount = 1000 + val keys = testData.addTagKeysData(keyCount) + saveAndPrepare() + + val allKeyIds = keys.map { it.id }.toList() + val keyIds = allKeyIds.take(500) + val newTags = listOf("tag1", "tag3", "a-tag", "b-tag") + + performProjectAuthPost( + "start-batch-job/tag-keys", + mapOf( + "keyIds" to keyIds, + "tags" to newTags, + ) + ).andIsOk + + waitForNotThrowing(pollTime = 1000, timeout = 10000) { + val all = keyService.getKeysWithTagsById(keyIds) + all.assert.hasSize(keyIds.size) + all.count { + it.keyMeta?.tags?.map { it.name }?.containsAll(newTags) == true + }.assert.isEqualTo(keyIds.size) + } + } + + @Test + @ProjectJWTAuthTestMethod + fun `it untags keys`() { + val keyCount = 1000 + val keys = testData.addTagKeysData(keyCount) + saveAndPrepare() + + val allKeyIds = keys.map { it.id }.toList() + val keyIds = allKeyIds.take(300) + val tagsToRemove = listOf("tag1", "a-tag", "b-tag") + + performProjectAuthPost( + "start-batch-job/untag-keys", + mapOf( + "keyIds" to keyIds, + "tags" to tagsToRemove + ) + ).andIsOk + + waitForNotThrowing(pollTime = 1000, timeout = 10000) { + val all = keyService.getKeysWithTagsById(keyIds) + all.assert.hasSize(keyIds.size) + all.count { + it.keyMeta?.tags?.map { it.name }?.any { tagsToRemove.contains(it) } == false && + it.keyMeta?.tags?.map { it.name }?.contains("tag3") == true + }.assert.isEqualTo(keyIds.size) + } + } + + @Test + @ProjectJWTAuthTestMethod + fun `it moves to other namespace`() { + val keys = testData.addNamespaceData() + saveAndPrepare() + + val allKeyIds = keys.map { it.id }.toList() + val keyIds = allKeyIds.take(700) + + performProjectAuthPost( + "start-batch-job/set-keys-namespace", + mapOf( + "keyIds" to keyIds, + "namespace" to "other-namespace" + ) + ).andIsOk.waitForJobCompleted() + + val all = keyService.find(keyIds) + all.count { it.namespace?.name == "other-namespace" }.assert.isEqualTo(keyIds.size) + } + + @Test + @ProjectJWTAuthTestMethod + fun `it fails on collision`() { + testData.addNamespaceData() + val key = testData.projectBuilder.addKey(keyName = "key").self + saveAndPrepare() + + val jobId = performProjectAuthPost( + "start-batch-job/set-keys-namespace", + mapOf( + "keyIds" to listOf(key.id), + "namespace" to "namespace" + ) + ).andIsOk.waitForJobCompleted().jobId + keyService.get(key.id).namespace.assert.isNull() + batchJobService.findJobDto(jobId)?.status.assert.isEqualTo(BatchJobStatus.FAILED) + } + + fun ResultActions.waitForJobCompleted() = andAssertThatJson { + node("id").isNumber.satisfies( + Consumer { + waitFor(pollTime = 2000) { + val job = batchJobService.findJobDto(it.toLong()) + job?.status?.completed == true + } + } + ) + } + + val ResultActions.jobId: Long + get() { + var jobId: Long? = null + this.andAssertThatJson { + node("id").isNumber.satisfies( + Consumer { + jobId = it.toLong() + } + ) + } + return jobId!! + } } diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/data/ActivityType.kt b/backend/data/src/main/kotlin/io/tolgee/activity/data/ActivityType.kt index 262e96887c..20dc990764 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/data/ActivityType.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/data/ActivityType.kt @@ -31,7 +31,12 @@ enum class ActivityType( CREATE_PROJECT, EDIT_PROJECT, NAMESPACE_EDIT, - BATCH_AUTO_TRANSLATE, - CLEAR_TRANSLATIONS, - COPY_TRANSLATIONS + BATCH_AUTO_TRANSLATE(true), + BATCH_CLEAR_TRANSLATIONS(true), + BATCH_COPY_TRANSLATIONS(true), + BATCH_SET_TRANSLATION_STATE(true), + BATCH_TAG_KEYS(true), + BATCH_UNTAG_KEYS(true), + BATCH_SET_KEYS_NAMESPACE(true) + ; } diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt index 30f8573492..822d883486 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt @@ -2,6 +2,7 @@ package io.tolgee.batch import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import io.sentry.Sentry +import io.tolgee.component.SavePointManager import io.tolgee.component.UsingRedisProvider import io.tolgee.model.batch.BatchJobChunkExecution import io.tolgee.model.batch.BatchJobChunkExecutionStatus @@ -18,6 +19,7 @@ import org.springframework.data.redis.core.StringRedisTemplate import org.springframework.stereotype.Service import org.springframework.transaction.PlatformTransactionManager import org.springframework.transaction.TransactionDefinition +import org.springframework.transaction.UnexpectedRollbackException import javax.persistence.EntityManager import javax.persistence.LockModeType @@ -34,7 +36,8 @@ class BatchJobActionService( private val batchJobChunkExecutionQueue: BatchJobChunkExecutionQueue, @Lazy private val redisTemplate: StringRedisTemplate, - private val concurrentExecutionLauncher: BatchJobConcurrentLauncher + private val concurrentExecutionLauncher: BatchJobConcurrentLauncher, + private val savePointManager: SavePointManager ) : Logging { companion object { const val MIN_TIME_BETWEEN_OPERATIONS = 10 @@ -49,38 +52,59 @@ class BatchJobActionService( concurrentExecutionLauncher.run { executionItem, coroutineContext -> var retryExecution: BatchJobChunkExecution? = null - val execution = executeInNewTransaction( - transactionManager, - isolationLevel = TransactionDefinition.ISOLATION_DEFAULT - ) { - catchingExceptions(executionItem) { + try { + val execution = executeInNewTransaction( + transactionManager, + isolationLevel = TransactionDefinition.ISOLATION_DEFAULT + ) { transactionStatus -> + catchingExceptions(executionItem) { - val lockedExecution = getPendingUnlockedExecutionItem(executionItem) - ?: return@executeInNewTransaction null + val lockedExecution = getPendingUnlockedExecutionItem(executionItem) + ?: return@executeInNewTransaction null - publishRemoveConsuming(executionItem) + publishRemoveConsuming(executionItem) - progressManager.handleJobRunning(lockedExecution.batchJob.id) - val batchJobDto = batchJobService.getJobDto(lockedExecution.batchJob.id) + progressManager.handleJobRunning(lockedExecution.batchJob.id) + val batchJobDto = batchJobService.getJobDto(lockedExecution.batchJob.id) - logger.debug("Job ${batchJobDto.id}: 🟡 Processing chunk ${lockedExecution.id}") - val util = ChunkProcessingUtil(lockedExecution, applicationContext, coroutineContext) - util.processChunk() + logger.debug("Job ${batchJobDto.id}: 🟡 Processing chunk ${lockedExecution.id}") + val savepoint = savePointManager.setSavepoint() + val util = ChunkProcessingUtil(lockedExecution, applicationContext, coroutineContext) + util.processChunk() - progressManager.handleProgress(lockedExecution) - entityManager.persist(lockedExecution) + if (transactionStatus.isRollbackOnly) { + logger.debug("Job ${batchJobDto.id}: 🛑 Rollbacking chunk ${lockedExecution.id}") + savePointManager.rollbackSavepoint(savepoint) + } - if (lockedExecution.retry) { - retryExecution = util.retryExecution - entityManager.persist(util.retryExecution) - } + progressManager.handleProgress(lockedExecution) + entityManager.persist(lockedExecution) + + if (lockedExecution.retry) { + retryExecution = util.retryExecution + entityManager.persist(util.retryExecution) + } - logger.debug("Job ${batchJobDto.id}: ✅ Processed chunk ${lockedExecution.id}") - return@executeInNewTransaction lockedExecution + logger.debug("Job ${batchJobDto.id}: ✅ Processed chunk ${lockedExecution.id}") + return@executeInNewTransaction lockedExecution + } + } + execution?.let { progressManager.handleChunkCompletedCommitted(it) } + addRetryExecutionToQueue(retryExecution) + } catch (e: Throwable) { + when (e) { + is UnexpectedRollbackException -> { + logger.debug( + "Job ${executionItem.jobId}: ⚠️ Chunk ${executionItem.chunkExecutionId}" + + " thrown UnexpectedRollbackException" + ) + } + else -> { + logger.error("Job ${executionItem.jobId}: ⚠️ Chunk ${executionItem.chunkExecutionId} thrown error", e) + Sentry.captureException(e) + } } } - execution?.let { progressManager.handleChunkCompletedCommitted(it) } - addRetryExecutionToQueue(retryExecution) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobType.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobType.kt index 21074d93a1..a0485f5bd3 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobType.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobType.kt @@ -5,7 +5,10 @@ import io.tolgee.batch.processors.AutoTranslationChunkProcessor import io.tolgee.batch.processors.ClearTranslationsChunkProcessor import io.tolgee.batch.processors.CopyTranslationsChunkProcessor import io.tolgee.batch.processors.DeleteKeysChunkProcessor +import io.tolgee.batch.processors.SetKeysNamespaceChunkProcessor import io.tolgee.batch.processors.SetTranslationsStateChunkProcessor +import io.tolgee.batch.processors.TagKeysChunkProcessor +import io.tolgee.batch.processors.UntagKeysChunkProcessor import kotlin.reflect.KClass enum class BatchJobType( @@ -31,21 +34,39 @@ enum class BatchJobType( processor = DeleteKeysChunkProcessor::class, ), SET_TRANSLATIONS_STATE( - activityType = ActivityType.SET_TRANSLATION_STATE, + activityType = ActivityType.BATCH_SET_TRANSLATION_STATE, chunkSize = 0, maxRetries = 3, processor = SetTranslationsStateChunkProcessor::class, ), CLEAR_TRANSLATIONS( - activityType = ActivityType.CLEAR_TRANSLATIONS, + activityType = ActivityType.BATCH_CLEAR_TRANSLATIONS, chunkSize = 0, maxRetries = 3, processor = ClearTranslationsChunkProcessor::class, ), COPY_TRANSLATIONS( - activityType = ActivityType.COPY_TRANSLATIONS, + activityType = ActivityType.BATCH_COPY_TRANSLATIONS, chunkSize = 0, maxRetries = 3, processor = CopyTranslationsChunkProcessor::class, + ), + TAG_KEYS( + activityType = ActivityType.BATCH_TAG_KEYS, + chunkSize = 0, + maxRetries = 3, + processor = TagKeysChunkProcessor::class, + ), + UNTAG_KEYS( + activityType = ActivityType.BATCH_UNTAG_KEYS, + chunkSize = 0, + maxRetries = 3, + processor = UntagKeysChunkProcessor::class, + ), + SET_KEYS_NAMESPACE( + activityType = ActivityType.BATCH_SET_KEYS_NAMESPACE, + chunkSize = 0, + maxRetries = 3, + processor = SetKeysNamespaceChunkProcessor::class, ); } diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/processors/SetKeysNamespaceChunkProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/batch/processors/SetKeysNamespaceChunkProcessor.kt new file mode 100644 index 0000000000..b5aaa3f6cc --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/processors/SetKeysNamespaceChunkProcessor.kt @@ -0,0 +1,67 @@ +package io.tolgee.batch.processors + +import io.tolgee.batch.BatchJobDto +import io.tolgee.batch.ChunkProcessor +import io.tolgee.batch.FailedDontRequeueException +import io.tolgee.batch.request.SetKeysNamespaceRequest +import io.tolgee.constants.Message +import io.tolgee.model.EntityWithId +import io.tolgee.model.batch.BatchJob +import io.tolgee.model.batch.SetKeysNamespaceParams +import io.tolgee.service.key.KeyService +import kotlinx.coroutines.ensureActive +import org.apache.commons.lang3.exception.ExceptionUtils.getRootCauseMessage +import org.springframework.stereotype.Component +import javax.persistence.EntityManager +import javax.persistence.PersistenceException +import kotlin.coroutines.CoroutineContext + +@Component +class SetKeysNamespaceChunkProcessor( + private val entityManager: EntityManager, + private val keyService: KeyService +) : ChunkProcessor { + override fun process( + job: BatchJobDto, + chunk: List, + coroutineContext: CoroutineContext, + onProgress: ((Int) -> Unit) + ) { + val subChunked = chunk.chunked(100) + var progress = 0 + val params = getParams(job) + subChunked.forEach { subChunk -> + coroutineContext.ensureActive() + try { + keyService.setNamespace(subChunk, params.namespace) + entityManager.flush() + } catch (e: PersistenceException) { + if (getRootCauseMessage(e).contains("key_project_id_name_namespace_id_idx")) { + throw FailedDontRequeueException(Message.KEY_EXISTS_IN_NAMESPACE, listOf(), e) + } + throw e + } + progress += subChunk.size + onProgress.invoke(progress) + } + } + + override fun getTarget(data: SetKeysNamespaceRequest): List { + return data.keyIds + } + + private fun getParams(job: BatchJobDto): SetKeysNamespaceParams { + return entityManager.createQuery( + """from SetKeysNamespaceParams tkp where tkp.batchJob.id = :batchJobId""", + SetKeysNamespaceParams::class.java + ).setParameter("batchJobId", job.id).singleResult + ?: throw IllegalStateException("No params found") + } + + override fun getParams(data: SetKeysNamespaceRequest, job: BatchJob): EntityWithId? { + return SetKeysNamespaceParams().apply { + this.batchJob = job + this.namespace = data.namespace + } + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/processors/TagKeysChunkProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/batch/processors/TagKeysChunkProcessor.kt new file mode 100644 index 0000000000..68daea7a3a --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/processors/TagKeysChunkProcessor.kt @@ -0,0 +1,56 @@ +package io.tolgee.batch.processors + +import io.tolgee.batch.BatchJobDto +import io.tolgee.batch.ChunkProcessor +import io.tolgee.batch.request.TagKeysRequest +import io.tolgee.model.EntityWithId +import io.tolgee.model.batch.BatchJob +import io.tolgee.model.batch.TagKeysParams +import io.tolgee.service.key.TagService +import kotlinx.coroutines.ensureActive +import org.springframework.stereotype.Component +import javax.persistence.EntityManager +import kotlin.coroutines.CoroutineContext + +@Component +class TagKeysChunkProcessor( + private val entityManager: EntityManager, + private val tagService: TagService +) : ChunkProcessor { + override fun process( + job: BatchJobDto, + chunk: List, + coroutineContext: CoroutineContext, + onProgress: ((Int) -> Unit) + ) { + val subChunked = chunk.chunked(100) + var progress: Int = 0 + var params = getParams(job) + subChunked.forEach { subChunk -> + coroutineContext.ensureActive() + tagService.tagKeysById(subChunk.associateWith { params.tags }) + entityManager.flush() + progress += subChunk.size + onProgress.invoke(progress) + } + } + + override fun getTarget(data: TagKeysRequest): List { + return data.keyIds + } + + private fun getParams(job: BatchJobDto): TagKeysParams { + return entityManager.createQuery( + """from TagKeysParams tkp where tkp.batchJob.id = :batchJobId""", + TagKeysParams::class.java + ).setParameter("batchJobId", job.id).singleResult + ?: throw IllegalStateException("No params found") + } + + override fun getParams(data: TagKeysRequest, job: BatchJob): EntityWithId? { + return TagKeysParams().apply { + this.batchJob = job + this.tags = data.tags + } + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/processors/UntagKeysChunkProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/batch/processors/UntagKeysChunkProcessor.kt new file mode 100644 index 0000000000..f6447d69c1 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/processors/UntagKeysChunkProcessor.kt @@ -0,0 +1,56 @@ +package io.tolgee.batch.processors + +import io.tolgee.batch.BatchJobDto +import io.tolgee.batch.ChunkProcessor +import io.tolgee.batch.request.UntagKeysRequest +import io.tolgee.model.EntityWithId +import io.tolgee.model.batch.BatchJob +import io.tolgee.model.batch.UntagKeysParams +import io.tolgee.service.key.TagService +import kotlinx.coroutines.ensureActive +import org.springframework.stereotype.Component +import javax.persistence.EntityManager +import kotlin.coroutines.CoroutineContext + +@Component +class UntagKeysChunkProcessor( + private val entityManager: EntityManager, + private val tagService: TagService +) : ChunkProcessor { + override fun process( + job: BatchJobDto, + chunk: List, + coroutineContext: CoroutineContext, + onProgress: ((Int) -> Unit) + ) { + val subChunked = chunk.chunked(100) + var progress: Int = 0 + var params = getParams(job) + subChunked.forEach { subChunk -> + coroutineContext.ensureActive() + tagService.untagKeys(subChunk.associateWith { params.tags }) + entityManager.flush() + progress += subChunk.size + onProgress.invoke(progress) + } + } + + override fun getTarget(data: UntagKeysRequest): List { + return data.keyIds + } + + private fun getParams(job: BatchJobDto): UntagKeysParams { + return entityManager.createQuery( + """from UntagKeysParams ukp where ukp.batchJob.id = :batchJobId""", + UntagKeysParams::class.java + ).setParameter("batchJobId", job.id).singleResult + ?: throw IllegalStateException("No params found") + } + + override fun getParams(data: UntagKeysRequest, job: BatchJob): EntityWithId? { + return UntagKeysParams().apply { + this.batchJob = job + this.tags = data.tags + } + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/request/SetKeysNamespaceRequest.kt b/backend/data/src/main/kotlin/io/tolgee/batch/request/SetKeysNamespaceRequest.kt new file mode 100644 index 0000000000..92ab20108e --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/request/SetKeysNamespaceRequest.kt @@ -0,0 +1,16 @@ +package io.tolgee.batch.request + +import io.tolgee.constants.ValidationConstants +import javax.validation.constraints.NotBlank +import javax.validation.constraints.NotEmpty +import javax.validation.constraints.Size + +class SetKeysNamespaceRequest { + @NotEmpty + var keyIds: List = listOf() + + @NotEmpty + @NotBlank + @Size(max = ValidationConstants.MAX_NAMESPACE_LENGTH) + var namespace: String = "" +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/request/TagKeysRequest.kt b/backend/data/src/main/kotlin/io/tolgee/batch/request/TagKeysRequest.kt new file mode 100644 index 0000000000..b3caa01abb --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/request/TagKeysRequest.kt @@ -0,0 +1,11 @@ +package io.tolgee.batch.request + +import javax.validation.constraints.NotEmpty + +class TagKeysRequest { + @NotEmpty + var keyIds: List = listOf() + + @NotEmpty + var tags: List = listOf() +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/request/UntagKeysRequest.kt b/backend/data/src/main/kotlin/io/tolgee/batch/request/UntagKeysRequest.kt new file mode 100644 index 0000000000..d21fca93d8 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/request/UntagKeysRequest.kt @@ -0,0 +1,11 @@ +package io.tolgee.batch.request + +import javax.validation.constraints.NotEmpty + +class UntagKeysRequest { + @NotEmpty + var keyIds: List = listOf() + + @NotEmpty + var tags: List = listOf() +} diff --git a/backend/data/src/main/kotlin/io/tolgee/component/SavePointManager.kt b/backend/data/src/main/kotlin/io/tolgee/component/SavePointManager.kt new file mode 100644 index 0000000000..d7299cb668 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/component/SavePointManager.kt @@ -0,0 +1,47 @@ +package io.tolgee.component + +import org.hibernate.Session +import org.hibernate.internal.SessionImpl +import org.hibernate.resource.transaction.backend.jdbc.internal.JdbcResourceLocalTransactionCoordinatorImpl +import org.springframework.stereotype.Component +import java.sql.Savepoint +import java.util.* +import javax.persistence.EntityManager + +@Component +class SavePointManager( + private val entityManager: EntityManager +) { + fun setSavepoint(): Savepoint? { + var savepoint: Savepoint? = null + getSession().doWork { connection -> savepoint = connection.setSavepoint(UUID.randomUUID().toString()) } + return savepoint + } + + fun rollbackSavepoint(savepoint: Savepoint?) { + getSession().doWork { connection -> + connection.rollback(savepoint) + } + + val session = (getSession().session as? SessionImpl) ?: throw IllegalStateException("Session is not SessionImpl") + val coordinatorGetter = session::class.java.getMethod("getTransactionCoordinator") + coordinatorGetter.isAccessible = true + val coordinator = coordinatorGetter.invoke(session) as? JdbcResourceLocalTransactionCoordinatorImpl + ?: throw IllegalStateException("Transaction coordinator is not JdbcResourceLocalTransactionCoordinatorImpl") + val delegateField = coordinator::class.java.getDeclaredField("physicalTransactionDelegate") + delegateField.isAccessible = true + val delegate = + delegateField.get(coordinator) as? JdbcResourceLocalTransactionCoordinatorImpl.TransactionDriverControlImpl + ?: throw IllegalStateException("Transaction delegate is not TransactionDriverControlImpl") + delegateField.isAccessible = false + val field = delegate::class.java.getDeclaredField("rollbackOnly") + field.isAccessible = true + field.set(delegate, false) + field.isAccessible = false + } + + fun getSession(): Session { + return entityManager.unwrap(Session::class.java) + ?: throw IllegalStateException("Session is null") + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/constants/Message.kt b/backend/data/src/main/kotlin/io/tolgee/constants/Message.kt index 4d0bf358a0..fbe201d71b 100644 --- a/backend/data/src/main/kotlin/io/tolgee/constants/Message.kt +++ b/backend/data/src/main/kotlin/io/tolgee/constants/Message.kt @@ -150,7 +150,9 @@ enum class Message { PROJECT_NOT_SELECTED, PLAN_HAS_SUBSCRIBERS, TRANSLATION_FAILED, - BATCH_JOB_NOT_FOUND + BATCH_JOB_NOT_FOUND, + KEY_EXISTS_IN_NAMESPACE, + TAG_IS_BLANK ; val code: String diff --git a/backend/data/src/main/kotlin/io/tolgee/constants/ValidationConstants.kt b/backend/data/src/main/kotlin/io/tolgee/constants/ValidationConstants.kt new file mode 100644 index 0000000000..760b54d9cf --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/constants/ValidationConstants.kt @@ -0,0 +1,6 @@ +package io.tolgee.constants + +object ValidationConstants { + const val MAX_TAG_LENGTH = 100 + const val MAX_NAMESPACE_LENGTH = 100 +} diff --git a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/TestDataService.kt b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/TestDataService.kt index 17a809d46c..057c580f08 100644 --- a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/TestDataService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/TestDataService.kt @@ -118,10 +118,10 @@ class TestDataService( organizationBuilder.self.name.let { name -> organizationService.deleteAllByName(name) } } - additionalTestDataSavers.forEach { + additionalTestDataSavers.forEach { dataSaver -> tryUntilItDoesntBreakConstraint { executeInNewTransaction(transactionManager) { - it.clean(builder) + dataSaver.clean(builder) } } } @@ -130,7 +130,7 @@ class TestDataService( private fun updateLanguageStats(builder: TestDataBuilder) { builder.data.projects.forEach { try { - executeInNewTransaction(transactionManager) { + executeInNewTransaction(transactionManager) { _ -> languageStatsService.refreshLanguageStats(it.self.id) entityManager.flush() } diff --git a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BatchJobsTestData.kt b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BatchJobsTestData.kt index c3c35fa314..157e90c5de 100644 --- a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BatchJobsTestData.kt +++ b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BatchJobsTestData.kt @@ -49,4 +49,31 @@ class BatchJobsTestData : BaseTestData() { }.self } } + + fun addTagKeysData(keyCount: Int = 100): List { + this.projectBuilder.addKey { + name = "a-key" + }.build { + addTag("a-tag") + }.self + return (1..keyCount).map { + this.projectBuilder.addKey { + name = "key$it" + }.build { + addTag("tag1") + addTag("tag2") + addTag("tag3") + }.self + } + } + + fun addNamespaceData(): List { + this.projectBuilder.addKey("namespace", "key") + return (1..500).map { + this.projectBuilder.addKey("namespace1", "key$it").self + } + + (1..500).map { + this.projectBuilder.addKey(keyName = "key-without-namespace-$it").self + } + } } diff --git a/backend/data/src/main/kotlin/io/tolgee/dtos/request/key/EditKeyDto.kt b/backend/data/src/main/kotlin/io/tolgee/dtos/request/key/EditKeyDto.kt index 91f07d41cd..cf63f672dc 100644 --- a/backend/data/src/main/kotlin/io/tolgee/dtos/request/key/EditKeyDto.kt +++ b/backend/data/src/main/kotlin/io/tolgee/dtos/request/key/EditKeyDto.kt @@ -2,6 +2,7 @@ package io.tolgee.dtos.request.key import com.fasterxml.jackson.annotation.JsonSetter import io.swagger.v3.oas.annotations.media.Schema +import io.tolgee.constants.ValidationConstants import org.hibernate.validator.constraints.Length import javax.validation.constraints.NotBlank @@ -10,7 +11,7 @@ data class EditKeyDto( @field:Length(max = 2000) var name: String = "", - @field:Length(max = 100) + @field:Length(max = ValidationConstants.MAX_NAMESPACE_LENGTH) @Schema(description = "The namespace of the key. (When empty or null default namespace will be used)") var namespace: String? = null, ) { diff --git a/backend/data/src/main/kotlin/io/tolgee/dtos/request/key/TagKeyDto.kt b/backend/data/src/main/kotlin/io/tolgee/dtos/request/key/TagKeyDto.kt index 46ce65a33c..18d9d2b6bd 100644 --- a/backend/data/src/main/kotlin/io/tolgee/dtos/request/key/TagKeyDto.kt +++ b/backend/data/src/main/kotlin/io/tolgee/dtos/request/key/TagKeyDto.kt @@ -1,10 +1,11 @@ package io.tolgee.dtos.request.key +import io.tolgee.constants.ValidationConstants import javax.validation.constraints.NotBlank import javax.validation.constraints.Size data class TagKeyDto( @field:NotBlank - @field:Size(max = 100) + @field:Size(max = ValidationConstants.MAX_TAG_LENGTH) val name: String = "" ) diff --git a/backend/data/src/main/kotlin/io/tolgee/model/batch/SetKeysNamespaceParams.kt b/backend/data/src/main/kotlin/io/tolgee/model/batch/SetKeysNamespaceParams.kt new file mode 100644 index 0000000000..b5018e9679 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/model/batch/SetKeysNamespaceParams.kt @@ -0,0 +1,21 @@ +package io.tolgee.model.batch + +import com.vladmihalcea.hibernate.type.json.JsonBinaryType +import io.tolgee.model.StandardAuditModel +import org.hibernate.annotations.TypeDef +import org.hibernate.annotations.TypeDefs +import javax.persistence.Entity +import javax.persistence.OneToOne +import javax.validation.constraints.NotEmpty + +@Entity +@TypeDefs( + value = [TypeDef(name = "jsonb", typeClass = JsonBinaryType::class)] +) +class SetKeysNamespaceParams : StandardAuditModel() { + @OneToOne(optional = false) + lateinit var batchJob: BatchJob + + @NotEmpty + var namespace: String = "" +} diff --git a/backend/data/src/main/kotlin/io/tolgee/model/batch/TagKeysParams.kt b/backend/data/src/main/kotlin/io/tolgee/model/batch/TagKeysParams.kt new file mode 100644 index 0000000000..f5e03769f5 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/model/batch/TagKeysParams.kt @@ -0,0 +1,23 @@ +package io.tolgee.model.batch + +import com.vladmihalcea.hibernate.type.json.JsonBinaryType +import io.tolgee.model.StandardAuditModel +import org.hibernate.annotations.Type +import org.hibernate.annotations.TypeDef +import org.hibernate.annotations.TypeDefs +import javax.persistence.Entity +import javax.persistence.OneToOne +import javax.validation.constraints.NotEmpty + +@Entity +@TypeDefs( + value = [TypeDef(name = "jsonb", typeClass = JsonBinaryType::class)] +) +class TagKeysParams : StandardAuditModel() { + @OneToOne(optional = false) + lateinit var batchJob: BatchJob + + @Type(type = "jsonb") + @NotEmpty + var tags: List = listOf() +} diff --git a/backend/data/src/main/kotlin/io/tolgee/model/batch/UntagKeysParams.kt b/backend/data/src/main/kotlin/io/tolgee/model/batch/UntagKeysParams.kt new file mode 100644 index 0000000000..1e27bc881f --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/model/batch/UntagKeysParams.kt @@ -0,0 +1,23 @@ +package io.tolgee.model.batch + +import com.vladmihalcea.hibernate.type.json.JsonBinaryType +import io.tolgee.model.StandardAuditModel +import org.hibernate.annotations.Type +import org.hibernate.annotations.TypeDef +import org.hibernate.annotations.TypeDefs +import javax.persistence.Entity +import javax.persistence.OneToOne +import javax.validation.constraints.NotEmpty + +@Entity +@TypeDefs( + value = [TypeDef(name = "jsonb", typeClass = JsonBinaryType::class)] +) +class UntagKeysParams : StandardAuditModel() { + @OneToOne(optional = false) + lateinit var batchJob: BatchJob + + @Type(type = "jsonb") + @NotEmpty + var tags: List = listOf() +} diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/KeyRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/KeyRepository.kt index bf5c77eec6..f1e1ec43d3 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/KeyRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/KeyRepository.kt @@ -125,10 +125,29 @@ interface KeyRepository : JpaRepository { ) fun getWithTags(keys: Set): List + @Query( + """ + select k from Key k + left join fetch k.keyMeta km + left join fetch km.tags + where k.id in :keyIds + """ + ) + fun getWithTagsByIds(keyIds: Iterable): Set + @Query( """ select k.project.id from Key k where k.id in :keysIds """ ) fun getProjectIdsForKeyIds(keysIds: List): List + + @Query( + """ + select k from Key k + left join fetch k.namespace + where k.id in :keyIds + """ + ) + fun getKeysWithNamespaces(keyIds: List): List } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/key/KeyService.kt b/backend/data/src/main/kotlin/io/tolgee/service/key/KeyService.kt index 01e8fb9e5f..b9651f8670 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/key/KeyService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/key/KeyService.kt @@ -163,6 +163,16 @@ class KeyService( return key } + @Transactional + fun setNamespace(keyIds: List, namespace: String?) { + val keys = keyRepository.getKeysWithNamespaces(keyIds) + val projectId = keys.map { it.project.id }.distinct().singleOrNull() ?: return + val namespaceEntity = namespaceService.findOrCreate(namespace, projectId) + + keys.forEach { it.namespace = namespaceEntity } + keyRepository.saveAll(keys) + } + fun checkKeyNotExisting(projectId: Long, name: String, namespace: String?) { if (findOptional(projectId, name, namespace).isPresent) { throw ValidationException(Message.KEY_EXISTS) @@ -243,6 +253,7 @@ class KeyService( fun getPaged(projectId: Long, pageable: Pageable): Page = keyRepository.getAllByProjectId(projectId, pageable) fun getKeysWithTags(keys: Set): List = keyRepository.getWithTags(keys) + fun getKeysWithTagsById(keysIds: Iterable): Set = keyRepository.getWithTagsByIds(keysIds) fun find(id: List): List { return keyRepository.findAllByIdIn(id) diff --git a/backend/data/src/main/kotlin/io/tolgee/service/key/TagService.kt b/backend/data/src/main/kotlin/io/tolgee/service/key/TagService.kt index 4e8b197046..876a50f95d 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/key/TagService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/key/TagService.kt @@ -47,20 +47,40 @@ class TagService( return tag } + /** + * @param map key entity to list of tags + */ fun tagKeys(map: Map>) { if (map.isEmpty()) { return } - val keysWithTags = keyService.getKeysWithTags(map.keys).associateBy { it.id } - val projectId = getSingleProjectId(keysWithTags) + val keysWithTags = keyService.getKeysWithTags(map.keys) + tagKeys(keysWithTags, map.mapKeys { it.key.id }) + } + + /** + * @param map keyId entity to list of tags + */ + fun tagKeysById(map: Map>) { + if (map.isEmpty()) { + return + } + + val keysWithTags = keyService.getKeysWithTagsById(map.keys) + tagKeys(keysWithTags, map) + } + + private fun tagKeys(keysWithFetchedTags: Iterable, map: Map>) { + val keysByIdMap = keysWithFetchedTags.associateBy { it.id } + val projectId = getSingleProjectId(keysByIdMap) val existingTags = this.getFromProject(projectId, map.values.flatten().toSet()).associateBy { it.name }.toMutableMap() - map.forEach { (key, tagsToAdd) -> + map.forEach { (keyId, tagsToAdd) -> tagsToAdd.forEach { tagToAdd -> - val keyWithData = keysWithTags[key.id] ?: throw NotFoundException(Message.KEY_NOT_FOUND) + val keyWithData = keysByIdMap[keyId] ?: throw NotFoundException(Message.KEY_NOT_FOUND) val keyMeta = keyMetaService.getOrCreateForKey(keyWithData) val tag = existingTags[tagToAdd]?.let { if (!keyMeta.tags.contains(it)) { @@ -70,7 +90,7 @@ class TagService( it } ?: let { Tag().apply { - project = key.project + project = keysByIdMap[keyId]?.project ?: throw NotFoundException(Message.KEY_NOT_FOUND) keyMetas.add(keyMeta) name = tagToAdd keyMeta.tags.add(this) @@ -83,6 +103,26 @@ class TagService( } } + fun untagKeys(map: Map>) { + if (map.isEmpty()) { + return + } + + val keysWithTags = keyService.getKeysWithTagsById(map.keys) + untagKeys(keysWithTags, map) + } + + private fun untagKeys(keysWithFetchedTags: Iterable, map: Map>) { + val keysByIdMap = keysWithFetchedTags.associateBy { it.id } + + map.forEach { (keyId, tagsToRemove) -> + keysByIdMap[keyId]?.keyMeta?.let { keyMeta -> + keyMeta.tags.removeIf { tagsToRemove.contains(it.name) } + keyMetaService.save(keyMeta) + } + } + } + private fun getSingleProjectId(keysWithTags: Map): Long { val projectIds = keysWithTags.map { it.value.project.id }.toSet() diff --git a/backend/data/src/main/kotlin/io/tolgee/util/transactionUtil.kt b/backend/data/src/main/kotlin/io/tolgee/util/transactionUtil.kt index 2919ab7f54..f652eecb96 100644 --- a/backend/data/src/main/kotlin/io/tolgee/util/transactionUtil.kt +++ b/backend/data/src/main/kotlin/io/tolgee/util/transactionUtil.kt @@ -3,6 +3,7 @@ package io.tolgee.util import org.springframework.dao.CannotAcquireLockException import org.springframework.transaction.PlatformTransactionManager import org.springframework.transaction.TransactionDefinition +import org.springframework.transaction.TransactionStatus import org.springframework.transaction.support.TransactionTemplate import javax.persistence.OptimisticLockException @@ -10,20 +11,20 @@ fun executeInNewTransaction( transactionManager: PlatformTransactionManager, isolationLevel: Int = TransactionDefinition.ISOLATION_DEFAULT, propagationBehavior: Int = TransactionDefinition.PROPAGATION_REQUIRES_NEW, - fn: () -> T + fn: (ts: TransactionStatus) -> T ): T { val tt = TransactionTemplate(transactionManager) tt.propagationBehavior = propagationBehavior tt.isolationLevel = isolationLevel - return tt.execute { - fn() + return tt.execute { ts -> + fn(ts) } as T } fun executeInNewTransaction( transactionManager: PlatformTransactionManager, - fn: () -> T + fn: (ts: TransactionStatus) -> T ): T { return executeInNewTransaction( transactionManager = transactionManager, diff --git a/backend/data/src/main/resources/db/changelog/schema.xml b/backend/data/src/main/resources/db/changelog/schema.xml index d73c4f5262..8e08407088 100644 --- a/backend/data/src/main/resources/db/changelog/schema.xml +++ b/backend/data/src/main/resources/db/changelog/schema.xml @@ -2727,4 +2727,73 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/backend/testing/src/main/kotlin/io/tolgee/AbstractSpringTest.kt b/backend/testing/src/main/kotlin/io/tolgee/AbstractSpringTest.kt index bca4be4dcb..2853d356fb 100644 --- a/backend/testing/src/main/kotlin/io/tolgee/AbstractSpringTest.kt +++ b/backend/testing/src/main/kotlin/io/tolgee/AbstractSpringTest.kt @@ -48,6 +48,7 @@ import org.springframework.cache.CacheManager import org.springframework.context.ApplicationContext import org.springframework.transaction.PlatformTransactionManager import org.springframework.transaction.TransactionDefinition +import org.springframework.transaction.TransactionStatus import org.springframework.transaction.support.TransactionTemplate @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -251,7 +252,7 @@ abstract class AbstractSpringTest : AbstractTransactionalTest() { internalProperties.fakeMtProviders = false } - fun executeInNewTransaction(fn: () -> T): T { + fun executeInNewTransaction(fn: (ts: TransactionStatus) -> T): T { return io.tolgee.util.executeInNewTransaction( transactionManager = platformTransactionManager, fn = fn,