From e6d082425ad9a523838cb3c0566817f424b6e9cf Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Tue, 2 Jan 2024 17:13:29 +0100 Subject: [PATCH 1/3] chore: Prepare for CT -> Refactoring & Possible performance improvements --- .../TranslationCommentController.kt | 32 +- .../translation/TranslationsController.kt | 11 +- .../tolgee/component/KeyComplexEditHelper.kt | 3 +- .../io/tolgee/controllers/ExportController.kt | 4 +- .../TranslationsControllerCachingTest.kt | 7 +- .../TranslationStatsJobTest.kt | 3 +- .../service/AllTranslationsServiceTest.kt | 45 ++ .../tolgee/service/TranslationServiceTest.kt | 35 +- .../io/tolgee/activity/ActivityHolder.kt | 2 + .../tolgee/component/OutdatedFlagListener.kt | 27 -- .../demoProject/DemoProjectCreator.kt | 29 +- .../kotlin/io/tolgee/dtos/KeyAndLanguage.kt | 27 ++ .../io/tolgee/events/OnTranslationsSet.kt | 19 - .../tolgee/model/translation/Translation.kt | 16 +- .../tolgee/repository/LanguageRepository.kt | 5 + .../repository/TranslationRepository.kt | 27 ++ .../io/tolgee/service/LanguageService.kt | 7 + .../io/tolgee/service/key/KeyService.kt | 13 +- .../service/key/ResolvingKeyImporter.kt | 53 ++- .../tolgee/service/key/utils/KeysImporter.kt | 10 +- .../service/machineTranslation/MtService.kt | 4 +- .../translation/AllTranslationsService.kt | 64 +++ .../translation/AutoTranslationService.kt | 2 +- .../translation/TranslationCommentService.kt | 38 ++ .../translation/TranslationMemoryService.kt | 68 ++- .../service/translation/TranslationService.kt | 439 ++++++++---------- .../translation/TranslationUpdateHelper.kt | 48 ++ .../main/resources/db/changelog/schema.xml | 19 + .../kotlin/io/tolgee/AbstractSpringTest.kt | 4 + 29 files changed, 650 insertions(+), 411 deletions(-) create mode 100644 backend/app/src/test/kotlin/io/tolgee/service/AllTranslationsServiceTest.kt delete mode 100644 backend/data/src/main/kotlin/io/tolgee/component/OutdatedFlagListener.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/dtos/KeyAndLanguage.kt delete mode 100644 backend/data/src/main/kotlin/io/tolgee/events/OnTranslationsSet.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/service/translation/AllTranslationsService.kt create mode 100644 backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationUpdateHelper.kt diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationCommentController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationCommentController.kt index 7d593c9d52..e4efd0ac10 100644 --- a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationCommentController.kt +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationCommentController.kt @@ -20,7 +20,6 @@ import io.tolgee.hateoas.translations.comments.TranslationCommentModelAssembler import io.tolgee.hateoas.translations.comments.TranslationWithCommentModel import io.tolgee.model.enums.Scope import io.tolgee.model.enums.TranslationCommentState -import io.tolgee.model.enums.TranslationState import io.tolgee.model.translation.Translation import io.tolgee.model.translation.TranslationComment import io.tolgee.security.ProjectHolder @@ -174,27 +173,12 @@ class TranslationCommentController( @RequestBody @Valid dto: TranslationCommentWithLangKeyDto, ): ResponseEntity { - val translation = translationService.getOrCreate(dto.keyId, dto.languageId) - if (translation.key.project.id != projectHolder.project.id) { - throw BadRequestException(io.tolgee.constants.Message.KEY_NOT_FROM_PROJECT) - } - - if (translation.language.project.id != projectHolder.project.id) { - throw BadRequestException(io.tolgee.constants.Message.LANGUAGE_NOT_FROM_PROJECT) - } - - // Translation was just created - if (translation.id == 0L) { - translation.state = TranslationState.UNTRANSLATED - } - - translationService.save(translation) - - val comment = translationCommentService.create(dto, translation, authenticationFacade.authenticatedUserEntity) + val comment = + translationCommentService.create(dto, projectHolder.project.id, authenticationFacade.authenticatedUserEntity) return ResponseEntity( TranslationWithCommentModel( comment = translationCommentModelAssembler.toModel(comment), - translation = translationModelAssembler.toModel(translation), + translation = translationModelAssembler.toModel(comment.translation), ), HttpStatus.CREATED, ) @@ -211,9 +195,13 @@ class TranslationCommentController( @RequestBody @Valid dto: TranslationCommentDto, ): ResponseEntity { - val translation = translationService.find(translationId) ?: throw NotFoundException() - translation.checkFromProject() - val comment = translationCommentService.create(dto, translation, authenticationFacade.authenticatedUserEntity) + val comment = + translationCommentService.create( + dto, + translationId, + projectHolder.project.id, + authenticationFacade.authenticatedUserEntity, + ) return ResponseEntity(translationCommentModelAssembler.toModel(comment), HttpStatus.CREATED) } diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationsController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationsController.kt index 321b695304..0500556f09 100644 --- a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationsController.kt +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/translation/TranslationsController.kt @@ -47,6 +47,7 @@ import io.tolgee.service.key.KeyService import io.tolgee.service.key.ScreenshotService import io.tolgee.service.queryBuilders.CursorUtil import io.tolgee.service.security.SecurityService +import io.tolgee.service.translation.AllTranslationsService import io.tolgee.service.translation.TranslationService import jakarta.validation.Valid import org.springdoc.core.annotations.ParameterObject @@ -104,6 +105,7 @@ class TranslationsController( private val activityHolder: ActivityHolder, private val activityService: ActivityService, private val projectTranslationLastModifiedManager: ProjectTranslationLastModifiedManager, + private val allTranslationsService: AllTranslationsService, ) : IController { @GetMapping(value = ["/{languages}"]) @Operation( @@ -160,7 +162,7 @@ When null, resulting file will be a flat key-value object. .filterViewPermissionByTag(projectId = projectHolder.project.id, languageTags = languages) val response = - translationService.getTranslations( + allTranslationsService.getAllTranslations( languageTags = permittedTags, namespace = ns, projectId = projectHolder.project.id, @@ -187,7 +189,8 @@ When null, resulting file will be a flat key-value object. val key = keyService.get(projectHolder.project.id, dto.key, dto.namespace) securityService.checkLanguageTranslatePermissionsByTag(dto.translations.keys, projectHolder.project.id) - val modifiedTranslations = translationService.setForKey(key, dto.translations) + val modifiedTranslations = + translationService.set(key, dto.translations, projectHolder.project.id) val translations = dto.languagesToReturn @@ -210,14 +213,14 @@ When null, resulting file will be a flat key-value object. dto: SetTranslationsWithKeyDto, ): SetTranslationsResponseModel { val key = - keyService.find(projectHolder.projectEntity.id, dto.key, dto.namespace)?.also { + keyService.find(projectHolder.project.id, dto.key, dto.namespace)?.also { activityHolder.activity = ActivityType.SET_TRANSLATIONS } ?: let { checkKeyEditScope() activityHolder.activity = ActivityType.CREATE_KEY keyService.create(projectHolder.projectEntity, dto.key, dto.namespace) } - val translations = translationService.setForKey(key, dto.translations) + val translations = translationService.set(key, dto.translations, projectHolder.project.id) return getSetTranslationsResponse(key, translations) } diff --git a/backend/api/src/main/kotlin/io/tolgee/component/KeyComplexEditHelper.kt b/backend/api/src/main/kotlin/io/tolgee/component/KeyComplexEditHelper.kt index f2a89ea116..5b3e6ebeca 100644 --- a/backend/api/src/main/kotlin/io/tolgee/component/KeyComplexEditHelper.kt +++ b/backend/api/src/main/kotlin/io/tolgee/component/KeyComplexEditHelper.kt @@ -157,10 +157,11 @@ class KeyComplexEditHelper( }.toMap() val translations = - translationService.setForKey( + translationService.set( key, oldTranslations = oldTranslations, translations = modifiedTranslations, + projectId = projectHolder.project.id, ) translations.forEach { diff --git a/backend/api/src/main/kotlin/io/tolgee/controllers/ExportController.kt b/backend/api/src/main/kotlin/io/tolgee/controllers/ExportController.kt index 94911fd267..d0dadc1113 100644 --- a/backend/api/src/main/kotlin/io/tolgee/controllers/ExportController.kt +++ b/backend/api/src/main/kotlin/io/tolgee/controllers/ExportController.kt @@ -10,6 +10,7 @@ import io.tolgee.security.authentication.AllowApiAccess import io.tolgee.security.authentication.AuthenticationFacade import io.tolgee.security.authorization.RequiresProjectPermissions import io.tolgee.service.security.PermissionService +import io.tolgee.service.translation.AllTranslationsService import io.tolgee.service.translation.TranslationService import io.tolgee.util.StreamingResponseBodyProvider import org.apache.tomcat.util.http.fileupload.IOUtils @@ -40,6 +41,7 @@ class ExportController( private val projectHolder: ProjectHolder, private val authenticationFacade: AuthenticationFacade, private val streamingResponseBodyProvider: StreamingResponseBodyProvider, + private val allTranslationsService: AllTranslationsService, ) : IController { @GetMapping(value = ["/jsonZip"], produces = ["application/zip"]) @Operation(summary = "Exports data as ZIP of jsons", deprecated = true) @@ -65,7 +67,7 @@ class ExportController( streamingResponseBodyProvider.createStreamingResponseBody { out: OutputStream -> val zipOutputStream = ZipOutputStream(out) val translations = - translationService.getTranslations( + allTranslationsService.getAllTranslations( allLanguages.map { it.tag }.toSet(), null, projectHolder.project.id, diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/translations/v2TranslationsController/TranslationsControllerCachingTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/translations/v2TranslationsController/TranslationsControllerCachingTest.kt index a45c94f8d2..591c70f340 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/translations/v2TranslationsController/TranslationsControllerCachingTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/translations/v2TranslationsController/TranslationsControllerCachingTest.kt @@ -68,7 +68,12 @@ class TranslationsControllerCachingTest : ProjectAuthControllerTest("/v2/project val newNow = Date(Date().time + 50000) setForcedDate(newNow) - translationService.setTranslation(testData.aKey, testData.englishLanguage, "This was changed!") + translationService.set( + testData.aKey, + testData.englishLanguage, + "This was changed!", + testData.project.id, + ) val newLastModified = performWithIsModifiedSince(lastModified).andIsOk.lastModified() assertEqualsDate(newLastModified, newNow) diff --git a/backend/app/src/test/kotlin/io/tolgee/jobs/migration/translationStats/TranslationStatsJobTest.kt b/backend/app/src/test/kotlin/io/tolgee/jobs/migration/translationStats/TranslationStatsJobTest.kt index bd471211e3..9c9a569bac 100644 --- a/backend/app/src/test/kotlin/io/tolgee/jobs/migration/translationStats/TranslationStatsJobTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/jobs/migration/translationStats/TranslationStatsJobTest.kt @@ -45,7 +45,8 @@ class TranslationStatsJobTest : AbstractSpringTest() { val instance = translationsStatsUpdateJobRunner.run() executeInNewTransaction { - val newTranslationId = translationService.setForKey(testData.aKey, mapOf("en" to "Hellooooo!"))["en"]!!.id + val newTranslationId = + translationService.set(testData.aKey, mapOf("en" to "Hellooooo!"), testData.project.id)["en"]!!.id entityManager .createNativeQuery( "update translation set word_count = null, character_count = null where id = $newTranslationId", diff --git a/backend/app/src/test/kotlin/io/tolgee/service/AllTranslationsServiceTest.kt b/backend/app/src/test/kotlin/io/tolgee/service/AllTranslationsServiceTest.kt new file mode 100644 index 0000000000..22e81d6662 --- /dev/null +++ b/backend/app/src/test/kotlin/io/tolgee/service/AllTranslationsServiceTest.kt @@ -0,0 +1,45 @@ +package io.tolgee.service + +import io.tolgee.AbstractSpringTest +import io.tolgee.dtos.request.key.CreateKeyDto +import io.tolgee.testing.assertions.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.transaction.annotation.Transactional +import java.util.* + +@SpringBootTest +@Transactional +class AllTranslationsServiceTest : AbstractSpringTest() { + @Transactional + @Test + fun getTranslations() { + val id = dbPopulator.populate("App").project.id + val data = + allTranslationsService.getAllTranslations( + languageTags = HashSet(Arrays.asList("en", "de")), + namespace = null, + projectId = id, + structureDelimiter = '.', + ) + assertThat(data["en"]).isInstanceOf(MutableMap::class.java) + } + + @Transactional + @Test + fun `returns correct map when collision`() { + val project = dbPopulator.populate("App").project + keyService.create(project, CreateKeyDto("folder.folder", null, mapOf("en" to "Ha"))) + keyService.create(project, CreateKeyDto("folder.folder.translation", null, mapOf("en" to "Ha"))) + + val viewData = + allTranslationsService.getAllTranslations( + languageTags = HashSet(Arrays.asList("en", "de")), + namespace = null, + projectId = project.id, + structureDelimiter = '.', + ) + @Suppress("UNCHECKED_CAST") + assertThat(viewData["en"] as Map).containsKey("folder.folder.translation") + } +} diff --git a/backend/app/src/test/kotlin/io/tolgee/service/TranslationServiceTest.kt b/backend/app/src/test/kotlin/io/tolgee/service/TranslationServiceTest.kt index 03bb64f652..a42087cfe2 100644 --- a/backend/app/src/test/kotlin/io/tolgee/service/TranslationServiceTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/service/TranslationServiceTest.kt @@ -2,7 +2,6 @@ package io.tolgee.service import io.tolgee.AbstractSpringTest import io.tolgee.development.testDataBuilder.data.TranslationsTestData -import io.tolgee.dtos.request.key.CreateKeyDto import io.tolgee.testing.assertions.Assertions.assertThat import org.junit.jupiter.api.Test import org.springframework.boot.test.context.SpringBootTest @@ -12,38 +11,6 @@ import java.util.* @SpringBootTest @Transactional class TranslationServiceTest : AbstractSpringTest() { - @Transactional - @Test - fun getTranslations() { - val id = dbPopulator.populate("App").project.id - val data = - translationService.getTranslations( - languageTags = HashSet(Arrays.asList("en", "de")), - namespace = null, - projectId = id, - structureDelimiter = '.', - ) - assertThat(data["en"]).isInstanceOf(MutableMap::class.java) - } - - @Transactional - @Test - fun `returns correct map when collision`() { - val project = dbPopulator.populate("App").project - keyService.create(project, CreateKeyDto("folder.folder", null, mapOf("en" to "Ha"))) - keyService.create(project, CreateKeyDto("folder.folder.translation", null, mapOf("en" to "Ha"))) - - val viewData = - translationService.getTranslations( - languageTags = HashSet(Arrays.asList("en", "de")), - namespace = null, - projectId = project.id, - structureDelimiter = '.', - ) - @Suppress("UNCHECKED_CAST") - assertThat(viewData["en"] as Map).containsKey("folder.folder.translation") - } - @Test fun `adds stats on translation save and update`() { val testData = @@ -73,7 +40,7 @@ class TranslationServiceTest : AbstractSpringTest() { fun `clears auto translation when set empty`() { val testData = TranslationsTestData() testDataService.saveTestData(testData.root) - translationService.setForKey(testData.aKey, mapOf("de" to "")) + translationService.set(testData.aKey, mapOf("de" to ""), testData.project.id) val translation = translationService.get(testData.aKeyGermanTranslation.id) assertThat(translation.auto).isFalse assertThat(translation.mtProvider).isNull() diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt index 499a7ae8f2..892236538c 100644 --- a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt +++ b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt @@ -53,6 +53,8 @@ open class ActivityHolder(val applicationContext: ApplicationContext) { activityRevision.describingRelations.associateBy { it.entityId to it.entityClass }.toMutableMap() } + open val modifiedEntityInstances = mutableListOf() + fun getDescribingRelationFromCache( entityId: Long, entityClass: String, diff --git a/backend/data/src/main/kotlin/io/tolgee/component/OutdatedFlagListener.kt b/backend/data/src/main/kotlin/io/tolgee/component/OutdatedFlagListener.kt deleted file mode 100644 index f362b2b6e7..0000000000 --- a/backend/data/src/main/kotlin/io/tolgee/component/OutdatedFlagListener.kt +++ /dev/null @@ -1,27 +0,0 @@ -package io.tolgee.component - -import io.tolgee.configuration.TransactionScopeConfig -import io.tolgee.events.OnTranslationsSet -import io.tolgee.service.translation.TranslationService -import org.springframework.context.annotation.Scope -import org.springframework.context.event.EventListener -import org.springframework.core.annotation.Order -import org.springframework.stereotype.Component - -@Component -@Scope(TransactionScopeConfig.SCOPE_TRANSACTION) -class OutdatedFlagListener( - private val translationService: TranslationService, -) { - @EventListener - @Order(1) - fun onEvent(event: OnTranslationsSet) { - val baseLanguage = event.key.project.baseLanguage ?: return - val oldBaseValue = event.oldValues[baseLanguage.tag] - val newBaseValue = event.translations.find { it.language.id == baseLanguage.id }?.text - if (oldBaseValue != newBaseValue) { - val excluded = event.translations.map { it.id }.toSet() - translationService.setOutdated(event.key, excluded) - } - } -} diff --git a/backend/data/src/main/kotlin/io/tolgee/component/demoProject/DemoProjectCreator.kt b/backend/data/src/main/kotlin/io/tolgee/component/demoProject/DemoProjectCreator.kt index fdef9c0745..4157268b5c 100644 --- a/backend/data/src/main/kotlin/io/tolgee/component/demoProject/DemoProjectCreator.kt +++ b/backend/data/src/main/kotlin/io/tolgee/component/demoProject/DemoProjectCreator.kt @@ -2,6 +2,7 @@ package io.tolgee.component.demoProject import io.tolgee.activity.ActivityHolder import io.tolgee.activity.data.ActivityType +import io.tolgee.dtos.KeyAndLanguage import io.tolgee.dtos.RelatedKeyDto import io.tolgee.dtos.request.KeyInScreenshotPositionDto import io.tolgee.dtos.request.ScreenshotInfoDto @@ -11,7 +12,6 @@ import io.tolgee.model.Project import io.tolgee.model.Screenshot import io.tolgee.model.enums.TranslationState import io.tolgee.model.key.Key -import io.tolgee.model.translation.Translation import io.tolgee.service.LanguageService import io.tolgee.service.bigMeta.BigMetaService import io.tolgee.service.key.KeyService @@ -54,11 +54,17 @@ class DemoProjectCreator( } private val translations by lazy { - DemoProjectData.translations.flatMap { (languageTag, translations) -> - translations.map { (key, text) -> - setTranslation(key, languageTag, text) - } - }.associateBy { it.language.tag to it.key.name } + val toSave = + DemoProjectData.translations.flatMap { (languageTag, translations) -> + translations.mapNotNull { (key, text) -> + KeyAndLanguage(keys[key] ?: return@mapNotNull null, languages[languageTag] ?: return@mapNotNull null) to text + } + }.toMap() + + translationService.set( + toSave, + project.id, + ).entries.associate { (it.key.language.tag to it.key.key.name) to it.value } } private fun addBigMeta() { @@ -79,17 +85,6 @@ class DemoProjectCreator( } } - private fun setTranslation( - keyName: String, - languageTag: String, - translation: String, - ): Translation { - val language = languages[languageTag]!! - return translationService.setTranslation(getOrCreateKey(keyName), language, translation).also { - it.state = TranslationState.REVIEWED - } - } - private fun addScreenshots() { val screenshot = saveScreenshot() diff --git a/backend/data/src/main/kotlin/io/tolgee/dtos/KeyAndLanguage.kt b/backend/data/src/main/kotlin/io/tolgee/dtos/KeyAndLanguage.kt new file mode 100644 index 0000000000..cef58ea64d --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/dtos/KeyAndLanguage.kt @@ -0,0 +1,27 @@ +package io.tolgee.dtos + +import io.tolgee.model.Language +import io.tolgee.model.key.Key + +data class KeyAndLanguage( + val key: Key, + val language: Language, +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as KeyAndLanguage + + if (key.id != other.key.id) return false + if (language.id != other.language.id) return false + + return true + } + + override fun hashCode(): Int { + var result = key.id.hashCode() + result = 31 * result + language.id.hashCode() + return result + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/events/OnTranslationsSet.kt b/backend/data/src/main/kotlin/io/tolgee/events/OnTranslationsSet.kt deleted file mode 100644 index 2829219a37..0000000000 --- a/backend/data/src/main/kotlin/io/tolgee/events/OnTranslationsSet.kt +++ /dev/null @@ -1,19 +0,0 @@ -package io.tolgee.events - -import io.tolgee.model.key.Key -import io.tolgee.model.translation.Translation -import org.springframework.context.ApplicationEvent - -/** - * This event is dispatched when base translation for some key provided. - * It is dispatched even when new key is created with base translation provided - */ -class OnTranslationsSet( - source: Any, - val key: Key, - /** - * Map of old values languageTag -> String - */ - val oldValues: Map, - val translations: List, -) : ApplicationEvent(source) diff --git a/backend/data/src/main/kotlin/io/tolgee/model/translation/Translation.kt b/backend/data/src/main/kotlin/io/tolgee/model/translation/Translation.kt index 08e8801b92..105a3e87fd 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/translation/Translation.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/translation/Translation.kt @@ -9,6 +9,7 @@ import io.tolgee.constants.MtServiceType import io.tolgee.exceptions.BadRequestException import io.tolgee.model.Language import io.tolgee.model.StandardAuditModel +import io.tolgee.model.UserAccount import io.tolgee.model.enums.TranslationState import io.tolgee.model.key.Key import io.tolgee.util.TranslationStatsUtil @@ -37,7 +38,7 @@ import org.hibernate.annotations.ColumnDefault ) @ActivityLoggedEntity @EntityListeners(Translation.Companion.UpdateStatsListener::class, Translation.Companion.StateListener::class) -@ActivityEntityDescribingPaths(paths = ["key", "language"]) +@ActivityEntityDescribingPaths(paths = ["key", "language", "author"]) class Translation( @Column(columnDefinition = "text") @ActivityLoggedProp @@ -81,6 +82,13 @@ class Translation( @field:ColumnDefault("false") var outdated: Boolean = false + @ActivityLoggedProp + @ManyToOne(fetch = FetchType.LAZY, optional = true) + var author: UserAccount? = null + + @ColumnDefault("true") + var active: Boolean = true + constructor(text: String? = null, key: Key, language: Language) : this(text) { this.key = key this.language = language @@ -154,9 +162,9 @@ class Translation( if (!translation.text.isNullOrEmpty() && translation.state == TranslationState.UNTRANSLATED) { translation.state = TranslationState.TRANSLATED } - if (translation.text.isNullOrEmpty() && - translation.state != TranslationState.UNTRANSLATED && translation.state != TranslationState.DISABLED - ) { + + if (translation.text.isNullOrEmpty() && translation.state != TranslationState.DISABLED) { + translation.text = null translation.state = TranslationState.UNTRANSLATED } } diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/LanguageRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/LanguageRepository.kt index 7e84bb395b..9591e1267c 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/LanguageRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/LanguageRepository.kt @@ -80,4 +80,9 @@ interface LanguageRepository : JpaRepository { projectId: Long, languageIds: List, ): List + + fun findByIdAndProjectId( + id: Long, + projectId: Long, + ): Language? } diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt index 6ce9db72e0..ecd6805dd0 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt @@ -209,4 +209,31 @@ interface TranslationRepository : JpaRepository { key: Key, languageTags: Collection, ): List + + @Query( + """ + from Translation t + where t.key.id in :keyIds + and t.language.id not in :excludeLanguageIds + and t.language.id <> :baseLanguageId + and t.text is not null + and t.text <> '' + """, + ) + fun getTranslationsToSetOutDated( + keyIds: Collection, + excludeTranslationIds: Collection, + baseLanguageId: Long, + ): List + + @Query( + """ + from Translation t + where t.key.project.id = :projectId + """, + ) + fun find( + translationId: Long, + projectId: Long, + ): Translation? } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/LanguageService.kt b/backend/data/src/main/kotlin/io/tolgee/service/LanguageService.kt index f734d6c1cc..38cfec157b 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/LanguageService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/LanguageService.kt @@ -253,4 +253,11 @@ class LanguageService( fun getViewsOfProjects(projectIds: List): List { return languageRepository.getViewsOfProjects(projectIds) } + + fun get( + id: Long, + projectId: Long, + ): Language { + return languageRepository.findByIdAndProjectId(id, projectId) ?: throw NotFoundException(Message.LANGUAGE_NOT_FOUND) + } } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/key/KeyService.kt b/backend/data/src/main/kotlin/io/tolgee/service/key/KeyService.kt index b797c8c7ca..d5851665bd 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/key/KeyService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/key/KeyService.kt @@ -113,7 +113,7 @@ class KeyService( if (it.isEmpty()) { return@let null } - translationService.setForKey(key, it) + translationService.set(key, it, project.id) } dto.states?.map { @@ -382,7 +382,7 @@ class KeyService( keyId: Long, languageIds: List, ): List { - val key = keyRepository.findByProjectIdAndId(projectId, keyId) ?: throw NotFoundException() + val key = get(keyId, projectId) enableRestOfLanguages(projectId, languageIds, key) return disableLanguages(projectId, languageIds, key) } @@ -408,11 +408,18 @@ class KeyService( ): List { val languages = languageRepository.findAllByProjectIdAndIdInOrderById(projectId, languageIds) languages.map { language -> - val translation = translationService.getOrCreate(key, language) + val translation = translationService.getOrCreate(key, language, projectId) translation.clear() translation.state = TranslationState.DISABLED translationService.save(translation) } return languages } + + fun get( + keyId: Long, + projectId: Long, + ): Key { + return keyRepository.findByProjectIdAndId(projectId, keyId) ?: throw NotFoundException(Message.KEY_NOT_FOUND) + } } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/key/ResolvingKeyImporter.kt b/backend/data/src/main/kotlin/io/tolgee/service/key/ResolvingKeyImporter.kt index 92f06a9a3a..aee4d23464 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/key/ResolvingKeyImporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/key/ResolvingKeyImporter.kt @@ -1,6 +1,7 @@ package io.tolgee.service.key import io.tolgee.constants.Message +import io.tolgee.dtos.KeyAndLanguage import io.tolgee.dtos.KeyImportResolvableResult import io.tolgee.dtos.request.ScreenshotInfoDto import io.tolgee.dtos.request.translation.importKeysResolvable.ImportKeysResolvableItemDto @@ -60,37 +61,41 @@ class ResolvingKeyImporter( private fun tryImport(): List { checkLanguagePermissions(keysToImport) - return keysToImport.map keys@{ keyToImport -> - val key = getOrCreateKey(keyToImport) + val translationsToSave = mutableMapOf() + val existingTranslations = mutableMapOf() - keyToImport.mapLanguageAsKey().forEach translations@{ (language, resolvable) -> - language ?: throw NotFoundException(Message.LANGUAGE_NOT_FOUND) - val existingTranslation = getExistingTranslation(key, language) + val result = + keysToImport.map keys@{ keyToImport -> + val key = getOrCreateKey(keyToImport) - val isEmpty = existingTranslation !== null && existingTranslation.text.isNullOrEmpty() + keyToImport.mapLanguageAsKey().forEach translations@{ (language, resolvable) -> + language ?: throw NotFoundException(Message.LANGUAGE_NOT_FOUND) + val existingTranslation = getExistingTranslation(key, language) - val isNew = existingTranslation == null + val isEmpty = existingTranslation !== null && existingTranslation.text.isNullOrEmpty() - val translationExists = !isEmpty && !isNew + val isNew = existingTranslation == null - if (validate(translationExists, resolvable, key, language)) return@translations + val translationExists = !isEmpty && !isNew - if (isEmpty || (!isNew && resolvable.resolution == ImportTranslationResolution.OVERRIDE)) { - translationService.setTranslation(existingTranslation!!, resolvable.text) - return@translations - } + if (validate(translationExists, resolvable, key, language)) return@translations - if (isNew) { - val translation = - Translation(resolvable.text).apply { - this.key = key - this.language = language - } - translationService.save(translation) + if (isEmpty || (!isNew && resolvable.resolution == ImportTranslationResolution.OVERRIDE) || isNew) { + translationsToSave[KeyAndLanguage(key, language)] = resolvable.text + existingTranslations[KeyAndLanguage(key, language)] = existingTranslation + return@translations + } } + key } - key - } + + translationService.set( + translationsToSave, + existingTranslations, + projectEntity.id, + ) + + return result } private fun importScreenshots(): Map { @@ -100,7 +105,7 @@ class ResolvingKeyImporter( } val images = imageUploadService.find(uploadedImagesIds) - checkImageUploadermissions(images) + checkImageUploadPermissions(images) val createdScreenshots = images.associate { @@ -151,7 +156,7 @@ class ResolvingKeyImporter( }.toMap() } - private fun checkImageUploadermissions(images: List) { + private fun checkImageUploadPermissions(images: List) { if (images.isNotEmpty()) { securityService.checkScreenshotsUploadPermission(projectEntity.id) } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/key/utils/KeysImporter.kt b/backend/data/src/main/kotlin/io/tolgee/service/key/utils/KeysImporter.kt index bcab87d94e..c2954a022f 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/key/utils/KeysImporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/key/utils/KeysImporter.kt @@ -1,5 +1,6 @@ package io.tolgee.service.key.utils +import io.tolgee.dtos.KeyAndLanguage import io.tolgee.dtos.request.translation.ImportKeysItemDto import io.tolgee.model.Project import io.tolgee.model.key.Key @@ -38,6 +39,7 @@ class KeysImporter( securityService.checkLanguageTranslatePermissionByTag(project.id, languageTags) val toTag = mutableMapOf>() + val translationsToSave = mutableMapOf() keys.forEach { keyDto -> val safeNamespace = getSafeNamespace(keyDto.namespace) @@ -56,11 +58,12 @@ class KeysImporter( this.namespace = namespaces[safeNamespace] } keyService.save(key) + keyDto.translations.entries.forEach { (languageTag, value) -> - languages[languageTag]?.let { language -> - translationService.setTranslation(key, language, value) - } + val language = languages[languageTag] ?: return@forEach + translationsToSave[KeyAndLanguage(key, language)] = value } + existing[safeNamespace to keyDto.name] = key if (!keyDto.tags.isNullOrEmpty()) { @@ -71,6 +74,7 @@ class KeysImporter( } } + translationService.set(translationsToSave, project.id) tagService.tagKeys(toTag) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/machineTranslation/MtService.kt b/backend/data/src/main/kotlin/io/tolgee/service/machineTranslation/MtService.kt index f7cf5961d8..e392b1181a 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/machineTranslation/MtService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/machineTranslation/MtService.kt @@ -17,6 +17,7 @@ import io.tolgee.model.key.Key import io.tolgee.service.bigMeta.BigMetaService import io.tolgee.service.key.KeyService import io.tolgee.service.project.ProjectService +import io.tolgee.service.translation.TranslationMemoryService import io.tolgee.service.translation.TranslationService import org.springframework.context.ApplicationEventPublisher import org.springframework.data.domain.PageRequest @@ -33,6 +34,7 @@ class MtService( private val tolgeeProperties: TolgeeProperties, private val bigMetaService: BigMetaService, private val keyService: KeyService, + private val translationMemoryService: TranslationMemoryService, ) { @Transactional fun getMachineTranslations( @@ -274,7 +276,7 @@ class MtService( text: String, keyId: Long?, ): List { - return translationService.getTranslationMemorySuggestions( + return translationMemoryService.getTranslationMemorySuggestions( sourceTranslationText = text, key = null, sourceLanguage = sourceLanguage, diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/AllTranslationsService.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/AllTranslationsService.kt new file mode 100644 index 0000000000..064d5cc3fc --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/AllTranslationsService.kt @@ -0,0 +1,64 @@ +package io.tolgee.service.translation + +import io.tolgee.helpers.TextHelper +import io.tolgee.model.views.SimpleTranslationView +import io.tolgee.repository.TranslationRepository +import org.springframework.stereotype.Component +import org.springframework.transaction.annotation.Transactional +import java.util.HashMap +import java.util.LinkedHashMap + +@Component +class AllTranslationsService( + private val translationRepository: TranslationRepository, +) { + @Transactional + @Suppress("UNCHECKED_CAST") + fun getAllTranslations( + languageTags: Set, + namespace: String?, + projectId: Long, + structureDelimiter: Char?, + ): Map { + val safeNamespace = if (namespace == "") null else namespace + val allByLanguages = translationRepository.getTranslations(languageTags, safeNamespace, projectId) + val langTranslations: HashMap = LinkedHashMap() + for (translation in allByLanguages) { + val map = + langTranslations + .computeIfAbsent( + translation.languageTag, + ) { LinkedHashMap() } as MutableMap + addToMap(translation, map, structureDelimiter) + } + return langTranslations + } + + @Suppress("UNCHECKED_CAST") + private fun addToMap( + translation: SimpleTranslationView, + map: MutableMap, + delimiter: Char?, + ) { + var currentMap = map + val path = TextHelper.splitOnNonEscapedDelimiter(translation.key, delimiter).toMutableList() + val name = path.removeLast() + for (folderName in path) { + val childMap = currentMap.computeIfAbsent(folderName) { LinkedHashMap() } + if (childMap is Map<*, *>) { + currentMap = childMap as MutableMap + continue + } + // there is already string value, so we cannot replace it by map, + // we have to save the key directly without nesting + map[translation.key] = translation.text + return + } + // The result already contains the key, so we have to add it to root without nesting + if (currentMap.containsKey(name)) { + map[translation.key] = translation.text + return + } + currentMap[name] = translation.text + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/AutoTranslationService.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/AutoTranslationService.kt index 7156567ffa..e2d1614352 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/translation/AutoTranslationService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/AutoTranslationService.kt @@ -165,7 +165,7 @@ class AutoTranslationService( val translations = adjustedConfigs.map { if (it.override) { - return@map it to translationService.getOrCreate(key, it.language) + return@map it to translationService.getOrCreate(key, it.language, key.project.id) } it to getUntranslatedTranslations(key, listOf(it.language)).firstOrNull() diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationCommentService.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationCommentService.kt index 2e51fe7abd..5932529fc5 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationCommentService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationCommentService.kt @@ -2,11 +2,14 @@ package io.tolgee.service.translation import io.tolgee.dtos.request.translation.comment.ITranslationCommentDto import io.tolgee.dtos.request.translation.comment.TranslationCommentDto +import io.tolgee.dtos.request.translation.comment.TranslationCommentWithLangKeyDto import io.tolgee.exceptions.NotFoundException import io.tolgee.model.UserAccount import io.tolgee.model.enums.TranslationCommentState +import io.tolgee.model.enums.TranslationState import io.tolgee.model.translation.Translation import io.tolgee.model.translation.TranslationComment +import io.tolgee.repository.TranslationRepository import io.tolgee.repository.translation.TranslationCommentRepository import io.tolgee.security.authentication.AuthenticationFacade import jakarta.persistence.EntityManager @@ -20,9 +23,44 @@ class TranslationCommentService( private val translationCommentRepository: TranslationCommentRepository, private val authenticationFacade: AuthenticationFacade, private val entityManager: EntityManager, + private val translationService: TranslationService, + private val translationRepository: TranslationRepository, ) { @Transactional fun create( + dto: TranslationCommentWithLangKeyDto, + projectId: Long, + author: UserAccount, + ): TranslationComment { + val translation = translationService.getOrCreate(dto.keyId, dto.languageId, projectId) + + if (translation.id == 0L) { + translation.state = TranslationState.UNTRANSLATED + } + + return create(dto, translation, author) + } + + @Transactional + fun create( + dto: ITranslationCommentDto, + translationId: Long, + projectId: Long, + author: UserAccount, + ): TranslationComment { + val translation = translationService.get(translationId, projectId) + + return TranslationComment( + text = dto.text, + state = dto.state, + translation = translation, + ).let { + it.author = author + create(it) + } + } + + private fun create( dto: ITranslationCommentDto, translation: Translation, author: UserAccount, diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationMemoryService.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationMemoryService.kt index 099259c26c..5d4dfc2dfa 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationMemoryService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationMemoryService.kt @@ -1,8 +1,13 @@ package io.tolgee.service.translation +import io.tolgee.constants.Message +import io.tolgee.exceptions.NotFoundException import io.tolgee.model.Language import io.tolgee.model.key.Key +import io.tolgee.model.translation.Translation import io.tolgee.model.views.TranslationMemoryItemView +import io.tolgee.repository.TranslationRepository +import io.tolgee.service.project.ProjectService import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable import org.springframework.stereotype.Service @@ -10,12 +15,25 @@ import org.springframework.stereotype.Service @Service class TranslationMemoryService( private val translationsService: TranslationService, + private val projectService: ProjectService, + private val translationRepository: TranslationRepository, ) { fun getAutoTranslatedValue( key: Key, targetLanguage: Language, ): TranslationMemoryItemView? { - return translationsService.getTranslationMemoryValue(key, targetLanguage) + val baseLanguage = + projectService.getOrCreateBaseLanguage(targetLanguage.project.id) + ?: throw NotFoundException(Message.BASE_LANGUAGE_NOT_FOUND) + + val baseTranslationText = findBaseTranslation(key)?.text ?: return null + + return translationRepository.getTranslationMemoryValue( + baseTranslationText, + key, + baseLanguage, + targetLanguage, + ).firstOrNull() } fun suggest( @@ -23,7 +41,11 @@ class TranslationMemoryService( targetLanguage: Language, pageable: Pageable, ): Page { - return translationsService.getTranslationMemorySuggestions(key, targetLanguage, pageable) + val baseTranslation = findBaseTranslation(key) ?: return Page.empty() + + val baseTranslationText = baseTranslation.text ?: return Page.empty(pageable) + + return getTranslationMemorySuggestions(baseTranslationText, key, targetLanguage, pageable) } fun suggest( @@ -31,11 +53,51 @@ class TranslationMemoryService( targetLanguage: Language, pageable: Pageable, ): Page { - return translationsService.getTranslationMemorySuggestions( + return getTranslationMemorySuggestions( baseTranslationText, null, targetLanguage, pageable, ) } + + fun getTranslationMemorySuggestions( + baseTranslationText: String, + key: Key?, + targetLanguage: Language, + pageable: Pageable, + ): Page { + val baseLanguage = + projectService.getOrCreateBaseLanguage(targetLanguage.project.id) + ?: throw NotFoundException(Message.BASE_LANGUAGE_NOT_FOUND) + + return getTranslationMemorySuggestions(baseTranslationText, key, baseLanguage, targetLanguage, pageable) + } + + fun getTranslationMemorySuggestions( + sourceTranslationText: String, + key: Key?, + sourceLanguage: Language, + targetLanguage: Language, + pageable: Pageable, + ): Page { + if ((sourceTranslationText.length) < 3) { + return Page.empty(pageable) + } + + return translationRepository.getTranslateMemorySuggestions( + baseTranslationText = sourceTranslationText, + key = key, + baseLanguage = sourceLanguage, + targetLanguage = targetLanguage, + pageable = pageable, + ) + } + + private fun findBaseTranslation(key: Key): Translation? { + projectService.getOrCreateBaseLanguage(key.project.id)?.let { + return translationsService.find(key, it).orElse(null) + } + return null + } } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt index 79d6f36c5a..aa6145a5da 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt @@ -2,12 +2,11 @@ package io.tolgee.service.translation import io.tolgee.configuration.tolgee.TolgeeProperties import io.tolgee.constants.Message +import io.tolgee.dtos.KeyAndLanguage import io.tolgee.dtos.request.translation.GetTranslationsParams import io.tolgee.dtos.request.translation.TranslationFilters -import io.tolgee.events.OnTranslationsSet import io.tolgee.exceptions.BadRequestException import io.tolgee.exceptions.NotFoundException -import io.tolgee.helpers.TextHelper import io.tolgee.model.Language import io.tolgee.model.Project import io.tolgee.model.enums.TranslationState @@ -15,8 +14,6 @@ import io.tolgee.model.key.Key import io.tolgee.model.translation.Translation import io.tolgee.model.translation.Translation_ import io.tolgee.model.views.KeyWithTranslationsView -import io.tolgee.model.views.SimpleTranslationView -import io.tolgee.model.views.TranslationMemoryItemView import io.tolgee.repository.TranslationRepository import io.tolgee.service.LanguageService import io.tolgee.service.dataImport.ImportService @@ -43,6 +40,7 @@ class TranslationService( private val translationViewDataProvider: TranslationViewDataProvider, private val entityManager: EntityManager, private val translationCommentService: TranslationCommentService, + private val translationUpdateHelper: TranslationUpdateHelper, ) { @set:Autowired @set:Lazy @@ -56,28 +54,6 @@ class TranslationService( @set:Lazy lateinit var projectService: ProjectService - @Transactional - @Suppress("UNCHECKED_CAST") - fun getTranslations( - languageTags: Set, - namespace: String?, - projectId: Long, - structureDelimiter: Char?, - ): Map { - val safeNamespace = if (namespace == "") null else namespace - val allByLanguages = translationRepository.getTranslations(languageTags, safeNamespace, projectId) - val langTranslations: HashMap = LinkedHashMap() - for (translation in allByLanguages) { - val map = - langTranslations - .computeIfAbsent( - translation.languageTag, - ) { LinkedHashMap() } as MutableMap - addToMap(translation, map, structureDelimiter) - } - return langTranslations - } - fun getAllByLanguageId(languageId: Long): List { return translationRepository.getAllByLanguageId(languageId) } @@ -97,25 +73,46 @@ class TranslationService( fun getOrCreate( key: Key, language: Language, + projectId: Long, ): Translation { - return find(key, language).orElseGet { - Translation(language = language, key = key) + return getOrCreate(listOf(KeyAndLanguage(key, language)), projectId).values.single() + } + + fun getOrCreate( + items: Collection, + projectId: Long, + ): Map { + val existing = translationUpdateHelper.getExistingTranslations(items, projectId) + + return items.associateWith { item -> + existing[item] ?: create(item.key, item.language) } } + private fun create( + key: Key, + language: Language, + ): Translation { + return Translation(language = language, key = key) + } + fun getOrCreate( keyId: Long, languageId: Long, + projectId: Long, ): Translation { - return translationRepository.findOneByKeyIdAndLanguageId(keyId, languageId) - ?: let { - val key = keyService.findOptional(keyId).orElseThrow { NotFoundException() } - val language = languageService.get(languageId) - Translation().apply { - this.key = key - this.language = language - } - } + val key = keyService.get(keyId, projectId) + val language = languageService.get(languageId, projectId) + + return getOrCreate( + listOf( + KeyAndLanguage( + key, + language, + ), + ), + projectId, + ).values.single() } fun find( @@ -150,46 +147,6 @@ class TranslationService( return translationViewDataProvider.getSelectAllKeys(projectId, languages, params) } - fun setTranslation( - key: Key, - languageTag: String?, - text: String?, - ): Translation? { - val language = - languageService.findByTag(languageTag!!, key.project) - .orElseThrow { NotFoundException(Message.LANGUAGE_NOT_FOUND) } - return setTranslation(key, language, text) - } - - fun setTranslation( - key: Key, - language: Language, - text: String?, - ): Translation { - val translation = getOrCreate(key, language) - setTranslation(translation, text) - key.translations.add(translation) - return translation - } - - fun setTranslation( - translation: Translation, - text: String?, - ): Translation { - if (translation.text !== text) { - translation.resetFlags() - } - translation.text = text - if (translation.state == TranslationState.UNTRANSLATED && !translation.text.isNullOrEmpty()) { - translation.state = TranslationState.TRANSLATED - } - if (text.isNullOrEmpty()) { - translation.state = TranslationState.UNTRANSLATED - translation.text = null - } - return save(translation) - } - fun save(translation: Translation): Translation { val translationTextLength = translation.text?.length ?: 0 if (translationTextLength > tolgeeProperties.maxTranslationTextLength) { @@ -199,95 +156,155 @@ class TranslationService( } @Transactional - fun setForKey( + fun set( + key: Key, + language: Language, + text: String, + projectId: Long, + ): Map { + return set(mapOf(KeyAndLanguage(key, language) to text), projectId) + } + + @Transactional + fun set( key: Key, translations: Map, + projectId: Long, ): Map { - val languages = languageService.findByTags(translations.keys, key.project.id) + val languages = languageService.findByTags(translations.keys, projectId) val oldTranslations = - getKeyTranslations(languages, key.project, key).associate { - languageByIdFromLanguages( + getKeyTranslations(languages, entityManager.getReference(Project::class.java, projectId), key).associate { + languages.byId( it.language.id, - languages, ) to it.text } - return setForKey( - key, - translations.map { languageByTagFromLanguages(it.key, languages) to it.value } - .toMap(), - oldTranslations, + return set( + key = key, + translations = + translations.map { languages.byTag(it.key) to it.value } + .toMap(), + oldTranslations = oldTranslations, + projectId = projectId, ).mapKeys { it.key.tag } } - fun findForKeyByLanguages( - key: Key, - languageTags: Collection, - ): List { - return translationRepository.findForKeyByLanguages(key, languageTags) - } - - private fun languageByTagFromLanguages( - tag: String, - languages: Collection, - ) = languages.find { it.tag == tag } ?: throw NotFoundException(Message.LANGUAGE_NOT_FOUND) - - private fun languageByIdFromLanguages( - id: Long, - languages: Collection, - ) = languages.find { it.id == id } ?: throw NotFoundException(Message.LANGUAGE_NOT_FOUND) - @Transactional - fun setForKey( + fun set( key: Key, translations: Map, oldTranslations: Map, + projectId: Long, ): Map { - val result = - translations.entries.associate { (language, value) -> - language to setTranslation(key, language, value) - }.filterValues { it != null }.mapValues { it.value } - - applicationEventPublisher.publishEvent( - OnTranslationsSet( - source = this, - key = key, - oldValues = oldTranslations.map { it.key.tag to it.value }.toMap(), - translations = result.values.toList(), - ), - ) + val newTranslations = + translations.mapKeys { KeyAndLanguage(key, it.key) } + val oldTranslationMap = oldTranslations.mapKeys { KeyAndLanguage(key, it.key) } + return setWithOld(newTranslations, oldTranslationMap, projectId).mapKeys { it.key.language } + } + + @Transactional + fun set( + translations: Map, + projectId: Long, + ): Map { + val existingTranslations = + translationUpdateHelper + .getExistingTranslations(translations.keys, projectId) - return result + return set(translations, existingTranslations, projectId) } - @Suppress("UNCHECKED_CAST") - private fun addToMap( - translation: SimpleTranslationView, - map: MutableMap, - delimiter: Char?, - ) { - var currentMap = map - val path = TextHelper.splitOnNonEscapedDelimiter(translation.key, delimiter).toMutableList() - val name = path.removeLast() - for (folderName in path) { - val childMap = currentMap.computeIfAbsent(folderName) { LinkedHashMap() } - if (childMap is Map<*, *>) { - currentMap = childMap as MutableMap - continue + fun set( + translations: Map, + existingTranslations: Map, + projectId: Long, + ): Map { + val oldTranslations = + existingTranslations + .mapKeys { it.key }.mapValues { it.value?.text } + + val translationEntities = getPreparedTranslations(existingTranslations, translations.keys) + + return set(translations, oldTranslations, projectId, translationEntities) + } + + private fun setWithOld( + newTranslations: Map, + oldTranslations: Map, + projectId: Long, + ): Map { + val translationEntities = getOrCreate(newTranslations.keys, projectId) + return set(newTranslations, oldTranslations, projectId, translationEntities) + } + + /** + * @param translationEntities map of new or existing translations prepared for text update + */ + private fun set( + newTranslations: Map, + oldTranslations: Map, + projectId: Long, + translationEntities: Map, + ): Map { + translationEntities.forEach { (key, language), translation -> + val newText = newTranslations[KeyAndLanguage(key, language)] + if (translation.text != newText) { + translation.resetFlags() } - // there is already string value, so we cannot replace it by map, - // we have to save the key directly without nesting - map[translation.key] = translation.text - return + translation.text = newText + + save(translation) + + key.translations.add(translation) } - // The result already contains the key, so we have to add it to root without nesting - if (currentMap.containsKey(name)) { - map[translation.key] = translation.text - return + handleOutdatedState(translationEntities, oldTranslations, projectId) + return translationEntities + } + + fun getPreparedTranslations( + existingTranslations: Map, + requiredKeysAndLanguages: Set, + ): Map { + return requiredKeysAndLanguages.associate { + KeyAndLanguage(it.key, it.language) to ( + existingTranslations[it] + ?: create(it.key, it.language) + ) } - currentMap[name] = translation.text } + fun handleOutdatedState( + newTranslations: Map, + oldTranslations: Map, + projectId: Long, + ) { + val baseLanguage = projectService.get(projectId).baseLanguage ?: return + + // we don't want to set outdated flag for just modified translations + val excluded = newTranslations.values.map { it.id } + val keys = + newTranslations.map { it.key.key }.toSet().filter { key -> + val oldBaseValue = oldTranslations[KeyAndLanguage(key, baseLanguage)] + val newBaseValue = newTranslations[KeyAndLanguage(key, baseLanguage)]?.text + oldBaseValue != newBaseValue + } + + this.setOutdated(keys.map { it.id }, excluded, baseLanguage.id) + } + + fun findForKeyByLanguages( + key: Key, + languageTags: Collection, + ): List { + return translationRepository.findForKeyByLanguages(key, languageTags) + } + + private fun Collection.byTag(tag: String) = + find { it.tag == tag } ?: throw NotFoundException(Message.LANGUAGE_NOT_FOUND) + + private fun Collection.byId(id: Long) = + find { it.id == id } ?: throw NotFoundException(Message.LANGUAGE_NOT_FOUND) + fun deleteByIdIn(ids: Collection) { importService.onExistingTranslationsRemoved(ids) translationCommentService.deleteByTranslationIdIn(ids) @@ -325,76 +342,6 @@ class TranslationService( } } - fun findBaseTranslation(key: Key): Translation? { - projectService.getOrCreateBaseLanguage(key.project.id)?.let { - return find(key, it).orElse(null) - } - return null - } - - fun getTranslationMemoryValue( - key: Key, - targetLanguage: Language, - ): TranslationMemoryItemView? { - val baseLanguage = - projectService.getOrCreateBaseLanguage(targetLanguage.project.id) - ?: throw NotFoundException(Message.BASE_LANGUAGE_NOT_FOUND) - - val baseTranslationText = findBaseTranslation(key)?.text ?: return null - - return translationRepository.getTranslationMemoryValue( - baseTranslationText, - key, - baseLanguage, - targetLanguage, - ).firstOrNull() - } - - fun getTranslationMemorySuggestions( - key: Key, - targetLanguage: Language, - pageable: Pageable, - ): Page { - val baseTranslation = findBaseTranslation(key) ?: return Page.empty() - - val baseTranslationText = baseTranslation.text ?: return Page.empty(pageable) - - return getTranslationMemorySuggestions(baseTranslationText, key, targetLanguage, pageable) - } - - fun getTranslationMemorySuggestions( - baseTranslationText: String, - key: Key?, - targetLanguage: Language, - pageable: Pageable, - ): Page { - val baseLanguage = - projectService.getOrCreateBaseLanguage(targetLanguage.project.id) - ?: throw NotFoundException(Message.BASE_LANGUAGE_NOT_FOUND) - - return getTranslationMemorySuggestions(baseTranslationText, key, baseLanguage, targetLanguage, pageable) - } - - fun getTranslationMemorySuggestions( - sourceTranslationText: String, - key: Key?, - sourceLanguage: Language, - targetLanguage: Language, - pageable: Pageable, - ): Page { - if ((sourceTranslationText.length) < 3) { - return Page.empty(pageable) - } - - return translationRepository.getTranslateMemorySuggestions( - baseTranslationText = sourceTranslationText, - key = key, - baseLanguage = sourceLanguage, - targetLanguage = targetLanguage, - pageable = pageable, - ) - } - @Transactional fun dismissAutoTranslated(translation: Translation) { translation.auto = false @@ -412,20 +359,17 @@ class TranslationService( } fun setOutdated( - key: Key, - excludeTranslationIds: Set = emptySet(), + keyIds: Collection, + excludeTranslationIds: Collection, + baseLanguageId: Long, ) { - val baseLanguage = key.project.baseLanguage - key.translations.forEach { - val isBase = it.language.id == baseLanguage?.id - val isEmpty = it.text.isNullOrEmpty() - val isExcluded = excludeTranslationIds.contains(it.id) - - if (!isBase && !isEmpty && !isExcluded) { - it.outdated = true - it.state = TranslationState.TRANSLATED - save(it) - } + val translations = + translationRepository + .getTranslationsToSetOutDated(keyIds, excludeTranslationIds, baseLanguageId) + + translations.forEach { + it.outdated = true + it.state = TranslationState.TRANSLATED } } @@ -471,7 +415,7 @@ class TranslationService( languageIds: List, state: TranslationState, ) { - val translations = getTargetTranslations(keyIds, languageIds) + val translations = getTargetTranslationsForBatch(keyIds, languageIds) translations.filter { it.state != TranslationState.DISABLED }.forEach { it.state = state } saveAll(translations) } @@ -485,7 +429,7 @@ class TranslationService( keyIds: List, languageIds: List, ) { - val translations = getTargetTranslations(keyIds, languageIds) + val translations = getTargetTranslationsForBatch(keyIds, languageIds) translations.forEach { it.clear() } @@ -497,9 +441,9 @@ class TranslationService( sourceLanguageId: Long, targetLanguageIds: List, ) { - val sourceTranslations = getTargetTranslations(keyIds, listOf(sourceLanguageId)).associateBy { it.key.id } + val sourceTranslations = getTargetTranslationsForBatch(keyIds, listOf(sourceLanguageId)).associateBy { it.key.id } val targetTranslations = - getTargetTranslations(keyIds, targetLanguageIds).onEach { + getTargetTranslationsForBatch(keyIds, targetLanguageIds).onEach { it.text = sourceTranslations[it.key.id]?.text if (!it.text.isNullOrEmpty()) { it.state = TranslationState.TRANSLATED @@ -511,25 +455,22 @@ class TranslationService( saveAll(targetTranslations) } - private fun getTargetTranslations( + private fun getTargetTranslationsForBatch( keyIds: List, targetLanguageIds: List, ): List { - val existing = getTranslations(keyIds, targetLanguageIds) - val existingMap = - existing.groupBy { it.key.id } - .map { entry -> - entry.key to - entry.value.associateBy { translation -> translation.language.id } - }.toMap() - return keyIds.flatMap { keyId -> - targetLanguageIds.map { languageId -> - existingMap[keyId]?.get(languageId) ?: getOrCreate( - entityManager.getReference(Key::class.java, keyId), - entityManager.getReference(Language::class.java, languageId), - ) - }.filter { it.state !== TranslationState.DISABLED } - } + val keyAndLanguages = + keyIds.flatMap { keyId -> + targetLanguageIds.map { languageId -> + KeyAndLanguage( + entityManager.getReference(Key::class.java, keyId), + entityManager.getReference(Language::class.java, languageId), + ) + } + } + + return getOrCreate(keyAndLanguages, keyAndLanguages.first().key.project.id).values + .filter { it.state !== TranslationState.DISABLED } } fun deleteAllByProject(projectId: Long) { @@ -541,4 +482,12 @@ class TranslationService( "language_id IN (SELECT id FROM language WHERE project_id = :projectId)", ).setParameter("projectId", projectId).executeUpdate() } + + fun get( + translationId: Long, + projectId: Long, + ): Translation { + return translationRepository.find(translationId, projectId) + ?: throw NotFoundException(Message.TRANSLATION_NOT_FOUND) + } } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationUpdateHelper.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationUpdateHelper.kt new file mode 100644 index 0000000000..d76c957c00 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationUpdateHelper.kt @@ -0,0 +1,48 @@ +package io.tolgee.service.translation + +import io.tolgee.dtos.KeyAndLanguage +import io.tolgee.model.Language_ +import io.tolgee.model.Project_ +import io.tolgee.model.key.Key_ +import io.tolgee.model.translation.Translation +import io.tolgee.model.translation.Translation_ +import jakarta.persistence.EntityManager +import jakarta.persistence.criteria.CriteriaBuilder +import jakarta.persistence.criteria.CriteriaQuery +import org.springframework.stereotype.Component + +@Component +class TranslationUpdateHelper( + private val entityManager: EntityManager, +) { + private fun getQueryToFindExistingTranslations( + items: Collection, + projectId: Long, + ): CriteriaQuery { + val cb: CriteriaBuilder = entityManager.criteriaBuilder + val query = cb.createQuery(Translation::class.java) + val root = query.from(Translation::class.java) + val key = root.join(Translation_.key) + val predicates = + items.map { item -> + cb.and( + cb.equal(key.get(Key_.id), item.key), + cb.equal(root.get(Translation_.language).get(Language_.id), item.language), + ) + } + val keyPredicates = cb.or(*predicates.toTypedArray()) + query.where(cb.and(keyPredicates, cb.equal(key.get(Key_.project).get(Project_.id), projectId))) + query.select(root) + return query + } + + fun getExistingTranslations( + items: Collection, + projectId: Long, + ): Map { + return entityManager + .createQuery(getQueryToFindExistingTranslations(items, projectId)).resultList.associateBy { + KeyAndLanguage(it.key, it.language) + } + } +} diff --git a/backend/data/src/main/resources/db/changelog/schema.xml b/backend/data/src/main/resources/db/changelog/schema.xml index 502f19c6d9..63aa164d03 100644 --- a/backend/data/src/main/resources/db/changelog/schema.xml +++ b/backend/data/src/main/resources/db/changelog/schema.xml @@ -3028,4 +3028,23 @@ create unique index import_author_project_unique on import(author_id, project_id) where deleted_at is null + + + + + + + + + + + + + + + + + + + diff --git a/backend/testing/src/main/kotlin/io/tolgee/AbstractSpringTest.kt b/backend/testing/src/main/kotlin/io/tolgee/AbstractSpringTest.kt index 257aca184b..8be98cc0b1 100644 --- a/backend/testing/src/main/kotlin/io/tolgee/AbstractSpringTest.kt +++ b/backend/testing/src/main/kotlin/io/tolgee/AbstractSpringTest.kt @@ -46,6 +46,7 @@ import io.tolgee.service.security.PatService import io.tolgee.service.security.PermissionService import io.tolgee.service.security.UserAccountService import io.tolgee.service.security.UserPreferencesService +import io.tolgee.service.translation.AllTranslationsService import io.tolgee.service.translation.TranslationCommentService import io.tolgee.service.translation.TranslationService import io.tolgee.testing.AbstractTransactionalTest @@ -88,6 +89,9 @@ abstract class AbstractSpringTest : AbstractTransactionalTest() { @Autowired protected lateinit var translationService: TranslationService + @Autowired + protected lateinit var allTranslationsService: AllTranslationsService + @Autowired protected lateinit var keyService: KeyService From e98b0045a1ae028b8ea38ad0a48637cf30579692 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Tue, 2 Jan 2024 17:30:00 +0100 Subject: [PATCH 2/3] chore: Prepare for CT -> Refactoring & Possible performance improvements --- .../main/resources/application-dbschema.yaml | 2 +- .../repository/TranslationRepository.kt | 4 +- .../service/translation/TranslationService.kt | 3 +- .../main/resources/db/changelog/schema.xml | 66 ++++++++++++++----- build.gradle | 2 +- 5 files changed, 53 insertions(+), 24 deletions(-) diff --git a/backend/app/src/main/resources/application-dbschema.yaml b/backend/app/src/main/resources/application-dbschema.yaml index 9af8283137..3b84590a9f 100644 --- a/backend/app/src/main/resources/application-dbschema.yaml +++ b/backend/app/src/main/resources/application-dbschema.yaml @@ -3,7 +3,7 @@ spring: drop-first: true change-log: classpath:db/changelog/schema.xml datasource: - url: jdbc:postgresql://localhost:55432/postgres + url: jdbc:postgresql://localhost:55438/postgres username: postgres password: postgres tolgee: diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt index ecd6805dd0..ea187a9096 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt @@ -214,7 +214,7 @@ interface TranslationRepository : JpaRepository { """ from Translation t where t.key.id in :keyIds - and t.language.id not in :excludeLanguageIds + and t.id not in :excludeTranslationIds and t.language.id <> :baseLanguageId and t.text is not null and t.text <> '' @@ -229,7 +229,7 @@ interface TranslationRepository : JpaRepository { @Query( """ from Translation t - where t.key.project.id = :projectId + where t.key.project.id = :projectId and t.id = :translationId """, ) fun find( diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt index aa6145a5da..5eeac06509 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt @@ -22,7 +22,6 @@ import io.tolgee.service.project.ProjectService import io.tolgee.service.queryBuilders.translationViewBuilder.TranslationViewDataProvider import jakarta.persistence.EntityManager import org.springframework.beans.factory.annotation.Autowired -import org.springframework.context.ApplicationEventPublisher import org.springframework.context.annotation.Lazy import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable @@ -36,9 +35,9 @@ class TranslationService( private val translationRepository: TranslationRepository, private val importService: ImportService, private val tolgeeProperties: TolgeeProperties, - private val applicationEventPublisher: ApplicationEventPublisher, private val translationViewDataProvider: TranslationViewDataProvider, private val entityManager: EntityManager, + @Lazy private val translationCommentService: TranslationCommentService, private val translationUpdateHelper: TranslationUpdateHelper, ) { diff --git a/backend/data/src/main/resources/db/changelog/schema.xml b/backend/data/src/main/resources/db/changelog/schema.xml index 63aa164d03..92cea4e3a5 100644 --- a/backend/data/src/main/resources/db/changelog/schema.xml +++ b/backend/data/src/main/resources/db/changelog/schema.xml @@ -3028,23 +3028,53 @@ create unique index import_author_project_unique on import(author_id, project_id) where deleted_at is null - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + CREATE UNIQUE INDEX translations_active ON translation (key_id, language_id) where active is true; + + + CREATE UNIQUE INDEX translations_reviewed ON translation (key_id, language_id) where state = 1; + + + update translation t set + author_id = + (select ar.author_id + from activity_revision ar + join activity_modified_entity ame + on ar.id = ame.activity_revision_id and ame.entity_id = t.id and + ame.entity_class = 'Translation' + order by ar.id desc + limit 1) + + + + + + + + + + + + + + + + diff --git a/build.gradle b/build.gradle index 83bfa4f0d0..be6c3ecf6e 100644 --- a/build.gradle +++ b/build.gradle @@ -90,7 +90,7 @@ project(':server-app').afterEvaluate { task startDbChangelogContainer { doLast { exec { - commandLine "docker", "run", "-e", "POSTGRES_PASSWORD=postgres", "-d", "-p55438:5432", "--name", dbSchemaContainerName, "postgres:13" + commandLine "docker", "run", "--rm", "-e" ,"POSTGRES_PASSWORD=postgres", "-d", "-p55438:5432", "--name", dbSchemaContainerName, "postgres:13" } Thread.sleep(5000) } From 3cc5f08026eb55220d443eefa9cc3287adc63ba9 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Tue, 2 Jan 2024 18:14:12 +0100 Subject: [PATCH 3/3] chore: Prepare for CT -> Refactoring & Possible performance improvements --- .../translation/TranslationUpdateHelper.kt | 5 ++--- .../src/main/resources/db/changelog/schema.xml | 15 --------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationUpdateHelper.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationUpdateHelper.kt index d76c957c00..140cfa123a 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationUpdateHelper.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationUpdateHelper.kt @@ -1,7 +1,6 @@ package io.tolgee.service.translation import io.tolgee.dtos.KeyAndLanguage -import io.tolgee.model.Language_ import io.tolgee.model.Project_ import io.tolgee.model.key.Key_ import io.tolgee.model.translation.Translation @@ -26,8 +25,8 @@ class TranslationUpdateHelper( val predicates = items.map { item -> cb.and( - cb.equal(key.get(Key_.id), item.key), - cb.equal(root.get(Translation_.language).get(Language_.id), item.language), + cb.equal(key, item.key), + cb.equal(root.get(Translation_.language), item.language), ) } val keyPredicates = cb.or(*predicates.toTypedArray()) diff --git a/backend/data/src/main/resources/db/changelog/schema.xml b/backend/data/src/main/resources/db/changelog/schema.xml index 92cea4e3a5..c68a187994 100644 --- a/backend/data/src/main/resources/db/changelog/schema.xml +++ b/backend/data/src/main/resources/db/changelog/schema.xml @@ -3062,19 +3062,4 @@ limit 1) - - - - - - - - - - - - - - -