feat: Batch operation, stores activity

tolgee · Jun 23, 2023 · 61cb27a · 61cb27a
1 parent 126b353
commit 61cb27a
Show file tree

Hide file tree

Showing 15 changed files with 88 additions and 48 deletions.
diff --git a/backend/api/src/main/kotlin/io/tolgee/component/ProjectTranslationLastModifiedManager.kt b/backend/api/src/main/kotlin/io/tolgee/component/ProjectTranslationLastModifiedManager.kt
@@ -26,7 +26,7 @@ class ProjectTranslationLastModifiedManager(
 
   @EventListener
   fun onActivity(event: OnProjectActivityEvent) {
-    event.activityHolder.activityRevision?.projectId?.let { projectId ->
+    event.activityRevision.projectId?.let { projectId ->
       getCache()?.put(projectId, currentDateProvider.date.time)
     }
   }

diff --git a/...nd/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchOperationControllerTest.kt b/...nd/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchOperationControllerTest.kt
@@ -4,6 +4,7 @@ import io.tolgee.ProjectAuthControllerTest
 import io.tolgee.development.testDataBuilder.data.BatchOperationsTestData
 import io.tolgee.fixtures.andIsOk
 import io.tolgee.fixtures.waitForNotThrowing
+import io.tolgee.model.batch.BatchJobChunkExecution
 import io.tolgee.model.translation.Translation
 import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod
 import io.tolgee.testing.assert
@@ -79,7 +80,7 @@ class BatchOperationControllerTest : ProjectAuthControllerTest("/v2/projects/")
   @Test
   @ProjectJWTAuthTestMethod
   fun `it deletes keys`() {
-    val keyCount = 10000
+    val keyCount = 1000
     val keys = testData.addTranslationOperationData(keyCount)
     saveAndPrepare()
 
@@ -93,7 +94,19 @@ class BatchOperationControllerTest : ProjectAuthControllerTest("/v2/projects/")
     ).andIsOk
 
     waitForNotThrowing(pollTime = 1000, timeout = 300000) {
-      keyService.getAll(testData.projectBuilder.self.id).assert.isEmpty()
+      val all = keyService.getAll(testData.projectBuilder.self.id)
+      all.assert.isEmpty()
+    }
+
+    waitForNotThrowing(pollTime = 1000, timeout = 300000) {
+      executeInNewTransaction {
+        val data = entityManager
+          .createQuery("""from BatchJobChunkExecution""", BatchJobChunkExecution::class.java)
+          .resultList
+
+        data.assert.hasSize(1)
+        data[0].activityRevision.assert.isNotNull
+      }
     }
   }
 }
diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityHolder.kt
@@ -1,44 +1,29 @@
 package io.tolgee.activity
 
 import io.tolgee.activity.data.ActivityType
-import io.tolgee.events.OnProjectActivityEvent
 import io.tolgee.model.EntityWithId
 import io.tolgee.model.activity.ActivityModifiedEntity
 import io.tolgee.model.activity.ActivityRevision
-import org.slf4j.LoggerFactory
-import org.springframework.context.ApplicationContext
-import javax.annotation.PreDestroy
 import kotlin.reflect.KClass
 
-open class ActivityHolder(
-  private val applicationContext: ApplicationContext
-) {
+open class ActivityHolder {
   open var activity: ActivityType? = null
 
   open var meta: MutableMap<String, Any?> = mutableMapOf()
 
   open var activityRevision: ActivityRevision? = null
 
-  private val logger = LoggerFactory.getLogger(this::class.java)
-
   open var modifiedCollections: MutableMap<Pair<EntityWithId, String>, List<Any?>?> = mutableMapOf()
 
   open var transactionRollbackOnly = false
 
   open var organizationId: Long? = null
 
-  @PreDestroy
-  open fun preDestroy() {
-    if (!transactionRollbackOnly) {
-      applicationContext.publishEvent(OnProjectActivityEvent(this))
-    }
-  }
-
   /**
    * This field stores all modified entities, it's stored before the transaction is committed
    */
   open var modifiedEntities:
-    MutableMap<
-      KClass<out EntityWithId>, MutableMap<Long, ActivityModifiedEntity>
-      > = mutableMapOf()
+    ModifiedEntitiesType = mutableMapOf()
 }
+
+typealias ModifiedEntitiesType = MutableMap<KClass<out EntityWithId>, MutableMap<Long, ActivityModifiedEntity>>
diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt b/backend/data/src/main/kotlin/io/tolgee/activity/ActivityService.kt
@@ -4,6 +4,7 @@ import io.tolgee.activity.data.ActivityType
 import io.tolgee.activity.projectActivityView.ProjectActivityViewDataProvider
 import io.tolgee.dtos.query_results.TranslationHistoryView
 import io.tolgee.events.OnProjectActivityStoredEvent
+import io.tolgee.model.activity.ActivityRevision
 import io.tolgee.model.views.activity.ProjectActivityView
 import io.tolgee.repository.activity.ActivityModifiedEntityRepository
 import org.springframework.context.ApplicationContext
@@ -20,9 +21,8 @@ class ActivityService(
   private val activityModifiedEntityRepository: ActivityModifiedEntityRepository
 ) {
   @Transactional
-  fun storeActivityData(activityHolder: ActivityHolder) {
-    val activityRevision = activityHolder.activityRevision ?: return
-    activityRevision.modifiedEntities = activityHolder.modifiedEntities.values.flatMap { it.values }.toMutableList()
+  fun storeActivityData(activityRevision: ActivityRevision, modifiedEntities: ModifiedEntitiesType) {
+    activityRevision.modifiedEntities = modifiedEntities.values.flatMap { it.values }.toMutableList()
 
     entityManager.persist(activityRevision)
     activityRevision.describingRelations.forEach {
@@ -31,6 +31,7 @@ class ActivityService(
     activityRevision.modifiedEntities.forEach { activityModifiedEntity ->
       entityManager.persist(activityModifiedEntity)
     }
+    entityManager.flush()
     applicationContext.publishEvent(OnProjectActivityStoredEvent(this, activityRevision))
   }
 

diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityInterceptor.kt b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityInterceptor.kt
@@ -1,6 +1,8 @@
 package io.tolgee.activity.iterceptor
 
+import io.tolgee.activity.ActivityHolder
 import io.tolgee.activity.data.RevisionType
+import io.tolgee.events.OnProjectActivityEvent
 import io.tolgee.util.Logging
 import org.hibernate.EmptyInterceptor
 import org.hibernate.Transaction
@@ -20,6 +22,20 @@ class ActivityInterceptor : EmptyInterceptor(), Logging {
     interceptedEventsManager.onAfterTransactionCompleted(tx)
   }
 
+  override fun beforeTransactionCompletion(tx: Transaction) {
+    if (tx.isActive) {
+      val holder = this.applicationContext.getBean(ActivityHolder::class.java)
+      val activityRevision = holder.activityRevision ?: return
+      applicationContext.publishEvent(
+        OnProjectActivityEvent(
+          activityRevision,
+          holder.modifiedEntities,
+          holder.organizationId
+        )
+      )
+    }
+  }
+
   override fun onSave(
     entity: Any?,
     id: Serializable?,

diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt
@@ -2,6 +2,7 @@ package io.tolgee.batch
 
 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import io.sentry.Sentry
+import io.tolgee.activity.ActivityHolder
 import io.tolgee.component.CurrentDateProvider
 import io.tolgee.component.UsingRedisProvider
 import io.tolgee.model.batch.BatchJobChunkExecution
@@ -37,7 +38,8 @@ class BatchJobActionService(
   private val transactionManager: PlatformTransactionManager,
   private val applicationContext: ApplicationContext,
   private val usingRedisProvider: UsingRedisProvider,
-  private val progressManager: ProgressManager
+  private val progressManager: ProgressManager,
+  private val activityHolder: ActivityHolder
 ) : Logging {
   companion object {
     const val MIN_TIME_BETWEEN_OPERATIONS = 100
@@ -95,6 +97,9 @@ class BatchJobActionService(
                     logger.debug("Job ${lockedExecution.batchJob.id}: 🟡 Processing chunk ${lockedExecution.id}")
                     val util = ChunkProcessingUtil(lockedExecution, applicationContext)
                     util.processChunk()
+                    val activityRevision = activityHolder.activityRevision
+                      ?: throw IllegalStateException("Activity revision not set")
+                    activityRevision.batchJobChunkExecution = lockedExecution
                     progressManager.handleProgress(lockedExecution)
                     entityManager.persist(lockedExecution)
                     if (lockedExecution.retry) {
@@ -174,10 +179,10 @@ class BatchJobActionService(
   fun populateQueue() {
     val data = entityManager.createQuery(
       """
-          from BatchJobChunkExecution bjce
-          join fetch bjce.batchJob
-          where bjce.status = :status
-          order by bjce.createdAt asc, bjce.executeAfter asc, bjce.id asc
+                        from BatchJobChunkExecution bjce
+                        join fetch bjce.batchJob
+                        where bjce.status = : status
+                        order by bjce.createdAt asc, bjce.executeAfter asc, bjce.id asc
       """.trimIndent(),
       BatchJobChunkExecution::class.java
     )
@@ -209,8 +214,8 @@ class BatchJobActionService(
   fun getItemIfCanAcquireLock(id: Long): BatchJobChunkExecution? {
     return entityManager.createQuery(
       """
-         from BatchJobChunkExecution bjce
-         where bjce.id = :id
+                        from BatchJobChunkExecution bjce
+                        where bjce . id = :id
       """.trimIndent(),
       BatchJobChunkExecution::class.java
     )

diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/ProgressManager.kt b/backend/data/src/main/kotlin/io/tolgee/batch/ProgressManager.kt
@@ -43,18 +43,15 @@ class ProgressManager(
   }
 
   fun handleJobStatus(job: BatchJob, completedChunks: Long, progress: Long) {
-    val completedChunksNotNull = completedChunks
+    logger.debug("Job ${job.id} completed chunks: $completedChunks of ${job.totalChunks}")
 
-    logger.debug("Job ${job.id} completed chunks: $completedChunksNotNull of ${job.totalChunks}")
-    if (job.totalChunks.toLong() != completedChunksNotNull) {
+    if (job.totalChunks.toLong() != completedChunks) {
       return
     }
 
-    val progressNotNull = progress
+    logger.debug("Job ${job.id} progress: $progress of ${job.totalItems}")
 
-    logger.debug("Job ${job.id} progress: $progressNotNull of ${job.totalItems}")
-
-    if (job.totalItems.toLong() != progressNotNull) {
+    if (job.totalItems.toLong() != progress) {
       job.status = BatchJobStatus.FAILED
       entityManager.persist(job)
       eventPublisher.publishEvent(OnBatchOperationFailed(job))

diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/processors/DeleteKeysChunkProcessor.kt b/backend/data/src/main/kotlin/io/tolgee/batch/processors/DeleteKeysChunkProcessor.kt
@@ -16,12 +16,11 @@ class DeleteKeysChunkProcessor(
   override fun process(job: BatchJob, chunk: List<Long>, onProgress: ((Int) -> Unit)) {
     val subChunked = chunk.chunked(100)
     var progress: Int = 0
-    subChunked.forEachIndexed { index, subChunk ->
+    subChunked.forEach { subChunk ->
       keyService.deleteMultiple(subChunk)
       entityManager.flush()
       progress += subChunk.size
       onProgress.invoke(progress)
-      return
     }
   }
 

diff --git a/backend/data/src/main/kotlin/io/tolgee/component/eventListeners/LanguageStatsListener.kt b/backend/data/src/main/kotlin/io/tolgee/component/eventListeners/LanguageStatsListener.kt
@@ -19,9 +19,9 @@ class LanguageStatsListener(
   @Async
   fun onActivity(event: OnProjectActivityEvent) {
     runSentryCatching {
-      val projectId = event.activityHolder.activityRevision?.projectId ?: return
+      val projectId = event.activityRevision.projectId ?: return
 
-      val modifiedEntityClasses = event.activityHolder.modifiedEntities.keys.toSet()
+      val modifiedEntityClasses = event.modifiedEntities.keys.toSet()
       val isStatsModified = modifiedEntityClasses.contains(Language::class) ||
         modifiedEntityClasses.contains(Translation::class) ||
         modifiedEntityClasses.contains(Key::class) ||

diff --git a/...d/data/src/main/kotlin/io/tolgee/component/eventListeners/StoreProjectActivityListener.kt b/...d/data/src/main/kotlin/io/tolgee/component/eventListeners/StoreProjectActivityListener.kt
@@ -11,6 +11,6 @@ class StoreProjectActivityListener(
 ) {
   @EventListener
   fun onActivity(event: OnProjectActivityEvent) {
-    activityService.storeActivityData(event.activityHolder)
+    activityService.storeActivityData(event.activityRevision, event.modifiedEntities)
   }
 }
diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/ActivityHolderConfig.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/ActivityHolderConfig.kt
@@ -9,6 +9,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
 import org.springframework.context.ApplicationContext
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
+import org.springframework.context.annotation.Lazy
 import org.springframework.context.annotation.Primary
 import org.springframework.context.annotation.Scope
 import org.springframework.context.annotation.ScopedProxyMode
@@ -32,6 +33,7 @@ class ActivityHolderConfig {
   }
 
   @Bean
+  @Lazy(false)
   @Primary
   @Scope(BeanDefinition.SCOPE_PROTOTYPE, proxyMode = ScopedProxyMode.TARGET_CLASS)
   fun activityHolder(applicationContext: ApplicationContext): ActivityHolder {

diff --git a/backend/data/src/main/kotlin/io/tolgee/events/OnProjectActivityEvent.kt b/backend/data/src/main/kotlin/io/tolgee/events/OnProjectActivityEvent.kt
@@ -1,8 +1,10 @@
 package io.tolgee.events
 
-import io.tolgee.activity.ActivityHolder
-import org.springframework.context.ApplicationEvent
+import io.tolgee.activity.ModifiedEntitiesType
+import io.tolgee.model.activity.ActivityRevision
 
 class OnProjectActivityEvent(
-  val activityHolder: ActivityHolder,
-) : ApplicationEvent(activityHolder)
+  val activityRevision: ActivityRevision,
+  val modifiedEntities: ModifiedEntitiesType,
+  val organizationId: Long?
+)
diff --git a/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityRevision.kt b/backend/data/src/main/kotlin/io/tolgee/model/activity/ActivityRevision.kt
@@ -3,6 +3,7 @@ package io.tolgee.model.activity
 import com.vladmihalcea.hibernate.type.json.JsonBinaryType
 import io.tolgee.activity.data.ActivityType
 import io.tolgee.component.CurrentDateProvider
+import io.tolgee.model.batch.BatchJobChunkExecution
 import org.hibernate.annotations.NotFound
 import org.hibernate.annotations.NotFoundAction
 import org.hibernate.annotations.Type
@@ -17,11 +18,13 @@ import javax.persistence.Entity
 import javax.persistence.EntityListeners
 import javax.persistence.EnumType
 import javax.persistence.Enumerated
+import javax.persistence.FetchType
 import javax.persistence.GeneratedValue
 import javax.persistence.GenerationType
 import javax.persistence.Id
 import javax.persistence.Index
 import javax.persistence.OneToMany
+import javax.persistence.OneToOne
 import javax.persistence.PrePersist
 import javax.persistence.SequenceGenerator
 import javax.persistence.Table
@@ -82,6 +85,9 @@ class ActivityRevision : java.io.Serializable {
   @OneToMany(mappedBy = "activityRevision")
   var modifiedEntities: MutableList<ActivityModifiedEntity> = mutableListOf()
 
+  @OneToOne(fetch = FetchType.LAZY)
+  var batchJobChunkExecution: BatchJobChunkExecution? = null
+
   companion object {
     @Configurable
     class ActivityRevisionListener {

diff --git a/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobChunkExecution.kt b/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobChunkExecution.kt
@@ -2,6 +2,7 @@ package io.tolgee.model.batch
 
 import com.vladmihalcea.hibernate.type.json.JsonBinaryType
 import io.tolgee.model.StandardAuditModel
+import io.tolgee.model.activity.ActivityRevision
 import org.hibernate.annotations.ColumnDefault
 import org.hibernate.annotations.Type
 import org.hibernate.annotations.TypeDef
@@ -11,7 +12,9 @@ import javax.persistence.Column
 import javax.persistence.Entity
 import javax.persistence.EnumType
 import javax.persistence.Enumerated
+import javax.persistence.FetchType
 import javax.persistence.ManyToOne
+import javax.persistence.OneToOne
 import javax.persistence.Table
 
 @Entity
@@ -43,4 +46,7 @@ class BatchJobChunkExecution : StandardAuditModel() {
 
   @ColumnDefault("false")
   var retry: Boolean = false
+
+  @OneToOne(fetch = FetchType.LAZY, mappedBy = "batchJobChunkExecution")
+  var activityRevision: ActivityRevision? = null
 }
diff --git a/backend/data/src/main/resources/db/changelog/schema.xml b/backend/data/src/main/resources/db/changelog/schema.xml
@@ -2599,4 +2599,12 @@
             </column>
         </addColumn>
     </changeSet>
+    <changeSet author="jenik (generated)" id="1687532552778-1">
+        <addColumn tableName="activity_revision">
+            <column name="batch_job_chunk_execution_id" type="int8"/>
+        </addColumn>
+    </changeSet>
+    <changeSet author="jenik (generated)" id="1687532552778-2">
+        <addForeignKeyConstraint baseColumnNames="batch_job_chunk_execution_id" baseTableName="activity_revision" constraintName="FK7rfcdgi5uj3k7bk02h7e3suft" deferrable="false" initiallyDeferred="false" referencedColumnNames="id" referencedTableName="batch_job_chunk_execution" validate="true"/>
+    </changeSet>
 </databaseChangeLog>