diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobManagementController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobManagementController.kt index 779b335e31..c017cddfd7 100644 --- a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobManagementController.kt +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobManagementController.kt @@ -20,6 +20,7 @@ import org.springdoc.api.annotations.ParameterObject import org.springframework.data.domain.Pageable import org.springframework.data.web.PagedResourcesAssembler import org.springframework.data.web.SortDefault +import org.springframework.hateoas.CollectionModel import org.springframework.hateoas.PagedModel import org.springframework.web.bind.annotation.CrossOrigin import org.springframework.web.bind.annotation.GetMapping @@ -65,6 +66,21 @@ class BatchJobManagementController( return pagedResourcesAssembler.toModel(views, batchJobModelAssembler) } + @GetMapping(value = ["current-batch-jobs"]) + @AccessWithApiKey() + @AccessWithAnyProjectPermission() + @Operation( + summary = "Returns all running and pending tasks. " + + "Completed tasks are returned only if they are not older than 1 hour. " + + "If user doesn't have permission to view all batch jobs, only their jobs are returned." + ) + fun currentJobs(): CollectionModel { + val views = batchJobService.getCurrentJobViews( + projectId = projectHolder.project.id, + ) + return batchJobModelAssembler.toCollectionModel(views) + } + @GetMapping(value = ["batch-jobs/{id}"]) @AccessWithApiKey() @AccessWithAnyProjectPermission() diff --git a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobController.kt b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobController.kt index 48b97982f8..66835081ad 100644 --- a/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobController.kt +++ b/backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobController.kt @@ -16,7 +16,7 @@ import io.tolgee.security.project_auth.AccessWithProjectPermission import io.tolgee.security.project_auth.ProjectHolder import io.tolgee.service.security.SecurityService import org.springframework.web.bind.annotation.CrossOrigin -import org.springframework.web.bind.annotation.PutMapping +import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.RequestBody import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RestController @@ -34,7 +34,7 @@ class StartBatchJobController( private val authenticationFacade: AuthenticationFacade, private val batchJobModelAssembler: BatchJobModelAssembler ) { - @PutMapping(value = ["/translate"]) + @PostMapping(value = ["/translate"]) @AccessWithApiKey() @AccessWithProjectPermission(Scope.BATCH_AUTO_TRANSLATE) @Operation(summary = "Translates provided keys to provided languages") @@ -49,7 +49,7 @@ class StartBatchJobController( ).model } - @PutMapping(value = ["/delete-keys"]) + @PostMapping(value = ["/delete-keys"]) @AccessWithApiKey() @AccessWithProjectPermission(Scope.KEYS_DELETE) @Operation(summary = "Translates provided keys to provided languages") diff --git a/backend/api/src/main/kotlin/io/tolgee/hateoas/batch/BatchJobModel.kt b/backend/api/src/main/kotlin/io/tolgee/hateoas/batch/BatchJobModel.kt index cb4e4bce30..b1ca0681e1 100644 --- a/backend/api/src/main/kotlin/io/tolgee/hateoas/batch/BatchJobModel.kt +++ b/backend/api/src/main/kotlin/io/tolgee/hateoas/batch/BatchJobModel.kt @@ -30,7 +30,10 @@ open class BatchJobModel( val author: SimpleUserAccountModel?, @Schema(description = "The time when the job created") - val createdAt: String, + val createdAt: Long, + + @Schema(description = "The time when the job was last updated (status change)") + val updatedAt: Long, @Schema(description = "The activity revision id, that stores the activity details of the job") val activityRevisionId: Long?, diff --git a/backend/api/src/main/kotlin/io/tolgee/hateoas/batch/BatchJobModelAssembler.kt b/backend/api/src/main/kotlin/io/tolgee/hateoas/batch/BatchJobModelAssembler.kt index 7f5ecf4a47..e483e46251 100644 --- a/backend/api/src/main/kotlin/io/tolgee/hateoas/batch/BatchJobModelAssembler.kt +++ b/backend/api/src/main/kotlin/io/tolgee/hateoas/batch/BatchJobModelAssembler.kt @@ -20,7 +20,8 @@ class BatchJobModelAssembler( progress = view.progress, totalItems = view.batchJob.totalItems, author = view.batchJob.author?.let { simpleUserAccountModelAssembler.toModel(it) }, - createdAt = view.batchJob.createdAt.toString(), + createdAt = view.batchJob.createdAt?.time ?: 0, + updatedAt = view.batchJob.updatedAt?.time ?: 0, activityRevisionId = view.batchJob.activityRevision?.id, errorMessage = view.errorMessage?.code ) diff --git a/backend/api/src/main/kotlin/io/tolgee/websocket/ActivityWebsocketListener.kt b/backend/api/src/main/kotlin/io/tolgee/websocket/ActivityWebsocketListener.kt index 52a8ef5835..8dc5794bb9 100644 --- a/backend/api/src/main/kotlin/io/tolgee/websocket/ActivityWebsocketListener.kt +++ b/backend/api/src/main/kotlin/io/tolgee/websocket/ActivityWebsocketListener.kt @@ -7,6 +7,7 @@ import io.tolgee.batch.events.OnBatchJobCancelled import io.tolgee.batch.events.OnBatchJobFailed import io.tolgee.batch.events.OnBatchJobProgress import io.tolgee.batch.events.OnBatchJobSucceeded +import io.tolgee.component.CurrentDateProvider import io.tolgee.constants.Message import io.tolgee.events.OnProjectActivityStoredEvent import io.tolgee.hateoas.user_account.SimpleUserAccountModelAssembler @@ -26,7 +27,8 @@ class ActivityWebsocketListener( private val websocketEventPublisher: WebsocketEventPublisher, private val simpleUserAccountModelAssembler: SimpleUserAccountModelAssembler, private val userAccountService: UserAccountService, - private val relationDescriptionExtractor: RelationDescriptionExtractor + private val relationDescriptionExtractor: RelationDescriptionExtractor, + private val currentDateProvider: CurrentDateProvider ) { @Async @@ -78,7 +80,8 @@ class ActivityWebsocketListener( data = data, sourceActivity = activityRevision.type, activityId = activityRevision.id, - dataCollapsed = data == null + dataCollapsed = data == null, + timestamp = currentDateProvider.date.time ) ) } @@ -97,7 +100,8 @@ class ActivityWebsocketListener( data = WebsocketProgressInfo(event.job.id, event.processed, event.total, realStatus), sourceActivity = null, activityId = null, - dataCollapsed = false + dataCollapsed = false, + timestamp = currentDateProvider.date.time ) ) } @@ -125,7 +129,8 @@ class ActivityWebsocketListener( data = WebsocketProgressInfo(event.job.id, null, null, event.job.status, errorMessage?.code), sourceActivity = null, activityId = null, - dataCollapsed = false + dataCollapsed = false, + timestamp = currentDateProvider.date.time ) ) } diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobManagementControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobManagementControllerTest.kt index 33af835f9c..dd5fd8af4a 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobManagementControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobManagementControllerTest.kt @@ -10,18 +10,22 @@ import io.tolgee.batch.JobChunkExecutionQueue import io.tolgee.batch.processors.TranslationChunkProcessor import io.tolgee.batch.request.BatchTranslateRequest import io.tolgee.batch.state.BatchJobStateProvider +import io.tolgee.component.CurrentDateProvider import io.tolgee.development.testDataBuilder.data.BatchJobsTestData import io.tolgee.fixtures.andAssertThatJson import io.tolgee.fixtures.andIsForbidden import io.tolgee.fixtures.andIsOk +import io.tolgee.fixtures.andPrettyPrint import io.tolgee.fixtures.isValidId import io.tolgee.fixtures.node import io.tolgee.fixtures.waitForNotThrowing +import io.tolgee.model.UserAccount import io.tolgee.model.batch.BatchJob import io.tolgee.model.batch.BatchJobStatus import io.tolgee.testing.ContextRecreatingTest import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod import io.tolgee.testing.assert +import io.tolgee.util.addMinutes import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -59,6 +63,9 @@ class BatchJobManagementControllerTest : ProjectAuthControllerTest("/v2/projects @Autowired lateinit var batchJobConcurrentLauncher: BatchJobConcurrentLauncher + @Autowired + lateinit var currentDateProvider: CurrentDateProvider + @BeforeEach fun setup() { testData = BatchJobsTestData() @@ -70,6 +77,7 @@ class BatchJobManagementControllerTest : ProjectAuthControllerTest("/v2/projects @AfterEach fun after() { batchJobConcurrentLauncher.pause = false + currentDateProvider.forcedDate = null } @Test @@ -82,7 +90,7 @@ class BatchJobManagementControllerTest : ProjectAuthControllerTest("/v2/projects batchJobConcurrentLauncher.pause = true - performProjectAuthPut( + performProjectAuthPost( "start-batch-job/translate", mapOf( "keyIds" to keyIds, @@ -207,6 +215,63 @@ class BatchJobManagementControllerTest : ProjectAuthControllerTest("/v2/projects } } + @Test + @ProjectJWTAuthTestMethod + fun `returns list of current jobs`() { + saveAndPrepare() + + var wait = true + whenever( + translationChunkProcessor.process(any(), any(), any(), any()) + ).then { + while (wait) { + Thread.sleep(100) + } + } + + val adminsJobs = (1..3).map { runChunkedJob(50) } + val anotherUsersJobs = (1..3).map { runChunkedJob(50, testData.anotherUser) } + + performProjectAuthGet("current-batch-jobs") + .andIsOk.andPrettyPrint.andAssertThatJson { + node("_embedded.batchJobs") { + isArray.hasSize(6) + node("[0].status").isEqualTo("RUNNING") + node("[1].status").isEqualTo("RUNNING") + node("[2].status").isEqualTo("PENDING") + } + } + + wait = false + + waitForNotThrowing(pollTime = 1000, timeout = 10000) { + val dtos = (adminsJobs + anotherUsersJobs).map { batchJobService.getJobDto(it.id) } + dtos.count { it.status == BatchJobStatus.SUCCESS }.assert.isEqualTo(6) + } + + performProjectAuthGet("current-batch-jobs") + .andIsOk.andAssertThatJson { + node("_embedded.batchJobs") { + isArray.hasSize(6) + node("[0].status").isEqualTo("SUCCESS") + } + } + + userAccount = testData.anotherUser + + performProjectAuthGet("current-batch-jobs") + .andIsOk.andAssertThatJson { + node("_embedded.batchJobs").isArray.hasSize(3) + } + + currentDateProvider.forcedDate = currentDateProvider.date.addMinutes(61) + + performProjectAuthGet("current-batch-jobs") + .andIsOk.andAssertThatJson { + node("_embedded.batchJobs").isAbsent() + } + } + @Test @ProjectJWTAuthTestMethod fun `returns single job`() { @@ -250,14 +315,14 @@ class BatchJobManagementControllerTest : ProjectAuthControllerTest("/v2/projects this.projectSupplier = { testData.projectBuilder.self } } - protected fun runChunkedJob(keyCount: Int): BatchJob { + protected fun runChunkedJob(keyCount: Int, author: UserAccount = testData.user): BatchJob { return executeInNewTransaction { batchJobService.startJob( request = BatchTranslateRequest().apply { keyIds = (1L..keyCount).map { it } }, project = testData.projectBuilder.self, - author = testData.user, + author = author, type = BatchJobType.TRANSLATION ) } diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt index c745ccbdbf..ecd0edafe0 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt @@ -56,7 +56,7 @@ class StartBatchJobControllerTest : ProjectAuthControllerTest("/v2/projects/") { val keyIds = keys.map { it.id }.toList() - performProjectAuthPut( + performProjectAuthPost( "start-batch-job/translate", mapOf( "keyIds" to keyIds, @@ -106,7 +106,7 @@ class StartBatchJobControllerTest : ProjectAuthControllerTest("/v2/projects/") { val keyIds = keys.map { it.id }.toList() - performProjectAuthPut( + performProjectAuthPost( "start-batch-job/delete-keys", mapOf( "keyIds" to keyIds, diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt index 6eae82a287..5dcdd3af14 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt @@ -1,17 +1,23 @@ package io.tolgee.batch +import io.tolgee.component.CurrentDateProvider import io.tolgee.constants.Message import io.tolgee.dtos.cacheable.UserAccountDto import io.tolgee.exceptions.NotFoundException +import io.tolgee.exceptions.PermissionException import io.tolgee.model.Project import io.tolgee.model.UserAccount import io.tolgee.model.batch.BatchJob import io.tolgee.model.batch.BatchJobChunkExecution import io.tolgee.model.batch.BatchJobStatus import io.tolgee.model.batch.IBatchJob +import io.tolgee.model.enums.Scope import io.tolgee.model.views.BatchJobView import io.tolgee.repository.BatchJobRepository +import io.tolgee.security.AuthenticationFacade +import io.tolgee.service.security.SecurityService import io.tolgee.util.Logging +import io.tolgee.util.addMinutes import io.tolgee.util.executeInNewTransaction import io.tolgee.util.logger import org.springframework.context.ApplicationContext @@ -33,7 +39,10 @@ class BatchJobService( private val cachingBatchJobService: CachingBatchJobService, @Lazy private val progressManager: ProgressManager, - private val jobChunkExecutionQueue: JobChunkExecutionQueue + private val jobChunkExecutionQueue: JobChunkExecutionQueue, + private val currentDateProvider: CurrentDateProvider, + private val securityService: SecurityService, + private val authenticationFacade: AuthenticationFacade ) : Logging { @Transactional @@ -108,7 +117,22 @@ class BatchJobService( val jobs = batchJobRepository.getJobs(projectId, userAccount?.id, pageable) val progresses = getProgresses(jobs) + val errorMessages = getErrorMessages(jobs) + + return jobs.map { + BatchJobView(it, progresses[it.id] ?: 0, errorMessages[it.id]) + } + } + fun getCurrentJobViews(projectId: Long): List { + val jobs: List = batchJobRepository.getCurrentJobs( + projectId, + userAccountId = getUserAccountIdForCurrentJobView(projectId), + oneHourAgo = currentDateProvider.date.addMinutes(-60), + completedStatuses = BatchJobStatus.values().filter { it.completed } + ) + + val progresses = getProgresses(jobs) val errorMessages = getErrorMessages(jobs) return jobs.map { @@ -116,6 +140,19 @@ class BatchJobService( } } + /** + * Returns user account id if user has no permission to view all jobs. + */ + private fun getUserAccountIdForCurrentJobView(projectId: Long): Long? { + val userAccount = authenticationFacade.userAccount + return try { + securityService.checkProjectPermission(projectId, Scope.BATCH_JOBS_VIEW, userAccount) + null + } catch (e: PermissionException) { + userAccount.id + } + } + fun getErrorMessages(jobs: Iterable): Map { val needsErrorMessage = jobs.filter { it.status == BatchJobStatus.FAILED }.map { it.id }.toList() diff --git a/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJob.kt b/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJob.kt index 12c59e3e52..7347211d80 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJob.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJob.kt @@ -11,6 +11,7 @@ import org.hibernate.annotations.Type import org.hibernate.annotations.TypeDef import org.hibernate.annotations.TypeDefs import javax.persistence.Entity +import javax.persistence.EnumType.STRING import javax.persistence.Enumerated import javax.persistence.FetchType import javax.persistence.ManyToOne @@ -36,9 +37,10 @@ class BatchJob : StandardAuditModel(), IBatchJob { var chunkSize: Int = 0 + @Enumerated(STRING) override var status: BatchJobStatus = BatchJobStatus.PENDING - @Enumerated + @Enumerated(STRING) var type: BatchJobType = BatchJobType.TRANSLATION @OneToOne(mappedBy = "batchJob", fetch = FetchType.LAZY) diff --git a/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobStatus.kt b/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobStatus.kt index b12366ffd9..d016ff0abf 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobStatus.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/batch/BatchJobStatus.kt @@ -1,9 +1,11 @@ package io.tolgee.model.batch -enum class BatchJobStatus { - PENDING, - RUNNING, - SUCCESS, - FAILED, - CANCELLED, +enum class BatchJobStatus( + val completed: Boolean +) { + PENDING(false), + RUNNING(false), + SUCCESS(true), + FAILED(true), + CANCELLED(true), } diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/BatchJobRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/BatchJobRepository.kt index 22a165dd91..4f6fa62bb1 100644 --- a/backend/data/src/main/kotlin/io/tolgee/repository/BatchJobRepository.kt +++ b/backend/data/src/main/kotlin/io/tolgee/repository/BatchJobRepository.kt @@ -1,12 +1,14 @@ package io.tolgee.repository import io.tolgee.model.batch.BatchJob +import io.tolgee.model.batch.BatchJobStatus import io.tolgee.model.views.JobErrorMessagesView import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable import org.springframework.data.jpa.repository.JpaRepository import org.springframework.data.jpa.repository.Query import org.springframework.stereotype.Repository +import java.util.* @Repository interface BatchJobRepository : JpaRepository { @@ -26,6 +28,24 @@ interface BatchJobRepository : JpaRepository { ) fun getJobs(projectId: Long, userAccountId: Long?, pageable: Pageable): Page + @Query( + value = """ + select j from BatchJob j + left join fetch j.author + left join fetch j.activityRevision + where j.project.id = :projectId + and (:userAccountId is null or j.author.id = :userAccountId) + and (j.status not in :completedStatuses or j.updatedAt > :oneHourAgo) + order by j.updatedAt + """ + ) + fun getCurrentJobs( + projectId: Long, + userAccountId: Long?, + oneHourAgo: Date, + completedStatuses: List + ): List + @Query( nativeQuery = true, value = """ diff --git a/backend/data/src/main/kotlin/io/tolgee/websocket/WebsocketEvent.kt b/backend/data/src/main/kotlin/io/tolgee/websocket/WebsocketEvent.kt index 8c92da85fb..6ef6a41904 100644 --- a/backend/data/src/main/kotlin/io/tolgee/websocket/WebsocketEvent.kt +++ b/backend/data/src/main/kotlin/io/tolgee/websocket/WebsocketEvent.kt @@ -7,5 +7,6 @@ data class WebsocketEvent( val data: Any? = null, val sourceActivity: ActivityType?, val activityId: Long?, - val dataCollapsed: Boolean + val dataCollapsed: Boolean, + val timestamp: Long ) diff --git a/backend/data/src/main/resources/db/changelog/schema.xml b/backend/data/src/main/resources/db/changelog/schema.xml index 77820c6dbb..708a485aba 100644 --- a/backend/data/src/main/resources/db/changelog/schema.xml +++ b/backend/data/src/main/resources/db/changelog/schema.xml @@ -2521,7 +2521,7 @@ - + @@ -2535,7 +2535,7 @@ - + @@ -2543,12 +2543,12 @@ - + - + @@ -2562,14 +2562,20 @@ + + + + - + + + - + @@ -2593,60 +2599,45 @@ - + + + + + + + + + + + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - + - - - - + + - - + +