From cc9c988759d0ca3cb72d28af926ca864566defe9 Mon Sep 17 00:00:00 2001 From: Chanyeong Lim Date: Tue, 27 Aug 2024 19:49:11 +0900 Subject: [PATCH 1/3] =?UTF-8?q?snu4t=EC=97=90=20=EA=B0=95=EC=9D=98=20?= =?UTF-8?q?=ED=8F=89=EC=A0=90=20=EB=8F=99=EA=B8=B0=ED=99=94=20(#111)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * LectureRepository에 findAllRatings 추가 * evInfo 동기화 Batch 추가 * 강의평 추가/삭제/수정 시 snu4t에 정보 적용 * add reactive mongodb dependency * JpaPagingItemReader 사용 * MongoService 분리 * processor 없이 처리 * 필요없는 job 삭제 --- .../snuttev/sync/SnuttLectureSyncJobConfig.kt | 8 ++ .../snuttev/sync/SnuttRatingSyncJobConfig.kt | 85 +++++++++++++++++++ core/build.gradle.kts | 1 + .../evaluation/service/EvaluationService.kt | 14 +++ .../lecture/repository/LectureRepository.kt | 9 ++ .../repository/SnuttLectureIdMapRepository.kt | 2 + .../snuttev/core/domain/mongo/MongoService.kt | 22 +++++ 7 files changed, 141 insertions(+) create mode 100644 batch/src/main/kotlin/com/wafflestudio/snuttev/sync/SnuttRatingSyncJobConfig.kt create mode 100644 core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/mongo/MongoService.kt diff --git a/batch/src/main/kotlin/com/wafflestudio/snuttev/sync/SnuttLectureSyncJobConfig.kt b/batch/src/main/kotlin/com/wafflestudio/snuttev/sync/SnuttLectureSyncJobConfig.kt index b5f858e..ff2d24e 100644 --- a/batch/src/main/kotlin/com/wafflestudio/snuttev/sync/SnuttLectureSyncJobConfig.kt +++ b/batch/src/main/kotlin/com/wafflestudio/snuttev/sync/SnuttLectureSyncJobConfig.kt @@ -37,6 +37,7 @@ class SnuttLectureSyncJobConfig( private val semesterLectureRepository: SemesterLectureRepository, private val lectureRepository: LectureRepository, private val snuttLectureIdMapRepository: SnuttLectureIdMapRepository, + private val ratingSyncJob: Job, ) { companion object { private const val JOB_NAME = "SYNC_JOB" @@ -78,6 +79,7 @@ class SnuttLectureSyncJobConfig( ), ), ) + .next(ratingSyncJobStep(jobRepository)) .build() } @@ -93,6 +95,7 @@ class SnuttLectureSyncJobConfig( .toMutableMap() return JobBuilder(JOB_NAME, jobRepository) .start(customReaderStep(jobRepository, Query())) + .next(ratingSyncJobStep(jobRepository)) .build() } @@ -169,6 +172,11 @@ class SnuttLectureSyncJobConfig( snuttLectureIdMapRepository.saveAll(items.map { it.snuttLectureIdMap }) } } + + private fun ratingSyncJobStep(jobRepository: JobRepository): Step = + StepBuilder(SnuttRatingSyncJobConfig.RATING_SYNC_JOB_NAME, jobRepository) + .job(ratingSyncJob) + .build() } data class SyncProcessResult( diff --git a/batch/src/main/kotlin/com/wafflestudio/snuttev/sync/SnuttRatingSyncJobConfig.kt b/batch/src/main/kotlin/com/wafflestudio/snuttev/sync/SnuttRatingSyncJobConfig.kt new file mode 100644 index 0000000..c373f50 --- /dev/null +++ b/batch/src/main/kotlin/com/wafflestudio/snuttev/sync/SnuttRatingSyncJobConfig.kt @@ -0,0 +1,85 @@ +package com.wafflestudio.snuttev.sync + +import com.wafflestudio.snuttev.core.domain.lecture.model.SnuttLectureIdMap +import com.wafflestudio.snuttev.core.domain.lecture.repository.LectureRepository +import jakarta.persistence.EntityManagerFactory +import org.springframework.batch.core.Job +import org.springframework.batch.core.Step +import org.springframework.batch.core.job.builder.JobBuilder +import org.springframework.batch.core.repository.JobRepository +import org.springframework.batch.core.step.builder.StepBuilder +import org.springframework.batch.item.ItemWriter +import org.springframework.batch.item.database.JpaPagingItemReader +import org.springframework.batch.item.database.builder.JpaPagingItemReaderBuilder +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.context.annotation.Profile +import org.springframework.data.mongodb.core.BulkOperations +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.orm.jpa.JpaTransactionManager + +@Configuration +@Profile(value = ["!test"]) +class SnuttRatingSyncJobConfig( + private val entityManagerFactory: EntityManagerFactory, + private val mongoTemplate: MongoTemplate, + private val lectureRepository: LectureRepository, +) { + companion object { + const val RATING_SYNC_JOB_NAME = "RATING_SYNC_JOB" + private const val CUSTOM_READER_JOB_STEP = RATING_SYNC_JOB_NAME + "_STEP" + private const val CHUNK_SIZE = 1000000 + } + + @Bean + fun ratingSyncJob(jobRepository: JobRepository): Job { + return JobBuilder(RATING_SYNC_JOB_NAME, jobRepository) + .start(customReaderStep(jobRepository)) + .build() + } + + private fun customReaderStep(jobRepository: JobRepository): Step { + return StepBuilder(CUSTOM_READER_JOB_STEP, jobRepository) + .chunk( + CHUNK_SIZE, + JpaTransactionManager().apply { + this.entityManagerFactory = this@SnuttRatingSyncJobConfig.entityManagerFactory + }, + ) + .reader(reader()) + .writer(writer()) + .build() + } + + private fun reader(): JpaPagingItemReader = + JpaPagingItemReaderBuilder() + .name("snuttLectureIdMapReader") + .entityManagerFactory(entityManagerFactory) + .queryString("SELECT s FROM SnuttLectureIdMap s JOIN FETCH s.semesterLecture") + .pageSize(CHUNK_SIZE) + .build() + + private fun writer(): ItemWriter { + return ItemWriter { items -> + val lectureIdtoLectureRatingMap = + lectureRepository.findAllRatingsByLectureIds( + items.mapNotNull { it.semesterLecture.lecture.id }, + ) + .associateBy { it.id } + val bulkOps = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, "lectures") + items.forEach { + val evInfo = lectureIdtoLectureRatingMap[it.semesterLecture.lecture.id] + bulkOps.updateOne( + Query(Criteria.where("_id").`is`(it.snuttId)), + Update().set("evInfo.evId", evInfo?.id) + .set("evInfo.avgRating", evInfo?.avgRating) + .set("evInfo.count", evInfo?.count), + ) + } + bulkOps.execute() + } + } +} diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 9070e49..3395cad 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -18,6 +18,7 @@ dependencies { implementation("software.amazon.awssdk:secretsmanager:2.20.66") implementation("software.amazon.awssdk:sts:2.20.66") + implementation("org.springframework.boot:spring-boot-starter-data-mongodb-reactive") implementation("org.springframework.boot:spring-boot-starter-data-redis") runtimeOnly("com.mysql:mysql-connector-j") diff --git a/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/evaluation/service/EvaluationService.kt b/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/evaluation/service/EvaluationService.kt index 7ed4fc4..0da8334 100644 --- a/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/evaluation/service/EvaluationService.kt +++ b/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/evaluation/service/EvaluationService.kt @@ -36,6 +36,8 @@ import com.wafflestudio.snuttev.core.domain.evaluation.repository.LectureEvaluat import com.wafflestudio.snuttev.core.domain.lecture.model.SemesterLecture import com.wafflestudio.snuttev.core.domain.lecture.repository.LectureRepository import com.wafflestudio.snuttev.core.domain.lecture.repository.SemesterLectureRepository +import com.wafflestudio.snuttev.core.domain.lecture.repository.SnuttLectureIdMapRepository +import com.wafflestudio.snuttev.core.domain.mongo.MongoService import com.wafflestudio.snuttev.core.domain.tag.repository.TagRepository import org.springframework.dao.DataIntegrityViolationException import org.springframework.data.repository.findByIdOrNull @@ -51,6 +53,8 @@ class EvaluationService internal constructor( private val evaluationReportRepository: EvaluationReportRepository, private val evaluationLikeRepository: EvaluationLikeRepository, private val cache: Cache, + private val snuttLectureIdMapRepository: SnuttLectureIdMapRepository, + private val mongoService: MongoService, ) { companion object { private const val DEFAULT_PAGE_SIZE = 20 @@ -77,6 +81,8 @@ class EvaluationService internal constructor( cache.deleteAll(CacheKey.EVALUATIONS_BY_TAG_PAGE) + updateEvInfosBySemesterLecture(semesterLecture) + return genLectureEvaluationDto(lectureEvaluation) } @@ -276,6 +282,7 @@ class EvaluationService internal constructor( } cache.deleteAll(CacheKey.EVALUATIONS_BY_TAG_PAGE) + updateEvInfosBySemesterLecture(evaluation.semesterLecture) val isLiked = evaluationLikeRepository.existsByLectureEvaluationAndUserId(evaluation, userId) return EvaluationWithSemesterResponse.of(evaluation, userId, isLiked) @@ -308,6 +315,7 @@ class EvaluationService internal constructor( lectureEvaluation.isHidden = true cache.deleteAll(CacheKey.EVALUATIONS_BY_TAG_PAGE) + updateEvInfosBySemesterLecture(lectureEvaluation.semesterLecture) } fun reportEvaluation( @@ -391,4 +399,10 @@ class EvaluationService internal constructor( content = evaluationReport.content, isHidden = evaluationReport.isHidden, ) + + private fun updateEvInfosBySemesterLecture(semesterLecture: SemesterLecture) { + val evInfo = lectureRepository.findAllRatingsByLectureIds(listOf(semesterLecture.lecture.id!!)).firstOrNull() + val snuttIds = snuttLectureIdMapRepository.findAllBySemesterLecture(semesterLecture).map { it.snuttId } + mongoService.updateEvInfoToSnuttIds(snuttIds, evInfo) + } } diff --git a/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/lecture/repository/LectureRepository.kt b/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/lecture/repository/LectureRepository.kt index 7df3af5..2ff17dd 100644 --- a/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/lecture/repository/LectureRepository.kt +++ b/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/lecture/repository/LectureRepository.kt @@ -32,4 +32,13 @@ interface LectureRepository : JpaRepository, LectureRepositoryCu """, ) fun findAllRatingsByLectureIds(ids: Iterable): List + + @Query( + """ + select new com.wafflestudio.snuttev.core.domain.lecture.model.LectureRatingDao( + sl.lecture.id, avg(le.rating), count(le.id) + ) from LectureEvaluation le right join le.semesterLecture sl where le.isHidden = false group by sl.lecture.id + """, + ) + fun findAllRatings(): List } diff --git a/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/lecture/repository/SnuttLectureIdMapRepository.kt b/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/lecture/repository/SnuttLectureIdMapRepository.kt index 114245f..dd8566f 100644 --- a/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/lecture/repository/SnuttLectureIdMapRepository.kt +++ b/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/lecture/repository/SnuttLectureIdMapRepository.kt @@ -1,5 +1,6 @@ package com.wafflestudio.snuttev.core.domain.lecture.repository +import com.wafflestudio.snuttev.core.domain.lecture.model.SemesterLecture import com.wafflestudio.snuttev.core.domain.lecture.model.SnuttLectureIdMap import org.springframework.data.jpa.repository.JpaRepository import org.springframework.data.jpa.repository.Query @@ -8,4 +9,5 @@ interface SnuttLectureIdMapRepository : JpaRepository { @Query("SELECT ttm FROM SnuttLectureIdMap ttm JOIN FETCH ttm.semesterLecture WHERE ttm.snuttId IN :snuttIds") fun findAllWithSemesterLectureBySnuttIdIn(snuttIds: List): List fun findBySnuttId(snuttId: String): SnuttLectureIdMap? + fun findAllBySemesterLecture(semesterLecture: SemesterLecture): List } diff --git a/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/mongo/MongoService.kt b/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/mongo/MongoService.kt new file mode 100644 index 0000000..77d1267 --- /dev/null +++ b/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/mongo/MongoService.kt @@ -0,0 +1,22 @@ +package com.wafflestudio.snuttev.core.domain.mongo + +import com.wafflestudio.snuttev.core.domain.lecture.model.LectureRatingDao +import org.springframework.data.mongodb.core.ReactiveMongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.stereotype.Service + +@Service +class MongoService( + private val mongoTemplate: ReactiveMongoTemplate, +) { + fun updateEvInfoToSnuttIds(snuttIds: List, evInfo: LectureRatingDao?) = + mongoTemplate.updateMulti( + Query(Criteria.where("_id").`in`(snuttIds)), + Update().set("evInfo.evId", evInfo?.id) + .set("evInfo.avgRating", evInfo?.avgRating) + .set("evInfo.count", evInfo?.count), + "lectures", + ).subscribe() +} From 2f00138d3a904ae4ce72fe7cae50681dd0206e56 Mon Sep 17 00:00:00 2001 From: Chanyeong Lim Date: Tue, 3 Sep 2024 15:04:40 +0900 Subject: [PATCH 2/3] Switch to non-reactive mongodb to fix batch (#112) --- batch/build.gradle.kts | 1 - build.gradle.kts | 1 + core/build.gradle.kts | 3 ++- .../snuttev/core/domain/mongo/MongoService.kt | 25 ++++++++++++------- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/batch/build.gradle.kts b/batch/build.gradle.kts index 919fee4..e9d15a0 100644 --- a/batch/build.gradle.kts +++ b/batch/build.gradle.kts @@ -2,7 +2,6 @@ dependencies { implementation(project(":core")) implementation("org.springframework.boot:spring-boot-starter-batch") - implementation("org.springframework.boot:spring-boot-starter-data-mongodb") runtimeOnly("org.postgresql:postgresql") runtimeOnly("com.h2database:h2") } diff --git a/build.gradle.kts b/build.gradle.kts index ae4aa3a..7d86f02 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -45,6 +45,7 @@ subprojects { implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") implementation("org.springframework.boot:spring-boot-starter-data-jpa") + implementation("org.springframework.boot:spring-boot-starter-data-mongodb") implementation("com.wafflestudio.truffle.sdk:truffle-spring-boot-starter:1.1.2") implementation("com.wafflestudio.truffle.sdk:truffle-logback:1.1.2") diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 3395cad..e63b511 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -18,9 +18,10 @@ dependencies { implementation("software.amazon.awssdk:secretsmanager:2.20.66") implementation("software.amazon.awssdk:sts:2.20.66") - implementation("org.springframework.boot:spring-boot-starter-data-mongodb-reactive") implementation("org.springframework.boot:spring-boot-starter-data-redis") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core") + runtimeOnly("com.mysql:mysql-connector-j") kapt("com.querydsl:querydsl-apt::jakarta") } diff --git a/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/mongo/MongoService.kt b/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/mongo/MongoService.kt index 77d1267..e735ff2 100644 --- a/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/mongo/MongoService.kt +++ b/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/mongo/MongoService.kt @@ -1,7 +1,10 @@ package com.wafflestudio.snuttev.core.domain.mongo import com.wafflestudio.snuttev.core.domain.lecture.model.LectureRatingDao -import org.springframework.data.mongodb.core.ReactiveMongoTemplate +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import org.springframework.data.mongodb.core.MongoTemplate import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.Update @@ -9,14 +12,18 @@ import org.springframework.stereotype.Service @Service class MongoService( - private val mongoTemplate: ReactiveMongoTemplate, + private val mongoTemplate: MongoTemplate, ) { fun updateEvInfoToSnuttIds(snuttIds: List, evInfo: LectureRatingDao?) = - mongoTemplate.updateMulti( - Query(Criteria.where("_id").`in`(snuttIds)), - Update().set("evInfo.evId", evInfo?.id) - .set("evInfo.avgRating", evInfo?.avgRating) - .set("evInfo.count", evInfo?.count), - "lectures", - ).subscribe() + CoroutineScope(Dispatchers.IO).launch { + runCatching { + mongoTemplate.updateMulti( + Query(Criteria.where("_id").`in`(snuttIds)), + Update().set("evInfo.evId", evInfo?.id) + .set("evInfo.avgRating", evInfo?.avgRating) + .set("evInfo.count", evInfo?.count), + "lectures", + ) + } + } } From 7b2a40b7db253c29c25ba11d7c6276c16f473fbd Mon Sep 17 00:00:00 2001 From: Chanyeong Lim Date: Wed, 4 Sep 2024 22:37:57 +0900 Subject: [PATCH 3/3] =?UTF-8?q?MongoService=EC=97=90=EC=84=9C=20coroutine?= =?UTF-8?q?=20=EC=A0=9C=EA=B1=B0=20(#113)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * MongoService CoroutineScope 제거 * Mock MongoService in test --- core/build.gradle.kts | 2 -- .../snuttev/core/domain/mongo/MongoService.kt | 21 +++++++------------ .../service/EvaluationServiceTest.kt | 3 +++ 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/core/build.gradle.kts b/core/build.gradle.kts index e63b511..9070e49 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -20,8 +20,6 @@ dependencies { implementation("org.springframework.boot:spring-boot-starter-data-redis") - implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core") - runtimeOnly("com.mysql:mysql-connector-j") kapt("com.querydsl:querydsl-apt::jakarta") } diff --git a/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/mongo/MongoService.kt b/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/mongo/MongoService.kt index e735ff2..ced6f7a 100644 --- a/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/mongo/MongoService.kt +++ b/core/src/main/kotlin/com/wafflestudio/snuttev/core/domain/mongo/MongoService.kt @@ -1,9 +1,6 @@ package com.wafflestudio.snuttev.core.domain.mongo import com.wafflestudio.snuttev.core.domain.lecture.model.LectureRatingDao -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch import org.springframework.data.mongodb.core.MongoTemplate import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query @@ -15,15 +12,13 @@ class MongoService( private val mongoTemplate: MongoTemplate, ) { fun updateEvInfoToSnuttIds(snuttIds: List, evInfo: LectureRatingDao?) = - CoroutineScope(Dispatchers.IO).launch { - runCatching { - mongoTemplate.updateMulti( - Query(Criteria.where("_id").`in`(snuttIds)), - Update().set("evInfo.evId", evInfo?.id) - .set("evInfo.avgRating", evInfo?.avgRating) - .set("evInfo.count", evInfo?.count), - "lectures", - ) - } + runCatching { + mongoTemplate.updateMulti( + Query(Criteria.where("_id").`in`(snuttIds)), + Update().set("evInfo.evId", evInfo?.id) + .set("evInfo.avgRating", evInfo?.avgRating) + .set("evInfo.count", evInfo?.count), + "lectures", + ) } } diff --git a/core/src/test/kotlin/com/wafflestudio/snuttev/core/domain/evaluation/service/EvaluationServiceTest.kt b/core/src/test/kotlin/com/wafflestudio/snuttev/core/domain/evaluation/service/EvaluationServiceTest.kt index 02361f2..7607a4c 100644 --- a/core/src/test/kotlin/com/wafflestudio/snuttev/core/domain/evaluation/service/EvaluationServiceTest.kt +++ b/core/src/test/kotlin/com/wafflestudio/snuttev/core/domain/evaluation/service/EvaluationServiceTest.kt @@ -14,6 +14,7 @@ import com.wafflestudio.snuttev.core.domain.lecture.model.Lecture import com.wafflestudio.snuttev.core.domain.lecture.model.SemesterLecture import com.wafflestudio.snuttev.core.domain.lecture.repository.LectureRepository import com.wafflestudio.snuttev.core.domain.lecture.repository.SemesterLectureRepository +import com.wafflestudio.snuttev.core.domain.mongo.MongoService import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatNoException import org.assertj.core.api.Assertions.assertThatThrownBy @@ -21,6 +22,7 @@ import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.test.mock.mockito.MockBean import org.springframework.data.repository.findByIdOrNull import org.springframework.transaction.annotation.Transactional import kotlin.random.Random @@ -34,6 +36,7 @@ class EvaluationServiceTest @Autowired constructor( private val lectureRepository: LectureRepository, private val semesterLectureRepository: SemesterLectureRepository, private val evaluationLikeRepository: EvaluationLikeRepository, + @MockBean private val mongoService: MongoService, ) { @BeforeEach fun setup() {