[#39] 마지막 저장 장소에 MongoDB 추가 (#40)

* feat: mongoDB 추가 확인 - 차후 필요 없는 것들 삭제할 예정 * detailnfoUpdateConsumer 변경 전 커밋 - 세부정보 저장할 때, rdb의 데이터를 가져옴 - dto로 변경한 다음 mongoDB로 저장 * feat: mysql의 데이터를 mongoDB로 옮기는 consumer 추가 - 새로운 토픽을 두어 update와 분리
f-lab-edu · Nov 15, 2024 · bb214e1 · bb214e1
1 parent f623277
commit bb214e1
Show file tree

Hide file tree

Showing 28 changed files with 298 additions and 65 deletions.
diff --git a/app/build.gradle b/app/build.gradle
@@ -25,6 +25,7 @@ dependencies {
     annotationProcessor 'org.projectlombok:lombok'
 
     implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
+    implementation 'org.springframework.boot:spring-boot-starter-data-mongodb'
     implementation 'org.hibernate:hibernate-spatial:6.6.0.Final'
 
     // Use JUnit Jupiter for testing.

diff --git a/app/src/main/java/org/healthmap/app/controller/TestController.java b/app/src/main/java/org/healthmap/app/controller/TestController.java
@@ -0,0 +1,29 @@
+package org.healthmap.app.controller;
+
+import lombok.RequiredArgsConstructor;
+import org.healthmap.db.mongodb.model.MedicalFacility;
+import org.healthmap.db.mongodb.repository.MedicalFacilityMongoRepository;
+import org.healthmap.db.mysql.model.MedicalFacilityEntity;
+import org.healthmap.db.mysql.repository.MedicalFacilityMysqlRepository;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+// mongoDB 적용 되는지 확인용 controller
+@RestController
+@RequiredArgsConstructor
+public class TestController {
+    private final MedicalFacilityMongoRepository medicalFacilityMongoRepository;
+    private final MedicalFacilityMysqlRepository medicalFacilityMysqlRepository;
+
+    @GetMapping("/mongo/test2")
+    public void contentTest2() {
+        MedicalFacilityEntity testEntity = medicalFacilityMysqlRepository.findById("JDQ4MTAxMiM1MSMkMiMkMCMkMDAkMzgxMTkxIzExIyQxIyQ3IyQ5MiQzNjEwMDIjNTEjJDEjJDIjJDgz").orElse(null);
+        MedicalFacility medicalFacility = MedicalFacility.of(testEntity.getId(), testEntity.getName(), testEntity.getAddress(), testEntity.getPhoneNumber(), testEntity.getUrl(),
+                testEntity.getType(), testEntity.getState(), testEntity.getCity(), testEntity.getTown(), testEntity.getPostNumber(), testEntity.getCoordinate(),
+                testEntity.getParking(), testEntity.getParkingEtc(), testEntity.getTreatmentMon(), testEntity.getTreatmentTue(), testEntity.getTreatmentWed(),
+                testEntity.getTreatmentThu(), testEntity.getTreatmentFri(), testEntity.getTreatmentSat(), testEntity.getTreatmentSun(), testEntity.getReceiveWeek(),
+                testEntity.getReceiveSat(), testEntity.getLunchWeek(), testEntity.getLunchSat(), testEntity.getNoTreatmentSun(), testEntity.getNoTreatmentHoliday(),
+                testEntity.getEmergencyDay(), testEntity.getEmergencyNight(), testEntity.getCreatedAt(), testEntity.getUpdatedAt());
+        medicalFacilityMongoRepository.save(medicalFacility);
+    }
+}
diff --git a/app/src/main/java/org/healthmap/app/service/HealthMapService.java b/app/src/main/java/org/healthmap/app/service/HealthMapService.java
@@ -5,7 +5,7 @@
 import org.geolatte.geom.Point;
 import org.healthmap.app.dto.HealthMapRequestDto;
 import org.healthmap.app.dto.HealthMapResponseDto;
-import org.healthmap.db.medicalfacility.MedicalFacilityRepository;
+import org.healthmap.db.mysql.repository.MedicalFacilityMysqlRepository;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
@@ -16,7 +16,7 @@
 @Service
 @Slf4j
 public class HealthMapService {
-    private final MedicalFacilityRepository medicalFacilityRepository;
+    private final MedicalFacilityMysqlRepository medicalFacilityRepository;
 
     // 2km 이내의 시설 찾기
     public List<HealthMapResponseDto> getNearByMedicalFacility(HealthMapRequestDto requestDto) {

diff --git a/app/src/main/resources/application.yml b/app/src/main/resources/application.yml
@@ -6,3 +6,11 @@ spring:
     driver-class-name: com.mysql.cj.jdbc.Driver
     url: jdbc:mysql://localhost:13306/healthmap?useSSL=false&allowPublicKeyRetrieval=true&characterEncoding=UTF-8&serverTimezone=Asia/Seoul
     username: root
+
+  data:
+    mongodb:
+      host: localhost
+      port: 27018
+      database: healthmap
+      auto-index-creation: true
+
diff --git a/common/src/main/java/org/healthmap/dto/FacilityIdDto.java b/common/src/main/java/org/healthmap/dto/FacilityIdDto.java
@@ -0,0 +1,14 @@
+package org.healthmap.dto;
+
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+@Getter
+@NoArgsConstructor
+public class FacilityIdDto {
+    private String id;
+
+    public FacilityIdDto(String id) {
+        this.id = id;
+    }
+}
diff --git a/consumer/build.gradle b/consumer/build.gradle
@@ -16,6 +16,7 @@ dependencies {
     annotationProcessor 'org.projectlombok:lombok'
 
     implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
+    implementation 'org.springframework.boot:spring-boot-starter-data-mongodb'
     implementation 'org.springframework.kafka:spring-kafka'
     implementation 'org.hibernate:hibernate-spatial:6.6.0.Final'
 

diff --git a/consumer/src/main/java/org/healthmap/service/BasicInfoSaveConsumer.java b/consumer/src/main/java/org/healthmap/service/BasicInfoSaveConsumer.java
@@ -5,8 +5,8 @@
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.common.TopicPartition;
 import org.healthmap.config.KafkaProperties;
-import org.healthmap.db.medicalfacility.MedicalFacilityEntity;
-import org.healthmap.db.medicalfacility.MedicalFacilityRepository;
+import org.healthmap.db.mysql.model.MedicalFacilityEntity;
+import org.healthmap.db.mysql.repository.MedicalFacilityMysqlRepository;
 import org.healthmap.dto.BasicInfoDto;
 import org.healthmap.openapi.service.MapApiService;
 import org.locationtech.jts.geom.Coordinate;
@@ -25,12 +25,12 @@
 public class BasicInfoSaveConsumer {
     private final KafkaTemplate<String, BasicInfoDto> kafkaTemplate;
     private final KafkaProperties kafkaProperties;
-    private final MedicalFacilityRepository medicalFacilityRepository;
+    private final MedicalFacilityMysqlRepository medicalFacilityRepository;
     private final MapApiService mapApiService;
     private final Point dummyPoint;
-    private AtomicInteger count = new AtomicInteger(0); // 동작 확인용
+    private final AtomicInteger count = new AtomicInteger(0); // 동작 확인용
 
-    public BasicInfoSaveConsumer(KafkaTemplate<String, BasicInfoDto> kafkaTemplate, KafkaProperties kafkaProperties, MedicalFacilityRepository medicalFacilityRepository, MapApiService mapApiService) {
+    public BasicInfoSaveConsumer(KafkaTemplate<String, BasicInfoDto> kafkaTemplate, KafkaProperties kafkaProperties, MedicalFacilityMysqlRepository medicalFacilityRepository, MapApiService mapApiService) {
         this.kafkaTemplate = kafkaTemplate;
         this.kafkaProperties = kafkaProperties;
         this.medicalFacilityRepository = medicalFacilityRepository;

diff --git a/consumer/src/main/java/org/healthmap/service/BasicInfoUpdateConsumer.java b/consumer/src/main/java/org/healthmap/service/BasicInfoUpdateConsumer.java
@@ -3,8 +3,8 @@
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.healthmap.config.KafkaProperties;
-import org.healthmap.db.medicalfacility.MedicalFacilityEntity;
-import org.healthmap.db.medicalfacility.MedicalFacilityRepository;
+import org.healthmap.db.mysql.model.MedicalFacilityEntity;
+import org.healthmap.db.mysql.repository.MedicalFacilityMysqlRepository;
 import org.healthmap.dto.BasicInfoDto;
 import org.springframework.kafka.annotation.KafkaListener;
 import org.springframework.kafka.core.KafkaTemplate;
@@ -20,13 +20,13 @@
 @Service
 @RequiredArgsConstructor
 public class BasicInfoUpdateConsumer {
-    private final MedicalFacilityRepository medicalFacilityRepository;
+    private final MedicalFacilityMysqlRepository medicalFacilityRepository;
     private final KafkaTemplate<String, BasicInfoDto> kafkaTemplate;
     private final KafkaProperties kafkaProperties;
     private final TransactionTemplate transactionTemplate;
     private final ExecutorService executorService = Executors.newFixedThreadPool(20);
-    private AtomicInteger count = new AtomicInteger(0);     // 동작 확인용
-    private AtomicInteger realCount = new AtomicInteger(0); // 동작 확인용
+    private final AtomicInteger count = new AtomicInteger(0);     // 동작 확인용
+    private final AtomicInteger realCount = new AtomicInteger(0); // 동작 확인용
 
     @KafkaListener(topics = "${kafka-config.consumer.basic-topic}",
             groupId = "${kafka-config.consumer.groupId}",

diff --git a/consumer/src/main/java/org/healthmap/service/DetailInfoUpdateConsumer.java b/consumer/src/main/java/org/healthmap/service/DetailInfoUpdateConsumer.java
@@ -3,8 +3,10 @@
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.healthmap.db.medicalfacility.MedicalFacilityRepository;
+import org.healthmap.config.KafkaProperties;
+import org.healthmap.db.mysql.repository.MedicalFacilityMysqlRepository;
 import org.healthmap.dto.BasicInfoDto;
+import org.healthmap.dto.FacilityIdDto;
 import org.healthmap.openapi.dto.FacilityDetailUpdateDto;
 import org.healthmap.openapi.error.OpenApiErrorCode;
 import org.healthmap.openapi.exception.OpenApiProblemException;
@@ -22,9 +24,10 @@
 @Service
 @RequiredArgsConstructor
 public class DetailInfoUpdateConsumer {
-    private final KafkaTemplate<String, String> kafkaTemplate;
+    private final KafkaTemplate<String, FacilityIdDto> kafkaTemplate;
+    private final KafkaProperties kafkaProperties;
     private final TransactionTemplate transactionTemplate;
-    private final MedicalFacilityRepository medicalFacilityRepository;
+    private final MedicalFacilityMysqlRepository medicalFacilityMysqlRepository;
     private final FacilityDetailApiService facilityDetailApiService;
     private final AtomicInteger count = new AtomicInteger(0);     // 동작 확인용
 
@@ -55,8 +58,8 @@ public void updateDetailInfo(ConsumerRecord<String, BasicInfoDto> record, Acknow
                     return;
                 }
             }
-            // 개수 확인용
-            kafkaTemplate.send("check", String.valueOf(count.get()));
+            // id만 전달
+            kafkaTemplate.send(kafkaProperties.getMigrationTopic(), new FacilityIdDto(id));
             ack.acknowledge();
         } catch (OpenApiProblemException oe) {
             if (oe.getOpenApiErrorCode() == OpenApiErrorCode.TOO_MANY_TRY) {
@@ -68,7 +71,7 @@ public void updateDetailInfo(ConsumerRecord<String, BasicInfoDto> record, Acknow
                     addRetryCount(record, retryCount);
                     ack.nack(Duration.ofMillis(500));
                 } else {
-                    kafkaTemplate.send("error-check", "-"); // 차후 제거
+                    kafkaTemplate.send("error-check", new FacilityIdDto("-")); // 차후 제거
                     ack.acknowledge();
                 }
             }
@@ -78,21 +81,19 @@ public void updateDetailInfo(ConsumerRecord<String, BasicInfoDto> record, Acknow
         }
     }
 
-    //TODO: 사용할지 말지 결정
     private int getRetryCount(ConsumerRecord<String, BasicInfoDto> record) {
         return record.headers().lastHeader("retry-count") != null
                 ? Integer.parseInt(new String(record.headers().lastHeader("retry-count").value()))
                 : 0;
     }
 
-    //TODO: 사용할지 말지 결정
     private void addRetryCount(ConsumerRecord<String, BasicInfoDto> record, int count) {
         record.headers().add("retry-count", Integer.toString(count).getBytes());
     }
 
 
     private void updateFacilityDetail(FacilityDetailUpdateDto dto) {
-        medicalFacilityRepository.updateDetail(
+        medicalFacilityMysqlRepository.updateDetail(
                 dto.getCode(), dto.getParking(), dto.getParkingEtc(), dto.getTreatmentMon(), dto.getTreatmentTue(), dto.getTreatmentWed(),
                 dto.getTreatmentThu(), dto.getTreatmentFri(), dto.getTreatmentSat(), dto.getTreatmentSun(), dto.getReceiveWeek(),
                 dto.getReceiveSat(), dto.getLunchWeek(), dto.getLunchSat(), dto.getNoTreatmentSun(), dto.getNoTreatmentHoliday(),

diff --git a/consumer/src/main/java/org/healthmap/service/MySqlToMongoConsumer.java b/consumer/src/main/java/org/healthmap/service/MySqlToMongoConsumer.java
@@ -0,0 +1,55 @@
+package org.healthmap.service;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.healthmap.db.mongodb.model.MedicalFacility;
+import org.healthmap.db.mongodb.repository.MedicalFacilityMongoRepository;
+import org.healthmap.db.mysql.model.MedicalFacilityEntity;
+import org.healthmap.db.mysql.repository.MedicalFacilityMysqlRepository;
+import org.healthmap.dto.FacilityIdDto;
+import org.springframework.kafka.annotation.KafkaListener;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.kafka.support.Acknowledgment;
+import org.springframework.stereotype.Service;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class MySqlToMongoConsumer {
+    private final KafkaTemplate<String, String> kafkaTemplate;
+    private final MedicalFacilityMysqlRepository medicalFacilityMysqlRepository;
+    private final MedicalFacilityMongoRepository medicalFacilityMongoRepository;
+    private final AtomicInteger count = new AtomicInteger(0);
+
+
+    @KafkaListener(topics = "${kafka-config.consumer.migration-topic}",
+            groupId = "${kafka-config.consumer.migration-groupId}",
+            containerFactory = "facilityIdContainerFactory")
+    public void mysqlToMongo(ConsumerRecord<String, FacilityIdDto> consumerRecord, Acknowledgment ack) {
+        String id = consumerRecord.value().getId();
+        MedicalFacilityEntity medicalFacilityEntity = medicalFacilityMysqlRepository.findById(id).orElse(null);
+
+        if (medicalFacilityEntity != null) {
+            MedicalFacility medicalFacility = convertEntityToDocument(medicalFacilityEntity);
+            medicalFacilityMongoRepository.save(medicalFacility);
+            count.incrementAndGet();
+            if (count.get() != 0 && count.get() % 100 == 0) {
+                log.info("migration count : {}", count.get());
+            }
+            ack.acknowledge();
+        }
+        kafkaTemplate.send("finished", id);
+    }
+
+    private MedicalFacility convertEntityToDocument(MedicalFacilityEntity entity) {
+        return MedicalFacility.of(entity.getId(), entity.getName(), entity.getAddress(), entity.getPhoneNumber(), entity.getUrl(),
+                entity.getType(), entity.getState(), entity.getCity(), entity.getTown(), entity.getPostNumber(), entity.getCoordinate(),
+                entity.getParking(), entity.getParkingEtc(), entity.getTreatmentMon(), entity.getTreatmentTue(), entity.getTreatmentWed(),
+                entity.getTreatmentThu(), entity.getTreatmentFri(), entity.getTreatmentSat(), entity.getTreatmentSun(),
+                entity.getReceiveWeek(), entity.getReceiveSat(), entity.getLunchWeek(), entity.getLunchSat(), entity.getNoTreatmentSun(),
+                entity.getNoTreatmentHoliday(), entity.getEmergencyDay(), entity.getEmergencyNight(), entity.getCreatedAt(), entity.getUpdatedAt());
+    }
+}
diff --git a/consumer/src/main/resources/application.yml b/consumer/src/main/resources/application.yml
@@ -11,3 +11,9 @@ spring:
   kafka:
     bootstrap-servers: localhost:9093
 
+  data:
+    mongodb:
+      host: localhost
+      port: 27018
+      database: healthmap
+      auto-index-creation: true
diff --git a/db/build.gradle b/db/build.gradle
@@ -17,6 +17,7 @@ dependencies {
 
     runtimeOnly 'com.mysql:mysql-connector-j'
     implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
+    implementation 'org.springframework.boot:spring-boot-starter-data-mongodb'
     implementation 'org.hibernate:hibernate-spatial:6.6.0.Final'
 
     testImplementation platform('org.junit:junit-bom:5.10.0')

diff --git a/db/src/main/java/org/healthmap/db/mongodb/model/MedicalFacility.java b/db/src/main/java/org/healthmap/db/mongodb/model/MedicalFacility.java
@@ -0,0 +1,73 @@
+package org.healthmap.db.mongodb.model;
+
+import jakarta.persistence.Id;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import org.locationtech.jts.geom.Point;
+import org.springframework.data.mongodb.core.geo.GeoJsonPoint;
+import org.springframework.data.mongodb.core.index.GeoSpatialIndexType;
+import org.springframework.data.mongodb.core.index.GeoSpatialIndexed;
+import org.springframework.data.mongodb.core.mapping.Document;
+
+import java.time.LocalDateTime;
+
+@Getter
+@AllArgsConstructor(access = AccessLevel.PROTECTED)
+@NoArgsConstructor
+@Document(collection = "medicalFacility")
+public class MedicalFacility {
+    @Id
+    private String id;
+    private String name;
+    private String address;
+    private String phoneNumber;
+    private String url;
+    private String type;
+    private String state;
+    private String city;
+    private String town;
+    private String postNumber;
+
+    @GeoSpatialIndexed(type = GeoSpatialIndexType.GEO_2DSPHERE) // 지리적 인덱스
+    private GeoJsonPoint coordinate;                            // MongoDB의 GeoJSON Point 타입
+
+    private String parking;
+    private String parkingEtc;
+    private String treatmentMon;
+    private String treatmentTue;
+    private String treatmentWed;
+    private String treatmentThu;
+    private String treatmentFri;
+    private String treatmentSat;
+    private String treatmentSun;
+    private String receiveWeek;
+    private String receiveSat;
+    private String lunchWeek;
+    private String lunchSat;
+    private String noTreatmentSun;
+    private String noTreatmentHoliday;
+    private String emergencyDay;
+    private String emergencyNight;
+    private LocalDateTime createdAt;
+    private LocalDateTime updatedAt;
+
+    public static MedicalFacility of(String id, String name, String address, String phoneNumber, String url, String type, String state, String city,
+                                     String town, String postNumber, Point coordinate, String parking, String parkingEtc, String treatmentMon,
+                                     String treatmentTue, String treatmentWed, String treatmentThu, String treatmentFri, String treatmentSat,
+                                     String treatmentSun, String receiveWeek, String receiveSat, String lunchWeek, String lunchSat,
+                                     String noTreatmentSun, String noTreatmentHoliday, String emergencyDay, String emergencyNight,
+                                     LocalDateTime createdAt, LocalDateTime updatedAt
+    ) {
+        GeoJsonPoint newCoordinate = new GeoJsonPoint(coordinate.getX(), coordinate.getY());
+
+        return new MedicalFacility(
+                id, name, address, phoneNumber, url, type, state, city, town,
+                postNumber, newCoordinate, parking, parkingEtc, treatmentMon, treatmentTue,
+                treatmentWed, treatmentThu, treatmentFri, treatmentSat, treatmentSun,
+                receiveWeek, receiveSat, lunchWeek, lunchSat, noTreatmentSun,
+                noTreatmentHoliday, emergencyDay, emergencyNight, createdAt, updatedAt
+        );
+    }
+}
diff --git a/db/src/main/java/org/healthmap/db/mongodb/repository/MedicalFacilityMongoRepository.java b/db/src/main/java/org/healthmap/db/mongodb/repository/MedicalFacilityMongoRepository.java
@@ -0,0 +1,7 @@
+package org.healthmap.db.mongodb.repository;
+
+import org.healthmap.db.mongodb.model.MedicalFacility;
+import org.springframework.data.mongodb.repository.MongoRepository;
+
+public interface MedicalFacilityMongoRepository extends MongoRepository<MedicalFacility, String> {
+}
diff --git a/...lthmap/db/medicalfacility/BaseEntity.java → .../healthmap/db/mysql/model/BaseEntity.java b/...lthmap/db/medicalfacility/BaseEntity.java → .../healthmap/db/mysql/model/BaseEntity.java
@@ -1,14 +1,16 @@
-package org.healthmap.db.medicalfacility;
+package org.healthmap.db.mysql.model;
 
 import jakarta.persistence.Column;
 import jakarta.persistence.EntityListeners;
 import jakarta.persistence.MappedSuperclass;
+import lombok.Getter;
 import org.springframework.data.annotation.CreatedDate;
 import org.springframework.data.annotation.LastModifiedDate;
 import org.springframework.data.jpa.domain.support.AuditingEntityListener;
 
 import java.time.LocalDateTime;
 
+@Getter
 @MappedSuperclass
 @EntityListeners(AuditingEntityListener.class)
 public class BaseEntity {

diff --git a/...edicalfacility/MedicalFacilityEntity.java → ...db/mysql/model/MedicalFacilityEntity.java b/...edicalfacility/MedicalFacilityEntity.java → ...db/mysql/model/MedicalFacilityEntity.java
@@ -1,4 +1,4 @@
-package org.healthmap.db.medicalfacility;
+package org.healthmap.db.mysql.model;
 
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;

diff --git a/...lity/CustomMedicalFacilityRepository.java → ...tory/CustomMedicalFacilityRepository.java b/...lity/CustomMedicalFacilityRepository.java → ...tory/CustomMedicalFacilityRepository.java
@@ -1,4 +1,4 @@
-package org.healthmap.db.medicalfacility;
+package org.healthmap.db.mysql.repository;
 
 import java.util.List;