diff --git a/build.gradle b/build.gradle index 2149a45..51dda73 100644 --- a/build.gradle +++ b/build.gradle @@ -31,16 +31,19 @@ task copyConfigSettings(type: Copy) { dependencies { + implementation 'org.springframework.boot:spring-boot-starter-web' implementation 'org.springframework.boot:spring-boot-starter-data-jpa' implementation 'org.springframework.boot:spring-boot-starter-security' - implementation 'org.springframework.boot:spring-boot-starter-web' implementation 'org.springframework.boot:spring-boot-starter-oauth2-client' + implementation 'org.springframework.boot:spring-boot-starter-batch' + implementation 'com.rometools:rome:1.15.0' compileOnly 'org.projectlombok:lombok' runtimeOnly 'com.h2database:h2' annotationProcessor 'org.projectlombok:lombok' testImplementation 'org.springframework.boot:spring-boot-starter-test' testImplementation 'org.springframework.restdocs:spring-restdocs-mockmvc' testImplementation 'org.springframework.security:spring-security-test' + testImplementation 'org.springframework.batch:spring-batch-test' } test { diff --git a/src/main/java/com/study/platform/PlatformApplication.java b/src/main/java/com/study/platform/PlatformApplication.java index 51f66df..ca4cb0e 100644 --- a/src/main/java/com/study/platform/PlatformApplication.java +++ b/src/main/java/com/study/platform/PlatformApplication.java @@ -1,12 +1,14 @@ package com.study.platform; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +@EnableBatchProcessing @SpringBootApplication public class PlatformApplication { - public static void main(String[] args) { + public static void main(final String[] args) { SpringApplication.run(PlatformApplication.class, args); } diff --git a/src/main/java/com/study/platform/blog/batch/job/FeedJobConfiguration.java b/src/main/java/com/study/platform/blog/batch/job/FeedJobConfiguration.java new file mode 100644 index 0000000..c3d2751 --- /dev/null +++ b/src/main/java/com/study/platform/blog/batch/job/FeedJobConfiguration.java @@ -0,0 +1,75 @@ +package com.study.platform.blog.batch.job; + +import com.study.platform.blog.domain.Blog; +import com.study.platform.blog.domain.Feed; +import com.study.platform.blog.service.FeedReader; +import com.study.platform.blog.service.dto.FeedDto; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.database.JpaItemWriter; +import org.springframework.batch.item.database.JpaPagingItemReader; +import org.springframework.batch.item.database.builder.JpaPagingItemReaderBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import javax.persistence.EntityManagerFactory; +import java.util.List; +import java.util.stream.Collectors; + +@Slf4j +@RequiredArgsConstructor +@Configuration +public class FeedJobConfiguration { + private final JobBuilderFactory jobBuilderFactory; + private final StepBuilderFactory stepBuilderFactory; + private final EntityManagerFactory entityManagerFactory; + private final FeedReader feedReader; + + @Bean + public Job feedJob() { + return jobBuilderFactory.get("feedJob") + .start(feedStop1()) + .build(); + } + + @Bean + public Step feedStop1() { + return this.stepBuilderFactory.get("feedStep1") + .>chunk(100) + .reader(feedItemReader()) + .processor(feedItemProcessor()) + .writer(feedItemWriter()) + .build(); + } + + private JpaPagingItemReader feedItemReader() { + return new JpaPagingItemReaderBuilder() + .name("feedItemReader") + .entityManagerFactory(entityManagerFactory) + .pageSize(100) + .queryString("SELECT b FROM Blog b") + .build(); + } + + private ItemProcessor> feedItemProcessor() { + return blog -> { + final List feeds = feedReader.getFeeds(blog.getLink()); + + return feeds.stream() + .map(feedDto -> feedDto.toEntity(blog)) + .collect(Collectors.toList()); + }; + } + + private JpaItemListWriter feedItemWriter() { + final JpaItemWriter jpaItemWriter = new JpaItemWriter<>(); + jpaItemWriter.setEntityManagerFactory(entityManagerFactory); + + return new JpaItemListWriter<>(jpaItemWriter); + } +} diff --git a/src/main/java/com/study/platform/blog/batch/job/JpaItemListWriter.java b/src/main/java/com/study/platform/blog/batch/job/JpaItemListWriter.java new file mode 100644 index 0000000..a4d51be --- /dev/null +++ b/src/main/java/com/study/platform/blog/batch/job/JpaItemListWriter.java @@ -0,0 +1,23 @@ +package com.study.platform.blog.batch.job; + +import lombok.RequiredArgsConstructor; +import org.springframework.batch.item.database.JpaItemWriter; + +import java.util.ArrayList; +import java.util.List; + +@RequiredArgsConstructor +public class JpaItemListWriter extends JpaItemWriter> { + private final JpaItemWriter jpaItemWriter; + + @Override + public void write(final List> items) { + final List list = new ArrayList<>(); + + for (final List item : items) { + list.addAll(item); + } + + this.jpaItemWriter.write(list); + } +} diff --git a/src/main/java/com/study/platform/blog/domain/Feed.java b/src/main/java/com/study/platform/blog/domain/Feed.java index 920829f..89e5f88 100644 --- a/src/main/java/com/study/platform/blog/domain/Feed.java +++ b/src/main/java/com/study/platform/blog/domain/Feed.java @@ -1,21 +1,14 @@ package com.study.platform.blog.domain; -import java.net.URL; -import java.time.LocalDateTime; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.JoinColumn; -import javax.persistence.ManyToOne; - import lombok.AccessLevel; import lombok.Builder; import lombok.Getter; import lombok.NoArgsConstructor; +import javax.persistence.*; +import java.net.URL; +import java.util.Date; + @Entity @NoArgsConstructor(access = AccessLevel.PROTECTED) @Getter @@ -30,16 +23,17 @@ public class Feed { private URL link; + @Lob private String description; @ManyToOne @JoinColumn(name = "blog_id") private Blog blog; - private LocalDateTime pubDate; + private Date pubDate; @Builder - public Feed(Long id, String title, URL link, String description, Blog blog, LocalDateTime pubDate) { + public Feed(final Long id, final String title, final URL link, final String description, final Blog blog, final Date pubDate) { this.id = id; this.title = title; this.link = link; diff --git a/src/main/java/com/study/platform/blog/service/FeedReader.java b/src/main/java/com/study/platform/blog/service/FeedReader.java new file mode 100644 index 0000000..603b164 --- /dev/null +++ b/src/main/java/com/study/platform/blog/service/FeedReader.java @@ -0,0 +1,18 @@ +package com.study.platform.blog.service; + +import java.io.IOException; +import java.net.URL; +import java.util.List; + +import org.springframework.stereotype.Service; + +import com.rometools.rome.feed.synd.SyndEntry; +import com.rometools.rome.io.FeedException; +import com.study.platform.blog.service.dto.FeedDto; + +@Service +public interface FeedReader { + List getFeeds(URL rssUrl) throws IOException, FeedException; + + FeedDto getFeed(SyndEntry syndEntry) throws IOException, FeedException; +} diff --git a/src/main/java/com/study/platform/blog/service/FeedReaderImpl.java b/src/main/java/com/study/platform/blog/service/FeedReaderImpl.java new file mode 100644 index 0000000..d9e9312 --- /dev/null +++ b/src/main/java/com/study/platform/blog/service/FeedReaderImpl.java @@ -0,0 +1,44 @@ +package com.study.platform.blog.service; + +import com.rometools.rome.feed.synd.SyndEntry; +import com.rometools.rome.feed.synd.SyndFeed; +import com.rometools.rome.io.FeedException; +import com.rometools.rome.io.SyndFeedInput; +import com.rometools.rome.io.XmlReader; +import com.study.platform.blog.service.dto.FeedDto; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.net.URL; +import java.util.List; +import java.util.stream.Collectors; + +@Component +public class FeedReaderImpl implements FeedReader { + + @Override + public List getFeeds(final URL rssUrl) throws IOException, FeedException { + final SyndFeedInput syndFeedInput = new SyndFeedInput(); + final SyndFeed syndFeed = syndFeedInput.build(new XmlReader(rssUrl)); + + final List feeds = syndFeed.getEntries(); + + return feeds.stream() + .map(this::getFeed) + .collect(Collectors.toList()); + } + + @Override + public FeedDto getFeed(final SyndEntry syndEntry) { + try { + return FeedDto.builder() + .title(syndEntry.getTitle()) + .link(new URL(syndEntry.getLink())) + .description(syndEntry.getDescription().getValue()) + .pubDate(syndEntry.getPublishedDate()) + .build(); + } catch (final Exception e) { + throw new IllegalArgumentException("해당 feed를 찾을수 없습니다."); + } + } +} diff --git a/src/main/java/com/study/platform/blog/service/dto/FeedDto.java b/src/main/java/com/study/platform/blog/service/dto/FeedDto.java new file mode 100644 index 0000000..b66c741 --- /dev/null +++ b/src/main/java/com/study/platform/blog/service/dto/FeedDto.java @@ -0,0 +1,38 @@ +package com.study.platform.blog.service.dto; + +import com.study.platform.blog.domain.Blog; +import com.study.platform.blog.domain.Feed; +import lombok.Builder; +import lombok.Getter; + +import java.net.URL; +import java.util.Date; + +@Getter +public class FeedDto { + private final String title; + + private final URL link; + + private final String description; + + private final Date pubDate; + + @Builder + public FeedDto(final String title, final URL link, final String description, final Date pubDate) { + this.title = title; + this.link = link; + this.description = description; + this.pubDate = pubDate; + } + + public Feed toEntity(final Blog blog) { + return Feed.builder() + .blog(blog) + .title(this.title) + .link(this.link) + .description(this.description) + .pubDate(this.pubDate) + .build(); + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 44c4f72..7124b15 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,6 +1,6 @@ spring: profiles: active: local - h2: - console: - enabled: true \ No newline at end of file + include: + - oauth + - db \ No newline at end of file diff --git a/src/test/java/com/study/platform/TestJobConfiguration.java b/src/test/java/com/study/platform/TestJobConfiguration.java new file mode 100644 index 0000000..72c082b --- /dev/null +++ b/src/test/java/com/study/platform/TestJobConfiguration.java @@ -0,0 +1,16 @@ +package com.study.platform; + +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.test.JobLauncherTestUtils; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@EnableBatchProcessing +@Configuration +public class TestJobConfiguration { + + @Bean + public JobLauncherTestUtils jobLauncherTestUtils() { + return new JobLauncherTestUtils(); + } +} diff --git a/src/test/java/com/study/platform/blog/batch/job/FeedJobConfigurationTest.java b/src/test/java/com/study/platform/blog/batch/job/FeedJobConfigurationTest.java new file mode 100644 index 0000000..9e2806c --- /dev/null +++ b/src/test/java/com/study/platform/blog/batch/job/FeedJobConfigurationTest.java @@ -0,0 +1,47 @@ +package com.study.platform.blog.batch.job; + +import com.study.platform.blog.domain.Blog; +import com.study.platform.blog.domain.BlogRepository; +import com.study.platform.blog.domain.FeedRepository; +import org.junit.jupiter.api.Test; +import org.springframework.batch.core.BatchStatus; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.test.JobLauncherTestUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; + +import java.net.URL; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertAll; + +@SpringBootTest +@TestPropertySource(properties = {"job.name" + "feedJob"}) +class FeedJobConfigurationTest { + + @Autowired + private JobLauncherTestUtils jobLauncherTestUtils; + + @Autowired + private BlogRepository blogRepository; + + @Autowired + private FeedRepository feedRepository; + + @Test + void blogFeedBatch() throws Exception { + // given + this.blogRepository.save(Blog.builder() + .title("test") + .link(new URL("https://rutgo-letsgo.tistory.com/rss")) + .build()); + + final JobExecution jobExecution = jobLauncherTestUtils.launchJob(); + + assertAll( + () -> assertThat(jobExecution.getStatus()).isEqualTo(BatchStatus.COMPLETED), + () -> assertThat(feedRepository.findAll()).hasSize(50) + ); + } +} \ No newline at end of file diff --git a/src/test/java/com/study/platform/blog/service/FeedReaderImplTest.java b/src/test/java/com/study/platform/blog/service/FeedReaderImplTest.java new file mode 100644 index 0000000..4fac7d2 --- /dev/null +++ b/src/test/java/com/study/platform/blog/service/FeedReaderImplTest.java @@ -0,0 +1,29 @@ +package com.study.platform.blog.service; + +import static org.assertj.core.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.*; + +import java.io.IOException; +import java.net.URL; +import java.util.List; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import com.rometools.rome.io.FeedException; +import com.study.platform.blog.service.dto.FeedDto; + +class FeedReaderImplTest { + + private final FeedReader feedReader = new FeedReaderImpl(); + + @DisplayName("BlogFeed 전체 가져오는 테스트") + @Test + void getFeeds() throws IOException, FeedException { + List feeds = feedReader.getFeeds(new URL("https://rutgo-letsgo.tistory.com/rss")); + + assertAll( + () -> assertThat(feeds).isNotEmpty() + ); + } +}