From e3814c7023164e3d6c45a33587b18addf6c1bbf4 Mon Sep 17 00:00:00 2001 From: Tom Gianos Date: Tue, 18 Apr 2017 16:53:51 -0700 Subject: [PATCH] Modify job cleanup query to limit amount of data returned via id projection (#493) --- .../jpa/repositories/JpaApplicationRepository.java | 2 -- .../core/jpa/repositories/JpaClusterRepository.java | 2 -- .../core/jpa/repositories/JpaCommandRepository.java | 2 -- .../jpa/repositories/JpaJobExecutionRepository.java | 3 +-- .../jpa/repositories/JpaJobMetadataRepository.java | 3 +-- .../genie/core/jpa/repositories/JpaJobRepository.java | 3 +-- .../jpa/repositories/JpaJobRequestRepository.java | 8 +++++--- .../jpa/services/JpaJobPersistenceServiceImpl.java | 11 ++++------- .../com/netflix/genie/GenieCoreTestApplication.java | 2 +- 9 files changed, 13 insertions(+), 23 deletions(-) diff --git a/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaApplicationRepository.java b/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaApplicationRepository.java index d710700500e..edb0e76fd6c 100644 --- a/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaApplicationRepository.java +++ b/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaApplicationRepository.java @@ -18,13 +18,11 @@ import com.netflix.genie.core.jpa.entities.ApplicationEntity; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; -import org.springframework.stereotype.Repository; /** * Application repository. * * @author tgianos */ -@Repository public interface JpaApplicationRepository extends JpaRepository, JpaSpecificationExecutor { } diff --git a/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaClusterRepository.java b/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaClusterRepository.java index 6bd7a931f96..15cc7e0c7bf 100644 --- a/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaClusterRepository.java +++ b/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaClusterRepository.java @@ -18,13 +18,11 @@ import com.netflix.genie.core.jpa.entities.ClusterEntity; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; -import org.springframework.stereotype.Repository; /** * Cluster repository. * * @author tgianos */ -@Repository public interface JpaClusterRepository extends JpaRepository, JpaSpecificationExecutor { } diff --git a/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaCommandRepository.java b/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaCommandRepository.java index b62de58609d..a651b75bcfc 100644 --- a/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaCommandRepository.java +++ b/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaCommandRepository.java @@ -18,13 +18,11 @@ import com.netflix.genie.core.jpa.entities.CommandEntity; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; -import org.springframework.stereotype.Repository; /** * Command repository. * * @author tgianos */ -@Repository public interface JpaCommandRepository extends JpaRepository, JpaSpecificationExecutor { } diff --git a/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaJobExecutionRepository.java b/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaJobExecutionRepository.java index 169d67bbd03..0619fc4e89d 100644 --- a/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaJobExecutionRepository.java +++ b/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaJobExecutionRepository.java @@ -18,7 +18,6 @@ import com.netflix.genie.core.jpa.entities.JobExecutionEntity; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; -import org.springframework.stereotype.Repository; import javax.validation.constraints.NotNull; import java.util.List; @@ -29,10 +28,10 @@ * @author tgianos * @since 3.0.0 */ -@Repository public interface JpaJobExecutionRepository extends JpaRepository, JpaSpecificationExecutor { /** * Deletes all job executions for the given ids. + * * @param ids list of ids for which the job requests should be deleted * @return no. of executions deleted */ diff --git a/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaJobMetadataRepository.java b/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaJobMetadataRepository.java index 1c4cafc577e..05b5d3b5687 100644 --- a/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaJobMetadataRepository.java +++ b/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaJobMetadataRepository.java @@ -18,7 +18,6 @@ import com.netflix.genie.core.jpa.entities.JobMetadataEntity; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; -import org.springframework.stereotype.Repository; import javax.validation.constraints.NotNull; import java.util.List; @@ -29,10 +28,10 @@ * @author tgianos * @since 3.0.0 */ -@Repository public interface JpaJobMetadataRepository extends JpaRepository, JpaSpecificationExecutor { /** * Deletes all job metadatas for the given ids. + * * @param ids list of ids for which the job requests should be deleted * @return no. of metadatas deleted */ diff --git a/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaJobRepository.java b/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaJobRepository.java index 065db2a2640..3ea07f6c89f 100644 --- a/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaJobRepository.java +++ b/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaJobRepository.java @@ -18,7 +18,6 @@ import com.netflix.genie.core.jpa.entities.JobEntity; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; -import org.springframework.stereotype.Repository; import javax.validation.constraints.NotNull; import java.util.List; @@ -28,10 +27,10 @@ * * @author tgianos */ -@Repository public interface JpaJobRepository extends JpaRepository, JpaSpecificationExecutor { /** * Deletes all jobs for the given ids. + * * @param ids list of ids for which the jobs should be deleted * @return no. of jobs deleted */ diff --git a/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaJobRequestRepository.java b/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaJobRequestRepository.java index 3e32935a87b..b1d55f811bb 100644 --- a/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaJobRequestRepository.java +++ b/genie-core/src/main/java/com/netflix/genie/core/jpa/repositories/JpaJobRequestRepository.java @@ -18,7 +18,8 @@ import com.netflix.genie.core.jpa.entities.JobRequestEntity; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; -import org.springframework.stereotype.Repository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import javax.validation.constraints.NotNull; import java.util.Date; @@ -30,7 +31,6 @@ * @author tgianos * @since 3.0.0 */ -@Repository public interface JpaJobRequestRepository extends JpaRepository, JpaSpecificationExecutor { /** @@ -39,10 +39,12 @@ public interface JpaJobRequestRepository extends JpaRepository findByCreatedBefore(@NotNull final Date date); + @Query("SELECT e.id FROM JobRequestEntity e WHERE e.created < :date") + List findByCreatedBefore(@NotNull @Param("date") final Date date); /** * Deletes all job requests for the given ids. + * * @param ids list of ids for which the job requests should be deleted * @return no. of requests deleted */ diff --git a/genie-core/src/main/java/com/netflix/genie/core/jpa/services/JpaJobPersistenceServiceImpl.java b/genie-core/src/main/java/com/netflix/genie/core/jpa/services/JpaJobPersistenceServiceImpl.java index a0066ffc217..a6e349abe9c 100644 --- a/genie-core/src/main/java/com/netflix/genie/core/jpa/services/JpaJobPersistenceServiceImpl.java +++ b/genie-core/src/main/java/com/netflix/genie/core/jpa/services/JpaJobPersistenceServiceImpl.java @@ -28,7 +28,6 @@ import com.netflix.genie.common.exceptions.GenieNotFoundException; import com.netflix.genie.common.exceptions.GeniePreconditionException; import com.netflix.genie.core.jpa.entities.ApplicationEntity; -import com.netflix.genie.core.jpa.entities.BaseEntity; import com.netflix.genie.core.jpa.entities.ClusterEntity; import com.netflix.genie.core.jpa.entities.CommandEntity; import com.netflix.genie.core.jpa.entities.JobEntity; @@ -53,7 +52,6 @@ import javax.validation.constraints.NotNull; import java.util.Date; import java.util.List; -import java.util.stream.Collectors; /** * JPA implementation of the job persistence service. @@ -297,11 +295,10 @@ public void setJobCompletionInformation( */ @Override public long deleteAllJobsCreatedBeforeDate(@NotNull final Date date) { - final List requests = jobRequestRepo.findByCreatedBefore(date); - final List ids = requests.stream().map(BaseEntity::getId).collect(Collectors.toList()); - jobExecutionRepo.deleteByIdIn(ids); - jobMetadataRepository.deleteByIdIn(ids); - jobRepo.deleteByIdIn(ids); + final List ids = this.jobRequestRepo.findByCreatedBefore(date); + this.jobExecutionRepo.deleteByIdIn(ids); + this.jobMetadataRepository.deleteByIdIn(ids); + this.jobRepo.deleteByIdIn(ids); return jobRequestRepo.deleteByIdIn(ids); } diff --git a/genie-core/src/test/java/com/netflix/genie/GenieCoreTestApplication.java b/genie-core/src/test/java/com/netflix/genie/GenieCoreTestApplication.java index d3951dd9c6a..a1a23cb02f1 100644 --- a/genie-core/src/test/java/com/netflix/genie/GenieCoreTestApplication.java +++ b/genie-core/src/test/java/com/netflix/genie/GenieCoreTestApplication.java @@ -91,7 +91,7 @@ public Executor processExecutor() { * Get the jobs dir as a Spring Resource. Will create if it doesn't exist. * * @return The job dir as a resource - * @throws IOException on error reading or creading the directory + * @throws IOException on error reading or creating the directory */ @Bean @ConditionalOnMissingBean