From 9d069aaaca9e1f1a77ee385223e789f461886061 Mon Sep 17 00:00:00 2001 From: kr565370 <112558283+kr565370@users.noreply.github.com> Date: Wed, 13 Dec 2023 17:10:34 +0000 Subject: [PATCH] Rename force compaction to compact all files --- docs/06-status.md | 4 ++-- ...eCreateCompactionJobs.java => CompactAllFiles.java} | 6 +++--- .../sleeper/compaction/job/creation/CreateJobs.java | 10 +++++----- .../compaction/job/creation/CreateJobsTest.java | 2 +- .../drivers/compaction/CompactionDriver.java | 2 +- .../utility/{forceCreateJobs.sh => compactAllFiles.sh} | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) rename java/clients/src/main/java/sleeper/clients/status/update/{ForceCreateCompactionJobs.java => CompactAllFiles.java} (95%) rename scripts/utility/{forceCreateJobs.sh => compactAllFiles.sh} (94%) diff --git a/docs/06-status.md b/docs/06-status.md index cd198b622a..85631133e4 100644 --- a/docs/06-status.md +++ b/docs/06-status.md @@ -127,12 +127,12 @@ To run this client you can run the following command: sleeper deployment utility/adminClient.sh ${INSTANCE_ID} ``` -## Force compaction job creation +## Compact all files If you want to fully compact all files in leaf partitions, but the compaction strategy is not compacting files in a partition, you can run the following script to force compactions to be created for files in leaf partitions that were skipped by the compaction strategy: ```bash -sleeper deployment utility/forceCreateJobs.sh ${NSTANCE_ID} +sleeper deployment utility/compactAllFiles.sh ${NSTANCE_ID} ``` \ No newline at end of file diff --git a/java/clients/src/main/java/sleeper/clients/status/update/ForceCreateCompactionJobs.java b/java/clients/src/main/java/sleeper/clients/status/update/CompactAllFiles.java similarity index 95% rename from java/clients/src/main/java/sleeper/clients/status/update/ForceCreateCompactionJobs.java rename to java/clients/src/main/java/sleeper/clients/status/update/CompactAllFiles.java index bfed306ed4..894793b1ee 100644 --- a/java/clients/src/main/java/sleeper/clients/status/update/ForceCreateCompactionJobs.java +++ b/java/clients/src/main/java/sleeper/clients/status/update/CompactAllFiles.java @@ -38,8 +38,8 @@ import java.io.IOException; -public class ForceCreateCompactionJobs { - private ForceCreateCompactionJobs() { +public class CompactAllFiles { + private CompactAllFiles() { } public static void main(String[] args) throws ObjectFactoryException, StateStoreException, IOException { @@ -59,7 +59,7 @@ public static void main(String[] args) throws ObjectFactoryException, StateStore Configuration conf = HadoopConfigurationProvider.getConfigurationForClient(instanceProperties); StateStoreProvider stateStoreProvider = new StateStoreProvider(dynamoDBClient, instanceProperties, conf); CompactionJobStatusStore jobStatusStore = CompactionJobStatusStoreFactory.getStatusStore(dynamoDBClient, instanceProperties); - CreateJobs jobCreator = CreateJobs.forceCompaction( + CreateJobs jobCreator = CreateJobs.compactAllFiles( new ObjectFactory(instanceProperties, s3Client, "/tmp"), instanceProperties, tablePropertiesProvider, stateStoreProvider, new SendCompactionJobToSqs(instanceProperties, sqsClient)::send, jobStatusStore); diff --git a/java/compaction/compaction-job-creation/src/main/java/sleeper/compaction/job/creation/CreateJobs.java b/java/compaction/compaction-job-creation/src/main/java/sleeper/compaction/job/creation/CreateJobs.java index 3cae8af150..f3cff23b96 100644 --- a/java/compaction/compaction-job-creation/src/main/java/sleeper/compaction/job/creation/CreateJobs.java +++ b/java/compaction/compaction-job-creation/src/main/java/sleeper/compaction/job/creation/CreateJobs.java @@ -65,7 +65,7 @@ public class CreateJobs { private final TablePropertiesProvider tablePropertiesProvider; private final StateStoreProvider stateStoreProvider; private final CompactionJobStatusStore jobStatusStore; - private final boolean forceCreateJobs; + private final boolean compactAllFiles; private CreateJobs(ObjectFactory objectFactory, InstanceProperties instanceProperties, @@ -73,17 +73,17 @@ private CreateJobs(ObjectFactory objectFactory, StateStoreProvider stateStoreProvider, JobSender jobSender, CompactionJobStatusStore jobStatusStore, - boolean forceCreateJobs) { + boolean compactAllFiles) { this.objectFactory = objectFactory; this.instanceProperties = instanceProperties; this.jobSender = jobSender; this.tablePropertiesProvider = tablePropertiesProvider; this.stateStoreProvider = stateStoreProvider; this.jobStatusStore = jobStatusStore; - this.forceCreateJobs = forceCreateJobs; + this.compactAllFiles = compactAllFiles; } - public static CreateJobs forceCompaction(ObjectFactory objectFactory, + public static CreateJobs compactAllFiles(ObjectFactory objectFactory, InstanceProperties instanceProperties, TablePropertiesProvider tablePropertiesProvider, StateStoreProvider stateStoreProvider, @@ -134,7 +134,7 @@ public void createJobsForTable(TableProperties tableProperties) throws StateStor List compactionJobs = compactionStrategy.createCompactionJobs(activeFileInfosWithJobId, activeFileInfosWithNoJobId, allPartitions); LOGGER.info("Used {} to create {} compaction jobs for table {}", compactionStrategy.getClass().getSimpleName(), compactionJobs.size(), tableId); - if (forceCreateJobs) { + if (compactAllFiles) { createJobsFromLeftoverFiles(tableProperties, activeFileInfosWithNoJobId, allPartitions, compactionJobs); } for (CompactionJob compactionJob : compactionJobs) { diff --git a/java/compaction/compaction-job-creation/src/test/java/sleeper/compaction/job/creation/CreateJobsTest.java b/java/compaction/compaction-job-creation/src/test/java/sleeper/compaction/job/creation/CreateJobsTest.java index 268bafeaf7..ed38a9eae1 100644 --- a/java/compaction/compaction-job-creation/src/test/java/sleeper/compaction/job/creation/CreateJobsTest.java +++ b/java/compaction/compaction-job-creation/src/test/java/sleeper/compaction/job/creation/CreateJobsTest.java @@ -322,7 +322,7 @@ private List createJobs() throws Exception { private List forceCreateJobs() throws Exception { List compactionJobs = new ArrayList<>(); - CreateJobs createJobs = CreateJobs.forceCompaction(ObjectFactory.noUserJars(), instanceProperties, + CreateJobs createJobs = CreateJobs.compactAllFiles(ObjectFactory.noUserJars(), instanceProperties, new FixedTablePropertiesProvider(tableProperties), new FixedStateStoreProvider(tableProperties, stateStore), compactionJobs::add, jobStatusStore); diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/compaction/CompactionDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/compaction/CompactionDriver.java index fb7d8b2f25..231ac62fc9 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/compaction/CompactionDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/compaction/CompactionDriver.java @@ -84,7 +84,7 @@ public List forceCreateJobsGetIds() { CompactionJobStatusStore store = CompactionJobStatusStoreFactory .getStatusStoreWithStronglyConsistentReads(dynamoDBClient, instance.getInstanceProperties()); Set jobsBefore = allJobIds(store).collect(Collectors.toSet()); - CreateJobs createJobs = CreateJobs.forceCompaction( + CreateJobs createJobs = CreateJobs.compactAllFiles( ObjectFactory.noUserJars(), instance.getInstanceProperties(), instance.getTablePropertiesProvider(), instance.getStateStoreProvider(), new SendCompactionJobToSqs(instance.getInstanceProperties(), sqsClient)::send, store); diff --git a/scripts/utility/forceCreateJobs.sh b/scripts/utility/compactAllFiles.sh similarity index 94% rename from scripts/utility/forceCreateJobs.sh rename to scripts/utility/compactAllFiles.sh index 4281d4a8e1..0107d5c145 100755 --- a/scripts/utility/forceCreateJobs.sh +++ b/scripts/utility/compactAllFiles.sh @@ -35,4 +35,4 @@ VERSION=$(cat "${TEMPLATE_DIR}/version.txt") echo "-------------------------------------------------------" echo "Forcing compaction job creation" echo "-------------------------------------------------------" -java -cp "${JAR_DIR}/clients-${VERSION}-utility.jar" sleeper.clients.status.update.ForceCreateCompactionJobs "${INSTANCE_ID}" +java -cp "${JAR_DIR}/clients-${VERSION}-utility.jar" sleeper.clients.status.update.CompactAllFiles "${INSTANCE_ID}"