feat: create partitions if needed

Do not send create partition commands to the HSMC if partition already exists. Reduces traffic on the HMS.
mcneiljt · Dec 27, 2023 · 35063a3 · 35063a3
1 parent f47bf6c
commit 35063a3
Showing 1 changed file with 7 additions and 2 deletions.
diff --git a/src/main/java/com/mcneilio/shokuyoku/driver/BasicEventDriver.java b/src/main/java/com/mcneilio/shokuyoku/driver/BasicEventDriver.java
@@ -1,6 +1,6 @@
 package com.mcneilio.shokuyoku.driver;
 
-
+import com.mcneilio.shokuyoku.util.PartitionManager;
 import com.mcneilio.shokuyoku.util.Statsd;
 import com.timgroup.statsd.StatsDClient;
 import org.apache.hadoop.conf.Configuration;
@@ -154,7 +154,12 @@ public String flush(boolean deleteFile) {
                 if (storageDriver != null){
                     storageDriver.addFile(date,  eventName, fileName, Paths.get(fileName));
                 }
-                //TODO: create hive partition
+
+                String s3_prefix = System.getenv("S3_PREFIX");
+                String prefix = s3_prefix.endsWith("/") ? s3_prefix : s3_prefix + "/";
+                String tablePath = "s3a://" + System.getenv("S3_BUCKET") + "/" + prefix + System.getenv("HIVE_DATABASE");
+                PartitionManager.createPartitionIfNeeded(tablePath, eventName, date);
+
                 if(deleteFile)
                     new File(fileName).delete();
                 this.fileName = null;