Merge branch 'develop' into 1759-periodically-execute-lambdas-to-keep…

…-a-warm-pool-ready

example/full/instance.properties

@@ -121,6 +121,14 @@ sleeper.table.properties.provider.timeout.minutes=60
 # consistent.
 sleeper.tables.index.dynamo.consistent.reads=true
 
+# The amount of memory for lambdas that create batches of tables to run some operation against, eg.
+# create compaction jobs, run garbage collection, perform partition splitting.
+sleeper.table.batching.lambdas.memory=1024
+
+# The timeout in seconds for lambdas that create batches of tables to run some operation against, eg.
+# create compaction jobs, run garbage collection, perform partition splitting.
+sleeper.table.batching.lambdas.timeout.seconds=60
+
 
 ## The following properties relate to standard ingest.
 

java/cdk/src/main/java/sleeper/cdk/stack/CompactionStack.java

@@ -115,6 +115,8 @@
 import static sleeper.configuration.properties.instance.CommonProperty.ACCOUNT;
 import static sleeper.configuration.properties.instance.CommonProperty.ID;
 import static sleeper.configuration.properties.instance.CommonProperty.REGION;
+import static sleeper.configuration.properties.instance.CommonProperty.TABLE_BATCHING_LAMBDAS_MEMORY_IN_MB;
+import static sleeper.configuration.properties.instance.CommonProperty.TABLE_BATCHING_LAMBDAS_TIMEOUT_IN_SECONDS;
 import static sleeper.configuration.properties.instance.CommonProperty.TASK_RUNNER_LAMBDA_MEMORY_IN_MB;
 import static sleeper.configuration.properties.instance.CommonProperty.TASK_RUNNER_LAMBDA_TIMEOUT_IN_SECONDS;
 import static sleeper.configuration.properties.instance.CommonProperty.VPC_ID;
@@ -267,8 +269,8 @@ private void lambdaToCreateCompactionJobsBatchedViaSQS(
                 .functionName(triggerFunctionName)
                 .description("Create batches of tables and send requests to create compaction jobs for those batches")
                 .runtime(JAVA_11)
-                .memorySize(256)
-                .timeout(Duration.seconds(30))
+                .memorySize(instanceProperties.getInt(TABLE_BATCHING_LAMBDAS_MEMORY_IN_MB))
+                .timeout(Duration.seconds(instanceProperties.getInt(TABLE_BATCHING_LAMBDAS_TIMEOUT_IN_SECONDS)))
                 .handler("sleeper.compaction.job.creation.lambda.CreateCompactionJobsTriggerLambda::handleRequest")
                 .environment(environmentVariables)
                 .reservedConcurrentExecutions(1)

java/cdk/src/main/java/sleeper/cdk/stack/GarbageCollectorStack.java

@@ -50,6 +50,8 @@
 import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.GARBAGE_COLLECTOR_QUEUE_URL;
 import static sleeper.configuration.properties.instance.CommonProperty.ID;
 import static sleeper.configuration.properties.instance.CommonProperty.JARS_BUCKET;
+import static sleeper.configuration.properties.instance.CommonProperty.TABLE_BATCHING_LAMBDAS_MEMORY_IN_MB;
+import static sleeper.configuration.properties.instance.CommonProperty.TABLE_BATCHING_LAMBDAS_TIMEOUT_IN_SECONDS;
 import static sleeper.configuration.properties.instance.GarbageCollectionProperty.GARBAGE_COLLECTOR_LAMBDA_MEMORY_IN_MB;
 import static sleeper.configuration.properties.instance.GarbageCollectionProperty.GARBAGE_COLLECTOR_PERIOD_IN_MINUTES;
 
@@ -89,8 +91,8 @@ public GarbageCollectorStack(
                 .runtime(Runtime.JAVA_11)
                 .handler("sleeper.garbagecollector.GarbageCollectorTriggerLambda::handleRequest")
                 .environment(Utils.createDefaultEnvironment(instanceProperties))
-                .memorySize(256)
-                .timeout(Duration.minutes(1))
+                .memorySize(instanceProperties.getInt(TABLE_BATCHING_LAMBDAS_MEMORY_IN_MB))
+                .timeout(Duration.seconds(instanceProperties.getInt(TABLE_BATCHING_LAMBDAS_TIMEOUT_IN_SECONDS)))
                 .logGroup(createLambdaLogGroup(this, "GarbageCollectorTriggerLogGroup", triggerFunctionName, instanceProperties)));
         IFunction handlerFunction = gcJar.buildFunction(this, "GarbageCollectorLambda", builder -> builder
                 .functionName(functionName)

java/cdk/src/main/java/sleeper/cdk/stack/PartitionSplittingStack.java

@@ -61,6 +61,8 @@
 import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.PARTITION_SPLITTING_TABLE_BATCH_QUEUE_URL;
 import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.PARTITION_SPLITTING_TRIGGER_LAMBDA_FUNCTION;
 import static sleeper.configuration.properties.instance.CommonProperty.ID;
+import static sleeper.configuration.properties.instance.CommonProperty.TABLE_BATCHING_LAMBDAS_MEMORY_IN_MB;
+import static sleeper.configuration.properties.instance.CommonProperty.TABLE_BATCHING_LAMBDAS_TIMEOUT_IN_SECONDS;
 import static sleeper.configuration.properties.instance.PartitionSplittingProperty.FIND_PARTITIONS_TO_SPLIT_LAMBDA_MEMORY_IN_MB;
 import static sleeper.configuration.properties.instance.PartitionSplittingProperty.FIND_PARTITIONS_TO_SPLIT_TIMEOUT_IN_SECONDS;
 import static sleeper.configuration.properties.instance.PartitionSplittingProperty.PARTITION_SPLITTING_TRIGGER_PERIOD_IN_MINUTES;
@@ -190,8 +192,8 @@ private void createTriggerFunction(InstanceProperties instanceProperties, Lambda
                 .functionName(triggerFunctionName)
                 .description("Creates batches of Sleeper tables to perform partition splitting for and puts them on a queue to be processed")
                 .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11)
-                .memorySize(256)
-                .timeout(Duration.seconds(30))
+                .memorySize(instanceProperties.getInt(TABLE_BATCHING_LAMBDAS_MEMORY_IN_MB))
+                .timeout(Duration.seconds(instanceProperties.getInt(TABLE_BATCHING_LAMBDAS_TIMEOUT_IN_SECONDS)))
                 .handler("sleeper.splitter.lambda.FindPartitionsToSplitTriggerLambda::handleRequest")
                 .environment(environmentVariables)
                 .reservedConcurrentExecutions(1)

java/cdk/src/main/java/sleeper/cdk/stack/TableMetricsStack.java

@@ -51,6 +51,8 @@
 import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.TABLE_METRICS_RULE;
 import static sleeper.configuration.properties.instance.CommonProperty.ID;
 import static sleeper.configuration.properties.instance.CommonProperty.JARS_BUCKET;
+import static sleeper.configuration.properties.instance.CommonProperty.TABLE_BATCHING_LAMBDAS_MEMORY_IN_MB;
+import static sleeper.configuration.properties.instance.CommonProperty.TABLE_BATCHING_LAMBDAS_TIMEOUT_IN_SECONDS;
 
 public class TableMetricsStack extends NestedStack {
     public TableMetricsStack(
@@ -70,8 +72,8 @@ public TableMetricsStack(
                 .runtime(Runtime.JAVA_11)
                 .handler("sleeper.metrics.TableMetricsTriggerLambda::handleRequest")
                 .environment(Utils.createDefaultEnvironment(instanceProperties))
-                .memorySize(256)
-                .timeout(Duration.minutes(1))
+                .memorySize(instanceProperties.getInt(TABLE_BATCHING_LAMBDAS_MEMORY_IN_MB))
+                .timeout(Duration.seconds(instanceProperties.getInt(TABLE_BATCHING_LAMBDAS_TIMEOUT_IN_SECONDS)))
                 .logGroup(createLambdaLogGroup(this, "MetricsTriggerLogGroup", triggerFunctionName, instanceProperties)));
         IFunction tableMetricsPublisher = metricsJar.buildFunction(this, "MetricsPublisher", builder -> builder
                 .functionName(publishFunctionName)

java/configuration/src/main/java/sleeper/configuration/properties/instance/CommonProperty.java

@@ -216,6 +216,20 @@ public interface CommonProperty {
             .validationPredicate(Utils::isTrueOrFalse)
             .propertyGroup(InstancePropertyGroup.COMMON)
             .build();
+    UserDefinedInstanceProperty TABLE_BATCHING_LAMBDAS_MEMORY_IN_MB = Index.propertyBuilder("sleeper.table.batching.lambdas.memory")
+            .description("The amount of memory for lambdas that create batches of tables to run some operation against, " +
+                    "eg. create compaction jobs, run garbage collection, perform partition splitting.")
+            .defaultValue("1024")
+            .validationPredicate(Utils::isPositiveInteger)
+            .propertyGroup(InstancePropertyGroup.COMMON)
+            .build();
+    UserDefinedInstanceProperty TABLE_BATCHING_LAMBDAS_TIMEOUT_IN_SECONDS = Index.propertyBuilder("sleeper.table.batching.lambdas.timeout.seconds")
+            .description("The timeout in seconds for lambdas that create batches of tables to run some operation against, " +
+                    "eg. create compaction jobs, run garbage collection, perform partition splitting.")
+            .defaultValue("60")
+            .validationPredicate(Utils::isPositiveInteger)
+            .propertyGroup(InstancePropertyGroup.COMMON)
+            .build();
 
     static List<UserDefinedInstanceProperty> getAll() {
         return Index.INSTANCE.getAll();

scripts/templates/instanceproperties.template

@@ -162,6 +162,14 @@ sleeper.table.properties.provider.timeout.minutes=60
 # consistent.
 sleeper.tables.index.dynamo.consistent.reads=true
 
+# The amount of memory for lambdas that create batches of tables to run some operation against, eg.
+# create compaction jobs, run garbage collection, perform partition splitting.
+sleeper.table.batching.lambdas.memory=1024
+
+# The timeout in seconds for lambdas that create batches of tables to run some operation against, eg.
+# create compaction jobs, run garbage collection, perform partition splitting.
+sleeper.table.batching.lambdas.timeout.seconds=60
+
 
 ## The following properties relate to standard ingest.