From d8b7b6f22db2c8375a5620c5561312e08ba9eaac Mon Sep 17 00:00:00 2001 From: kr565370 <112558283+kr565370@users.noreply.github.com> Date: Wed, 13 Mar 2024 15:23:34 +0000 Subject: [PATCH 1/3] Rename sleeper.table common properties to sleeper.common.table --- example/full/instance.properties | 16 ++++++------- .../properties/instance/CommonProperty.java | 23 +++++++++---------- scripts/templates/instanceproperties.template | 16 ++++++------- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/example/full/instance.properties b/example/full/instance.properties index a043f924c3..04e2725063 100644 --- a/example/full/instance.properties +++ b/example/full/instance.properties @@ -111,23 +111,23 @@ sleeper.metadata.s3.dynamo.pointintimerecovery=false # This specifies whether point in time recovery is enabled for the Sleeper table index. This is set on # the DynamoDB tables. -sleeper.tables.index.dynamo.pointintimerecovery=false - -# The timeout in minutes for when the table properties provider cache should be cleared, forcing table -# properties to be reloaded from S3. -sleeper.table.properties.provider.timeout.minutes=60 +sleeper.common.table.index.dynamo.pointintimerecovery=false # This specifies whether queries and scans against the table index DynamoDB tables are strongly # consistent. -sleeper.tables.index.dynamo.consistent.reads=true +sleeper.common.table.index.dynamo.consistent.reads=true + +# The timeout in minutes for when the table properties provider cache should be cleared, forcing table +# properties to be reloaded from S3. +sleeper.common.table.properties.provider.timeout.minutes=60 # The amount of memory for lambdas that create batches of tables to run some operation against, eg. # create compaction jobs, run garbage collection, perform partition splitting. -sleeper.table.batching.lambdas.memory=1024 +sleeper.common.table.batching.lambdas.memory=1024 # The timeout in seconds for lambdas that create batches of tables to run some operation against, eg. # create compaction jobs, run garbage collection, perform partition splitting. -sleeper.table.batching.lambdas.timeout.seconds=60 +sleeper.common.table.batching.lambdas.timeout.seconds=60 ## The following properties relate to standard ingest. diff --git a/java/configuration/src/main/java/sleeper/configuration/properties/instance/CommonProperty.java b/java/configuration/src/main/java/sleeper/configuration/properties/instance/CommonProperty.java index 48694dc303..4907a914a9 100644 --- a/java/configuration/src/main/java/sleeper/configuration/properties/instance/CommonProperty.java +++ b/java/configuration/src/main/java/sleeper/configuration/properties/instance/CommonProperty.java @@ -194,36 +194,35 @@ public interface CommonProperty { .validationPredicate(Utils::isTrueOrFalse) .propertyGroup(InstancePropertyGroup.COMMON) .runCdkDeployWhenChanged(true).build(); - UserDefinedInstanceProperty TABLE_INDEX_DYNAMO_POINT_IN_TIME_RECOVERY = Index.propertyBuilder("sleeper.tables.index.dynamo.pointintimerecovery") + UserDefinedInstanceProperty TABLE_INDEX_DYNAMO_POINT_IN_TIME_RECOVERY = Index.propertyBuilder("sleeper.common.table.index.dynamo.pointintimerecovery") .description("This specifies whether point in time recovery is enabled for the Sleeper table index. " + "This is set on the DynamoDB tables.") .defaultValue("false") .validationPredicate(Utils::isTrueOrFalse) .propertyGroup(InstancePropertyGroup.COMMON) .runCdkDeployWhenChanged(true).build(); - - UserDefinedInstanceProperty TABLE_PROPERTIES_PROVIDER_TIMEOUT_IN_MINS = Index.propertyBuilder("sleeper.table.properties.provider.timeout.minutes") - .description("The timeout in minutes for when the table properties provider cache should be cleared, " + - "forcing table properties to be reloaded from S3.") - .defaultValue("60") - .validationPredicate(Utils::isPositiveInteger) - .propertyGroup(InstancePropertyGroup.COMMON) - .build(); - UserDefinedInstanceProperty TABLE_INDEX_DYNAMO_STRONGLY_CONSISTENT_READS = Index.propertyBuilder("sleeper.tables.index.dynamo.consistent.reads") + UserDefinedInstanceProperty TABLE_INDEX_DYNAMO_STRONGLY_CONSISTENT_READS = Index.propertyBuilder("sleeper.common.table.index.dynamo.consistent.reads") .description("This specifies whether queries and scans against the table index DynamoDB tables " + "are strongly consistent.") .defaultValue("true") .validationPredicate(Utils::isTrueOrFalse) .propertyGroup(InstancePropertyGroup.COMMON) .build(); - UserDefinedInstanceProperty TABLE_BATCHING_LAMBDAS_MEMORY_IN_MB = Index.propertyBuilder("sleeper.table.batching.lambdas.memory") + UserDefinedInstanceProperty TABLE_PROPERTIES_PROVIDER_TIMEOUT_IN_MINS = Index.propertyBuilder("sleeper.common.table.properties.provider.timeout.minutes") + .description("The timeout in minutes for when the table properties provider cache should be cleared, " + + "forcing table properties to be reloaded from S3.") + .defaultValue("60") + .validationPredicate(Utils::isPositiveInteger) + .propertyGroup(InstancePropertyGroup.COMMON) + .build(); + UserDefinedInstanceProperty TABLE_BATCHING_LAMBDAS_MEMORY_IN_MB = Index.propertyBuilder("sleeper.common.table.batching.lambdas.memory") .description("The amount of memory for lambdas that create batches of tables to run some operation against, " + "eg. create compaction jobs, run garbage collection, perform partition splitting.") .defaultValue("1024") .validationPredicate(Utils::isPositiveInteger) .propertyGroup(InstancePropertyGroup.COMMON) .build(); - UserDefinedInstanceProperty TABLE_BATCHING_LAMBDAS_TIMEOUT_IN_SECONDS = Index.propertyBuilder("sleeper.table.batching.lambdas.timeout.seconds") + UserDefinedInstanceProperty TABLE_BATCHING_LAMBDAS_TIMEOUT_IN_SECONDS = Index.propertyBuilder("sleeper.common.table.batching.lambdas.timeout.seconds") .description("The timeout in seconds for lambdas that create batches of tables to run some operation against, " + "eg. create compaction jobs, run garbage collection, perform partition splitting.") .defaultValue("60") diff --git a/scripts/templates/instanceproperties.template b/scripts/templates/instanceproperties.template index 28028b6a65..1b674b3193 100644 --- a/scripts/templates/instanceproperties.template +++ b/scripts/templates/instanceproperties.template @@ -152,23 +152,23 @@ sleeper.metadata.s3.dynamo.pointintimerecovery=false # This specifies whether point in time recovery is enabled for the Sleeper table index. This is set on # the DynamoDB tables. -sleeper.tables.index.dynamo.pointintimerecovery=false - -# The timeout in minutes for when the table properties provider cache should be cleared, forcing table -# properties to be reloaded from S3. -sleeper.table.properties.provider.timeout.minutes=60 +sleeper.common.table.index.dynamo.pointintimerecovery=false # This specifies whether queries and scans against the table index DynamoDB tables are strongly # consistent. -sleeper.tables.index.dynamo.consistent.reads=true +sleeper.common.table.index.dynamo.consistent.reads=true + +# The timeout in minutes for when the table properties provider cache should be cleared, forcing table +# properties to be reloaded from S3. +sleeper.common.table.properties.provider.timeout.minutes=60 # The amount of memory for lambdas that create batches of tables to run some operation against, eg. # create compaction jobs, run garbage collection, perform partition splitting. -sleeper.table.batching.lambdas.memory=1024 +sleeper.common.table.batching.lambdas.memory=1024 # The timeout in seconds for lambdas that create batches of tables to run some operation against, eg. # create compaction jobs, run garbage collection, perform partition splitting. -sleeper.table.batching.lambdas.timeout.seconds=60 +sleeper.common.table.batching.lambdas.timeout.seconds=60 ## The following properties relate to standard ingest. From ba0246a305c8f67eda262ca7090468f0adaae0e1 Mon Sep 17 00:00:00 2001 From: kr565370 <112558283+kr565370@users.noreply.github.com> Date: Wed, 13 Mar 2024 16:08:12 +0000 Subject: [PATCH 2/3] Revert table index property changes --- .../configuration/properties/instance/CommonProperty.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/configuration/src/main/java/sleeper/configuration/properties/instance/CommonProperty.java b/java/configuration/src/main/java/sleeper/configuration/properties/instance/CommonProperty.java index 4907a914a9..5480f58c69 100644 --- a/java/configuration/src/main/java/sleeper/configuration/properties/instance/CommonProperty.java +++ b/java/configuration/src/main/java/sleeper/configuration/properties/instance/CommonProperty.java @@ -194,14 +194,14 @@ public interface CommonProperty { .validationPredicate(Utils::isTrueOrFalse) .propertyGroup(InstancePropertyGroup.COMMON) .runCdkDeployWhenChanged(true).build(); - UserDefinedInstanceProperty TABLE_INDEX_DYNAMO_POINT_IN_TIME_RECOVERY = Index.propertyBuilder("sleeper.common.table.index.dynamo.pointintimerecovery") + UserDefinedInstanceProperty TABLE_INDEX_DYNAMO_POINT_IN_TIME_RECOVERY = Index.propertyBuilder("sleeper.tables.index.dynamo.pointintimerecovery") .description("This specifies whether point in time recovery is enabled for the Sleeper table index. " + "This is set on the DynamoDB tables.") .defaultValue("false") .validationPredicate(Utils::isTrueOrFalse) .propertyGroup(InstancePropertyGroup.COMMON) .runCdkDeployWhenChanged(true).build(); - UserDefinedInstanceProperty TABLE_INDEX_DYNAMO_STRONGLY_CONSISTENT_READS = Index.propertyBuilder("sleeper.common.table.index.dynamo.consistent.reads") + UserDefinedInstanceProperty TABLE_INDEX_DYNAMO_STRONGLY_CONSISTENT_READS = Index.propertyBuilder("sleeper.tables.index.dynamo.consistent.reads") .description("This specifies whether queries and scans against the table index DynamoDB tables " + "are strongly consistent.") .defaultValue("true") From 8e909a8d8be719d42acff0854a7940163e590ef5 Mon Sep 17 00:00:00 2001 From: kr565370 <112558283+kr565370@users.noreply.github.com> Date: Wed, 13 Mar 2024 16:12:38 +0000 Subject: [PATCH 3/3] Remove common from property prefix --- example/full/instance.properties | 10 +++++----- .../properties/instance/CommonProperty.java | 6 +++--- scripts/templates/instanceproperties.template | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/example/full/instance.properties b/example/full/instance.properties index 04e2725063..03a7fb4274 100644 --- a/example/full/instance.properties +++ b/example/full/instance.properties @@ -111,23 +111,23 @@ sleeper.metadata.s3.dynamo.pointintimerecovery=false # This specifies whether point in time recovery is enabled for the Sleeper table index. This is set on # the DynamoDB tables. -sleeper.common.table.index.dynamo.pointintimerecovery=false +sleeper.tables.index.dynamo.pointintimerecovery=false # This specifies whether queries and scans against the table index DynamoDB tables are strongly # consistent. -sleeper.common.table.index.dynamo.consistent.reads=true +sleeper.tables.index.dynamo.consistent.reads=true # The timeout in minutes for when the table properties provider cache should be cleared, forcing table # properties to be reloaded from S3. -sleeper.common.table.properties.provider.timeout.minutes=60 +sleeper.cache.table.properties.provider.timeout.minutes=60 # The amount of memory for lambdas that create batches of tables to run some operation against, eg. # create compaction jobs, run garbage collection, perform partition splitting. -sleeper.common.table.batching.lambdas.memory=1024 +sleeper.batch.table.lambdas.memory=1024 # The timeout in seconds for lambdas that create batches of tables to run some operation against, eg. # create compaction jobs, run garbage collection, perform partition splitting. -sleeper.common.table.batching.lambdas.timeout.seconds=60 +sleeper.batch.table.lambdas.timeout.seconds=60 ## The following properties relate to standard ingest. diff --git a/java/configuration/src/main/java/sleeper/configuration/properties/instance/CommonProperty.java b/java/configuration/src/main/java/sleeper/configuration/properties/instance/CommonProperty.java index 5480f58c69..4b77640654 100644 --- a/java/configuration/src/main/java/sleeper/configuration/properties/instance/CommonProperty.java +++ b/java/configuration/src/main/java/sleeper/configuration/properties/instance/CommonProperty.java @@ -208,21 +208,21 @@ public interface CommonProperty { .validationPredicate(Utils::isTrueOrFalse) .propertyGroup(InstancePropertyGroup.COMMON) .build(); - UserDefinedInstanceProperty TABLE_PROPERTIES_PROVIDER_TIMEOUT_IN_MINS = Index.propertyBuilder("sleeper.common.table.properties.provider.timeout.minutes") + UserDefinedInstanceProperty TABLE_PROPERTIES_PROVIDER_TIMEOUT_IN_MINS = Index.propertyBuilder("sleeper.cache.table.properties.provider.timeout.minutes") .description("The timeout in minutes for when the table properties provider cache should be cleared, " + "forcing table properties to be reloaded from S3.") .defaultValue("60") .validationPredicate(Utils::isPositiveInteger) .propertyGroup(InstancePropertyGroup.COMMON) .build(); - UserDefinedInstanceProperty TABLE_BATCHING_LAMBDAS_MEMORY_IN_MB = Index.propertyBuilder("sleeper.common.table.batching.lambdas.memory") + UserDefinedInstanceProperty TABLE_BATCHING_LAMBDAS_MEMORY_IN_MB = Index.propertyBuilder("sleeper.batch.table.lambdas.memory") .description("The amount of memory for lambdas that create batches of tables to run some operation against, " + "eg. create compaction jobs, run garbage collection, perform partition splitting.") .defaultValue("1024") .validationPredicate(Utils::isPositiveInteger) .propertyGroup(InstancePropertyGroup.COMMON) .build(); - UserDefinedInstanceProperty TABLE_BATCHING_LAMBDAS_TIMEOUT_IN_SECONDS = Index.propertyBuilder("sleeper.common.table.batching.lambdas.timeout.seconds") + UserDefinedInstanceProperty TABLE_BATCHING_LAMBDAS_TIMEOUT_IN_SECONDS = Index.propertyBuilder("sleeper.batch.table.lambdas.timeout.seconds") .description("The timeout in seconds for lambdas that create batches of tables to run some operation against, " + "eg. create compaction jobs, run garbage collection, perform partition splitting.") .defaultValue("60") diff --git a/scripts/templates/instanceproperties.template b/scripts/templates/instanceproperties.template index 1b674b3193..a28499a1fa 100644 --- a/scripts/templates/instanceproperties.template +++ b/scripts/templates/instanceproperties.template @@ -152,23 +152,23 @@ sleeper.metadata.s3.dynamo.pointintimerecovery=false # This specifies whether point in time recovery is enabled for the Sleeper table index. This is set on # the DynamoDB tables. -sleeper.common.table.index.dynamo.pointintimerecovery=false +sleeper.tables.index.dynamo.pointintimerecovery=false # This specifies whether queries and scans against the table index DynamoDB tables are strongly # consistent. -sleeper.common.table.index.dynamo.consistent.reads=true +sleeper.tables.index.dynamo.consistent.reads=true # The timeout in minutes for when the table properties provider cache should be cleared, forcing table # properties to be reloaded from S3. -sleeper.common.table.properties.provider.timeout.minutes=60 +sleeper.cache.table.properties.provider.timeout.minutes=60 # The amount of memory for lambdas that create batches of tables to run some operation against, eg. # create compaction jobs, run garbage collection, perform partition splitting. -sleeper.common.table.batching.lambdas.memory=1024 +sleeper.batch.table.lambdas.memory=1024 # The timeout in seconds for lambdas that create batches of tables to run some operation against, eg. # create compaction jobs, run garbage collection, perform partition splitting. -sleeper.common.table.batching.lambdas.timeout.seconds=60 +sleeper.batch.table.lambdas.timeout.seconds=60 ## The following properties relate to standard ingest.