diff --git a/example/full/instance.properties b/example/full/instance.properties index 775e8ca27a..609a5ab2f2 100644 --- a/example/full/instance.properties +++ b/example/full/instance.properties @@ -96,6 +96,10 @@ sleeper.metadata.dynamo.pointintimerecovery=false # revision DynamoDB table. sleeper.metadata.s3.dynamo.pointintimerecovery=false +# This specifies whether point in time recovery is enabled for the Sleeper table index. This is set on +# the DynamoDB tables. +sleeper.tables.index.dynamo.pointintimerecovery=false + # The timeout in minutes for when the table properties provider cache should be cleared, forcing table # properties to be reloaded from S3. sleeper.table.properties.provider.timeout.minutes=60 diff --git a/java/cdk/src/main/java/sleeper/cdk/SleeperCdkApp.java b/java/cdk/src/main/java/sleeper/cdk/SleeperCdkApp.java index a15657c0fe..ca1bf09f04 100644 --- a/java/cdk/src/main/java/sleeper/cdk/SleeperCdkApp.java +++ b/java/cdk/src/main/java/sleeper/cdk/SleeperCdkApp.java @@ -39,6 +39,7 @@ import sleeper.cdk.stack.S3StateStoreStack; import sleeper.cdk.stack.StateStoreStacks; import sleeper.cdk.stack.TableDataStack; +import sleeper.cdk.stack.TableIndexStack; import sleeper.cdk.stack.TableMetricsStack; import sleeper.cdk.stack.TopicStack; import sleeper.cdk.stack.VpcStack; @@ -124,6 +125,7 @@ public void create() { // Stack for tables dataStack = new TableDataStack(this, "TableData", instanceProperties); + new TableIndexStack(this, "TableIndex", instanceProperties); stateStoreStacks = new StateStoreStacks( new DynamoDBStateStoreStack(this, "DynamoDBStateStore", instanceProperties), new S3StateStoreStack(this, "S3StateStore", instanceProperties, dataStack)); diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/TableIndexStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/TableIndexStack.java new file mode 100644 index 0000000000..7baa52073c --- /dev/null +++ b/java/cdk/src/main/java/sleeper/cdk/stack/TableIndexStack.java @@ -0,0 +1,81 @@ +/* + * Copyright 2022-2023 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.cdk.stack; + +import software.amazon.awscdk.NestedStack; +import software.amazon.awscdk.RemovalPolicy; +import software.amazon.awscdk.services.dynamodb.Attribute; +import software.amazon.awscdk.services.dynamodb.AttributeType; +import software.amazon.awscdk.services.dynamodb.BillingMode; +import software.amazon.awscdk.services.dynamodb.ITable; +import software.amazon.awscdk.services.dynamodb.Table; +import software.constructs.Construct; + +import sleeper.configuration.properties.instance.InstanceProperties; +import sleeper.table.index.dynamodb.DynamoDBTableIndex; + +import static sleeper.cdk.Utils.removalPolicy; +import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.TABLE_ID_INDEX_DYNAMO_TABLENAME; +import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.TABLE_NAME_INDEX_DYNAMO_TABLENAME; +import static sleeper.configuration.properties.instance.CommonProperty.ID; +import static sleeper.configuration.properties.instance.CommonProperty.TABLE_INDEX_DYNAMO_POINT_IN_TIME_RECOVERY; + +public class TableIndexStack extends NestedStack { + + private final ITable indexByNameDynamoTable; + private final ITable indexByIdDynamoTable; + + public TableIndexStack(Construct scope, String id, InstanceProperties instanceProperties) { + super(scope, id); + String instanceId = instanceProperties.get(ID); + RemovalPolicy removalPolicy = removalPolicy(instanceProperties); + + indexByNameDynamoTable = Table.Builder + .create(this, "IndexByNameDynamoDBTable") + .tableName(String.join("-", "sleeper", instanceId, "table-index-by-name")) + .removalPolicy(removalPolicy) + .billingMode(BillingMode.PAY_PER_REQUEST) + .partitionKey(Attribute.builder() + .name(DynamoDBTableIndex.TABLE_NAME_FIELD) + .type(AttributeType.STRING) + .build()) + .pointInTimeRecovery(instanceProperties.getBoolean(TABLE_INDEX_DYNAMO_POINT_IN_TIME_RECOVERY)) + .build(); + instanceProperties.set(TABLE_NAME_INDEX_DYNAMO_TABLENAME, indexByNameDynamoTable.getTableName()); + + indexByIdDynamoTable = Table.Builder + .create(this, "IndexByIdDynamoDBTable") + .tableName(String.join("-", "sleeper", instanceId, "table-index-by-id")) + .removalPolicy(removalPolicy) + .billingMode(BillingMode.PAY_PER_REQUEST) + .partitionKey(Attribute.builder() + .name(DynamoDBTableIndex.TABLE_ID_FIELD) + .type(AttributeType.STRING) + .build()) + .pointInTimeRecovery(instanceProperties.getBoolean(TABLE_INDEX_DYNAMO_POINT_IN_TIME_RECOVERY)) + .build(); + instanceProperties.set(TABLE_ID_INDEX_DYNAMO_TABLENAME, indexByIdDynamoTable.getTableName()); + } + + public ITable getIndexByNameDynamoTable() { + return indexByNameDynamoTable; + } + + public ITable getIndexByIdDynamoTable() { + return indexByIdDynamoTable; + } +} diff --git a/java/configuration/src/main/java/sleeper/configuration/properties/instance/CdkDefinedInstanceProperty.java b/java/configuration/src/main/java/sleeper/configuration/properties/instance/CdkDefinedInstanceProperty.java index f305393f34..05edbc3773 100644 --- a/java/configuration/src/main/java/sleeper/configuration/properties/instance/CdkDefinedInstanceProperty.java +++ b/java/configuration/src/main/java/sleeper/configuration/properties/instance/CdkDefinedInstanceProperty.java @@ -46,6 +46,16 @@ public interface CdkDefinedInstanceProperty extends InstanceProperty { .propertyGroup(InstancePropertyGroup.COMMON) .build(); + // Tables + CdkDefinedInstanceProperty TABLE_NAME_INDEX_DYNAMO_TABLENAME = Index.propertyBuilder("sleeper.tables.name.index.dynamo.table") + .description("The name of the DynamoDB table indexing tables by their name.") + .propertyGroup(InstancePropertyGroup.COMMON) + .build(); + CdkDefinedInstanceProperty TABLE_ID_INDEX_DYNAMO_TABLENAME = Index.propertyBuilder("sleeper.tables.id.index.dynamo.table") + .description("The name of the DynamoDB table indexing tables by their ID.") + .propertyGroup(InstancePropertyGroup.COMMON) + .build(); + // DynamoDBStateStore CdkDefinedInstanceProperty ACTIVE_FILEINFO_TABLENAME = Index.propertyBuilder("sleeper.metadata.dynamo.active.table") .description("The name of the DynamoDB table holding metadata of active files in Sleeper tables.") diff --git a/java/configuration/src/main/java/sleeper/configuration/properties/instance/CommonProperty.java b/java/configuration/src/main/java/sleeper/configuration/properties/instance/CommonProperty.java index 68976a927c..3ace0b0c8a 100644 --- a/java/configuration/src/main/java/sleeper/configuration/properties/instance/CommonProperty.java +++ b/java/configuration/src/main/java/sleeper/configuration/properties/instance/CommonProperty.java @@ -173,6 +173,13 @@ public interface CommonProperty { .validationPredicate(Utils::isTrueOrFalse) .propertyGroup(InstancePropertyGroup.COMMON) .runCdkDeployWhenChanged(true).build(); + UserDefinedInstanceProperty TABLE_INDEX_DYNAMO_POINT_IN_TIME_RECOVERY = Index.propertyBuilder("sleeper.tables.index.dynamo.pointintimerecovery") + .description("This specifies whether point in time recovery is enabled for the Sleeper table index. " + + "This is set on the DynamoDB tables.") + .defaultValue("false") + .validationPredicate(Utils::isTrueOrFalse) + .propertyGroup(InstancePropertyGroup.COMMON) + .runCdkDeployWhenChanged(true).build(); UserDefinedInstanceProperty TABLE_PROPERTIES_PROVIDER_TIMEOUT_IN_MINS = Index.propertyBuilder("sleeper.table.properties.provider.timeout.minutes") .description("The timeout in minutes for when the table properties provider cache should be cleared, " + diff --git a/java/configuration/src/test/java/sleeper/configuration/properties/InstancePropertiesTestHelper.java b/java/configuration/src/test/java/sleeper/configuration/properties/InstancePropertiesTestHelper.java index 47b77b22cf..632ef4ee8d 100644 --- a/java/configuration/src/test/java/sleeper/configuration/properties/InstancePropertiesTestHelper.java +++ b/java/configuration/src/test/java/sleeper/configuration/properties/InstancePropertiesTestHelper.java @@ -32,6 +32,8 @@ import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.PARTITION_TABLENAME; import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.READY_FOR_GC_FILEINFO_TABLENAME; import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.REVISION_TABLENAME; +import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.TABLE_ID_INDEX_DYNAMO_TABLENAME; +import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.TABLE_NAME_INDEX_DYNAMO_TABLENAME; import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.VERSION; import static sleeper.configuration.properties.instance.CommonProperty.ACCOUNT; import static sleeper.configuration.properties.instance.CommonProperty.ID; @@ -80,6 +82,8 @@ public static InstanceProperties createTestInstanceProperties() { instanceProperties.set(READY_FOR_GC_FILEINFO_TABLENAME, id + "-rfgcf"); instanceProperties.set(PARTITION_TABLENAME, id + "-p"); instanceProperties.set(REVISION_TABLENAME, id + "-rv"); + instanceProperties.set(TABLE_NAME_INDEX_DYNAMO_TABLENAME, id + "-tni"); + instanceProperties.set(TABLE_ID_INDEX_DYNAMO_TABLENAME, id + "-tii"); return instanceProperties; } diff --git a/java/core/src/main/java/sleeper/core/table/TableAlreadyExistsException.java b/java/core/src/main/java/sleeper/core/table/TableAlreadyExistsException.java new file mode 100644 index 0000000000..7402dfc577 --- /dev/null +++ b/java/core/src/main/java/sleeper/core/table/TableAlreadyExistsException.java @@ -0,0 +1,24 @@ +/* + * Copyright 2022-2023 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.core.table; + +public class TableAlreadyExistsException extends RuntimeException { + + public TableAlreadyExistsException(String tableName) { + super("Table already exists: " + tableName); + } +} diff --git a/java/core/src/main/java/sleeper/core/table/TableId.java b/java/core/src/main/java/sleeper/core/table/TableId.java new file mode 100644 index 0000000000..4d76467aa0 --- /dev/null +++ b/java/core/src/main/java/sleeper/core/table/TableId.java @@ -0,0 +1,67 @@ +/* + * Copyright 2022-2023 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.core.table; + +import java.util.Objects; + +public class TableId { + + private final String tableUniqueId; + private final String tableName; + + private TableId(String tableUniqueId, String tableName) { + this.tableUniqueId = tableUniqueId; + this.tableName = tableName; + } + + public static TableId uniqueIdAndName(String tableUniqueId, String tableName) { + return new TableId(tableUniqueId, tableName); + } + + public String getTableName() { + return tableName; + } + + public String getTableUniqueId() { + return tableUniqueId; + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + TableId tableId1 = (TableId) object; + return Objects.equals(tableUniqueId, tableId1.tableUniqueId) && Objects.equals(tableName, tableId1.tableName); + } + + @Override + public int hashCode() { + return Objects.hash(tableUniqueId, tableName); + } + + @Override + public String toString() { + return "TableId{" + + "tableUniqueId='" + tableUniqueId + '\'' + + ", tableName='" + tableName + '\'' + + '}'; + } +} diff --git a/java/core/src/main/java/sleeper/core/table/TableIdGenerator.java b/java/core/src/main/java/sleeper/core/table/TableIdGenerator.java new file mode 100644 index 0000000000..eff279c2fe --- /dev/null +++ b/java/core/src/main/java/sleeper/core/table/TableIdGenerator.java @@ -0,0 +1,45 @@ +/* + * Copyright 2022-2023 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.core.table; + +import org.apache.commons.codec.binary.Hex; + +import java.security.SecureRandom; +import java.util.Random; + +public class TableIdGenerator { + + private final Random random; + + public TableIdGenerator() { + random = new SecureRandom(); + } + + private TableIdGenerator(int seed) { + random = new Random(seed); + } + + public static TableIdGenerator fromRandomSeed(int seed) { + return new TableIdGenerator(seed); + } + + public String generateString() { + byte[] bytes = new byte[4]; + random.nextBytes(bytes); + return Hex.encodeHexString(bytes); + } +} diff --git a/java/core/src/main/java/sleeper/core/table/TableIndex.java b/java/core/src/main/java/sleeper/core/table/TableIndex.java new file mode 100644 index 0000000000..c5e1bbd7cc --- /dev/null +++ b/java/core/src/main/java/sleeper/core/table/TableIndex.java @@ -0,0 +1,30 @@ +/* + * Copyright 2022-2023 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.core.table; + +import java.util.Optional; +import java.util.stream.Stream; + +public interface TableIndex { + TableId createTable(String tableName) throws TableAlreadyExistsException; + + Stream streamAllTables(); + + Optional getTableByName(String tableName); + + Optional getTableByUniqueId(String tableUniqueId); +} diff --git a/java/core/src/test/java/sleeper/core/table/InMemoryTableIndex.java b/java/core/src/test/java/sleeper/core/table/InMemoryTableIndex.java new file mode 100644 index 0000000000..c496ff80e0 --- /dev/null +++ b/java/core/src/test/java/sleeper/core/table/InMemoryTableIndex.java @@ -0,0 +1,57 @@ +/* + * Copyright 2022-2023 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.core.table; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; +import java.util.stream.Stream; + +public class InMemoryTableIndex implements TableIndex { + + private final Map idByName = new TreeMap<>(); + private final Map nameById = new HashMap<>(); + private int nextIdNumber = 1; + + @Override + public TableId createTable(String tableName) { + if (idByName.containsKey(tableName)) { + throw new TableAlreadyExistsException(tableName); + } + TableId id = TableId.uniqueIdAndName("table-" + nextIdNumber, tableName); + nextIdNumber++; + idByName.put(tableName, id); + nameById.put(id.getTableUniqueId(), id); + return id; + } + + @Override + public Stream streamAllTables() { + return idByName.values().stream(); + } + + @Override + public Optional getTableByName(String tableName) { + return Optional.ofNullable(idByName.get(tableName)); + } + + @Override + public Optional getTableByUniqueId(String tableUniqueId) { + return Optional.ofNullable(nameById.get(tableUniqueId)); + } +} diff --git a/java/core/src/test/java/sleeper/core/table/InMemoryTableIndexTest.java b/java/core/src/test/java/sleeper/core/table/InMemoryTableIndexTest.java new file mode 100644 index 0000000000..cd9cc5dfa3 --- /dev/null +++ b/java/core/src/test/java/sleeper/core/table/InMemoryTableIndexTest.java @@ -0,0 +1,136 @@ +/* + * Copyright 2022-2023 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.core.table; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.tuple; + +public class InMemoryTableIndexTest { + + private final TableIndex store = new InMemoryTableIndex(); + + @Nested + @DisplayName("Create a table") + class CreateTable { + @Test + void shouldCreateATable() { + TableId tableId = store.createTable("test-table"); + + assertThat(store.streamAllTables()) + .containsExactly(tableId); + } + + @Test + void shouldFailToCreateATableWhichAlreadyExists() { + store.createTable("duplicate-table"); + + assertThatThrownBy(() -> store.createTable("duplicate-table")) + .isInstanceOf(TableAlreadyExistsException.class); + } + + @Test + void shouldGenerateNumericTableIds() { + TableId tableIdA = store.createTable("table-a"); + TableId tableIdB = store.createTable("table-b"); + + assertThat(List.of(tableIdA, tableIdB)) + .extracting(TableId::getTableName, TableId::getTableUniqueId) + .containsExactly( + tuple("table-a", "table-1"), + tuple("table-b", "table-2")); + } + } + + @Nested + @DisplayName("Look up a table") + class LookupTable { + + @Test + void shouldGetTableByName() { + TableId tableId = store.createTable("test-table"); + + assertThat(store.getTableByName("test-table")) + .contains(tableId); + } + + @Test + void shouldGetNoTableByName() { + store.createTable("existing-table"); + + assertThat(store.getTableByName("not-a-table")) + .isEmpty(); + } + + @Test + void shouldGetTableById() { + TableId tableId = store.createTable("test-table"); + + assertThat(store.getTableByUniqueId(tableId.getTableUniqueId())) + .contains(tableId); + } + + @Test + void shouldGetNoTableById() { + store.createTable("existing-table"); + + assertThat(store.getTableByUniqueId("not-a-table")) + .isEmpty(); + } + } + + @Nested + @DisplayName("List tables") + class ListTables { + + @Test + void shouldGetTablesOrderedByName() { + store.createTable("some-table"); + store.createTable("a-table"); + store.createTable("this-table"); + store.createTable("other-table"); + + assertThat(store.streamAllTables()) + .extracting(TableId::getTableName) + .containsExactly( + "a-table", + "other-table", + "some-table", + "this-table"); + } + + @Test + void shouldGetTableIds() { + TableId table1 = store.createTable("first-table"); + TableId table2 = store.createTable("second-table"); + + assertThat(store.streamAllTables()) + .containsExactly(table1, table2); + } + + @Test + void shouldGetNoTables() { + assertThat(store.streamAllTables()).isEmpty(); + } + } +} diff --git a/java/core/src/test/java/sleeper/core/table/TableIdGeneratorTest.java b/java/core/src/test/java/sleeper/core/table/TableIdGeneratorTest.java new file mode 100644 index 0000000000..3313f2661f --- /dev/null +++ b/java/core/src/test/java/sleeper/core/table/TableIdGeneratorTest.java @@ -0,0 +1,45 @@ +/* + * Copyright 2022-2023 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.core.table; + +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TableIdGeneratorTest { + + @Test + void shouldGenerate32BitHexId() { + assertThat(TableIdGenerator.fromRandomSeed(0).generateString()) + .isEqualTo("60b420bb"); + } + + @Test + void shouldGenerateAnother32BitHexId() { + assertThat(TableIdGenerator.fromRandomSeed(123).generateString()) + .isEqualTo("ddf121b9"); + } + + @Test + void shouldGenerateDifferentIdsFromSameGenerator() { + TableIdGenerator generator = TableIdGenerator.fromRandomSeed(42); + assertThat(List.of(generator.generateString(), generator.generateString())) + .containsExactly("359d41ba", "f78afe0d"); + } +} diff --git a/java/dynamodb-tools/src/main/java/sleeper/dynamodb/tools/DynamoDBUtils.java b/java/dynamodb-tools/src/main/java/sleeper/dynamodb/tools/DynamoDBUtils.java index a437ebbe7b..d1a4ebdd95 100644 --- a/java/dynamodb-tools/src/main/java/sleeper/dynamodb/tools/DynamoDBUtils.java +++ b/java/dynamodb-tools/src/main/java/sleeper/dynamodb/tools/DynamoDBUtils.java @@ -28,6 +28,7 @@ import com.amazonaws.services.dynamodbv2.model.ResourceInUseException; import com.amazonaws.services.dynamodbv2.model.ScanRequest; import com.amazonaws.services.dynamodbv2.model.ScanResult; +import com.amazonaws.services.dynamodbv2.model.Tag; import com.amazonaws.services.dynamodbv2.model.TimeToLiveSpecification; import com.amazonaws.services.dynamodbv2.model.UpdateTimeToLiveRequest; import org.slf4j.Logger; @@ -39,6 +40,7 @@ import java.util.Optional; import java.util.function.Function; import java.util.function.UnaryOperator; +import java.util.stream.Collectors; import java.util.stream.Stream; public class DynamoDBUtils { @@ -57,15 +59,30 @@ public static void initialiseTable( String tableName, List attributeDefinitions, List keySchemaElements) { + initialiseTable(dynamoDB, tableName, attributeDefinitions, keySchemaElements, Map.of()); + } + public static void initialiseTable( + AmazonDynamoDB dynamoDB, + String tableName, + List attributeDefinitions, + List keySchemaElements, + Map tags) { CreateTableRequest request = new CreateTableRequest() .withTableName(tableName) .withAttributeDefinitions(attributeDefinitions) .withKeySchema(keySchemaElements) .withBillingMode(BillingMode.PAY_PER_REQUEST); + String message = ""; + if (!tags.isEmpty()) { + request = request.withTags(tags.entrySet().stream() + .map(e -> new Tag().withKey(e.getKey()).withValue(e.getValue())) + .collect(Collectors.toUnmodifiableList())); + message = " with tags " + tags; + } try { CreateTableResult result = dynamoDB.createTable(request); - LOGGER.info("Created table {}", result.getTableDescription().getTableName()); + LOGGER.info("Created table {} {}", result.getTableDescription().getTableName(), message); } catch (ResourceInUseException e) { if (e.getMessage().contains("Table already exists")) { LOGGER.warn("Table {} already exists", tableName); diff --git a/java/dynamodb-tools/src/test/java/sleeper/dynamodb/tools/DynamoDBTestBase.java b/java/dynamodb-tools/src/test/java/sleeper/dynamodb/tools/DynamoDBTestBase.java index c589ddaa39..04173f5f50 100644 --- a/java/dynamodb-tools/src/test/java/sleeper/dynamodb/tools/DynamoDBTestBase.java +++ b/java/dynamodb-tools/src/test/java/sleeper/dynamodb/tools/DynamoDBTestBase.java @@ -19,27 +19,22 @@ import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -import org.testcontainers.containers.GenericContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -import sleeper.core.CommonTestConstants; - import static sleeper.dynamodb.tools.GenericContainerAwsV1ClientHelper.buildAwsV1Client; @Testcontainers public abstract class DynamoDBTestBase { - private static final int DYNAMO_PORT = 8000; protected static AmazonDynamoDB dynamoDBClient; @Container - public static GenericContainer dynamoDb = new GenericContainer(CommonTestConstants.DYNAMODB_LOCAL_CONTAINER) - .withExposedPorts(DYNAMO_PORT); + public static DynamoDBContainer dynamoDb = new DynamoDBContainer(); @BeforeAll public static void initDynamoClient() { - dynamoDBClient = buildAwsV1Client(dynamoDb, DYNAMO_PORT, AmazonDynamoDBClientBuilder.standard()); + dynamoDBClient = buildAwsV1Client(dynamoDb, dynamoDb.getDynamoPort(), AmazonDynamoDBClientBuilder.standard()); } @AfterAll diff --git a/java/statestore/src/main/java/sleeper/statestore/dynamodb/DynamoDBStateStoreCreator.java b/java/statestore/src/main/java/sleeper/statestore/dynamodb/DynamoDBStateStoreCreator.java index b943e33dbe..24e63f114d 100644 --- a/java/statestore/src/main/java/sleeper/statestore/dynamodb/DynamoDBStateStoreCreator.java +++ b/java/statestore/src/main/java/sleeper/statestore/dynamodb/DynamoDBStateStoreCreator.java @@ -17,23 +17,15 @@ import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; import com.amazonaws.services.dynamodbv2.model.AttributeDefinition; -import com.amazonaws.services.dynamodbv2.model.BillingMode; -import com.amazonaws.services.dynamodbv2.model.CreateTableRequest; -import com.amazonaws.services.dynamodbv2.model.CreateTableResult; import com.amazonaws.services.dynamodbv2.model.KeySchemaElement; import com.amazonaws.services.dynamodbv2.model.KeyType; -import com.amazonaws.services.dynamodbv2.model.ResourceInUseException; import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType; -import com.amazonaws.services.dynamodbv2.model.Tag; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import sleeper.configuration.properties.instance.InstanceProperties; +import sleeper.dynamodb.tools.DynamoDBUtils; -import java.util.Collection; import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.ACTIVE_FILEINFO_TABLENAME; import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.PARTITION_TABLENAME; @@ -49,19 +41,12 @@ * normally done using CDK. */ public class DynamoDBStateStoreCreator { - private static final Logger LOGGER = LoggerFactory.getLogger(DynamoDBStateStoreCreator.class); - private final AmazonDynamoDB dynamoDB; private final InstanceProperties instanceProperties; - private final Collection tags; + private final AmazonDynamoDB dynamoDB; public DynamoDBStateStoreCreator(InstanceProperties instanceProperties, AmazonDynamoDB dynamoDB) { + this.instanceProperties = Objects.requireNonNull(instanceProperties, "instanceProperties must not be null"); this.dynamoDB = Objects.requireNonNull(dynamoDB, "dynamoDB must not be null"); - this.instanceProperties = instanceProperties; - this.tags = instanceProperties.getTags() - .entrySet() - .stream() - .map(e -> new Tag().withKey(e.getKey()).withValue(e.getValue())) - .collect(Collectors.toList()); } public void create() { @@ -100,25 +85,8 @@ private void initialiseTable( String tableName, List attributeDefinitions, List keySchemaElements) { - CreateTableRequest request = new CreateTableRequest() - .withTableName(tableName) - .withAttributeDefinitions(attributeDefinitions) - .withKeySchema(keySchemaElements) - .withBillingMode(BillingMode.PAY_PER_REQUEST); - String message = ""; - if (!tags.isEmpty()) { - request = request.withTags(tags); - message = " with tags " + tags; - } - try { - CreateTableResult result = dynamoDB.createTable(request); - LOGGER.info("Created table {} {}", result.getTableDescription().getTableName(), message); - } catch (ResourceInUseException e) { - if (e.getMessage().contains("Table already exists")) { - LOGGER.warn("Table {} already exists", tableName); - } else { - throw e; - } - } + DynamoDBUtils.initialiseTable(dynamoDB, + tableName, attributeDefinitions, keySchemaElements, + instanceProperties.getTags()); } } diff --git a/java/tables/pom.xml b/java/tables/pom.xml index c0751dc383..37a3b4b4e4 100644 --- a/java/tables/pom.xml +++ b/java/tables/pom.xml @@ -42,6 +42,11 @@ statestore ${project.parent.version} + + sleeper + dynamodb-tools + ${project.parent.version} + sleeper @@ -57,6 +62,13 @@ test-jar test + + sleeper + dynamodb-tools + ${project.parent.version} + test-jar + test + org.testcontainers localstack diff --git a/java/tables/src/main/java/sleeper/table/index/dynamodb/DynamoDBTableIdFormat.java b/java/tables/src/main/java/sleeper/table/index/dynamodb/DynamoDBTableIdFormat.java new file mode 100644 index 0000000000..029763d84b --- /dev/null +++ b/java/tables/src/main/java/sleeper/table/index/dynamodb/DynamoDBTableIdFormat.java @@ -0,0 +1,46 @@ +/* + * Copyright 2022-2023 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.table.index.dynamodb; + +import com.amazonaws.services.dynamodbv2.model.AttributeValue; + +import sleeper.core.table.TableId; + +import java.util.Map; + +import static sleeper.dynamodb.tools.DynamoDBAttributes.createStringAttribute; +import static sleeper.dynamodb.tools.DynamoDBAttributes.getStringAttribute; + +class DynamoDBTableIdFormat { + private DynamoDBTableIdFormat() { + } + + static final String TABLE_NAME_FIELD = "TableName"; + static final String TABLE_ID_FIELD = "TableId"; + + public static Map getItem(TableId id) { + return Map.of( + TABLE_ID_FIELD, createStringAttribute(id.getTableUniqueId()), + TABLE_NAME_FIELD, createStringAttribute(id.getTableName())); + } + + public static TableId readItem(Map item) { + return TableId.uniqueIdAndName( + getStringAttribute(item, TABLE_ID_FIELD), + getStringAttribute(item, TABLE_NAME_FIELD)); + } +} diff --git a/java/tables/src/main/java/sleeper/table/index/dynamodb/DynamoDBTableIdStoreCreator.java b/java/tables/src/main/java/sleeper/table/index/dynamodb/DynamoDBTableIdStoreCreator.java new file mode 100644 index 0000000000..9f4831fc8f --- /dev/null +++ b/java/tables/src/main/java/sleeper/table/index/dynamodb/DynamoDBTableIdStoreCreator.java @@ -0,0 +1,66 @@ +/* + * Copyright 2022-2023 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.table.index.dynamodb; + +import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; +import com.amazonaws.services.dynamodbv2.model.AttributeDefinition; +import com.amazonaws.services.dynamodbv2.model.KeySchemaElement; +import com.amazonaws.services.dynamodbv2.model.KeyType; +import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType; + +import sleeper.configuration.properties.instance.InstanceProperties; +import sleeper.dynamodb.tools.DynamoDBUtils; + +import java.util.List; +import java.util.Objects; + +import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.TABLE_ID_INDEX_DYNAMO_TABLENAME; +import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.TABLE_NAME_INDEX_DYNAMO_TABLENAME; +import static sleeper.table.index.dynamodb.DynamoDBTableIdFormat.TABLE_ID_FIELD; +import static sleeper.table.index.dynamodb.DynamoDBTableIdFormat.TABLE_NAME_FIELD; + +public class DynamoDBTableIdStoreCreator { + private final InstanceProperties instanceProperties; + private final AmazonDynamoDB dynamoDB; + + private DynamoDBTableIdStoreCreator(InstanceProperties instanceProperties, AmazonDynamoDB dynamoDB) { + this.instanceProperties = Objects.requireNonNull(instanceProperties, "instanceProperties must not be null"); + this.dynamoDB = Objects.requireNonNull(dynamoDB, "dynamoDB must not be null"); + } + + public static void create(AmazonDynamoDB dynamoDBClient, InstanceProperties instanceProperties) { + new DynamoDBTableIdStoreCreator(instanceProperties, dynamoDBClient).create(); + } + + public void create() { + initialiseTable(instanceProperties.get(TABLE_NAME_INDEX_DYNAMO_TABLENAME), + List.of(new AttributeDefinition(TABLE_NAME_FIELD, ScalarAttributeType.S)), + List.of(new KeySchemaElement(TABLE_NAME_FIELD, KeyType.HASH))); + initialiseTable(instanceProperties.get(TABLE_ID_INDEX_DYNAMO_TABLENAME), + List.of(new AttributeDefinition(TABLE_ID_FIELD, ScalarAttributeType.S)), + List.of(new KeySchemaElement(TABLE_ID_FIELD, KeyType.HASH))); + } + + private void initialiseTable( + String tableName, + List attributeDefinitions, + List keySchemaElements) { + DynamoDBUtils.initialiseTable(dynamoDB, + tableName, attributeDefinitions, keySchemaElements, + instanceProperties.getTags()); + } +} diff --git a/java/tables/src/main/java/sleeper/table/index/dynamodb/DynamoDBTableIndex.java b/java/tables/src/main/java/sleeper/table/index/dynamodb/DynamoDBTableIndex.java new file mode 100644 index 0000000000..207d665777 --- /dev/null +++ b/java/tables/src/main/java/sleeper/table/index/dynamodb/DynamoDBTableIndex.java @@ -0,0 +1,134 @@ +/* + * Copyright 2022-2023 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.table.index.dynamodb; + +import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; +import com.amazonaws.services.dynamodbv2.model.AttributeValue; +import com.amazonaws.services.dynamodbv2.model.CancellationReason; +import com.amazonaws.services.dynamodbv2.model.ComparisonOperator; +import com.amazonaws.services.dynamodbv2.model.Condition; +import com.amazonaws.services.dynamodbv2.model.ConsumedCapacity; +import com.amazonaws.services.dynamodbv2.model.Put; +import com.amazonaws.services.dynamodbv2.model.QueryRequest; +import com.amazonaws.services.dynamodbv2.model.QueryResult; +import com.amazonaws.services.dynamodbv2.model.ReturnConsumedCapacity; +import com.amazonaws.services.dynamodbv2.model.ScanRequest; +import com.amazonaws.services.dynamodbv2.model.TransactWriteItem; +import com.amazonaws.services.dynamodbv2.model.TransactWriteItemsRequest; +import com.amazonaws.services.dynamodbv2.model.TransactWriteItemsResult; +import com.amazonaws.services.dynamodbv2.model.TransactionCanceledException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import sleeper.configuration.properties.instance.InstanceProperties; +import sleeper.core.table.TableAlreadyExistsException; +import sleeper.core.table.TableId; +import sleeper.core.table.TableIdGenerator; +import sleeper.core.table.TableIndex; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; + +import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.TABLE_ID_INDEX_DYNAMO_TABLENAME; +import static sleeper.configuration.properties.instance.CdkDefinedInstanceProperty.TABLE_NAME_INDEX_DYNAMO_TABLENAME; +import static sleeper.dynamodb.tools.DynamoDBAttributes.createStringAttribute; +import static sleeper.dynamodb.tools.DynamoDBUtils.streamPagedItems; + +public class DynamoDBTableIndex implements TableIndex { + private static final Logger LOGGER = LoggerFactory.getLogger(DynamoDBTableIndex.class); + + public static final String TABLE_NAME_FIELD = DynamoDBTableIdFormat.TABLE_NAME_FIELD; + public static final String TABLE_ID_FIELD = DynamoDBTableIdFormat.TABLE_ID_FIELD; + + private final AmazonDynamoDB dynamoDB; + private final String nameIndexDynamoTableName; + private final String idIndexDynamoTableName; + private final TableIdGenerator idGenerator = new TableIdGenerator(); + + public DynamoDBTableIndex(AmazonDynamoDB dynamoDB, InstanceProperties instanceProperties) { + this.dynamoDB = dynamoDB; + this.nameIndexDynamoTableName = instanceProperties.get(TABLE_NAME_INDEX_DYNAMO_TABLENAME); + this.idIndexDynamoTableName = instanceProperties.get(TABLE_ID_INDEX_DYNAMO_TABLENAME); + } + + @Override + public TableId createTable(String tableName) throws TableAlreadyExistsException { + TableId id = TableId.uniqueIdAndName(idGenerator.generateString(), tableName); + Map idItem = DynamoDBTableIdFormat.getItem(id); + TransactWriteItemsRequest request = new TransactWriteItemsRequest() + .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL) + .withTransactItems( + new TransactWriteItem().withPut(new Put() + .withTableName(nameIndexDynamoTableName) + .withItem(idItem) + .withConditionExpression("attribute_not_exists(#tablename)") + .withExpressionAttributeNames(Map.of("#tablename", TABLE_NAME_FIELD))), + new TransactWriteItem().withPut(new Put() + .withTableName(idIndexDynamoTableName) + .withItem(idItem) + .withConditionExpression("attribute_not_exists(#tableid)") + .withExpressionAttributeNames(Map.of("#tableid", TABLE_ID_FIELD)))); + try { + TransactWriteItemsResult result = dynamoDB.transactWriteItems(request); + List consumedCapacity = result.getConsumedCapacity(); + double totalCapacity = consumedCapacity.stream().mapToDouble(ConsumedCapacity::getCapacityUnits).sum(); + LOGGER.debug("Created table {} with ID {}, capacity consumed = {}", + tableName, id.getTableUniqueId(), totalCapacity); + return id; + } catch (TransactionCanceledException e) { + CancellationReason nameIndexReason = e.getCancellationReasons().get(0); + if ("ConditionalCheckFailed".equals(nameIndexReason.getCode())) { + throw new TableAlreadyExistsException(tableName); + } else { + throw e; + } + } + } + + @Override + public Stream streamAllTables() { + return streamPagedItems(dynamoDB, + new ScanRequest() + .withTableName(nameIndexDynamoTableName) + .withConsistentRead(true)) + .map(DynamoDBTableIdFormat::readItem) + .sorted(Comparator.comparing(TableId::getTableName)); + } + + @Override + public Optional getTableByName(String tableName) { + QueryResult result = dynamoDB.query(new QueryRequest() + .withTableName(nameIndexDynamoTableName) + .addKeyConditionsEntry(TABLE_NAME_FIELD, new Condition() + .withAttributeValueList(createStringAttribute(tableName)) + .withComparisonOperator(ComparisonOperator.EQ))); + return result.getItems().stream().map(DynamoDBTableIdFormat::readItem).findFirst(); + } + + @Override + public Optional getTableByUniqueId(String tableUniqueId) { + QueryResult result = dynamoDB.query(new QueryRequest() + .withTableName(idIndexDynamoTableName) + .addKeyConditionsEntry(TABLE_ID_FIELD, new Condition() + .withAttributeValueList(createStringAttribute(tableUniqueId)) + .withComparisonOperator(ComparisonOperator.EQ))); + return result.getItems().stream().map(DynamoDBTableIdFormat::readItem).findFirst(); + } +} diff --git a/java/tables/src/test/java/sleeper/table/index/dynamodb/DynamoDBTableIndexIT.java b/java/tables/src/test/java/sleeper/table/index/dynamodb/DynamoDBTableIndexIT.java new file mode 100644 index 0000000000..b808562edc --- /dev/null +++ b/java/tables/src/test/java/sleeper/table/index/dynamodb/DynamoDBTableIndexIT.java @@ -0,0 +1,135 @@ +/* + * Copyright 2022-2023 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sleeper.table.index.dynamodb; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import sleeper.configuration.properties.instance.InstanceProperties; +import sleeper.core.table.TableAlreadyExistsException; +import sleeper.core.table.TableId; +import sleeper.core.table.TableIndex; +import sleeper.dynamodb.tools.DynamoDBTestBase; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static sleeper.configuration.properties.InstancePropertiesTestHelper.createTestInstanceProperties; + +public class DynamoDBTableIndexIT extends DynamoDBTestBase { + + private final InstanceProperties instanceProperties = createTestInstanceProperties(); + private final TableIndex store = new DynamoDBTableIndex(dynamoDBClient, instanceProperties); + + @BeforeEach + void setUp() { + DynamoDBTableIdStoreCreator.create(dynamoDBClient, instanceProperties); + } + + @Nested + @DisplayName("Create a table") + class CreateTable { + @Test + void shouldCreateATable() { + TableId tableId = store.createTable("test-table"); + + assertThat(store.streamAllTables()) + .containsExactly(tableId); + } + + @Test + void shouldFailToCreateATableWhichAlreadyExists() { + store.createTable("duplicate-table"); + + assertThatThrownBy(() -> store.createTable("duplicate-table")) + .isInstanceOf(TableAlreadyExistsException.class); + } + } + + @Nested + @DisplayName("Look up a table") + class LookupTable { + + @Test + void shouldGetTableByName() { + TableId tableId = store.createTable("test-table"); + + assertThat(store.getTableByName("test-table")) + .contains(tableId); + } + + @Test + void shouldGetNoTableByName() { + store.createTable("existing-table"); + + assertThat(store.getTableByName("not-a-table")) + .isEmpty(); + } + + @Test + void shouldGetTableById() { + TableId tableId = store.createTable("test-table"); + + assertThat(store.getTableByUniqueId(tableId.getTableUniqueId())) + .contains(tableId); + } + + @Test + void shouldGetNoTableById() { + store.createTable("existing-table"); + + assertThat(store.getTableByUniqueId("not-a-table")) + .isEmpty(); + } + } + + @Nested + @DisplayName("List tables") + class ListTables { + + @Test + void shouldGetTablesOrderedByName() { + store.createTable("some-table"); + store.createTable("a-table"); + store.createTable("this-table"); + store.createTable("other-table"); + + assertThat(store.streamAllTables()) + .extracting(TableId::getTableName) + .containsExactly( + "a-table", + "other-table", + "some-table", + "this-table"); + } + + @Test + void shouldGetTableIds() { + TableId table1 = store.createTable("first-table"); + TableId table2 = store.createTable("second-table"); + + assertThat(store.streamAllTables()) + .containsExactly(table1, table2); + } + + @Test + void shouldGetNoTables() { + assertThat(store.streamAllTables()).isEmpty(); + } + } +} diff --git a/scripts/templates/instanceproperties.template b/scripts/templates/instanceproperties.template index cbd962587c..46de6c22bd 100644 --- a/scripts/templates/instanceproperties.template +++ b/scripts/templates/instanceproperties.template @@ -137,6 +137,10 @@ sleeper.metadata.dynamo.pointintimerecovery=false # revision DynamoDB table. sleeper.metadata.s3.dynamo.pointintimerecovery=false +# This specifies whether point in time recovery is enabled for the Sleeper table index. This is set on +# the DynamoDB tables. +sleeper.tables.index.dynamo.pointintimerecovery=false + # The timeout in minutes for when the table properties provider cache should be cleared, forcing table # properties to be reloaded from S3. sleeper.table.properties.provider.timeout.minutes=60