Skip to content

Commit

Permalink
Move all references to use PopulateInstanceProperties
Browse files Browse the repository at this point in the history
  • Loading branch information
@patchwork01
patchwork01 committed Nov 1, 2024
1 parent ab7fe27 commit a2c9952
Show file tree
Hide file tree
Showing 5 changed files with 9 additions and 154 deletions.
6 changes: 2 additions & 4 deletions java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,11 +132,9 @@ public void deploy() throws IOException, InterruptedException {
LOGGER.info("jarsDirectory: {}", jarsDirectory);
LOGGER.info("sleeperVersion: {}", sleeperVersion);
LOGGER.info("deployPaused: {}", deployPaused);
InstanceProperties instanceProperties = PopulateInstancePropertiesAws.builder()
.sts(sts).regionProvider(regionProvider)
.deployInstanceConfig(deployInstanceConfiguration)
InstanceProperties instanceProperties = PopulateInstancePropertiesAws.builder(sts, regionProvider)
.instanceId(instanceId).vpcId(vpcId).subnetIds(subnetIds)
.build().populate();
.build().populate(deployInstanceConfiguration.getInstanceProperties());
extraInstanceProperties.accept(instanceProperties);
validate(instanceProperties, deployInstanceConfiguration.getTableProperties());

Expand Down
145 changes: 1 addition & 144 deletions java/clients/src/main/java/sleeper/clients/deploy/PopulateInstancePropertiesAws.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,159 +20,16 @@
import com.amazonaws.services.securitytoken.model.GetCallerIdentityRequest;
import software.amazon.awssdk.regions.providers.AwsRegionProvider;

import sleeper.core.deploy.DeployInstanceConfiguration;
import sleeper.core.deploy.PopulateInstanceProperties;
import sleeper.core.deploy.SleeperScheduleRule;
import sleeper.core.properties.instance.InstanceProperties;
import sleeper.core.properties.validation.LambdaDeployType;

import java.nio.file.Path;
import java.util.Optional;
import java.util.Properties;
import java.util.function.Supplier;

import static java.util.Objects.requireNonNull;
import static org.apache.commons.lang3.ObjectUtils.requireNonEmpty;
import static sleeper.core.properties.PropertiesUtils.loadProperties;
import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.CONFIG_BUCKET;
import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.QUERY_RESULTS_BUCKET;
import static sleeper.core.properties.instance.CommonProperty.ACCOUNT;
import static sleeper.core.properties.instance.CommonProperty.ECR_REPOSITORY_PREFIX;
import static sleeper.core.properties.instance.CommonProperty.ID;
import static sleeper.core.properties.instance.CommonProperty.JARS_BUCKET;
import static sleeper.core.properties.instance.CommonProperty.LAMBDA_DEPLOY_TYPE;
import static sleeper.core.properties.instance.CommonProperty.REGION;
import static sleeper.core.properties.instance.CommonProperty.SUBNETS;
import static sleeper.core.properties.instance.CommonProperty.VPC_ID;
import static sleeper.core.properties.instance.CompactionProperty.ECR_COMPACTION_REPO;
import static sleeper.core.properties.instance.EKSProperty.BULK_IMPORT_REPO;
import static sleeper.core.properties.instance.EMRServerlessProperty.BULK_IMPORT_EMR_SERVERLESS_CUSTOM_IMAGE_REPO;
import static sleeper.core.properties.instance.IngestProperty.ECR_INGEST_REPO;

public class PopulateInstancePropertiesAws {
private final Supplier<String> accountSupplier;
private final AwsRegionProvider regionProvider;
private final String instanceId;
private final String vpcId;
private final String subnetIds;
private final InstanceProperties properties;
private final Properties tagsProperties;

private PopulateInstancePropertiesAws(Builder builder) {
accountSupplier = requireNonNull(builder.accountSupplier, "accountSupplier must not be null");
regionProvider = requireNonNull(builder.regionProvider, "regionProvider must not be null");
instanceId = requireNonEmpty(builder.instanceId, "instanceId must not be empty");
vpcId = requireNonEmpty(builder.vpcId, "vpcId must not be empty");
subnetIds = requireNonEmpty(builder.subnetIds, "subnetIds must not be empty");
properties = Optional.ofNullable(builder.properties).orElseGet(InstanceProperties::new);
tagsProperties = Optional.ofNullable(builder.tagsProperties).orElseGet(properties::getTagsProperties);
private PopulateInstancePropertiesAws() {
}

public static PopulateInstanceProperties.Builder builder(AWSSecurityTokenService sts, AwsRegionProvider regionProvider) {
return PopulateInstanceProperties.builder()
.accountSupplier(sts.getCallerIdentity(new GetCallerIdentityRequest())::getAccount)
.regionIdSupplier(() -> regionProvider.getRegion().id());
}

public static Builder builder() {
return new Builder();
}

public InstanceProperties populate() {
InstanceProperties instanceProperties = populateDefaultsFromInstanceId(properties, instanceId);
tagsProperties.setProperty("InstanceID", instanceId);
instanceProperties.loadTags(tagsProperties);
instanceProperties.set(ACCOUNT, accountSupplier.get());
instanceProperties.set(REGION, regionProvider.getRegion().id());
instanceProperties.set(VPC_ID, vpcId);
instanceProperties.set(SUBNETS, subnetIds);
return instanceProperties;
}

public static InstanceProperties generateTearDownDefaultsFromInstanceId(String instanceId) {
InstanceProperties instanceProperties = populateDefaultsFromInstanceId(new InstanceProperties(), instanceId);
instanceProperties.setEnum(LAMBDA_DEPLOY_TYPE, LambdaDeployType.CONTAINER);
instanceProperties.set(CONFIG_BUCKET, InstanceProperties.getConfigBucketFromInstanceId(instanceId));
instanceProperties.set(QUERY_RESULTS_BUCKET, String.format("sleeper-%s-query-results", instanceId));
SleeperScheduleRule.getCloudWatchRuleDefaults(instanceId)
.forEach(rule -> instanceProperties.set(rule.getProperty(), rule.getPropertyValue()));
return instanceProperties;
}

public static InstanceProperties populateDefaultsFromInstanceId(InstanceProperties properties, String instanceId) {
properties.set(ID, instanceId);
properties.set(JARS_BUCKET, String.format("sleeper-%s-jars", instanceId));
String ecrPrefix = Optional.ofNullable(properties.get(ECR_REPOSITORY_PREFIX)).orElse(instanceId);
properties.set(ECR_COMPACTION_REPO, ecrPrefix + "/compaction-job-execution");
properties.set(ECR_INGEST_REPO, ecrPrefix + "/ingest");
properties.set(BULK_IMPORT_REPO, ecrPrefix + "/bulk-import-runner");
properties.set(BULK_IMPORT_EMR_SERVERLESS_CUSTOM_IMAGE_REPO, ecrPrefix + "/bulk-import-runner-emr-serverless");
return properties;
}

public static final class Builder {
private Supplier<String> accountSupplier;
private AwsRegionProvider regionProvider;
private String instanceId;
private String vpcId;
private String subnetIds;
private InstanceProperties properties;
private Properties tagsProperties;

private Builder() {
}

public Builder sts(AWSSecurityTokenService sts) {
return accountSupplier(sts.getCallerIdentity(new GetCallerIdentityRequest())::getAccount);
}

public Builder accountSupplier(Supplier<String> accountSupplier) {
this.accountSupplier = accountSupplier;
return this;
}

public Builder regionProvider(AwsRegionProvider regionProvider) {
this.regionProvider = regionProvider;
return this;
}

public Builder instanceId(String instanceId) {
this.instanceId = instanceId;
return this;
}

public Builder vpcId(String vpcId) {
this.vpcId = vpcId;
return this;
}

public Builder subnetIds(String subnetIds) {
this.subnetIds = subnetIds;
return this;
}

public Builder instanceProperties(InstanceProperties properties) {
this.properties = properties;
return this;
}

public Builder instanceProperties(Path propertiesPath) {
this.properties = InstanceProperties.createWithoutValidation(loadProperties(propertiesPath));
return this;
}

public Builder tagsProperties(Properties tagsProperties) {
this.tagsProperties = tagsProperties;
return this;
}

public Builder deployInstanceConfig(DeployInstanceConfiguration deployInstanceConfig) {
return instanceProperties(deployInstanceConfig.getInstanceProperties())
.tagsProperties(deployInstanceConfig.getInstanceProperties().getTagsProperties());
}

public PopulateInstancePropertiesAws build() {
return new PopulateInstancePropertiesAws(this);
}
}
}
4 changes: 2 additions & 2 deletions java/clients/src/main/java/sleeper/clients/docker/DeployDockerInstance.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,13 @@
import org.eclipse.jetty.io.RuntimeIOException;
import software.amazon.awssdk.services.sqs.SqsClient;

import sleeper.clients.deploy.PopulateInstancePropertiesAws;
import sleeper.clients.docker.stack.CompactionDockerStack;
import sleeper.clients.docker.stack.ConfigurationDockerStack;
import sleeper.clients.docker.stack.IngestDockerStack;
import sleeper.clients.docker.stack.TableDockerStack;
import sleeper.clients.status.update.AddTable;
import sleeper.configuration.properties.S3InstanceProperties;
import sleeper.core.deploy.PopulateInstanceProperties;
import sleeper.core.properties.instance.InstanceProperties;
import sleeper.core.properties.table.TableProperties;
import sleeper.core.properties.validation.DefaultAsyncCommitBehaviour;
Expand Down Expand Up @@ -98,7 +98,7 @@ public static void main(String[] args) throws Exception {
}

public void deploy(String instanceId) {
InstanceProperties instanceProperties = PopulateInstancePropertiesAws.populateDefaultsFromInstanceId(
InstanceProperties instanceProperties = PopulateInstanceProperties.populateDefaultsFromInstanceId(
new InstanceProperties(), instanceId);
TableProperties tableProperties = generateTableProperties(instanceProperties);
extraTableProperties.accept(tableProperties);
Expand Down
4 changes: 2 additions & 2 deletions java/clients/src/main/java/sleeper/clients/teardown/TearDownInstance.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import sleeper.clients.deploy.PopulateInstancePropertiesAws;
import sleeper.clients.util.ClientUtils;
import sleeper.configuration.properties.S3InstanceProperties;
import sleeper.core.deploy.PopulateInstanceProperties;
import sleeper.core.properties.instance.InstanceProperties;
import sleeper.core.properties.local.LoadLocalProperties;

Expand Down Expand Up @@ -139,7 +139,7 @@ public static InstanceProperties loadInstancePropertiesOrGenerateDefaults(Amazon
return S3InstanceProperties.loadGivenInstanceIdNoValidation(s3, instanceId);
} catch (AmazonS3Exception e) {
LOGGER.info("Failed to download configuration, using default properties");
return PopulateInstancePropertiesAws.generateTearDownDefaultsFromInstanceId(instanceId);
return PopulateInstanceProperties.generateTearDownDefaultsFromInstanceId(instanceId);
}
}

Expand Down
4 changes: 2 additions & 2 deletions ...st-drivers/src/main/java/sleeper/systemtest/drivers/cdk/TearDownSystemTestDeployment.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,12 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import sleeper.clients.deploy.PopulateInstancePropertiesAws;
import sleeper.clients.teardown.RemoveECRRepositories;
import sleeper.clients.teardown.RemoveJarsBucket;
import sleeper.clients.teardown.ShutdownSystemProcesses;
import sleeper.clients.teardown.TearDownClients;
import sleeper.clients.teardown.WaitForStackToDelete;
import sleeper.core.deploy.PopulateInstanceProperties;
import sleeper.systemtest.configuration.SystemTestStandaloneProperties;
import sleeper.systemtest.dsl.instance.SystemTestParameters;

Expand Down Expand Up @@ -73,7 +73,7 @@ public void cleanupAfterAllInstancesAndStackDeleted() throws InterruptedExceptio
LOGGER.info("Removing the Jars bucket and docker containers");
RemoveJarsBucket.remove(clients.getS3v2(), properties.get(SYSTEM_TEST_JARS_BUCKET));
RemoveECRRepositories.remove(clients.getEcr(),
PopulateInstancePropertiesAws.generateTearDownDefaultsFromInstanceId(properties.get(SYSTEM_TEST_ID)),
PopulateInstanceProperties.generateTearDownDefaultsFromInstanceId(properties.get(SYSTEM_TEST_ID)),
List.of(properties.get(SYSTEM_TEST_REPO)));
}

Expand Down

0 comments on commit a2c9952

Please sign in to comment.