diff --git a/README.md b/README.md index 9dfc719..43ebf1f 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,14 @@ | __Works on__ | LocalStack v2 | +### UPDATE + +The Terraform configuration file now randomly generates names for the bucket, in order to avoid conflicts +at a global scale on AWS. This name shall be written out to a properties file, which the app will pick up +and use for the S3 client. Furthermore, the name is also passed as an environment variable to the Lambda function by Terraform, +so there's no need to worry about managing it. + + ## Introduction This application was created for demonstration purposes to highlight the ease of switching from @@ -137,7 +145,7 @@ $ export AWS_SECRET_ACCESS_KEY=[your_aws_secret_access_key_id] Make sure you have Terraform [installed](https://developer.hashicorp.com/terraform/downloads) -Under setup/terraform run: +Under `terraform` run: ``` $ terraform init @@ -210,12 +218,12 @@ to get rid of any files that keep track of the resources' state. Then: ``` $ tflocal init -$ tflocal plan -var 'env=dev' +$ tflocal plan $ tflocal apply ``` -What we're doing here is just passing an environmental variable to let the Lambda -know this is the `dev` environment. +We run the exact same commands for the exact same file. We no longer need to pass any environment +variables, since the bucket name is generated and passed by Terraform. ### Starting the backend diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..1a31f4d --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "shipment-list-demo", + "lockfileVersion": 3, + "requires": true, + "packages": {} +} diff --git a/setup/terraform/vars.tf b/setup/terraform/vars.tf deleted file mode 100644 index ccddc47..0000000 --- a/setup/terraform/vars.tf +++ /dev/null @@ -1,11 +0,0 @@ -variable "env" { - type = string - description = "dev env" - default = "" -} - -variable "sns_sub_endpoint" { - type = string - description = "SNS subscriber endpoint" - default = "https://localhost:8081/sns/notifications" -} \ No newline at end of file diff --git a/shipment-picture-lambda-validator/pom.xml b/shipment-picture-lambda-validator/pom.xml index 43945d9..1403514 100644 --- a/shipment-picture-lambda-validator/pom.xml +++ b/shipment-picture-lambda-validator/pom.xml @@ -55,6 +55,12 @@ thumbnailator 0.4.19 + + org.projectlombok + lombok + 1.18.22 + compile + diff --git a/shipment-picture-lambda-validator/src/main/java/dev/ancaghenade/shipmentpicturelambdavalidator/Location.java b/shipment-picture-lambda-validator/src/main/java/dev/ancaghenade/shipmentpicturelambdavalidator/Location.java new file mode 100644 index 0000000..1918699 --- /dev/null +++ b/shipment-picture-lambda-validator/src/main/java/dev/ancaghenade/shipmentpicturelambdavalidator/Location.java @@ -0,0 +1,16 @@ +package dev.ancaghenade.shipmentpicturelambdavalidator; + +import lombok.Getter; +import software.amazon.awssdk.regions.Region; + +@Getter +public enum Location { + + + REGION(Region.US_EAST_1); + + private final Region region; + Location(Region region) { + this.region = region; + } +} diff --git a/shipment-picture-lambda-validator/src/main/java/dev/ancaghenade/shipmentpicturelambdavalidator/S3ClientHelper.java b/shipment-picture-lambda-validator/src/main/java/dev/ancaghenade/shipmentpicturelambdavalidator/S3ClientHelper.java index c979238..b15a56e 100644 --- a/shipment-picture-lambda-validator/src/main/java/dev/ancaghenade/shipmentpicturelambdavalidator/S3ClientHelper.java +++ b/shipment-picture-lambda-validator/src/main/java/dev/ancaghenade/shipmentpicturelambdavalidator/S3ClientHelper.java @@ -2,22 +2,20 @@ import java.io.IOException; import java.net.URI; -import software.amazon.awssdk.regions.Region; +import java.util.Objects; import software.amazon.awssdk.services.s3.S3Client; public class S3ClientHelper { - private static final String ENVIRONMENT = System.getenv("ENVIRONMENT"); - private static PropertiesProvider properties = new PropertiesProvider(); + private static final String LOCALSTACK_HOSTNAME = System.getenv("LOCALSTACK_HOSTNAME"); public static S3Client getS3Client() throws IOException { var clientBuilder = S3Client.builder(); - if (properties.getProperty("environment.dev").equals(ENVIRONMENT)) { - + if (Objects.nonNull(LOCALSTACK_HOSTNAME)) { return clientBuilder - .region(Region.of(properties.getProperty("aws.region"))) - .endpointOverride(URI.create(properties.getProperty("s3.endpoint"))) + .region(Location.REGION.getRegion()) + .endpointOverride(URI.create(String.format("http://%s:4566", LOCALSTACK_HOSTNAME))) .forcePathStyle(true) .build(); } else { diff --git a/shipment-picture-lambda-validator/src/main/java/dev/ancaghenade/shipmentpicturelambdavalidator/SNSClientHelper.java b/shipment-picture-lambda-validator/src/main/java/dev/ancaghenade/shipmentpicturelambdavalidator/SNSClientHelper.java index 2a63e36..96cab5d 100644 --- a/shipment-picture-lambda-validator/src/main/java/dev/ancaghenade/shipmentpicturelambdavalidator/SNSClientHelper.java +++ b/shipment-picture-lambda-validator/src/main/java/dev/ancaghenade/shipmentpicturelambdavalidator/SNSClientHelper.java @@ -1,30 +1,29 @@ package dev.ancaghenade.shipmentpicturelambdavalidator; -import java.io.IOException; import java.net.URI; -import software.amazon.awssdk.regions.Region; +import java.util.Objects; import software.amazon.awssdk.services.sns.SnsClient; public class SNSClientHelper { - private static final String ENVIRONMENT = System.getenv("ENVIRONMENT"); - private static PropertiesProvider properties = new PropertiesProvider(); - + private static final String LOCALSTACK_HOSTNAME = System.getenv("LOCALSTACK_HOSTNAME"); private static String snsTopicArn; - public static SnsClient getSnsClient() throws IOException { + public static SnsClient getSnsClient() { var clientBuilder = SnsClient.builder(); - if (properties.getProperty("environment.dev").equals(ENVIRONMENT)) { - snsTopicArn = properties.getProperty("sns.arn.dev"); + if (Objects.nonNull(LOCALSTACK_HOSTNAME)) { + snsTopicArn = String.format("arn:aws:sns:%s:000000000000:update_shipment_picture_topic", + Location.REGION.getRegion()); return clientBuilder - .region(Region.of(properties.getProperty("aws.region"))) - .endpointOverride(URI.create(properties.getProperty("sns.endpoint"))) + .region(Location.REGION.getRegion()) + .endpointOverride(URI.create(String.format("http://%s:4566", LOCALSTACK_HOSTNAME))) .build(); } else { - snsTopicArn = properties.getProperty("sns.arn.prod"); + snsTopicArn = String.format("arn:aws:sns:%s:%s:update_shipment_picture_topic", + Location.REGION.getRegion(), "932043840972"); return clientBuilder.build(); } } diff --git a/shipment-picture-lambda-validator/src/main/java/dev/ancaghenade/shipmentpicturelambdavalidator/ServiceHandler.java b/shipment-picture-lambda-validator/src/main/java/dev/ancaghenade/shipmentpicturelambdavalidator/ServiceHandler.java index 827a3e9..649fa8f 100644 --- a/shipment-picture-lambda-validator/src/main/java/dev/ancaghenade/shipmentpicturelambdavalidator/ServiceHandler.java +++ b/shipment-picture-lambda-validator/src/main/java/dev/ancaghenade/shipmentpicturelambdavalidator/ServiceHandler.java @@ -14,9 +14,11 @@ import java.util.Objects; import javax.imageio.ImageIO; import org.apache.http.entity.ContentType; +import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.sns.SnsClient; import software.amazon.awssdk.services.sns.model.PublishRequest; @@ -24,8 +26,7 @@ public class ServiceHandler implements RequestStreamHandler { - private static final String BUCKET_NAME = "shipment-picture-bucket"; - + private static final String BUCKET_NAME = System.getenv("BUCKET"); public ServiceHandler() { } @@ -50,9 +51,15 @@ public void handleRequest(InputStream inputStream, OutputStream outputStream, .key(objectKey) .build(); - var s3ObjectResponse = s3Client.getObject( - getObjectRequest); - + ResponseInputStream s3ObjectResponse; + try { + s3ObjectResponse = s3Client.getObject( + getObjectRequest); + } catch (Exception e) { + e.printStackTrace(); + context.getLogger().log(e.getMessage()); + return; + } context.getLogger().log("Object fetched"); // Check if the image was already processed @@ -150,11 +157,7 @@ private S3Client acquireS3Client() { } private SnsClient acquireSnsClient() { - try { - return SNSClientHelper.getSnsClient(); - } catch (IOException e) { - throw new RuntimeException(e); - } + return SNSClientHelper.getSnsClient(); } } diff --git a/shipment-picture-lambda-validator/src/main/resources/lambda_update_script.sh b/shipment-picture-lambda-validator/src/main/resources/lambda_update_script.sh index c7b5ac5..9576475 100644 --- a/shipment-picture-lambda-validator/src/main/resources/lambda_update_script.sh +++ b/shipment-picture-lambda-validator/src/main/resources/lambda_update_script.sh @@ -3,8 +3,8 @@ awslocal lambda update-function-code --function-name shipment-picture-lambda-validator \ --zip-file fileb://target/shipment-picture-lambda-validator.jar \ - --region eu-central-1 + --region us-east-1 aws lambda update-function-code --function-name shipment-picture-lambda-validator \ --zip-file fileb://target/shipment-picture-lambda-validator.jar \ - --region eu-central-1 \ No newline at end of file + --region us-east-1 \ No newline at end of file diff --git a/src/main/java/dev/ancaghenade/shipmentlistdemo/buckets/BucketName.java b/src/main/java/dev/ancaghenade/shipmentlistdemo/buckets/BucketName.java index aff8987..c8054d4 100644 --- a/src/main/java/dev/ancaghenade/shipmentlistdemo/buckets/BucketName.java +++ b/src/main/java/dev/ancaghenade/shipmentlistdemo/buckets/BucketName.java @@ -1,15 +1,25 @@ package dev.ancaghenade.shipmentlistdemo.buckets; -import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; +import org.springframework.stereotype.Component; -@Getter -public enum BucketName { +@Component +@Configuration +@PropertySource(value = "classpath:buckets.properties") +public class BucketName { - SHIPMENT_PICTURE("shipment-picture-bucket"); + @Value("${shipment-picture-bucket}") + private String shipmentPictureBucket; + @Value("${shipment-picture-bucket-validator}") + private String shipmentPictureValidatorBucket; - private final String bucketName; + public String getShipmentPictureBucket() { + return shipmentPictureBucket; + } - BucketName(String bucketName) { - this.bucketName = bucketName; + public String getShipmentPictureValidatorBucket() { + return shipmentPictureValidatorBucket; } } diff --git a/src/main/java/dev/ancaghenade/shipmentlistdemo/repository/DynamoDBService.java b/src/main/java/dev/ancaghenade/shipmentlistdemo/repository/DynamoDBService.java index 300bdd3..7f5bfbd 100644 --- a/src/main/java/dev/ancaghenade/shipmentlistdemo/repository/DynamoDBService.java +++ b/src/main/java/dev/ancaghenade/shipmentlistdemo/repository/DynamoDBService.java @@ -39,6 +39,7 @@ public Optional getShipment(String shipmentId) { public String delete(String shipmentId) { shipmentTable.deleteItem(Key.builder().partitionValue(shipmentId).build()); + return "Shipment has been deleted"; } diff --git a/src/main/java/dev/ancaghenade/shipmentlistdemo/repository/S3StorageService.java b/src/main/java/dev/ancaghenade/shipmentlistdemo/repository/S3StorageService.java index 30eb28e..50ff5c9 100644 --- a/src/main/java/dev/ancaghenade/shipmentlistdemo/repository/S3StorageService.java +++ b/src/main/java/dev/ancaghenade/shipmentlistdemo/repository/S3StorageService.java @@ -3,16 +3,24 @@ import dev.ancaghenade.shipmentlistdemo.buckets.BucketName; import dev.ancaghenade.shipmentlistdemo.util.FileUtil; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; +import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Error; +import software.amazon.awssdk.services.s3.model.S3Object; @Service public class S3StorageService { @@ -20,16 +28,18 @@ public class S3StorageService { private final S3Client s3; private static final Logger LOGGER = LoggerFactory.getLogger(S3StorageService.class); + private final BucketName bucketName; @Autowired - public S3StorageService(S3Client s3) { + public S3StorageService(S3Client s3, BucketName bucketName) { this.s3 = s3; + this.bucketName = bucketName; } public void save(String path, String fileName, MultipartFile multipartFile) throws IOException { PutObjectRequest putObjectRequest = PutObjectRequest.builder() - .bucket(BucketName.SHIPMENT_PICTURE.getBucketName()) + .bucket(bucketName.getShipmentPictureBucket()) .key(path + "/" + fileName) .contentType(multipartFile.getContentType()) .contentLength(multipartFile.getSize()) @@ -42,7 +52,7 @@ public void save(String path, String fileName, public byte[] download(String key) throws IOException { GetObjectRequest getObjectRequest = GetObjectRequest.builder() - .bucket(BucketName.SHIPMENT_PICTURE.getBucketName()) + .bucket(bucketName.getShipmentPictureBucket()) .key(key) .build(); byte[] object = new byte[0]; @@ -54,4 +64,34 @@ public byte[] download(String key) throws IOException { return object; } + public void delete(String folderPrefix) { + List keysToDelete = new ArrayList<>(); + s3.listObjectsV2Paginator( + builder -> builder.bucket(bucketName.getShipmentPictureBucket()) + .prefix(folderPrefix + "/")) + .contents().stream() + .map(S3Object::key) + .forEach(key -> keysToDelete.add(ObjectIdentifier.builder().key(key).build())); + + DeleteObjectsRequest deleteRequest = DeleteObjectsRequest.builder() + .bucket(bucketName.getShipmentPictureBucket()) + .delete(builder -> builder.objects(keysToDelete).build()) + .build(); + + try { + DeleteObjectsResponse response = s3.deleteObjects(deleteRequest); + List errors = response.errors(); + if (!errors.isEmpty()) { + LOGGER.error("Errors occurred while deleting objects:"); + errors.forEach(error -> System.out.println("Object: " + error.key() + + ", Error Code: " + error.code() + + ", Error Message: " + error.message())); + } else { + LOGGER.info("Objects deleted successfully."); + } + } catch (SdkException e) { + LOGGER.error("Error occurred during object deletion: " + e.getMessage()); + } + } + } diff --git a/src/main/java/dev/ancaghenade/shipmentlistdemo/service/ShipmentService.java b/src/main/java/dev/ancaghenade/shipmentlistdemo/service/ShipmentService.java index 8ae0725..791d264 100644 --- a/src/main/java/dev/ancaghenade/shipmentlistdemo/service/ShipmentService.java +++ b/src/main/java/dev/ancaghenade/shipmentlistdemo/service/ShipmentService.java @@ -34,6 +34,7 @@ public List getAllShipments() { } public String deleteShipment(String shipmentId) { + s3StorageService.delete(shipmentId); return dynamoDBService.delete(shipmentId); } @@ -59,12 +60,12 @@ public void uploadShipmentImage(String shipmentId, MultipartFile file) { } catch (IOException e) { throw new IllegalStateException(e); } - shipment.setImageLink(fileName); + shipment.setImageLink(format("%s/%s", path, fileName)); dynamoDBService.upsert(shipment); } - public byte[] downloadShipmentImage(String shipmentId) { + public byte[] downloadShipmentImage(String shipmentId) throws IllegalStateException { Shipment shipment = dynamoDBService.getShipment(shipmentId).stream() .findFirst() .orElseThrow( diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 6762248..88516c3 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -8,4 +8,4 @@ aws: endpoint: http://localhost.localstack.cloud:4566/ sqs: endpoint: http://localhost:4566/000000000000 - region: eu-central-1 \ No newline at end of file + region: us-east-1 \ No newline at end of file diff --git a/src/main/resources/application-prod.yml b/src/main/resources/application-prod.yml index a3487cd..28d3176 100644 --- a/src/main/resources/application-prod.yml +++ b/src/main/resources/application-prod.yml @@ -8,4 +8,4 @@ aws: endpoint: https://s3.eu-central-1.amazonaws.com sqs: endpoint: https://sqs.eu-central-1.amazonaws.com - region: eu-central-1 \ No newline at end of file + region: us-east-1 \ No newline at end of file diff --git a/src/main/resources/buckets.properties b/src/main/resources/buckets.properties new file mode 100755 index 0000000..a40a93f --- /dev/null +++ b/src/main/resources/buckets.properties @@ -0,0 +1,2 @@ +shipment-picture-bucket=shipment-picture-bucket-concise-malamute +shipment-picture-bucket-validator=shipment-picture-lambda-validator-bucket-concise-malamute diff --git a/setup/terraform/cleanup.sh b/terraform/cleanup.sh similarity index 100% rename from setup/terraform/cleanup.sh rename to terraform/cleanup.sh diff --git a/setup/terraform/data.json b/terraform/data.json similarity index 100% rename from setup/terraform/data.json rename to terraform/data.json diff --git a/setup/terraform/locals.tf b/terraform/locals.tf similarity index 100% rename from setup/terraform/locals.tf rename to terraform/locals.tf diff --git a/setup/terraform/main.tf b/terraform/main.tf similarity index 83% rename from setup/terraform/main.tf rename to terraform/main.tf index a9dda73..1d2f68d 100644 --- a/setup/terraform/main.tf +++ b/terraform/main.tf @@ -7,12 +7,21 @@ terraform { } } provider "aws" { - region = "eu-central-1" + region = "us-east-1" +} + +provider "random" { + version = "3.1.0" +} + +resource "random_pet" "random_name" { + length = 2 + separator = "-" } # S3 bucket resource "aws_s3_bucket" "shipment_picture_bucket" { - bucket = "shipment-picture-bucket" + bucket = "shipment-picture-bucket-${random_pet.random_name.id}" force_destroy = true lifecycle { prevent_destroy = false @@ -48,7 +57,7 @@ resource "aws_dynamodb_table_item" "shipment" { # Define a bucket for the lambda zip resource "aws_s3_bucket" "lambda_code_bucket" { - bucket = "shipment-picture-lambda-validator-bucket" + bucket = "shipment-picture-lambda-validator-bucket-${random_pet.random_name.id}" force_destroy = true lifecycle { prevent_destroy = false @@ -57,7 +66,7 @@ resource "aws_s3_bucket" "lambda_code_bucket" { # Lambda source code resource "aws_s3_bucket_object" "lambda_code" { - source = "../../shipment-picture-lambda-validator/target/shipment-picture-lambda-validator.jar" + source = "../shipment-picture-lambda-validator/target/shipment-picture-lambda-validator.jar" bucket = aws_s3_bucket.lambda_code_bucket.id key = "shipment-picture-lambda-validator.jar" } @@ -74,7 +83,7 @@ resource "aws_lambda_function" "shipment_picture_lambda_validator" { timeout = 60 environment { variables = { - ENVIRONMENT = var.env + BUCKET = aws_s3_bucket.shipment_picture_bucket.bucket } } } @@ -150,8 +159,8 @@ resource "aws_iam_role_policy" "lambda_exec_policy" { "sns:Publish" ], "Resource": [ - "arn:aws:s3:::shipment-picture-bucket", - "arn:aws:s3:::shipment-picture-bucket/*", + "arn:aws:s3:::shipment-picture-bucket-${random_pet.random_name.id}", + "arn:aws:s3:::shipment-picture-bucket-${random_pet.random_name.id}/*", "${aws_sns_topic.update_shipment_picture_topic.arn}" ] } @@ -218,4 +227,15 @@ resource "aws_sns_topic_subscription" "my_topic_subscription" { confirmation_timeout_in_minutes = 1 } +# save generated bucket name to properties file +resource "local_file" "properties_file" { + content = <<-EOT + shipment-picture-bucket=${aws_s3_bucket.shipment_picture_bucket.bucket} + shipment-picture-bucket-validator=${aws_s3_bucket.lambda_code_bucket.bucket} + EOT + depends_on = [aws_s3_bucket.shipment_picture_bucket] + + filename = "../src/main/resources/buckets.properties" +} +