diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..fbe8633 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,44 @@ +name: Run Integration Tests + +on: + push: + paths-ignore: + - 'README.md' + branches: + - main + pull_request: + branches: + - main + +jobs: + cdk: + name: Chaos Testing + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '21' + + - name: Build Lambdas + run: cd lambda-functions && mvn clean package shade:shade + + - name: Spin up LocalStack + run: | + docker-compose up -d + sleep 100 + env: + LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }} + + - name: Run Integration Tests + run: | + pip3 install boto3 pytest + pytest + env: + AWS_DEFAULT_REGION: us-east-1 + AWS_REGION: us-east-1 + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..130ecfc --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +# LocalStack + +volume/ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d9a2c6e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,30 @@ +version: "3.9" + +services: + localstack: + networks: + - ls_network + container_name: localstack + image: localstack/localstack-pro:latest + ports: + - "127.0.0.1:4566:4566" # LocalStack Gateway + - "127.0.0.1:4510-4559:4510-4559" # external services port range + - "127.0.0.1:443:443" + environment: + - DOCKER_HOST=unix:///var/run/docker.sock #unix socket to communicate with the docker daemon + - LOCALSTACK_HOST=localstack # where services are available from other containers + - LAMBDA_DOCKER_NETWORK=ls_network + - LOCALSTACK_AUTH_TOKEN=${LOCALSTACK_AUTH_TOKEN:?} + - LAMBDA_RUNTIME_ENVIRONMENT_TIMEOUT=60 + - PERSIST_ALL=false + - EXTENSION_AUTO_INSTALL=localstack-extension-outages + - LAMBDA_RUNTIME_ENVIRONMENT_TIMEOUT=600 + volumes: + - "./volume:/var/lib/localstack" + - "/var/run/docker.sock:/var/run/docker.sock" + - "./lambda-functions/target/product-lambda.jar:/etc/localstack/init/ready.d/target/product-lambda.jar" + - "./init-resources.sh:/etc/localstack/init/ready.d/init-resources.sh" + +networks: + ls_network: + name: ls_network diff --git a/init-resources.sh b/init-resources.sh new file mode 100755 index 0000000..4f1746c --- /dev/null +++ b/init-resources.sh @@ -0,0 +1,132 @@ +#!/bin/sh + +apt-get -y install jq + +echo "Create DynamoDB table..." +awslocal dynamodb create-table \ + --table-name Products \ + --attribute-definitions AttributeName=id,AttributeType=S \ + --key-schema AttributeName=id,KeyType=HASH \ + --provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5 \ + --region us-east-1 + +echo "Add Product Lambda..." +awslocal lambda create-function \ + --function-name add-product \ + --runtime java17 \ + --handler lambda.AddProduct::handleRequest \ + --memory-size 1024 \ + --zip-file fileb:///etc/localstack/init/ready.d/target/product-lambda.jar \ + --region us-east-1 \ + --role arn:aws:iam::000000000000:role/productRole \ + --environment Variables={AWS_REGION=us-east-1} + +echo "Get Product Lambda..." +awslocal lambda create-function \ + --function-name get-product \ + --runtime java17 \ + --handler lambda.GetProduct::handleRequest \ + --memory-size 1024 \ + --zip-file fileb:///etc/localstack/init/ready.d/target/product-lambda.jar \ + --region us-east-1 \ + --role arn:aws:iam::000000000000:role/productRole \ + --environment Variables={AWS_REGION=us-east-1} + +export REST_API_ID=12345 + +echo "Create Rest API..." +awslocal apigateway create-rest-api --name quote-api-gateway --tags '{"_custom_id_":"12345"}' --region us-east-1 + +echo "Export Parent ID..." +export PARENT_ID=$(awslocal apigateway get-resources --rest-api-id $REST_API_ID --region=us-east-1 | jq -r '.items[0].id') + +echo "Export Resource ID..." +export RESOURCE_ID=$(awslocal apigateway create-resource --rest-api-id $REST_API_ID --parent-id $PARENT_ID --path-part "productApi" --region=us-east-1 | jq -r '.id') + +echo "RESOURCE ID:" +echo $RESOURCE + +echo "Put GET Method..." +awslocal apigateway put-method \ +--rest-api-id $REST_API_ID \ +--resource-id $RESOURCE_ID \ +--http-method GET \ +--request-parameters "method.request.path.productApi=true" \ +--authorization-type "NONE" \ +--region=us-east-1 + +echo "Put POST Method..." +awslocal apigateway put-method \ +--rest-api-id $REST_API_ID \ +--resource-id $RESOURCE_ID \ +--http-method POST \ +--request-parameters "method.request.path.productApi=true" \ +--authorization-type "NONE" \ +--region=us-east-1 + + +echo "Update GET Method..." +awslocal apigateway update-method \ + --rest-api-id $REST_API_ID \ + --resource-id $RESOURCE_ID \ + --http-method GET \ + --patch-operations "op=replace,path=/requestParameters/method.request.querystring.param,value=true" \ + --region=us-east-1 + + +echo "Put POST Method Integration..." +awslocal apigateway put-integration \ + --rest-api-id $REST_API_ID \ + --resource-id $RESOURCE_ID \ + --http-method POST \ + --type AWS_PROXY \ + --integration-http-method POST \ + --uri arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:000000000000:function:add-product/invocations \ + --passthrough-behavior WHEN_NO_MATCH \ + --region=us-east-1 + +echo "Put GET Method Integration..." +awslocal apigateway put-integration \ + --rest-api-id $REST_API_ID \ + --resource-id $RESOURCE_ID \ + --http-method GET \ + --type AWS_PROXY \ + --integration-http-method GET \ + --uri arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:000000000000:function:get-product/invocations \ + --passthrough-behavior WHEN_NO_MATCH \ + --region=us-east-1 + +echo "Create DEV Deployment..." +awslocal apigateway create-deployment \ + --rest-api-id $REST_API_ID \ + --stage-name dev \ + --region=us-east-1 + +awslocal sns create-topic --name ProductEventsTopic + +awslocal sqs create-queue --queue-name ProductEventsQueue + +awslocal sqs get-queue-attributes --queue-url http://localhost:4566/000000000000/ProductEventsQueue --attribute-names QueueArn + +awslocal sns subscribe \ + --topic-arn arn:aws:sns:us-east-1:000000000000:ProductEventsTopic \ + --protocol sqs \ + --notification-endpoint arn:aws:sqs:us-east-1:000000000000:ProductEventsQueue + +awslocal lambda create-function \ + --function-name process-product-events \ + --runtime java17 \ + --handler lambda.DynamoDBWriterLambda::handleRequest \ + --memory-size 1024 \ + --zip-file fileb:///etc/localstack/init/ready.d/target/product-lambda.jar \ + --region us-east-1 \ + --role arn:aws:iam::000000000000:role/productRole + +awslocal lambda create-event-source-mapping \ + --function-name process-product-events \ + --batch-size 10 \ + --event-source-arn arn:aws:sqs:us-east-1:000000000000:ProductEventsQueue + +awslocal sqs set-queue-attributes \ + --queue-url http://localhost:4566/000000000000/ProductEventsQueue \ + --attributes VisibilityTimeout=10 diff --git a/lambda-functions/pom.xml b/lambda-functions/pom.xml new file mode 100644 index 0000000..a4a9347 --- /dev/null +++ b/lambda-functions/pom.xml @@ -0,0 +1,128 @@ + + + + 4.0.0 + + product-lambda + cloud.localstack + jar + 1.0-SNAPSHOT + + + 17 + 17 + false + + + + + software.amazon.awssdk + lambda + + + com.amazonaws + aws-lambda-java-core + 1.2.2 + + + software.amazon.awssdk + protocol-core + 2.20.69 + + + software.amazon.awssdk + s3 + + + software.amazon.awssdk + dynamodb + 2.20.68 + + + com.amazonaws + aws-lambda-java-events + 3.11.3 + + + + com.fasterxml.jackson.core + jackson-core + 2.13.3 + + + + + com.fasterxml.jackson.core + jackson-databind + 2.13.3 + + + + + com.fasterxml.jackson.core + jackson-annotations + 2.15.1 + + + software.amazon.awssdk + sns + 2.20.69 + + + + org.slf4j + slf4j-api + 2.0.7 + + + + ch.qos.logback + logback-classic + 1.4.7 + + + + + + + + software.amazon.awssdk + bom + 2.20.47 + pom + import + + + + + + product-lambda + + + src/main/resources + true + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.3 + + false + + + + package + + shade + + + + + + + \ No newline at end of file diff --git a/lambda-functions/src/main/java/lambda/AddProduct.java b/lambda-functions/src/main/java/lambda/AddProduct.java new file mode 100644 index 0000000..95c5511 --- /dev/null +++ b/lambda-functions/src/main/java/lambda/AddProduct.java @@ -0,0 +1,102 @@ +package lambda; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyRequestEvent; +import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; +import java.util.HashMap; +import java.util.Map; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.ConditionalCheckFailedException; +import software.amazon.awssdk.services.dynamodb.model.DynamoDbException; +import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; +import software.amazon.awssdk.services.sns.model.PublishRequest; + +public class AddProduct extends ProductApi implements + RequestHandler { + + private static final String TABLE_NAME = "Products"; + private static final String PRODUCT_ID = "id"; + + @Override + public APIGatewayProxyResponseEvent handleRequest(APIGatewayProxyRequestEvent requestEvent, + Context context) { + + Map productData; + try { + productData = objectMapper.readValue(requestEvent.getBody(), HashMap.class); + } catch (JsonMappingException e) { + throw new RuntimeException(e); + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { + throw new RuntimeException(e); + } + + HashMap itemValues = new HashMap<>(); + itemValues.put("id", AttributeValue.builder().s(productData.get("id")).build()); + itemValues.put("name", AttributeValue.builder().s(productData.get("name")).build()); + itemValues.put("price", AttributeValue.builder().n(productData.get("price")).build()); + itemValues.put("description", + AttributeValue.builder().s(productData.get("description")).build()); + + PutItemRequest putItemRequest = PutItemRequest.builder() + .tableName(TABLE_NAME) + .item(itemValues) + .conditionExpression("attribute_not_exists(id) OR id = :id") + .expressionAttributeValues( + Map.of(":id", AttributeValue.builder().s(productData.get("id")).build())) + .build(); + + Map headers = new HashMap<>(); + headers.put("Content-Type", "application/json"); + + try { + ddb.putItem(putItemRequest); + return new APIGatewayProxyResponseEvent().withStatusCode(200) + .withBody("Product added/updated successfully.") + .withIsBase64Encoded(false).withHeaders(headers); + } catch (ConditionalCheckFailedException e) { + return new APIGatewayProxyResponseEvent().withStatusCode(409) + .withBody("Product with the given ID already exists.") + .withIsBase64Encoded(false).withHeaders(headers); + } catch (DynamoDbException e) { + context.getLogger().log("Error: " + e.getMessage()); + // Publish message to SNS topic if DynamoDB operation fails. + String productDataJson; + try { + productDataJson = objectMapper.writeValueAsString(productData); + } catch (JsonProcessingException ex) { + throw new RuntimeException(ex); + } + PublishRequest publishRequest = PublishRequest.builder() + .message(productDataJson) + .topicArn(topicArn) + .build(); + context.getLogger().log("Sending to queue: " + productDataJson); + + snsClient.publish(publishRequest); + + return new APIGatewayProxyResponseEvent().withStatusCode(200) + .withBody("A DynamoDB error occurred. Message sent to queue.") + .withIsBase64Encoded(false).withHeaders(headers); + } catch (AwsServiceException ex) { + context.getLogger().log("AwsServiceException exception: " + ex.getMessage()); + return new APIGatewayProxyResponseEvent().withStatusCode(500) + .withBody(ex.getMessage()) + .withIsBase64Encoded(false).withHeaders(headers); + } catch (RuntimeException e) { + context.getLogger().log("Runtime exception: " + e.getMessage()); + return new APIGatewayProxyResponseEvent().withStatusCode(500) + .withBody(e.getMessage()) + .withIsBase64Encoded(false).withHeaders(headers); + } catch (Exception e) { + context.getLogger().log("Generic exception: " + e.getMessage()); + return new APIGatewayProxyResponseEvent().withStatusCode(500) + .withBody(e.getMessage()) + .withIsBase64Encoded(false).withHeaders(headers); + } + + } +} \ No newline at end of file diff --git a/lambda-functions/src/main/java/lambda/DynamoDBWriterLambda.java b/lambda-functions/src/main/java/lambda/DynamoDBWriterLambda.java new file mode 100644 index 0000000..5a6af98 --- /dev/null +++ b/lambda-functions/src/main/java/lambda/DynamoDBWriterLambda.java @@ -0,0 +1,64 @@ +package lambda; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.HashMap; +import java.util.Map; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.DynamoDbException; +import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; +import software.amazon.awssdk.services.dynamodb.model.PutItemResponse; + +public class DynamoDBWriterLambda extends ProductApi implements RequestHandler { + + private final ObjectMapper objectMapper = new ObjectMapper(); + + private static final String TABLE_NAME = "Products"; + + @Override + public Void handleRequest(SQSEvent event, Context context) { + + for (SQSEvent.SQSMessage msg : event.getRecords()) { + try { + JsonNode rootNode = objectMapper.readTree(msg.getBody()); + String messageContent = rootNode.get("Message").asText(); + + Map productData; + try { + productData = objectMapper.readValue(messageContent, HashMap.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + HashMap itemValues = new HashMap<>(); + itemValues.put("id", AttributeValue.builder().s(productData.get("id")).build()); + itemValues.put("name", AttributeValue.builder().s(productData.get("name")).build()); + itemValues.put("price", AttributeValue.builder().n(productData.get("price")).build()); + itemValues.put("description", + AttributeValue.builder().s(productData.get("description")).build()); + + // Put the item into the DynamoDB table + PutItemRequest putItemRequest = PutItemRequest.builder() + .tableName(TABLE_NAME) + .item(itemValues) + .build(); + PutItemResponse putItemResult = ddb.putItem(putItemRequest); + context.getLogger().log("Successfully processed message, result: " + putItemResult); + + } catch (DynamoDbException dbe) { + // Service unavailable, let the message go back to the queue after visibility timeout + context.getLogger().log( + "DynamoDB service is unavailable, message will be retried. Error: " + + dbe.getMessage()); + throw dbe; + } catch (Exception e) { + context.getLogger().log("Exception: Error processing the message: " + e.getMessage()); + } + } + return null; + } + +} diff --git a/lambda-functions/src/main/java/lambda/GetProduct.java b/lambda-functions/src/main/java/lambda/GetProduct.java new file mode 100644 index 0000000..ef9558d --- /dev/null +++ b/lambda-functions/src/main/java/lambda/GetProduct.java @@ -0,0 +1,71 @@ +package lambda; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyRequestEvent; +import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.HashMap; +import java.util.Map; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.DynamoDbException; +import software.amazon.awssdk.services.dynamodb.model.GetItemRequest; +import software.amazon.awssdk.services.dynamodb.model.GetItemResponse; + +public class GetProduct extends ProductApi implements + RequestHandler { + + private static final String TABLE_NAME = "Products"; + private static final String PRODUCT_ID = "id"; + private final ObjectMapper objectMapper = new ObjectMapper(); + + + @Override + public APIGatewayProxyResponseEvent handleRequest(APIGatewayProxyRequestEvent requestEvent, + Context context) { + String productId = requestEvent.getQueryStringParameters().get(PRODUCT_ID); + System.out.println(requestEvent); + System.out.println("PRODUCT ID: " + productId); + + HashMap valueMap = new HashMap<>(); + valueMap.put("id", AttributeValue.fromS(productId)); + + GetItemRequest getItemRequest = GetItemRequest.builder() + .tableName(TABLE_NAME) + .key(valueMap) + .build(); + + try { + GetItemResponse getItemResponse = ddb.getItem(getItemRequest); + if (getItemResponse.item() != null && !getItemResponse.item().isEmpty()) { + // Convert the result to JSON format + + Map responseBody = new HashMap<>(); + getItemResponse.item().forEach((k, v) -> responseBody.put(k, convertAttributeValue(v))); + + return new APIGatewayProxyResponseEvent().withStatusCode(200) + .withBody(objectMapper.writeValueAsString(responseBody)); + } else { + return new APIGatewayProxyResponseEvent().withStatusCode(404).withBody("Product not found"); + } + } catch (DynamoDbException | JsonProcessingException e) { + context.getLogger().log("Error: " + e.getMessage()); + return new APIGatewayProxyResponseEvent().withStatusCode(500) + .withBody("Internal server error"); + } + } + + private Object convertAttributeValue(AttributeValue value) { + if (value.s() != null) { + return value.s(); + } + if (value.n() != null) { + return value.n(); + } + if (value.b() != null) { + return value.b(); + } + return null; + } +} \ No newline at end of file diff --git a/lambda-functions/src/main/java/lambda/ProductApi.java b/lambda-functions/src/main/java/lambda/ProductApi.java new file mode 100644 index 0000000..eeafcae --- /dev/null +++ b/lambda-functions/src/main/java/lambda/ProductApi.java @@ -0,0 +1,47 @@ +package lambda; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.net.URI; + +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.retry.RetryPolicy; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.sns.SnsClient; + +public class ProductApi { + + protected static final String LOCALSTACK_HOSTNAME = System.getenv("LOCALSTACK_HOSTNAME"); + protected static final String AWS_REGION = System.getenv("AWS_REGION"); + protected static final String topicArn = "arn:aws:sns:us-east-1:000000000000:ProductEventsTopic"; + protected ObjectMapper objectMapper = new ObjectMapper(); + + // Define a custom retry policy + // Set maximum number of retries + RetryPolicy customRetryPolicy = RetryPolicy.builder() + .numRetries(3) + .build(); + + // Apply the custom retry policy to ClientOverrideConfiguration + ClientOverrideConfiguration clientOverrideConfig = ClientOverrideConfiguration.builder() + .retryPolicy(customRetryPolicy) + .build(); + + protected SnsClient snsClient = SnsClient.builder() + .endpointOverride(URI.create(String.format("http://%s:4566", LOCALSTACK_HOSTNAME))) + .credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create("test", "test"))) + .region(Region.of(AWS_REGION)) + .build(); + + protected DynamoDbClient ddb = DynamoDbClient.builder() + .endpointOverride(URI.create(String.format("http://%s:4566", LOCALSTACK_HOSTNAME))) + .credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create("test", "test"))) + .region(Region.of(AWS_REGION)) + .endpointDiscoveryEnabled(true) + .overrideConfiguration(clientOverrideConfig) + .build(); +} diff --git a/lambda-functions/src/main/resources/lambda_update_script.sh b/lambda-functions/src/main/resources/lambda_update_script.sh new file mode 100644 index 0000000..38b7d93 --- /dev/null +++ b/lambda-functions/src/main/resources/lambda_update_script.sh @@ -0,0 +1,4 @@ + +awslocal lambda update-function-code --function-name process-product-events \ + --zip-file fileb://target/product-lambda.jar \ + --region us-east-1 diff --git a/lambda-functions/target/product-lambda.jar b/lambda-functions/target/product-lambda.jar new file mode 100644 index 0000000..158257c Binary files /dev/null and b/lambda-functions/target/product-lambda.jar differ diff --git a/tests/test_outage.py b/tests/test_outage.py new file mode 100644 index 0000000..48f786f --- /dev/null +++ b/tests/test_outage.py @@ -0,0 +1,82 @@ +import pytest +import time +import boto3 +import requests + +# Replace with your LocalStack endpoint +LOCALSTACK_ENDPOINT = "http://localhost:4566" + +# Replace with your LocalStack DynamoDB table name +DYNAMODB_TABLE_NAME = "Products" + +# Replace with your Lambda function names +LAMBDA_FUNCTIONS = ["add-product", "get-product", "process-product-events"] + + +@pytest.fixture(scope="module") +def dynamodb_resource(): + return boto3.resource("dynamodb", endpoint_url=LOCALSTACK_ENDPOINT) + + +@pytest.fixture(scope="module") +def lambda_client(): + return boto3.client("lambda", endpoint_url=LOCALSTACK_ENDPOINT) + + +def test_dynamodb_table_exists(dynamodb_resource): + tables = dynamodb_resource.tables.all() + table_names = [table.name for table in tables] + assert DYNAMODB_TABLE_NAME in table_names + + +def test_lambda_functions_exist(lambda_client): + functions = lambda_client.list_functions()["Functions"] + function_names = [func["FunctionName"] for func in functions] + assert all(func_name in function_names for func_name in LAMBDA_FUNCTIONS) + + +def test_dynamodb_outage(): + outage_payload = [{"service": "dynamodb", "region": "us-east-1"}] + requests.post( + "http://outages.localhost.localstack.cloud:4566/outages", json=outage_payload + ) + + # Make a request to DynamoDB and assert an error + url = "http://12345.execute-api.localhost.localstack.cloud:4566/dev/productApi" + headers = {"Content-Type": "application/json"} + data = { + "id": "prod-1002", + "name": "Super Widget", + "price": "29.99", + "description": "A versatile widget that can be used for a variety of purposes. Durable, reliable, and affordable.", + } + + response = requests.post(url, headers=headers, json=data) + + assert "error" in response.text + + # Check if outage is running + outage_status = requests.get( + "http://outages.localhost.localstack.cloud:4566/outages" + ).json() + assert outage_payload == outage_status + + # Stop the outage + requests.post("http://outages.localhost.localstack.cloud:4566/outages", json=[]) + + # Check if outage is stopped + outage_status = requests.get( + "http://outages.localhost.localstack.cloud:4566/outages" + ).json() + assert not outage_status + + # Wait for a few seconds + time.sleep(60) + + # Query if there are items in DynamoDB table + dynamodb = boto3.resource("dynamodb", endpoint_url=LOCALSTACK_ENDPOINT) + table = dynamodb.Table(DYNAMODB_TABLE_NAME) + response = table.scan() + items = response["Items"] + print(items) + assert "Super Widget" in [item["name"] for item in items]