diff --git a/cloud/AWS/datastream/lambda_functions/checker_lambda.zip b/cloud/AWS/datastream/lambda_functions/checker_lambda.zip new file mode 100644 index 00000000..ad13bc67 Binary files /dev/null and b/cloud/AWS/datastream/lambda_functions/checker_lambda.zip differ diff --git a/cloud/AWS/datastream/lambda_functions/commander_lambda.zip b/cloud/AWS/datastream/lambda_functions/commander_lambda.zip new file mode 100644 index 00000000..46724f25 Binary files /dev/null and b/cloud/AWS/datastream/lambda_functions/commander_lambda.zip differ diff --git a/cloud/AWS/datastream/lambda_functions/poller_lambda.zip b/cloud/AWS/datastream/lambda_functions/poller_lambda.zip new file mode 100644 index 00000000..2680b834 Binary files /dev/null and b/cloud/AWS/datastream/lambda_functions/poller_lambda.zip differ diff --git a/cloud/AWS/datastream/lambda_functions/starter_lambda.zip b/cloud/AWS/datastream/lambda_functions/starter_lambda.zip new file mode 100644 index 00000000..72e633ed Binary files /dev/null and b/cloud/AWS/datastream/lambda_functions/starter_lambda.zip differ diff --git a/cloud/AWS/datastream/lambda_functions/stopper_lambda.zip b/cloud/AWS/datastream/lambda_functions/stopper_lambda.zip new file mode 100644 index 00000000..f50f91d6 Binary files /dev/null and b/cloud/AWS/datastream/lambda_functions/stopper_lambda.zip differ diff --git a/cloud/AWS/datastream/main.tf b/cloud/AWS/datastream/main.tf new file mode 100644 index 00000000..f140d6a1 --- /dev/null +++ b/cloud/AWS/datastream/main.tf @@ -0,0 +1,369 @@ +provider "aws" { + region = var.region +} + +terraform { + required_version = ">= 0.12" +} + +variable "region" {} +variable "starter_lambda_name" {} +variable "commander_lambda_name" {} +variable "poller_lambda_name" {} +variable "checker_lambda_name" {} +variable "stopper_lambda_name" {} +variable "lambda_policy_name" {} +variable "lambda_role_name" {} +variable "sm_name" {} +variable "runtime" {} + +resource "aws_iam_policy" "datastreamlambda_policy" { + name = var.lambda_policy_name + description = "Policy with permissions for datastreamlambda" + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Action = [ + "iam:GetRole", + "iam:PassRole" + ], + Resource = "*" + }, + { + Effect = "Allow", + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + Resource = "arn:aws:logs:*:*:*" + }, + { + Effect = "Allow", + Action = [ + "ssm:SendCommand", + "ssm:DescribeInstanceInformation", + "ssm:GetCommandInvocation", + "ssm:GetDeployablePatchSnapshotForInstance", + "ssm:GetDocument", + "ssm:DescribeDocument", + "ssm:GetManifest", + "ssm:GetParameter", + "ssm:GetParameters", + "ssm:ListAssociations", + "ssm:ListInstanceAssociations", + "ssm:PutInventory", + "ssm:PutComplianceItems", + "ssm:PutConfigurePackageResult", + "ssm:UpdateAssociationStatus", + "ssm:UpdateInstanceAssociationStatus", + "ssm:UpdateInstanceInformation" + ], + Resource = "*" + }, + { + Effect = "Allow", + Action = [ + "s3:*" + ], + Resource = "*" + }, + { + Effect = "Allow", + Action = [ + "ec2messages:AcknowledgeMessage", + "ec2messages:DeleteMessage", + "ec2messages:FailMessage", + "ec2messages:GetEndpoint", + "ec2messages:GetMessages", + "ec2messages:SendReply" + ], + Resource = "*" + }, + { + Effect = "Allow", + Action = [ + "ssmmessages:CreateControlChannel", + "ssmmessages:CreateDataChannel", + "ssmmessages:OpenControlChannel", + "ssmmessages:OpenDataChannel" + ], + Resource = "*" + }, + { + Effect = "Allow" + Action = [ + "ec2:*" + ] + Resource = "*" + } + ] + }) +} + + +resource "aws_iam_role" "datastreamlambda_role" { + name = var.lambda_role_name + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow", + Principal = { + Service = "lambda.amazonaws.com" + }, + Action = "sts:AssumeRole" + }] + }) +} + +resource "aws_iam_policy_attachment" "datastream_attachment" { + name = "datastream_attachment" + roles = [aws_iam_role.datastreamlambda_role.name] + policy_arn = aws_iam_policy.datastreamlambda_policy.arn +} + + +resource "aws_lambda_function" "starter_lambda" { + function_name = var.starter_lambda_name + role = aws_iam_role.datastreamlambda_role.arn + handler = "lambda_function.lambda_handler" + runtime = var.runtime + filename = "${path.module}/lambda_functions/starter_lambda.zip" + timeout = 180 +} + +resource "aws_lambda_function" "commander_lambda" { + function_name = var.commander_lambda_name + role = aws_iam_role.datastreamlambda_role.arn + handler = "lambda_function.lambda_handler" + runtime = var.runtime + filename = "${path.module}/lambda_functions/commander_lambda.zip" + timeout = 60 +} + +resource "aws_lambda_function" "poller_lambda" { + function_name = var.poller_lambda_name + role = aws_iam_role.datastreamlambda_role.arn + handler = "lambda_function.lambda_handler" + runtime = var.runtime + filename = "${path.module}/lambda_functions/poller_lambda.zip" + timeout = 900 +} + +resource "aws_lambda_function" "checker_lambda" { + function_name = var.checker_lambda_name + role = aws_iam_role.datastreamlambda_role.arn + handler = "lambda_function.lambda_handler" + runtime = var.runtime + filename = "${path.module}/lambda_functions/checker_lambda.zip" + timeout = 60 +} + +resource "aws_lambda_function" "stopper_lambda" { + function_name = var.stopper_lambda_name + role = aws_iam_role.datastreamlambda_role.arn + handler = "lambda_function.lambda_handler" + runtime = var.runtime + filename = "${path.module}/lambda_functions/stopper_lambda.zip" + timeout = 180 +} + + +resource "aws_iam_role" "iam_for_sfn" { + name = "statemachine_role" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow", + Principal = { + Service = "states.amazonaws.com" + }, + Action = "sts:AssumeRole" + }] + }) +} + +resource "aws_iam_policy" "lambda_invoke_policy" { + name = "lambda_invoke_policy" + description = "Policy to allow invoking Lambda functions" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow", + Action = "lambda:InvokeFunction", + Resource = [ "*" + ] + }] + }) +} + +resource "aws_iam_role_policy_attachment" "lambda_invoke_attachment" { + role = aws_iam_role.iam_for_sfn.name + policy_arn = aws_iam_policy.lambda_invoke_policy.arn +} + + +resource "aws_sfn_state_machine" "my_state_machine" { + name = var.sm_name + role_arn = aws_iam_role.iam_for_sfn.arn + definition = < /home/ec2-user/ngen-datastream/log.txt"], + "commands" : [ + "mkdir -p /home/ec2-user/ngen-datastream/data/mount", + "mount-s3 ngen-datastream /home/ec2-user/ngen-datastream/data/mount", + "/home/ec2-user/ngen-datastream/scripts/stream.sh -c /home/ec2-user/ngen-datastream/configs/conf_datastream_daily.sh > /home/ec2-user/ngen-datastream/log.txt" + ], "bucket" : "ngen-datastream", "obj_key" : "daily/DATE/ngen-run/outputs/cat-1.csv" } diff --git a/cloud/AWS/start_ami/lambda_handler.py b/cloud/AWS/start_ami/lambda_function.py similarity index 100% rename from cloud/AWS/start_ami/lambda_handler.py rename to cloud/AWS/start_ami/lambda_function.py diff --git a/cloud/AWS/startup_ec2.sh b/cloud/AWS/startup_ec2.sh index dbee5e87..9863b715 100644 --- a/cloud/AWS/startup_ec2.sh +++ b/cloud/AWS/startup_ec2.sh @@ -25,8 +25,8 @@ sudo dnf install docker -y sudo systemctl start docker sudo usermod -aG docker ${USER} mkdir -p /home/ec2-user/ngen-datastream/data/mount - sudo yum install -y ./mount-s3.rpm + echo "cd docker && sudo docker build -t awiciroh/ngen-deps:latest -f Dockerfile.ngen-deps --no-cache . && docker build -t awiciroh/t-route:latest -f Dockerfile.t-route . --no-cache && docker build -t awiciroh/ngen -f Dockerfile.ngen . --no-cache && docker build -t awiciroh/ciroh-ngen-image:latest-local -f Dockerfile . --no-cache" echo "copy that ^^ and log out of session, log back in and run that command" diff --git a/cloud/AWS/state_machine/datastream.asl.json b/cloud/AWS/state_machine/datastream.asl.json deleted file mode 100644 index 161371d6..00000000 --- a/cloud/AWS/state_machine/datastream.asl.json +++ /dev/null @@ -1,166 +0,0 @@ -{ - "Comment": "A description of my state machine", - "StartAt": "EC2StarterFromAMI", - "States": { - "EC2StarterFromAMI": { - "Type": "Task", - "Resource": "arn:aws:states:::lambda:invoke", - "OutputPath": "$.Payload", - "Parameters": { - "Payload.$": "$", - "FunctionName": "arn:aws:lambda:us-east-2:857712214391:function:ec2start_ami:$LATEST" - }, - "Retry": [ - { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException", - "States.Timeout" - ], - "BackoffRate": 2, - "Comment": "Try restarting", - "IntervalSeconds": 2, - "MaxAttempts": 10 - } - ], - "Next": "Commander" - }, - "Commander": { - "Type": "Task", - "Resource": "arn:aws:states:::lambda:invoke", - "OutputPath": "$.Payload", - "Parameters": { - "Payload.$": "$", - "FunctionName": "arn:aws:lambda:us-east-2:857712214391:function:ec2commander:$LATEST" - }, - "Retry": [ - { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException" - ], - "IntervalSeconds": 1, - "MaxAttempts": 3, - "BackoffRate": 2 - } - ], - "Next": "EC2Poller", - "Catch": [ - { - "ErrorEquals": [ - "States.TaskFailed" - ], - "Comment": "In case of failure", - "Next": "EC2Stopper", - "ResultPath": "$.failedInput" - } - ] - }, - "EC2Poller": { - "Type": "Task", - "Resource": "arn:aws:states:::lambda:invoke", - "OutputPath": "$.Payload", - "Parameters": { - "Payload.$": "$", - "FunctionName": "arn:aws:lambda:us-east-2:857712214391:function:CommandPoller:$LATEST" - }, - "Retry": [ - { - "ErrorEquals": [ - "States.Timeout" - ], - "IntervalSeconds": 1, - "Comment": "Retry for a long time just incase forcingprocessor takes awhile", - "BackoffRate": 1, - "MaxAttempts": 100 - } - ], - "Next": "Choice", - "Catch": [ - { - "ErrorEquals": [ - "States.TaskFailed" - ], - "Next": "EC2Stopper", - "Comment": "Kill EC2 incase of failure", - "ResultPath": "$.failedInput" - } - ] - }, - "Choice": { - "Type": "Choice", - "Choices": [ - { - "Variable": "$.ii_pass", - "BooleanEquals": true, - "Next": "RunChecker" - }, - { - "Variable": "$.ii_pass", - "BooleanEquals": false, - "Next": "EC2Poller" - } - ] - }, - "RunChecker": { - "Type": "Task", - "Resource": "arn:aws:states:::lambda:invoke", - "OutputPath": "$.Payload", - "Parameters": { - "Payload.$": "$", - "FunctionName": "arn:aws:lambda:us-east-2:857712214391:function:S3ObjectChecker:$LATEST" - }, - "Retry": [ - { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException" - ], - "IntervalSeconds": 1, - "MaxAttempts": 3, - "BackoffRate": 2 - } - ], - "Next": "EC2Stopper", - "Catch": [ - { - "ErrorEquals": [ - "States.TaskFailed" - ], - "Comment": "Kill EC2 incase of failure", - "Next": "EC2Stopper", - "ResultPath": "$.failedInput" - } - ] - }, - "EC2Stopper": { - "Type": "Task", - "Resource": "arn:aws:states:::lambda:invoke", - "OutputPath": "$.Payload", - "Parameters": { - "Payload.$": "$", - "FunctionName": "arn:aws:lambda:us-east-2:857712214391:function:EC2Stopper:$LATEST" - }, - "Retry": [ - { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException" - ], - "IntervalSeconds": 1, - "MaxAttempts": 3, - "BackoffRate": 2 - } - ], - "End": true - } - } - } \ No newline at end of file diff --git a/cloud/AWS/stopper/lambda_function.py b/cloud/AWS/stopper/lambda_function.py new file mode 100644 index 00000000..2d76c84e --- /dev/null +++ b/cloud/AWS/stopper/lambda_function.py @@ -0,0 +1,19 @@ +import boto3 + +client_ec2 = boto3.client('ec2') + +def lambda_handler(event, context): + """ + Generic Poller funcion + """ + + instance_id = event['instance_parameters']['InstanceId'] + + print(f'Shutting down processor {instance_id}') + client_ec2.stop_instances(InstanceIds=[instance_id]) + + waiter = client_ec2.get_waiter('instance_stopped') + waiter.wait(InstanceIds=[instance_id]) + + print(f'Instance {instance_id} has been successfully stopped.') + diff --git a/cloud/AWS/streamcommander/lambda_handler.py b/cloud/AWS/streamcommander/lambda_function.py similarity index 94% rename from cloud/AWS/streamcommander/lambda_handler.py rename to cloud/AWS/streamcommander/lambda_function.py index 09ea3291..091de0b6 100644 --- a/cloud/AWS/streamcommander/lambda_handler.py +++ b/cloud/AWS/streamcommander/lambda_function.py @@ -37,7 +37,7 @@ def lambda_handler(event, context): InstanceIds=[instance_id], DocumentName='AWS-RunShellScript', Parameters={'commands': event['commands'], - "executionTimeout": [f"{3600*24}" for x in range(len(event['commands']))] + "executionTimeout": [f"{3600*24}"] } ) wait_for_command_response(response,instance_id) diff --git a/scripts/stream.sh b/scripts/stream.sh index 287a989c..21ca3569 100755 --- a/scripts/stream.sh +++ b/scripts/stream.sh @@ -53,6 +53,7 @@ while [ "$#" -gt 0 ]; do -i|--id-type) SUBSET_ID_TYPE="$2"; shift 2;; -I|--id) SUBSET_ID="$2"; shift 2;; -v|--version) HYDROFABRIC_VERSION="$2"; shift 2;; + -S|--s3-mount) S3_MOUNT="$2"; shift 2;; -c|--conf-file) CONF_FILE="$2"; shift 2;; *) usage;; esac