From a3bbfb0df23ab2e53ebbe1272a020b79339a364b Mon Sep 17 00:00:00 2001 From: Jordan Laser Date: Tue, 6 Feb 2024 13:30:03 -0700 Subject: [PATCH] terraform for datastream in AWS --- .../lambda_functions/checker_lambda.zip | Bin 0 -> 508 bytes .../lambda_functions/commander_lambda.zip | Bin 0 -> 817 bytes .../lambda_functions/poller_lambda.zip | Bin 0 -> 649 bytes .../lambda_functions/starter_lambda.zip | Bin 0 -> 814 bytes .../lambda_functions/stopper_lambda.zip | Bin 0 -> 433 bytes cloud/AWS/datastream/main.tf | 369 ++++++++++++++++++ cloud/AWS/datastream/variables.tfvars | 10 + cloud/AWS/execution_dailyrun.json | 8 +- .../{lambda_handler.py => lambda_function.py} | 0 cloud/AWS/startup_ec2.sh | 2 +- cloud/AWS/state_machine/datastream.asl.json | 166 -------- cloud/AWS/stopper/lambda_function.py | 19 + .../{lambda_handler.py => lambda_function.py} | 2 +- scripts/stream.sh | 1 + 14 files changed, 407 insertions(+), 170 deletions(-) create mode 100644 cloud/AWS/datastream/lambda_functions/checker_lambda.zip create mode 100644 cloud/AWS/datastream/lambda_functions/commander_lambda.zip create mode 100644 cloud/AWS/datastream/lambda_functions/poller_lambda.zip create mode 100644 cloud/AWS/datastream/lambda_functions/starter_lambda.zip create mode 100644 cloud/AWS/datastream/lambda_functions/stopper_lambda.zip create mode 100644 cloud/AWS/datastream/main.tf create mode 100644 cloud/AWS/datastream/variables.tfvars rename cloud/AWS/start_ami/{lambda_handler.py => lambda_function.py} (100%) delete mode 100644 cloud/AWS/state_machine/datastream.asl.json create mode 100644 cloud/AWS/stopper/lambda_function.py rename cloud/AWS/streamcommander/{lambda_handler.py => lambda_function.py} (94%) diff --git a/cloud/AWS/datastream/lambda_functions/checker_lambda.zip b/cloud/AWS/datastream/lambda_functions/checker_lambda.zip new file mode 100644 index 0000000000000000000000000000000000000000..ad13bc6781797dc5b019f6ba1ee9ed609f035c4c GIT binary patch literal 508 zcmWIWW@Zs#U|`^2VDxp1VA=4+!-WPO^ODnh;7+GF0GcbUO)}UDL!v-R?@7JIBHc9Mw!JAwI$3Vv~zV}_6 zX5HJg!%Xw!+Sd~Q%5SS=i+@Xy(|a=Kci}tFl)D+tD%V$B@BAV@#Z;wpg^ET>neT~f zpS}5)URjqpZ9>5$Ca>NV(wIy?|v&rS`W5pN~>vjdue9SW!`a7KcQWpi7kt5DBEc;^C-xTC0?+Wmweg>q-8XJKec+C7 zTF!~7<&Ijl`|2yJtCrjhb<#O&8+r8go(DO{LjxNdXg9fd~#vl1YhR!x92c@^^Tc3cjEUjjtymEW+n}51M4>E|7Q;H wW@M6M#uXnDz<6R{07ehPl12~uH8gAf}KGsrOHB<3ciB*v$e<|UV8=I7}ZRECCd zGBD5UJCq8-r4`%^j4Ush85qFC)R5Th+Xf=F-`5K?Dy4h--tMtw3F7cR&XwG=&A^Sr zZR2|BwcAoucKUMue7}2>uBPAb4L^_WlYd_KLZ&->Ye-wpl8Ki)%G(}_y0lIbJvoyx zd!pT=HyWqB_zo?p@;b4{S$(edvMR6RcI#GeFN{ren&RvjCt;qc7&`0Zsw5?U(HBu% zyc>VdC|JlVEV;x+{=8BXzxe`_u)P^;yWR#p-}xp+KRS-7GJv;);kU$BDW2au`fS`6 zyxz7b-pV}o_xWfOHjkS#b{TeXS*!IY{NanSIo%_v=W266Fup);mVcY(k2~MElW$#o zx4iOkW2|zn?i-NAdQog9_AREu8EbA*jarwgCFO;u0o{_99*2|lAl?|lT)iL{YhriW^QKd%cnC;q_)1#j}_9CnizShV`5A8-GyFaiGfhRqem43( zFZuZT-=B4Y-=j57|MvNK>%xRzm;KG|2XEJ$yOVcM`rojHZ~tv*UbFG}*M^$Khun-e zX}SG=`MkTEKYTvl;;*|mIKO_o)yhg#)nlM%d$V|i#u>{yDqP`JK8|-ld9ZrkEoDk{c5*X99nkd%bV|!rPD5N4CgyF zS5KVZTx99vV-siFzZBkcsq_cqMW2mc7p9yqlze#K)^~<=$)W{%k!{IaWYzP|Y+QeF z^#zkv@@l`h1H2iTqB74CMQCSEFiY_n;yb9v;88i!ctK##=-9|=Bj z-*)!<>l`i1n?E+Ow*=TTPUvM=6W!X>HJ7pf($%dSUcFGlGz5T3C++$PC(2si7W$H|e|41=&bnX3r zrY?Pw{oDy36c!##zw+aiPO0`Rn}pZ7<-safdsp~-$-M74*r5M#UQ4uKziY(-zn4do zCjK)|IGBAhZ?2-w>L_i`(+kA~rUj}-!dAZb&x8PBMfWQSIW6L*p|3pm*vfx> zKd1fCJv&oN^R=2K98W5BeHv0$&q=pWYTvUVUBf;`Z+_rM`;NHT&2OjiCVzNe*WSUx z`s(hJt!GaEco#h*cCJmw{ASiO-jQ$KtSwLfZ!*zc=4~`#Zd&RiQfDXVSexzXERQ=!isl zXow{DB=rjiMw?CAsNE~Y>2ua0Y3vbl9h+I63%X)+;uqV$`4 z-3z{o2~A#lH{Neg)5Tv8s!I9NeE%%UxnR29f@7{`s5|S5m`}_p#yx9hKI{G9&bE18 z)sf3A6;GzJ{hX>Tt8Tm|Qr}!`^9%hSzUD1`IjxHx-ae5YHaC%7>S1T_qAz0qtjw1_ z)|-(PaPIVk3+A5;XI_|M5V%Krj?U$ly62B}Y~Ho7T;{t}UYg@Vt#r*3aq|+l%)ZwV z!g}G?YRxN-XZZ~yEBjt6uMT;heS2ln*M$i;ITx>W`1PS@R+~zGOWW#WWgAZ*=`Zt4f>yBvupPTwX;^e<(B{`)Il`W#D z4I^iVTZkx2=)L^yzV!tA=2`x`+&}3Y`x*Z4Z>@M>cwG7iwfl_)v$_=zWv*d=I;Z-B zN7i$-=e!;1!g3Z}oo@R-KV5R)GFxRcFA|UvbIc0?w?-rt|F;RhJmH;N#qlCjW^s)p`t(ZE-YAm__wnGbWW>N;j^Z zbg*jTnw!FNc05)VvN}I&*4*$PI+2sNsV{drHZQhqrpHWek7{GatxS8j^X=pRWWUhu z&duFn^WI#^6N!KG>tBex*=(bocXx0FI-O}gBD&JIV&n0?UBaiX8uYHN_?x{^^j1Z^ z^nnAmB^_*kbdrwNUHiqjI#%@mOYs12MkYCCT!AP73{D0HV0bbtX#}yb1t}{eNYR2b Tz?+o~q=OL%BY?Cs$Pfkqz-pz{ literal 0 HcmV?d00001 diff --git a/cloud/AWS/datastream/main.tf b/cloud/AWS/datastream/main.tf new file mode 100644 index 00000000..f140d6a1 --- /dev/null +++ b/cloud/AWS/datastream/main.tf @@ -0,0 +1,369 @@ +provider "aws" { + region = var.region +} + +terraform { + required_version = ">= 0.12" +} + +variable "region" {} +variable "starter_lambda_name" {} +variable "commander_lambda_name" {} +variable "poller_lambda_name" {} +variable "checker_lambda_name" {} +variable "stopper_lambda_name" {} +variable "lambda_policy_name" {} +variable "lambda_role_name" {} +variable "sm_name" {} +variable "runtime" {} + +resource "aws_iam_policy" "datastreamlambda_policy" { + name = var.lambda_policy_name + description = "Policy with permissions for datastreamlambda" + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Action = [ + "iam:GetRole", + "iam:PassRole" + ], + Resource = "*" + }, + { + Effect = "Allow", + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + Resource = "arn:aws:logs:*:*:*" + }, + { + Effect = "Allow", + Action = [ + "ssm:SendCommand", + "ssm:DescribeInstanceInformation", + "ssm:GetCommandInvocation", + "ssm:GetDeployablePatchSnapshotForInstance", + "ssm:GetDocument", + "ssm:DescribeDocument", + "ssm:GetManifest", + "ssm:GetParameter", + "ssm:GetParameters", + "ssm:ListAssociations", + "ssm:ListInstanceAssociations", + "ssm:PutInventory", + "ssm:PutComplianceItems", + "ssm:PutConfigurePackageResult", + "ssm:UpdateAssociationStatus", + "ssm:UpdateInstanceAssociationStatus", + "ssm:UpdateInstanceInformation" + ], + Resource = "*" + }, + { + Effect = "Allow", + Action = [ + "s3:*" + ], + Resource = "*" + }, + { + Effect = "Allow", + Action = [ + "ec2messages:AcknowledgeMessage", + "ec2messages:DeleteMessage", + "ec2messages:FailMessage", + "ec2messages:GetEndpoint", + "ec2messages:GetMessages", + "ec2messages:SendReply" + ], + Resource = "*" + }, + { + Effect = "Allow", + Action = [ + "ssmmessages:CreateControlChannel", + "ssmmessages:CreateDataChannel", + "ssmmessages:OpenControlChannel", + "ssmmessages:OpenDataChannel" + ], + Resource = "*" + }, + { + Effect = "Allow" + Action = [ + "ec2:*" + ] + Resource = "*" + } + ] + }) +} + + +resource "aws_iam_role" "datastreamlambda_role" { + name = var.lambda_role_name + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow", + Principal = { + Service = "lambda.amazonaws.com" + }, + Action = "sts:AssumeRole" + }] + }) +} + +resource "aws_iam_policy_attachment" "datastream_attachment" { + name = "datastream_attachment" + roles = [aws_iam_role.datastreamlambda_role.name] + policy_arn = aws_iam_policy.datastreamlambda_policy.arn +} + + +resource "aws_lambda_function" "starter_lambda" { + function_name = var.starter_lambda_name + role = aws_iam_role.datastreamlambda_role.arn + handler = "lambda_function.lambda_handler" + runtime = var.runtime + filename = "${path.module}/lambda_functions/starter_lambda.zip" + timeout = 180 +} + +resource "aws_lambda_function" "commander_lambda" { + function_name = var.commander_lambda_name + role = aws_iam_role.datastreamlambda_role.arn + handler = "lambda_function.lambda_handler" + runtime = var.runtime + filename = "${path.module}/lambda_functions/commander_lambda.zip" + timeout = 60 +} + +resource "aws_lambda_function" "poller_lambda" { + function_name = var.poller_lambda_name + role = aws_iam_role.datastreamlambda_role.arn + handler = "lambda_function.lambda_handler" + runtime = var.runtime + filename = "${path.module}/lambda_functions/poller_lambda.zip" + timeout = 900 +} + +resource "aws_lambda_function" "checker_lambda" { + function_name = var.checker_lambda_name + role = aws_iam_role.datastreamlambda_role.arn + handler = "lambda_function.lambda_handler" + runtime = var.runtime + filename = "${path.module}/lambda_functions/checker_lambda.zip" + timeout = 60 +} + +resource "aws_lambda_function" "stopper_lambda" { + function_name = var.stopper_lambda_name + role = aws_iam_role.datastreamlambda_role.arn + handler = "lambda_function.lambda_handler" + runtime = var.runtime + filename = "${path.module}/lambda_functions/stopper_lambda.zip" + timeout = 180 +} + + +resource "aws_iam_role" "iam_for_sfn" { + name = "statemachine_role" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow", + Principal = { + Service = "states.amazonaws.com" + }, + Action = "sts:AssumeRole" + }] + }) +} + +resource "aws_iam_policy" "lambda_invoke_policy" { + name = "lambda_invoke_policy" + description = "Policy to allow invoking Lambda functions" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow", + Action = "lambda:InvokeFunction", + Resource = [ "*" + ] + }] + }) +} + +resource "aws_iam_role_policy_attachment" "lambda_invoke_attachment" { + role = aws_iam_role.iam_for_sfn.name + policy_arn = aws_iam_policy.lambda_invoke_policy.arn +} + + +resource "aws_sfn_state_machine" "my_state_machine" { + name = var.sm_name + role_arn = aws_iam_role.iam_for_sfn.arn + definition = < /home/ec2-user/ngen-datastream/log.txt"], + "commands" : [ + "mkdir -p /home/ec2-user/ngen-datastream/data/mount", + "mount-s3 ngen-datastream /home/ec2-user/ngen-datastream/data/mount", + "/home/ec2-user/ngen-datastream/scripts/stream.sh -c /home/ec2-user/ngen-datastream/configs/conf_datastream_daily.sh > /home/ec2-user/ngen-datastream/log.txt" + ], "bucket" : "ngen-datastream", "obj_key" : "daily/DATE/ngen-run/outputs/cat-1.csv" } diff --git a/cloud/AWS/start_ami/lambda_handler.py b/cloud/AWS/start_ami/lambda_function.py similarity index 100% rename from cloud/AWS/start_ami/lambda_handler.py rename to cloud/AWS/start_ami/lambda_function.py diff --git a/cloud/AWS/startup_ec2.sh b/cloud/AWS/startup_ec2.sh index dbee5e87..9863b715 100644 --- a/cloud/AWS/startup_ec2.sh +++ b/cloud/AWS/startup_ec2.sh @@ -25,8 +25,8 @@ sudo dnf install docker -y sudo systemctl start docker sudo usermod -aG docker ${USER} mkdir -p /home/ec2-user/ngen-datastream/data/mount - sudo yum install -y ./mount-s3.rpm + echo "cd docker && sudo docker build -t awiciroh/ngen-deps:latest -f Dockerfile.ngen-deps --no-cache . && docker build -t awiciroh/t-route:latest -f Dockerfile.t-route . --no-cache && docker build -t awiciroh/ngen -f Dockerfile.ngen . --no-cache && docker build -t awiciroh/ciroh-ngen-image:latest-local -f Dockerfile . --no-cache" echo "copy that ^^ and log out of session, log back in and run that command" diff --git a/cloud/AWS/state_machine/datastream.asl.json b/cloud/AWS/state_machine/datastream.asl.json deleted file mode 100644 index 161371d6..00000000 --- a/cloud/AWS/state_machine/datastream.asl.json +++ /dev/null @@ -1,166 +0,0 @@ -{ - "Comment": "A description of my state machine", - "StartAt": "EC2StarterFromAMI", - "States": { - "EC2StarterFromAMI": { - "Type": "Task", - "Resource": "arn:aws:states:::lambda:invoke", - "OutputPath": "$.Payload", - "Parameters": { - "Payload.$": "$", - "FunctionName": "arn:aws:lambda:us-east-2:857712214391:function:ec2start_ami:$LATEST" - }, - "Retry": [ - { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException", - "States.Timeout" - ], - "BackoffRate": 2, - "Comment": "Try restarting", - "IntervalSeconds": 2, - "MaxAttempts": 10 - } - ], - "Next": "Commander" - }, - "Commander": { - "Type": "Task", - "Resource": "arn:aws:states:::lambda:invoke", - "OutputPath": "$.Payload", - "Parameters": { - "Payload.$": "$", - "FunctionName": "arn:aws:lambda:us-east-2:857712214391:function:ec2commander:$LATEST" - }, - "Retry": [ - { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException" - ], - "IntervalSeconds": 1, - "MaxAttempts": 3, - "BackoffRate": 2 - } - ], - "Next": "EC2Poller", - "Catch": [ - { - "ErrorEquals": [ - "States.TaskFailed" - ], - "Comment": "In case of failure", - "Next": "EC2Stopper", - "ResultPath": "$.failedInput" - } - ] - }, - "EC2Poller": { - "Type": "Task", - "Resource": "arn:aws:states:::lambda:invoke", - "OutputPath": "$.Payload", - "Parameters": { - "Payload.$": "$", - "FunctionName": "arn:aws:lambda:us-east-2:857712214391:function:CommandPoller:$LATEST" - }, - "Retry": [ - { - "ErrorEquals": [ - "States.Timeout" - ], - "IntervalSeconds": 1, - "Comment": "Retry for a long time just incase forcingprocessor takes awhile", - "BackoffRate": 1, - "MaxAttempts": 100 - } - ], - "Next": "Choice", - "Catch": [ - { - "ErrorEquals": [ - "States.TaskFailed" - ], - "Next": "EC2Stopper", - "Comment": "Kill EC2 incase of failure", - "ResultPath": "$.failedInput" - } - ] - }, - "Choice": { - "Type": "Choice", - "Choices": [ - { - "Variable": "$.ii_pass", - "BooleanEquals": true, - "Next": "RunChecker" - }, - { - "Variable": "$.ii_pass", - "BooleanEquals": false, - "Next": "EC2Poller" - } - ] - }, - "RunChecker": { - "Type": "Task", - "Resource": "arn:aws:states:::lambda:invoke", - "OutputPath": "$.Payload", - "Parameters": { - "Payload.$": "$", - "FunctionName": "arn:aws:lambda:us-east-2:857712214391:function:S3ObjectChecker:$LATEST" - }, - "Retry": [ - { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException" - ], - "IntervalSeconds": 1, - "MaxAttempts": 3, - "BackoffRate": 2 - } - ], - "Next": "EC2Stopper", - "Catch": [ - { - "ErrorEquals": [ - "States.TaskFailed" - ], - "Comment": "Kill EC2 incase of failure", - "Next": "EC2Stopper", - "ResultPath": "$.failedInput" - } - ] - }, - "EC2Stopper": { - "Type": "Task", - "Resource": "arn:aws:states:::lambda:invoke", - "OutputPath": "$.Payload", - "Parameters": { - "Payload.$": "$", - "FunctionName": "arn:aws:lambda:us-east-2:857712214391:function:EC2Stopper:$LATEST" - }, - "Retry": [ - { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException" - ], - "IntervalSeconds": 1, - "MaxAttempts": 3, - "BackoffRate": 2 - } - ], - "End": true - } - } - } \ No newline at end of file diff --git a/cloud/AWS/stopper/lambda_function.py b/cloud/AWS/stopper/lambda_function.py new file mode 100644 index 00000000..2d76c84e --- /dev/null +++ b/cloud/AWS/stopper/lambda_function.py @@ -0,0 +1,19 @@ +import boto3 + +client_ec2 = boto3.client('ec2') + +def lambda_handler(event, context): + """ + Generic Poller funcion + """ + + instance_id = event['instance_parameters']['InstanceId'] + + print(f'Shutting down processor {instance_id}') + client_ec2.stop_instances(InstanceIds=[instance_id]) + + waiter = client_ec2.get_waiter('instance_stopped') + waiter.wait(InstanceIds=[instance_id]) + + print(f'Instance {instance_id} has been successfully stopped.') + diff --git a/cloud/AWS/streamcommander/lambda_handler.py b/cloud/AWS/streamcommander/lambda_function.py similarity index 94% rename from cloud/AWS/streamcommander/lambda_handler.py rename to cloud/AWS/streamcommander/lambda_function.py index 09ea3291..091de0b6 100644 --- a/cloud/AWS/streamcommander/lambda_handler.py +++ b/cloud/AWS/streamcommander/lambda_function.py @@ -37,7 +37,7 @@ def lambda_handler(event, context): InstanceIds=[instance_id], DocumentName='AWS-RunShellScript', Parameters={'commands': event['commands'], - "executionTimeout": [f"{3600*24}" for x in range(len(event['commands']))] + "executionTimeout": [f"{3600*24}"] } ) wait_for_command_response(response,instance_id) diff --git a/scripts/stream.sh b/scripts/stream.sh index 287a989c..21ca3569 100755 --- a/scripts/stream.sh +++ b/scripts/stream.sh @@ -53,6 +53,7 @@ while [ "$#" -gt 0 ]; do -i|--id-type) SUBSET_ID_TYPE="$2"; shift 2;; -I|--id) SUBSET_ID="$2"; shift 2;; -v|--version) HYDROFABRIC_VERSION="$2"; shift 2;; + -S|--s3-mount) S3_MOUNT="$2"; shift 2;; -c|--conf-file) CONF_FILE="$2"; shift 2;; *) usage;; esac