diff --git a/cloud/AWS/execution_dailyrun.json b/cloud/AWS/datastream/execution_dailyrun.json similarity index 67% rename from cloud/AWS/execution_dailyrun.json rename to cloud/AWS/datastream/execution_dailyrun.json index 5d9561f2..092c665f 100644 --- a/cloud/AWS/execution_dailyrun.json +++ b/cloud/AWS/datastream/execution_dailyrun.json @@ -1,7 +1,7 @@ { "instance_parameters" : { - "ImageId" : "ami-0d93002aa2fb716c2", + "ImageId" : "ami-0f543bde58455541d", "InstanceType" : "c5n.18xlarge", "KeyName" : "Processor", "MinCount" : 1, @@ -34,9 +34,9 @@ "region" : "us-east-2", "commands" : [ - "mkdir -p /home/ec2-user/ngen-datastream/data/mount", - "mount-s3 ngen-datastream /home/ec2-user/ngen-datastream/data/mount", - "/home/ec2-user/ngen-datastream/scripts/stream.sh -c /home/ec2-user/ngen-datastream/configs/conf_datastream_daily.sh > /home/ec2-user/ngen-datastream/log.txt" + "runuser -l ec2-user -c 'mkdir -p /home/ec2-user/ngen-datastream/data/mount'", + "runuser -l ec2-user -c 'mount-s3 ngen-datastream /home/ec2-user/ngen-datastream/data/mount'", + "runuser -l ec2-user -c '/home/ec2-user/ngen-datastream/scripts/stream.sh -c /home/ec2-user/ngen-datastream/configs/conf_datastream_daily.sh > /home/ec2-user/ngen-datastream/log.txt'" ], "bucket" : "ngen-datastream", "obj_key" : "daily/DATE/ngen-run/outputs/cat-1.csv" diff --git a/cloud/AWS/datastream/main.tf b/cloud/AWS/datastream/main.tf index f140d6a1..5683fa4a 100644 --- a/cloud/AWS/datastream/main.tf +++ b/cloud/AWS/datastream/main.tf @@ -16,6 +16,10 @@ variable "lambda_policy_name" {} variable "lambda_role_name" {} variable "sm_name" {} variable "runtime" {} +variable "schedule_name" {} +variable "scheduler_policy_name" {} +variable "scheduler_role_name" {} +variable "execution_name" {} resource "aws_iam_policy" "datastreamlambda_policy" { name = var.lambda_policy_name @@ -103,7 +107,6 @@ resource "aws_iam_policy" "datastreamlambda_policy" { }) } - resource "aws_iam_role" "datastreamlambda_role" { name = var.lambda_role_name assume_role_policy = jsonencode({ @@ -206,7 +209,7 @@ resource "aws_iam_role_policy_attachment" "lambda_invoke_attachment" { } -resource "aws_sfn_state_machine" "my_state_machine" { +resource "aws_sfn_state_machine" "datastream_state_machine" { name = var.sm_name role_arn = aws_iam_role.iam_for_sfn.arn definition = <