add schedule

cloud/AWS/execution_dailyrun.json → cloud/AWS/datastream/execution_dailyrun.json

@@ -1,7 +1,7 @@
 {
   "instance_parameters" :
   {
-    "ImageId"            : "ami-0d93002aa2fb716c2",
+    "ImageId"            : "ami-0f543bde58455541d",
     "InstanceType"       : "c5n.18xlarge",
     "KeyName"            : "Processor",
     "MinCount"           : 1,
@@ -34,9 +34,9 @@
 
   "region"   : "us-east-2",
   "commands"  : [
-    "mkdir -p /home/ec2-user/ngen-datastream/data/mount",
-    "mount-s3 ngen-datastream /home/ec2-user/ngen-datastream/data/mount",
-    "/home/ec2-user/ngen-datastream/scripts/stream.sh -c /home/ec2-user/ngen-datastream/configs/conf_datastream_daily.sh > /home/ec2-user/ngen-datastream/log.txt"
+    "runuser -l ec2-user -c 'mkdir -p /home/ec2-user/ngen-datastream/data/mount'",
+    "runuser -l ec2-user -c 'mount-s3 ngen-datastream /home/ec2-user/ngen-datastream/data/mount'",
+    "runuser -l ec2-user -c '/home/ec2-user/ngen-datastream/scripts/stream.sh -c /home/ec2-user/ngen-datastream/configs/conf_datastream_daily.sh > /home/ec2-user/ngen-datastream/log.txt'"
   ],
   "bucket"   : "ngen-datastream",
   "obj_key"  : "daily/DATE/ngen-run/outputs/cat-1.csv"

cloud/AWS/datastream/main.tf

@@ -16,6 +16,10 @@ variable "lambda_policy_name" {}
 variable "lambda_role_name" {}
 variable "sm_name" {}
 variable "runtime" {}
+variable "schedule_name" {}
+variable "scheduler_policy_name" {}
+variable "scheduler_role_name" {}
+variable "execution_name" {}
 
 resource "aws_iam_policy" "datastreamlambda_policy" {
   name        = var.lambda_policy_name
@@ -103,7 +107,6 @@ resource "aws_iam_policy" "datastreamlambda_policy" {
   })
 }
 
-
 resource "aws_iam_role" "datastreamlambda_role" {
   name = var.lambda_role_name
   assume_role_policy = jsonencode({
@@ -206,7 +209,7 @@ resource "aws_iam_role_policy_attachment" "lambda_invoke_attachment" {
 }
 
 
-resource "aws_sfn_state_machine" "my_state_machine" {
+resource "aws_sfn_state_machine" "datastream_state_machine" {
   name       = var.sm_name
   role_arn   = aws_iam_role.iam_for_sfn.arn
   definition = <<EOF
@@ -367,3 +370,62 @@ resource "aws_sfn_state_machine" "my_state_machine" {
 }
 EOF
 }
+
+resource "aws_iam_policy" "scheduler_policy" {
+  name        = var.scheduler_policy_name
+  description = "Policy with permissions for statemachine execution"
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+            "Effect": "Allow",
+            Action = [
+                "states:StartExecution",
+                "events:PutTargets",
+                "events:PutRule",
+                "events:PutPermission"
+              ],
+            "Resource": ["*"]
+        }
+    ]
+  })
+}
+
+resource "aws_iam_role" "scheduler_role" {
+  name = var.scheduler_role_name
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect = "Allow",
+      Principal = {
+        Service = "scheduler.amazonaws.com"
+      },
+      Action = "sts:AssumeRole"
+    }]
+  })
+}
+
+resource "aws_iam_policy_attachment" "datastream_scheduler_attachment" {
+  name       = "datastream_scheduler_attachment"
+  roles      = [aws_iam_role.scheduler_role.name]
+  policy_arn = aws_iam_policy.scheduler_policy.arn
+}
+
+resource "aws_scheduler_schedule" "data_stream_schedule" {
+  name       = var.schedule_name
+  group_name = "default"
+
+  flexible_time_window {
+    mode = "OFF"
+  }
+
+  schedule_expression = "cron(0 1 * * ? *)"  
+  schedule_expression_timezone  = "America/New_York"
+
+  target {
+    arn      = aws_sfn_state_machine.datastream_state_machine.arn
+    role_arn = aws_iam_role.scheduler_role.arn
+    input = file("${path.module}/${var.execution_name}")
+  }
+
+}

cloud/AWS/datastream/variables.tfvars

@@ -8,3 +8,7 @@ stopper_lambda_name   = "EC2Stopper_tf"
 lambda_policy_name    = "datastream_lambda_policy_tf"
 lambda_role_name      = "datastream_lambda_role_tf"
 runtime               = "python3.12"
+schedule_name         = "ngen-datastream-schedule"
+scheduler_policy_name = "datastream-scheduler-policy"
+scheduler_role_name   = "datastream-scheduler-role"
+execution_name        = "execution_dailyrun.json"

cloud/AWS/startup_ec2.sh

@@ -1,32 +1,27 @@
 #!/bin/bash
 arch=$(dpkg --print-architecture)
-if [ "$arch" == "amd64" ]; then
-    curl -L -O https://github.com/LynkerIntel/hfsubset/releases/download/hfsubset-release-12/hfsubset-linux_amd64.tar.gz
-    curl -L -O https://s3.amazonaws.com/mountpoint-s3-release/latest/x86_64/mount-s3.rpm
-else
-    echo "no release for hfsubset on arm64"
-    curl -L -O https://s3.amazonaws.com/mountpoint-s3-release/latest/arm64/mount-s3.rpm
-fi
+echo "Arch: "$arch
+curl -L -O https://github.com/LynkerIntel/hfsubset/releases/download/hfsubset-release-12/hfsubset-linux_amd64.tar.gz
+curl -L -O https://s3.amazonaws.com/mountpoint-s3-release/latest/x86_64/mount-s3.rpm
 sudo dnf update -y
 sudo dnf install git pip pigz awscli -y
 tar -xzvf hfsubset-linux_amd64.tar.gz
 rm hfsubset-linux_amd64.tar.gz
 sudo mv hfsubset /usr/bin/hfsubset
 git clone https://github.com/CIROH-UA/ngen-datastream.git
-python3 -m pip install --upgrade pip
-pip3 install -r /home/ec2-user/ngen-datastream/python/requirements.txt --no-cache
+# python3 -m pip install --upgrade pip
+# pip3 install -r /home/ec2-user/ngen-datastream/python/requirements.txt --no-cache
 aws configure
 aws configure set s3.max_concurrent_requests 256
-mkdir docker
-aws s3 sync s3://ngen-datastream/docker/docker ~/docker
 sudo dnf update
 sudo dnf -y install dnf-plugins-core
 sudo dnf install docker -y
 sudo systemctl start docker
-sudo usermod -aG docker ${USER}
-mkdir -p /home/ec2-user/ngen-datastream/data/mount
-sudo yum install -y ./mount-s3.rpm
-
-echo "cd docker && sudo docker build -t awiciroh/ngen-deps:latest -f Dockerfile.ngen-deps --no-cache . && docker build -t awiciroh/t-route:latest -f Dockerfile.t-route . --no-cache && docker build -t awiciroh/ngen -f Dockerfile.ngen . --no-cache && docker build -t awiciroh/ciroh-ngen-image:latest-local -f Dockerfile . --no-cache"
-echo "copy that ^^ and log out of session, log back in and run that command"
+sudo usermod -aG docker ec2-user
+sudo newgrp docker
+mkdir docker
+aws s3 sync s3://ngen-datastream/docker/docker ~/docker
+cd docker 
+sudo docker build -t awiciroh/ngen-deps:latest -f Dockerfile.ngen-deps --no-cache . && docker build -t awiciroh/t-route:latest -f Dockerfile.t-route . --no-cache && docker build -t awiciroh/ngen -f Dockerfile.ngen . --no-cache && docker build -t awiciroh/ciroh-ngen-image:latest-local -f Dockerfile . --no-cache
+echo "done!"