move AWS into cloud and pip install to startup

CIROH-UA · Feb 2, 2024 · 9628689 · 9628689
1 parent 77a6d7a
commit 9628689
Show file tree

Hide file tree

Showing 18 changed files with 67 additions and 65 deletions.
diff --git a/AWS/user_data.sh b/AWS/user_data.sh
diff --git a/AWS/user_data.txt b/AWS/user_data.txt
diff --git a/AWS/Trust.json → cloud/AWS/Trust.json b/AWS/Trust.json → cloud/AWS/Trust.json
diff --git a/AWS/checker/lambda_function.py → cloud/AWS/checker/lambda_function.py b/AWS/checker/lambda_function.py → cloud/AWS/checker/lambda_function.py
diff --git a/AWS/execution_dailyrun.json → cloud/AWS/execution_dailyrun.json b/AWS/execution_dailyrun.json → cloud/AWS/execution_dailyrun.json
diff --git a/AWS/forcingcommander/lambda_function.py → ...d/AWS/forcingcommander/lambda_function.py b/AWS/forcingcommander/lambda_function.py → ...d/AWS/forcingcommander/lambda_function.py
diff --git a/AWS/index.html → cloud/AWS/index.html b/AWS/index.html → cloud/AWS/index.html
diff --git a/AWS/instance_profile.json → cloud/AWS/instance_profile.json b/AWS/instance_profile.json → cloud/AWS/instance_profile.json
diff --git a/AWS/poller/lambda_function.py → cloud/AWS/poller/lambda_function.py b/AWS/poller/lambda_function.py → cloud/AWS/poller/lambda_function.py
diff --git a/AWS/starter/lambda_function.py → cloud/AWS/starter/lambda_function.py b/AWS/starter/lambda_function.py → cloud/AWS/starter/lambda_function.py
diff --git a/cloud/AWS/startup_ec2.sh b/cloud/AWS/startup_ec2.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+sudo dnf update -y
+sudo dnf install git pip pigz awscli -y
+curl -L -O https://github.com/LynkerIntel/hfsubset/releases/download/hfsubset-release-12/hfsubset-linux_amd64.tar.gz
+tar -xzvf hfsubset-linux_amd64.tar.gz
+sudo mv hfsubset /usr/bin/hfsubset
+git clone https://github.com/CIROH-UA/ngen-datastream.git
+python3 -m pip install --upgrade pip
+pip3 install -r $PACAKGE_DIR/requirements.txt --no-cache
+aws configure
+aws configure set s3.max_concurrent_requests 256
+mkdir docker
+aws s3 sync s3://ngen-datastream/docker/docker ~/docker
+sudo dnf update
+sudo dnf -y install dnf-plugins-core
+sudo dnf install docker -y
+sudo systemctl start docker
+sudo usermod -aG docker ${USER}
+echo "cd docker && sudo docker build -t awiciroh/ngen-deps:latest -f Dockerfile.ngen-deps --no-cache . && docker build -t awiciroh/t-route:latest -f Dockerfile.t-route . --no-cache && docker build -t awiciroh/ngen -f Dockerfile.ngen . --no-cache && docker build -t awiciroh/ciroh-ngen-image:latest-local -f Dockerfile . --no-cache"
+echo "copy that ^^ and log out of session, log back in and run that command"
+
diff --git a/AWS/stopper/lambda_function.py → cloud/AWS/stopper/lambda_function.py b/AWS/stopper/lambda_function.py → cloud/AWS/stopper/lambda_function.py
diff --git a/AWS/tarballer/lambda_function.py → cloud/AWS/tarballer/lambda_function.py b/AWS/tarballer/lambda_function.py → cloud/AWS/tarballer/lambda_function.py
diff --git a/AWS/validator/lambda_function.py → cloud/AWS/validator/lambda_function.py b/AWS/validator/lambda_function.py → cloud/AWS/validator/lambda_function.py
diff --git a/cloud/start_up_debian.sh b/cloud/start_up_debian.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+sudo apt update -y
+sudo apt-get update -y
+sudo apt install git pigz awscli -y
+git clone https://github.com/CIROH-UA/ngen-datastream.git
+curl -L -O https://github.com/LynkerIntel/hfsubset/releases/download/hfsubset-release-12/hfsubset-linux_amd64.tar.gz
+tar -xzvf hfsubset-linux_amd64.tar.gz
+sudo mv hfsubset /usr/bin/hfsubset
+aws configure
+mkdir docker
+aws s3 sync s3://ngen-datastream/docker/docker ~/docker
+sudo apt-get update -y
+sudo apt-get install ca-certificates curl gnupg -y
+sudo install -m 0755 -d /etc/apt/keyrings -y
+curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg 
+sudo chmod a+r /etc/apt/keyrings/docker.gpg
+echo   "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" |   sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+sudo apt-get update
+sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+sudo usermod -aG docker ${USER}
+
+echo "log out and back in"
+
+#!/bin/bash
+cd docker
+sudo docker build -t awiciroh/ngen-deps:latest -f Dockerfile.ngen-deps --no-cache . && docker build -t awiciroh/t-route:latest -f Dockerfile.t-route . --no-cache && docker build -t awiciroh/ngen -f Dockerfile.ngen . --no-cache && docker build -t awiciroh/ciroh-ngen-image:latest-local -f Dockerfile . --no-cache
diff --git a/cloud/startup_fedora.sh b/cloud/startup_fedora.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+sudo dnf update -y
+sudo dnf install git pip pigz awscli -y
+curl -L -O https://github.com/LynkerIntel/hfsubset/releases/download/hfsubset-release-12/hfsubset-linux_amd64.tar.gz
+tar -xzvf hfsubset-linux_amd64.tar.gz
+mv hfsubset /usr/bin/hfsubset
+git clone https://github.com/CIROH-UA/ngen-datastream.git
+aws configure
+aws configure set s3.max_concurrent_requests 256
+mkdir docker
+aws s3 sync s3://ngen-datastream/docker/docker ~/docker
+sudo dnf update -y
+sudo dnf -y install dnf-plugins-core
+sudo dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo
+sudo dnf install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y
+sudo systemctl start docker
+sudo usermod -aG docker ${USER}
+echo "cd docker && sudo docker build -t awiciroh/ngen-deps:latest -f Dockerfile.ngen-deps --no-cache . && docker build -t awiciroh/t-route:latest -f Dockerfile.t-route . --no-cache && docker build -t awiciroh/ngen -f Dockerfile.ngen . --no-cache && docker build -t awiciroh/ciroh-ngen-image:latest-local -f Dockerfile . --no-cache"
+echo "copy that ^^ and log out of session, log back in and run that command"
+
diff --git a/requirements.txt → python/requirements.txt b/requirements.txt → python/requirements.txt
diff --git a/scripts/stream.sh b/scripts/stream.sh
@@ -93,13 +93,6 @@ fi
 mkdir -p $DATA_PATH
 NGEN_RUN_PATH="${DATA_PATH%/}/ngen-run"
 
-if [ -e "$DATA_PATH" ]; then
-    :
-else
-    echo "$DATA_PATH doesn't not exist!"
-    exit 1
-fi
-
 DATASTREAM_CONF_PATH="${DATA_PATH%/}/datastream-configs"
 DATASTREAM_RESOURCES="${DATA_PATH%/}/datastream-resources"
 mkdir -p $DATASTREAM_CONF_PATH