diff --git a/AWS/user_data.sh b/AWS/user_data.sh deleted file mode 100644 index 3e83c3b2..00000000 --- a/AWS/user_data.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# cli command to start instance -# aws ec2 run-instances --user-data file://user_data.txt --instance-type c5n.18xlarge --count 1 --image-id ami-08cba41c585e4a2e2 --region us-east-2 --key-name Processor --iam-instance-profile '{"Name":"Processor"}' --security-group-ids "sg-0fc864d44ef677a07" --profile jlaser_ciroh - -echo "EXECUTING USER DATA" - -cd /home/ec2-user - -sudo dnf install git -y -python3 -m venv ./venv-datastream -git clone https://github.com/CIROH-UA/ngen-datastream.git -source ./venv-datastream/bin/activate && pip3 install --upgrade pip -pip3 install -r ./ngen-datastream/requirements.txt -deactivate - -python3 -m venv ./venv-ngen-cal -git clone --branch realization_validation https://github.com/JordanLaserGit/ngen-cal.git -source ./venv-ngen-cal/bin/activate && pip3 install --upgrade pip -pip3 install -r ./ngen-cal/requirements.txt -pip3 install -e ./ngen-cal/python/ngen_conf -deactivate - -# sudo dnf install go -y -# go install github.com/aaraney/ht@latest - -touch /tmp/userdata_complete - -echo "USERDATA COMPLETE" diff --git a/AWS/user_data.txt b/AWS/user_data.txt deleted file mode 100755 index 3e83c3b2..00000000 --- a/AWS/user_data.txt +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# cli command to start instance -# aws ec2 run-instances --user-data file://user_data.txt --instance-type c5n.18xlarge --count 1 --image-id ami-08cba41c585e4a2e2 --region us-east-2 --key-name Processor --iam-instance-profile '{"Name":"Processor"}' --security-group-ids "sg-0fc864d44ef677a07" --profile jlaser_ciroh - -echo "EXECUTING USER DATA" - -cd /home/ec2-user - -sudo dnf install git -y -python3 -m venv ./venv-datastream -git clone https://github.com/CIROH-UA/ngen-datastream.git -source ./venv-datastream/bin/activate && pip3 install --upgrade pip -pip3 install -r ./ngen-datastream/requirements.txt -deactivate - -python3 -m venv ./venv-ngen-cal -git clone --branch realization_validation https://github.com/JordanLaserGit/ngen-cal.git -source ./venv-ngen-cal/bin/activate && pip3 install --upgrade pip -pip3 install -r ./ngen-cal/requirements.txt -pip3 install -e ./ngen-cal/python/ngen_conf -deactivate - -# sudo dnf install go -y -# go install github.com/aaraney/ht@latest - -touch /tmp/userdata_complete - -echo "USERDATA COMPLETE" diff --git a/AWS/Trust.json b/cloud/AWS/Trust.json similarity index 100% rename from AWS/Trust.json rename to cloud/AWS/Trust.json diff --git a/AWS/checker/lambda_function.py b/cloud/AWS/checker/lambda_function.py similarity index 100% rename from AWS/checker/lambda_function.py rename to cloud/AWS/checker/lambda_function.py diff --git a/AWS/execution_dailyrun.json b/cloud/AWS/execution_dailyrun.json similarity index 100% rename from AWS/execution_dailyrun.json rename to cloud/AWS/execution_dailyrun.json diff --git a/AWS/forcingcommander/lambda_function.py b/cloud/AWS/forcingcommander/lambda_function.py similarity index 100% rename from AWS/forcingcommander/lambda_function.py rename to cloud/AWS/forcingcommander/lambda_function.py diff --git a/AWS/index.html b/cloud/AWS/index.html similarity index 100% rename from AWS/index.html rename to cloud/AWS/index.html diff --git a/AWS/instance_profile.json b/cloud/AWS/instance_profile.json similarity index 100% rename from AWS/instance_profile.json rename to cloud/AWS/instance_profile.json diff --git a/AWS/poller/lambda_function.py b/cloud/AWS/poller/lambda_function.py similarity index 100% rename from AWS/poller/lambda_function.py rename to cloud/AWS/poller/lambda_function.py diff --git a/AWS/starter/lambda_function.py b/cloud/AWS/starter/lambda_function.py similarity index 100% rename from AWS/starter/lambda_function.py rename to cloud/AWS/starter/lambda_function.py diff --git a/cloud/AWS/startup_ec2.sh b/cloud/AWS/startup_ec2.sh new file mode 100644 index 00000000..ff1da992 --- /dev/null +++ b/cloud/AWS/startup_ec2.sh @@ -0,0 +1,21 @@ +#!/bin/bash +sudo dnf update -y +sudo dnf install git pip pigz awscli -y +curl -L -O https://github.com/LynkerIntel/hfsubset/releases/download/hfsubset-release-12/hfsubset-linux_amd64.tar.gz +tar -xzvf hfsubset-linux_amd64.tar.gz +sudo mv hfsubset /usr/bin/hfsubset +git clone https://github.com/CIROH-UA/ngen-datastream.git +python3 -m pip install --upgrade pip +pip3 install -r $PACAKGE_DIR/requirements.txt --no-cache +aws configure +aws configure set s3.max_concurrent_requests 256 +mkdir docker +aws s3 sync s3://ngen-datastream/docker/docker ~/docker +sudo dnf update +sudo dnf -y install dnf-plugins-core +sudo dnf install docker -y +sudo systemctl start docker +sudo usermod -aG docker ${USER} +echo "cd docker && sudo docker build -t awiciroh/ngen-deps:latest -f Dockerfile.ngen-deps --no-cache . && docker build -t awiciroh/t-route:latest -f Dockerfile.t-route . --no-cache && docker build -t awiciroh/ngen -f Dockerfile.ngen . --no-cache && docker build -t awiciroh/ciroh-ngen-image:latest-local -f Dockerfile . --no-cache" +echo "copy that ^^ and log out of session, log back in and run that command" + diff --git a/AWS/stopper/lambda_function.py b/cloud/AWS/stopper/lambda_function.py similarity index 100% rename from AWS/stopper/lambda_function.py rename to cloud/AWS/stopper/lambda_function.py diff --git a/AWS/tarballer/lambda_function.py b/cloud/AWS/tarballer/lambda_function.py similarity index 100% rename from AWS/tarballer/lambda_function.py rename to cloud/AWS/tarballer/lambda_function.py diff --git a/AWS/validator/lambda_function.py b/cloud/AWS/validator/lambda_function.py similarity index 100% rename from AWS/validator/lambda_function.py rename to cloud/AWS/validator/lambda_function.py diff --git a/cloud/start_up_debian.sh b/cloud/start_up_debian.sh new file mode 100644 index 00000000..3ce57f9a --- /dev/null +++ b/cloud/start_up_debian.sh @@ -0,0 +1,26 @@ +#!/bin/bash +sudo apt update -y +sudo apt-get update -y +sudo apt install git pigz awscli -y +git clone https://github.com/CIROH-UA/ngen-datastream.git +curl -L -O https://github.com/LynkerIntel/hfsubset/releases/download/hfsubset-release-12/hfsubset-linux_amd64.tar.gz +tar -xzvf hfsubset-linux_amd64.tar.gz +sudo mv hfsubset /usr/bin/hfsubset +aws configure +mkdir docker +aws s3 sync s3://ngen-datastream/docker/docker ~/docker +sudo apt-get update -y +sudo apt-get install ca-certificates curl gnupg -y +sudo install -m 0755 -d /etc/apt/keyrings -y +curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +sudo chmod a+r /etc/apt/keyrings/docker.gpg +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt-get update +sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +sudo usermod -aG docker ${USER} + +echo "log out and back in" + +#!/bin/bash +cd docker +sudo docker build -t awiciroh/ngen-deps:latest -f Dockerfile.ngen-deps --no-cache . && docker build -t awiciroh/t-route:latest -f Dockerfile.t-route . --no-cache && docker build -t awiciroh/ngen -f Dockerfile.ngen . --no-cache && docker build -t awiciroh/ciroh-ngen-image:latest-local -f Dockerfile . --no-cache \ No newline at end of file diff --git a/cloud/startup_fedora.sh b/cloud/startup_fedora.sh new file mode 100644 index 00000000..5b570d9e --- /dev/null +++ b/cloud/startup_fedora.sh @@ -0,0 +1,20 @@ +#!/bin/bash +sudo dnf update -y +sudo dnf install git pip pigz awscli -y +curl -L -O https://github.com/LynkerIntel/hfsubset/releases/download/hfsubset-release-12/hfsubset-linux_amd64.tar.gz +tar -xzvf hfsubset-linux_amd64.tar.gz +mv hfsubset /usr/bin/hfsubset +git clone https://github.com/CIROH-UA/ngen-datastream.git +aws configure +aws configure set s3.max_concurrent_requests 256 +mkdir docker +aws s3 sync s3://ngen-datastream/docker/docker ~/docker +sudo dnf update -y +sudo dnf -y install dnf-plugins-core +sudo dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo +sudo dnf install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y +sudo systemctl start docker +sudo usermod -aG docker ${USER} +echo "cd docker && sudo docker build -t awiciroh/ngen-deps:latest -f Dockerfile.ngen-deps --no-cache . && docker build -t awiciroh/t-route:latest -f Dockerfile.t-route . --no-cache && docker build -t awiciroh/ngen -f Dockerfile.ngen . --no-cache && docker build -t awiciroh/ciroh-ngen-image:latest-local -f Dockerfile . --no-cache" +echo "copy that ^^ and log out of session, log back in and run that command" + diff --git a/requirements.txt b/python/requirements.txt similarity index 100% rename from requirements.txt rename to python/requirements.txt diff --git a/scripts/stream.sh b/scripts/stream.sh index 760c5f1b..df4d1db7 100755 --- a/scripts/stream.sh +++ b/scripts/stream.sh @@ -93,13 +93,6 @@ fi mkdir -p $DATA_PATH NGEN_RUN_PATH="${DATA_PATH%/}/ngen-run" -if [ -e "$DATA_PATH" ]; then - : -else - echo "$DATA_PATH doesn't not exist!" - exit 1 -fi - DATASTREAM_CONF_PATH="${DATA_PATH%/}/datastream-configs" DATASTREAM_RESOURCES="${DATA_PATH%/}/datastream-resources" mkdir -p $DATASTREAM_CONF_PATH