Skip to content

Commit

Permalink
Merge pull request #154 from mit-submit/main
Browse files Browse the repository at this point in the history
SQL/grafana fixes
  • Loading branch information
ludomori99 authored Nov 25, 2023
2 parents c14ab32 + a51d76d commit 4535ba5
Show file tree
Hide file tree
Showing 102 changed files with 1,952 additions and 463 deletions.
74 changes: 26 additions & 48 deletions .github/workflows/dev-ci-cd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,54 +30,27 @@ jobs:
- name: Create Secrets Files
run: |
mkdir -p ${{ github.workspace }}/deploy/dev/secrets/
touch ${{ github.workspace }}/deploy/dev/secrets/imap_user.txt
echo "${{ secrets.DEV_IMAP_USER }}" >> ${{ github.workspace }}/deploy/dev/secrets/imap_user.txt
chmod 400 ${{ github.workspace }}/deploy/dev/secrets/imap_user.txt
touch ${{ github.workspace }}/deploy/dev/secrets/imap_pw.txt
echo "${{ secrets.DEV_IMAP_PW }}" >> ${{ github.workspace }}/deploy/dev/secrets/imap_pw.txt
chmod 400 ${{ github.workspace }}/deploy/dev/secrets/imap_pw.txt
touch ${{ github.workspace }}/deploy/dev/secrets/cleo_url.txt
echo "${{ secrets.DEV_CLEO_URL }}" >> ${{ github.workspace }}/deploy/dev/secrets/cleo_url.txt
chmod 400 ${{ github.workspace }}/deploy/dev/secrets/cleo_url.txt
touch ${{ github.workspace }}/deploy/dev/secrets/cleo_user.txt
echo "${{ secrets.DEV_CLEO_USER }}" >> ${{ github.workspace }}/deploy/dev/secrets/cleo_user.txt
chmod 400 ${{ github.workspace }}/deploy/dev/secrets/cleo_user.txt
touch ${{ github.workspace }}/deploy/dev/secrets/cleo_pw.txt
echo "${{ secrets.DEV_CLEO_PW }}" >> ${{ github.workspace }}/deploy/dev/secrets/cleo_pw.txt
chmod 400 ${{ github.workspace }}/deploy/dev/secrets/cleo_pw.txt
touch ${{ github.workspace }}/deploy/dev/secrets/cleo_project.txt
echo "${{ secrets.DEV_CLEO_PROJECT }}" >> ${{ github.workspace }}/deploy/dev/secrets/cleo_project.txt
chmod 400 ${{ github.workspace }}/deploy/dev/secrets/cleo_project.txt
touch ${{ github.workspace }}/deploy/dev/secrets/sender_server.txt
echo "${{ secrets.DEV_SENDER_SERVER }}" >> ${{ github.workspace }}/deploy/dev/secrets/sender_server.txt
chmod 400 ${{ github.workspace }}/deploy/dev/secrets/sender_server.txt
touch ${{ github.workspace }}/deploy/dev/secrets/sender_port.txt
echo "${{ secrets.DEV_SENDER_PORT }}" >> ${{ github.workspace }}/deploy/dev/secrets/sender_port.txt
chmod 400 ${{ github.workspace }}/deploy/dev/secrets/sender_port.txt
touch ${{ github.workspace }}/deploy/dev/secrets/sender_replyto.txt
echo "${{ secrets.DEV_SENDER_REPLYTO }}" >> ${{ github.workspace }}/deploy/dev/secrets/sender_replyto.txt
chmod 400 ${{ github.workspace }}/deploy/dev/secrets/sender_replyto.txt
touch ${{ github.workspace }}/deploy/dev/secrets/sender_user.txt
echo "${{ secrets.DEV_SENDER_USER }}" >> ${{ github.workspace }}/deploy/dev/secrets/sender_user.txt
chmod 400 ${{ github.workspace }}/deploy/dev/secrets/sender_user.txt
touch ${{ github.workspace }}/deploy/dev/secrets/sender_pw.txt
echo "${{ secrets.DEV_SENDER_PW }}" >> ${{ github.workspace }}/deploy/dev/secrets/sender_pw.txt
chmod 400 ${{ github.workspace }}/deploy/dev/secrets/sender_pw.txt
touch ${{ github.workspace }}/deploy/dev/secrets/flask_uploader_app_secret_key.txt
echo "${{ secrets.DEV_FLASK_UPLOADER_APP_SECRET_KEY }}" >> ${{ github.workspace }}/deploy/dev/secrets/flask_uploader_app_secret_key.txt
chmod 400 ${{ github.workspace }}/deploy/dev/secrets/flask_uploader_app_secret_key.txt
touch ${{ github.workspace }}/deploy/dev/secrets/uploader_salt.txt
echo "${{ secrets.DEV_UPLOADER_SALT }}" >> ${{ github.workspace }}/deploy/dev/secrets/uploader_salt.txt
chmod 400 ${{ github.workspace }}/deploy/dev/secrets/uploader_salt.txt
touch ${{ github.workspace }}/deploy/dev/secrets/openai_api_key.txt
echo "${{ secrets.OPENAI_API_KEY }}" >> ${{ github.workspace }}/deploy/dev/secrets/openai_api_key.txt
chmod 400 ${{ github.workspace }}/deploy/dev/secrets/openai_api_key.txt
touch ${{ github.workspace }}/deploy/dev/secrets/hf_token.txt
echo "${{ secrets.HF_TOKEN }}" >> ${{ github.workspace }}/deploy/dev/secrets/hf_token.txt
chmod 400 ${{ github.workspace }}/deploy/dev/secrets/hf_token.txt
touch ${{ github.workspace }}/deploy/dev/secrets/pg_password.txt
echo "${{ secrets.DEV_PG_PASSWORD }}" >> ${{ github.workspace }}/deploy/dev/secrets/pg_password.txt
chmod 400 ${{ github.workspace }}/deploy/dev/secrets/pg_password.txt
workspace=${{ github.workspace }}
env="dev"
sed -i "s/WORKSPACE/${workspace//\//\\/}/" ${workspace}/deploy/create_secret.sh
sed -i "s/ENV/${env}/" ${workspace}/deploy/create_secret.sh
/bin/bash ${workspace}/deploy/create_secret.sh imap_user.txt ${{ secrets.DEV_IMAP_USER }}
/bin/bash ${workspace}/deploy/create_secret.sh imap_pw.txt ${{ secrets.DEV_IMAP_PW }}
/bin/bash ${workspace}/deploy/create_secret.sh cleo_url.txt ${{ secrets.DEV_CLEO_URL }}
/bin/bash ${workspace}/deploy/create_secret.sh cleo_user.txt ${{ secrets.DEV_CLEO_USER }}
/bin/bash ${workspace}/deploy/create_secret.sh cleo_pw.txt ${{ secrets.DEV_CLEO_PW }}
/bin/bash ${workspace}/deploy/create_secret.sh cleo_project.txt ${{ secrets.DEV_CLEO_PROJECT }}
/bin/bash ${workspace}/deploy/create_secret.sh sender_server.txt ${{ secrets.DEV_SENDER_SERVER }}
/bin/bash ${workspace}/deploy/create_secret.sh sender_port.txt ${{ secrets.DEV_SENDER_PORT }}
/bin/bash ${workspace}/deploy/create_secret.sh sender_replyto.txt ${{ secrets.DEV_SENDER_REPLYTO }}
/bin/bash ${workspace}/deploy/create_secret.sh sender_user.txt ${{ secrets.DEV_SENDER_USER }}
/bin/bash ${workspace}/deploy/create_secret.sh sender_pw.txt ${{ secrets.DEV_SENDER_PW }}
/bin/bash ${workspace}/deploy/create_secret.sh flask_uploader_app_secret_key.txt ${{ secrets.DEV_FLASK_UPLOADER_APP_SECRET_KEY }}
/bin/bash ${workspace}/deploy/create_secret.sh uploader_salt.txt ${{ secrets.DEV_UPLOADER_SALT }}
/bin/bash ${workspace}/deploy/create_secret.sh openai_api_key.txt ${{ secrets.OPENAI_API_KEY }}
/bin/bash ${workspace}/deploy/create_secret.sh hf_token.txt ${{ secrets.HF_TOKEN }}
/bin/bash ${workspace}/deploy/create_secret.sh pg_password.txt ${{ secrets.DEV_PG_PASSWORD }}
/bin/bash ${workspace}/deploy/create_secret.sh grafana_password.txt ${{ secrets.DEV_GRAFANA_PG_PASSWORD }}
# create env file to set tag(s) for docker-compose
- name: Create Env File
Expand All @@ -87,6 +60,11 @@ jobs:
export tag="${tag//\//-}.${GITHUB_SHA}"
echo "TAG=${tag}" >> ${{ github.workspace }}/deploy/dev/.env
# create deployment directory if it doesn't already exist
- name: Create Directory
run: |
ssh submit06 "mkdir -p ~/A2rchi-dev/"
# stop any existing docker compose that's running
- name: Stop Docker Compose
run: |
Expand Down
82 changes: 82 additions & 0 deletions .github/workflows/prod-65830-ci-cd.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
name: Deploy A2rchi Prod for 6.5830
run-name: ${{ github.actor }} deploys A2rchi for 6.5830 to prod
on:
push:
branches:
- release-65830
jobs:
deploy-prod-system:
runs-on: ubuntu-latest
env:
SSH_AUTH_SOCK: /tmp/ssh_agent.sock
steps:
# boilerplate message and pull repository to CI runner
- run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event."
- uses: actions/checkout@v3
- run: echo "The ${{ github.repository }} repository has been cloned to the runner."

# setup SSH
- name: Setup SSH
run: |
mkdir -p /home/runner/.ssh/
echo "${{ secrets.SSH_PRIVATE_KEY_MDRUSSO }}" > /home/runner/.ssh/id_rsa_submit
chmod 600 /home/runner/.ssh/id_rsa_submit
echo "${{ secrets.SSH_SUBMIT_KNOWN_HOSTS }}" > ~/.ssh/known_hosts
cp ${{ github.workspace }}/deploy/ssh_config /home/runner/.ssh/config
ssh-agent -a $SSH_AUTH_SOCK > /dev/null
ssh-add /home/runner/.ssh/id_rsa_submit
# create secrets files for docker-compose
- name: Create Secrets Files
run: |
mkdir -p ${{ github.workspace }}/deploy/prod-65830/secrets/
workspace=${{ github.workspace }}
env="prod-65830"
sed -i "s/WORKSPACE/${workspace//\//\\/}/" ${workspace}/deploy/create_secret.sh
sed -i "s/ENV/${env}/" ${workspace}/deploy/create_secret.sh
/bin/bash ${workspace}/deploy/create_secret.sh flask_uploader_app_secret_key.txt ${{ secrets.PROD_FLASK_UPLOADER_APP_SECRET_KEY }}
/bin/bash ${workspace}/deploy/create_secret.sh uploader_salt.txt ${{ secrets.PROD_UPLOADER_SALT }}
/bin/bash ${workspace}/deploy/create_secret.sh openai_api_key.txt ${{ secrets.OPENAI_API_KEY }}
/bin/bash ${workspace}/deploy/create_secret.sh hf_token.txt ${{ secrets.HF_TOKEN }}
/bin/bash ${workspace}/deploy/create_secret.sh piazza_email.txt ${{ secrets.PROD_65830_PIAZZA_EMAIL }}
/bin/bash ${workspace}/deploy/create_secret.sh piazza_password.txt ${{ secrets.PROD_65830_PIAZZA_PASSWORD }}
/bin/bash ${workspace}/deploy/create_secret.sh slack_webhook.txt ${{ secrets.PROD_65830_SLACK_WEBHOOK }}
# create env file to set tag(s) for docker-compose
- name: Create Env File
run: |
touch ${{ github.workspace }}/deploy/prod-65830/.env
export tag="${GITHUB_REF#refs/heads/}"
export tag="${tag//\//-}.${GITHUB_SHA}"
echo "TAG=${tag}" >> ${{ github.workspace }}/deploy/prod-65830/.env
# create deployment directory if it doesn't already exist
- name: Create Directory
run: |
ssh submit06 "mkdir -p ~/A2rchi-prod-65830/"
# stop any existing docker compose that's running
- name: Stop Docker Compose
run: |
ssh submit06 'bash -s' < ${{ github.workspace }}/deploy/prod-65830/prod-65830-stop.sh
# copy repository to machine
- name: Copy Repository
run: |
rsync -e ssh -r ${{ github.workspace}}/* --exclude .git/ --delete submit06:~/A2rchi-prod-65830/
# run deploy script
- name: Run Deploy Script
run: |
export tag="${GITHUB_REF#refs/heads/}"
export tag="${tag//\//-}.${GITHUB_SHA}"
sed -i "s/BASE_TAG/${tag}/" ${{ github.workspace }}/deploy/prod-65830/prod-65830-install.sh
ssh submit06 'bash -s' < ${{ github.workspace }}/deploy/prod-65830/prod-65830-install.sh
# clean up secret files
- name: Remove Secrets from Runner
run: |
rm ${{ github.workspace }}/deploy/prod-65830/secrets/*.txt
# print job status
- run: echo "🍏 This job's status is ${{ job.status }}."
30 changes: 15 additions & 15 deletions .github/workflows/prod-801-ci-cd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,21 +30,16 @@ jobs:
- name: Create Secrets Files
run: |
mkdir -p ${{ github.workspace }}/deploy/prod-801/secrets/
touch ${{ github.workspace }}/deploy/prod-801/secrets/flask_uploader_app_secret_key.txt
echo "${{ secrets.PROD_FLASK_UPLOADER_APP_SECRET_KEY }}" >> ${{ github.workspace }}/deploy/prod-801/secrets/flask_uploader_app_secret_key.txt
chmod 400 ${{ github.workspace }}/deploy/prod-801/secrets/flask_uploader_app_secret_key.txt
touch ${{ github.workspace }}/deploy/prod-801/secrets/uploader_salt.txt
echo "${{ secrets.PROD_UPLOADER_SALT }}" >> ${{ github.workspace }}/deploy/prod-801/secrets/uploader_salt.txt
chmod 400 ${{ github.workspace }}/deploy/prod-801/secrets/uploader_salt.txt
touch ${{ github.workspace }}/deploy/prod-801/secrets/openai_api_key.txt
echo "${{ secrets.OPENAI_API_KEY }}" >> ${{ github.workspace }}/deploy/prod-801/secrets/openai_api_key.txt
chmod 400 ${{ github.workspace }}/deploy/prod-801/secrets/openai_api_key.txt
touch ${{ github.workspace }}/deploy/prod-801/secrets/hf_token.txt
echo "${{ secrets.HF_TOKEN }}" >> ${{ github.workspace }}/deploy/prod-801/secrets/hf_token.txt
chmod 400 ${{ github.workspace }}/deploy/prod-801/secrets/hf_token.txt
touch ${{ github.workspace }}/deploy/prod-801/secrets/pg_password.txt
echo "${{ secrets.PROD_801_PG_PASSWORD }}" >> ${{ github.workspace }}/deploy/prod-801/secrets/pg_password.txt
chmod 400 ${{ github.workspace }}/deploy/prod-801/secrets/pg_password.txt
workspace=${{ github.workspace }}
env="prod-801"
sed -i "s/WORKSPACE/${workspace//\//\\/}/" ${workspace}/deploy/create_secret.sh
sed -i "s/ENV/${env}/" ${workspace}/deploy/create_secret.sh
/bin/bash ${workspace}/deploy/create_secret.sh flask_uploader_app_secret_key.txt ${{ secrets.PROD_FLASK_UPLOADER_APP_SECRET_KEY }}
/bin/bash ${workspace}/deploy/create_secret.sh uploader_salt.txt ${{ secrets.PROD_UPLOADER_SALT }}
/bin/bash ${workspace}/deploy/create_secret.sh openai_api_key.txt ${{ secrets.OPENAI_API_KEY }}
/bin/bash ${workspace}/deploy/create_secret.sh hf_token.txt ${{ secrets.HF_TOKEN }}
/bin/bash ${workspace}/deploy/create_secret.sh pg_password.txt ${{ secrets.PROD_801_PG_PASSWORD }}
/bin/bash ${workspace}/deploy/create_secret.sh grafana_password.txt ${{ secrets.PROD_801_GRAFANA_PG_PASSWORD }}
# create env file to set tag(s) for docker-compose
- name: Create Env File
Expand All @@ -54,6 +49,11 @@ jobs:
export tag="${tag//\//-}.${GITHUB_SHA}"
echo "TAG=${tag}" >> ${{ github.workspace }}/deploy/prod-801/.env
# create deployment directory if it doesn't already exist
- name: Create Directory
run: |
ssh submit-t3desk "mkdir -p ~/A2rchi-prod-801/"
# stop any existing docker compose that's running
- name: Stop Docker Compose
run: |
Expand Down
Loading

0 comments on commit 4535ba5

Please sign in to comment.