Skip to content

(feat): Implement Email #276

(feat): Implement Email

(feat): Implement Email #276

name: Create/Destroy Test VM
on:
pull_request_target:
types: [opened, reopened, synchronize, labeled, unlabeled, closed]
jobs:
##################################################################################
# The following job is for creating a new DO droplet, creating DNS records for it
# and copying the project files to it.
##################################################################################
create_test_vm:
if: |
(
(
github.event.action == 'opened' ||
github.event.action == 'reopened' ||
github.event.action == 'synchronize'
) && contains(github.event.pull_request.labels.*.name, 'create-test-vm')
) ||
(
github.event.action == 'labeled' &&
github.event.label.name == 'create-test-vm'
)
runs-on: ubuntu-latest
steps:
- uses: unstructuredstudio/zubhub/.github/actions/comment_action@master
with:
token: ${{ secrets.GITHUB_TOKEN }}
issue_number: ${{ github.event.pull_request.number }}
owner: ${{ github.repository_owner }}
repo: ${{ github.event.repository.name }}
message: |
@${{ github.actor }} is creating a test VM for this PR 🚀🚀🚀
This may take a few minutes so relax and grab a cup of coffee ☕
We will notify you when the VM is ready.
- name: Get checkout commit ref and repo
run: |
# for context read this https://github.com/Arelle/Arelle/pull/938/files#r1377926250
merge_commit_sha=${{ github.event.pull_request.merge_commit_sha }}
head_ref=${{ github.event.pull_request.head.ref }}
target_repo=${{ github.repository }}
pr_repo=${{ github.event.pull_request.head.repo.full_name }}
CHECKOUT_COMMIT_REF=$([ "$merge_commit_sha" != "" ] && \
echo "$merge_commit_sha" || echo "$head_ref")
CHECKOUT_REPO=$([ "$merge_commit_sha" != "" ] && \
echo "$target_repo" || echo "$pr_repo")
echo "CHECKOUT_COMMIT_REF=$CHECKOUT_COMMIT_REF" >> $GITHUB_ENV
echo "CHECKOUT_REPO=$CHECKOUT_REPO" >> $GITHUB_ENV
# careful with this as you are checking out the source branch of the PR
# which is a potential security risk. Only do this when you know what you are doing
# See https://stackoverflow.com/questions/75873833/how-to-protect-github-secrets-in-pull-request-actions-from-malicious-pull-reques
# See https://stackoverflow.com/questions/74957218/what-is-the-difference-between-pull-request-and-pull-request-target-event-in-git
# See https://dev.to/suzukishunsuke/secure-github-actions-by-pullrequesttarget-641#:~:text=pull_request_target%20is%20one%20of%20the,the%20pull%20request's%20base%20branch.
- uses: unstructuredstudio/zubhub/.github/actions/checkout@master
with:
ref: ${{ env.CHECKOUT_COMMIT_REF }}
repository: ${{ env.CHECKOUT_REPO }}
- name: Create new droplet and DNS records
id: create_droplet
uses: unstructuredstudio/zubhub/.github/actions/doctl_action@master
with:
token: ${{ secrets.DO_ACCESS_TOKEN }}
script: |
# This script will be executed inside a composite action.
# rename GITHUB_OUTPUT so that it's clear that it's a
# composite output and should be handled differently from $GITHUB_OUTPUT
COMPOSITE_OUTPUT=$GITHUB_OUTPUT
NEW_DROPLET_NAME=zubhub-test-${{ github.event.pull_request.number }}
FRONTEND_DOMAIN=${NEW_DROPLET_NAME}
API_DOMAIN=api.${FRONTEND_DOMAIN}
MEDIA_DOMAIN=media.${FRONTEND_DOMAIN}
echo "NEW_DROPLET_NAME=$NEW_DROPLET_NAME" >> $COMPOSITE_OUTPUT
echo "FRONTEND_DOMAIN=$FRONTEND_DOMAIN" >> $COMPOSITE_OUTPUT
echo "API_DOMAIN=$API_DOMAIN" >> $COMPOSITE_OUTPUT
echo "MEDIA_DOMAIN=$MEDIA_DOMAIN" >> $COMPOSITE_OUTPUT
# check if droplet already exists and exit script if it does
NEW_DROPLET_IP=$(doctl compute droplet get $NEW_DROPLET_NAME \
--format PublicIPv4 --no-header 2>/dev/null || true)
if [[ -n "$NEW_DROPLET_IP" ]] ; then
echo "Droplet already exists. Save droplet IP to env variable and exit..."
echo "NEW_DROPLET_IP=$NEW_DROPLET_IP" >> $COMPOSITE_OUTPUT
exit 0
fi
# create new droplet
doctl compute droplet create $NEW_DROPLET_NAME --image \
${{ secrets.ZUBHUB_TEST_SNAPSHOT_ID }} --tag-name zubhub-test --size s-1vcpu-1gb \
--region nyc1 --enable-monitoring --ssh-keys ${{ secrets.DO_PUBLIC_SSHKEY_FP }} --wait
sleep 30s
NEW_DROPLET_IP=$(doctl compute droplet get $NEW_DROPLET_NAME \
--format PublicIPv4 --no-header)
echo "NEW_DROPLET_IP=$NEW_DROPLET_IP" >> $COMPOSITE_OUTPUT
# we only need records for frontend, media server and api server
doctl compute domain records create unstructured.studio --record-type A --record-name \
$FRONTEND_DOMAIN --record-data $NEW_DROPLET_IP --record-ttl 600
doctl compute domain records create unstructured.studio --record-type A --record-name \
$API_DOMAIN --record-data $NEW_DROPLET_IP --record-ttl 600
doctl compute domain records create unstructured.studio --record-type A --record-name \
$MEDIA_DOMAIN --record-data $NEW_DROPLET_IP --record-ttl 600
- uses: unstructuredstudio/zubhub/.github/actions/scp_action@master
env:
NEW_DROPLET_IP: ${{ fromJson(steps.create_droplet.outputs.JSON_STRING).NEW_DROPLET_IP }}
with:
host: ${{env.NEW_DROPLET_IP}}
username: ${{ vars.DO_BACKEND_USERNAME }}
key: ${{ secrets.DO_SSHKEY }}
source: "."
target: "/home/zubhub"
- name: Set output
id: set_output
env:
NEW_DROPLET_IP: ${{ fromJson(steps.create_droplet.outputs.JSON_STRING).NEW_DROPLET_IP }}
FRONTEND_DOMAIN: ${{ fromJson(steps.create_droplet.outputs.JSON_STRING).FRONTEND_DOMAIN }}
API_DOMAIN: ${{ fromJson(steps.create_droplet.outputs.JSON_STRING).API_DOMAIN }}
MEDIA_DOMAIN: ${{ fromJson(steps.create_droplet.outputs.JSON_STRING).MEDIA_DOMAIN }}
run: |
echo "NEW_DROPLET_IP=$NEW_DROPLET_IP" >> $GITHUB_OUTPUT
echo "FRONTEND_DOMAIN=$FRONTEND_DOMAIN" >> $GITHUB_OUTPUT
echo "API_DOMAIN=$API_DOMAIN" >> $GITHUB_OUTPUT
echo "MEDIA_DOMAIN=$MEDIA_DOMAIN" >> $GITHUB_OUTPUT
echo "CHECKOUT_COMMIT_REF=${{ env.CHECKOUT_COMMIT_REF }}" >> $GITHUB_OUTPUT
echo "CHECKOUT_REPO=${{ env.CHECKOUT_REPO }}" >> $GITHUB_OUTPUT
outputs:
NEW_DROPLET_IP: ${{ steps.set_output.outputs.NEW_DROPLET_IP }}
FRONTEND_DOMAIN: ${{ steps.set_output.outputs.FRONTEND_DOMAIN }}
API_DOMAIN: ${{ steps.set_output.outputs.API_DOMAIN }}
MEDIA_DOMAIN: ${{ steps.set_output.outputs.MEDIA_DOMAIN }}
CHECKOUT_COMMIT_REF: ${{ steps.set_output.outputs.CHECKOUT_COMMIT_REF }}
CHECKOUT_REPO: ${{ steps.set_output.outputs.CHECKOUT_REPO }}
#################################################################################
#################################################################################
# The following job is for building docker images and pushing them to dockerhub
##################################################################################
build_and_push:
needs: create_test_vm
runs-on: ubuntu-latest
strategy:
matrix:
service: ['frontend', 'web', 'celery', 'media']
steps:
- name: Use output
id: use_output
run: |
echo "NEW_DROPLET_IP=${{ needs.create_test_vm.outputs.NEW_DROPLET_IP }}" >> $GITHUB_ENV
echo "FRONTEND_DOMAIN=${{ needs.create_test_vm.outputs.FRONTEND_DOMAIN }}" >> $GITHUB_ENV
echo "API_DOMAIN=${{ needs.create_test_vm.outputs.API_DOMAIN }}" >> $GITHUB_ENV
echo "MEDIA_DOMAIN=${{ needs.create_test_vm.outputs.MEDIA_DOMAIN }}" >> $GITHUB_ENV
echo "CHECKOUT_COMMIT_REF=${{ needs.create_test_vm.outputs.CHECKOUT_COMMIT_REF }}" >> $GITHUB_ENV
echo "CHECKOUT_REPO=${{ needs.create_test_vm.outputs.CHECKOUT_REPO }}" >> $GITHUB_ENV
# careful with this as you are checking out the source branch of the PR
# which is a potential security risk. Only do this when you know what you are doing
# See https://stackoverflow.com/questions/75873833/how-to-protect-github-secrets-in-pull-request-actions-from-malicious-pull-reques
# See https://stackoverflow.com/questions/74957218/what-is-the-difference-between-pull-request-and-pull-request-target-event-in-git
# See https://dev.to/suzukishunsuke/secure-github-actions-by-pullrequesttarget-641#:~:text=pull_request_target%20is%20one%20of%20the,the%20pull%20request's%20base%20branch.
- uses: unstructuredstudio/zubhub/.github/actions/checkout@master
with:
ref: ${{ env.CHECKOUT_COMMIT_REF }}
repository: ${{ env.CHECKOUT_REPO }}
- name: create a .env file in the zubhub_frontend/zubhub/ directory
run: |
# create env file for frontend.
# This vm is only for testing so we don't need to worry about secrets being exposed
# (provided we don't put sensitive info in the env file)
cat << EOF > ./zubhub_frontend/zubhub/.env
GENERATE_SOURCEMAP=false
DISABLE_ESLINT_PLUGIN=true
REACT_APP_NODE_ENV=development
REACT_APP_BACKEND_DEVELOPMENT_URL=https://${{env.API_DOMAIN}}.unstructured.studio
REACT_APP_BACKEND_PRODUCTION_URL=https://${{env.API_DOMAIN}}.unstructured.studio
REACT_APP_DOSPACE_ACCESS_KEY_ID=
REACT_APP_DOSPACE_ACCESS_SECRET_KEY=
REACT_APP_VIDEO_UPLOAD_URL=
REACT_APP_VIDEO_FOLDER_NAME=videos
REACT_APP_DEV_VIDEO_FOLDER_NAME=dev_videos
REACT_APP_VIDEO_UPLOAD_PRESET_NAME=video_upload_preset
REACT_APP_DEV_VIDEO_UPLOAD_PRESET_NAME=dev_video_upload_preset
EOF
- name: Build and push ${{ matrix.service }}
uses: unstructuredstudio/zubhub/.github/actions/docker_build_and_push@master
with:
username: ${{ vars.DOCKERHUB_USERNAME }}
token: ${{ secrets.DOCKERHUB_TOKEN }}
# for frontend, context is ./zubhub_frontend/zubhub/
# for backend services, context is ./zubhub_backend/
context: ./zubhub_${{ matrix.service == 'frontend' && 'frontend/zubhub' || 'backend' }}/
# for frontend, Dockerfile is ./zubhub_frontend/zubhub/Dockerfile.prod
# for backend services, Dockerfile is ./zubhub_backend/compose/<service>/prod/Dockerfile
file: ./zubhub_${{ matrix.service == 'frontend' && 'frontend/zubhub' || format('backend/compose/{0}', matrix.service) }}/${{ matrix.service != 'frontend' && 'prod/' || '' }}Dockerfile${{ matrix.service == 'frontend' && '.prod' || '' }}
push: true
tags: unstructuredstudio/zubhub-test_${{ matrix.service }}:latest
##################################################################################
#################################################################################
# The following job is for deploying the project on the test droplet
##################################################################################
deploy:
needs:
- create_test_vm
- build_and_push
runs-on: ubuntu-latest
steps:
- name: Use output
id: use_output
run: |
echo "NEW_DROPLET_IP=${{ needs.create_test_vm.outputs.NEW_DROPLET_IP }}" >> $GITHUB_ENV
echo "FRONTEND_DOMAIN=${{ needs.create_test_vm.outputs.FRONTEND_DOMAIN }}" >> $GITHUB_ENV
echo "API_DOMAIN=${{ needs.create_test_vm.outputs.API_DOMAIN }}" >> $GITHUB_ENV
echo "MEDIA_DOMAIN=${{ needs.create_test_vm.outputs.MEDIA_DOMAIN }}" >> $GITHUB_ENV
- name: Executing remote command
uses: unstructuredstudio/zubhub/.github/actions/ssh_action@master
with:
host: ${{env.NEW_DROPLET_IP}}
username: ${{ vars.DO_BACKEND_USERNAME }}
key: ${{ secrets.DO_SSHKEY }}
script: |
# create env file for backend
cat << EOF > /home/zubhub_backend/.env
ENVIRONMENT=production
DEFAULT_FRONTEND_DOMAIN=${{env.FRONTEND_DOMAIN}}.unstructured.studio
DEFAULT_BACKEND_DOMAIN=${{env.API_DOMAIN}}.unstructured.studio
DEFAULT_DISPLAY_NAME=ZubHub
DEFAULT_FRONTEND_PROTOCOL=https
DEFAULT_BACKEND_PROTOCOL=https
SECRET_KEY=random string
DEBUG=1
STORE_MEDIA_LOCALLY=1
MEDIA_SECRET=random string
DEFAULT_MEDIA_SERVER_PROTOCOL=https
DEFAULT_MEDIA_SERVER_DOMAIN=${{env.MEDIA_DOMAIN}}.unstructured.studio
POSTGRES_NAME=postgres
POSTGRES_USER=postgres
POSTGRES_PASSWORD=postgres
POSTGRES_HOST=db
RABBITMQ_USERNAME=admin
RABBITMQ_PASSWORD=admin
RABBITMQ_MANAGEMENT_ALLOW_WEB_ACCESS=true
CELERY_BROKER=amqp://admin:admin@rabbitmq:5672/
CELERY_BACKEND=django-db
PROXY_COUNT=0
DETECT_MISCONFIG=0
SUPERUSER_PASSWORD=dummy_password
EOF
# deploy project
cd /home/zubhub
# Use sed to prepend "NODE_ENV=development" to the frontend build command.
# This is neccessary to avoid long build times on the test droplet.
sed -i '/"build":/ s/node /NODE_ENV=development node /' \
/home/zubhub/zubhub_frontend/zubhub/package.json
sudo bash /home/zubhub/test_vm_deployment/deploy_test_fullstack.sh
- uses: unstructuredstudio/zubhub/.github/actions/comment_action@master
with:
token: ${{ secrets.GITHUB_TOKEN }}
issue_number: ${{ github.event.pull_request.number }}
owner: ${{ github.repository_owner }}
repo: ${{ github.event.repository.name }}
message: |
Test VM is ready ✅✅✅
You can access it with the url:
https://${{env.FRONTEND_DOMAIN}}.unstructured.studio
######################################################################################
#################################################################################
# The following job is for destroying the test droplet and its DNS records
##################################################################################
destroy_test_vm:
if: |
(
github.event.action == 'closed' &&
contains(github.event.pull_request.labels.*.name, 'create-test-vm')
) ||
(
github.event.action == 'unlabeled' &&
github.event.label.name == 'create-test-vm'
)
runs-on: ubuntu-latest
steps:
- name: Get pr number and droplet ip
id: get_pr_number_and_droplet_ip
uses: unstructuredstudio/zubhub/.github/actions/doctl_action@master
with:
token: ${{ secrets.DO_ACCESS_TOKEN }}
script: |
# This script will be executed inside a composite action.
# rename GITHUB_OUTPUT so that it's clear that it's a
# composite output and should be handled differently from $GITHUB_OUTPUT
COMPOSITE_OUTPUT=$GITHUB_OUTPUT
# for events like unlabelled, github.event.pull_request.number is available
PR_NUMBER=${{ github.event.pull_request.number }}
if [[ -z "$PR_NUMBER" ]] ; then
# github.event.pull_request.number is not available in closed event
sudo apt-get install jq -y # TODO: remove this since jq. seems jq is installed by default
PR_NUMBER=$(curl --silent --show-error -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
https://api.github.com/repos/${{ github.repository }}/commits/${{ github.sha }}/pulls | \
jq -r '.[0].number')
fi
if [[ -z "$PR_NUMBER" ]] ; then
echo "PR_NUMBER is not available. Exiting..."
exit 1
fi
DROPLET_NAME=zubhub-test-$PR_NUMBER
# Get droplet ip
DROPLET_IP=$(doctl compute droplet get $DROPLET_NAME \
--format PublicIPv4 --no-header 2>/dev/null || true)
if [[ -z "$DROPLET_IP" ]] ; then
echo "Droplet does not exist. Exiting..."
exit 0
fi
echo "DROPLET_IP=$DROPLET_IP" >> $COMPOSITE_OUTPUT
echo "DROPLET_NAME=$DROPLET_NAME" >> $COMPOSITE_OUTPUT
- uses: unstructuredstudio/zubhub/.github/actions/comment_action@master
env:
DROPLET_IP: ${{ fromJson(steps.get_pr_number_and_droplet_ip.outputs.JSON_STRING).DROPLET_IP }}
if: ${{ env.DROPLET_IP != '' }}
with:
token: ${{ secrets.GITHUB_TOKEN }}
issue_number: ${{ github.event.pull_request.number }}
owner: ${{ github.repository_owner }}
repo: ${{ github.event.repository.name }}
message: |
@${{ github.actor }} is deleting test VM ...
- name: delete DNS records and test droplet
uses: unstructuredstudio/zubhub/.github/actions/doctl_action@master
env:
DROPLET_IP: ${{ fromJson(steps.get_pr_number_and_droplet_ip.outputs.JSON_STRING).DROPLET_IP }}
DROPLET_NAME: ${{ fromJson(steps.get_pr_number_and_droplet_ip.outputs.JSON_STRING).DROPLET_NAME }}
with:
token: ${{ secrets.DO_ACCESS_TOKEN }}
script: |
# Get the domain records
records=$(doctl compute domain records list unstructured.studio --format ID,Data --no-header)
# Loop through the records
while IFS= read -r record; do
# Split the record into ID and IP
ID=$(echo $record | cut -d' ' -f1)
IP=$(echo $record | cut -d' ' -f2)
# Delete the record if it points to the droplet IP
if [[ -n "$ID" && "$IP" == "${{env.DROPLET_IP}}" ]]; then
doctl compute domain records delete unstructured.studio $ID --force
fi
done <<< "$records"
# delete test droplet
if [[ -n "${{ env.DROPLET_NAME }}" ]] ; then
doctl compute droplet delete ${{ env.DROPLET_NAME }} --force
fi
- uses: unstructuredstudio/zubhub/.github/actions/comment_action@master
env:
DROPLET_IP: ${{ fromJson(steps.get_pr_number_and_droplet_ip.outputs.JSON_STRING).DROPLET_IP }}
if: ${{ env.DROPLET_IP != '' }}
with:
token: ${{ secrets.GITHUB_TOKEN }}
issue_number: ${{ github.event.pull_request.number }}
owner: ${{ github.repository_owner }}
repo: ${{ github.event.repository.name }}
message: |
Test VM deleted ✅✅✅
######################################################################################