Merge pull request #54 from mantidproject/fixing-ci-tests

Fix and re-enable CI tests

.github/workflows/ci.yml

@@ -3,41 +3,38 @@ name: CI - Ubuntu
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
-    branches:
-      - master
 
 jobs:
   webtests:
     runs-on: ubuntu-latest
     steps:
       - name: Setup Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: 3.x
           architecture: x64
-      - name: Checkout head 
-        uses: actions/checkout@v2
+      - name: Checkout head
+        uses: actions/checkout@v4
       - name: Run webtests
-        run: chmod +x ./run_web_tests.sh && ./run_web_tests.sh
+        run: bash ./run_web_tests.sh
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   flake8:
     runs-on: ubuntu-latest
     steps:
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: 3.x
           architecture: x64
-      - name: Checkout head 
-        uses: actions/checkout@v2
+      - name: Checkout head
+        uses: actions/checkout@v4
       - name: Install flake8
         run: pip install flake8
       - name: Run flake8
         run: cd web && flake8
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-

bin/boot.sh

@@ -18,11 +18,27 @@ fi
 
 # Create empty directories, so they are not owned by root
 mkdir -p "$SOURCE_DIR/pgdata"
-mkdir -p "$SOURCE_DIR/webdata"
 
 # Start external network
 create_external_net
 # Build services
 docker-compose --project-name ${PROJECT_NAME} build
 # Bring up the stack and detach
-docker-compose --project-name ${PROJECT_NAME} up -d
+docker-compose --project-name ${PROJECT_NAME} up --detach
+
+# Replace this loop with `--wait-timeout` when it is available
+# with 'docker compose' v2
+sleep_counter=0
+while [ $(docker-compose ps --quiet --filter status=running | wc -l | xargs) -ne 4 ]
+do
+  sleep 1
+  let counter++
+  if [[ $counter -eq 30 ]]; then
+    echo "Services have not come up to a running state."
+    exit 1
+  fi
+  echo "Waiting for all services to be running."
+done
+
+
+echo "Services up and running."

bin/shutdown.sh

@@ -12,8 +12,7 @@ docker-compose down
 # the web data volume shouldn't really be persistent as all of the files
 # come from an image
 echo "Removing webdata volume so it is rebuilt on next startup"
-rm -r "${SOURCE_DIR}/webdata"
-docker volume rm errorreports_webdata
+docker volume rm ${PROJECT_NAME}_webdata
 
 echo "Removing external network nginx_net"
 docker network rm nginx_net

blank.env

@@ -1,20 +1,17 @@
-# Default config
-DB_NAME=django
+DEBUG=false
+HOST_PORT=8083
 
+# Django settings. Passwords are in the password management tool
+DB_NAME=django
 # Generate with pwgen -s 32 1
 SECRET_KEY=<Not Set>
-
-# These are located in the password management tool
 DB_USER=<Not Set>
 DB_PASS=<Not Set>
-
 DB_SERVICE=postgres
 DB_PORT=5432
 
 # Can be found Slack settings
 SLACK_WEBHOOK_URL=<Not Set>
-
-HOST_PORT=8083
 SLACK_ERROR_REPORTS_CHANNEL=#error-reports
 SLACK_ERROR_REPORTS_USERNAME="Error Reporter"
 SLACK_ERROR_REPORTS_EMOJI=:sadmantid:

docker-compose.yml

@@ -46,9 +46,6 @@ services:
       SECRET_KEY: ${SECRET_KEY}
       # Define this in .env for development mode. DO NOT USE IN PRODUCTION
       DEBUG: ${DEBUG}
-      MAIL_PASS: ${MAIL_PASS}
-      MAIL_PORT: ${MAIL_PORT}
-      ERROR_EMAIL: ${ERROR_EMAIL}
 
   nginx-errorreports:
     restart: always

run_web_tests.sh

@@ -1,4 +1,48 @@
-cp .test_env .env
-./bin/boot.sh
-sleep 30
-docker exec -t errorreports_web_1 sh -c "python manage.py test"
+ENV_FILE=.env
+DB_DIR=pgdata
+DJANGO_SERVICE_NAME=web
+CLEAN_ENV=false
+CLEAN_SERVICES=false
+
+if [[ ! -f "$ENV_FILE" ]]; then
+  if [[ -d ${DB_DIR} ]]; then
+    echo "A $ENV_FILE file could not be found yet a '${DB_DIR}' directory exists."
+    echo "This implies a database has been created but the credentials are missing."
+    echo "Tests will fail as they won't be able to access the database."
+    echo "IF IN A DEVELOPMENT ENVIRONMENT remove ${DB_DIR} and re-run this script."
+    exit 1
+  fi
+
+  echo "No $ENV_FILE file and no DB found. Generating .env file for tests."
+  echo "Both .env file and DB will be cleaned "
+  cat blank.env |\
+  sed -e "s@DEBUG=false@DEBUG=true@" |\
+  sed -e "s@SECRET_KEY=<Not Set>@SECRET_KEY=123456789abcdefghijkl@" |\
+  sed -e 's@DB_USER=<Not Set>@DB_USER=testuser@' |\
+  sed -e 's@DB_PASS=<Not Set>@DB_PASS=testuserpasswd@' |\
+  sed -e 's@SLACK_WEBHOOK_URL=<Not Set>@SLACK_WEBHOOK_URL=@' > ${ENV_FILE}
+  CLEAN_ENV=true
+else
+  echo "Running tests using an existing environment file..."
+fi
+
+if [[ -z $(docker-compose ps --quiet --filter status=running $DJANGO_SERVICE_NAME) ]]; then
+  echo "Booting services."
+  ./bin/boot.sh
+  CLEAN_SERVICES=true
+else
+   echo "Services already running.."
+fi
+
+echo "Executing web tests..."
+docker-compose exec $DJANGO_SERVICE_NAME sh -c "python manage.py test $@"
+
+# Clean up
+if [[ $CLEAN_SERVICES == true ]]; then
+  echo "Shutting down services that were auto started."
+  ./bin/shutdown.sh
+fi
+if [[ $CLEAN_ENV == true ]]; then
+  echo "Cleaning auto-generated .env file"
+  rm -f $ENV_FILE
+fi

web/services/admin.py

@@ -1,6 +1,6 @@
 from django.contrib import admin
 from django.urls import reverse
-from django.utils.safestring import mark_safe   
+from django.utils.safestring import mark_safe
 # Register your models here.
 from services.models import ErrorReport, UserDetails
 

web/services/tasks.py

@@ -22,7 +22,7 @@ def send_notification_to_slack(name,
     :param additional_text: Any additional text provided
     """
     slack_webhook_url = settings.SLACK_WEBHOOK_URL
-    if slack_webhook_url is None:
+    if not slack_webhook_url:
         return
     text = """Name: {}  Email: {}
         Additional text:

web/settings.py

@@ -25,7 +25,7 @@
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = os.getenv('DEBUG', 'NO').lower() in ('on', 'true', 'y', 'yes')
 
-DEFAULT_AUTO_FIELD='django.db.models.BigAutoField'
+DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
 
 ALLOWED_HOSTS = ['*']