Merge pull request #4223 from unicef/feature/project_structure

Feature/project structure

.github/helpers/.env-selenium

@@ -0,0 +1,15 @@
+REDIS_INSTANCE=redis:6379
+PYTHONUNBUFFERED=1
+SECRET_KEY=secretkey
+ENV=dev
+DEBUG=true
+CELERY_BROKER_URL=redis://redis:6379/0
+CELERY_RESULT_BACKEND=redis://redis:6379/0
+CACHE_LOCATION=redis://redis:6379/1
+DATABASE_URL=postgis://postgres:postgres@db:5432/postgres
+DATABASE_URL_HUB_MIS=postgis://postgres:postgres@db:5432/mis_datahub
+DATABASE_URL_HUB_CA=postgis://postgres:postgres@db:5432/ca_datahub
+DATABASE_URL_HUB_ERP=postgis://postgres:postgres@db:5432/erp_datahub
+DATABASE_URL_HUB_REGISTRATION=postgis://postgres:postgres@db:5432/rdi_datahub
+USE_DUMMY_EXCHANGE_RATES=yes
+CELERY_TASK_ALWAYS_EAGER=true

.github/helpers/.env-unit

@@ -0,0 +1,20 @@
+ENV=dev
+SECRET_KEY=31283012-38dasda
+POSTGRES_DB=postgres
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+DATABASE_URL=postgis://postgres:postgres@db:5432/postgres
+DATABASE_URL_HUB_MIS=postgis://postgres:postgres@db:5432/mis_datahub
+DATABASE_URL_HUB_CA=postgis://postgres:postgres@db:5432/ca_datahub
+DATABASE_URL_HUB_ERP=postgis://postgres:postgres@db:5432/erp_datahub
+DATABASE_URL_HUB_REGISTRATION=postgis://postgres:postgres@db:5432/rdi_datahub
+POSTGRES_SSL_MODE=off
+EMAIL_HOST=TBD
+EMAIL_HOST_USER=TBD
+EMAIL_HOST_PASSWORD=TBD
+KOBO_KF_URL=https://kobo.humanitarianresponse.info
+KOBO_KC_URL=https://kobo.humanitarianresponse.info
+KOBO_MASTER_API_TOKEN=token
+EXCHANGE_RATES_API_KEY=token
+REDIS_INSTANCE=redis:6379
+PYTHONUNBUFFERED=1

deployment/.gitignore → .github/helpers/.gitignore

@@ -1 +1,3 @@
 pipeline-*secret*
+
+.env

backend/dev.sh → .github/helpers/dev.sh

@@ -20,8 +20,7 @@ else
         --cov-report xml:coverage.xml \
         --randomly-seed=42 \
         --create-db \
-        hct_mis_api/ \
-        tests/
+        /tests/unit/
       ;;
     "lint")
       mkdir -p ./lint-results

.github/helpers/docker-compose.selenium.yml

@@ -0,0 +1,48 @@
+volumes:
+  backend-web-app:
+
+services:
+  backend:
+    volumes:
+      - ../src/report/screenshot/:/code/screenshot/
+      - ../src/report/:/code/report/
+      - type: volume
+        source: backend-web-app
+        target: /code/hct_mis_api/apps/web
+        volume:
+          nocopy: false
+    depends_on:
+      db:
+        condition: service_started
+      redis:
+        condition: service_started
+      elasticsearch:
+        condition: service_started
+      celery_worker:
+        condition: service_started
+      init_fe:
+        condition: service_completed_successfully
+    env_file:
+      - .env-selenium
+
+  init_fe:
+    image: ${dist_backend_image}
+    volumes:
+      - backend-web-app:/tmp/
+    command: |
+      sh -c "
+        cp -r ./hct_mis_api/apps/web/* /tmp/
+      "
+    restart: "no"
+
+  celery_worker:
+    image: ${dev_backend_image}
+    restart: unless-stopped
+    command: "celery -A hct_mis_api.apps.core.celery worker -E -l info"
+    depends_on:
+      - db
+      - redis
+      - elasticsearch
+    env_file:
+      - .env-selenium
+

compose.selenium.local.yml → .github/helpers/docker-compose.tst.yml

@@ -1,23 +1,21 @@
-version: "3.7"
-
-volumes:
-  backend-data-selenium:
-  backend-web-app-selenium:
-  db-selenium-data:
-  data_es-selenium:
-
 services:
-  redis:
-    restart: always
-    image: redis:4.0.11-alpine3.8
-    ports:
-      - 6379:6379
 
-  db_selenium:
+  backend:
+    image: ${dev_backend_image}
+    volumes:
+      - ../../src/test-coverage:/code/backend/test-coverage
+      - ./dev.sh:/code/dev.sh
+    depends_on:
+      - db
+      - redis
+      - elasticsearch
+    env_file:
+      - .env-unit
+
+  db:
     image: kartoza/postgis:14-3
     volumes:
-      - db-selenium-data:/var/lib/postgresql/data
-      - ./postgres/init:/docker-entrypoint-initdb.d
+      - ../../development_tools/postgres/init:/docker-entrypoint-initdb.d
     environment:
       - POSTGRES_MULTIPLE_DATABASES=unicef_hct_mis_cashassist,rdi_datahub,mis_datahub,erp_datahub,ca_datahub
       - POSTGRES_DB=postgres
@@ -27,14 +25,16 @@ services:
       - POSTGRES_HOST_AUTH_METHOD=trust
       - POSTGRES_SSL_MODE=off
     ports:
-      - 5432:5432
+      - "5433:5432"
+
+  redis:
+    restart: unless-stopped
+    image: redis:4.0.11-alpine3.8
 
   elasticsearch:
     image: unicef/hct-elasticsearch
-    container_name: elasticsearch
     build:
-      context: elasticsearch
-      dockerfile: Dockerfile
+      context: ../../development_tools/elasticsearch
     environment:
       - node.name=es01
       - cluster.name=es-docker-cluster
@@ -46,7 +46,3 @@ services:
       memlock:
         soft: -1
         hard: -1
-    volumes:
-      - data_es-selenium:/usr/share/elasticsearch/data
-    ports:
-      - 9200:9200

.github/workflows/ci.yml

@@ -33,18 +33,18 @@ jobs:
         uses: actions/cache@v2
         with:
           path: |
-            frontend/node_modules
-            frontend/.yarn
-            frontend/.yarn/cache
+            src/frontend/node_modules
+            src/frontend/.yarn
+            src/frontend/.yarn/cache
           key: ${{ runner.os }}-node-${{ hashFiles('frontend/yarn.lock') }}
           restore-keys: |
             ${{ runner.os }}-node-
       - run: yarn install --frozen-lockfile
-        working-directory: frontend
+        working-directory: src/frontend
       - run: yarn lint
-        working-directory: frontend
+        working-directory: src/frontend
       - run: yarn test --watchAll=false
-        working-directory: frontend
+        working-directory: src/frontend
 
   build_and_push_dev:
     runs-on: ubuntu-latest
@@ -207,7 +207,7 @@ jobs:
             --push ./"
           
           if [ "${{ github.ref }}" = "refs/heads/master" ]; then
-            version=$(python3 -c "import sys; version=None; [version:=line.split('=')[1].strip().strip('\"') for line in open('backend/pyproject.toml', 'r') if line.strip().startswith('version =')]; print(version if version else sys.exit(1))")
+            version=$(python3 -c "import sys; version=None; [version:=line.split('=')[1].strip().strip('\"') for line in open('src/pyproject.toml', 'r') if line.strip().startswith('version =')]; print(version if version else sys.exit(1))")
             tagged_image=${{ vars.DOCKERHUB_ORGANIZATION }}/hope:core-$version
             build_command="$build_command -t $tagged_image"
           fi
@@ -228,13 +228,15 @@ jobs:
 
       - name: Unit tests
         run: |
-          backend_image=${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev docker compose \
-            -f ./deployment/docker-compose.tst.yml \
+          # a little hack to allow to have 1 compose file running 2 different test suites because --env-file didn't work as docs suggested it should
+          dev_backend_image=${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev docker compose \
+            -f ./.github/helpers/docker-compose.tst.yml \
+            --profile unit \
             run backend ./dev.sh test
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         with:
-          files: ./backend/coverage.xml
+          files: ./src/coverage.xml
           flags: unittests
           token: ${{ secrets.CODECOV_TOKEN }}
           verbose: true
@@ -253,27 +255,32 @@ jobs:
 
       - name: E2E tests
         run: |
-          compose_file=./deployment/docker-compose.selenium-night.yml
+          extra_options=""
+          
           if [ "${{ github.event_name }}" = "pull_request" ]; then
-            compose_file=./deployment/docker-compose.selenium.yml
+            extra_options="-m 'not night'"
           fi
           dist_backend_image=${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dist dev_backend_image=${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev docker compose \
-            -f $compose_file \
-            run selenium
+            -f ./.github/helpers/docker-compose.tst.yml \
+            -f ./.github/helpers/docker-compose.selenium.yml \
+            run backend bash -c "
+              waitforit -host=db -port=5432 -timeout=30
+              pytest -svvv $extra_options /tests/selenium --cov-report xml:./coverage.xml --html-report=./report/report.html --randomly-seed=42
+            "
       - name: Upload Artifact
         uses: actions/upload-artifact@v4
         if: always()
         continue-on-error: true
         with:
           name: report
-          path: ./backend/report/
+          path: ./src/report/
           retention-days: 5
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         if: always()
         continue-on-error: true
         with:
-          files: ./backend/coverage.xml
+          files: ./src/coverage.xml
           flags: e2e
           token: ${{ secrets.CODECOV_TOKEN }}
           verbose: true

.gitignore

@@ -46,7 +46,7 @@ pip-delete-this-directory.txt
 # Unit test / coverage reports
 htmlcov/
 .tox/
-.coverage
+.coveragep
 .coverage.*
 .cache
 nosetests.xml
@@ -91,6 +91,7 @@ celerybeat-schedule
 *.sage.py
 
 # Environments
+development_tools/.env
 .env
 .envrc
 .venv