Add CORS in nginx

- Fix flake8 issues.
- Add missing environment variables used.

.circleci/config.yml

@@ -13,7 +13,8 @@ jobs:
           name: build docker base image
           command: |
             export DOCKER_BUILDKIT=1
-            docker-compose build --progress plain base
+            cp .env-sample .env
+            docker-compose build --progress plain
             docker-compose pull db redis
       - run:
           name: Validate if there are no pending django migrations.

.env-sample

@@ -0,0 +1,4 @@
+# Required
+DJANGO_SECRET_KEY=RANDOM-STRING-FOR-SECRET-KEYS
+
+# For other, look at main/settings.py:env for available options.

.gitignore

@@ -6,6 +6,7 @@ env
 
 # secret settings env variables
 .env*
+!.env-sample
 
 # python stuff
 *.pyc

api/admin.py

@@ -1,4 +1,3 @@
-import os
 import csv
 import time
 from django.contrib.gis import admin as geoadmin
@@ -7,24 +6,25 @@
 from django.utils.translation import ugettext_lazy as _
 from django.utils.html import format_html_join, format_html
 from django.utils.safestring import mark_safe
-from api.event_sources import SOURCES
-from api.admin_classes import RegionRestrictedAdmin
-from django_admin_listfilter_dropdown.filters import RelatedDropdownFilter
-import api.models as models
 from django.contrib.auth.admin import UserAdmin
 from django.contrib.auth.models import User
+from django.http import HttpResponse, HttpResponseRedirect
+from django.conf import settings
+from django_admin_listfilter_dropdown.filters import RelatedDropdownFilter
 from rest_framework.authtoken.admin import TokenAdmin
 from rest_framework.authtoken.models import Token
-from django.http import HttpResponse, HttpResponseRedirect
-from .forms import ActionForm
+from reversion_compare.admin import CompareVersionAdmin
 # from reversion.models import Revision
 
-from reversion_compare.admin import CompareVersionAdmin
+from api.event_sources import SOURCES
+from api.admin_classes import RegionRestrictedAdmin
+import api.models as models
 from lang.admin import TranslationAdmin, TranslationInlineModelAdmin
-
 from api.management.commands.index_and_notify import Command as Notify
 from notifications.models import RecordType, SubscriptionType
 
+from .forms import ActionForm
+
 
 class ProfileInline(admin.StackedInline):
     model = models.Profile
@@ -423,16 +423,18 @@ class AppealFilterAdmin(CompareVersionAdmin):
     list_display = ('name', 'value')
     search_fields = ('name', 'value')
 
+
 class UserCountryAdmin(CompareVersionAdmin):
-    list_display = ('user','country')
-    #search_fields = ('user','country')
+    list_display = ('user', 'country')
+    # search_fields = ('user','country')
     model = models.UserCountry
 
+
 class UserRegionAdmin(CompareVersionAdmin):
-    list_display = ['user','get_firstname','get_lastname','get_email','region',]
+    list_display = ['user', 'get_firstname', 'get_lastname', 'get_email', 'region']
 
     def get_firstname(self, obj):
-        return obj.user.first_name 
+        return obj.user.first_name
     get_firstname.short_description = 'First name'
     get_firstname.admin_order_field = 'user__first_name'
 
@@ -446,10 +448,10 @@ def get_email(self, obj):
     get_email.short_description = 'Email'
     get_email.admin_order_field = 'user__email'
 
-
-    #search_fields = ('user','country')
+    # search_fields = ('user','country')
     model = models.UserRegion
 
+
 class GeneralDocumentAdmin(CompareVersionAdmin, RegionRestrictedAdmin, TranslationAdmin):
     search_fields = ('name',)
 
@@ -783,5 +785,6 @@ def get_actions(self, request):
 admin.site.register(models.UserCountry, UserCountryAdmin)
 admin.site.register(models.UserRegion, UserRegionAdmin)
 # admin.site.register(Revision, RevisionAdmin)
-admin.site.site_url = 'https://' + os.environ.get('FRONTEND_URL')
+
+admin.site.site_url = 'https://' + settings.FRONTEND_URL
 admin.widgets.RelatedFieldWidgetWrapper.template_name = 'related_widget_wrapper.html'

api/esconnection.py

@@ -1,9 +1,8 @@
-import os
 from elasticsearch import Elasticsearch
+from django.conf import settings
 
-host = os.environ.get('ES_HOST')
-if host is not None:
-    ES_CLIENT = Elasticsearch([host], timeout=2, max_retries=3, retry_on_timeout=True)
+if settings.ELASTIC_SEARCH_HOST is not None:
+    ES_CLIENT = Elasticsearch([settings.ELASTIC_SEARCH_HOST], timeout=2, max_retries=3, retry_on_timeout=True)
 else:
     print('Warning: No elasticsearch host found, will not index elasticsearch')
     ES_CLIENT = None

api/logger.py

@@ -1,10 +1,14 @@
 import logging
 import sys
-import os
+
+from django.conf import settings
+
 from azure_storage_logging.handlers import BlobStorageTimedRotatingFileHandler as storage
 
-formatter = logging.Formatter(fmt='%(asctime)s %(levelname)-8s %(message)s',
-                              datefmt='%Y-%m-%d %H:%M:%S')
+formatter = logging.Formatter(
+    fmt='%(asctime)s %(levelname)-8s %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S'
+)
 
 screen_handler = logging.StreamHandler(stream=sys.stdout)
 screen_handler.setFormatter(formatter)
@@ -13,16 +17,19 @@
 logger.setLevel('DEBUG')
 logger.addHandler(screen_handler)
 
-if (os.environ.get('AZURE_STORAGE_ACCOUNT') is not None and
-        os.environ.get('AZURE_STORAGE_KEY') is not None):
-    handler = storage(account_name=os.environ.get('AZURE_STORAGE_ACCOUNT'),
-                      account_key=os.environ.get('AZURE_STORAGE_KEY'),
-                      filename='go.log',
-                      when='M',
-                      interval=90,
-                      container='logs',
-                      encoding='utf-8'
-                      )
+if (
+    settings.AZURE_STORAGE_ACCOUNT is not None and
+    settings.AZURE_STORAGE_KEY is not None
+):
+    handler = storage(
+        account_name=settings.AZURE_STORAGE_ACCOUNT,
+        account_key=settings.AZURE_STORAGE_KEY,
+        filename='go.log',
+        when='M',
+        interval=90,
+        container='logs',
+        encoding='utf-8'
+    )
     handler.setFormatter(formatter)
     logger.addHandler(handler)
 else:

api/management/commands/index_and_notify.py

@@ -1,10 +1,13 @@
+import html
 from datetime import datetime, timezone, timedelta
+
 from django.db.models import Q, F, ExpressionWrapper, DurationField, Sum
 from django.db.models.query import QuerySet
 from django.core.management.base import BaseCommand
 from django.contrib.auth.models import User
 from django.conf import settings
 from django.template.loader import render_to_string
+
 from elasticsearch.helpers import bulk
 from utils.elasticsearch import construct_es_data
 from api.esconnection import ES_CLIENT
@@ -14,8 +17,6 @@
 from notifications.hello import get_hello
 from notifications.notification import send_notification
 from deployments.models import PersonnelDeployment, ERU, Personnel
-from main.frontend import frontend_url
-import html
 
 time_5_minutes = timedelta(minutes=5)
 time_1_day = timedelta(days=1)
@@ -123,7 +124,8 @@ def gather_subscribers(self, records, rtype, stype):
             subscribers = User.objects.filter(subscription__rtype=rtype_of_subscr,
                                               subscription__stype=stype, is_active=True).values('email')
 
-        # For FOLLOWED_EVENTs and DEPLOYMENTs we do not collect other generic (d*, country, region) subscriptions, just one. This part is not called.
+        # For FOLLOWED_EVENTs and DEPLOYMENTs we do not collect other generic (d*, country, region) subscriptions, just one.
+        # This part is not called.
         if rtype_of_subscr != RecordType.FOLLOWED_EVENT and \
            rtype_of_subscr != RecordType.SURGE_ALERT and \
            rtype_of_subscr != RecordType.SURGE_DEPLOYMENT_MESSAGES:
@@ -136,7 +138,9 @@ def gather_subscribers(self, records, rtype, stype):
 
             lookups = dtypes + countries + regions
             if len(lookups):
-                subscribers = (subscribers | User.objects.filter(subscription__lookup_id__in=lookups, is_active=True).values('email')).distinct()
+                subscribers = (
+                    subscribers | User.objects.filter(subscription__lookup_id__in=lookups, is_active=True).values('email')
+                ).distinct()
         emails = list(set([subscriber['email'] for subscriber in subscribers]))
         return emails
 
@@ -148,21 +152,26 @@ def get_template(self, rtype=99):
     # Get the front-end url of the resource
     def get_resource_uri(self, record, rtype):
         # Determine the front-end URL
-        resource_uri = frontend_url
-        if rtype == RecordType.SURGE_ALERT or rtype == RecordType.FIELD_REPORT: # Pointing to event instead of field report %s/%s/%s - Munu asked - ¤
-            belonging_event = record.event.id if record.event is not None else 57 # Very rare – giving a non-existent | manually created surge – no event
-            resource_uri = '%s/emergencies/%s#surge' % (frontend_url, belonging_event)
+        resource_uri = settings.FRONTEND_URL
+        if (
+            rtype == RecordType.SURGE_ALERT or rtype == RecordType.FIELD_REPORT
+        ):  # Pointing to event instead of field report %s/%s/%s - Munu asked - ¤
+            belonging_event = (
+                record.event.id if record.event is not None else 57
+            )  # Very rare – giving a non-existent | manually created surge – no event
+            resource_uri = '%s/emergencies/%s#surge' % (settings.FRONTEND_URL, belonging_event)
         elif rtype == RecordType.SURGE_DEPLOYMENT_MESSAGES:
-            resource_uri = '%s/%s' % (frontend_url, 'deployments')  # can be further sophisticated
+            resource_uri = '%s/%s' % (settings.FRONTEND_URL, 'deployments')  # can be further sophisticated
         elif rtype == RecordType.APPEAL and (
                 record.event is not None and not record.needs_confirmation):
             # Appeals with confirmed emergencies link to that emergency
-            resource_uri = '%s/emergencies/%s' % (frontend_url, record.event.id)
+            resource_uri = '%s/emergencies/%s' % (settings.FRONTEND_URL, record.event.id)
         elif rtype != RecordType.APPEAL:
             # One-by-one followed or globally subscribed emergencies
             resource_uri = '%s/%s/%s' % (
-                frontend_url,
-                'emergencies' if rtype == RecordType.EVENT or rtype == RecordType.FOLLOWED_EVENT else 'reports', # this else never occurs, see ¤
+                settings.FRONTEND_URL,
+                # this else never occurs, see ¤
+                'emergencies' if rtype == RecordType.EVENT or rtype == RecordType.FOLLOWED_EVENT else 'reports',
                 record.id
             )
         return resource_uri
@@ -191,13 +200,16 @@ def get_record_title(self, record, rtype):
                     sendMe = sendMe + ' (' + country + ')'
             return sendMe
         elif rtype == RecordType.SURGE_ALERT:
-            duration = (record.end-record.start).days
+            duration = (record.end - record.start).days
             if duration > 29:
                 durationMonth = (record.end - record.start).days // 30
                 duration = f"{durationMonth} month{'s' if durationMonth > 1 else ''}"
             else:
                 duration = f"{(record.end - record.start).days} days"
-            return f"{record.operation if record.operation_en else record.event.name}, {duration} starting on {record.start.date()}"
+            return (
+                f'{record.operation if record.operation_en else record.event.name}'
+                f', {duration} starting on {record.start.date()}'
+            )
             # go-frontend/issues/2041: del ' (' + record.atype.name + ', ' + record.category.name.lower() +')'
         elif rtype == RecordType.SURGE_DEPLOYMENT_MESSAGES:
             return '%s, %s' % (record.country_deployed_to, record.region_deployed_to)
@@ -296,7 +308,10 @@ def get_weekly_digest_latest_deployments(self):
         #         alert_to_add = {
         #             'type': 'Alert',
         #             'operation': alert.operation,
-        #             'event_url': '{}/emergencies/{}#overview'.format(frontend_url, event.id) if event else frontend_url,
+        #            'event_url': (
+        #                '{}/emergencies/{}#overview'.format(settings.FRONTEND_URL, event.id) if event else
+        #                settings.FRONTEND_URL,
+        #            ),
         #             'society_from': '',
         #             'deployed_to': '',
         #             'name': '',
@@ -313,7 +328,10 @@ def get_weekly_digest_latest_deployments(self):
             country_from = Country.objects.get(id=pers.country_from_id) if pers.country_from_id is not None else None
             dep_to_add = {
                 'operation': event.name if event else '',
-                'event_url': '{}/emergencies/{}#overview'.format(frontend_url, event.id) if event else frontend_url,
+                'event_url': (
+                    '{}/emergencies/{}#overview'.format(settings.FRONTEND_URL, event.id) if event else
+                    settings.FRONTEND_URL
+                ),
                 'society_from': country_from.society_name if country_from else '',
                 'name': pers.name,
                 'role': pers.role,
@@ -329,7 +347,9 @@ def get_weekly_digest_highlights(self):
         events = Event.objects.filter(is_featured=True, updated_at__gte=dig_time).order_by('-updated_at')
         ret_highlights = []
         for ev in events:
-            amount_requested = Appeal.objects.filter(event_id=ev.id).aggregate(Sum('amount_requested'))['amount_requested__sum'] or '--'
+            amount_requested = (
+                Appeal.objects.filter(event_id=ev.id).aggregate(Sum('amount_requested'))['amount_requested__sum'] or '--'
+            )
             amount_funded = Appeal.objects.filter(event_id=ev.id).aggregate(Sum('amount_funded'))['amount_funded__sum'] or '--'
             coverage = '--'
 
@@ -340,7 +360,9 @@ def get_weekly_digest_highlights(self):
                 'hl_id': ev.id,
                 'hl_name': ev.name,
                 'hl_last_update': ev.updated_at,
-                'hl_people': Appeal.objects.filter(event_id=ev.id).aggregate(Sum('num_beneficiaries'))['num_beneficiaries__sum'] or '--',
+                'hl_people': (
+                    Appeal.objects.filter(event_id=ev.id).aggregate(Sum('num_beneficiaries'))['num_beneficiaries__sum'] or '--'
+                ),
                 'hl_funding': amount_requested,
                 'hl_deployed_eru': ERU.objects.filter(event_id=ev.id).aggregate(Sum('units'))['units__sum'] or '--',
                 'hl_deployed_sp': PersonnelDeployment.objects.filter(event_deployed_to_id=ev.id).count(),
@@ -470,17 +492,19 @@ def construct_template_record(self, rtype, record):
                     volunteers += int(f.num_volunteers or 0)
                     delegates += int(f.num_expats_delegates or 0)
             resource_uri, follow_url = self.get_resource_uri(record, rtype), None
-            if resource_uri != frontend_url:
-                # instead of '{}/account#notifications'.format(frontend_url):
+            if resource_uri != settings.FRONTEND_URL:
+                # instead of '{}/account#notifications'.format(settings.FRONTEND_URL):
                 follow_url = resource_uri + '/follow'
                 resource_uri += '#overview'
             rec_obj = {
-                'frontend_url': frontend_url,
+                'frontend_url': settings.FRONTEND_URL,
                 'resource_uri': resource_uri,
                 'follow_url': follow_url,
                 'admin_uri': self.get_admin_uri(record, rtype),
                 'title': self.get_record_title(record, rtype),
-                'situation_overview': Event.objects.values_list('summary', flat=True).get(id=record.event_id) if record.event_id is not None else '',
+                'situation_overview': (
+                    Event.objects.values_list('summary', flat=True).get(id=record.event_id) if record.event_id is not None else ''
+                ),
                 'key_figures': {
                     'people_targeted': float(record.num_beneficiaries),
                     'funding_req': float(record.amount_requested),
@@ -513,7 +537,7 @@ def construct_template_record(self, rtype, record):
                 'admin_uri': self.get_admin_uri(record, rtype),
                 'title': self.get_record_title(record, rtype),
                 'content': shortened,
-        }
+            }
         else:  # The default (old) template
             rec_obj = {
                 'resource_uri': self.get_resource_uri(record, rtype),
@@ -698,13 +722,24 @@ def notify(self, records, rtype, stype, uid=None):
             if len(recipients):
                 # check if email is not in events_sent_to{event_id: recipients}
                 if not emails:
-                    logger.info('Silent about the one-by-one subscribed %s – user %s has not set email address' % (record_type, uid))
-                # Recently we do not allow EDIT (modif.) subscription, so it is irrelevant recently (do not check the 1+ events in loop) :
+                    logger.info(
+                        'Silent about the one-by-one subscribed %s – user %s has not set email address' % (record_type, uid)
+                    )
+                # Recently we do not allow EDIT (modif.) subscription
+                # , so it is irrelevant recently (do not check the 1+ events in loop) :
                 elif (records[0].id not in events_sent_to) or (emails[0] not in events_sent_to[records[0].id]):
-                    logger.info('Notifying %s subscriber about %s one-by-one subscribed %s' % (len(emails), record_count, record_type))
+                    logger.info(
+                        'Notifying %s subscriber about %s one-by-one subscribed %s' % (
+                            len(emails),
+                            record_count,
+                            record_type,
+                        )
+                    )
                     send_notification(subject, recipients, html, RTYPE_NAMES[rtype] + ' notification - ' + subject)
                 else:
-                    logger.info('Silent about a one-by-one subscribed %s – user already notified via generic subscription' % (record_type))
+                    logger.info(
+                        'Silent about a one-by-one subscribed %s – user already notified via generic subscription' % (record_type)
+                    )
 
     def index_records(self, records, to_create=True):
         self.bulk([construct_es_data(record, is_create=to_create) for record in list(records)])
@@ -732,7 +767,9 @@ def filter_just_created(self, queryset):
                 record.updated_at.replace(microsecond=0) == record.created_at.replace(microsecond=0))]
 
     def check_ingest_issues(self, having_ingest_issue):
-        # having_ingest_issue = CronJob.objects.raw('SELECT * FROM api_cronjob WHERE status=' + str(CronJobStatus.ERRONEOUS.value))
+        # having_ingest_issue = CronJob.objects.raw(
+        #     f'SELECT * FROM api_cronjob WHERE status={CronJobStatus.ERRONEOUS.value}'
+        # )
         ingest_issue_id = having_ingest_issue[0].id if len(having_ingest_issue) > 0 else -1
         ingestor_name = having_ingest_issue[0].name if len(having_ingest_issue) > 0 else ''
         if len(having_ingest_issue) > 0:
@@ -756,7 +793,9 @@ def handle(self, *args, **options):
         condU = Q(updated_at__gte=time_diff)
         condR = Q(real_data_update__gte=time_diff)  # instead of modified at
         cond2 = ~Q(previous_update__gte=time_diff_1_day)  # negate (~) no previous_update in the last day, so send once a day
-        condF = Q(auto_generated_source='New field report')  # exclude those events that were generated from field reports, to avoid 2x notif.
+        condF = Q(
+            auto_generated_source='New field report'
+        )  # exclude those events that were generated from field reports, to avoid 2x notif.
         condE = Q(status=CronJobStatus.ERRONEOUS)
 
         # ".annotate(diff...)" - To check if a record was newly created, we check if