diff --git a/.flake8 b/.flake8 index 212f18055..8c3b32a18 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,5 @@ [flake8] -ignore = C901, W504 +extend-ignore = C901, E203, E701 max-line-length = 130 exclude = .git,__pycache__,old,build,dist,*migrations*,*snapshots* max-complexity = 10 diff --git a/api/management/commands/add_action_tooltips.py b/api/management/commands/add_action_tooltips.py index 1d2726f05..a5e64365f 100644 --- a/api/management/commands/add_action_tooltips.py +++ b/api/management/commands/add_action_tooltips.py @@ -14,76 +14,76 @@ def handle(self, *args, **options): tts = [ [ "National Society readiness", - "Includes adapting community preparedness, response and DRR measures in view of the pandemic safety measures, especially when they are dealing with compounding disasters. Also includes adjusting contingency plans, updating Risk assessments identifying the different services that will be provided during the current disaster and how do these services fit into the National contingency plans.", + "Includes adapting community preparedness, response and DRR measures in view of the pandemic safety measures, especially when they are dealing with compounding disasters. Also includes adjusting contingency plans, updating Risk assessments identifying the different services that will be provided during the current disaster and how do these services fit into the National contingency plans.", # noqa: E501 ], [ "National Society sustainability", - "Includes identifying Core organizational budget to maintain the minimum structure of a National Society, available unrestricted reserves and unrestricted income and developing and executing business continuity plans.", + "Includes identifying Core organizational budget to maintain the minimum structure of a National Society, available unrestricted reserves and unrestricted income and developing and executing business continuity plans.", # noqa: E501 ], [ "Support to volunteers", - "Includes providing insurance for volunteers as well as access to PPE for volunteers to safely fulfil their duties.", + "Includes providing insurance for volunteers as well as access to PPE for volunteers to safely fulfil their duties.", # nqa: E501 # noqa: E501 ], [ "Epidemic control measures", - "Includes testing, contact tracing and support for quarantine of contacts and isolation of COVID-19 cases not requiring clinical treatment.", + "Includes testing, contact tracing and support for quarantine of contacts and isolation of COVID-19 cases not requiring clinical treatment.", # noqa: E501 ], [ "Risk communication, community engagement, and health and hygiene promotion", - "Includes approaches such as community-based activities, mass media (local radios, TV, press), social media outreach or face-to-face and interpersonal communication (door to door, community dialogues, community meetings) to promote hygiene and other risk reduction interventions.", + "Includes approaches such as community-based activities, mass media (local radios, TV, press), social media outreach or face-to-face and interpersonal communication (door to door, community dialogues, community meetings) to promote hygiene and other risk reduction interventions.", # noqa: E501 ], [ "Community-based surveillance (CBS) for COVID-19", - "Includes staff and volunteers who have completed a training and who are actively reporting health risks through the RC/RC approach to CBS. It does not include point-of-entry screenings, call centers, active surveillance or contact tracing.", + "Includes staff and volunteers who have completed a training and who are actively reporting health risks through the RC/RC approach to CBS. It does not include point-of-entry screenings, call centers, active surveillance or contact tracing.", # noqa: E501 ], [ "Infection prevention and control (IPC) and WASH (health facilities)", - "Includes any form of infection prevention and control (IPC) support provided to health facilities. Support may include donation of personal protective equipment or medical supplies, or other medical, logistics, or coordination support, support for triage of COVID-19 cases, installation of WASH infrastructure to facilitate IPC best practices, and IPC training.", + "Includes any form of infection prevention and control (IPC) support provided to health facilities. Support may include donation of personal protective equipment or medical supplies, or other medical, logistics, or coordination support, support for triage of COVID-19 cases, installation of WASH infrastructure to facilitate IPC best practices, and IPC training.", # noqa: E501 ], [ "Infection prevention and control (IPC) and WASH (community)", - "Includes community WASH activities such as establishing or rehabilitating a drinking water source, toilet, and/or a handwashing facility.", + "Includes community WASH activities such as establishing or rehabilitating a drinking water source, toilet, and/or a handwashing facility.", # noqa: E501 ], [ "Mental health and psychosocial support services (MHPSS)", - "Includes direct mental health and psychosocial support services to community members provided through National Society volunteers or staff.", + "Includes direct mental health and psychosocial support services to community members provided through National Society volunteers or staff.", # noqa: E501 ], [ "Isolation and clinical case management for COVID-19 cases", - "Includes support to health facilities that are actively treating COVID-19 cases or providing observational support and care to COVID-19 cases in isolation. Support can range from donations to staff and volunteer support.", + "Includes support to health facilities that are actively treating COVID-19 cases or providing observational support and care to COVID-19 cases in isolation. Support can range from donations to staff and volunteer support.", # noqa: E501 ], [ "Ambulance services for COVID-19 cases", - "Includes suspected or confirmed COVID-19 patients who have received ambulance transport services by the National Society.", + "Includes suspected or confirmed COVID-19 patients who have received ambulance transport services by the National Society.", # noqa: E501 ], [ "Maintain access to essential health services (community health)", - "Includes supporting essential community health services that have been impacted/reduced as a result of COVID-19, such as community-based malaria interventions, NCD support, and immunization campaigns to counteract the reduction in community health services and/or increased need for community health services resulting from COVID-19.", + "Includes supporting essential community health services that have been impacted/reduced as a result of COVID-19, such as community-based malaria interventions, NCD support, and immunization campaigns to counteract the reduction in community health services and/or increased need for community health services resulting from COVID-19.", # noqa: E501 ], [ "Maintain access to essential health services (clinical and paramedical)", - "Includes supporting health facilities supported to maintain routine essential services such as MCH, NCD, malaria treatment, and other essential services. Support can be through either physical or technical support of volunteers or staff, or support through PPE or medical supplies donation.", + "Includes supporting health facilities supported to maintain routine essential services such as MCH, NCD, malaria treatment, and other essential services. Support can be through either physical or technical support of volunteers or staff, or support through PPE or medical supplies donation.", # noqa: E501 ], [ "Management of the dead", - "Includes direct burial or cremation of human remains of COVID-19 cases, and supervision of safe burial or cremation in the community.", + "Includes direct burial or cremation of human remains of COVID-19 cases, and supervision of safe burial or cremation in the community.", # noqa: E501 ], ["Support COVID-19 Vaccination", "Includes activities such as service delivery, campaigns, or distribution."], [ "Community engagement and accountability (CEA), including community feedback mechanisms", - "Includes systems and approaches to collect community feedback which could mean recorded /tracked suggestions, comments, complaints, concerns, perceptions, praise, question collected through feedback systems and/or through community perception surveys.", + "Includes systems and approaches to collect community feedback which could mean recorded /tracked suggestions, comments, complaints, concerns, perceptions, praise, question collected through feedback systems and/or through community perception surveys.", # noqa: E501 ], [ "Livelihoods, cash support & food aid", - "Includes immediate food assistance or measures to protect households’ livelihoods by provisioning for lost sources of income to meet their basic needs and avoid further assets depletion. This can include all types of cash assistance to address basic needs. Also, activities for skills development such as entrepreneurship, marketing and coaching.", + "Includes immediate food assistance or measures to protect households’ livelihoods by provisioning for lost sources of income to meet their basic needs and avoid further assets depletion. This can include all types of cash assistance to address basic needs. Also, activities for skills development such as entrepreneurship, marketing and coaching.", # noqa: E501 ], [ "Social care and cohesion, and support to vulnerable groups", - "Includes analysis of the specific needs of marginalized groups in the needs assessment, and/or following the PGI minimum standards and/or using equivalent guide on assessing and meeting the needs of marginalized groups.", + "Includes analysis of the specific needs of marginalized groups in the needs assessment, and/or following the PGI minimum standards and/or using equivalent guide on assessing and meeting the needs of marginalized groups.", # noqa: E501 ], [ "Shelter and urban settlements", - "Includes for example, household items to improve shelter conditions (clothing, bed linen, mattress, blanket, etc.), emergency shelter (tents or tool kits), accommodation in collective facilities (camps, collective centers or quarantine facilities), cash and voucher assistance to cover payment of accommodation (rent or loans), utilities and other related cost directly related with the accommodation.", + "Includes for example, household items to improve shelter conditions (clothing, bed linen, mattress, blanket, etc.), emergency shelter (tents or tool kits), accommodation in collective facilities (camps, collective centers or quarantine facilities), cash and voucher assistance to cover payment of accommodation (rent or loans), utilities and other related cost directly related with the accommodation.", # noqa: E501 ], ] diff --git a/api/management/commands/generate-admin2-shp.py b/api/management/commands/generate-admin2-shp.py index 402c01ced..27993f9b0 100644 --- a/api/management/commands/generate-admin2-shp.py +++ b/api/management/commands/generate-admin2-shp.py @@ -4,7 +4,7 @@ class Command(BaseCommand): - help = "Converting Shapefiles from different sources to custom file for go-api. To run, python manage.py generate-admin2-shp input.shp output.shp --source=fews" + help = "Converting Shapefiles from different sources to custom file for go-api. To run, python manage.py generate-admin2-shp input.shp output.shp --source=fews" # noqa: E501 missing_args_message = "Filename is missing. A shapefile with valid admin polygons is required." @@ -18,7 +18,7 @@ def handle(self, *args, **options): output_filename = options["filename"][1] try: gdf = gpd.read_file(input_filename) - except: + except Exception: raise CommandError("Could not open file") if options["source"] == "fews": gdf.rename(columns={"shapeName": "name"}, inplace=True) diff --git a/api/management/commands/import-admin0-data.py b/api/management/commands/import-admin0-data.py index d025a3e6b..76b8f6da9 100644 --- a/api/management/commands/import-admin0-data.py +++ b/api/management/commands/import-admin0-data.py @@ -1,11 +1,7 @@ import csv -import json -import os -import sys from django.contrib.gis.gdal import DataSource from django.contrib.gis.geos import GEOSGeometry, MultiPolygon, Point -from django.contrib.gis.utils import LayerMapping from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist from django.core.management.base import BaseCommand, CommandError from django.db import transaction @@ -14,7 +10,7 @@ class Command(BaseCommand): - help = "import a shapefile of adminstrative boundary level 0 data to the GO database. To run, python manage.py import-admin0-data input.shp" + help = "import a shapefile of adminstrative boundary level 0 data to the GO database. To run, python manage.py import-admin0-data input.shp" # noqa: E501 missing_args_message = "Filename is missing. A shapefile with valid admin polygons is required." region_enum = {"Africa": 0, "Americas": 1, "Asia-Pacific": 2, "Europe": 3, "Middle East and North Africa": 4} @@ -25,11 +21,11 @@ def add_arguments(self, parser): parser.add_argument( "--update-bbox", action="store_true", - help="Update the bbox of the country geometry. Used if you want to overwrite changes that are made by users via the Django Admin", + help="Update the bbox of the country geometry. Used if you want to overwrite changes that are made by users via the Django Admin", # noqa: E501 ) parser.add_argument( "--update-centroid", - help="Update the centroid of the country using a CSV file provided. If the CSV does not have the country iso, then we use the geometric centroid", + help="Update the centroid of the country using a CSV file provided. If the CSV does not have the country iso, then we use the geometric centroid", # noqa: E501 ) parser.add_argument("--import-missing", help="Import missing countries for iso codes mentioned in this file.") parser.add_argument("--update-iso3", help="Import missing iso3 codes from this file.") @@ -73,7 +69,7 @@ def handle(self, *args, **options): try: data = DataSource(filename) - except: + except Exception: raise CommandError("Could not open file") fields = data[0].fields diff --git a/api/management/commands/import-admin1-data.py b/api/management/commands/import-admin1-data.py index 74fa51ef4..4d8f0376e 100644 --- a/api/management/commands/import-admin1-data.py +++ b/api/management/commands/import-admin1-data.py @@ -1,10 +1,7 @@ import csv -import os -import sys from django.contrib.gis.gdal import DataSource from django.contrib.gis.geos import GEOSGeometry, MultiPolygon -from django.contrib.gis.utils import LayerMapping from django.core.exceptions import ObjectDoesNotExist from django.core.management.base import BaseCommand, CommandError from django.db import transaction @@ -13,7 +10,7 @@ class Command(BaseCommand): - help = "import a shapefile of administrative boundary level 1 data to the GO database. To run, python manage.py import-admin1-data input.shp" + help = "import a shapefile of administrative boundary level 1 data to the GO database. To run, python manage.py import-admin1-data input.shp" # noqa: E501 missing_args_message = "Filename is missing. A shapefile with valid admin polygons is required." @@ -23,12 +20,12 @@ def add_arguments(self, parser): parser.add_argument( "--update-bbox", action="store_true", - help="Update the bbox of the district geometry. Used if you want to overwrite changes that are made by users via the Django Admin", + help="Update the bbox of the district geometry. Used if you want to overwrite changes that are made by users via the Django Admin", # noqa: E501 ) parser.add_argument( "--update-centroid", action="store_true", - help="Update the centroid of the district geometry. Used if you want to overwrite changes that are made by users via the Django Admin", + help="Update the centroid of the district geometry. Used if you want to overwrite changes that are made by users via the Django Admin", # noqa: E501 ) parser.add_argument("--import-missing", help="Import missing districts for codes mentioned in this file.") parser.add_argument("--import-all", action="store_true", help="Import all districts in the shapefile, if possible.") @@ -55,15 +52,13 @@ def handle(self, *args, **options): try: data = DataSource(filename) - except: + except Exception: raise CommandError("Could not open file") # loop through each feature in the shapefile for feature in data[0]: code = feature.get("ADMIN01COD") name = feature.get("ADMIN01NAM") - country_iso2 = feature.get("ISO2") - country_name = feature.get("COUNTRY") geom_wkt = feature.geom.wkt geom = GEOSGeometry(geom_wkt, srid=4326) @@ -91,7 +86,7 @@ def handle(self, *args, **options): # if there are more than one district with the same code, filter also using name if len(districts) > 1: district = District.objects.filter(code=code, name__icontains=name) - # if we get a match, update geometry. otherwise consider this as missing because it's possible the names aren't matching. + # if we get a match, update geometry. otherwise consider this as missing because it's possible the names aren't matching. # noqa: E501 if len(district): # update geom, centroid and bbox d = district[0] diff --git a/api/management/commands/import-admin2-data.py b/api/management/commands/import-admin2-data.py index c2f1d8b22..b17e9f838 100644 --- a/api/management/commands/import-admin2-data.py +++ b/api/management/commands/import-admin2-data.py @@ -10,7 +10,7 @@ class Command(BaseCommand): - help = "import a shapefile of administrative boundary level 2 data to the GO database. To run, python manage.py import-admin2-data input.shp" + help = "import a shapefile of administrative boundary level 2 data to the GO database. To run, python manage.py import-admin2-data input.shp" # noqa: E501 missing_args_message = "Filename is missing. A shapefile with valid admin polygons is required." @@ -20,12 +20,12 @@ def add_arguments(self, parser): parser.add_argument( "--update-bbox", action="store_true", - help="Update the bbox of the admin2 geometry. Used if you want to overwrite changes that are made by users via the Django Admin", + help="Update the bbox of the admin2 geometry. Used if you want to overwrite changes that are made by users via the Django Admin", # noqa: E501 ) parser.add_argument( "--update-centroid", action="store_true", - help="Update the centroid of the admin2 geometry. Used if you want to overwrite changes that are made by users via the Django Admin", + help="Update the centroid of the admin2 geometry. Used if you want to overwrite changes that are made by users via the Django Admin", # noqa: E501 ) parser.add_argument("--import-missing", help="Import missing admin2 boundaries for codes mentioned in this file.") parser.add_argument( @@ -54,7 +54,7 @@ def handle(self, *args, **options): try: data = DataSource(filename) - except: + except Exception: raise CommandError("Could not open file") # loop through each feature in the shapefile @@ -88,7 +88,7 @@ def handle(self, *args, **options): # if there are more than one admin2 with the same code, filter also using name if len(admin2_objects) > 1: admins2_names = Admin2.objects.filter(code=code, name__icontains=name) - # if we get a match, update geometry. otherwise consider this as missing because it's possible the names aren't matching. + # if we get a match, update geometry. otherwise consider this as missing because it's possible the names aren't matching. # noqa: E501 if len(admins2_names): # update geom, centroid and bbox self.update_admin2_columns(options, admins2_names[0], geom, centroid, bbox) @@ -135,7 +135,7 @@ def add_admin2(self, options, import_missing, feature, geom, centroid, bbox): admin2.save() if options["update_geom"]: self.update_geom(admin2, geom) - except IntegrityError as e: + except IntegrityError: print(f"Duplicate object {admin2.name}") pass diff --git a/api/management/commands/import-fdrs.py b/api/management/commands/import-fdrs.py index e1ee3a6e8..c0425ec5c 100644 --- a/api/management/commands/import-fdrs.py +++ b/api/management/commands/import-fdrs.py @@ -1,7 +1,7 @@ import csv from django.core.exceptions import ObjectDoesNotExist -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from django.db import transaction from api.models import Country diff --git a/api/management/commands/import-gec-code.py b/api/management/commands/import-gec-code.py index bcd5fb23f..2048dece0 100644 --- a/api/management/commands/import-gec-code.py +++ b/api/management/commands/import-gec-code.py @@ -1,7 +1,7 @@ import csv from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from django.db import transaction from api.models import Country, GECCode diff --git a/api/management/commands/import_admin2s_csv.py b/api/management/commands/import_admin2s_csv.py index f76aff914..550fd97da 100644 --- a/api/management/commands/import_admin2s_csv.py +++ b/api/management/commands/import_admin2s_csv.py @@ -26,7 +26,7 @@ def handle(self, *args, **options): adm2_code = row["GOadm2code"] try: adm2 = Admin2.objects.get(code=adm2_code) - except: + except Exception: adm2 = Admin2() adm2.code = adm2_code adm2.admin1 = district diff --git a/api/management/commands/import_countries_csv.py b/api/management/commands/import_countries_csv.py index eebf7aa6c..734484a77 100644 --- a/api/management/commands/import_countries_csv.py +++ b/api/management/commands/import_countries_csv.py @@ -1,7 +1,7 @@ import csv import os -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from django.db import transaction from api.models import Country @@ -71,7 +71,7 @@ def handle(self, *args, **options): all_ids.append(id) try: country = Country.objects.get(pk=id) - except: + except Exception: country = Country() for key in row.keys(): # print(key) diff --git a/api/management/commands/import_districts_csv.py b/api/management/commands/import_districts_csv.py index 57b77b346..37db1935c 100644 --- a/api/management/commands/import_districts_csv.py +++ b/api/management/commands/import_districts_csv.py @@ -1,9 +1,9 @@ import csv -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from django.db import transaction -from api.models import Country, District +from api.models import District def get_bool(value): @@ -45,7 +45,7 @@ def handle(self, *args, **options): all_ids.append(id) try: district = District.objects.get(pk=id) - except: + except Exception: district = District() for key in row.keys(): # print(key) @@ -60,7 +60,7 @@ def handle(self, *args, **options): try: district.save() print("SUCCESS", district.name) - except: + except Exception: print("FAILED", district.name) failures += 1 print("done importing districts, with %d failures" % failures) diff --git a/api/management/commands/import_ns_organizational_capacity.py b/api/management/commands/import_ns_organizational_capacity.py index d28df12c2..a5f01395b 100644 --- a/api/management/commands/import_ns_organizational_capacity.py +++ b/api/management/commands/import_ns_organizational_capacity.py @@ -1,7 +1,5 @@ -import csv - import pandas as pd -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from api.models import Country, CountryOrganizationalCapacity diff --git a/api/management/commands/index_and_notify.py b/api/management/commands/index_and_notify.py index ee4af2583..246d9199c 100644 --- a/api/management/commands/index_and_notify.py +++ b/api/management/commands/index_and_notify.py @@ -25,7 +25,7 @@ from deployments.models import ERU, Personnel, PersonnelDeployment from main.sentry import SentryMonitor from notifications.hello import get_hello -from notifications.models import RecordType, Subscription, SubscriptionType, SurgeAlert +from notifications.models import RecordType, Subscription, SubscriptionType from notifications.notification import send_notification from utils.elasticsearch import construct_es_data @@ -34,7 +34,7 @@ time_1_week = timedelta(days=7) # for digest mode digest_time = int(10314) # weekday - hour - min for digest timing (5 minutes once a week, Monday dawn) daily_retro = int(654) # hour - min for daily retropective email timing (5 minutes a day) | Should not contain a leading 0! -max_length = 860 # after this length (at the first space) we cut the sent content. In case of HTML tags we remove tags (to avoid chunked tags) +max_length = 860 # after this length (at the first space) we cut the sent content. In case of HTML tags we remove tags (to avoid chunked tags) # noqa: E501 events_sent_to = {} # to document sent events before re-sending them via specific following template_types = { 99: "design/generic_notification.html", @@ -901,13 +901,15 @@ def check_ingest_issues(self, having_ingest_issue): send_notification( "API monitor – ingest issues!", ["im@ifrc.org"], # Could be an ENV var - "Ingest issue(s) occured, one of them is " - + ingestor_name - + ", via CronJob log record id: https://" - + settings.BASE_URL - + "/admin/api/cronjob/" - + str(ingest_issue_id) - + ". Please fix it ASAP.", + ( + "Ingest issue(s) occured, one of them is " + + ingestor_name + + ", via CronJob log record id: https://" + + settings.BASE_URL + + "/admin/api/cronjob/" + + str(ingest_issue_id) + + ". Please fix it ASAP." + ), "Ingestion error", ) logger.info( @@ -953,8 +955,8 @@ def handle(self, *args, **options): diff=ExpressionWrapper(F("updated_at") - F("created_at"), output_field=DurationField()) ).filter(condU & cond2 & Q(diff__gt=timedelta(minutes=1))) - new_surgealerts = SurgeAlert.objects.filter(cond1) - new_pers_deployments = PersonnelDeployment.objects.filter(cond1) + # new_surgealerts = SurgeAlert.objects.filter(cond1) + # new_pers_deployments = PersonnelDeployment.objects.filter(cond1) # Merge Weekly Digest into one mail instead of separate ones if self.is_digest_mode(): @@ -966,8 +968,8 @@ def handle(self, *args, **options): # self.notify(updated_appeals, RecordType.APPEAL, SubscriptionType.EDIT) self.notify(new_events, RecordType.EVENT, SubscriptionType.NEW) # self.notify(updated_events, RecordType.EVENT, SubscriptionType.EDIT) - # temporary switched off while historical data is uploaded: self.notify(new_surgealerts, RecordType.SURGE_ALERT, SubscriptionType.NEW) - # temporary switched off while historical data is uploaded: self.notify(new_pers_deployments, RecordType.SURGE_DEPLOYMENT_MESSAGES, SubscriptionType.NEW) + # temporary switched off while historical data is uploaded: self.notify(new_surgealerts, RecordType.SURGE_ALERT, SubscriptionType.NEW) # noqa: E501 + # temporary switched off while historical data is uploaded: self.notify(new_pers_deployments, RecordType.SURGE_DEPLOYMENT_MESSAGES, SubscriptionType.NEW) # noqa: E501 # Followed Events if self.is_daily_checkup_time(): diff --git a/api/management/commands/index_elasticsearch.py b/api/management/commands/index_elasticsearch.py index 061e70d8b..4bdfb8b70 100644 --- a/api/management/commands/index_elasticsearch.py +++ b/api/management/commands/index_elasticsearch.py @@ -1,5 +1,4 @@ from django.core.management.base import BaseCommand -from elasticsearch import Elasticsearch from elasticsearch.client import IndicesClient from elasticsearch.helpers import bulk diff --git a/api/management/commands/ingest_admin1_population.py b/api/management/commands/ingest_admin1_population.py index 7e1e53015..5ce6df000 100644 --- a/api/management/commands/ingest_admin1_population.py +++ b/api/management/commands/ingest_admin1_population.py @@ -2,7 +2,7 @@ from django.core.management.base import BaseCommand from api.logger import logger -from api.models import Country, District +from api.models import District class Command(BaseCommand): diff --git a/api/management/commands/ingest_appeal_docs.py b/api/management/commands/ingest_appeal_docs.py index 1fcb97c37..abced2a4e 100644 --- a/api/management/commands/ingest_appeal_docs.py +++ b/api/management/commands/ingest_appeal_docs.py @@ -1,7 +1,6 @@ from collections import defaultdict from datetime import datetime, timezone -import brotli from bs4 import BeautifulSoup from dateutil.relativedelta import relativedelta from django.core.exceptions import ObjectDoesNotExist @@ -69,7 +68,7 @@ def handle(self, *args, **options): else: body = { "name": "ingest_appeal_docs", - "message": f"Error ingesting appeals_docs on url: {baseurl}, error_code: {smoke_response.status} – {smoke_response.reason}", + "message": f"Error ingesting appeals_docs on url: {baseurl}, error_code: {smoke_response.status} – {smoke_response.reason}", # noqa: E501 "status": CronJobStatus.ERRONEOUS, } CronJob.sync_cron(body) diff --git a/api/management/commands/ingest_appeals.py b/api/management/commands/ingest_appeals.py index aeea03bb9..c032f8e75 100644 --- a/api/management/commands/ingest_appeals.py +++ b/api/management/commands/ingest_appeals.py @@ -231,7 +231,7 @@ def parse_appeal_record(self, r, **options): # if there is more than one detail, the start date should be the *earliest # end date should be the *latest details = sorted(r["Details"], key=lambda x: self.parse_date(x["APD_startDate"])) - detail0 = details[0] # first + # detail0 = details[0] # first detail1 = details[-1] # last start_date = self.parse_date(r["APP_startDate"]) # not self.parse_date(detail0['APD_startDate']) end_date = self.parse_date(r["APP_endDate"]) # not self.parse_date(detail1['APD_endDate']) diff --git a/api/management/commands/ingest_deployments.py b/api/management/commands/ingest_deployments.py index 44dc07e1c..f74efd51a 100644 --- a/api/management/commands/ingest_deployments.py +++ b/api/management/commands/ingest_deployments.py @@ -1,12 +1,13 @@ from datetime import datetime, timezone import requests -from django.core.exceptions import ObjectDoesNotExist + +# from django.core.exceptions import ObjectDoesNotExist from django.core.management.base import BaseCommand from api.logger import logger -from api.models import Appeal, AppealDocument -from deployments.models import ERU, DeployedPerson, Personnel, PersonnelDeployment + +# from api.models import Appeal, AppealDocument class Command(BaseCommand): @@ -20,7 +21,7 @@ def parse_date(self, date_string): def handle(self, *args, **options): logger.info("Starting Deployment ingest") - # url = 'https://proxy.hxlstandard.org/data.json?url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1CBvledFYc_uwlvHTvJE0SYS7_mPGU2L-zhrqbB4KNIA%2Fedit%23gid%3D0&header-row=1' # not enough. + # url = 'https://proxy.hxlstandard.org/data.json?url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1CBvledFYc_uwlvHTvJE0SYS7_mPGU2L-zhrqbB4KNIA%2Fedit%23gid%3D0&header-row=1' # not enough. # noqa: E501 url = ( "https://proxy.hxlstandard.org/data.json?tagger-match-all=on&" + "tagger-01-header=year&" @@ -73,7 +74,7 @@ def handle(self, *args, **options): + "tagger-24-tag=%23c6&" + "tagger-25-header=comments&" + "tagger-25-tag=%23c7&" - + "url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1CBvledFYc_uwlvHTvJE0SYS7_mPGU2L-zhrqbB4KNIA%2Fedit%23gid%3D0&" + + "url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1CBvledFYc_uwlvHTvJE0SYS7_mPGU2L-zhrqbB4KNIA%2Fedit%23gid%3D0&" # noqa: E501 + "header-row=1" ) @@ -81,16 +82,16 @@ def handle(self, *args, **options): if response.status_code != 200: logger.error("Error querying Deployment HXL API") raise Exception("Error querying Deployment HXL API") - records = response.json() + # records = response.json() # some logging variables not_found = [] existing = [] created = [] - columns = [a.replace("*", "").replace(" ", "") for a in records[0]] - # ['Year', 'AppealCode', 'Region', 'Country', 'Location', 'Disastertype', 'Disastername', 'Name', 'DeployingNS/IFRCOffice', 'Gender', 'Language', 'Position', 'Type', 'SupportedbyNS', 'Availability', 'Expstartdate', 'expduration', 'Alert', 'Deploymentmessage', 'Startofmission', 'Endofmission', 'DeploymentDuration', 'Deployed', 'Rotation', 'Comments'] - # 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 + # columns = [a.replace("*", "").replace(" ", "") for a in records[0]] + # ['Year', 'AppealCode', 'Region', 'Country', 'Location', 'Disastertype', 'Disastername', 'Name', 'DeployingNS/IFRCOffice', 'Gender', 'Language', 'Position', 'Type', 'SupportedbyNS', 'Availability', 'Expstartdate', 'expduration', 'Alert', 'Deploymentmessage', 'Startofmission', 'Endofmission', 'DeploymentDuration', 'Deployed', 'Rotation', 'Comments'] # noqa: E501 + # 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 # noqa: E501 # if empty name -> Alert, otherwise -> Deployment diff --git a/api/management/commands/ingest_disaster_law.py b/api/management/commands/ingest_disaster_law.py index c45fbb98c..313f7e72f 100644 --- a/api/management/commands/ingest_disaster_law.py +++ b/api/management/commands/ingest_disaster_law.py @@ -1,15 +1,9 @@ -import datetime as dt -import json - -import pandas as pd import requests -import xmltodict from bs4 import BeautifulSoup -from dateutil.parser import parse from django.core.management.base import BaseCommand from api.logger import logger -from api.models import Country, CountryICRCPresence, CronJob, CronJobStatus +from api.models import Country, CronJob, CronJobStatus class Command(BaseCommand): @@ -25,7 +19,7 @@ def handle(self, *args, **kwargs): soup = BeautifulSoup(response.content, "html.parser") try: country_options = soup.find("select", {"data-drupal-selector": "edit-country"}).find_all("option") - except Exception as err: + except Exception: continue # Loop through countries and get information @@ -44,7 +38,7 @@ def handle(self, *args, **kwargs): country_soup = BeautifulSoup(country_page.content, "html.parser") description = country_soup.find("div", {"class": "field--name-field-paragraphs"}).find_all("p") description = "\n".join([para.text for para in description]) - except Exception as err: + except Exception: pass # Add all information to the country list country_list.append({"Country": country_name, "ID": country_id, "URL": country_url, "Description": description}) diff --git a/api/management/commands/ingest_gdacs.py b/api/management/commands/ingest_gdacs.py index c2a359146..c356d1cfc 100644 --- a/api/management/commands/ingest_gdacs.py +++ b/api/management/commands/ingest_gdacs.py @@ -1,5 +1,3 @@ -import datetime as dt - import requests import xmltodict from dateutil.parser import parse diff --git a/api/management/commands/ingest_icrc.py b/api/management/commands/ingest_icrc.py index 96561ad1d..f6e65d90c 100644 --- a/api/management/commands/ingest_icrc.py +++ b/api/management/commands/ingest_icrc.py @@ -1,11 +1,5 @@ -import datetime as dt -import json - -import pandas as pd import requests -import xmltodict from bs4 import BeautifulSoup -from dateutil.parser import parse from django.core.management.base import BaseCommand from api.logger import logger @@ -19,7 +13,7 @@ def handle(self, *args, **kwargs): logger.info("Strating ICRC data ingest") response = requests.get(url="https://www.icrc.org/en/where-we-work", headers={"User-Agent": ""}) if response.status_code != 200: - text_to_log = "Error querying ICRC feed at " + url + text_to_log = "Error querying ICRC feed at https://www.icrc.org/en/where-we-work" logger.error(text_to_log) logger.error(response.content) body = { @@ -51,7 +45,7 @@ def handle(self, *args, **kwargs): country_page.raise_for_status() country_soup = BeautifulSoup(country_page.content, "html.parser") description = country_soup.find("div", {"class": "block-introduction"}).find_all()[2].text.strip() - except Exception as err: + except Exception: pass # Append all the information to the list country_list.append( diff --git a/api/management/commands/ingest_ns_capacity.py b/api/management/commands/ingest_ns_capacity.py index aaaca8379..69b7d80d4 100644 --- a/api/management/commands/ingest_ns_capacity.py +++ b/api/management/commands/ingest_ns_capacity.py @@ -1,10 +1,6 @@ -import json - import requests -import xmltodict from django.conf import settings from django.core.management.base import BaseCommand -from requests.auth import HTTPBasicAuth from api.logger import logger from api.models import Country, CountryCapacityStrengthening, CronJob, CronJobStatus @@ -20,7 +16,7 @@ def handle(self, *args, **kwargs): OCAC_DATA_API = f"https://data-api.ifrc.org/api/ocacpublic?apiKey={settings.FDRS_APIKEY}" resp_ocac = requests.get(OCAC_DATA_API) if resp_ocac.status_code != 200: - text_to_log = "Error querying OCAC at " + url + text_to_log = "Error querying OCAC at " + OCAC_DATA_API logger.error(text_to_log) logger.error(resp_ocac.content) body = { @@ -32,7 +28,6 @@ def handle(self, *args, **kwargs): raise Exception("Error querying OCAC_DATA_API") resp_ocac_data = resp_ocac.json() - final_output = [] ocaa_count = 0 for item in resp_ocac_data: ocaa_count += 1 diff --git a/api/management/commands/ingest_ns_contact.py b/api/management/commands/ingest_ns_contact.py index 8b22668b3..1d01dd759 100644 --- a/api/management/commands/ingest_ns_contact.py +++ b/api/management/commands/ingest_ns_contact.py @@ -38,13 +38,13 @@ def handle(self, *args, **kwargs): added = 0 dict_data = xmltodict.parse(response.content) for data in dict_data["ArrayOfNationalSocietiesMain"]["NationalSocietiesMain"]: - address_1 = data["ADD_address1"] if type(data["ADD_address1"]) == str else None - address_2 = data["ADD_address2"] if type(data["ADD_address2"]) == str else None - city_code = data["ADD_city_code"] if type(data["ADD_city_code"]) == str else None - phone = data["ADD_phone"] if type(data["ADD_phone"]) == str else None - website = data["ADD_webSite"] if type(data["ADD_webSite"]) == str else None - emails = data["ADD_email"] if type(data["ADD_email"]) == str and data["ADD_email"] != None else None - founded_date = data["ADD_orgCreation"] if type(data["ADD_orgCreation"]) == str else None + address_1 = data["ADD_address1"] if isinstance(data["ADD_address1"], str) else None + address_2 = data["ADD_address2"] if isinstance(data["ADD_address2"], str) else None + city_code = data["ADD_city_code"] if isinstance(data["ADD_city_code"], str) else None + phone = data["ADD_phone"] if isinstance(data["ADD_phone"], str) else None + website = data["ADD_webSite"] if isinstance(data["ADD_webSite"], str) else None + emails = data["ADD_email"] if (isinstance(data["ADD_email"], str) and data["ADD_email"] is not None) else None + founded_date = data["ADD_orgCreation"] if isinstance(data["ADD_orgCreation"], str) else None iso = data["ADD_country_code"] # # get the country and try to update the data for those country country = Country.objects.filter(iso=iso.upper()).first() diff --git a/api/management/commands/ingest_ns_directory.py b/api/management/commands/ingest_ns_directory.py index c5d404f72..f42421a11 100644 --- a/api/management/commands/ingest_ns_directory.py +++ b/api/management/commands/ingest_ns_directory.py @@ -1,5 +1,3 @@ -import json - import requests import xmltodict from django.conf import settings @@ -37,7 +35,7 @@ def handle(self, *args, **kwargs): added = 0 dict_data = xmltodict.parse(response.content) for data in dict_data["ArrayOfNationalSocietiesContacts"]["NationalSocietiesContacts"]: - country_name = data["CON_country"] if type(data["CON_country"]) == str else None + country_name = data["CON_country"] if isinstance(data["CON_country"], str) else None if country_name is not None: country = Country.objects.filter(name__icontains=country_name).first() if country: @@ -45,11 +43,13 @@ def handle(self, *args, **kwargs): data = { "first_name": ( data["CON_firstName"] - if type(data["CON_firstName"]) == str and data["CON_firstName"] != None + if isinstance(data["CON_firstName"], str) and data["CON_firstName"] is not None else None ), "last_name": ( - data["CON_lastName"] if type(data["CON_lastName"]) == str and data["CON_lastName"] != None else None + data["CON_lastName"] + if isinstance(data["CON_lastName"], str) and data["CON_lastName"] is not None + else None ), "position": data["CON_title"], "country": country, diff --git a/api/management/commands/ingest_ns_document.py b/api/management/commands/ingest_ns_document.py index 7a525cd8a..0864b02b7 100644 --- a/api/management/commands/ingest_ns_document.py +++ b/api/management/commands/ingest_ns_document.py @@ -7,13 +7,7 @@ from django.core.management.base import BaseCommand from api.logger import logger -from api.models import ( - Country, - CountryDirectory, - CountryKeyDocument, - CronJob, - CronJobStatus, -) +from api.models import Country, CountryKeyDocument, CronJob, CronJobStatus class Command(BaseCommand): diff --git a/api/management/commands/ingest_ns_initiatives.py b/api/management/commands/ingest_ns_initiatives.py index d86212a75..d759702e5 100644 --- a/api/management/commands/ingest_ns_initiatives.py +++ b/api/management/commands/ingest_ns_initiatives.py @@ -1,5 +1,3 @@ -import json - import numpy as np import pandas as pd import requests diff --git a/api/management/commands/ingest_who.py b/api/management/commands/ingest_who.py index c90e2a581..bdc0704c8 100644 --- a/api/management/commands/ingest_who.py +++ b/api/management/commands/ingest_who.py @@ -1,13 +1,10 @@ -import datetime as dt - import requests import xmltodict from dateutil.parser import parse from django.core.management.base import BaseCommand -from api.event_sources import SOURCES from api.logger import logger -from api.models import Country, CronJob, CronJobStatus, Event, Region +from api.models import Country, CronJob, CronJobStatus, Event class Command(BaseCommand): @@ -18,7 +15,6 @@ def handle(self, *args, **options): logger.info("Querying WHO RSS feed for new emergency data") # get latest - nspace = "{https://www.who.int}" ur2 = [] ur2.append("https://www.who.int/feeds/entity/csr/don/en/rss.xml") ur2.append("https://www.who.int/feeds/entity/hac/en/rss.xml") @@ -185,5 +181,5 @@ def handle(self, *args, **options): # delete from api_event_regions where event_id in (select id from api_event where auto_generated_source like 'www.who.int%'); # delete from api_event where auto_generated_source like 'www.who.int%'; # -- -# select * from api_event_countries a join api_country b on (a.country_id=b.id) where event_id in (select id from api_event where auto_generated_source like 'www.who.int%'); -# select name from api_event a left join api_event_countries b on (a.id=b.event_id) where b.event_id is null and auto_generated_source like 'www.who.int%'; -- what country is not found +# select * from api_event_countries a join api_country b on (a.country_id=b.id) where event_id in (select id from api_event where auto_generated_source like 'www.who.int%'); # noqa: E501 +# select name from api_event a left join api_event_countries b on (a.id=b.event_id) where b.event_id is null and auto_generated_source like 'www.who.int%'; -- what country is not found # noqa: E501 diff --git a/api/management/commands/revoke_staff_status.py b/api/management/commands/revoke_staff_status.py index b2ebb0533..5c965ca4d 100644 --- a/api/management/commands/revoke_staff_status.py +++ b/api/management/commands/revoke_staff_status.py @@ -1,4 +1,3 @@ -import requests from django.contrib.auth.models import Group, User from django.core.management.base import BaseCommand from sentry_sdk.crons import monitor diff --git a/api/management/commands/scrape_pdfs.py b/api/management/commands/scrape_pdfs.py index 21da3b850..8696e646e 100644 --- a/api/management/commands/scrape_pdfs.py +++ b/api/management/commands/scrape_pdfs.py @@ -530,7 +530,7 @@ def clean_data_and_save(self, scraped_data): raw_disaster_risk_reduction_people_reached=data["sector"] .get(_s.disaster_Risk_reduction, {}) .get(_sfd.people_reached), - # raw_disaster_risk_reduction_people_targeted=data['sector'].get(_s.disaster_Risk_reduction, {}).get(_sfd.people_targeted), + # raw_disaster_risk_reduction_people_targeted=data['sector'].get(_s.disaster_Risk_reduction, {}).get(_sfd.people_targeted), # noqa: E501 raw_disaster_risk_reduction_requirements=data["sector"] .get(_s.disaster_Risk_reduction, {}) .get(_sfd.requirements), @@ -546,7 +546,7 @@ def clean_data_and_save(self, scraped_data): raw_livelihoods_and_basic_needs_people_reached=data["sector"] .get(_s.livelihoods_and_basic_needs, {}) .get(_sfd.people_reached), - # raw_livelihoods_and_basic_needs_people_targeted=data['sector'].get(_s.livelihoods_and_basic_needs, {}).get(_sfd.people_targeted), + # raw_livelihoods_and_basic_needs_people_targeted=data['sector'].get(_s.livelihoods_and_basic_needs, {}).get(_sfd.people_targeted), # noqa: E501 raw_livelihoods_and_basic_needs_requirements=data["sector"] .get(_s.livelihoods_and_basic_needs, {}) .get(_sfd.requirements), @@ -564,7 +564,7 @@ def clean_data_and_save(self, scraped_data): raw_protection_gender_and_inclusion_people_reached=data["sector"] .get(_s.protection_gender_inclusion, {}) .get(_sfd.people_reached), - # raw_protection_gender_and_inclusion_people_targeted=data['sector'].get(_s.protection_gender_inclusion, {}).get(_sfd.people_targeted), + # raw_protection_gender_and_inclusion_people_targeted=data['sector'].get(_s.protection_gender_inclusion, {}).get(_sfd.people_targeted), # noqa: E501 raw_protection_gender_and_inclusion_requirements=data["sector"] .get(_s.protection_gender_inclusion, {}) .get(_sfd.requirements), @@ -578,7 +578,7 @@ def clean_data_and_save(self, scraped_data): raw_water_sanitation_and_hygiene_people_reached=data["sector"] .get(_s.Water_sanitation_hygiene, {}) .get(_sfd.people_reached), - # raw_water_sanitation_and_hygiene_people_targeted=data['sector'].get(_s.Water_sanitation_hygiene, {}).get(_sfd.people_targeted), + # raw_water_sanitation_and_hygiene_people_targeted=data['sector'].get(_s.Water_sanitation_hygiene, {}).get(_sfd.people_targeted), # noqa: E501 raw_water_sanitation_and_hygiene_requirements=data["sector"] .get(_s.Water_sanitation_hygiene, {}) .get(_sfd.requirements), @@ -603,7 +603,7 @@ def clean_data_and_save(self, scraped_data): disaster_risk_reduction_people_reached=cleaners.clean_number( data["sector"].get(_s.disaster_Risk_reduction, {}).get(_sfd.people_reached) ), - # disaster_risk_reduction_people_targeted=cleaners.clean_number(data['sector'].get(_s.disaster_Risk_reduction, {}).get(_sfd.people_targeted)), + # disaster_risk_reduction_people_targeted=cleaners.clean_number(data['sector'].get(_s.disaster_Risk_reduction, {}).get(_sfd.people_targeted)), # noqa: E501 disaster_risk_reduction_requirements=cleaners.clean_number( data["sector"].get(_s.disaster_Risk_reduction, {}).get(_sfd.requirements) ), @@ -621,14 +621,14 @@ def clean_data_and_save(self, scraped_data): livelihoods_and_basic_needs_people_reached=cleaners.clean_number( data["sector"].get(_s.livelihoods_and_basic_needs, {}).get(_sfd.people_reached) ), - # livelihoods_and_basic_needs_people_targeted=cleaners.clean_number(data['sector'].get(_s.livelihoods_and_basic_needs, {}).get(_sfd.people_targeted)), + # livelihoods_and_basic_needs_people_targeted=cleaners.clean_number(data['sector'].get(_s.livelihoods_and_basic_needs, {}).get(_sfd.people_targeted)), # noqa: E501 livelihoods_and_basic_needs_requirements=cleaners.clean_number( data["sector"].get(_s.livelihoods_and_basic_needs, {}).get(_sfd.requirements) ), migration_female=cleaners.clean_number(data["sector"].get(_s.migration, {}).get(_sfd.female)), migration_male=cleaners.clean_number(data["sector"].get(_s.migration, {}).get(_sfd.male)), migration_people_reached=cleaners.clean_number(data["sector"].get(_s.migration, {}).get(_sfd.people_reached)), - # migration_people_targeted=cleaners.clean_number(data['sector'].get(_s.migration, {}).get(_sfd.people_targeted)), + # migration_people_targeted=cleaners.clean_number(data['sector'].get(_s.migration, {}).get(_sfd.people_targeted)), # noqa: E501 migration_requirements=cleaners.clean_number(data["sector"].get(_s.migration, {}).get(_sfd.requirements)), protection_gender_and_inclusion_female=cleaners.clean_number( data["sector"].get(_s.protection_gender_inclusion, {}).get(_sfd.female) @@ -639,7 +639,7 @@ def clean_data_and_save(self, scraped_data): protection_gender_and_inclusion_people_reached=cleaners.clean_number( data["sector"].get(_s.protection_gender_inclusion, {}).get(_sfd.people_reached) ), - # protection_gender_and_inclusion_people_targeted=cleaners.clean_number(data['sector'].get(_s.protection_gender_inclusion, {}).get(_sfd.people_targeted)), + # protection_gender_and_inclusion_people_targeted=cleaners.clean_number(data['sector'].get(_s.protection_gender_inclusion, {}).get(_sfd.people_targeted)), # noqa: E501 protection_gender_and_inclusion_requirements=cleaners.clean_number( data["sector"].get(_s.protection_gender_inclusion, {}).get(_sfd.requirements) ), @@ -657,7 +657,7 @@ def clean_data_and_save(self, scraped_data): water_sanitation_and_hygiene_people_reached=cleaners.clean_number( data["sector"].get(_s.Water_sanitation_hygiene, {}).get(_sfd.people_reached) ), - # water_sanitation_and_hygiene_people_targeted=cleaners.clean_number(data['sector'].get(_s.Water_sanitation_hygiene, {}).get(_sfd.people_targeted)), + # water_sanitation_and_hygiene_people_targeted=cleaners.clean_number(data['sector'].get(_s.Water_sanitation_hygiene, {}).get(_sfd.people_targeted)), # noqa: E501 water_sanitation_and_hygiene_requirements=cleaners.clean_number( data["sector"].get(_s.Water_sanitation_hygiene, {}).get(_sfd.requirements) ), @@ -747,7 +747,7 @@ def clean_data_and_save(self, scraped_data): current_operation_budget=cleaners.clean_number(data["meta"].get(_mfd.current_operation_budget)), dref_allocated=cleaners.clean_number(data["meta"].get(_mfd.dref_allocated)), glide_number=( - data["meta"].get(_mfd.glide_number)[:18] if data["meta"].get(_mfd.glide_number) != None else None + data["meta"].get(_mfd.glide_number)[:18] if data["meta"].get(_mfd.glide_number) is not None else None ), num_of_people_to_be_assisted=cleaners.clean_number(data["meta"].get(_mfd.num_of_people_to_be_assisted)), disaster_risk_reduction_female=cleaners.clean_number( @@ -830,7 +830,7 @@ def clean_data_and_save(self, scraped_data): ) ea_to_add.append(new_ea) - ## None of the records get inserted if any of them fails, but it would be faster this way + # # None of the records get inserted if any of them fails, but it would be faster this way # logger.info('Adding new EPoA records to DB (count: {})'.format(len(epoa_to_add))) # EmergencyOperationsDataset.objects.bulk_create(epoa_to_add) # logger.info('Adding new OU records to DB (count: {})'.format(len(ou_to_add))) diff --git a/api/management/commands/set_in_search_init.py b/api/management/commands/set_in_search_init.py index fa8e0532b..6639d8488 100644 --- a/api/management/commands/set_in_search_init.py +++ b/api/management/commands/set_in_search_init.py @@ -13,12 +13,10 @@ class Command(BaseCommand): def handle(self, *args, **options): try: # Update countries which should appear in search - inc_c = Country.objects.filter(independent=True, is_deprecated=False, record_type=1).update(in_search=True) + Country.objects.filter(independent=True, is_deprecated=False, record_type=1).update(in_search=True) # Update countries which should NOT appear in search # independent can be null too thus why negated check - exc_c = Country.objects.filter(~Q(independent=True) | Q(is_deprecated=True) | ~Q(record_type=1)).update( - in_search=False - ) + Country.objects.filter(~Q(independent=True) | Q(is_deprecated=True) | ~Q(record_type=1)).update(in_search=False) logger.info("Successfully set in_search for Countries") except Exception as ex: logger.error(f"Failed to set in_search for Countries. Error: {str(ex)}") diff --git a/api/management/commands/set_num_affected.py b/api/management/commands/set_num_affected.py index 9c1f355f1..f2267c350 100644 --- a/api/management/commands/set_num_affected.py +++ b/api/management/commands/set_num_affected.py @@ -1,6 +1,6 @@ from django.core.management.base import BaseCommand -from api.models import Appeal, Event +from api.models import Event class Command(BaseCommand): diff --git a/api/management/commands/sync_appealdocs.py b/api/management/commands/sync_appealdocs.py index 99505f99d..889805c49 100644 --- a/api/management/commands/sync_appealdocs.py +++ b/api/management/commands/sync_appealdocs.py @@ -7,7 +7,13 @@ from sentry_sdk.crons import monitor from api.logger import logger -from api.models import Appeal, AppealDocument, CronJob, CronJobStatus +from api.models import ( + Appeal, + AppealDocument, + AppealDocumentType, + CronJob, + CronJobStatus, +) from main.sentry import SentryMonitor CRON_NAME = "sync_appealdocs" diff --git a/api/management/commands/sync_molnix.py b/api/management/commands/sync_molnix.py index fd4c4d2ed..9929263f5 100644 --- a/api/management/commands/sync_molnix.py +++ b/api/management/commands/sync_molnix.py @@ -1,8 +1,6 @@ -import json - from dateutil import parser as date_parser from django.conf import settings -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from django.db import transaction from sentry_sdk.crons import monitor @@ -185,12 +183,12 @@ def get_go_event(tags): event_id = tag["name"].replace("OP-", "").strip() try: event_id_int = int(event_id) - except: + except Exception: logger.warning("%s tag is not a valid OP- tag" % event_id) continue try: event = Event.objects.get(id=event_id_int) - except: + except Exception: logger.warning("Emergency with ID %d not found" % event_id_int) prt("Emergency not found", 0, event_id_int) continue @@ -202,12 +200,12 @@ def get_go_country(countries, country_id): """ Given a Molnix country ID, returns GO country id """ - if not country_id in countries: + if country_id not in countries: return None iso = countries[country_id] try: country = Country.objects.get(iso=iso, independent=True) - except: + except Exception: logger.warning("Country with unknown ISO: %s" % iso) return None return country @@ -296,7 +294,7 @@ def sync_deployments(molnix_deployments, molnix_api, countries): try: personnel = Personnel.objects.get(molnix_id=md["id"]) created = False - except: + except Exception: personnel = Personnel(molnix_id=md["id"]) created = True # print('personnel found', personnel) @@ -308,7 +306,7 @@ def sync_deployments(molnix_deployments, molnix_api, countries): continue try: deployment = PersonnelDeployment.objects.get(is_molnix=True, event_deployed_to=event) - except: + except Exception: logger.warning("Did not import Deployment with Molnix ID %d. Invalid Event." % md["id"]) prt("Did not import Deployment. Invalid Event", md["id"]) continue @@ -317,7 +315,7 @@ def sync_deployments(molnix_deployments, molnix_api, countries): try: if md["position_id"]: surge_alert = SurgeAlert.objects.get(molnix_id=md["position_id"]) - except: + except Exception: logger.warning("%d deployment did not find SurgeAlert with Molnix position_id %d." % (md["id"], md["position_id"])) prt("Deployment did not find SurgeAlert", md["id"], md["position_id"]) continue @@ -328,7 +326,7 @@ def sync_deployments(molnix_deployments, molnix_api, countries): try: if md["person"] and "sex" in md["person"]: gender = md["person"]["sex"] - except: + except Exception: logger.warning("Did not find gender info in %d" % md["id"]) continue @@ -341,7 +339,7 @@ def sync_deployments(molnix_deployments, molnix_api, countries): and "city" in md["contact"]["addresses"][0] ): location = md["contact"]["addresses"][0]["city"] - except: + except Exception: logger.warning("Did not find city info in %d" % md["id"]) continue @@ -384,7 +382,7 @@ def sync_deployments(molnix_deployments, molnix_api, countries): country_name = NS_MATCHING_OVERRIDES[incoming_name] try: country_from = Country.objects.get(name_en=country_name) - except: + except Exception: warning = "Mismatch in NS name: %s" % md["incoming"]["name"] logger.warning(warning) warnings.append(warning) @@ -392,7 +390,7 @@ def sync_deployments(molnix_deployments, molnix_api, countries): try: country_from = Country.objects.get(society_name=incoming_name, independent=True) # maybe somewhen: .filter(society_name__iexact=incoming_name, independent=True).first() - except: + except Exception: # FIXME: Catch possibility of .get() returning multiple records # even though that should ideally never happen warning = "NS Name not found for Deployment ID: %d with secondment_incoming %s" % ( diff --git a/api/management/commands/triggers_to_db.py b/api/management/commands/triggers_to_db.py index 15bce0e61..4e5b36231 100644 --- a/api/management/commands/triggers_to_db.py +++ b/api/management/commands/triggers_to_db.py @@ -1,9 +1,6 @@ -from django.conf import settings from django.core.management.base import BaseCommand from django.db import connection -from api.logger import logger - class Command(BaseCommand): help = "Set triggers for updating previous_updated fields in api_event, _appeal, _fieldreport tables" @@ -11,7 +8,7 @@ class Command(BaseCommand): @staticmethod def apply(*_): return - ################## only one time change ################### + # ################# only one time change ################### with connection.cursor() as cursor: cursor.execute( """ diff --git a/api/management/commands/update-admin0-with-id.py b/api/management/commands/update-admin0-with-id.py index 8c1181ac3..152c0f1b3 100644 --- a/api/management/commands/update-admin0-with-id.py +++ b/api/management/commands/update-admin0-with-id.py @@ -8,7 +8,7 @@ class Command(BaseCommand): - help = "update admin0 geometries from a shapefile that has ICRC column headers along with ids from Go database. To run, python manage.py update-admin0-with-id input.shp" + help = "update admin0 geometries from a shapefile that has ICRC column headers along with ids from Go database. To run, python manage.py update-admin0-with-id input.shp" # noqa: E501 missing_args_message = "Filename is missing." @@ -21,7 +21,7 @@ def handle(self, *args, **options): try: data = DataSource(filename) - except: + except Exception: raise CommandError("Could not open file") for feature in data[0]: diff --git a/api/management/commands/update-mapbox-tilesets.py b/api/management/commands/update-mapbox-tilesets.py index 0c9efe4e7..008a99fef 100644 --- a/api/management/commands/update-mapbox-tilesets.py +++ b/api/management/commands/update-mapbox-tilesets.py @@ -7,7 +7,7 @@ class Command(BaseCommand): - help = "This command produces a countries.geojson and districts.geojson, and uploads them to Mapbox. It is the source for all GO Maps." + help = "This command produces a countries.geojson and districts.geojson, and uploads them to Mapbox. It is the source for all GO Maps." # noqa: E501 missing_args_message = "Argument missing. Specify --update-countries, --update-districts or --update-all." @@ -58,8 +58,8 @@ def update_countries(self, staging): print("Exporting countries...") try: - os.remove(f"/tmp/countries.geojson") - except FileNotFoundError as e: + os.remove("/tmp/countries.geojson") + except FileNotFoundError: pass subprocess.check_call( @@ -70,7 +70,7 @@ def update_countries(self, staging): "/tmp/countries.geojson", self.connection_string, "-sql", - "select cd.country_id, cd.geom, c.name, c.name_es, c.name_fr, c.name_ar, c.iso, c.region_id, c.iso3, c.independent, c.is_deprecated, c.disputed, c.fdrs, c.record_type from api_countrygeoms cd, api_country c where cd.country_id = c.id and c.record_type=1", + "select cd.country_id, cd.geom, c.name, c.name_es, c.name_fr, c.name_ar, c.iso, c.region_id, c.iso3, c.independent, c.is_deprecated, c.disputed, c.fdrs, c.record_type from api_countrygeoms cd, api_country c where cd.country_id = c.id and c.record_type=1", # noqa: E501 ] ) print("Countries written to /tmp/countries.geojson") @@ -82,8 +82,8 @@ def update_countries(self, staging): print("Exporting country centroids...") try: - os.remove(f"/tmp/country-centroids.geojson") - except FileNotFoundError as e: + os.remove("/tmp/country-centroids.geojson") + except FileNotFoundError: pass subprocess.check_call( @@ -96,7 +96,7 @@ def update_countries(self, staging): "/tmp/country-centroids.geojson", self.connection_string, "-sql", - "select id as country_id, name_en as name, name_ar, name_es, name_fr, independent, disputed, is_deprecated, iso, iso3, record_type, fdrs, region_id, centroid from api_country where centroid is not null", + "select id as country_id, name_en as name, name_ar, name_es, name_fr, independent, disputed, is_deprecated, iso, iso3, record_type, fdrs, region_id, centroid from api_country where centroid is not null", # noqa: E501 ] ) except Exception as e: @@ -105,7 +105,7 @@ def update_countries(self, staging): try: print("Update Mapbox tileset source for countries...") - tileset_source_name = f"go-countries-src-staging" if staging else f"go-countries-src" + tileset_source_name = "go-countries-src-staging" if staging else "go-countries-src" subprocess.check_call( ["tilesets", "upload-source", "--replace", "go-ifrc", tileset_source_name, "/tmp/countries.geojson"] ) @@ -115,7 +115,7 @@ def update_countries(self, staging): try: print("Update Mapbox tileset for countries... and sleeping a minute") - tileset_name = f"go-ifrc.go-countries-staging" if staging else f"go-ifrc.go-countries" + tileset_name = "go-ifrc.go-countries-staging" if staging else "go-ifrc.go-countries" subprocess.check_call(["tilesets", "publish", tileset_name]) time.sleep(60) except Exception as e: @@ -124,7 +124,7 @@ def update_countries(self, staging): try: print("Update Mapbox tileset source for country centroids...") - tileset_source_name = f"go-country-centroids-staging" if staging else f"go-country-centroids" + tileset_source_name = "go-country-centroids-staging" if staging else "go-country-centroids" subprocess.check_call( [ "tilesets", @@ -135,16 +135,16 @@ def update_countries(self, staging): "/tmp/country-centroids.geojson", ] ) - except Exception as e: + except Exception: print("Failed to update tileset source for country centroids") raise try: print("Update Mapbox tileset for country centroids... and sleeping a minute") - tileset_name = f"go-ifrc.go-country-centroids-staging" if staging else f"go-ifrc.go-country-centroids" + tileset_name = "go-ifrc.go-country-centroids-staging" if staging else "go-ifrc.go-country-centroids" subprocess.check_call(["tilesets", "publish", tileset_name]) time.sleep(60) - except Exception as e: + except Exception: print("Failed to update tileset for country centroids") raise @@ -153,8 +153,8 @@ def update_districts(self, staging): print("Exporting districts...") try: - os.remove(f"/tmp/distrcits.geojson") - except FileNotFoundError as e: + os.remove("/tmp/distrcits.geojson") + except FileNotFoundError: pass # FIXME eventually should be name_en, name_es etc. @@ -168,7 +168,7 @@ def update_districts(self, staging): "/tmp/districts.geojson", self.connection_string, "-sql", - "select cd.district_id, cd.geom, c.name, c.code, c.country_id, c.is_enclave, c.is_deprecated, country.iso as country_iso, country.iso3 as country_iso3, country.name as country_name, country.name_es as country_name_es, country.name_fr as country_name_fr, country.name_ar as country_name_ar from api_districtgeoms cd, api_district c, api_country country where cd.district_id = c.id and cd.geom is not null and country.id=c.country_id", + "select cd.district_id, cd.geom, c.name, c.code, c.country_id, c.is_enclave, c.is_deprecated, country.iso as country_iso, country.iso3 as country_iso3, country.name as country_name, country.name_es as country_name_es, country.name_fr as country_name_fr, country.name_ar as country_name_ar from api_districtgeoms cd, api_district c, api_country country where cd.district_id = c.id and cd.geom is not null and country.id=c.country_id", # noqa: E501 ] ) print("Districts written to /tmp/districts.geojson") @@ -180,8 +180,8 @@ def update_districts(self, staging): print("Exporting district centroids...") try: - os.remove(f"/tmp/district-centroids.geojson") - except FileNotFoundError as e: + os.remove("/tmp/district-centroids.geojson") + except FileNotFoundError: pass # FIXME eventually should be name_en, name_es etc. @@ -195,7 +195,7 @@ def update_districts(self, staging): "/tmp/district-centroids.geojson", self.connection_string, "-sql", - "select d.id as district_id, d.country_id as country_id, d.name, d.code, d.is_deprecated, d.is_enclave, c.iso as country_iso, c.iso3 as country_iso3, c.name as country_name, c.name_es as country_name_es, c.name_fr as country_name_fr, c.name_ar as country_name_ar, d.centroid from api_district d join api_country c on d.country_id=c.id where d.centroid is not null", + "select d.id as district_id, d.country_id as country_id, d.name, d.code, d.is_deprecated, d.is_enclave, c.iso as country_iso, c.iso3 as country_iso3, c.name as country_name, c.name_es as country_name_es, c.name_fr as country_name_fr, c.name_ar as country_name_ar, d.centroid from api_district d join api_country c on d.country_id=c.id where d.centroid is not null", # noqa: E501 ] ) except Exception as e: @@ -204,7 +204,7 @@ def update_districts(self, staging): try: print("Update Mapbox tileset source for districts...") - tileset_source_name = f"go-districts-src-staging" if staging else f"go-districts-src-1" + tileset_source_name = "go-districts-src-staging" if staging else "go-districts-src-1" subprocess.check_call( ["tilesets", "upload-source", "--replace", "go-ifrc", tileset_source_name, "/tmp/districts.geojson"] ) @@ -214,27 +214,27 @@ def update_districts(self, staging): try: print("Update Mapbox tileset for districts... and sleeping a minute") - tileset_name = f"go-ifrc.go-districts-staging" if staging else f"go-ifrc.go-districts-1" + tileset_name = "go-ifrc.go-districts-staging" if staging else "go-ifrc.go-districts-1" subprocess.check_call(["tilesets", "publish", tileset_name]) time.sleep(60) - except Exception as e: + except Exception: print("Failed to update tileset for districts") raise try: print("Update Mapbox tileset source for district centroids...") - tileset_source_name = f"go-district-centroids-staging" if staging else f"go-district-centroids" + tileset_source_name = "go-district-centroids-staging" if staging else "go-district-centroids" subprocess.check_call( ["tilesets", "upload-source", "--replace", "go-ifrc", tileset_source_name, "/tmp/district-centroids.geojson"] ) - except Exception as e: + except Exception: print("Failed to update tileset source for district centroid") raise try: print("Update Mapbox tileset for district centroids... [no sleep]") - tileset_name = f"go-ifrc.go-district-centroids-staging" if staging else f"go-ifrc.go-district-centroids" + tileset_name = "go-ifrc.go-district-centroids-staging" if staging else "go-ifrc.go-district-centroids" subprocess.check_call(["tilesets", "publish", tileset_name]) - except Exception as e: + except Exception: print("Failed to update tileset for distrct centroids") raise @@ -334,7 +334,7 @@ def prepare_admin2_geojson(self, iso): # query the database and create geojson try: os.remove(f"/tmp/{iso}.geojson") - except FileNotFoundError as e: + except FileNotFoundError: pass status = subprocess.run( @@ -345,7 +345,7 @@ def prepare_admin2_geojson(self, iso): f"/tmp/{iso}.geojson", self.connection_string, "-sql", - f"select d.id as admin1_id, d.name as admin1_name, ad.name, ad.id, ad.code, adg.geom from api_country as c, api_district as d, api_admin2 as ad, api_admin2geoms as adg where c.id=d.country_id and c.iso3='{iso}' and ad.admin1_id=d.id and adg.admin2_id = ad.id", + f"select d.id as admin1_id, d.name as admin1_name, ad.name, ad.id, ad.code, adg.geom from api_country as c, api_district as d, api_admin2 as ad, api_admin2geoms as adg where c.id=d.country_id and c.iso3='{iso}' and ad.admin1_id=d.id and adg.admin2_id = ad.id", # noqa: E501 ] ) if status: @@ -357,7 +357,7 @@ def prepare_admin2_geojson(self, iso): f"/tmp/{iso}-centroids.geojson", self.connection_string, "-sql", - f"select d.id as admin1_id, d.name as admin1_name, ad.name, ad.id, ad.code, ad.centroid from api_country as c, api_district as d, api_admin2 as ad where c.id=d.country_id and c.iso3='{iso}' and ad.admin1_id=d.id", + f"select d.id as admin1_id, d.name as admin1_name, ad.name, ad.id, ad.code, ad.centroid from api_country as c, api_district as d, api_admin2 as ad where c.id=d.country_id and c.iso3='{iso}' and ad.admin1_id=d.id", # noqa: E501 ] ) diff --git a/api/management/commands/update-sovereign-and-disputed.py b/api/management/commands/update-sovereign-and-disputed.py index 5a039e3dd..a397f78a4 100644 --- a/api/management/commands/update-sovereign-and-disputed.py +++ b/api/management/commands/update-sovereign-and-disputed.py @@ -1,14 +1,14 @@ import csv -from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist -from django.core.management.base import BaseCommand, CommandError +from django.core.exceptions import ObjectDoesNotExist +from django.core.management.base import BaseCommand from django.db import transaction from api.models import Country class Command(BaseCommand): - help = "update the sovereign country and disputed status for all the countries from a CSV file. To run, python manage.py update-sovereign-and-disputed file.csv" + help = "update the sovereign country and disputed status for all the countries from a CSV file. To run, python manage.py update-sovereign-and-disputed file.csv" # noqa: E501 missing_args_message = "Filename is missing. A CSV file is required." def add_arguments(self, parser): diff --git a/api/molnix_utils.py b/api/molnix_utils.py index 15110e5e3..85a53f808 100644 --- a/api/molnix_utils.py +++ b/api/molnix_utils.py @@ -82,7 +82,7 @@ def get_deployments(self): def get_position(self, id): try: return self.call_api(path="positions/%d" % id) - except: + except Exception: return None def get_countries(self): diff --git a/api/scrapers/cleaners.py b/api/scrapers/cleaners.py index 5040e66c8..d25d377be 100644 --- a/api/scrapers/cleaners.py +++ b/api/scrapers/cleaners.py @@ -12,7 +12,7 @@ def clean_number(num_in_str): try: num = int(re.sub(",", "", match.group())) return num - except: + except Exception: return None else: return None @@ -41,7 +41,7 @@ def clean_date(date_in_str): try: date = datetime.strptime(match.group(), "%d %B %Y") return date - except: + except Exception: return None else: return None diff --git a/api/scrapers/config.py b/api/scrapers/config.py index 1ccb80f63..9c1215d9d 100644 --- a/api/scrapers/config.py +++ b/api/scrapers/config.py @@ -97,7 +97,7 @@ class MetaFields: "Operation manager:", "National Society Focal Point:", "Amount requested for the appeal:", - "IFRC Project Manager \(responsible for implementation, compliances, monitoring and reporting\):", # noqa + r"IFRC Project Manager \(responsible for implementation, compliances, monitoring and reporting\):", # noqa "Period covered by this update:", ], } diff --git a/api/serializers.py b/api/serializers.py index 1f6c66eeb..371f5370a 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -38,7 +38,6 @@ CountryDirectory, CountryICRCPresence, CountryKeyDocument, - CountryKeyFigure, CountryLink, CountryOfFieldReportToReview, CountryOrganizationalCapacity, @@ -434,19 +433,6 @@ class Meta: ) -class CountryKeyFigureSerializer(ModelSerializer): - class Meta: - model = CountryKeyFigure - fields = ( - "country", - "figure", - "deck", - "source", - "visibility", - "id", - ) - - class RegionSnippetTableauSerializer(ModelSerializer): region = RegionSerializer() visibility_display = serializers.CharField(source="get_visibility_display", read_only=True) diff --git a/api/tasks.py b/api/tasks.py index 70010362b..653c08c5f 100644 --- a/api/tasks.py +++ b/api/tasks.py @@ -78,7 +78,7 @@ def generate_url(url, export_id, user, title): - """ + """ # noqa: E501 try: with tempfile.TemporaryDirectory() as tmp_dir: with sync_playwright() as p: diff --git a/api/test_scrapers.py b/api/test_scrapers.py index c9971f09d..4a124317a 100644 --- a/api/test_scrapers.py +++ b/api/test_scrapers.py @@ -1,5 +1,4 @@ import time -from datetime import timedelta from django.contrib.auth.models import User from django.test import TestCase @@ -15,7 +14,7 @@ SubscriptionType, ) -from .models import Appeal, Event, FieldReport +from .models import Appeal, FieldReport def get_user(): @@ -196,7 +195,7 @@ def setUp(self): country1 = Country.objects.create(name="1", region=region) country2 = Country.objects.create(name="2", region=region) appeal1 = Appeal.objects.create(aid="test1", name="appeal", atype=2, code="1", country=country1) - appeal2 = Appeal.objects.create(aid="test2", name="appeal", atype=2, code="2", country=country2) + Appeal.objects.create(aid="test2", name="appeal", atype=2, code="2", country=country2) time.sleep(1) appeal1.name = "something new" appeal1.save() diff --git a/api/test_views.py b/api/test_views.py index 44b639a00..9d0a2227d 100644 --- a/api/test_views.py +++ b/api/test_views.py @@ -333,12 +333,8 @@ def test_historical_events(self): appeal1 = AppealFactory.create( event=event1, dtype=dtype1, num_beneficiaries=9000, amount_requested=10000, amount_funded=1899999 ) - appeal2 = AppealFactory.create( - event=event2, dtype=dtype2, num_beneficiaries=90023, amount_requested=100440, amount_funded=12299999 - ) - appeal2 = AppealFactory.create( - event=event3, dtype=dtype2, num_beneficiaries=91000, amount_requested=10000888, amount_funded=678888 - ) + AppealFactory.create(event=event2, dtype=dtype2, num_beneficiaries=90023, amount_requested=100440, amount_funded=12299999) + AppealFactory.create(event=event3, dtype=dtype2, num_beneficiaries=91000, amount_requested=10000888, amount_funded=678888) response = self.client.get("/api/v2/go-historical/").json() self.assertEqual(response["count"], 3) # should give event that have appeal associated with and num_affected=null self.assertEqual(sorted([event1.id, event2.id, event3.id]), sorted([data["id"] for data in response["results"]])) @@ -362,8 +358,8 @@ def test_admin2_api(self): country = models.Country.objects.create(name="Nepal", iso3="NPL", region=region) admin1_1 = models.District.objects.create(name="admin1 1", code="NPL01", country=country) admin1_2 = models.District.objects.create(name="admin1 2", code="NPL02", country=country) - admin2_1 = models.Admin2.objects.create(name="test 1", admin1=admin1_1, code="1") - admin2_2 = models.Admin2.objects.create(name="test 2", admin1=admin1_2, code="2") + models.Admin2.objects.create(name="test 1", admin1=admin1_1, code="1") + models.Admin2.objects.create(name="test 2", admin1=admin1_2, code="2") # test fetching all admin2-s response = self.client.get("/api/v2/admin2/").json() diff --git a/api/views.py b/api/views.py index a5d2baac8..6ea078360 100644 --- a/api/views.py +++ b/api/views.py @@ -4,10 +4,9 @@ from django.conf import settings from django.contrib.auth import authenticate from django.contrib.auth.models import User -from django.db import models -from django.db.models import Avg, Case, Count, F, OuterRef, Q, Subquery, Sum, When +from django.db.models import Case, Count, F, Q, Sum, When from django.db.models.fields import IntegerField -from django.db.models.functions import Coalesce, TruncMonth, TruncYear +from django.db.models.functions import TruncMonth, TruncYear from django.http import HttpResponse, JsonResponse from django.template.loader import render_to_string from django.utils import timezone @@ -28,10 +27,6 @@ AggregateHeaderFiguresInputSerializer, AggregateHeaderFiguresSerializer, AreaAggregateSerializer, - CountryDisasterTypeCountSerializer, - CountryDisasterTypeMonthlySerializer, - CountryKeyFigureInputSerializer, - CountryKeyFigureSerializer, ProjectPrimarySectorsSerializer, ProjectSecondarySectorsSerializer, SearchInputSerializer, diff --git a/country_plan/management/commands/ingest_country_plan_file.py b/country_plan/management/commands/ingest_country_plan_file.py index 51de2371d..a911c505a 100644 --- a/country_plan/management/commands/ingest_country_plan_file.py +++ b/country_plan/management/commands/ingest_country_plan_file.py @@ -1,6 +1,6 @@ import logging from datetime import datetime -from typing import Optional, Tuple, Union +from typing import Optional, Tuple import requests from django.conf import settings diff --git a/databank/management/commands/fdrs_annual_income.py b/databank/management/commands/fdrs_annual_income.py index f17ed925f..a41d962cb 100644 --- a/databank/management/commands/fdrs_annual_income.py +++ b/databank/management/commands/fdrs_annual_income.py @@ -15,7 +15,7 @@ class Command(BaseCommand): def handle(self, *args, **kwargs): for overview in CountryOverview.objects.all(): country_fdrs_code = overview.country.fdrs - fdrs_api = f"https://data-api.ifrc.org/api/data?indicator=KPI_IncomeLC_CHF&KPI_Don_Code={country_fdrs_code}&apiKey={settings.FDRS_APIKEY}" + fdrs_api = f"https://data-api.ifrc.org/api/data?indicator=KPI_IncomeLC_CHF&KPI_Don_Code={country_fdrs_code}&apiKey={settings.FDRS_APIKEY}" # noqa: E501 fdrs_entities = requests.get(fdrs_api) if fdrs_entities.status_code != 200: return diff --git a/databank/management/commands/ingest_acaps.py b/databank/management/commands/ingest_acaps.py index 7ca6bfc42..28ab823ef 100644 --- a/databank/management/commands/ingest_acaps.py +++ b/databank/management/commands/ingest_acaps.py @@ -1,15 +1,12 @@ -import datetime as dt import time import pandas as pd import requests -import xmltodict -from dateutil.parser import parse from django.conf import settings from django.core.management.base import BaseCommand from api.logger import logger -from api.models import Country, CountryType, CronJob, CronJobStatus, DisasterType +from api.models import CountryType from databank.models import AcapsSeasonalCalender, CountryOverview diff --git a/databank/management/commands/ingest_climate.py b/databank/management/commands/ingest_climate.py index 8bd433419..f90914864 100644 --- a/databank/management/commands/ingest_climate.py +++ b/databank/management/commands/ingest_climate.py @@ -17,7 +17,7 @@ def handle(self, *args, **options): country_iso3 = co.country.iso3 if country_iso3: response = requests.get( - f"https://climateknowledgeportal.worldbank.org/api/v1/cru-x0.5_climatology_tasmin,tas,tasmax,pr_climatology_monthly_1991-2020_mean_historical_cru_ts4.07_mean/{country_iso3}?_format=json" + f"https://climateknowledgeportal.worldbank.org/api/v1/cru-x0.5_climatology_tasmin,tas,tasmax,pr_climatology_monthly_1991-2020_mean_historical_cru_ts4.07_mean/{country_iso3}?_format=json" # noqa: E501 ) response.raise_for_status() try: @@ -52,6 +52,6 @@ def handle(self, *args, **options): "precipitation": v[0], } CountryKeyClimate.objects.create(overview=co, **data) - except Exception as ex: + except Exception: logger.error("Error in ingesting climate data", exc_info=True) continue diff --git a/databank/management/commands/ingest_unicef.py b/databank/management/commands/ingest_unicef.py index e675de694..786b6f0b7 100644 --- a/databank/management/commands/ingest_unicef.py +++ b/databank/management/commands/ingest_unicef.py @@ -13,7 +13,7 @@ class Command(BaseCommand): def handle(self, *args, **kwargs): for overview in CO.objects.all(): - UNICEF_API = f"https://sdmx.data.unicef.org/ws/public/sdmxapi/rest/data/UNICEF,DM,1.0/{overview.country.iso3}.DM_POP_U18._T._T.?format=sdmx-json" + UNICEF_API = f"https://sdmx.data.unicef.org/ws/public/sdmxapi/rest/data/UNICEF,DM,1.0/{overview.country.iso3}.DM_POP_U18._T._T.?format=sdmx-json" # noqa: E501 unicef_entities = requests.get(UNICEF_API) if unicef_entities.status_code != 200: continue diff --git a/databank/management/commands/ingest_worldbank.py b/databank/management/commands/ingest_worldbank.py index b0e5d3527..9a6eef5f2 100644 --- a/databank/management/commands/ingest_worldbank.py +++ b/databank/management/commands/ingest_worldbank.py @@ -8,7 +8,7 @@ from api.models import Country, CountryType from databank.models import CountryOverview as CO -from .sources.utils import catch_error, get_country_by_iso3 +from .sources.utils import get_country_by_iso3 logger = logging.getLogger(__name__) @@ -55,7 +55,7 @@ def handle(self, *args, **kwargs): "page": page, }, ) - except requests.exceptions.HTTPError as err: + except requests.exceptions.HTTPError: continue try: data_list = response.json()[1] diff --git a/databank/management/commands/sources/utils.py b/databank/management/commands/sources/utils.py index 403f9eb06..c9c82d96a 100644 --- a/databank/management/commands/sources/utils.py +++ b/databank/management/commands/sources/utils.py @@ -19,7 +19,7 @@ def _caller(*args, **kwargs): try: with transaction.atomic(): return func(*args, **kwargs) - except Exception as e: + except Exception: # Log error to cronjob CronJob.sync_cron( { @@ -27,7 +27,7 @@ def _caller(*args, **kwargs): "message": ( f"Error querying {source_name}." + (f" For Country: {country}." if country else "") - + f"\n\n" + + "\n\n" + traceback.format_exc() ), "status": CronJobStatus.ERRONEOUS, diff --git a/databank/management/commands/sources/world_bank.py b/databank/management/commands/sources/world_bank.py index e8d6d7fca..11137df3d 100644 --- a/databank/management/commands/sources/world_bank.py +++ b/databank/management/commands/sources/world_bank.py @@ -2,7 +2,6 @@ import logging import requests -from django.conf import settings from api.models import Country from databank.models import CountryOverview as CO @@ -33,7 +32,6 @@ def prefetch(): data = {} - url = WORLD_BANK_API page = 1 now = datetime.datetime.now() daterange = f"{now.year - 1}:{now.year}" diff --git a/databank/models.py b/databank/models.py index 15b3460c2..2bcd3e1cf 100644 --- a/databank/models.py +++ b/databank/models.py @@ -467,4 +467,4 @@ class Meta: verbose_name_plural = _("Country Climate") def __str__(self): - return f"{self.overview.country.name} - { self.year} - {self.get_month_display()}" + return f"{self.overview.country.name} - {self.year} - {self.get_month_display()}" diff --git a/databank/tests.py b/databank/tests.py index 7ce503c2d..a39b155ac 100644 --- a/databank/tests.py +++ b/databank/tests.py @@ -1,3 +1 @@ -from django.test import TestCase - # Create your tests here. diff --git a/deployments/filters.py b/deployments/filters.py index 882d65391..0403414ad 100644 --- a/deployments/filters.py +++ b/deployments/filters.py @@ -1,5 +1,3 @@ -from functools import reduce - import django_filters as filters from django.contrib.auth.models import User from django.db.models import F, Q diff --git a/dref/views.py b/dref/views.py index f8e774d78..56bd5888c 100644 --- a/dref/views.py +++ b/dref/views.py @@ -1,11 +1,8 @@ -from itertools import chain -from operator import attrgetter - import django.utils.timezone as timezone from django.contrib.auth.models import Permission from django.db import models from django.utils.translation import gettext -from drf_spectacular.utils import extend_schema, extend_schema_view +from drf_spectacular.utils import extend_schema from rest_framework import ( mixins, permissions, diff --git a/eap/admin.py b/eap/admin.py index 8c38f3f3d..846f6b406 100644 --- a/eap/admin.py +++ b/eap/admin.py @@ -1,3 +1 @@ -from django.contrib import admin - # Register your models here. diff --git a/eap/tests.py b/eap/tests.py index 7ce503c2d..a39b155ac 100644 --- a/eap/tests.py +++ b/eap/tests.py @@ -1,3 +1 @@ -from django.test import TestCase - # Create your tests here. diff --git a/eap/views.py b/eap/views.py index 91ea44a21..60f00ef0e 100644 --- a/eap/views.py +++ b/eap/views.py @@ -1,3 +1 @@ -from django.shortcuts import render - # Create your views here. diff --git a/flash_update/utils.py b/flash_update/utils.py index 381e8ece3..07468b7ea 100644 --- a/flash_update/utils.py +++ b/flash_update/utils.py @@ -18,7 +18,7 @@ def render_to_pdf(template_src, context_dict={}): if not pdf.err: file = {"filename": "flash_update.pdf", "file": result.getvalue()} return file - except Exception as e: + except Exception: logger.error("Error in rendering html to pdf", exc_info=True) return None diff --git a/lang/serializers.py b/lang/serializers.py index 904914aaf..5b3bb1e09 100644 --- a/lang/serializers.py +++ b/lang/serializers.py @@ -98,7 +98,7 @@ def _get_language_clear_validated_data(cls, instance, validated_data, included_f cleared = False for field, current_lang_field in included_fields_lang.items(): old_value = getattr(instance, current_lang_field) or "" - if type(validated_data) == dict: + if isinstance(validated_data, dict): new_value = validated_data.get(current_lang_field) or validated_data.get(field) or "" else: # NOTE: Assuming it's model instance new_value = getattr(validated_data, current_lang_field, getattr(validated_data, field, None)) or "" @@ -108,7 +108,7 @@ def _get_language_clear_validated_data(cls, instance, validated_data, included_f lang_field = build_localized_fieldname(field, lang) if lang_field == current_lang_field: continue - if type(validated_data) == dict: + if isinstance(validated_data, dict): validated_data[lang_field] = "" else: # NOTE: Assuming it's model instance setattr(validated_data, lang_field, "") diff --git a/lang/tasks.py b/lang/tasks.py index fa6e9abb9..10ff176c5 100644 --- a/lang/tasks.py +++ b/lang/tasks.py @@ -137,7 +137,7 @@ def show_characters_counts(cls, only_models: typing.Optional[typing.List[models. total_count += count logger.info(f"\t\t {field} - {count}") logger.info(f"Total Count: {total_count}") - logger.info(f"Estimated Cost (AWS): {(len(AVAILABLE_LANGUAGES) -1) * total_count * 0.000015}") + logger.info(f"Estimated Cost (AWS): {(len(AVAILABLE_LANGUAGES) - 1) * total_count * 0.000015}") def run(self, batch_size=None, only_models: typing.Optional[typing.List[models.Model]] = None): """ diff --git a/local_units/management/commands/import-delegation-offices-csv.py b/local_units/management/commands/import-delegation-offices-csv.py index a0bf9bacf..1781224ca 100644 --- a/local_units/management/commands/import-delegation-offices-csv.py +++ b/local_units/management/commands/import-delegation-offices-csv.py @@ -12,7 +12,7 @@ from ...models import DelegationOffice, DelegationOfficeType # Don't use the received header's always changing upperlowercase header, use a row like this: -# name type city address postcode latitude longitude iso iso3 country ifrc_region_number ifrc_region_name society_name society_url url_ifrc ns_same_location multiple_ifrc_offices office_tier hod_first_name hod_last_name hod_mobile_number hod_email assistant_name assistant_email +# name type city address postcode latitude longitude iso iso3 country ifrc_region_number ifrc_region_name society_name society_url url_ifrc ns_same_location multiple_ifrc_offices office_tier hod_first_name hod_last_name hod_mobile_number hod_email assistant_name assistant_email # noqa: E501 class Command(BaseCommand): diff --git a/local_units/test_views.py b/local_units/test_views.py index cc31d1a2f..35f10b1e8 100644 --- a/local_units/test_views.py +++ b/local_units/test_views.py @@ -305,7 +305,7 @@ def test_create_local_unit_health(self): "draft": False, "validated": True, "postcode": "", - "address_loc": "Silele Clinic is is in Hosea Inkhundla under the Shiselweni, Sigombeni is in Nkom'iyahlaba Inkhundla under the Manzini region and Mahwalala is in the Mbabane West Inkhundla under the Hhohho region.", + "address_loc": "Silele Clinic is is in Hosea Inkhundla under the Shiselweni, Sigombeni is in Nkom'iyahlaba Inkhundla under the Manzini region and Mahwalala is in the Mbabane West Inkhundla under the Hhohho region.", # noqa: E501 "address_en": "", "city_loc": "", "city_en": "", @@ -326,7 +326,7 @@ def test_create_local_unit_health(self): "focal_point_phone_number": "26876088546", "focal_point_position": "Programmes Manager", "other_facility_type": None, - "speciality": "Initiate TB treatment, Cervical Cancer Screening and testing and diagnostic and treatment for people living with HIV and follow up care through the ART programme which the government supports very well", + "speciality": "Initiate TB treatment, Cervical Cancer Screening and testing and diagnostic and treatment for people living with HIV and follow up care through the ART programme which the government supports very well", # noqa: E501 "is_teaching_hospital": False, "is_in_patient_capacity": False, "is_isolation_rooms_wards": False, @@ -348,7 +348,7 @@ def test_create_local_unit_health(self): "midwife": 9, "other_medical_heal": True, "other_profiles": None, - "feedback": "first question of initial question did not provide for the option to write the name of the NS. It is written LRC yet it should allow Baphalali Eswatini Red Cross Society (BERCS) to be inscribed in the box.", + "feedback": "first question of initial question did not provide for the option to write the name of the NS. It is written LRC yet it should allow Baphalali Eswatini Red Cross Society (BERCS) to be inscribed in the box.", # noqa: E501 "affiliation": affiliation.id, "functionality": functionality.id, "health_facility_type": health_facility_type.id, diff --git a/main/exception_handler.py b/main/exception_handler.py index 2ff51a7d8..a351863f4 100644 --- a/main/exception_handler.py +++ b/main/exception_handler.py @@ -25,7 +25,7 @@ def custom_exception_handler(exc, context): # For 500 errors, we create new response and add extra attributes to sentry if not response: # Expected ReadOnlyError - if type(exc) == DjangoReadOnlyError: + if isinstance(exc, DjangoReadOnlyError): response_data = { "errors": {"non_field_errors": ["We are in maintenance mode, come back a bit later – site is in read only mode"]}, } diff --git a/main/filters.py b/main/filters.py index 53827ad6b..6f0a7bdef 100644 --- a/main/filters.py +++ b/main/filters.py @@ -1,4 +1,4 @@ -from django.db.models import F, OrderBy +from django.db.models import F from django_filters import rest_framework as filters from rest_framework.filters import OrderingFilter diff --git a/main/settings.py b/main/settings.py index 817da7931..d979be11b 100644 --- a/main/settings.py +++ b/main/settings.py @@ -610,7 +610,7 @@ SPECTACULAR_SETTINGS = { "TITLE": "IFRC-GO API", - "DESCRIPTION": 'Please see the GO Wiki for an overview of API usage, or the interactive Swagger page.', + "DESCRIPTION": 'Please see the GO Wiki for an overview of API usage, or the interactive Swagger page.', # noqa: E501 "VERSION": "1.0.0", "SERVE_INCLUDE_SCHEMA": False, "ENUM_ADD_EXPLICIT_BLANK_NULL_CHOICE": False, diff --git a/main/validators.py b/main/validators.py index 3f8b05403..611e2a3b9 100644 --- a/main/validators.py +++ b/main/validators.py @@ -6,7 +6,7 @@ class NumberValidator(object): def validate(self, password, user=None): - if not re.findall("\d", password): + if not re.findall(r"\d", password): raise ValidationError( _("The password must contain at least 1 digit, 0-9."), code="password_no_number", @@ -42,11 +42,11 @@ def get_help_text(self): class SymbolValidator(object): def validate(self, password, user=None): - if not re.findall("[()[\]{}|\\`~!@#$%^&*_\-+=;:'\",<>./?]", password): + if not re.findall(r"[()[\]{}|\\`~!@#$%^&*_\-+=;:'\",<>./?]", password): raise ValidationError( - _("The password must contain at least 1 symbol: " + "()[]{}|\`~!@#$%^&*_-+=;:'\",<>./?"), + _("The password must contain at least 1 symbol: " + r"()[]{}|\`~!@#$%^&*_-+=;:'\",<>./?"), code="password_no_symbol", ) def get_help_text(self): - return _("Your password must contain at least 1 symbol: " + "()[]{}|\`~!@#$%^&*_-+=;:'\",<>./?") + return _("Your password must contain at least 1 symbol: " + r"()[]{}|\`~!@#$%^&*_-+=;:'\",<>./?") diff --git a/manage.py b/manage.py index ad5d3e7ed..784a7f4aa 100755 --- a/manage.py +++ b/manage.py @@ -11,7 +11,7 @@ # issue is really that Django is missing to avoid masking other # exceptions on Python 2. try: - import django + import django # noqa: F401 except ImportError: raise ImportError( "Couldn't import Django. Are you sure it's installed and " diff --git a/per/models.py b/per/models.py index 48ba5bf35..bf73e8e71 100644 --- a/per/models.py +++ b/per/models.py @@ -708,7 +708,8 @@ def is_user_admin(user): return ( user is not None and not user.is_anonymous - and (user.is_superuser or user.groups.filter(name="OpsLearning Admin").exists()) + and user.is_superuser + or user.groups.filter(name="OpsLearning Admin").exists() ) def save(self, *args, **kwargs): diff --git a/registrations/tests.py b/registrations/tests.py index 7ce503c2d..a39b155ac 100644 --- a/registrations/tests.py +++ b/registrations/tests.py @@ -1,3 +1 @@ -from django.test import TestCase - # Create your tests here. diff --git a/utils/test_utils.py b/utils/test_utils.py index afbbaa3ea..d23cffe43 100644 --- a/utils/test_utils.py +++ b/utils/test_utils.py @@ -13,7 +13,7 @@ from api.factories import disaster_type as dtFactory from api.factories import event as eventFactory from api.factories import field_report as fieldReportFactory -from api.models import ERPGUID, Country, DisasterType, Event, FieldReport, Region +from api.models import ERPGUID from main.mock import erp_request_side_effect_mock from main.utils import DjangoReversionDataFixHelper from per.factories import OverviewFactory as PerOverviewFactory @@ -29,7 +29,7 @@ def test_not_successful(self, erp_request_side_effect_mock): report = fieldReportFactory.FieldReportFactory.create(rid="test", dtype=dtype, event=event) event.countries.set([]) report.countries.set([]) - result = erp.push_fr_data(report) + erp.push_fr_data(report) # self.assertEqual(erp_request_side_effect_mock.called, False) # this should work, but CircleCI calls this mock @patch("requests.post", side_effect=erp_request_side_effect_mock) @@ -41,7 +41,7 @@ def test_successful(self, erp_request_side_effect_mock): report = fieldReportFactory.FieldReportFactory.create(rid="test", dtype=dtype, event=event) report.countries.add(country1) report.countries.add(country2) - result = erp.push_fr_data(report) + erp.push_fr_data(report) ERP = ERPGUID.objects.get(api_guid="FindThisGUID") self.assertEqual(ERP.api_guid, "FindThisGUID") self.assertEqual(ERP.field_report_id, report.id)