Merge pull request #5642 from tsmock/dependency-cleanup

Update dependencies (backend)

.circleci/config.yml

@@ -35,7 +35,7 @@ jobs:
 
   backend-code-check-PEP8:
     docker:
-      - image: cimg/python:3.9
+      - image: cimg/python:3.10
     steps:
       - checkout
       - setup_remote_docker
@@ -47,7 +47,7 @@ jobs:
 
   backend-code-check-Black:
     docker:
-      - image: cimg/python:3.9
+      - image: cimg/python:3.10
     steps:
       - checkout
       - setup_remote_docker
@@ -61,12 +61,14 @@ jobs:
     resource_class: large
     docker:
 
-      - image: cimg/python:3.9
+      - image: cimg/python:3.10
         environment:
           SQLALCHEMY_DATABASE_URI: postgresql://taskingmanager@localhost/test_tm
           POSTGRES_TEST_DB: test_tm
           POSTGRES_USER: taskingmanager
           POSTGRES_ENDPOINT: localhost
+          TM_ORG_CODE: "CICode"
+          TM_ORG_NAME: "CircleCI Test Organisation"
 
       - image: cimg/postgres:14.2-postgis
         environment:
@@ -87,18 +89,19 @@ jobs:
             -c "CREATE EXTENSION postgis;"
       - run: pip install --upgrade pip pdm
       - run: pdm config --global python.use_venv False
-      - run: pdm export --prod --without-hashes > requirements.txt
+      - run: pdm export --dev --without-hashes > requirements.txt
       - run: pip install -r requirements.txt
       - run: mkdir --mode 766 -p /tmp/logs
       - run: mkdir ${CIRCLE_WORKING_DIRECTORY}/tests/backend/results
       - run: find ./tests/backend -name "test*.py" -exec chmod -x {} \;
       - run: echo "export TM_LOG_DIR=/tmp/logs" >> $BASH_ENV
+      - run: coverage erase
       - run:
           name: Run backend functional tests
           command: |
-            nosetests ./tests/backend --with-xunit \
-              --xunit-file ${CIRCLE_WORKING_DIRECTORY}/tests/backend/results/unitresults.xml \
-              --with-coverage --cover-erase --cover-package=./backend
+            coverage run --source ./backend -m pytest \
+              --rootdir ./tests/backend \
+              --junit-xml ${CIRCLE_WORKING_DIRECTORY}/tests/backend/results/unitresults.xml
       - run: coverage xml -o ${CIRCLE_WORKING_DIRECTORY}/tests/backend/results/coverage.xml
       - store_test_results:
           path: tests/backend/results/unitresults.xml
@@ -161,10 +164,10 @@ jobs:
   build:
     working_directory: /home/circleci/app
     docker:
-      - image: cimg/python:3.9.14-node
+      - image: cimg/python:3.10.7-node
         environment:
           SQLALCHEMY_DATABASE_URI: postgresql://taskingmanager@localhost/test_tm
-      - image: cimg/postgres:14.2-postgis
+      - image: cimg/postgres:14.7-postgis
         environment:
           POSTGRES_USER: taskingmanager
           POSTGRES_DB: test_tm
@@ -184,17 +187,17 @@ jobs:
       - run: cd ${CIRCLE_WORKING_DIRECTORY}
       - run: pip install --upgrade pip pdm
       - run: pdm config --global python.use_venv False
-      - run: pdm export --prod --without-hashes > requirements.txt
+      - run: pdm export --dev --without-hashes > requirements.txt
       - run: pip install -r requirements.txt
       - run: mkdir ${CIRCLE_WORKING_DIRECTORY}/tests/backend/results
       - run: find ./tests/backend -name "test*.py" -exec chmod -x {} \;
       - run: coverage erase
       - run:
           name: Run backend tests
           command: |
-            nosetests ./tests/backend --with-xunit \
-              --xunit-file ${CIRCLE_WORKING_DIRECTORY}/tests/backend/results/unitresults.xml \
-              --with-coverage --cover-erase --cover-package=./backend
+            coverage run --source ./backend -m pytest \
+              --rootdir ./tests/backend \
+              --junit-xml ${CIRCLE_WORKING_DIRECTORY}/tests/backend/results/unitresults.xml
       - run: coverage xml -o ${CIRCLE_WORKING_DIRECTORY}/tests/backend/results/coverage.xml
       - store_test_results:
           path: tests/backend/results/unitresults.xml
@@ -234,7 +237,7 @@ jobs:
     working_directory: /home/circleci/tasking-manager
     resource_class: medium
     docker:
-      - image: cimg/python:3.9.14-node
+      - image: cimg/python:3.10.7-node
     steps:
       - checkout
       - setup_remote_docker
@@ -271,7 +274,7 @@ jobs:
     working_directory: /home/circleci/tasking-manager
     resource_class: large
     docker:
-      - image: cimg/python:3.9.14-node
+      - image: cimg/python:3.10.7-node
     parameters:
       stack_name:
         description: "the name of the stack for cfn-config"

.dockerignore

@@ -4,5 +4,8 @@
 **/__pypackages__
 **/node_modules
 **/npm-debug.log
-logs/
+**/venv
+docs-old/
 docs/
+frontend/
+logs/

.flaskenv

@@ -0,0 +1,5 @@
+# This file is not read for Tasking Manager variables. Use tasking-manager.env instead.
+# Flask (the web server framework) uses this file to figure out what the entry point is,
+# if it is non-standard.
+# For more information, see https://flask.palletsprojects.com/en/2.0.x/cli/#environment-variables-from-dotenv
+FLASK_APP=manage.py

Dockerfile

@@ -1,5 +1,5 @@
 ARG ALPINE_IMG_TAG=3.17
-ARG PYTHON_IMG_TAG=3.9
+ARG PYTHON_IMG_TAG=3.10
 
 FROM docker.io/python:${PYTHON_IMG_TAG}-alpine${ALPINE_IMG_TAG} as base
 ARG APP_VERSION=0.1.0

backend/__init__.py

@@ -1,4 +1,18 @@
 import logging
+
+# gevent.monkey.patch_ssl is required. gevent message as follows:
+# MonkeyPatchWarning: Monkey-patching ssl after ssl has already been imported may
+# lead to errors, including RecursionError on Python 3.6. It may also silently
+# lead to incorrect behaviour on Python 3.7. Please monkey-patch earlier.
+# See https://github.com/gevent/gevent/issues/1016.
+try:
+    from gevent import monkey
+
+    monkey.patch_ssl()
+except ImportError as e:
+    logging.warning("Not using gevent")
+    logging.info(e)
+
 import os
 from logging.handlers import RotatingFileHandler
 

backend/api/organisations/resources.py

@@ -462,8 +462,8 @@ def get(self):
             )
 
         # Validate abbreviated.
-        omit_managers = strtobool(request.args.get("omitManagerList", "false"))
-        omit_stats = strtobool(request.args.get("omitOrgStats", "true"))
+        omit_managers = bool(strtobool(request.args.get("omitManagerList", "false")))
+        omit_stats = bool(strtobool(request.args.get("omitOrgStats", "true")))
         # Obtain organisations
         try:
             results_dto = OrganisationService.get_organisations_as_dto(

backend/api/projects/actions.py

@@ -15,6 +15,8 @@
 from backend.services.users.authentication_service import token_auth, tm
 from backend.services.interests_service import InterestService
 from backend.models.postgis.utils import InvalidGeoJson
+
+from shapely import GEOSException
 from shapely.errors import TopologicalError
 
 
@@ -432,6 +434,16 @@ def post(self):
                 "error": "Invalid geometry. Polygon is self intersecting",
                 "SubCode": "SelfIntersectingAOI",
             }, 400
+        except GEOSException as wrapped:
+            if (
+                isinstance(wrapped.args[0], str)
+                and "Self-intersection" in wrapped.args[0]
+            ):
+                return {
+                    "error": "Invalid geometry. Polygon is self intersecting",
+                    "SubCode": "SelfIntersectingAOI",
+                }, 400
+            return {"error": str(wrapped), "SubCode": "InternalServerError"}
         except Exception as e:
             error_msg = f"IntersectingTiles GET API - unhandled error: {str(e)}"
             current_app.logger.critical(error_msg)

backend/api/projects/resources.py

@@ -106,7 +106,7 @@ def get(self, project_id):
                         io.BytesIO(geojson.dumps(project_dto).encode("utf-8")),
                         mimetype="application/json",
                         as_attachment=True,
-                        attachment_filename=f"project_{str(project_id)}.json",
+                        download_name=f"project_{str(project_id)}.json",
                     )
 
                 return project_dto, 200
@@ -1032,7 +1032,7 @@ def get(self, project_id):
                     io.BytesIO(geojson.dumps(project_dto).encode("utf-8")),
                     mimetype="application/json",
                     as_attachment=True,
-                    attachment_filename=f"project_{str(project_id)}.json",
+                    download_name=f"project_{str(project_id)}.json",
                 )
 
             return project_dto, 200
@@ -1154,7 +1154,7 @@ def get(self, project_id):
                     io.BytesIO(geojson.dumps(project_aoi).encode("utf-8")),
                     mimetype="application/json",
                     as_attachment=True,
-                    attachment_filename=f"{str(project_id)}.geojson",
+                    download_name=f"{str(project_id)}.geojson",
                 )
 
             return project_aoi, 200

backend/api/tasks/actions.py

@@ -171,7 +171,9 @@ def post(self, project_id, task_id):
                 description: Internal Server Error
         """
         try:
-            stop_task = StopMappingTaskDTO(request.get_json())
+            stop_task = StopMappingTaskDTO(
+                request.get_json() if request.is_json else {}
+            )
             stop_task.user_id = token_auth.current_user()
             stop_task.task_id = task_id
             stop_task.project_id = project_id

backend/api/tasks/resources.py

@@ -122,7 +122,7 @@ def get(self, project_id):
                     io.BytesIO(tasks_json),
                     mimetype="application/json",
                     as_attachment=True,
-                    attachment_filename=f"{str(project_id)}-tasks.geojson",
+                    download_name=f"{str(project_id)}-tasks.geojson",
                 )
 
             return tasks_json, 200
@@ -268,7 +268,7 @@ def get(self, project_id):
                     io.BytesIO(xml),
                     mimetype="text.xml",
                     as_attachment=True,
-                    attachment_filename=f"HOT-project-{project_id}.osm",
+                    download_name=f"HOT-project-{project_id}.osm",
                 )
 
             return Response(xml, mimetype="text/xml", status=200)
@@ -341,7 +341,7 @@ def get(self, project_id):
                     io.BytesIO(xml),
                     mimetype="text.xml",
                     as_attachment=True,
-                    attachment_filename=f"HOT-project-{project_id}.gpx",
+                    download_name=f"HOT-project-{project_id}.gpx",
                 )
 
             return Response(xml, mimetype="text/xml", status=200)

backend/models/postgis/custom_editors.py

@@ -23,7 +23,7 @@ def save(self):
     @staticmethod
     def get_by_project_id(project_id: int):
         """Get custom editor by it's project id"""
-        return CustomEditor.query.get(project_id)
+        return db.session.get(CustomEditor, project_id)
 
     @classmethod
     def create_from_dto(cls, project_id: int, dto: CustomEditorDTO):

backend/models/postgis/interests.py

@@ -30,7 +30,7 @@ class Interest(db.Model):
     @staticmethod
     def get_by_id(interest_id: int):
         """Get interest by id"""
-        interest = Interest.query.get(interest_id)
+        interest = db.session.get(Interest, interest_id)
         if interest is None:
             raise NotFound(f"Interest id {interest_id} not found")
 

backend/models/postgis/licenses.py

@@ -29,7 +29,7 @@ class License(db.Model):
     @staticmethod
     def get_by_id(license_id: int):
         """Get license by id"""
-        map_license = License.query.get(license_id)
+        map_license = db.session.get(License, license_id)
 
         if map_license is None:
             raise NotFound()

backend/models/postgis/mapping_issues.py

@@ -20,7 +20,7 @@ def __init__(self, name):
     @staticmethod
     def get_by_id(category_id: int):
         """Get category by id"""
-        return MappingIssueCategory.query.get(category_id)
+        return db.session.get(MappingIssueCategory, category_id)
 
     @classmethod
     def create_from_dto(cls, dto: MappingIssueCategoryDTO) -> int:

backend/models/postgis/organisation.py

@@ -131,7 +131,7 @@ def get(organisation_id: int):
         :param organisation_id: organisation ID in scope
         :return: Organisation if found otherwise None
         """
-        return Organisation.query.get(organisation_id)
+        return db.session.get(Organisation, organisation_id)
 
     @staticmethod
     def get_organisation_by_name(organisation_name: str):

backend/models/postgis/priority_area.py

@@ -29,10 +29,9 @@ def from_dict(cls, area_poly: dict):
         if type(pa_geojson) is not geojson.Polygon:
             raise InvalidGeoJson("Priority Areas must be supplied as Polygons")
 
-        is_valid_geojson = geojson.is_valid(pa_geojson)
-        if is_valid_geojson["valid"] == "no":
+        if not pa_geojson.is_valid:
             raise InvalidGeoJson(
-                f"Priority Area: Invalid Polygon - {is_valid_geojson['message']}"
+                "Priority Area: Invalid Polygon - " + ", ".join(pa_geojson.errors())
             )
 
         pa = cls()
@@ -42,5 +41,6 @@ def from_dict(cls, area_poly: dict):
 
     def get_as_geojson(self):
         """Helper to translate geometry back to a GEOJson Poly"""
-        pa_geojson = db.engine.execute(self.geometry.ST_AsGeoJSON()).scalar()
+        with db.engine.connect() as conn:
+            pa_geojson = conn.execute(self.geometry.ST_AsGeoJSON()).scalar()
         return geojson.loads(pa_geojson)