Merge pull request #7 from kartoza/feat-pyqgis

Feat pyqgis

.flake8

@@ -0,0 +1,9 @@
+[flake8]
+exclude = */docs/*,*/.tox/*,*/.venv/*,*/.pycharm_helpers/*,*/migrations/*,docs/*,fabfile.py,*/__init__.py,django_project/core/settings/*.py
+max-line-length = 79
+
+# E12x continuation line indentation
+# E251 no spaces around keyword / parameter equals
+# E303 too many blank lines (3)
+# E402 module level import not at top of file
+ignore = E125,E126,E251,E303,E402,W504,W60,F405

.github/workflows/build-and-test.yml

@@ -75,7 +75,7 @@ jobs:
           python manage.py collectstatic --noinput --verbosity 0
           export DJANGO_SETTINGS_MODULE=core.settings.test && coverage run manage.py test && coverage xml
           EOF
-          docker cp cplus_api_dev_django:/home/web/django_project/coverage.xml ../coverage.xml
+          docker cp cplus-api-dev-django:/home/web/django_project/coverage.xml ../coverage.xml
       - name: Show Coverage
         if: ${{ github.event_name == 'pull_request' }}
         uses: orgoro/coverage@v3

deployment/.template.env

@@ -7,4 +7,5 @@ DATABASE_HOST=db
 REDIS_HOST=redis
 REDIS_PASSWORD=redis_password
 RABBITMQ_HOST=rabbitmq
-SENTRY_DSN=sentry_dsn
+SENTRY_DSN=sentry_dsn
+STORAGE_EMULATOR_HOST=http://gcs:4443

deployment/.template.test.env

@@ -7,4 +7,5 @@ DATABASE_HOST=db
 REDIS_HOST=redis
 REDIS_PASSWORD=redis_password
 RABBITMQ_HOST=rabbitmq
-SENTRY_DSN=sentry_dsn
+SENTRY_DSN=sentry_dsn
+STORAGE_EMULATOR_HOST=http://gcs:4443

deployment/docker-compose.override.devcontainer.yml

@@ -38,6 +38,7 @@ services:
     volumes:
       - ../:/home/web/project
       - ../django_project:/home/web/django_project
+      - ./volumes/media:/home/web/media
     links:
       - gcs
 
@@ -58,12 +59,10 @@ services:
       dockerfile: deployment/docker/Dockerfile
       target: vscode
     entrypoint: []
-    environment:
-      - STORAGE_EMULATOR_HOST=http://gcs:4443
     volumes:
       - ../:/home/web/project
-      - ./volumes/static:/home/web/static
       - ./volumes/media:/home/web/media
+      - ./volumes/static:/home/web/static
     links:
       - db
       - worker

deployment/docker-compose.override.template.yml

@@ -19,32 +19,67 @@ services:
     volumes:
       - ../django_project:/home/web/django_project
       - ./volumes/static:/home/web/static
-      - ./volumes/media:/home/web/media
 
   celery_beat:
+    image: kartoza/${COMPOSE_PROJECT_NAME:-django_project}_worker_dev
+    build:
+      context: ../
+      dockerfile: deployment/docker/Dockerfile
+      target: worker
     volumes:
+      - ../:/home/web/project
       - ../django_project:/home/web/django_project
 
   worker:
+    image: kartoza/${COMPOSE_PROJECT_NAME:-django_project}_worker_dev
+    build:
+      context: ../
+      dockerfile: deployment/docker/Dockerfile
+      target: worker
     volumes:
+      - ../:/home/web/project
       - ../django_project:/home/web/django_project
+      - ./volumes/media:/home/web/media
 
   dev:
+    image: kartoza/${COMPOSE_PROJECT_NAME:-django_project}_dev
     build:
       context: ../
       dockerfile: deployment/docker/Dockerfile
       target: dev
+    entrypoint: []
     volumes:
       - ../django_project:/home/web/django_project
       - ./volumes/static:/home/web/static
       - ./volumes/media:/home/web/media
+    links:
+      - db
+      - worker
+      - gcs
 
   nginx:
     volumes:
       - ./nginx/sites-enabled:/etc/nginx/conf.d:ro
       - ./volumes/static:/home/web/static
-      - ./volumes/media:/home/web/media
     ports:
       - "${HTTP_PORT:-8888}:80"
     links:
       - django
+
+  gcs:
+    container_name: googlecloudstorage
+    image: fsouza/fake-gcs-server:1.47.8
+    command:
+    - '-scheme'
+    - 'http'
+    - '-port'
+    - '4443'
+    - '-public-host'
+    - '127.0.0.1:4443'
+    - '-external-url'
+    - 'http://127.0.0.1:4443'
+    volumes:
+      - ./volumes/gcs_data:/storage
+      - ./gcs_emulator:/data
+    ports:
+      - 4443:4443

deployment/docker-compose.override.test.yml

@@ -26,29 +26,20 @@ services:
       - 4443:4443
 
   worker:
-    image: kartoza/${COMPOSE_PROJECT_NAME:-django_project}_worker_dev
-    build:
-      context: ../
-      dockerfile: deployment/docker/Dockerfile
-      target: worker
+    image: kartoza/cplus-api:test
     volumes:
       - ../django_project:/home/web/django_project
       - ./volumes/media:/home/web/media
     links:
       - gcs
 
   dev:
-    image: kartoza/${COMPOSE_PROJECT_NAME:-django_project}_dev
-    build:
-      context: ../
-      dockerfile: deployment/docker/Dockerfile
-      target: dev
+    image: kartoza/cplus-api:test
     entrypoint: []
-    environment:
-      - STORAGE_EMULATOR_HOST=http://gcs:4443
     volumes:
       - ../django_project:/home/web/django_project
       - ./volumes/static:/home/web/static
+      - ./volumes/media:/home/web/media
     links:
       - db
       - worker

deployment/docker-compose.yml

@@ -1,38 +1,41 @@
 version: '3.9'
-name: 'cplus'
 
 volumes:
+  conf-data:
   static-data:
   media-data:
-  conf-data:
   database:
   nginx-cache:
   backups-data:
   data-volume:
 
+x-common-variables: &common-variables
+  # editable in .env
+  DATABASE_NAME: ${DATABASE_NAME:-django}
+  DATABASE_USERNAME: ${DATABASE_USERNAME:-docker}
+  DATABASE_PASSWORD: ${DATABASE_PASSWORD:-docker}
+  DATABASE_HOST: ${DATABASE_HOST:-db}
+  REDIS_HOST: ${REDIS_HOST:-redis}
+  REDIS_PASSWORD: ${REDIS_PASSWORD:-redis_password}
+  RABBITMQ_HOST: ${RABBITMQ_HOST:-rabbitmq}
+  DJANGO_SETTINGS_MODULE: ${DJANGO_SETTINGS_MODULE:-core.settings.prod}
+  INITIAL_FIXTURES: ${INITIAL_FIXTURES:-False}
+  CSRF_TRUSTED_ORIGINS: ${CSRF_TRUSTED_ORIGINS:-[]}
+  SENTRY_ENVIRONMENT: ${SENTRY_ENVIRONMENT:-production}
+  SENTRY_DSN: ${SENTRY_DSN:-}
+  # Email where alters should be sent. This will be used by let's encrypt and as the django admin email.
+  ADMIN_USERNAME: ${ADMIN_USERNAME:-admin}
+  ADMIN_PASSWORD: ${ADMIN_PASSWORD:-admin}
+  ADMIN_EMAIL: ${ADMIN_EMAIL:[email protected]}
+  # worker variables
+  CPLUS_QUEUE_CONCURRENCY: ${CPLUS_QUEUE_CONCURRENCY:-1}
+
+
 x-common-django:
   &default-common-django
   image: kartoza/${COMPOSE_PROJECT_NAME:-django_project}:${DJANGO_TAG:-1.0.0}
   environment:
-    # editable in .env
-    - DATABASE_NAME=${DATABASE_NAME:-django}
-    - DATABASE_USERNAME=${DATABASE_USERNAME:-docker}
-    - DATABASE_PASSWORD=${DATABASE_PASSWORD:-docker}
-    - DATABASE_HOST=${DATABASE_HOST:-db}
-    - REDIS_HOST=${REDIS_HOST:-redis}
-    - REDIS_PASSWORD=${REDIS_PASSWORD:-redis_password}
-    - RABBITMQ_HOST=${RABBITMQ_HOST:-rabbitmq}
-    - DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE:-core.settings.prod}
-    - INITIAL_FIXTURES=${INITIAL_FIXTURES:-False}
-    - CSRF_TRUSTED_ORIGINS=${CSRF_TRUSTED_ORIGINS:-[]}
-    - SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT:-production}
-    - SENTRY_DSN=${SENTRY_DSN:-}
-    # Email where alters should be sent. This will be used by let's encrypt and as the django admin email.
-    - ADMIN_USERNAME=${ADMIN_USERNAME:-admin}
-    - ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin}
-    - ADMIN_EMAIL=${ADMIN_EMAIL:[email protected]}
-    # worker variables
-    - CPLUS_QUEUE_CONCURRENCY=${CPLUS_QUEUE_CONCURRENCY:-1}
+    <<: *common-variables
   restart: on-failure
 
 services:
@@ -79,7 +82,6 @@ services:
     command: 'uwsgi --ini /uwsgi.conf'
     volumes:
       - static-data:/home/web/static
-      - media-data:/home/web/media
     links:
       - db
       - redis
@@ -99,8 +101,11 @@ services:
     container_name: "cplus-api-worker"
     entrypoint: []
     command: '/bin/bash -c /home/web/django_project/worker_entrypoint.sh'
-    links:
-      - celery_beat
+    environment:
+      <<: *common-variables
+      CPLUS_WORKER: 1
+    volumes:
+      - media-data:/home/web/media
 
   dev:
     image: kartoza/${COMPOSE_PROJECT_NAME:-django_project}_dev
@@ -121,8 +126,7 @@ services:
     hostname: nginx
     volumes:
       - conf-data:/etc/nginx/conf.d:ro
-      - static-data:/home/web/static
-      - media-data:/home/web/media
       - nginx-cache:/home/web/nginx_cache
+      - static-data:/home/web/static
     links:
       - django

deployment/docker/Dockerfile

@@ -1,12 +1,21 @@
-FROM python:3.12.0-slim-bookworm AS prod
+FROM python:3.11.8-slim-bookworm AS prod
 
 RUN apt-get update -y && \
     apt-get install -y --no-install-recommends \
     gcc gettext cron \
     spatialite-bin libsqlite3-mod-spatialite \
     python3-dev python3-gdal python3-psycopg2 python3-ldap \
     python3-pip python3-pil python3-lxml python3-pylibmc \
-    uwsgi uwsgi-plugin-python3
+    uwsgi uwsgi-plugin-python3 wget \
+    gnupg software-properties-common
+
+# qgis python3-qgis qgis-plugin-grass
+RUN mkdir -m755 -p /etc/apt/keyrings
+RUN wget -O /etc/apt/keyrings/qgis-archive-keyring.gpg https://download.qgis.org/downloads/qgis-archive-keyring.gpg
+ADD deployment/docker/qgis.sources /etc/apt/sources.list.d/qgis.sources
+
+RUN apt-get update -y && \
+    apt-get install -y qgis python-qgis qgis-plugin-grass
 
 # Install pip packages
 ADD deployment/docker/requirements.txt /requirements.txt
@@ -16,6 +25,9 @@ RUN pip3 install --upgrade pip && pip install --upgrade pip
 ARG CPUCOUNT=1
 RUN pip3 install -r /requirements.txt
 
+# Fix pyqgis missing core lib
+# RUN cp /usr/lib/python3/dist-packages/qgis/_core.cpython-311-x86_64-linux-gnu.so /usr/lib/python3/dist-packages/qgis/_core.so
+
 ADD django_project /home/web/django_project
 
 EXPOSE 8080

deployment/docker/qgis.sources

@@ -0,0 +1,6 @@
+Types: deb deb-src
+URIs: https://qgis.org/debian
+Suites: bookworm
+Architectures: amd64
+Components: main
+Signed-By: /etc/apt/keyrings/qgis-archive-keyring.gpg

deployment/docker/requirements.txt

@@ -47,3 +47,6 @@ django-storages[google]
 
 # drf yasg
 drf-yasg==1.21.7
+
+# fix pyqgis
+PyQt5-sip

django_project/core/celery.py

@@ -2,6 +2,7 @@
 
 import os
 import logging
+import sys
 from celery import Celery, signals
 from celery.utils.serialization import strtobool
 from celery.worker.control import inspect_command
@@ -32,25 +33,29 @@
 # celery-is-rerunning-long-running-completed-tasks-over-and-over
 app.conf.broker_transport_options = {'visibility_timeout': 3 * 3600}
 
+# use max task = 1 to avoid memory leak from qgis processing tools
+app.conf.worker_max_tasks_per_child = 1
+
 # ------------------------------------
 # Task event handlers
 # ------------------------------------
 
+
 @signals.after_task_publish.connect
 def task_sent_handler(sender=None, headers=None, body=None, **kwargs):
     # task is sent to celery, but might not be queued to worker yet
     info = headers if 'task' in headers else body
-    task_id = info['id']
-    task_args = info['argsrepr'] if 'argsrepr' in info else ''
+    # task_id = info['id']
+    # task_args = info['argsrepr'] if 'argsrepr' in info else ''
     if info['task'] in EXCLUDED_TASK_LIST:
         return
 
 
 @signals.task_received.connect
 def task_received_handler(sender, request=None, **kwargs):
     # task should be queued
-    task_id = request.id if request else None
-    task_args = request.args
+    # task_id = request.id if request else None
+    # task_args = request.args
     task_name = request.name if request else ''
     if task_name in EXCLUDED_TASK_LIST:
         return
@@ -70,7 +75,7 @@ def task_success_handler(sender, **kwargs):
     task_name = sender.name if sender else ''
     if task_name in EXCLUDED_TASK_LIST:
         return
-    task_id = sender.request.id
+    # task_id = sender.request.id
 
 
 @signals.task_failure.connect
@@ -86,7 +91,7 @@ def task_revoked_handler(sender, request = None, **kwargs):
     task_name = sender.name if sender else ''
     if task_name in EXCLUDED_TASK_LIST:
         return
-    task_id = request.id if request else None
+    # task_id = request.id if request else None
 
 
 @signals.task_internal_error.connect
@@ -122,6 +127,7 @@ def task_retry_handler(sender, reason, **kwargs):
 #     },
 # }
 
+
 @inspect_command(
     alias='dump_conf',
     signature='[include_defaults=False]',
@@ -134,3 +140,15 @@ def conf(state, with_defaults=False, **kwargs):
     (Celery makes an attempt to remove sensitive info,but it is not foolproof)
     """
     return {'error': 'Config inspection has been disabled.'}
+
+
+is_worker = os.environ.get('CPLUS_WORKER', 0)
+if is_worker:
+    # init qgis
+    sys.path.insert(0, '/usr/share/qgis/python/plugins')
+    sys.path.insert(0, '/usr/share/qgis/python')
+    sys.path.append('/usr/lib/python3/dist-packages')
+    os.environ["QT_QPA_PLATFORM"] = "offscreen"
+    from qgis.core import *  # noqa
+    QgsApplication.setPrefixPath("/usr/bin/qgis", True)
+    logger.info('*******QGIS INIT DONE*********')

django_project/core/models/preferences.py

@@ -1,6 +1,5 @@
 """Model for Website Preferences."""
 from django.db import models
-from django.utils.translation import gettext_lazy as _
 
 from core.models.singleton import SingletonModel