Add celery (#643)

* basis celery settings

* added lint to bootstrap test. Fixed imports

* fix some comments

* moved to rabbit

* build with optional push

* better optional push

* simplified ci

* removed redundant steps

* Revert "moved to rabbit"

This reverts commit 1dc88f5

* bump build-push-action

* drop CELERY_TASK_ALWAYS_EAGER from .env.ci

* fixed image name in ci

* better compose

* fixed toml-sort

.github/workflows/ci.yml

@@ -72,7 +72,7 @@ jobs:
         uses: docker/setup-buildx-action@v3
 
       - name: make sure docker image is buildable
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           build-args: |
             PYTHON_VERSION=${{ env.python-version }}

hooks/post_gen_project.sh

@@ -10,4 +10,5 @@ poetry run python src/manage.py startapp some_app
 poetry run python src/manage.py makemigrations -n "initial"
 poetry run python src/manage.py migrate
 
+make lint
 make test

{{ cookiecutter.name }}/.github/workflows/ci.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - main
   pull_request:
 
 jobs:
@@ -81,14 +82,53 @@ jobs:
       - name: setup buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: make sure docker image is buildable
-        uses: docker/build-push-action@v5
+      - name: Generate image identifier
+        id: image-identifier
+        uses: ASzc/change-string-case-action@v6
         with:
+          string: ${{ github.repository_owner }}
+
+      - name: Build web backend image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          target: web
+          push: false
+          tags: |
+            ghcr.io/${{ steps.image-identifier.outputs.lowercase }}-backend:latest
+            ghcr.io/${{ steps.image-identifier.outputs.lowercase }}-backend:${{ github.sha }}
+          build-args: |
+            PYTHON_VERSION=${{ env.python-version }}
+            RELEASE=${{ github.sha }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Build web worker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          target: worker
+          push: false
+          tags: |
+            ghcr.io/${{ steps.image-identifier.outputs.lowercase }}-worker:latest
+            ghcr.io/${{ steps.image-identifier.outputs.lowercase }}-worker:${{ github.sha }}
           build-args: |
             PYTHON_VERSION=${{ env.python-version }}
+            RELEASE=${{ github.sha }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
+
+      - name: Build web scheduler image
+        uses: docker/build-push-action@v6
+        with:
           context: .
-          # make sure you log to the container registry before pushing
-          # see https://github.com/docker/login-action
+          target: scheduler
           push: false
+          tags: |
+            ghcr.io/${{ steps.image-identifier.outputs.lowercase }}-scheduler:latest
+            ghcr.io/${{ steps.image-identifier.outputs.lowercase }}-scheduler:${{ github.sha }}
+          build-args: |
+            PYTHON_VERSION=${{ env.python-version }}
+            RELEASE=${{ github.sha }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

{{ cookiecutter.name }}/Dockerfile

@@ -27,8 +27,7 @@ RUN poetry export --format=requirements.txt > requirements.txt
 # Base image with django dependecines
 #
 FROM python:${PYTHON_VERSION}-slim-bookworm as base
-LABEL maintainer="{{ cookiecutter.email }}"
-LABEL com.datadoghq.ad.logs='[{"service": "django", "source": "uwsgi"}]'
+LABEL maintainer="[email protected]"
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV PYTHONUNBUFFERED 1
@@ -53,3 +52,25 @@ RUN ./manage.py collectstatic --noinput
 
 FROM base as web
 CMD ./manage.py migrate && uwsgi --master --http :8000 --module app.wsgi --workers 2 --threads 2 --harakiri 25 --max-requests 1000 --log-x-forwarded-for
+
+
+FROM base as worker
+
+ENV _CELERY_APP=app.celery
+HEALTHCHECK --interval=15s --timeout=15s --start-period=5s --retries=3 \
+  CMD celery -A ${_CELERY_APP} inspect ping -d celery@$HOSTNAME
+
+CMD celery -A ${_CELERY_APP} worker -c ${CONCURENCY:-2} -n "celery@%h" --max-tasks-per-child ${MAX_REQUESTS_PER_CHILD:-50} --time-limit ${TIME_LIMIT:-900} --soft-time-limit ${SOFT_TIME_LIMIT:-45}
+
+
+FROM base as scheduler
+
+ENV _SCHEDULER_DB_PATH=/var/db/scheduler
+USER root
+RUN mkdir -p ${_SCHEDULER_DB_PATH} && chown nobody ${_SCHEDULER_DB_PATH}
+VOLUME ${_SCHEDULER_DB_PATH}
+USER nobody
+
+ENV _CELERY_APP=app.celery
+HEALTHCHECK NONE
+CMD celery -A ${_CELERY_APP} beat --pidfile=/tmp/celerybeat.pid --schedule=${_SCHEDULER_DB_PATH}/celerybeat-schedule.db

{{ cookiecutter.name }}/docker-compose.yml

@@ -13,3 +13,36 @@ services:
     image: redis:6-alpine
     ports:
       - 6379:6379
+
+  backend:
+    build:
+      context: .
+      target: web
+      args: &default-build-args
+        PYTHON_VERSION: "3.11.6"
+    environment: &default-environment
+      DATABASE_URL: postgres://postgres@postgres:5432/postgres
+      CELERY_BROKER_URL: redis://redis:6379/0
+    ports:
+      - 8000:8000
+    depends_on: &default-depends_on
+      - postgres
+      - redis
+
+  worker:
+    build:
+      context: .
+      target: worker
+      args: *default-build-args
+    environment: *default-environment
+    depends_on: *default-depends_on
+
+
+  scheduler:
+    build:
+      context: .
+      target: scheduler
+      args: *default-build-args
+    environment: *default-environment
+    depends_on: *default-depends_on
+

{{ cookiecutter.name }}/mypy.ini

@@ -49,3 +49,5 @@ ignore_missing_imports = on
 [mypy-ipware.*]
 ignore_missing_imports = on
 
+[mypy-celery.*]
+ignore_missing_imports = on

{{ cookiecutter.name }}/pyproject.toml

@@ -3,15 +3,16 @@ build-backend = "poetry.core.masonry.api"
 requires = ["poetry-core"]
 
 [tool.poetry]
-authors = ["{{ cookiecutter.author }} <{{ cookiecutter.email }}>"]
-description = "{{ cookiecutter.description }}"
-name = "{{ cookiecutter.name }}"
+authors = ["Fedor Borshev <[email protected]>"]
+description = ""
+name = "testproject"
 package-mode = false
 readme = "README.md"
 version = "0.0.0-dev"
 
 [tool.poetry.dependencies]
 bcrypt = "^4.1.3"
+celery = "5.4.0"
 django = "^4.2.14"
 django-axes = "^6.5.1"
 django-behaviors = "^0.5.1"

{{ cookiecutter.name }}/src/app/.env.ci

@@ -1,3 +1,5 @@
 DATABASE_URL=postgres://postgres:@localhost:5432/postgres
 DEBUG=off
 SECRET_KEY={{ random_ascii_string(48, punctuation=True) }}
+
+CELERY_BROKER_URL=redis://localhost:6379/0

{{ cookiecutter.name }}/src/app/celery.py

@@ -0,0 +1,14 @@
+import os
+
+from celery import Celery
+from django.conf import settings
+
+__all__ = ["celery"]
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "app.settings")
+
+celery = Celery("app")
+celery.config_from_object("django.conf:settings", namespace="CELERY")
+celery.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
+
+celery.conf.beat_schedule = {}

{{ cookiecutter.name }}/src/app/conf/celery.py

@@ -0,0 +1,8 @@
+from app.conf.environ import env
+from app.conf.timezone import TIME_ZONE
+
+CELERY_BROKER_URL = env("CELERY_BROKER_URL", cast=str, default="redis://localhost:6379/0")
+CELERY_TASK_ALWAYS_EAGER = env("CELERY_TASK_ALWAYS_EAGER", cast=bool, default=env("DEBUG"))
+CELERY_TIMEZONE = TIME_ZONE
+CELERY_ENABLE_UTC = False
+CELERY_TASK_ACKS_LATE = True

{{ cookiecutter.name }}/src/app/settings.py

@@ -18,6 +18,7 @@
     "conf/auth.py",
     "conf/boilerplate.py",
     "conf/db.py",
+    "conf/celery.py",
     "conf/healthchecks.py",
     "conf/http.py",
     "conf/i18n.py",