From 3d012558b456bafd1091f64a6cadc35d07007466 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 16 May 2018 16:52:45 -0400 Subject: [PATCH 1/5] Split out configuration and execution of celery beat to be independent of the ordinary celery workers. This is because we generally only want to run one celery beat instance per environment. TEQ-20 --- CHANGES.rst | 42 ++++++++++++++++++++++++++ README.rst | 15 +++++++++- defaults/main.yml | 1 + tasks/celery.yml | 76 +++++++++++++++++++++++++++++++++-------------- tasks/main.yml | 4 +-- 5 files changed, 113 insertions(+), 25 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index d3afc4e..b7895d6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,6 +3,46 @@ Tequila-django Changes +v 0.9.n+1 on Month Day, Year +---------------------------- + +* Allow a celery beat instance to be deployed and configured + independently of the celery worker instances. This is because it is + desirable to have only one beat instance, generally. + + It is recommended to add a new group to your inventory files, + e.g. ``[beat]``, that has exactly one instance in it, which may be + an instance that is also in your ``[worker]`` group. You may then + create a new playbook, e.g. ``deployment/playbooks/beat.yml``, + containing something like:: + + --- + - hosts: beat + become: yes + roles: + - { role: tequila-django, is_celery_beat: true } + + If you go this route, do not forget to add this new beat.yml file to + the list invoked by your site.yml playbook. Note that if you + incorporate this change this way, the tequila-django role will be + executed multiple times on instances that incorporate these + differing variants. Alternatively, one can fold together the + invocation of tequila-django into a single playbook that uses group + checking to set the parameters used, like so:: + + --- + - hosts: web:worker:beat + become: yes + roles: + - role: tequila-django + is_web: "{{ 'web' in group_names }}" + is_worker: "{{ 'worker' in group_names }}" + is_celery_beat: "{{ 'beat' in group_names }}" + + This has the advantage of requiring only one loop through the + tequila-django tasks per server instance. + + v 0.9.n on Month Day, Year -------------------------- @@ -16,6 +56,7 @@ v 0.9.n on Month Day, Year command introduced in Celery 3.1, and by default it uses the ``django-celery-monitor`` app to capture worker events. + v 0.9.11 on March 19, 2018 -------------------------- @@ -41,6 +82,7 @@ v 0.9.11 on March 19, 2018 `_ section. + v 0.9.10 on Mar 2, 2018 ----------------------- diff --git a/README.rst b/README.rst index 31a8dd8..78f9b18 100644 --- a/README.rst +++ b/README.rst @@ -82,8 +82,10 @@ The following variables are used by the ``tequila-django`` role: - ``env_name`` **required** - ``domain`` **required** - ``additional_domains`` **default:** empty list -- ``is_web`` **default:** ``false`` (**required:** one of ``is_web`` or ``is_worker`` set to ``true``) +- ``is_web`` **default:** ``false`` (**required:** one of ``is_web`` + or ``is_worker`` or ``is_celery_beat`` set to ``true``) - ``is_worker`` **default:** ``false`` +- ``is_celery_beat`` **default:** ``false`` - ``python_version`` **default:** ``"2.7"`` - ``root_dir`` **default:** ``"/var/www/{{ project_name }}"`` - ``source_dir`` **default:** ``"{{ root_dir }}/src"`` @@ -151,6 +153,17 @@ for the presence of any packages in this variable and will throw an error if found. This behavior can be disabled by setting ``ignore_devdependencies`` to ``true``. +The ``is_celery_beat`` variable is used to specify which server +instance will run celery beat, a worker dedicated to running tasks +that are specified to execute at specific times. Generally, you only +want one instance running celery beat at a time, to prevent scheduled +tasks from attempting to be executed more than once. It is +recommended to set aside an inventory group, e.g. ``[beat]``, to +distinguish this instance from your ordinary celery workers in their +own group, e.g. ``[workers]``. Your playbook(s) may then set +``is_celery_beat``, ``is_worker``, and ``is_web`` based on the +instances' inventory group membership. + The ``celery_events`` and ``celery_camera_class`` variables are used to enable and configure Celery event monitoring using the "snapshots" system, which allows worker activity to be tracked in a less expensive diff --git a/defaults/main.yml b/defaults/main.yml index a8c7bde..1758ffe 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -19,6 +19,7 @@ use_newrelic: false source_is_local: false is_web: false is_worker: false +is_celery_beat: false ignore_devdependencies: false extra_env: {} celery_events: false diff --git a/tasks/celery.yml b/tasks/celery.yml index c15f3c8..83b711c 100644 --- a/tasks/celery.yml +++ b/tasks/celery.yml @@ -1,20 +1,37 @@ --- -- name: configure celery and celery beat - template: src=celery.conf - dest=/etc/supervisor/conf.d/{{ project_name }}-celery-{{ item.name }}.conf - owner=root - group=root - mode=0600 - with_items: - - { name: "default", command: "worker", flags: "{{ celery_worker_extra_args|default('--loglevel=INFO') }}{% if celery_events %} --events{% endif %}" } - - { name: "beat", command: "beat", flags: "--schedule={{ root_dir }}/celerybeat-schedule.db --pidfile=/var/run/celery/celerybeat.pid --loglevel=INFO" } +- name: configure celery + template: + src: celery.conf + dest: /etc/supervisor/conf.d/{{ project_name }}-celery-default.conf + owner: root + group: root + mode: 0600 + when: is_worker + vars: + name: "default" + command: "worker" + flags: "{{ celery_worker_extra_args|default('--loglevel=INFO') }}{% if celery_events %} --events{% endif %}" + +- name: configure celery beat + template: + src: celery.conf + dest: /etc/supervisor/conf.d/{{ project_name }}-celery-beat.conf + owner: root + group: root + mode: 0600 + when: is_celery_beat + vars: + name: "beat" + command: "beat" + flags: "--schedule={{ root_dir }}/celerybeat-schedule.db --pidfile=/var/run/celery/celerybeat.pid --loglevel=INFO" - name: configure celery events - template: src=celery.conf - dest=/etc/supervisor/conf.d/{{ project_name }}-celery-events.conf - owner=root - group=root - mode=0600 + template: + src: celery.conf + dest: /etc/supervisor/conf.d/{{ project_name }}-celery-events.conf + owner: root + group: root + mode: 0600 when: celery_events vars: name: "events" @@ -39,14 +56,29 @@ command: systemd-tmpfiles --create when: celery_tmpfile|changed -- name: ensure celery is present - supervisorctl: name={{ project_name }}-celery-{{ item }} state=present +- name: ensure celery events is present and restart + supervisorctl: + name: "{{ project_name }}-celery-events" + state: "{{ item }}" + when: celery_events + with_items: + - present + - restarted + +- name: ensure celery beat is present and restart + supervisorctl: + name: "{{ project_name }}-celery-beat" + state: "{{ item }}" + when: is_celery_beat with_items: - - default - - beat + - present + - restarted -- name: restart celery - supervisorctl: name={{ project_name }}-celery-{{ item }} state=restarted +- name: ensure celery is present and restart + supervisorctl: + name: "{{ project_name }}-celery-default" + state: "{{ item }}" + when: is_worker with_items: - - default - - beat + - present + - restarted diff --git a/tasks/main.yml b/tasks/main.yml index 815c3df..29e1266 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -201,10 +201,10 @@ ansible_ssh_pipelining: true - name: django server is either a web server or a worker - assert: { that: "is_web or is_worker" } + assert: { that: "is_web or is_worker or is_celery_beat" } - include: celery.yml - when: is_worker + when: is_worker or is_celery_beat - include: web.yml when: is_web From 2512e56c40ee362794c5442b60f6435e64bcd8d6 Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Wed, 26 Sep 2018 11:45:51 -0400 Subject: [PATCH 2/5] remove extra doc from bad merge --- README.rst | 8 -------- 1 file changed, 8 deletions(-) diff --git a/README.rst b/README.rst index aed5064..1076965 100644 --- a/README.rst +++ b/README.rst @@ -160,14 +160,6 @@ every web instance, since they'll be getting in each other's way. This variable set to ``true`` causes the ``collectstatic`` task to be run only once. -Due to `some `_ `issues -`_ discovered with -npm not managing package installation when new packages are added to -the ``devDependencies`` object in package.json, tequila-django checks -for the presence of any packages in this variable and will throw an -error if found. This behavior can be disabled by setting -``ignore_devdependencies`` to ``true``. - The ``is_celery_beat`` variable is used to specify which server instance will run celery beat, a worker dedicated to running tasks that are specified to execute at specific times. Generally, you only From 99700f0c77332c50f158da3889ad4f91c0117619 Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Wed, 26 Sep 2018 11:48:45 -0400 Subject: [PATCH 3/5] remove stray devdependencies ref --- defaults/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/defaults/main.yml b/defaults/main.yml index 944ab91..eafecb8 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -20,7 +20,6 @@ source_is_local: false is_web: false is_worker: false is_celery_beat: false -ignore_devdependencies: false extra_env: {} celery_events: false celery_camera_class: "django_celery_monitor.camera.Camera" From b5aa09f6f8e94a16e99eb9b9afe290bd9932e452 Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Wed, 26 Sep 2018 15:42:14 -0400 Subject: [PATCH 4/5] add playbook example to README --- README.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.rst b/README.rst index 1076965..73340c8 100644 --- a/README.rst +++ b/README.rst @@ -171,6 +171,18 @@ own group, e.g. ``[workers]``. Your playbook(s) may then set ``is_celery_beat``, ``is_worker``, and ``is_web`` based on the instances' inventory group membership. +One can fold together the invocation of tequila-django into a single playbook that uses group +checking to set the parameters used, like so:: + + --- + - hosts: web:worker:beat + become: yes + roles: + - role: tequila-django + is_web: "{{ 'web' in group_names }}" + is_worker: "{{ 'worker' in group_names }}" + is_celery_beat: "{{ 'beat' in group_names }}" + The ``celery_events`` and ``celery_camera_class`` variables are used to enable and configure Celery event monitoring using the "snapshots" system, which allows worker activity to be tracked in a less expensive From fb11cf4ba00e5c432bbf8a273c7348647d32b4aa Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Wed, 26 Sep 2018 15:45:54 -0400 Subject: [PATCH 5/5] update release notes to reference 0.9.18 --- CHANGES.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 77c25f3..2e21819 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,12 @@ Changes v 0.9.n+1 on Month Day, Year ---------------------------- +* TBD + + +v 0.9.18 on Sep 26, 2018 +---------------------------- + * Allow a celery beat instance to be deployed and configured independently of the celery worker instances. This is because it is desirable to have only one beat instance, generally.