Merge pull request #33 from caktus/one-celery-beat

One celery beat instance

CHANGES.rst

@@ -3,6 +3,51 @@ Tequila-django
 
 Changes
 
+v 0.9.n+1 on Month Day, Year
+----------------------------
+
+* TBD
+
+
+v 0.9.18 on Sep 26, 2018
+----------------------------
+
+* Allow a celery beat instance to be deployed and configured
+  independently of the celery worker instances.  This is because it is
+  desirable to have only one beat instance, generally.
+
+  It is recommended to add a new group to your inventory files,
+  e.g. ``[beat]``, that has exactly one instance in it, which may be
+  an instance that is also in your ``[worker]`` group.  You may then
+  create a new playbook, e.g. ``deployment/playbooks/beat.yml``,
+  containing something like::
+
+    ---
+    - hosts: beat
+      become: yes
+      roles:
+        - { role: tequila-django, is_celery_beat: true }
+
+  If you go this route, do not forget to add this new beat.yml file to
+  the list invoked by your site.yml playbook.  Note that if you
+  incorporate this change this way, the tequila-django role will be
+  executed multiple times on instances that incorporate these
+  differing variants.  Alternatively, one can fold together the
+  invocation of tequila-django into a single playbook that uses group
+  checking to set the parameters used, like so::
+
+    ---
+    - hosts: web:worker:beat
+      become: yes
+      roles:
+        - role: tequila-django
+          is_web: "{{ 'web' in group_names }}"
+          is_worker: "{{ 'worker' in group_names }}"
+          is_celery_beat: "{{ 'beat' in group_names }}"
+
+  This has the advantage of requiring only one loop through the
+  tequila-django tasks per server instance.
+
 v 0.9.17 on Sep 11, 2018
 ------------------------
 
@@ -82,6 +127,7 @@ v 0.9.11 on March 19, 2018
   <http://docs.celeryproject.org/en/latest/whatsnew-4.0.html#lowercase-setting-names>`_
   section.
 
+
 v 0.9.10 on Mar 2, 2018
 -----------------------
 

README.rst

@@ -91,8 +91,10 @@ The following variables are used by the ``tequila-django`` role:
 - ``env_name`` **required**
 - ``domain`` **required**
 - ``additional_domains`` **default:** empty list
-- ``is_web`` **default:** ``false`` (**required:** one of ``is_web`` or ``is_worker`` set to ``true``)
+- ``is_web`` **default:** ``false`` (**required:** one of ``is_web``
+  or ``is_worker`` or ``is_celery_beat`` set to ``true``)
 - ``is_worker`` **default:** ``false``
+- ``is_celery_beat`` **default:** ``false``
 - ``python_version`` **default:** ``"2.7"``
 - ``root_dir`` **default:** ``"/var/www/{{ project_name }}"``
 - ``source_dir`` **default:** ``"{{ root_dir }}/src"``
@@ -158,6 +160,29 @@ every web instance, since they'll be getting in each other's way.
 This variable set to ``true`` causes the ``collectstatic`` task to be
 run only once.
 
+The ``is_celery_beat`` variable is used to specify which server
+instance will run celery beat, a worker dedicated to running tasks
+that are specified to execute at specific times.  Generally, you only
+want one instance running celery beat at a time, to prevent scheduled
+tasks from attempting to be executed more than once.  It is
+recommended to set aside an inventory group, e.g. ``[beat]``, to
+distinguish this instance from your ordinary celery workers in their
+own group, e.g. ``[workers]``.  Your playbook(s) may then set
+``is_celery_beat``, ``is_worker``, and ``is_web`` based on the
+instances' inventory group membership.
+
+One can fold together the invocation of tequila-django into a single playbook that uses group
+checking to set the parameters used, like so::
+
+  ---
+  - hosts: web:worker:beat
+    become: yes
+    roles:
+      - role: tequila-django
+        is_web: "{{ 'web' in group_names }}"
+        is_worker: "{{ 'worker' in group_names }}"
+        is_celery_beat: "{{ 'beat' in group_names }}"
+
 The ``celery_events`` and ``celery_camera_class`` variables are used
 to enable and configure Celery event monitoring using the "snapshots"
 system, which allows worker activity to be tracked in a less expensive

defaults/main.yml

@@ -19,6 +19,7 @@ use_newrelic: false
 source_is_local: false
 is_web: false
 is_worker: false
+is_celery_beat: false
 extra_env: {}
 celery_events: false
 celery_camera_class: "django_celery_monitor.camera.Camera"

tasks/celery.yml

@@ -1,32 +1,37 @@
 ---
 - name: configure celery
-  template: src=celery.conf
-            dest=/etc/supervisor/conf.d/{{ project_name }}-celery-{{ name }}.conf
-            owner=root
-            group=root
-            mode=0600
+  template:
+    src: celery.conf
+    dest: /etc/supervisor/conf.d/{{ project_name }}-celery-default.conf
+    owner: root
+    group: root
+    mode: 0600
+  when: is_worker
   vars:
     name: "default"
     command: "worker"
     flags: "{{ celery_worker_extra_args|default('--loglevel=INFO') }}{% if celery_events %} --events{% endif %}"
 
 - name: configure celery beat
-  template: src=celery.conf
-            dest=/etc/supervisor/conf.d/{{ project_name }}-celery-{{ name }}.conf
-            owner=root
-            group=root
-            mode=0600
+  template:
+    src: celery.conf
+    dest: /etc/supervisor/conf.d/{{ project_name }}-celery-beat.conf
+    owner: root
+    group: root
+    mode: 0600
+  when: is_celery_beat
   vars:
     name: "beat"
     command: "beat"
     flags: "--schedule={{ root_dir }}/celerybeat-schedule.db --pidfile=/var/run/celery/celerybeat.pid --loglevel=INFO"
 
 - name: configure celery events
-  template: src=celery.conf
-            dest=/etc/supervisor/conf.d/{{ project_name }}-celery-events.conf
-            owner=root
-            group=root
-            mode=0600
+  template:
+    src: celery.conf
+    dest: /etc/supervisor/conf.d/{{ project_name }}-celery-events.conf
+    owner: root
+    group: root
+    mode: 0600
   when: celery_events
   vars:
     name: "events"
@@ -51,14 +56,29 @@
   command: systemd-tmpfiles --create
   when: celery_tmpfile is changed
 
-- name: ensure celery is present
-  supervisorctl: name={{ project_name }}-celery-{{ item }} state=present
+- name: ensure celery events is present and restart
+  supervisorctl:
+    name: "{{ project_name }}-celery-events"
+    state: "{{ item }}"
+  when: celery_events
+  with_items:
+    - present
+    - restarted
+
+- name: ensure celery beat is present and restart
+  supervisorctl:
+    name: "{{ project_name }}-celery-beat"
+    state: "{{ item }}"
+  when: is_celery_beat
   with_items:
-    - default
-    - beat
+    - present
+    - restarted
 
-- name: restart celery
-  supervisorctl: name={{ project_name }}-celery-{{ item }} state=restarted
+- name: ensure celery is present and restart
+  supervisorctl:
+    name: "{{ project_name }}-celery-default"
+    state: "{{ item }}"
+  when: is_worker
   with_items:
-    - default
-    - beat
+    - present
+    - restarted

tasks/main.yml

@@ -223,10 +223,10 @@
     ansible_ssh_pipelining: true
 
 - name: django server is either a web server or a worker
-  assert: { that: "is_web or is_worker" }
+  assert: { that: "is_web or is_worker or is_celery_beat" }
 
 - include: celery.yml
-  when: is_worker
+  when: is_worker or is_celery_beat
 
 - include: web.yml
   when: is_web