diff --git a/CHANGES.rst b/CHANGES.rst index 908ef36..2e21819 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,6 +3,51 @@ Tequila-django Changes +v 0.9.n+1 on Month Day, Year +---------------------------- + +* TBD + + +v 0.9.18 on Sep 26, 2018 +---------------------------- + +* Allow a celery beat instance to be deployed and configured + independently of the celery worker instances. This is because it is + desirable to have only one beat instance, generally. + + It is recommended to add a new group to your inventory files, + e.g. ``[beat]``, that has exactly one instance in it, which may be + an instance that is also in your ``[worker]`` group. You may then + create a new playbook, e.g. ``deployment/playbooks/beat.yml``, + containing something like:: + + --- + - hosts: beat + become: yes + roles: + - { role: tequila-django, is_celery_beat: true } + + If you go this route, do not forget to add this new beat.yml file to + the list invoked by your site.yml playbook. Note that if you + incorporate this change this way, the tequila-django role will be + executed multiple times on instances that incorporate these + differing variants. Alternatively, one can fold together the + invocation of tequila-django into a single playbook that uses group + checking to set the parameters used, like so:: + + --- + - hosts: web:worker:beat + become: yes + roles: + - role: tequila-django + is_web: "{{ 'web' in group_names }}" + is_worker: "{{ 'worker' in group_names }}" + is_celery_beat: "{{ 'beat' in group_names }}" + + This has the advantage of requiring only one loop through the + tequila-django tasks per server instance. + v 0.9.17 on Sep 11, 2018 ------------------------ @@ -82,6 +127,7 @@ v 0.9.11 on March 19, 2018 `_ section. + v 0.9.10 on Mar 2, 2018 ----------------------- diff --git a/README.rst b/README.rst index dbed27c..73340c8 100644 --- a/README.rst +++ b/README.rst @@ -91,8 +91,10 @@ The following variables are used by the ``tequila-django`` role: - ``env_name`` **required** - ``domain`` **required** - ``additional_domains`` **default:** empty list -- ``is_web`` **default:** ``false`` (**required:** one of ``is_web`` or ``is_worker`` set to ``true``) +- ``is_web`` **default:** ``false`` (**required:** one of ``is_web`` + or ``is_worker`` or ``is_celery_beat`` set to ``true``) - ``is_worker`` **default:** ``false`` +- ``is_celery_beat`` **default:** ``false`` - ``python_version`` **default:** ``"2.7"`` - ``root_dir`` **default:** ``"/var/www/{{ project_name }}"`` - ``source_dir`` **default:** ``"{{ root_dir }}/src"`` @@ -158,6 +160,29 @@ every web instance, since they'll be getting in each other's way. This variable set to ``true`` causes the ``collectstatic`` task to be run only once. +The ``is_celery_beat`` variable is used to specify which server +instance will run celery beat, a worker dedicated to running tasks +that are specified to execute at specific times. Generally, you only +want one instance running celery beat at a time, to prevent scheduled +tasks from attempting to be executed more than once. It is +recommended to set aside an inventory group, e.g. ``[beat]``, to +distinguish this instance from your ordinary celery workers in their +own group, e.g. ``[workers]``. Your playbook(s) may then set +``is_celery_beat``, ``is_worker``, and ``is_web`` based on the +instances' inventory group membership. + +One can fold together the invocation of tequila-django into a single playbook that uses group +checking to set the parameters used, like so:: + + --- + - hosts: web:worker:beat + become: yes + roles: + - role: tequila-django + is_web: "{{ 'web' in group_names }}" + is_worker: "{{ 'worker' in group_names }}" + is_celery_beat: "{{ 'beat' in group_names }}" + The ``celery_events`` and ``celery_camera_class`` variables are used to enable and configure Celery event monitoring using the "snapshots" system, which allows worker activity to be tracked in a less expensive diff --git a/defaults/main.yml b/defaults/main.yml index 2bb3290..eafecb8 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -19,6 +19,7 @@ use_newrelic: false source_is_local: false is_web: false is_worker: false +is_celery_beat: false extra_env: {} celery_events: false celery_camera_class: "django_celery_monitor.camera.Camera" diff --git a/tasks/celery.yml b/tasks/celery.yml index 0c882b1..8b29a92 100644 --- a/tasks/celery.yml +++ b/tasks/celery.yml @@ -1,32 +1,37 @@ --- - name: configure celery - template: src=celery.conf - dest=/etc/supervisor/conf.d/{{ project_name }}-celery-{{ name }}.conf - owner=root - group=root - mode=0600 + template: + src: celery.conf + dest: /etc/supervisor/conf.d/{{ project_name }}-celery-default.conf + owner: root + group: root + mode: 0600 + when: is_worker vars: name: "default" command: "worker" flags: "{{ celery_worker_extra_args|default('--loglevel=INFO') }}{% if celery_events %} --events{% endif %}" - name: configure celery beat - template: src=celery.conf - dest=/etc/supervisor/conf.d/{{ project_name }}-celery-{{ name }}.conf - owner=root - group=root - mode=0600 + template: + src: celery.conf + dest: /etc/supervisor/conf.d/{{ project_name }}-celery-beat.conf + owner: root + group: root + mode: 0600 + when: is_celery_beat vars: name: "beat" command: "beat" flags: "--schedule={{ root_dir }}/celerybeat-schedule.db --pidfile=/var/run/celery/celerybeat.pid --loglevel=INFO" - name: configure celery events - template: src=celery.conf - dest=/etc/supervisor/conf.d/{{ project_name }}-celery-events.conf - owner=root - group=root - mode=0600 + template: + src: celery.conf + dest: /etc/supervisor/conf.d/{{ project_name }}-celery-events.conf + owner: root + group: root + mode: 0600 when: celery_events vars: name: "events" @@ -51,14 +56,29 @@ command: systemd-tmpfiles --create when: celery_tmpfile is changed -- name: ensure celery is present - supervisorctl: name={{ project_name }}-celery-{{ item }} state=present +- name: ensure celery events is present and restart + supervisorctl: + name: "{{ project_name }}-celery-events" + state: "{{ item }}" + when: celery_events + with_items: + - present + - restarted + +- name: ensure celery beat is present and restart + supervisorctl: + name: "{{ project_name }}-celery-beat" + state: "{{ item }}" + when: is_celery_beat with_items: - - default - - beat + - present + - restarted -- name: restart celery - supervisorctl: name={{ project_name }}-celery-{{ item }} state=restarted +- name: ensure celery is present and restart + supervisorctl: + name: "{{ project_name }}-celery-default" + state: "{{ item }}" + when: is_worker with_items: - - default - - beat + - present + - restarted diff --git a/tasks/main.yml b/tasks/main.yml index fd29d56..2623704 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -223,10 +223,10 @@ ansible_ssh_pipelining: true - name: django server is either a web server or a worker - assert: { that: "is_web or is_worker" } + assert: { that: "is_web or is_worker or is_celery_beat" } - include: celery.yml - when: is_worker + when: is_worker or is_celery_beat - include: web.yml when: is_web