From 52df9991c7647e8bb070f3f73cc406a636204077 Mon Sep 17 00:00:00 2001 From: Saurabh Date: Wed, 10 Jul 2024 18:39:38 +0530 Subject: [PATCH 1/2] feat: support for custom queues and workers --- press/docker/config/supervisor.conf | 28 +++++++++++++++++++ .../deploy_candidate/deploy_candidate.py | 20 +++++++++++-- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/press/docker/config/supervisor.conf b/press/docker/config/supervisor.conf index 1670ba9263..7387cdebe4 100644 --- a/press/docker/config/supervisor.conf +++ b/press/docker/config/supervisor.conf @@ -163,6 +163,27 @@ process_name=%(program_name)s-%(process_num)d {% endif %} +{% if custom_workers_config %} + +{% for worker_name, worker_details in custom_workers_config.items() %} +[program:-frappe-bench-{{ worker_name }}-worker] +command=bench worker --queue {{ worker_name }} +priority=4 +autostart=true +autorestart=true +stdout_logfile=/home/frappe/frappe-bench/logs/{{ worker_name }}.log +stderr_logfile=/home/frappe/frappe-bench/logs/{{ worker_name }}.error.log +user=frappe +stopwaitsecs={{ worker_details["timeout"] }} +directory=/home/frappe/frappe-bench +killasgroup=true +numprocs=1 +process_name=%(program_name)s-%(process_num)d +{% endfor %} + +{% endif %} + + [program:frappe-bench-redis-cache] command=redis-server /home/frappe/frappe-bench/config/redis-cache.conf priority=1 @@ -213,6 +234,13 @@ programs=frappe-bench-frappe-schedule,frappe-bench-frappe-default-worker,frappe- {% endif %} +{% if custom_workers_config %} + +[group:frappe-bench-custom_workers] +programs={%- for worker_name in workers -%}frappe-bench-{{ worker_name }}-worker,{%- endfor %} + +{% endif %} + [group:frappe-bench-redis] programs=frappe-bench-redis-cache,frappe-bench-redis-queue diff --git a/press/press/doctype/deploy_candidate/deploy_candidate.py b/press/press/doctype/deploy_candidate/deploy_candidate.py index 979456cccc..6e3eefbf06 100644 --- a/press/press/doctype/deploy_candidate/deploy_candidate.py +++ b/press/press/doctype/deploy_candidate/deploy_candidate.py @@ -859,6 +859,8 @@ def _update_app_releases(self) -> None: app.pullable_release = release_pair["new"]["name"] def _prepare_build_context(self, no_push: bool): + rg: ReleaseGroup = frappe.get_cache_doc("Release Group", self.group) + repo_path_map = self._clone_repos() pmf = get_package_manager_files(repo_path_map) self._run_prebuild_validations_and_update_step(pmf) @@ -881,7 +883,7 @@ def _prepare_build_context(self, no_push: bool): self._copy_config_files() self._generate_redis_cache_config() - self._generate_supervisor_config() + self._generate_supervisor_config(rg) self._generate_apps_txt() self.generate_ssh_keys() @@ -1158,13 +1160,25 @@ def _generate_redis_cache_config(self): content = frappe.render_template(redis_cache_conf_template, {"doc": self}, is_path=True) f.write(content) - def _generate_supervisor_config(self): + def _generate_supervisor_config(self, rg): supervisor_conf = os.path.join(self.build_directory, "config", "supervisor.conf") with open(supervisor_conf, "w") as f: supervisor_conf_template = "press/docker/config/supervisor.conf" - content = frappe.render_template(supervisor_conf_template, {"doc": self}, is_path=True) + custom_workers_config = self._get_custom_workers(rg) + content = frappe.render_template( + supervisor_conf_template, + {"doc": self, "custom_workers_config": custom_workers_config}, + is_path=True, + ) f.write(content) + def _get_custom_workers(self, rg): + if rg.common_site_config: + common_site_config = json.loads(self.common_site_config) or frappe._dict() + + if "workers" in common_site_config: + return common_site_config["workers"] + def _generate_apps_txt(self): apps_txt = os.path.join(self.build_directory, "apps.txt") with open(apps_txt, "w") as f: From b00381318468d066d56eb72c05359e93d1067a43 Mon Sep 17 00:00:00 2001 From: Saurabh Date: Wed, 28 Aug 2024 12:41:07 +0530 Subject: [PATCH 2/2] fix: worker group string --- press/docker/config/supervisor.conf | 10 ++++----- .../deploy_candidate/deploy_candidate.py | 22 ++++++++++++++----- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/press/docker/config/supervisor.conf b/press/docker/config/supervisor.conf index 7387cdebe4..d1b61b35b4 100644 --- a/press/docker/config/supervisor.conf +++ b/press/docker/config/supervisor.conf @@ -163,9 +163,9 @@ process_name=%(program_name)s-%(process_num)d {% endif %} -{% if custom_workers_config %} +{% if custom_workers %} -{% for worker_name, worker_details in custom_workers_config.items() %} +{% for worker_name, config in custom_workers.items() %} [program:-frappe-bench-{{ worker_name }}-worker] command=bench worker --queue {{ worker_name }} priority=4 @@ -174,7 +174,7 @@ autorestart=true stdout_logfile=/home/frappe/frappe-bench/logs/{{ worker_name }}.log stderr_logfile=/home/frappe/frappe-bench/logs/{{ worker_name }}.error.log user=frappe -stopwaitsecs={{ worker_details["timeout"] }} +stopwaitsecs={{ config["timeout"] }} directory=/home/frappe/frappe-bench killasgroup=true numprocs=1 @@ -234,10 +234,10 @@ programs=frappe-bench-frappe-schedule,frappe-bench-frappe-default-worker,frappe- {% endif %} -{% if custom_workers_config %} +{% if custom_workers_group %} [group:frappe-bench-custom_workers] -programs={%- for worker_name in workers -%}frappe-bench-{{ worker_name }}-worker,{%- endfor %} +programs={{ custom_workers_group}} {% endif %} diff --git a/press/press/doctype/deploy_candidate/deploy_candidate.py b/press/press/doctype/deploy_candidate/deploy_candidate.py index 6e3eefbf06..758a965474 100644 --- a/press/press/doctype/deploy_candidate/deploy_candidate.py +++ b/press/press/doctype/deploy_candidate/deploy_candidate.py @@ -1164,20 +1164,32 @@ def _generate_supervisor_config(self, rg): supervisor_conf = os.path.join(self.build_directory, "config", "supervisor.conf") with open(supervisor_conf, "w") as f: supervisor_conf_template = "press/docker/config/supervisor.conf" - custom_workers_config = self._get_custom_workers(rg) + custom_workers = self._get_custom_workers(rg) + content = frappe.render_template( supervisor_conf_template, - {"doc": self, "custom_workers_config": custom_workers_config}, + { + "doc": self, + "custom_workers": custom_workers, + "custom_workers_group": self._get_custom_workers_group(custom_workers), + }, is_path=True, ) f.write(content) def _get_custom_workers(self, rg): if rg.common_site_config: - common_site_config = json.loads(self.common_site_config) or frappe._dict() + common_site_config = json.loads(rg.common_site_config) or frappe._dict() + return common_site_config.get("workers", frappe._dict()) + + return frappe._dict() - if "workers" in common_site_config: - return common_site_config["workers"] + def _get_custom_workers_group(self, custom_workers): + group = [] + if custom_workers: + for worker_name in custom_workers: + group.append(f"frappe-bench-{ worker_name }-worker") + return ", ".join(group) def _generate_apps_txt(self): apps_txt = os.path.join(self.build_directory, "apps.txt")