From cad52d235a31ae1fbd904d2c580faefb06166137 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 24 Aug 2023 20:31:08 +0300 Subject: [PATCH 1/6] PMM-12221 User isn't logged out during upgrade anymore. --- .../playbook/tasks/files/maintenance.html | 13 ++ update/ansible/playbook/tasks/init.yml | 2 +- .../tasks/roles/initialization/tasks/main.yml | 9 ++ .../tasks/roles/nginx/files/conf.d/pmm.conf | 23 ++++ update/ansible/playbook/tasks/update.yml | 122 +++++++----------- 5 files changed, 95 insertions(+), 74 deletions(-) create mode 100644 update/ansible/playbook/tasks/files/maintenance.html diff --git a/update/ansible/playbook/tasks/files/maintenance.html b/update/ansible/playbook/tasks/files/maintenance.html new file mode 100644 index 0000000000..7521e39c31 --- /dev/null +++ b/update/ansible/playbook/tasks/files/maintenance.html @@ -0,0 +1,13 @@ + + + + + + + Maintenance Mode + + +

We're Currently upgrading PMM

+

We apologize for the inconvenience. Please check back later.

+ + \ No newline at end of file diff --git a/update/ansible/playbook/tasks/init.yml b/update/ansible/playbook/tasks/init.yml index 828b8055c6..5c27f29b2d 100644 --- a/update/ansible/playbook/tasks/init.yml +++ b/update/ansible/playbook/tasks/init.yml @@ -4,6 +4,6 @@ become: true gather_facts: true tasks: - - name: Run initializaion role + - name: Run initialization role include_role: name: initialization diff --git a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml index 888d71ef64..8cb4186e4b 100644 --- a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml @@ -86,6 +86,15 @@ state: present when: not ansible_check_mode +- name: Create working directory for Alertmanager + file: path=/srv/alertmanager/data state=directory owner=pmm group=pmm + +- name: Create working directory for VictoriaMetrics + file: path=/srv/victoriametrics/data state=directory owner=pmm group=pmm + +- name: Create empty configuration file for VictoriaMetrics + file: path=/etc/victoriametrics-promscrape.yml state=touch owner=pmm group=pmm + - name: Run SQLite -> Postgres only for docker upgrade block: - name: Check that the SQLite grafana database exists diff --git a/update/ansible/playbook/tasks/roles/nginx/files/conf.d/pmm.conf b/update/ansible/playbook/tasks/roles/nginx/files/conf.d/pmm.conf index 4b136d2ce3..81768fa357 100644 --- a/update/ansible/playbook/tasks/roles/nginx/files/conf.d/pmm.conf +++ b/update/ansible/playbook/tasks/roles/nginx/files/conf.d/pmm.conf @@ -43,6 +43,29 @@ ssl_trusted_certificate /srv/nginx/ca-certs.pem; ssl_dhparam /srv/nginx/dhparam.pem; + # this block checks for maintenance.html file and if exists redirects all requests to maintenance page + # there are two exceptions for it /v1/Updates/Status and /auth_request endpoints + set $skip_maintenance 0; + + if (-f /usr/share/pmm-server/maintenance/maintenance.html) { + set $skip_maintenance 1; + } + + if ($request_uri ~* "^/v1/Updates/Status|^/auth_request") { + set $skip_maintenance 0; + } + + if ($skip_maintenance = 1) { + return 503; + } + + error_page 503 @maintenance; + + location @maintenance { + root /usr/share/pmm-server/maintenance; + rewrite ^(.*)$ /maintenance.html break; + } + # Enable passing of the remote user's IP address to all # proxied services using the X-Forwarded-For header diff --git a/update/ansible/playbook/tasks/update.yml b/update/ansible/playbook/tasks/update.yml index df4cd7b7c2..19826fb88c 100644 --- a/update/ansible/playbook/tasks/update.yml +++ b/update/ansible/playbook/tasks/update.yml @@ -39,12 +39,25 @@ is_docker: True when: is_docker is undefined tasks: - # we need install this package here because it contain VERSION file + - name: Enable maintenance mode + copy: + src: maintenance.html + dest: /usr/share/pmm-server/maintenance/ + mode: 0644 + + # we need to install this package here because it contains VERSION file - name: Remove percona-dashboard without architecture yum: name: percona-dashboards.*noarch state: absent + # see https://jira.percona.com/browse/PMM-8492 for details about a issue + - name: Delete experimental repo file in 2.16 version + file: + path: "/etc/yum.repos.d/percona-original-experimental.repo" + state: absent + register: experimental_repo_existed + - name: Update percona-dashboards package yum: name: @@ -52,6 +65,34 @@ - percona-grafana state: latest + - name: Cleanup yum metadata + command: yum clean metadata + register: yum_clean_result + changed_when: "'Cleaning repos' in yum_clean_result.stdout" + when: experimental_repo_existed.changed + tags: + - skip_ansible_lint # '503 Tasks that run when changed should likely be handlers'. + # The handler looks bad in this case + + # TODO: join with the command above + - name: Cleanup yum metadata + command: yum clean metadata + become: true + tags: + - skip_ansible_lint + + # Split download and update to produce a bit more of progress output. + - name: Download pmm2 packages + yum: + name: "{{ pmm_packages }}" + state: latest + download_only: yes + + - name: Update pmm2 packages + yum: + name: "{{ pmm_packages }}" + state: latest + - name: Create supervisord dir file: path: /etc/supervisord.d/ @@ -62,27 +103,12 @@ src: pmm.ini dest: /etc/supervisord.d/pmm.ini + # restart pmm-managed-init and pmm-managed first as they may update supervisord configuration on start - name: Generate new supervisor config command: pmm-managed-init register: managed_init_result changed_when: True - - name: Update postgresql version_1 - command: sed -i 's:-D /srv/postgres$:-D /srv/postgres14:' /etc/supervisord.d/pmm.ini - changed_when: true - - - name: Update postgresql version_2 - command: sed -i 's:/usr/pgsql/bin/postgres:/usr/pgsql-14/bin/postgres:' /etc/supervisord.d/pmm.ini - changed_when: true - - - name: Update postgresql version_3 - command: sed -i 's:/srv/logs/postgresql.log:/srv/logs/postgresql14.log:' /etc/supervisord.d/pmm.ini - changed_when: true - - - name: Update postgresql version_4 - command: sed -i 's:/usr/pgsql-11/bin/postgres:/usr/pgsql-14/bin/postgres:' /etc/supervisord.d/pmm.ini - changed_when: true - - name: Disable pmm-update-perform-init ini_file: path: /etc/supervisord.d/pmm.ini @@ -195,48 +221,6 @@ - name: Check reread results debug: var=reread_result.stdout_lines - # see https://jira.percona.com/browse/PMM-8492 for details about a issue - - name: Delete experimental repo file in 2.16 version - file: - path: "/etc/yum.repos.d/percona-original-experimental.repo" - state: absent - register: experimental_repo_existed - - - name: Cleanup yum metadata - command: yum clean metadata - register: yum_clean_result - changed_when: "'Cleaning repos' in yum_clean_result.stdout" - when: experimental_repo_existed.changed - tags: - - skip_ansible_lint # '503 Tasks that run when changed should likely be handlers'. - # The handler looks bad in this case - - # TODO: join with the command above - - name: Cleanup yum metadata - command: yum clean metadata - become: true - tags: - - skip_ansible_lint - - # Split download and update to produce a bit more of progress output. - - name: Download pmm2 packages - yum: - name: "{{ pmm_packages }}" - state: latest - download_only: yes - - - name: Update pmm2 packages - yum: - name: "{{ pmm_packages }}" - state: latest - - - # restart pmm-managed-init and pmm-managed first as they may update supervisord configuration on start - - name: Generate new supervisor config - command: pmm-managed-init - register: managed_init_result - changed_when: True - - name: Restart pmm-managed EL7 when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' command: supervisorctl {{ item }} pmm-managed @@ -254,7 +238,7 @@ # give pmm-managed time to update supervisord configuration, # and give update UI time to catch up after pmm-managed restart - name: Wait for pmm-managed - pause: seconds=5 + pause: seconds=10 - name: Update system packages yum: @@ -325,19 +309,6 @@ regexp: "set -o errexit" replace: "" - # TODO: Create /srv/alertmanager/data and /srv/victoriametrics/data in pmm-managed - # to support Docker-way (swap image) updates. - # https://jira.percona.com/browse/PMM-7024 - - - name: Create working directory for Alertmanager - file: path=/srv/alertmanager/data state=directory owner=pmm group=pmm - - - name: Create working directory for VictoriaMetrics - file: path=/srv/victoriametrics/data state=directory owner=pmm group=pmm - - - name: Create empty configuration file for VictoriaMetrics - file: path=/etc/victoriametrics-promscrape.yml state=touch owner=pmm group=pmm - - name: Reread supervisord configuration again EL7 when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' command: supervisorctl reread @@ -460,6 +431,11 @@ register: maintail_result changed_when: False + - name: Disable maintenance mode + file: + state: absent + path: /usr/share/pmm-server/maintenance/maintenance.html + - name: Print supervisord logs debug: var=maintail_result.stdout_lines From 3ffa52bcb61690e71b1ce835bf314c1e7383d8d9 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 25 Aug 2023 01:11:18 +0300 Subject: [PATCH 2/6] Update update/ansible/playbook/tasks/roles/nginx/files/conf.d/pmm.conf Co-authored-by: Alex Tymchuk --- update/ansible/playbook/tasks/roles/nginx/files/conf.d/pmm.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update/ansible/playbook/tasks/roles/nginx/files/conf.d/pmm.conf b/update/ansible/playbook/tasks/roles/nginx/files/conf.d/pmm.conf index 81768fa357..af66541da7 100644 --- a/update/ansible/playbook/tasks/roles/nginx/files/conf.d/pmm.conf +++ b/update/ansible/playbook/tasks/roles/nginx/files/conf.d/pmm.conf @@ -43,7 +43,7 @@ ssl_trusted_certificate /srv/nginx/ca-certs.pem; ssl_dhparam /srv/nginx/dhparam.pem; - # this block checks for maintenance.html file and if exists redirects all requests to maintenance page + # this block checks for maintenance.html file and, if it exists, it redirects all requests to the maintenance page # there are two exceptions for it /v1/Updates/Status and /auth_request endpoints set $skip_maintenance 0; From 7407f9cc77661ee87adae2e09538fb558738fdf7 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 25 Aug 2023 01:11:35 +0300 Subject: [PATCH 3/6] Update update/ansible/playbook/tasks/update.yml Co-authored-by: Alex Tymchuk --- update/ansible/playbook/tasks/update.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/update/ansible/playbook/tasks/update.yml b/update/ansible/playbook/tasks/update.yml index 19826fb88c..db484e0cae 100644 --- a/update/ansible/playbook/tasks/update.yml +++ b/update/ansible/playbook/tasks/update.yml @@ -45,7 +45,6 @@ dest: /usr/share/pmm-server/maintenance/ mode: 0644 - # we need to install this package here because it contains VERSION file - name: Remove percona-dashboard without architecture yum: name: percona-dashboards.*noarch From 5537e23abaffcc6be699fce7d26177291bed4d88 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Tue, 29 Aug 2023 15:32:40 +0300 Subject: [PATCH 4/6] Update update/ansible/playbook/tasks/files/maintenance.html Co-authored-by: Alex Tymchuk --- update/ansible/playbook/tasks/files/maintenance.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/update/ansible/playbook/tasks/files/maintenance.html b/update/ansible/playbook/tasks/files/maintenance.html index 7521e39c31..0ae49a8998 100644 --- a/update/ansible/playbook/tasks/files/maintenance.html +++ b/update/ansible/playbook/tasks/files/maintenance.html @@ -7,7 +7,7 @@ Maintenance Mode -

We're Currently upgrading PMM

-

We apologize for the inconvenience. Please check back later.

+

PMM is being upgraded

+

Please wait until the upgrade process is finished.

\ No newline at end of file From 2e75f24ae555f3e3abccf05de8297b28a5b88069 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 30 Aug 2023 12:50:35 +0300 Subject: [PATCH 5/6] Update update/ansible/playbook/tasks/files/maintenance.html --- update/ansible/playbook/tasks/files/maintenance.html | 1 + 1 file changed, 1 insertion(+) diff --git a/update/ansible/playbook/tasks/files/maintenance.html b/update/ansible/playbook/tasks/files/maintenance.html index 0ae49a8998..8056396816 100644 --- a/update/ansible/playbook/tasks/files/maintenance.html +++ b/update/ansible/playbook/tasks/files/maintenance.html @@ -4,6 +4,7 @@ + Maintenance Mode From 2d9e0dd57dee203123953c3d3a1e4962d4c7d8f3 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 30 Aug 2023 13:13:41 +0300 Subject: [PATCH 6/6] PMM-12221 rename variable. --- .../playbook/tasks/roles/nginx/files/conf.d/pmm.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/update/ansible/playbook/tasks/roles/nginx/files/conf.d/pmm.conf b/update/ansible/playbook/tasks/roles/nginx/files/conf.d/pmm.conf index af66541da7..dfdbfe8f11 100644 --- a/update/ansible/playbook/tasks/roles/nginx/files/conf.d/pmm.conf +++ b/update/ansible/playbook/tasks/roles/nginx/files/conf.d/pmm.conf @@ -45,17 +45,17 @@ # this block checks for maintenance.html file and, if it exists, it redirects all requests to the maintenance page # there are two exceptions for it /v1/Updates/Status and /auth_request endpoints - set $skip_maintenance 0; + set $maintenance_mode 0; if (-f /usr/share/pmm-server/maintenance/maintenance.html) { - set $skip_maintenance 1; + set $maintenance_mode 1; } if ($request_uri ~* "^/v1/Updates/Status|^/auth_request") { - set $skip_maintenance 0; + set $maintenance_mode 0; } - if ($skip_maintenance = 1) { + if ($maintenance_mode = 1) { return 503; }