Revert "PMM-12175 Revert #2017 Migrate Grafana to Postgres (#2272)" (#…

…2400) This reverts commit 8493ee0.
percona · Aug 8, 2023 · 1b80abb · 1b80abb
1 parent 6a12659
commit 1b80abb
Show file tree

Hide file tree

Showing 8 changed files with 250 additions and 8 deletions.
diff --git a/build/ansible/roles/pmm2-images/tasks/main.yml b/build/ansible/roles/pmm2-images/tasks/main.yml
@@ -29,7 +29,7 @@
 
 # local yum repo for building of pmm server docker image in autobuild jobs
 - name: PMM                        | Add local YUM repository
-  when: ansible_virtualization_type == "docker"
+  when: ansible_virtualization_type == "docker" 
   yum_repository:
     name: local
     description: Local YUM repository - x86_64
@@ -104,11 +104,11 @@
         group: "{{ item.group }}"
         non_unique: true
       loop:
-        - { name: pmm, uid: 1000, comment: "PMM Server", shell: "/bin/false", home: "/home/pmm", group: pmm }
-        - { name: nginx, uid: 999, comment: "nginx user", shell: "/sbin/nologin", home: "/var/cache/nginx", group: nginx }
-        - { name: grafana, uid: 998, comment: "Grafana Dashboard", shell: "/sbin/nologin", home: "/etc/grafana", group: grafana }
-        - { name: clickhouse, uid: 997, comment: "Clickhouse server", shell: "/sbin/nologin", home: "/var/lib/clickhouse", group: clickhouse }
-        - { name: pmm-agent, uid: 996, comment: "pmm-agent", shell: "/bin/false", home: "/usr/local/percona/", group: pmm-agent }
+        - { name: pmm, uid: 1000, comment: "PMM Server", shell: "/bin/false", home: "/home/pmm", group: pmm, }
+        - { name: nginx, uid: 999, comment: "nginx user", shell: "/sbin/nologin", home: "/var/cache/nginx", group: nginx, }
+        - { name: grafana, uid: 998, comment: "Grafana Dashboard", shell: "/sbin/nologin", home: "/etc/grafana", group: grafana, }
+        - { name: clickhouse, uid: 997, comment: "Clickhouse server", shell: "/sbin/nologin", home: "/var/lib/clickhouse", group: clickhouse, }
+        - { name: pmm-agent, uid: 996, comment: "pmm-agent", shell: "/bin/false", home: "/usr/local/percona/", group: pmm-agent, }
   when: ansible_virtualization_type == "docker"
 
 - name: Create directories        | Create dirs
@@ -149,6 +149,7 @@
       - pmm-update
       - dbaas-controller
       - dbaas-tools
+      - grafana-db-migrator
       - pmm-dump
       - vmproxy
     state: installed

diff --git a/build/packages/rpm/server/SPECS/grafana-db-migrator.spec b/build/packages/rpm/server/SPECS/grafana-db-migrator.spec
@@ -46,5 +46,4 @@ install -m 755 dist/grafana-db-migrator %{buildroot}%{_sbindir}/
 - Add fixes for CHAR fields
 
 * Tue Nov 02 2021 Nikita Beletskii <[email protected]> - 1.0.1-1
-- Creating package for grafana-db-migrator
-
+- Creating package for grafana-db-migrator
diff --git a/build/scripts/build-server-rpm-all b/build/scripts/build-server-rpm-all
@@ -12,11 +12,13 @@ ${bin_dir}/build-server-rpm pmm-update pmm
 ${bin_dir}/build-server-rpm dbaas-controller
 ${bin_dir}/build-server-rpm dbaas-tools
 ${bin_dir}/build-server-rpm pmm-dump
+${bin_dir}/build-server-rpm grafana-db-migrator
 ${bin_dir}/build-server-rpm vmproxy pmm
 
 # 3rd-party
 ${bin_dir}/build-server-rpm victoriametrics
 ${bin_dir}/build-server-rpm alertmanager
 ${bin_dir}/build-server-rpm grafana
+# ${bin_dir}/build-server-rpm grafana-db-migrator
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/update/ansible/playbook/tasks/roles/grafana/files/grafana.ini b/update/ansible/playbook/tasks/roles/grafana/files/grafana.ini
@@ -1,6 +1,18 @@
 ##################### Grafana Configuration #####################
 # Only changed settings. You can find default settings in /usr/share/grafana/conf/defaults.ini
 
+#################################### Database ####################################
+[database]
+# You can configure the database connection by specifying type, host, name, user and password
+# as separate properties or as on string using the url properties.
+
+# Either "mysql", "postgres" or "sqlite3", it's your choice
+type = postgres
+host = localhost
+user = grafana
+# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;"""
+password = grafana
+
 [paths]
 # Directory where grafana will automatically scan and look for plugins
 plugins = /srv/grafana/plugins

diff --git a/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml b/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml
@@ -23,6 +23,42 @@
     group: grafana
     mode: "0444"
 
+- name: Check that the SQLite grafana database exists
+  stat:
+    path: /srv/grafana/grafana.db
+  register: sqlite_grafana
+
+- name: Temporary change database to SQLite
+  block:
+    - name: Remove database options (SQLite is default)
+      ini_file:
+        dest: /etc/grafana/grafana.ini
+        section: database
+        option: type
+        value: absent
+
+    - name: Remove database host
+      ini_file:
+        dest: /etc/grafana/grafana.ini
+        section: database
+        option: host
+        state: absent
+
+    - name: Remove database user
+      ini_file:
+        dest: /etc/grafana/grafana.ini
+        section: database
+        option: user
+        state: absent
+
+    - name: Remove database password
+      ini_file:
+        dest: /etc/grafana/grafana.ini
+        section: database
+        option: password
+        state: absent
+  when: sqlite_grafana.stat.exists
+
 - name: Create provisioning directory
   file:
     path: "/usr/share/grafana/conf/provisioning/{{ item }}"

diff --git a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml
@@ -1,5 +1,10 @@
 ---
 # This role contains tasks executed during initialization of PMM Server
+- name: Determine type of upgrade
+  set_fact:
+    ui_upgrade: False
+  when: ui_upgrade is undefined
+
 
 # PMM-10858 - In certain environments, including AWS EC2, some of the
 # EPEL repository mirrors do not respond within the time limit defined
@@ -65,3 +70,35 @@
   include_role:
     name: postgres
   when: is_postgres_11.stat.exists
+
+- name: Create grafana database in postgres
+  postgresql_db:
+    name: grafana
+    state: present
+
+- name: Create grafana user in postgres
+  postgresql_user:
+    db: grafana
+    name: grafana
+    password: grafana
+    priv: 'ALL'
+    expires: infinity
+    state: present
+  when: not ansible_check_mode
+
+- name: Run SQLite -> Postgres only for docker upgrade
+  block:
+    - name: Check that the SQLite grafana database exists
+      stat:
+        path: /srv/grafana/grafana.db
+      register: is_database_sqlite
+
+    - name: Migrate Grafana database from SQLite to Postgresql
+      include_role:
+        name: sqlite-to-postgres
+      when: is_database_sqlite.stat.exists
+      tags:
+        - skip_ansible_lint # '503 Tasks that run when changed should likely be handlers'.
+  # We use current_version_file['failed'] because we don't want to run this on creating container
+  when: not ui_upgrade and current_version_file['failed'] == false
+
diff --git a/update/ansible/playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml b/update/ansible/playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml
@@ -0,0 +1,122 @@
+---
+- name: Create Grafana backup dir
+  file:
+    path: "/srv/backup/grafana"
+    state: directory
+    owner: grafana
+    group: grafana
+    mode: '0700'
+
+- name: Stop grafana before upgrade
+  supervisorctl:
+    name: 'grafana'
+    state: stopped
+
+- name: Create backup for SQLite Grafana database
+  copy:
+    src: /srv/grafana/grafana.db
+    dest: "/srv/backup/grafana/grafana.db"
+    owner: grafana
+    group: grafana
+    mode: '0700'
+
+- name: Remove all ` symbols in grafana dashboard description
+  command: sqlite3 /srv/grafana/grafana.db -cmd ".timeout 60000" "UPDATE dashboard SET data = REPLACE(data, '`', '');"
+  changed_when: True
+
+- name: Disable provisioning before change database
+  ini_file:
+    dest: /etc/grafana/grafana.ini
+    section: paths
+    option: provisioning
+    value: conf/provisioning_disable
+
+- name: Switch to postgres
+  ini_file:
+    dest: /etc/grafana/grafana.ini
+    section: database
+    option: type
+    value: postgres
+
+- name: Set database host
+  ini_file:
+    dest: /etc/grafana/grafana.ini
+    section: database
+    option: host
+    value: localhost
+
+- name: Set database user
+  ini_file:
+    dest: /etc/grafana/grafana.ini
+    section: database
+    option: user
+    value: grafana
+
+- name: Set database password
+  ini_file:
+    dest: /etc/grafana/grafana.ini
+    section: database
+    option: password
+    value: grafana
+
+- name: Start grafana again
+  supervisorctl:
+    name: grafana
+    state: restarted
+  ignore_errors: yes
+
+- name: Check if initial data were created
+  postgresql_query:
+    db: grafana
+    query: SELECT 1 FROM org WHERE id=1
+  retries: 3
+  delay: 3
+  register: psql_result
+  until: psql_result.rowcount == 1
+  when: not ansible_check_mode
+
+- name: Wait for grafana database initialization
+  pause:
+    seconds: 10
+
+- name: Stop grafana before upgrade
+  supervisorctl:
+    name: grafana
+    state: stopped
+
+- name: Remove default admin user
+  postgresql_query:
+    db: grafana
+    query: DELETE FROM public.user WHERE login='admin'
+  when: not ansible_check_mode
+
+- name: Run grafana migrator
+  command: grafana-db-migrator --change-char-to-text /srv/grafana/grafana.db "postgres://grafana:grafana@localhost:5432/grafana?sslmode=disable"
+  register: migrator_output
+  changed_when: "'All done' in migrator_output.stdout"
+
+- name: Enable provisioning after change database
+  ini_file:
+    dest: /etc/grafana/grafana.ini
+    section: paths
+    option: provisioning
+    value: conf/provisioning
+
+- name: Start grafana again
+  supervisorctl:
+    name: grafana
+    state: restarted
+
+- name: Wait for grafana initialization
+  pause:
+    seconds: 5
+
+- name: Fix database/folder relationship
+  command: grafana-db-migrator --fix-folders-id /srv/grafana/grafana.db "postgres://grafana:grafana@localhost:5432/grafana?sslmode=disable"
+  register: migrator_output
+  changed_when: "'All done' in migrator_output.stdout"
+
+- name: Remove SQLite Grafana database
+  file:
+    path: /srv/grafana/grafana.db
+    state: absent
diff --git a/update/ansible/playbook/tasks/update.yml b/update/ansible/playbook/tasks/update.yml
@@ -20,6 +20,7 @@
       - pmm2-client
       - pmm-dump
       - vmproxy
+      - grafana-db-migrator
   pre_tasks:
     - name: detect /srv/pmm-distribution
       stat:
@@ -149,9 +150,16 @@
       when: is_docker and not is_supervisor_running.stat.exists and (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9'
       shell: /usr/local/bin/supervisord -c /etc/supervisord.conf &
 
+    - name: Wait until postgres port is present before continuing
+      wait_for:
+        host: localhost
+        port: 5432
+
     - name: Run initialization playbook
       include_role:
         name: initialization
+      vars:
+        ui_upgrade: True
 
     - name: Enable crond service
       when: not is_docker
@@ -415,6 +423,29 @@
     - name: Print other services's logs
       debug: var=update_result.stdout_lines
 
+    - name: Check that the SQLite grafana database exists
+      stat:
+        path: /srv/grafana/grafana.db
+      register: is_database_sqlite
+
+    - name: Migrate Grafana database from SQLite to Postgresql
+      include_role:
+        name: sqlite-to-postgres
+      when: is_database_sqlite.stat.exists
+      tags:
+        - skip_ansible_lint # '503 Tasks that run when changed should likely be handlers'.
+
+    - name: Fix grafana fields type
+      postgresql_query:
+        db: grafana
+        query: "{{ item }}"
+      loop:
+        - ALTER TABLE tag ALTER COLUMN key TYPE text;
+        - ALTER TABLE tag ALTER COLUMN value TYPE text;
+        - ALTER TABLE api_key ALTER COLUMN key TYPE text;
+        - ALTER TABLE api_key ALTER COLUMN name TYPE text;
+      when: not ansible_check_mode
+
     # SIGUSR2 is sent to supervisord by pmm-managed right before the update for logging to work correctly.
     # We use that fact to show what was restarted during the update.
     - name: Get supervisord logs EL7
@@ -436,3 +467,5 @@
       file:
         state: absent
         path: /var/cache/yum
+
+