diff --git a/.ansible-lint b/.ansible-lint
index 5ce0b4f..ff3d85e 100644
--- a/.ansible-lint
+++ b/.ansible-lint
@@ -1,2 +1,3 @@
 skip_list:
   - 106
+  - 503
diff --git a/handlers/main.yml b/handlers/main.yml
deleted file mode 100644
index 453d514..0000000
--- a/handlers/main.yml
+++ /dev/null
@@ -1,24 +0,0 @@
----
-- name: dirsrv restart "{{ dirsrv_serverid }}"
-  service:
-    name: "dirsrv@{{ dirsrv_serverid }}"
-    state: restarted
-
-- name: dirsrv schema reload "{{ dirsrv_serverid }}"
-  ldap_entry:
-    # This runs after dirsrv restart, if a restart is needed,
-    # so TLS status enforcing may have changed.
-    # Restarting the server also reloads the schema, but I see no easy way to
-    # avoid this task if the other handler has been executed...
-    server_uri: "{{ dirsrv_server_uri }}"
-    validate_certs: "{{ dirsrv_tls_certificate_trusted }}"
-    start_tls: "{{ dirsrv_tls_enforced }}"
-    bind_dn: "{{ dirsrv_rootdn }}"
-    bind_pw: "{{ dirsrv_rootdn_password }}"
-    dn: "cn=ansible-managed schema reload,cn=schema reload task,cn=tasks,cn=config"
-    objectClass:
-      - extensibleObject
-    attributes:
-      cn: ansible-managed schema reload
-      schemadir: "/etc/dirsrv/slapd-{{ dirsrv_serverid }}/schema/"
-    state: present
diff --git a/tasks/configure_authentication.yml b/tasks/configure_authentication.yml
index c5e6231..6caf8b6 100644
--- a/tasks/configure_authentication.yml
+++ b/tasks/configure_authentication.yml
@@ -29,7 +29,7 @@
         values: "{{ item.value }}"
         state: exact
       loop: "{{ dirsrv_ldapi_config }}"
-      notify: dirsrv restart "{{ dirsrv_serverid }}"
+      register: dirsrv_restart_condition_auth_1
 
   rescue:
     - name: Configure LDAPI over LDAPI
@@ -42,7 +42,7 @@
         values: "{{ item.value }}"
         state: exact
       loop: "{{ dirsrv_ldapi_config }}"
-      notify: dirsrv restart "{{ dirsrv_serverid }}"
+      register: dirsrv_restart_condition_auth_2
 
 # Documentation: https://directory.fedoraproject.org/docs/389ds/design/sasl-mechanism-configuration.html
 # TODO: "none" means "everything is allowed"... how to allow none (or just EXTERNAL that is always enabled but not always allowed or whatever?)
diff --git a/tasks/configure_plugins.yml b/tasks/configure_plugins.yml
index d9ed65b..9a92871 100644
--- a/tasks/configure_plugins.yml
+++ b/tasks/configure_plugins.yml
@@ -25,7 +25,7 @@
     - "cn=UID numbers,cn=Distributed Numeric Assignment Plugin,cn=plugins,cn=config"
     - "cn=GID numbers,cn=Distributed Numeric Assignment Plugin,cn=plugins,cn=config"
   when: "'Distributed Numeric Assignment Plugin' in dirsrv_plugins_enabled and not dirsrv_plugins_enabled['Distributed Numeric Assignment Plugin']"
-  notify: dirsrv restart "{{ dirsrv_serverid }}"
+  register: dirsrv_restart_condition_plugins_1
 
 - name: Manage the dna shared ranges OUs, required when we have a replica
   block:
@@ -101,7 +101,7 @@
       loop:
         - "cn=UID numbers,cn=Distributed Numeric Assignment Plugin,cn=plugins,cn=config"
         - "cn=GID numbers,cn=Distributed Numeric Assignment Plugin,cn=plugins,cn=config"
-      notify: dirsrv restart "{{ dirsrv_serverid }}"
+      register: dirsrv_restart_condition_plugins_2
 
     - name: Configure DNA plugin parameters (UID)
       ldap_attr:
@@ -123,7 +123,7 @@
         - { name: "dnanextvalue", value: "{{ dirsrv_dna_plugin.uid_min }}" }
         - { name: "dnamaxvalue", value: "{{ dirsrv_dna_plugin.uid_max }}" }
         - { name: "dnasharedcfgdn", value: "cn=Account UIDs,ou=Ranges,{{ dirsrv_suffix }}" }
-      notify: dirsrv restart "{{ dirsrv_serverid }}"
+      register: dirsrv_restart_condition_plugins_3
 
     - name: Configure DNA plugin parameters (GID)
       ldap_attr:
@@ -145,6 +145,6 @@
         - { name: "dnanextvalue", value: "{{ dirsrv_dna_plugin.gid_min }}" }
         - { name: "dnamaxvalue", value: "{{ dirsrv_dna_plugin.gid_max }}" }
         - { name: "dnasharedcfgdn", value: "cn=Account GIDs,ou=Ranges,{{ dirsrv_suffix }}" }
-      notify: dirsrv restart "{{ dirsrv_serverid }}"
+      register: dirsrv_restart_condition_plugins_4
 
   when: "'Distributed Numeric Assignment Plugin' in dirsrv_plugins_enabled and dirsrv_plugins_enabled['Distributed Numeric Assignment Plugin']"
diff --git a/tasks/configure_schema.yml b/tasks/configure_schema.yml
index 3f64ca9..3dd0ec8 100644
--- a/tasks/configure_schema.yml
+++ b/tasks/configure_schema.yml
@@ -9,7 +9,7 @@
     group: dirsrv
   loop: "{{ dirsrv_custom_schema }}"
   tags: [ dirsrv_schema ]
-  notify: dirsrv schema reload "{{ dirsrv_serverid }}"
+  register: dirsrv_restart_condition_schema_reload_1
 
 - name: Search other schema files
   find:
@@ -25,6 +25,6 @@
     path: "{{ item.path }}"
     state: absent
   loop: "{{ dirsrv_found_files['files'] }}"
-  notify: dirsrv schema reload "{{ dirsrv_serverid }}"
+  register: dirsrv_restart_condition_schema_reload_2
   tags: [ dirsrv_schema ]
   when: not dirsrv_allow_other_schema_files
diff --git a/tasks/configure_tls.yml b/tasks/configure_tls.yml
index 39b0fd5..c674f08 100644
--- a/tasks/configure_tls.yml
+++ b/tasks/configure_tls.yml
@@ -167,7 +167,7 @@
         {{ dirsrv_pkutil_authflag }}
         -W ''
       when: dirsrv_nss_key_count_too_many
-      notify: dirsrv restart "{{ dirsrv_serverid }}"
+      register: dirsrv_restart_condition_tls_1
 
       # I *think* this may be needed for self-signed certificates, but not entirely sure...
       # - name: Set certificate parameters
@@ -233,7 +233,7 @@
     state: absent
   when: not dirsrv_tls_enabled
   tags: [ dirsrv_tls ]
-  notify: dirsrv restart "{{ dirsrv_serverid }}"
+  register: dirsrv_restart_condition_tls_2
 
 - name: Create RSA configuration for TLS (if enabled)
   ldap_entry:
@@ -249,7 +249,7 @@
     state: present
   when: dirsrv_tls_enabled | bool
   tags: [ dirsrv_tls ]
-  notify: dirsrv restart "{{ dirsrv_serverid }}"
+  register: dirsrv_restart_condition_tls_3
 
 - name: Configure RSA parameters
   ldap_attr:
@@ -269,7 +269,7 @@
     - { name: "nsSSLActivation", value: "on" }
   when: dirsrv_tls_enabled | bool
   tags: [ dirsrv_tls ]
-  notify: dirsrv restart "{{ dirsrv_serverid }}"
+  register: dirsrv_restart_condition_tls_4
 
 # Note: https://www.port389.org/docs/389ds/howto/howto-ssl.html says
 # that other settings will not activate if nsslapd-security is off.
@@ -313,7 +313,7 @@
     - { name: "nsssl2", value: "off" }
     - { name: "nsssl3", value: "off" }
     - { name: "sslVersionMin", value: "TLS{{ dirsrv_tls_min_version }}" }
-  notify: dirsrv restart "{{ dirsrv_serverid }}"
+  register: dirsrv_restart_condition_tls_5
   when: dirsrv_tls_enabled | bool
   tags: [ dirsrv_tls ]
 
@@ -363,6 +363,6 @@
     name: "nsslapd-securelistenhost"
     values: "{{ dirsrv_secure_listen_host }}"
     state: exact
-  notify: dirsrv restart "{{ dirsrv_serverid }}"
+  register: dirsrv_restart_condition_tls_6
   when: dirsrv_secure_listen_host != None
   tags: [ dirsrv_tls ]
diff --git a/tasks/configure_tls_enforcing.yml b/tasks/configure_tls_enforcing.yml
index fff6275..79a238c 100644
--- a/tasks/configure_tls_enforcing.yml
+++ b/tasks/configure_tls_enforcing.yml
@@ -16,7 +16,7 @@
         - { name: "nsslapd-minssf", value: "{{ dirsrv_tls_minssf if dirsrv_tls_enabled and dirsrv_tls_enforced else '0' }}" }
       failed_when: false
       tags: [ dirsrv_tls ]
-      notify: dirsrv restart "{{ dirsrv_serverid }}"
+      register: dirsrv_restart_condition_tls_enforcing_1
 
   rescue:
     - name: Configure enforcing of TLS, over TLS
@@ -34,4 +34,4 @@
         - { name: "nsslapd-require-secure-binds", value: "{{ 'on' if dirsrv_tls_enabled and dirsrv_tls_enforced else 'off' }}" }
         - { name: "nsslapd-minssf", value: "{{ dirsrv_tls_minssf if dirsrv_tls_enabled and dirsrv_tls_enforced else '0' }}" }
       tags: [ dirsrv_tls ]
-      notify: dirsrv restart "{{ dirsrv_serverid }}"
+      register: dirsrv_restart_condition_tls_enforcing_2
diff --git a/tasks/main.yml b/tasks/main.yml
index 2008916..1b2bd07 100644
--- a/tasks/main.yml
+++ b/tasks/main.yml
@@ -60,7 +60,7 @@
     name: "nsslapd-listenhost"
     values: "{{ dirsrv_listen_host }}"
     state: exact
-  notify: dirsrv restart "{{ dirsrv_serverid }}"
+  register: dirsrv_restart_condition_main
   when: dirsrv_listen_host != None
 
 - include: configure_authentication.yml
@@ -75,14 +75,53 @@
 
 - include: configure_tls.yml
 
-# Restart server and/or reload schema, to enable TLS and schema changes.
-# Required to enforce TLS as TLS must be enabled beforehand (requiring a restart).
-- meta: flush_handlers
-  tags: [ dirsrv_tls, dirsrv_cert, dirsrv_schema ]
-
+# Required also to enforce TLS as TLS must be enabled beforehand (requiring a restart).
+- name: "Restart {{ dirsrv_serverid }} to enable TLS, authentication, plugins"
+  service:
+    name: "dirsrv@{{ dirsrv_serverid }}"
+    state: restarted
+  # Cannot use an handler due to https://github.com/lvps/389ds-server/issues/16
+  # Cannot use use any fancy woarkaround due to https://github.com/ansible/ansible/issues/63822
+  # Cannot use a generic name for the handler since https://github.com/lvps/389ds-server/pull/14, it *has* to be tied to the server
+  when: "(dirsrv_restart_condition_main is defined and dirsrv_restart_condition_main.changed) or\
+        (dirsrv_restart_condition_plugins_1 is defined and dirsrv_restart_condition_plugins_1.changed) or\
+        (dirsrv_restart_condition_plugins_2 is defined and dirsrv_restart_condition_plugins_2.changed) or\
+        (dirsrv_restart_condition_plugins_3 is defined and dirsrv_restart_condition_plugins_3.changed) or\
+        (dirsrv_restart_condition_plugins_4 is defined and dirsrv_restart_condition_plugins_4.changed) or\
+        (dirsrv_restart_condition_auth_1 is defined and dirsrv_restart_condition_auth_1.changed) or\
+        (dirsrv_restart_condition_auth_2 is defined and dirsrv_restart_condition_auth_2.changed) or\
+        (dirsrv_restart_condition_tls_1 is defined and dirsrv_restart_condition_tls_1.changed) or\
+        (dirsrv_restart_condition_tls_2 is defined and dirsrv_restart_condition_tls_2.changed) or\
+        (dirsrv_restart_condition_tls_3 is defined and dirsrv_restart_condition_tls_3.changed) or\
+        (dirsrv_restart_condition_tls_4 is defined and dirsrv_restart_condition_tls_4.changed) or\
+        (dirsrv_restart_condition_tls_5 is defined and dirsrv_restart_condition_tls_5.changed) or\
+        (dirsrv_restart_condition_tls_6 is defined and dirsrv_restart_condition_tls_6.changed)"
+  register: dirsrv_restart_condition_has_restarted
+
+- name: "Reload schema on {{ dirsrv_serverid }}"
+  ldap_entry:
+    server_uri: "{{ dirsrv_server_uri }}"
+    validate_certs: "{{ dirsrv_tls_certificate_trusted }}"
+    start_tls: "{{ dirsrv_tls_enforced }}"
+    bind_dn: "{{ dirsrv_rootdn }}"
+    bind_pw: "{{ dirsrv_rootdn_password }}"
+    dn: "cn=ansible-managed schema reload,cn=schema reload task,cn=tasks,cn=config"
+    objectClass:
+      - extensibleObject
+    attributes:
+      cn: ansible-managed schema reload
+      schemadir: "/etc/dirsrv/slapd-{{ dirsrv_serverid }}/schema/"
+    state: present
+  when: "not dirsrv_restart_condition_has_restarted and (\
+        (dirsrv_restart_condition_schema_reload_1 is defined and dirsrv_restart_condition_schema_reload_1.changed) or\
+        (dirsrv_restart_condition_schema_reload_2 is defined and dirsrv_restart_condition_schema_reload_2.changed)\
+        )"
 - include: configure_tls_enforcing.yml
 
-# Restart server to enforce TLS.
 # This only happens if TLS is being enforced.
-- meta: flush_handlers
-  tags: [ dirsrv_tls, dirsrv_cert, dirsrv_schema ]
+- name: "Restart {{ dirsrv_serverid }} to enforce TLS"
+  service:
+    name: "dirsrv@{{ dirsrv_serverid }}"
+    state: restarted
+  when: "(dirsrv_restart_condition_tls_enforcing_1 is defined and dirsrv_restart_condition_tls_enforcing_1.changed) or\
+        (dirsrv_restart_condition_tls_enforcing_2 is defined and dirsrv_restart_condition_tls_enforcing_2.changed)"