Merge pull request #680 from Netcracker/fix/prepare_dns_etc_hosts

[CPDEV-103936] Fix prepare_dns_etc_hosts for work with unavailable nodes

documentation/Installation.md

@@ -548,6 +548,7 @@ The following parameters are supported:
 | `expect.pods.plugins.retries`    | int  | no        | 150           | `300`   | Number of retires to check pods readiness in `plugins`                                                             |
 | `nodes.ready.timeout`            | int  | no        | 5             | `10`    | Timeout between `nodes.ready.retries` for cluster node readiness waiting                                           |
 | `nodes.ready.retries`            | int  | no        | 15            | `60`    | Number of retries to check a cluster node readiness                                                                |
+| `ignore_unavailable_nodes_for_etchosts_update` | boolean  | no        | false            | false    | Allow to run `prepare.dns.etc_hosts`, `update.etc_hosts` tasks when some nodes are unavailable. **Do not change the default value unless you know what for!**                    |
 
 
 ### node_defaults

kubemarine/procedures/install.py

@@ -208,15 +208,26 @@ def system_prepare_dns_resolv_conf(group: NodeGroup) -> None:
 def system_prepare_dns_etc_hosts(cluster: KubernetesCluster) -> None:
     remained_offline = cluster.nodes['all'].get_online_nodes(False)
     if not remained_offline.is_empty():
-        raise Exception("Nodes %s are not reachable" % remained_offline.get_hosts())
+        if not cluster.inventory['globals']['ignore_unavailable_nodes_for_etchosts_update']:
+            # generally all the nodes must be available during this task running
+            raise Exception("Nodes %s are not reachable" % remained_offline.get_hosts())
+        else:
+            # in some cases we allow to update /etc/hosts only at the available nodes
+            cluster.log.warning("Nodes %s are not reachable. You can update their /etc/hosts by running install job "
+                "with the task prepare.dns.etc_hosts only when these nodes become available." % remained_offline.get_hosts())
 
     config = system.generate_etc_hosts_config(cluster.inventory, 'etc_hosts')
     config += system.generate_etc_hosts_config(cluster.inventory, 'etc_hosts_generated')
 
     utils.dump_file(cluster, config, 'etc_hosts')
     cluster.log.debug("\nUploading...")
 
-    group = cluster.nodes['all']
+    if not cluster.inventory['globals']['ignore_unavailable_nodes_for_etchosts_update']:
+        # generally we update /etc/hosts at all the nodes
+        group = cluster.nodes['all']
+    else:
+        # update /etc/hosts only at the available nodes
+        group = cluster.nodes['all'].get_online_nodes(True)
 
     system.update_etc_hosts(group, config=config)
     cluster.log.debug(group.sudo("ls -la /etc/hosts"))

kubemarine/resources/configurations/defaults.yaml

@@ -18,6 +18,7 @@ globals:
       retries: 15
   timeout_download:
     60
+  ignore_unavailable_nodes_for_etchosts_update: false
 
 node_defaults:
   boot:

kubemarine/resources/schemas/definitions/globals.json

@@ -82,6 +82,11 @@
       "type": "integer",
       "default": 60,
       "description": "Timeout for the thirdparties download on nodes."
+    },
+    "ignore_unavailable_nodes_for_etchosts_update": {
+      "type": "boolean",
+      "default": false,
+      "description": "Switcher to allow /etc/hosts update at the cluster nodes when there are unavailable nodes"
     }
   },
   "definitions": {