Merge pull request #400 from rundeck-plugins/issue_rpl-36_child-groups

RPL-36: Fix - Ansible process sub children

functional-test/src/test/groovy/functional/ChildGroupsSpec.groovy

@@ -0,0 +1,45 @@
+package functional
+
+import functional.base.BaseTestConfiguration
+import org.testcontainers.spock.Testcontainers
+
+@Testcontainers
+class ChildGroupsSpec extends BaseTestConfiguration {
+
+    static String NODENAME  = 'nodename'
+    static String HOSTNAME  = 'hostname'
+    static String TAGS      = 'tags'
+    static String PROJ_NAME = 'ansible-child-groups'
+    static String NODE_1    = 'one.example.com'
+    static String TAGS_1    = 'dbservers, east, prod'
+    static String NODE_2    = 'three.example.com'
+    static String TAGS_2    = 'dbservers, test, west'
+    static String NODE_3    = 'mail.example.com'
+    static String TAGS_3    = 'ungrouped'
+
+    def setupSpec() {
+        startCompose()
+        configureRundeck(PROJ_NAME, NODE_1)
+    }
+
+    void "child groups"() {
+        when:
+        def result = client.apiCall {api-> api.listNodes(PROJ_NAME,'.*')}
+
+        then:
+        result != null
+        result.size() == 7
+        result.get(NODE_1) != null
+        result.get(NODE_1).getAttributes().get(NODENAME) == NODE_1
+        result.get(NODE_1).getAttributes().get(HOSTNAME) == NODE_1
+        result.get(NODE_1).getAttributes().get(TAGS)     == TAGS_1
+        result.get(NODE_2) != null
+        result.get(NODE_2).getAttributes().get(NODENAME) == NODE_2
+        result.get(NODE_2).getAttributes().get(HOSTNAME) == NODE_2
+        result.get(NODE_2).getAttributes().get(TAGS)     == TAGS_2
+        result.get(NODE_3) != null
+        result.get(NODE_3).getAttributes().get(NODENAME) == NODE_3
+        result.get(NODE_3).getAttributes().get(HOSTNAME) == NODE_3
+        result.get(NODE_3).getAttributes().get(TAGS)     == TAGS_3
+    }
+}

functional-test/src/test/resources/docker/ansible-child-groups/ansible.cfg

@@ -0,0 +1,6 @@
+[defaults]
+inventory=/home/rundeck/ansible-child-groups/inventory_child.yaml
+interpreter_python=/usr/bin/python3
+
+
+

functional-test/src/test/resources/docker/ansible-child-groups/inventory_child.yaml

@@ -0,0 +1,27 @@
+ungrouped:
+  hosts:
+    mail.example.com:
+webservers:
+  hosts:
+    foo.example.com:
+    bar.example.com:
+dbservers:
+  hosts:
+    one.example.com:
+    two.example.com:
+    three.example.com:
+east:
+  hosts:
+    foo.example.com:
+    one.example.com:
+    two.example.com:
+west:
+  hosts:
+    bar.example.com:
+    three.example.com:
+prod:
+  children:
+    east:
+test:
+  children:
+    west:

functional-test/src/test/resources/docker/docker-compose.yml

@@ -35,6 +35,7 @@ services:
     - ./ansible:/home/rundeck/ansible:rw
     - ./ansible-list:/home/rundeck/ansible-list:rw
     - ./ansible-yaml-parsing:/home/rundeck/ansible-yaml-parsing:rw
+    - ./ansible-child-groups:/home/rundeck/ansible-child-groups:rw
 
 volumes:
   rundeck-data:

functional-test/src/test/resources/project-import/ansible-child-groups/rundeck-child-groups/files/acls/node-acl.aclpolicy

@@ -0,0 +1,8 @@
+by:
+  urn: project:ansible-yaml-parsing
+for:
+  storage:
+    - match: 
+        path: 'keys/.*'
+      allow: [read] 
+description: Allow access to key storage

functional-test/src/test/resources/project-import/ansible-child-groups/rundeck-child-groups/files/etc/project.properties

@@ -0,0 +1,31 @@
+#edit below
+project.disable.executions=false
+project.disable.schedule=false
+project.execution.history.cleanup.batch=500
+project.execution.history.cleanup.enabled=false
+project.execution.history.cleanup.retention.days=60
+project.execution.history.cleanup.retention.minimum=50
+project.execution.history.cleanup.schedule=0 0 0 1/1 * ? *
+project.jobs.gui.groupExpandLevel=1
+project.later.executions.disable.value=0
+project.later.executions.disable=false
+project.later.executions.enable.value=
+project.later.executions.enable=false
+project.later.schedule.disable.value=
+project.later.schedule.disable=false
+project.later.schedule.enable.value=
+project.later.schedule.enable=false
+project.name=ansible-yaml-parsing
+project.nodeCache.enabled=false
+project.nodeCache.firstLoadSynch=true
+project.output.allowUnsanitized=false
+project.retry-counter=3
+project.ssh-authentication=privateKey
+resources.source.1.type=local
+resources.source.2.config.ansible-config-file-path=/home/rundeck/ansible-child-groups/ansible.cfg
+resources.source.2.config.ansible-gather-facts=false
+resources.source.2.config.ansible-ignore-errors=true
+resources.source.2.config.ansible-inventory=/home/rundeck/ansible-child-groups/inventory_child.yaml
+resources.source.2.type=com.batix.rundeck.plugins.AnsibleResourceModelSourceFactory
+service.FileCopier.default.provider=sshj-scp
+service.NodeExecutor.default.provider=sshj-ssh

src/main/groovy/com/rundeck/plugins/ansible/plugin/AnsibleResourceModelSource.java

@@ -1,6 +1,7 @@
 package com.rundeck.plugins.ansible.plugin;
 
 import com.dtolabs.rundeck.core.common.Framework;
+import com.dtolabs.rundeck.core.common.INodeEntry;
 import com.dtolabs.rundeck.core.common.INodeSet;
 import com.dtolabs.rundeck.core.common.NodeEntryImpl;
 import com.dtolabs.rundeck.core.common.NodeSetImpl;
@@ -48,13 +49,15 @@
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Properties;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static com.rundeck.plugins.ansible.ansible.InventoryList.ALL;
 import static com.rundeck.plugins.ansible.ansible.InventoryList.CHILDREN;
@@ -132,6 +135,8 @@ public class AnsibleResourceModelSource implements ResourceModelSource, ProxyRun
   @Setter
   private AnsibleInventoryList.AnsibleInventoryListBuilder ansibleInventoryListBuilder = null;
 
+  private Map<String, NodeEntryImpl> ansibleNodes = new HashMap<>();
+
   public AnsibleResourceModelSource(final Framework framework) {
       this.framework = framework;
   }
@@ -714,44 +719,134 @@ public void ansibleInventoryList(NodeSetImpl nodes, AnsibleRunner.AnsibleRunnerB
 
     if (isTagMapValid(all, ALL)) {
       Map<String, Object> children = InventoryList.getValue(all, CHILDREN);
+      processChildren(children, new HashSet<>());
+    }
 
-      if (isTagMapValid(children, CHILDREN)) {
-        for (Map.Entry<String, Object> pair : children.entrySet()) {
-          String hostGroup = pair.getKey();
-          Map<String, Object> hostNames = InventoryList.getType(pair.getValue());
-          Map<String, Object> hosts = InventoryList.getValue(hostNames, HOSTS);
-
-          if (isTagMapValid(hosts, HOSTS)) {
-            for (Map.Entry<String, Object> hostNode : hosts.entrySet()) {
-              NodeEntryImpl node = new NodeEntryImpl();
-              node.setTags(Set.of(hostGroup));
-              String hostName = hostNode.getKey();
-              node.setHostname(hostName);
-              node.setNodename(hostName);
-              Map<String, Object> nodeValues = InventoryList.getType(hostNode.getValue());
-
-              InventoryList.tagHandle(NodeTag.HOSTNAME, node, nodeValues);
-              InventoryList.tagHandle(NodeTag.USERNAME, node, nodeValues);
-              InventoryList.tagHandle(NodeTag.OS_FAMILY, node, nodeValues);
-              InventoryList.tagHandle(NodeTag.OS_NAME, node, nodeValues);
-              InventoryList.tagHandle(NodeTag.OS_ARCHITECTURE, node, nodeValues);
-              InventoryList.tagHandle(NodeTag.OS_VERSION, node, nodeValues);
-              InventoryList.tagHandle(NodeTag.DESCRIPTION, node, nodeValues);
-
-              nodeValues.forEach((key, value) -> {
-                if (value != null) {
-                  node.setAttribute(key, value.toString());
-                }
-              });
+    ansibleNodes.forEach((k, node) -> nodes.putNode(node));
+    ansibleNodes.clear();
+  }
 
-              nodes.putNode(node);
-            }
-          }
-        }
+  /**
+   * Processes the given set of nodes and populates the children map with the results.
+   *
+   * @param children a map to be populated with the processed children nodes
+   * @param tags    a set of tags to filter the nodes
+   * @throws ResourceModelSourceException if an error occurs while processing the nodes
+   */
+  public void processChildren(Map<String, Object> children, HashSet<String> tags) throws ResourceModelSourceException {
+    if (!isTagMapValid(children, CHILDREN)) {
+      return;
+    }
+
+    for (Map.Entry<String, Object> pair : children.entrySet()) {
+
+      String hostGroup = pair.getKey();
+      tags.add(hostGroup);
+      Map<String, Object> hostNames = InventoryList.getType(pair.getValue());
+
+      if (hostNames.containsKey(CHILDREN)) {
+        Map<String, Object> subChildren = InventoryList.getValue(hostNames, CHILDREN);
+        processChildren(subChildren, tags);
+      } else {
+        processHosts(hostNames, tags);
+        tags.clear();
       }
     }
   }
 
+  /**
+   * Processes the hosts within the given host names map and adds them to the nodes set.
+   *
+   * @param hostNames the map containing host names and their attributes
+   * @param tags      the set of tags to apply to the nodes
+   * @throws ResourceModelSourceException if an error occurs while processing the nodes
+   */
+  public void processHosts(Map<String, Object> hostNames, HashSet<String> tags) throws ResourceModelSourceException {
+    Map<String, Object> hosts = InventoryList.getValue(hostNames, HOSTS);
+
+    if (!isTagMapValid(hosts, HOSTS)) {
+      return;
+    }
+
+    for (Map.Entry<String, Object> hostNode : hosts.entrySet()) {
+      NodeEntryImpl node = createNodeEntry(hostNode);
+      addNode(node, tags);
+    }
+  }
+
+  /**
+   * Creates a NodeEntryImpl object from the given host node entry and tags.
+   *
+   * @param hostNode the entry containing the host name and its attributes
+   * @return the created NodeEntryImpl object
+   */
+  public NodeEntryImpl createNodeEntry(Map.Entry<String, Object> hostNode) throws ResourceModelSourceException {
+    NodeEntryImpl node = new NodeEntryImpl();
+    String hostName = hostNode.getKey();
+    node.setHostname(hostName);
+    node.setNodename(hostName);
+    Map<String, Object> nodeValues = InventoryList.getType(hostNode.getValue());
+
+    applyNodeTags(node, nodeValues);
+    nodeValues.forEach((key, value) -> {
+      if (value != null) {
+        node.setAttribute(key, value.toString());
+      }
+    });
+
+    return node;
+  }
+
+  /**
+   * Applies predefined tags to the given node based on the provided node values.
+   *
+   * @param node       the node to which the tags will be applied
+   * @param nodeValues the map containing the node's attributes
+   */
+  public void applyNodeTags(NodeEntryImpl node, Map<String, Object> nodeValues) throws ResourceModelSourceException {
+    InventoryList.tagHandle(NodeTag.HOSTNAME, node, nodeValues);
+    InventoryList.tagHandle(NodeTag.USERNAME, node, nodeValues);
+    InventoryList.tagHandle(NodeTag.OS_FAMILY, node, nodeValues);
+    InventoryList.tagHandle(NodeTag.OS_NAME, node, nodeValues);
+    InventoryList.tagHandle(NodeTag.OS_ARCHITECTURE, node, nodeValues);
+    InventoryList.tagHandle(NodeTag.OS_VERSION, node, nodeValues);
+    InventoryList.tagHandle(NodeTag.DESCRIPTION, node, nodeValues);
+  }
+
+  /**
+   * Adds a node to the ansibleNodes map, merging tags if the node already exists.
+   *
+   * @param node The node to add.
+   * @param tags The tags to associate with the node.
+   */
+  public void addNode(NodeEntryImpl node, Set<String> tags) {
+    ansibleNodes.compute(node.getNodename(), (key, existingNode) -> {
+      if (existingNode != null) {
+        Set<String> mergedTags = new HashSet<>(getStringTags(existingNode));
+        mergedTags.addAll(tags);
+        existingNode.setTags(Set.copyOf(mergedTags));
+        return existingNode;
+      } else {
+        node.setTags(Set.copyOf(tags));
+        return node;
+      }
+    });
+  }
+
+  /**
+   * Retrieves the tags from a node and converts them to strings.
+   *
+   * @param node The node whose tags are to be retrieved.
+   * @return A set of strings representing the node's tags.  Returns an empty set if the node has no tags.
+   */
+  public Set<String> getStringTags(NodeEntryImpl node) {
+    Set<String> tags = new HashSet<>();
+    for (Object tag : node.getTags()) {
+      tags.add(tag.toString());
+    }
+    return tags;
+  }
+
   /**
    * Gets Ansible nodes from inventory
    * @return Ansible nodes