WHIRR-760

core/src/main/java/org/apache/whirr/ClusterController.java

@@ -329,9 +329,15 @@ private Cluster.Instance toInstance(NodeMetadata metadata, Cluster cluster, Clus
       }
     } catch (NoSuchElementException e) {
     }
-
+
+    Set<String> publicAddresses = metadata.getPublicAddresses();
+    //If we want the cluster to configure against the private address we set it as both
+    if(spec.isInternalIp()){
+      publicAddresses = metadata.getPrivateAddresses();
+    }
+
     return new Cluster.Instance(credentials, roles,
-      Iterables.getFirst(metadata.getPublicAddresses().size() > 0 ? metadata.getPublicAddresses() : metadata.getPrivateAddresses(), null),
+      Iterables.getFirst(publicAddresses.size() > 0 ? publicAddresses : metadata.getPrivateAddresses(), null),
       Iterables.getFirst(metadata.getPrivateAddresses(), null),
       metadata.getId(), metadata);
   }

core/src/main/java/org/apache/whirr/ClusterSpec.java

@@ -199,7 +199,15 @@ public enum Property {
 
     AWS_EC2_PLACEMENT_GROUP(String.class, false, "If given, use this existing EC2 placement group. (aws-ec2 specific option)"),
 
-    QUIET(Boolean.class, false,  "Adjust user level, console logging verbosity.");
+    QUIET(Boolean.class, false,  "Adjust user level, console logging verbosity."),
+
+    SSH_ALLOW_PASSWORD(Boolean.class, false,  "If password login is allowed"),
+
+    USE_INTERNAL_IP(Boolean.class, false,  "If it should use in internal ip for its setup instead of the external"),
+
+    ALLOW_OTHER_SUDOERS(Boolean.class, false,  "If we should only add to the sudoers file"),
+
+    NETWORKS(String.class, false,  "What network to use");
 
     private Class<?> type;
     private boolean multipleArguments;
@@ -336,6 +344,13 @@ public static ClusterSpec withNoDefaults(Configuration conf)
 
   private boolean isQuiet;
 
+  private boolean isAllowPassword;
+
+  private boolean isInternalIp;
+
+  private boolean isAllowOtherSudoers;
+  private String networks;
+
   public ClusterSpec() throws ConfigurationException {
     this(new PropertiesConfiguration());
   }
@@ -413,6 +428,15 @@ public ClusterSpec(Configuration userConfig, boolean loadDefaults, Map<String,No
 
     setQuiet(config.getBoolean(Property.QUIET.getConfigName(), Boolean.FALSE));
 
+    setAllowPassword(config.getBoolean(Property.SSH_ALLOW_PASSWORD.getConfigName(), Boolean.FALSE));
+
+    setInternalIp(config.getBoolean(Property.USE_INTERNAL_IP.getConfigName(), Boolean.FALSE));
+
+    setAllowOtherSudoers(config.getBoolean(Property.ALLOW_OTHER_SUDOERS.getConfigName(), Boolean.FALSE));
+
+    setNetworks(config.getString(Property.NETWORKS.getConfigName()));
+
+
     Map<String, List<String>> fr = new HashMap<String, List<String>>();
     String firewallPrefix = Property.FIREWALL_RULES.getConfigName();
     Pattern firewallRuleKeyPattern = Pattern.compile("^".concat(Pattern.quote(firewallPrefix).concat("(?:\\.(.+))?$")));
@@ -626,6 +650,22 @@ public String getBlobStoreProvider() {
   public boolean isQuiet() {
     return isQuiet;
   }
+
+  public boolean isAllowPassword() {
+    return isAllowPassword;
+  }
+
+  public boolean isInternalIp() {
+    return isInternalIp;
+  }
+
+  public String getNetworks() {
+	  return networks;
+	}
+
+  public boolean isAllowOtherSudoers() {
+    return isAllowOtherSudoers;
+  }
 
   /**
    * Probably jclouds should provide a similar mechanism
@@ -1005,7 +1045,24 @@ public void setByonNodes(Map<String,Node> byonNodes) {
   public void setQuiet(boolean isQuiet) {
     this.isQuiet = isQuiet;
   }
-
+
+  public void setAllowPassword(boolean isAllowPassword) {
+    this.isAllowPassword = isAllowPassword;
+  }
+
+  public void setInternalIp(boolean isInternalIp) {
+    this.isInternalIp = isInternalIp;
+  }
+
+  public void setAllowOtherSudoers(boolean isAllowOtherSudoers) {
+    this.isAllowOtherSudoers = isAllowOtherSudoers;
+  }
+
+
+  public void setNetworks(String networks) {
+	  this.networks = networks;
+  }
+
   public void setHandlerListener(ClusterActionHandlerListener handlerListener) {
     this.handlerListener = handlerListener;
   }
@@ -1174,4 +1231,5 @@ public String toString() {
       .add("byonNodes", getByonNodes())
       .toString();
   }
+
 }

core/src/main/java/org/apache/whirr/actions/BootstrapClusterAction.java

@@ -85,12 +85,14 @@ protected void doAction(Map<InstanceTemplate, ClusterActionEvent> eventMap)
 
     ExecutorService executorService = Executors.newCachedThreadPool();    
     Map<InstanceTemplate, Future<Set<? extends NodeMetadata>>> futures = Maps.newHashMap();
-
+    boolean isInternalIp = false;
     // initialize startup processes per InstanceTemplates
     for (Entry<InstanceTemplate, ClusterActionEvent> entry : eventMap.entrySet()) {
       final InstanceTemplate instanceTemplate = entry.getKey();
       final ClusterSpec clusterSpec = entry.getValue().getClusterSpec();
-
+      if(clusterSpec.isInternalIp()){
+        isInternalIp = true;
+      }
       final int maxNumberOfRetries = clusterSpec.getMaxStartupRetries();
       StatementBuilder statementBuilder = entry.getValue().getStatementBuilder();
 
@@ -111,7 +113,6 @@ protected void doAction(Map<InstanceTemplate, ClusterActionEvent> eventMap)
               computeService, template, executorService, nodeStarterFactory));
       futures.put(instanceTemplate, nodesFuture);
     }
-
     Set<Instance> instances = Sets.newLinkedHashSet();
     for (Entry<InstanceTemplate, Future<Set<? extends NodeMetadata>>> entry :
         futures.entrySet()) {
@@ -125,7 +126,7 @@ protected void doAction(Map<InstanceTemplate, ClusterActionEvent> eventMap)
         throw new IOException(e);
       }
       Set<String> roles = entry.getKey().getRoles();
-      instances.addAll(getInstances(roles, nodes));
+      instances.addAll(getInstances(roles, nodes, isInternalIp));
     }
     Cluster cluster = new Cluster(instances);
     for (ClusterActionEvent event : eventMap.values()) {
@@ -134,13 +135,17 @@ protected void doAction(Map<InstanceTemplate, ClusterActionEvent> eventMap)
   }
 
   private Set<Instance> getInstances(final Set<String> roles,
-      Set<? extends NodeMetadata> nodes) {
+      Set<? extends NodeMetadata> nodes, final boolean isInternal) {
     return Sets.newLinkedHashSet(Collections2.transform(Sets.newLinkedHashSet(nodes),
         new Function<NodeMetadata, Instance>() {
       @Override
       public Instance apply(NodeMetadata node) {
+        Set<String> publicIp = node.getPublicAddresses();
+        if(isInternal){
+          publicIp = node.getPrivateAddresses();
+        }
         return new Instance(node.getCredentials(), roles,
-                            Iterables.get(node.getPublicAddresses().size() > 0 ? node.getPublicAddresses() : node.getPrivateAddresses(), 0),
+                            Iterables.get(publicIp.size() > 0 ? publicIp: node.getPrivateAddresses(), 0),
                             Iterables.get(node.getPrivateAddresses().size() > 0 ? node.getPrivateAddresses() : node.getPublicAddresses(), 0),
             node.getId(), node);
       }

core/src/main/java/org/apache/whirr/compute/BootstrapTemplate.java

@@ -25,15 +25,21 @@
 import static org.jclouds.scriptbuilder.domain.Statements.newStatementList;
 import static org.jclouds.scriptbuilder.statements.ssh.SshStatements.sshdConfig;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import com.google.common.base.Joiner;
 import com.google.common.base.Splitter;
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+
 import org.apache.whirr.ClusterSpec;
 import org.apache.whirr.InstanceTemplate;
 import org.apache.whirr.service.jclouds.StatementBuilder;
 import org.jclouds.aws.ec2.compute.AWSEC2ComputeService;
 import org.jclouds.aws.ec2.compute.AWSEC2TemplateOptions;
+import org.jclouds.cloudstack.compute.options.CloudStackTemplateOptions;
 import org.jclouds.compute.ComputeService;
 import org.jclouds.compute.ComputeServiceContext;
 import org.jclouds.compute.domain.Template;
@@ -43,6 +49,7 @@
 import org.jclouds.ec2.compute.predicates.EC2ImagePredicates;
 import org.jclouds.scriptbuilder.domain.OsFamily;
 import org.jclouds.scriptbuilder.domain.Statement;
+import org.jclouds.scriptbuilder.domain.StatementList;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -63,7 +70,8 @@ public static Template build(
 
     statementBuilder.name(name);
     ensureUserExistsAndAuthorizeSudo(statementBuilder, clusterSpec.getClusterUser(),
-        clusterSpec.getPublicKey(), clusterSpec.getPrivateKey());
+        clusterSpec.getPublicKey(), clusterSpec.getPrivateKey(), clusterSpec.isAllowPassword(),
+        clusterSpec.isAllowOtherSudoers());
     Statement bootstrap = statementBuilder.build(clusterSpec);
 
     if (LOG.isDebugEnabled()) {
@@ -73,23 +81,37 @@ public static Template build(
     TemplateBuilder templateBuilder = computeService.templateBuilder().from(
         instanceTemplate.getTemplate() != null ? instanceTemplate.getTemplate() :
         clusterSpec.getTemplate());
+
     Template template = templateBuilder.build();
+    String networks = clusterSpec.getNetworks();
+    if(!Strings.isNullOrEmpty(networks)) {
+    	template.getOptions().networks(networks);
+    }
     template.getOptions().runScript(bootstrap);
     return setSpotInstancePriceIfSpecified(
       computeService.getContext(), clusterSpec, template, instanceTemplate
     );
   }
 
   private static void ensureUserExistsAndAuthorizeSudo(
-      StatementBuilder builder, String user, String publicKey, String privateKey
+      StatementBuilder builder, String user, String publicKey, String privateKey, boolean isAllowedPassword,
+      boolean isAllowOtherSudoers
   ) {
     builder.addExport("NEW_USER", user);
     builder.addExport("DEFAULT_HOME", "/home/users");
-    builder.addStatement(0, newStatementList(
-        ensureUserExistsWithPublicAndPrivateKey(user, publicKey, privateKey),
-        makeSudoersOnlyPermitting(user),
-        disablePasswordBasedAuth())
-    );
+
+    List<Statement> statements = new ArrayList<Statement>();
+    statements.add(ensureUserExistsWithPublicAndPrivateKey(user, publicKey, privateKey));
+    if(isAllowOtherSudoers){
+      statements.add(addSudoersUser(user));
+    }else{
+      statements.add(makeSudoersOnlyPermitting(user));  
+    }
+
+    if(!isAllowedPassword){
+      statements.add(disablePasswordBasedAuth());
+    }
+    builder.addStatement(0, new StatementList(statements));
   }
 
   /**
@@ -178,6 +200,20 @@ private static Statement makeSudoersOnlyPermitting(String username) {
         )
     );
   }
+  // must be used inside InitBuilder, as this sets the shell variables used in this statement
+  private static Statement addSudoersUser(String username) {
+    return newStatementList(
+  interpret(
+        "touch /etc/sudoers.d/whirr",
+        "chmod 0440 /etc/sudoers.d/whirr",
+        "chown root /etc/sudoers.d/whirr\n"),
+      appendFile(
+        "/etc/sudoers.d/whirr",
+        ImmutableSet.of(
+          username + " ALL = (ALL) NOPASSWD: ALL")
+        )
+    );
+  }
 
   private static Statement disablePasswordBasedAuth() {
     return sshdConfig(ImmutableMap.of("PasswordAuthentication","no"));

services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java

@@ -124,10 +124,14 @@ static Configuration buildMapReduceConfiguration(ClusterSpec clusterSpec,
 
     setIfAbsent(config, "mapred.local.dir",
         appendToDataDirectories(dataDirectories, "/hadoop/mapred/local"));
-
+
+    String host = cluster.getInstanceMatching(role(HadoopNameNodeClusterActionHandler.ROLE)).getPublicHostName();
+    setIfAbsent(config, "mapreduce.jobhistory.webapp.address", host + ":19888");
+    setIfAbsent(config, "mapreduce.jobhistory.address", host + ":10020");
+
     Set<Instance> taskTrackers = cluster
       .getInstancesMatching(role(HadoopTaskTrackerClusterActionHandler.ROLE));
-    
+
     if (!taskTrackers.isEmpty()) {
 
       Hardware hardware = Iterables.getFirst(taskTrackers, null)

services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilderTest.java

@@ -45,6 +45,7 @@
 
 import java.util.List;
 
+import static org.apache.whirr.RolePredicates.role;
 import static org.hamcrest.Matchers.is;
 import static org.junit.Assert.assertThat;
 import static org.mockito.Mockito.mock;
@@ -165,6 +166,11 @@ public void testMapReduce() throws Exception {
     assertThat(conf.getString("mapred.reduce.tasks"), is("15"));
     assertThat(conf.getString("mapred.local.dir"),
         is("/data0/hadoop/mapred/local,/data1/hadoop/mapred/local"));
+
+    String host = cluster.getInstanceMatching(
+        role(HadoopNameNodeClusterActionHandler.ROLE)).getPublicHostName();
+    assertThat(conf.getString("mapreduce.jobhistory.webapp.address"), is(host + ":19888"));
+    assertThat(conf.getString("mapreduce.jobhistory.address"), is(host + ":10020"));
   }
 
   @Test