From 67234d51a129e0b2acc60d965780815067700b59 Mon Sep 17 00:00:00 2001 From: ashley-taylor Date: Wed, 9 Apr 2014 14:50:59 +1200 Subject: [PATCH 1/3] WHIRR-760 allow other sudoers access allow password ssh access configure all systems against the private ip address --- .../org/apache/whirr/ClusterController.java | 10 +++- .../java/org/apache/whirr/ClusterSpec.java | 46 ++++++++++++++++++- .../whirr/actions/BootstrapClusterAction.java | 17 ++++--- .../whirr/compute/BootstrapTemplate.java | 43 ++++++++++++++--- 4 files changed, 99 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/apache/whirr/ClusterController.java b/core/src/main/java/org/apache/whirr/ClusterController.java index 39c430ee..62968027 100644 --- a/core/src/main/java/org/apache/whirr/ClusterController.java +++ b/core/src/main/java/org/apache/whirr/ClusterController.java @@ -329,9 +329,15 @@ private Cluster.Instance toInstance(NodeMetadata metadata, Cluster cluster, Clus } } catch (NoSuchElementException e) { } - + + Set publicAddresses = metadata.getPublicAddresses(); + //If we want the cluster to configure against the private address we set it as both + if(spec.isInternalIp()){ + publicAddresses = metadata.getPrivateAddresses(); + } + return new Cluster.Instance(credentials, roles, - Iterables.getFirst(metadata.getPublicAddresses().size() > 0 ? metadata.getPublicAddresses() : metadata.getPrivateAddresses(), null), + Iterables.getFirst(publicAddresses.size() > 0 ? publicAddresses : metadata.getPrivateAddresses(), null), Iterables.getFirst(metadata.getPrivateAddresses(), null), metadata.getId(), metadata); } diff --git a/core/src/main/java/org/apache/whirr/ClusterSpec.java b/core/src/main/java/org/apache/whirr/ClusterSpec.java index 32f1059c..95141ee5 100644 --- a/core/src/main/java/org/apache/whirr/ClusterSpec.java +++ b/core/src/main/java/org/apache/whirr/ClusterSpec.java @@ -199,7 +199,13 @@ public enum Property { AWS_EC2_PLACEMENT_GROUP(String.class, false, "If given, use this existing EC2 placement group. (aws-ec2 specific option)"), - QUIET(Boolean.class, false, "Adjust user level, console logging verbosity."); + QUIET(Boolean.class, false, "Adjust user level, console logging verbosity."), + + SSH_ALLOW_PASSWORD(Boolean.class, false, "If password login is allowed"), + + USE_INTERNAL_IP(Boolean.class, false, "If it should use in internal ip for its setup instead of the external"), + + ALLOW_OTHER_SUDOERS(Boolean.class, false, "If we should only add to the sudoers file"); private Class type; private boolean multipleArguments; @@ -336,6 +342,12 @@ public static ClusterSpec withNoDefaults(Configuration conf) private boolean isQuiet; + private boolean isAllowPassword; + + private boolean isInternalIp; + + private boolean isAllowOtherSudoers; + public ClusterSpec() throws ConfigurationException { this(new PropertiesConfiguration()); } @@ -413,6 +425,12 @@ public ClusterSpec(Configuration userConfig, boolean loadDefaults, Map> fr = new HashMap>(); String firewallPrefix = Property.FIREWALL_RULES.getConfigName(); Pattern firewallRuleKeyPattern = Pattern.compile("^".concat(Pattern.quote(firewallPrefix).concat("(?:\\.(.+))?$"))); @@ -626,6 +644,18 @@ public String getBlobStoreProvider() { public boolean isQuiet() { return isQuiet; } + + public boolean isAllowPassword() { + return isAllowPassword; + } + + public boolean isInternalIp() { + return isInternalIp; + } + + public boolean isAllowOtherSudoers() { + return isAllowOtherSudoers; + } /** * Probably jclouds should provide a similar mechanism @@ -1005,7 +1035,19 @@ public void setByonNodes(Map byonNodes) { public void setQuiet(boolean isQuiet) { this.isQuiet = isQuiet; } - + + public void setAllowPassword(boolean isAllowPassword) { + this.isAllowPassword = isAllowPassword; + } + + public void setInternalIp(boolean isInternalIp) { + this.isInternalIp = isInternalIp; + } + + public void setAllowOtherSudoers(boolean isAllowOtherSudoers) { + this.isAllowOtherSudoers = isAllowOtherSudoers; + } + public void setHandlerListener(ClusterActionHandlerListener handlerListener) { this.handlerListener = handlerListener; } diff --git a/core/src/main/java/org/apache/whirr/actions/BootstrapClusterAction.java b/core/src/main/java/org/apache/whirr/actions/BootstrapClusterAction.java index 9f67734d..33d03676 100644 --- a/core/src/main/java/org/apache/whirr/actions/BootstrapClusterAction.java +++ b/core/src/main/java/org/apache/whirr/actions/BootstrapClusterAction.java @@ -85,12 +85,14 @@ protected void doAction(Map eventMap) ExecutorService executorService = Executors.newCachedThreadPool(); Map>> futures = Maps.newHashMap(); - + boolean isInternalIp = false; // initialize startup processes per InstanceTemplates for (Entry entry : eventMap.entrySet()) { final InstanceTemplate instanceTemplate = entry.getKey(); final ClusterSpec clusterSpec = entry.getValue().getClusterSpec(); - + if(clusterSpec.isInternalIp()){ + isInternalIp = true; + } final int maxNumberOfRetries = clusterSpec.getMaxStartupRetries(); StatementBuilder statementBuilder = entry.getValue().getStatementBuilder(); @@ -111,7 +113,6 @@ protected void doAction(Map eventMap) computeService, template, executorService, nodeStarterFactory)); futures.put(instanceTemplate, nodesFuture); } - Set instances = Sets.newLinkedHashSet(); for (Entry>> entry : futures.entrySet()) { @@ -125,7 +126,7 @@ protected void doAction(Map eventMap) throw new IOException(e); } Set roles = entry.getKey().getRoles(); - instances.addAll(getInstances(roles, nodes)); + instances.addAll(getInstances(roles, nodes, isInternalIp)); } Cluster cluster = new Cluster(instances); for (ClusterActionEvent event : eventMap.values()) { @@ -134,13 +135,17 @@ protected void doAction(Map eventMap) } private Set getInstances(final Set roles, - Set nodes) { + Set nodes, final boolean isInternal) { return Sets.newLinkedHashSet(Collections2.transform(Sets.newLinkedHashSet(nodes), new Function() { @Override public Instance apply(NodeMetadata node) { + Set publicIp = node.getPublicAddresses(); + if(isInternal){ + publicIp = node.getPrivateAddresses(); + } return new Instance(node.getCredentials(), roles, - Iterables.get(node.getPublicAddresses().size() > 0 ? node.getPublicAddresses() : node.getPrivateAddresses(), 0), + Iterables.get(publicIp.size() > 0 ? publicIp: node.getPrivateAddresses(), 0), Iterables.get(node.getPrivateAddresses().size() > 0 ? node.getPrivateAddresses() : node.getPublicAddresses(), 0), node.getId(), node); } diff --git a/core/src/main/java/org/apache/whirr/compute/BootstrapTemplate.java b/core/src/main/java/org/apache/whirr/compute/BootstrapTemplate.java index 8d8c61c0..4df9939e 100644 --- a/core/src/main/java/org/apache/whirr/compute/BootstrapTemplate.java +++ b/core/src/main/java/org/apache/whirr/compute/BootstrapTemplate.java @@ -25,10 +25,14 @@ import static org.jclouds.scriptbuilder.domain.Statements.newStatementList; import static org.jclouds.scriptbuilder.statements.ssh.SshStatements.sshdConfig; +import java.util.ArrayList; +import java.util.List; + import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; + import org.apache.whirr.ClusterSpec; import org.apache.whirr.InstanceTemplate; import org.apache.whirr.service.jclouds.StatementBuilder; @@ -43,6 +47,7 @@ import org.jclouds.ec2.compute.predicates.EC2ImagePredicates; import org.jclouds.scriptbuilder.domain.OsFamily; import org.jclouds.scriptbuilder.domain.Statement; +import org.jclouds.scriptbuilder.domain.StatementList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,7 +68,8 @@ public static Template build( statementBuilder.name(name); ensureUserExistsAndAuthorizeSudo(statementBuilder, clusterSpec.getClusterUser(), - clusterSpec.getPublicKey(), clusterSpec.getPrivateKey()); + clusterSpec.getPublicKey(), clusterSpec.getPrivateKey(), clusterSpec.isAllowPassword(), + clusterSpec.isAllowOtherSudoers()); Statement bootstrap = statementBuilder.build(clusterSpec); if (LOG.isDebugEnabled()) { @@ -81,15 +87,24 @@ public static Template build( } private static void ensureUserExistsAndAuthorizeSudo( - StatementBuilder builder, String user, String publicKey, String privateKey + StatementBuilder builder, String user, String publicKey, String privateKey, boolean isAllowedPassword, + boolean isAllowOtherSudoers ) { builder.addExport("NEW_USER", user); builder.addExport("DEFAULT_HOME", "/home/users"); - builder.addStatement(0, newStatementList( - ensureUserExistsWithPublicAndPrivateKey(user, publicKey, privateKey), - makeSudoersOnlyPermitting(user), - disablePasswordBasedAuth()) - ); + + List statements = new ArrayList(); + statements.add(ensureUserExistsWithPublicAndPrivateKey(user, publicKey, privateKey)); + if(isAllowOtherSudoers){ + statements.add(addSudoersUser(user)); + }else{ + statements.add(makeSudoersOnlyPermitting(user)); + } + + if(!isAllowedPassword){ + statements.add(disablePasswordBasedAuth()); + } + builder.addStatement(0, new StatementList(statements)); } /** @@ -178,6 +193,20 @@ private static Statement makeSudoersOnlyPermitting(String username) { ) ); } + // must be used inside InitBuilder, as this sets the shell variables used in this statement + private static Statement addSudoersUser(String username) { + return newStatementList( + interpret( + "touch /etc/sudoers.d/whirr", + "chmod 0440 /etc/sudoers.d/whirr", + "chown root /etc/sudoers.d/whirr\n"), + appendFile( + "/etc/sudoers.d/whirr", + ImmutableSet.of( + username + " ALL = (ALL) NOPASSWD: ALL") + ) + ); + } private static Statement disablePasswordBasedAuth() { return sshdConfig(ImmutableMap.of("PasswordAuthentication","no")); From 1bc56c5017095c6df032c00865ddfb2cde051ae2 Mon Sep 17 00:00:00 2001 From: Lae Chen Date: Fri, 16 Jan 2015 10:24:59 +1300 Subject: [PATCH 2/3] [WHIRR-761] Add MapReduce job history configurations --- .../whirr/service/hadoop/HadoopConfigurationBuilder.java | 8 ++++++-- .../service/hadoop/HadoopConfigurationBuilderTest.java | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java b/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java index b61314f5..dce36f56 100644 --- a/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java +++ b/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java @@ -124,10 +124,14 @@ static Configuration buildMapReduceConfiguration(ClusterSpec clusterSpec, setIfAbsent(config, "mapred.local.dir", appendToDataDirectories(dataDirectories, "/hadoop/mapred/local")); - + + String host = cluster.getInstanceMatching(role(HadoopNameNodeClusterActionHandler.ROLE)).getPublicHostName(); + setIfAbsent(config, "mapreduce.jobhistory.webapp.address", host + ":19888"); + setIfAbsent(config, "mapreduce.jobhistory.address", host + ":10020"); + Set taskTrackers = cluster .getInstancesMatching(role(HadoopTaskTrackerClusterActionHandler.ROLE)); - + if (!taskTrackers.isEmpty()) { Hardware hardware = Iterables.getFirst(taskTrackers, null) diff --git a/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilderTest.java b/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilderTest.java index 4903f64e..fd6842fc 100644 --- a/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilderTest.java +++ b/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilderTest.java @@ -45,6 +45,7 @@ import java.util.List; +import static org.apache.whirr.RolePredicates.role; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; import static org.mockito.Mockito.mock; @@ -165,6 +166,11 @@ public void testMapReduce() throws Exception { assertThat(conf.getString("mapred.reduce.tasks"), is("15")); assertThat(conf.getString("mapred.local.dir"), is("/data0/hadoop/mapred/local,/data1/hadoop/mapred/local")); + + String host = cluster.getInstanceMatching( + role(HadoopNameNodeClusterActionHandler.ROLE)).getPublicHostName(); + assertThat(conf.getString("mapreduce.jobhistory.webapp.address"), is(host + ":19888")); + assertThat(conf.getString("mapreduce.jobhistory.address"), is(host + ":10020")); } @Test From 0fa6052bd318b4101e78ede46b7bef71fde3e960 Mon Sep 17 00:00:00 2001 From: ashley-taylor Date: Wed, 18 Feb 2015 15:27:43 +1300 Subject: [PATCH 3/3] ability to set network. Needed to work with rackspace cloud and rackconnect --- .../java/org/apache/whirr/ClusterSpec.java | 18 +++++++++++++++++- .../whirr/compute/BootstrapTemplate.java | 7 +++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/whirr/ClusterSpec.java b/core/src/main/java/org/apache/whirr/ClusterSpec.java index 95141ee5..b8f45ad2 100644 --- a/core/src/main/java/org/apache/whirr/ClusterSpec.java +++ b/core/src/main/java/org/apache/whirr/ClusterSpec.java @@ -205,7 +205,9 @@ public enum Property { USE_INTERNAL_IP(Boolean.class, false, "If it should use in internal ip for its setup instead of the external"), - ALLOW_OTHER_SUDOERS(Boolean.class, false, "If we should only add to the sudoers file"); + ALLOW_OTHER_SUDOERS(Boolean.class, false, "If we should only add to the sudoers file"), + + NETWORKS(String.class, false, "What network to use"); private Class type; private boolean multipleArguments; @@ -347,6 +349,7 @@ public static ClusterSpec withNoDefaults(Configuration conf) private boolean isInternalIp; private boolean isAllowOtherSudoers; + private String networks; public ClusterSpec() throws ConfigurationException { this(new PropertiesConfiguration()); @@ -431,6 +434,9 @@ public ClusterSpec(Configuration userConfig, boolean loadDefaults, Map> fr = new HashMap>(); String firewallPrefix = Property.FIREWALL_RULES.getConfigName(); Pattern firewallRuleKeyPattern = Pattern.compile("^".concat(Pattern.quote(firewallPrefix).concat("(?:\\.(.+))?$"))); @@ -652,6 +658,10 @@ public boolean isAllowPassword() { public boolean isInternalIp() { return isInternalIp; } + + public String getNetworks() { + return networks; + } public boolean isAllowOtherSudoers() { return isAllowOtherSudoers; @@ -1048,6 +1058,11 @@ public void setAllowOtherSudoers(boolean isAllowOtherSudoers) { this.isAllowOtherSudoers = isAllowOtherSudoers; } + + public void setNetworks(String networks) { + this.networks = networks; + } + public void setHandlerListener(ClusterActionHandlerListener handlerListener) { this.handlerListener = handlerListener; } @@ -1216,4 +1231,5 @@ public String toString() { .add("byonNodes", getByonNodes()) .toString(); } + } diff --git a/core/src/main/java/org/apache/whirr/compute/BootstrapTemplate.java b/core/src/main/java/org/apache/whirr/compute/BootstrapTemplate.java index 4df9939e..102e65a3 100644 --- a/core/src/main/java/org/apache/whirr/compute/BootstrapTemplate.java +++ b/core/src/main/java/org/apache/whirr/compute/BootstrapTemplate.java @@ -30,6 +30,7 @@ import com.google.common.base.Joiner; import com.google.common.base.Splitter; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -38,6 +39,7 @@ import org.apache.whirr.service.jclouds.StatementBuilder; import org.jclouds.aws.ec2.compute.AWSEC2ComputeService; import org.jclouds.aws.ec2.compute.AWSEC2TemplateOptions; +import org.jclouds.cloudstack.compute.options.CloudStackTemplateOptions; import org.jclouds.compute.ComputeService; import org.jclouds.compute.ComputeServiceContext; import org.jclouds.compute.domain.Template; @@ -79,7 +81,12 @@ public static Template build( TemplateBuilder templateBuilder = computeService.templateBuilder().from( instanceTemplate.getTemplate() != null ? instanceTemplate.getTemplate() : clusterSpec.getTemplate()); + Template template = templateBuilder.build(); + String networks = clusterSpec.getNetworks(); + if(!Strings.isNullOrEmpty(networks)) { + template.getOptions().networks(networks); + } template.getOptions().runScript(bootstrap); return setSpotInstancePriceIfSpecified( computeService.getContext(), clusterSpec, template, instanceTemplate