WHIRR-347. Support provider-independent environment variables for cloud credentials (asavu)

git-svn-id: https://svn.apache.org/repos/asf/whirr/trunk@1243842 13f79535-47bb-0310-9956-ffa450edef68

Author: Andrei Savu

.gitignore

@@ -8,3 +8,4 @@ target
 **/*.log
 services/voldemort/lib/
 whirr.log
+conf/credentials

CHANGES.txt

@@ -53,6 +53,9 @@ Trunk (unreleased changes)
     WHIRR-422. Integration tests should fail or succeed in a limited 
     amount of time (asavu)
 
+    WHIRR-347. Support provider-independent environment variables for 
+    cloud credentials (asavu)
+
   BUG FIXES
 
     WHIRR-367. Wrong groupId for zookeeper (Joe Crobak via asavu)

bin/whirr

@@ -24,6 +24,23 @@ case "`uname`" in
 	CYGWIN*) cygwin=true;
 esac
 
+if [ -e ~/.whirr/credentials ]; then
+    . ~/.whirr/credentials
+elif [ -e $BIN/../conf/credentials ]; then
+    . $BIN/../conf/credentials
+fi
+
+# Override cloud provider credentials as needed
+export WHIRR_PROVIDER=${WHIRR_PROVIDER:-$PROVIDER}
+export WHIRR_IDENTITY=${WHIRR_IDENTITY:-$IDENTITY}
+export WHIRR_CREDENTIAL=${WHIRR_CREDENTIAL:-$CREDENTIAL}
+
+# Override blob store credentials as needed
+export WHIRR_BLOBSTORE_PROVIDER=${WHIRR_BLOBSTORE_PROVIDER:-$BLOBSTORE_PROVIDER}
+export WHIRR_BLOBSTORE_IDENTITY=${WHIRR_BLOBSTORE_IDENTITY:-$BLOBSTORE_IDENTITY}
+export WHIRR_BLOBSTORE_CREDENTIAL=${WHIRR_BLOBSTORE_CREDENTIAL:-$BLOBSTORE_CREDENTIAL}
+
+# Build the classpath
 if [ -d "$BIN/../cli/target/lib" ]; then
   CLASSPATH="$BIN/../conf/:$BIN/..:$BIN/../cli/target/lib/*:$BIN/../cli/target/*"
 else
@@ -36,5 +53,8 @@ if $cygwin; then
   CLASSPATH=`cygpath -wp $CLASSPATH`;
 fi
 
+echo Starting cluster on "$WHIRR_PROVIDER" using identity "$WHIRR_IDENTITY"
+
+# Start the application
 java $LOGGING_OPTS $WHIRR_CLI_OPTS -cp "$CLASSPATH" org.apache.whirr.cli.Main "$@"
 

cli/src/main/java/org/apache/whirr/cli/command/CleanupClusterCommand.java

@@ -52,7 +52,7 @@ public CleanupClusterCommand(ClusterControllerFactory factory,
   public int run(InputStream in, PrintStream out, PrintStream err,
       List<String> args) throws Exception {
 
-    OptionSet optionSet = parser.parse(args.toArray(new String[0]));
+    OptionSet optionSet = parser.parse(args.toArray(new String[args.size()]));
     if (!optionSet.nonOptionArguments().isEmpty()) {
       printUsage(err);
       return -1;

cli/src/main/java/org/apache/whirr/cli/command/DestroyClusterCommand.java

@@ -46,7 +46,7 @@ public DestroyClusterCommand(ClusterControllerFactory factory) {
   public int run(InputStream in, PrintStream out, PrintStream err,
                  List<String> args) throws Exception {
 
-    OptionSet optionSet = parser.parse(args.toArray(new String[0]));
+    OptionSet optionSet = parser.parse(args.toArray(new String[args.size()]));
 
     if (!optionSet.nonOptionArguments().isEmpty()) {
       printUsage(err);

conf/credentials.sample

@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# In this file users can store their cloud login credentials for convenience
+# If this file exists it is sourced by Whirr scripts.
+#
+# Whirr will look for this file in the home directory first (~/.whirr/credentials)
+# If not found it will look for it in the conf directory.
+#
+# Preset environment variables will take precedence over these files.
+#
+# VARIABLES:
+#
+# PROVIDER - The cloud provider to use in Whirr
+# IDENTITY - The identity to use in Whirr
+# CREDENTIAL - The credential to use in Whirr
+#
+# BLOBSTORE_* - Overrides the base variables (e.g. PROVIDER) specifically for
+# blobstore contexts. Base variables are still used for compute access
+# If BLOBSTORE_* variables are not defined Whirr will use the base variables
+# for blobstore access.
+
+# Users can assign these variables the values they want to assign to
+# their WHIRR_* variable counterparts. If a WHIRR_* variable is found in env
+# then it takes precedence (to be able to do one-off overrides on recipes)
+# otherwise WHIRR_* variables take the value from this file. Finally .properties
+# files override both this file and previous env variables
+
+# Set cloud provider connection details
+
+PROVIDER=
+IDENTITY=
+CREDENTIAL=
+
+# Set blob store connection details. If not defined they are computed
+# from the cloud provider connection details defined above
+
+# BLOBSTORE_PROVIDER=
+# BLOBSTORE_IDENTITY=
+# BLOBSTORE_CREDENTIAL=
+

core/src/main/java/org/apache/whirr/ClusterSpec.java

@@ -434,7 +434,8 @@ private Configuration composeWithDefaults(Configuration userConfig)
       throws ConfigurationException {
     CompositeConfiguration composed = new CompositeConfiguration();
     composed.addConfiguration(userConfig);
-    composed.addConfiguration(new PropertiesConfiguration(getClass().getClassLoader().getResource(DEFAULT_PROPERTIES)));
+    composed.addConfiguration(new PropertiesConfiguration(
+        getClass().getClassLoader().getResource(DEFAULT_PROPERTIES)));
     return composed;
   }
 

core/src/main/java/org/apache/whirr/command/AbstractClusterCommand.java

@@ -18,22 +18,15 @@
 
 package org.apache.whirr.command;
 
-import static org.apache.whirr.ClusterSpec.Property.CLUSTER_NAME;
-import static org.apache.whirr.ClusterSpec.Property.IDENTITY;
-import static org.apache.whirr.ClusterSpec.Property.INSTANCE_TEMPLATES;
-
 import com.google.common.collect.Maps;
-
 import java.io.IOException;
 import java.io.PrintStream;
 import java.util.EnumSet;
 import java.util.Map;
-
 import joptsimple.ArgumentAcceptingOptionSpec;
 import joptsimple.OptionParser;
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
-
 import org.apache.commons.configuration.CompositeConfiguration;
 import org.apache.commons.configuration.Configuration;
 import org.apache.commons.configuration.ConfigurationException;
@@ -43,6 +36,12 @@
 import org.apache.whirr.ClusterControllerFactory;
 import org.apache.whirr.ClusterSpec;
 import org.apache.whirr.ClusterSpec.Property;
+import static org.apache.whirr.ClusterSpec.Property.CLUSTER_NAME;
+import static org.apache.whirr.ClusterSpec.Property.CREDENTIAL;
+import static org.apache.whirr.ClusterSpec.Property.IDENTITY;
+import static org.apache.whirr.ClusterSpec.Property.INSTANCE_TEMPLATES;
+import static org.apache.whirr.ClusterSpec.Property.PRIVATE_KEY_FILE;
+import static org.apache.whirr.ClusterSpec.Property.PROVIDER;
 import org.apache.whirr.state.ClusterStateStore;
 import org.apache.whirr.state.ClusterStateStoreFactory;
 import org.slf4j.Logger;
@@ -100,12 +99,14 @@ protected ClusterSpec getClusterSpec(OptionSet optionSet) throws ConfigurationEx
     for (Map.Entry<Property, OptionSpec<?>> entry : optionSpecs.entrySet()) {
       Property property = entry.getKey();
       OptionSpec<?> option = entry.getValue();
+      Object value;
       if (property.hasMultipleArguments()) {
-        optionsConfig.setProperty(property.getConfigName(),
-            optionSet.valuesOf(option));
+        value = optionSet.valuesOf(option);
       } else {
-        optionsConfig.setProperty(property.getConfigName(),
-            optionSet.valueOf(option));
+        value = optionSet.valueOf(option);
+      }
+      if (value != null) {
+        optionsConfig.setProperty(property.getConfigName(), value);
       }
     }
     CompositeConfiguration config = new CompositeConfiguration();
@@ -114,14 +115,17 @@ protected ClusterSpec getClusterSpec(OptionSet optionSet) throws ConfigurationEx
       Configuration defaults = new PropertiesConfiguration(optionSet.valueOf(configOption));
       config.addConfiguration(defaults);
     }
+    ClusterSpec clusterSpec = new ClusterSpec(config);
 
-    for (Property required : EnumSet.of(CLUSTER_NAME, IDENTITY, INSTANCE_TEMPLATES)) {
-      if (config.getString(required.getConfigName()) == null) {
+    for (Property required : EnumSet.of(CLUSTER_NAME, PROVIDER, IDENTITY, CREDENTIAL,
+        INSTANCE_TEMPLATES, PRIVATE_KEY_FILE)) {
+      if (clusterSpec.getConfiguration().getString(required.getConfigName()) == null) {
         throw new IllegalArgumentException(String.format("Option '%s' not set.",
             required.getSimpleName()));
       }
     }
-    return new ClusterSpec(config);
+
+    return clusterSpec;
   }
 
   /**

core/src/main/resources/whirr-default.properties

@@ -15,6 +15,10 @@ whirr.private-key-file=${sys:user.home}/.ssh/id_rsa
 # whirr.public-key-file = ${whirr.private-key-file}.pub
 # this is created in ClusterSpec class constructor
 
+whirr.provider=${env:WHIRR_PROVIDER}
+whirr.identity=${env:WHIRR_IDENTITY}
+whirr.credential=${env:WHIRR_CREDENTIAL}
+
 whirr.version=${version}
 
 whirr.max-startup-retries=1

core/src/test/java/org/apache/whirr/command/AbstractClusterCommandTest.java

@@ -18,21 +18,20 @@
 
 package org.apache.whirr.command;
 
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertThat;
+import joptsimple.OptionSet;
+import org.apache.whirr.ClusterControllerFactory;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.util.KeyPair;
+import org.junit.Test;
 
 import java.io.File;
 import java.io.InputStream;
 import java.io.PrintStream;
 import java.util.List;
 import java.util.Map;
 
-import joptsimple.OptionSet;
-
-import org.apache.whirr.ClusterControllerFactory;
-import org.apache.whirr.ClusterSpec;
-import org.apache.whirr.util.KeyPair;
-import org.junit.Test;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
 
 public class AbstractClusterCommandTest {
 
@@ -62,6 +61,7 @@ public int run(InputStream in, PrintStream out, PrintStream err,
    * Ensure that an invalid service name uses the default (after logging a
    * warning).
    */
+  @Test
   public void testCreateServerWithInvalidClusterControllerName() throws Exception {
     AbstractClusterCommand clusterCommand = new AbstractClusterCommand("name",
         "description", new ClusterControllerFactory()) {

recipes/cassandra-ec2.properties recipes/cassandra.properties

@@ -16,7 +16,7 @@
 #
 
 #
-# Cassandra Cluster on AWS EC2
+# Setup an Apache Cassandra Cluster
 # 
 
 # Read the Configuration Guide for more info:
@@ -25,14 +25,15 @@
 # Change the cluster name here
 whirr.cluster-name=cassandra
 
+# Setup your cloud credentials by copying conf/credentials.sample
+# to ~/.whirr/credentials and editing as needed
+
+# Change the name of cluster admin user
+whirr.cluster-user=${sys:user.name}
+
 # Change the number of machines in the cluster here
 whirr.instance-templates=3 cassandra
 
-# For EC2 set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables.
-whirr.provider=aws-ec2
-whirr.identity=${env:AWS_ACCESS_KEY_ID}
-whirr.credential=${env:AWS_SECRET_ACCESS_KEY}
-
 # By default use the user system SSH keys. Override them here.
 # whirr.private-key-file=${sys:user.home}/.ssh/id_rsa
 # whirr.public-key-file=${whirr.private-key-file}.pub