diff --git a/admin/installer/cloudformation.py b/admin/installer/cloudformation.py index fc53125fce..6c12540f67 100644 --- a/admin/installer/cloudformation.py +++ b/admin/installer/cloudformation.py @@ -141,7 +141,7 @@ def _validate_cluster_size(size): # Keys corresponding to CloudFormation user Inputs. access_key_id_param = template.add_parameter(Parameter( "AmazonAccessKeyID", - Description="Your Amazon AWS access key ID (mandatory)", + Description="Required: Your Amazon AWS access key ID", Type="String", NoEcho=True, AllowedPattern="[\w]+", @@ -150,25 +150,32 @@ def _validate_cluster_size(size): )) secret_access_key_param = template.add_parameter(Parameter( "AmazonSecretAccessKey", - Description="Your Amazon AWS secret access key (mandatory)", + Description="Required: Your Amazon AWS secret access key", Type="String", NoEcho=True, MinLength="1", )) keyname_param = template.add_parameter(Parameter( "EC2KeyPair", - Description="Name of an existing EC2 KeyPair to enable SSH " - "access to the instance (mandatory)", + Description="Required: Name of an existing EC2 KeyPair to enable SSH " + "access to the instance", + Type="AWS::EC2::KeyPair::KeyName", +)) +template.add_parameter(Parameter( + "S3AccessPolicy", + Description="Required: Is current IAM user allowed to access S3? " + "S3 access is required to distribute Flocker and Docker " + "configuration amongst stack nodes. Reference: " + "http://docs.aws.amazon.com/IAM/latest/UserGuide/" + "access_permissions.html Stack creation will fail if user " + "cannot access S3", Type="String", - MinLength="1", - AllowedPattern="[\x20-\x7E]*", - MaxLength="255", - ConstraintDescription="can contain only ASCII characters.", + AllowedValues=["Yes"], )) volumehub_token = template.add_parameter(Parameter( "VolumeHubToken", Description=( - "Your Volume Hub token (optional). " + "Optional: Your Volume Hub token. " "You'll find the token at https://volumehub.clusterhq.com/v1/token." ), Type="String", diff --git a/flocker/acceptance/endtoend/installer/postgres/docker-compose-node1.yml b/admin/test/installer/postgres/docker-compose-node1.yml similarity index 100% rename from flocker/acceptance/endtoend/installer/postgres/docker-compose-node1.yml rename to admin/test/installer/postgres/docker-compose-node1.yml diff --git a/flocker/acceptance/endtoend/installer/postgres/docker-compose-node2.yml b/admin/test/installer/postgres/docker-compose-node2.yml similarity index 100% rename from flocker/acceptance/endtoend/installer/postgres/docker-compose-node2.yml rename to admin/test/installer/postgres/docker-compose-node2.yml diff --git a/flocker/acceptance/endtoend/test_installer.py b/admin/test/test_installer.py similarity index 79% rename from flocker/acceptance/endtoend/test_installer.py rename to admin/test/test_installer.py index a89cd6cca8..faa59dd4fe 100644 --- a/flocker/acceptance/endtoend/test_installer.py +++ b/admin/test/test_installer.py @@ -18,11 +18,12 @@ from eliot import Message -from ...common.runner import run_ssh -from ...common import gather_deferreds, loop_until, retry_failure -from ...testtools import AsyncTestCase, async_runner, random_name -from ..testtools import connected_cluster - +from flocker.common.runner import run_ssh +from flocker.common import gather_deferreds, loop_until, retry_failure +from flocker.testtools import AsyncTestCase, async_runner, random_name +from flocker.acceptance.testtools import ( + connected_cluster, acceptance_yaml_for_test, extract_substructure_for_test +) RECREATE_STATEMENT = 'create table test(i int);' INSERT_STATEMENT = 'insert into test values(1);' @@ -34,6 +35,8 @@ POSTGRESQL_USERNAME = 'flocker' POSTGRESQL_PASSWORD = 'flocker' +CLOUDFORMATION_TEMPLATE_URL = "https://s3.amazonaws.com/installer.downloads.clusterhq.com/flocker-cluster.cloudformation.json" # noqa + def remote_command(client_ip, command): """ @@ -106,31 +109,54 @@ def remote_postgres(client_ip, host, command): ) -def get_stack_report(stack_id): +def aws_output(args, aws_config): + """ + Run the ``aws`` command line tool with the supplied subcommand ``args`` and + the supplied ``aws_config`` as environment variables. + + :param list args: The list of ``aws`` arguments (including sub-command). + :param dict aws_config: environment variables to be merged with the current + process environment before running the ``aws`` sub-command. + :returns: The ``bytes`` output of the ``aws`` command. + """ + environment = os.environ.copy() + environment.update(aws_config) + return check_output( + ['aws'] + args, + env=environment + ) + + +def get_stack_report(stack_id, aws_config): """ Get information about a CloudFormation stack. :param unicode stack_id: The AWS cloudformation stack ID. + :param dict aws_config: environment variables to be merged with the current + process environment before running the ``aws`` sub-command. :returns: A ``dict`` of information about the stack. """ - output = check_output( - ['aws', 'cloudformation', 'describe-stacks', - '--stack-name', stack_id] + output = aws_output( + ['cloudformation', 'describe-stacks', + '--stack-name', stack_id], + aws_config ) results = json.loads(output) return results['Stacks'][0] -def wait_for_stack_status(stack_id, target_status): +def wait_for_stack_status(stack_id, target_status, aws_config): """ Poll the status of a CloudFormation stack. :param unicode stack_id: The AWS cloudformation stack ID. :param unicode target_status: The desired stack status. + :param dict aws_config: environment variables to be merged with the current + process environment before running the ``aws`` sub-command. :returns: A ``Deferred`` which fires when the stack has ``target_status``. """ def predicate(): - stack_report = get_stack_report(stack_id) + stack_report = get_stack_report(stack_id, aws_config) current_status = stack_report['StackStatus'] Message.log( function='wait_for_stack_status', @@ -146,42 +172,49 @@ def predicate(): repeat(10, 120)) -def create_cloudformation_stack(template_url, access_key_id, - secret_access_key, parameters): +def create_cloudformation_stack(template_url, parameters, aws_config): """ Create a CloudFormation stack. - :param unicode stack_id: The AWS cloudformation stack ID. + :param unicode template_url: Cloudformation template URL on S3. + :param dict parameters: The parameters required by the template. + :param dict aws_config: environment variables to be merged with the current + process environment before running the ``aws`` sub-command. + :returns: A ``Deferred`` which fires when the stack has been created. """ # Request stack creation. stack_name = CLOUDFORMATION_STACK_NAME + str(int(time.time())) - output = check_output( - ['aws', 'cloudformation', 'create-stack', + output = aws_output( + ['cloudformation', 'create-stack', '--disable-rollback', '--parameters', json.dumps(parameters), '--stack-name', stack_name, - '--template-url', template_url] + '--template-url', template_url], + aws_config ) output = json.loads(output) stack_id = output['StackId'] Message.new(cloudformation_stack_id=stack_id) - return wait_for_stack_status(stack_id, 'CREATE_COMPLETE') + return wait_for_stack_status(stack_id, 'CREATE_COMPLETE', aws_config) -def delete_cloudformation_stack(stack_id): +def delete_cloudformation_stack(stack_id, aws_config): """ Delete a CloudFormation stack. :param unicode stack_id: The AWS cloudformation stack ID. + :param dict aws_config: environment variables to be merged with the current + process environment before running the ``aws`` sub-command. :returns: A ``Deferred`` which fires when the stack has been deleted. """ - check_output( - ['aws', 'cloudformation', 'delete-stack', - '--stack-name', stack_id] + aws_output( + ['cloudformation', 'delete-stack', + '--stack-name', stack_id], + aws_config, ) - return wait_for_stack_status(stack_id, 'DELETE_COMPLETE') + return wait_for_stack_status(stack_id, 'DELETE_COMPLETE', aws_config) def get_output(outputs, key): @@ -235,40 +268,56 @@ def _stack_from_environment(self): def _new_stack(self): """ Create a new CloudFormation stack from a template URL supplied as an - environment variable. AWS credentials and CloudFormation parameter - values must also be supplied as environment variables. + environment variable. AWS credentials and CloudFormation parameters are + gathered from an ``acceptance.yml`` style configuration file. """ - template_url = os.environ.get('CLOUDFORMATION_TEMPLATE_URL') - if template_url is None: - self.skipTest( - 'CLOUDFORMATION_TEMPLATE_URL environment variable not found. ' - ) - access_key_id = os.environ['ACCESS_KEY_ID'] - secret_access_key = os.environ['SECRET_ACCESS_KEY'] + config = extract_substructure_for_test( + test_case=self, + substructure=dict( + aws=dict( + access_key=u"", + secret_access_token=u"", + keyname=u"", + region=u"" + ), + ), + config=acceptance_yaml_for_test(self) + ) + template_url = os.environ.get( + 'CLOUDFORMATION_TEMPLATE_URL', CLOUDFORMATION_TEMPLATE_URL + ) + parameters = [ { 'ParameterKey': 'EC2KeyPair', - 'ParameterValue': os.environ['KEY_PAIR'] + 'ParameterValue': config["aws"]["keyname"] }, { 'ParameterKey': 'AmazonAccessKeyID', - 'ParameterValue': os.environ['ACCESS_KEY_ID'] + 'ParameterValue': config["aws"]["access_key"] }, { 'ParameterKey': 'AmazonSecretAccessKey', - 'ParameterValue': os.environ['SECRET_ACCESS_KEY'] + 'ParameterValue': config["aws"]["secret_access_token"] }, { 'ParameterKey': 'VolumeHubToken', - 'ParameterValue': os.environ['VOLUMEHUB_TOKEN'] + 'ParameterValue': os.environ.get('VOLUMEHUB_TOKEN', '') + }, + { + 'ParameterKey': 'S3AccessPolicy', + 'ParameterValue': 'Yes' } ] - d = create_cloudformation_stack( - template_url, - access_key_id, secret_access_key, parameters + aws_config = dict( + AWS_ACCESS_KEY_ID=config["aws"]["access_key"], + AWS_SECRET_ACCESS_KEY=config["aws"]["secret_access_token"], + AWS_DEFAULT_REGION=config["aws"]["region"], ) + d = create_cloudformation_stack(template_url, parameters, aws_config) + def set_stack_variables(stack_report): outputs = stack_report['Outputs'] stack_id = stack_report['StackId'] @@ -277,7 +326,9 @@ def set_stack_variables(stack_report): self, variable_name, get_output(outputs, stack_output_name) ) if 'KEEP_STACK' not in os.environ: - self.addCleanup(delete_cloudformation_stack, stack_id) + self.addCleanup( + delete_cloudformation_stack, stack_id, aws_config + ) d.addCallback(set_stack_variables) return d diff --git a/build.yaml b/build.yaml index d1a6a6e910..bc5f99891b 100644 --- a/build.yaml +++ b/build.yaml @@ -1280,7 +1280,7 @@ job_type: } timeout: 30 directories_to_delete: [] - notify_slack: '#engineering' + notify_slack: '#nightly-builds' run_docker_build_ubuntu_trusty_fpm: at: '0 1 * * *' @@ -1309,7 +1309,7 @@ job_type: } timeout: 30 directories_to_delete: [] - notify_slack: '#engineering' + notify_slack: '#nightly-builds' build_vagrant_basebox_for_osx_yosemite: at: '0 5 * * *' @@ -1357,7 +1357,7 @@ job_type: } timeout: 120 directories_to_delete: [] - notify_slack: '#engineering' + notify_slack: '#nightly-builds' run_client_installation_on_OSX: at: '30 7 * * *' @@ -1399,7 +1399,7 @@ job_type: # actually run the tests. Also has to be less than the timeout for the # main multijob, see jobs.groovy.j2:545. timeout: 50 - notify_slack: '#engineering' + notify_slack: '#nightly-builds' run_acceptance_on_AWS_CentOS_7_with_EBS: at: '0 5 * * *' @@ -1430,7 +1430,7 @@ job_type: # as of Dec 2015. timeout: 120 directories_to_delete: [] - notify_slack: '#engineering' + notify_slack: '#nightly-builds' run_acceptance_on_AWS_Ubuntu_Trusty_with_EBS: at: '0 6 * * *' @@ -1458,7 +1458,7 @@ job_type: # but slightly shorter since Ubuntu runs the tests faster. timeout: 90 directories_to_delete: [] - notify_slack: '#engineering' + notify_slack: '#nightly-builds' run_acceptance_on_GCE_CentOS_7_with_LOOPBACK: at: '0 5 * * *' @@ -1485,7 +1485,7 @@ job_type: # Reasoning as for run_acceptance_on_AWS_CentOS_7_with_EBS timeout: 120 directories_to_delete: [] - notify_slack: '#engineering' + notify_slack: '#nightly-builds' run_acceptance_on_GCE_Ubuntu_Trusty_with_LOOPBACK: at: '0 6 * * *' @@ -1512,7 +1512,7 @@ job_type: # Reasoning as for run_acceptance_on_AWS_Ubuntu_Trusty_with_EBS timeout: 90 directories_to_delete: [] - notify_slack: '#engineering' + notify_slack: '#nightly-builds' run_acceptance_on_Rackspace_CentOS_7_with_Cinder: at: '0 5 * * *' @@ -1541,7 +1541,7 @@ job_type: # Reasoning as for run_acceptance_on_AWS_CentOS_7_with_EBS timeout: 120 directories_to_delete: [] - notify_slack: '#engineering' + notify_slack: '#nightly-builds' run_acceptance_on_Rackspace_Ubuntu_Trusty_with_Cinder: at: '0 6 * * *' @@ -1570,7 +1570,7 @@ job_type: # Reasoning as for run_acceptance_on_AWS_Ubuntu_Trusty_with_EBS timeout: 90 directories_to_delete: [] - notify_slack: '#engineering' + notify_slack: '#nightly-builds' run_sphinx_link_check: at: '0 8 * * *' @@ -1583,7 +1583,7 @@ job_type: } timeout: 10 archive_artifacts: ["docs/_build/linkcheck/output.txt"] - notify_slack: '#engineering' + notify_slack: '#nightly-builds' cleanup_cloud_resources: at: '0 8 * * *' @@ -1600,6 +1600,33 @@ job_type: "cleanup_cloud_resources.stderr" ] + run_admin_test_test_installer: + at: '0 8 * * *' + on_nodes_with_labels: 'aws-ubuntu-trusty-T2Medium' + module: admin.test.test_installer + with_steps: + - { type: 'shell', + cli: [ *hashbang, + *add_shell_functions, + *setup_pip_cache, + *cleanup, + *setup_venv, + *setup_flocker_modules, + *setup_coverage, + *setup_aws_env_vars, + 'export ACCEPTANCE_YAML=/tmp/acceptance.yaml', + *run_trial_with_coverage, + *run_coverage, + *convert_results_to_junit ] + } + clean_repo: true + archive_artifacts: *flocker_artifacts + publish_test_results: true + # CloudFormation stack creation takes 5-10 minutes and the test itself + # may take another 10. Give 30 minutes to be sure. + timeout: 30 + directories_to_delete: [] + #-----------------------------------------------------------------------------# diff --git a/docs/gettinginvolved/acceptance-testing.rst b/docs/gettinginvolved/acceptance-testing.rst index f56305c551..768dbbaaa3 100644 --- a/docs/gettinginvolved/acceptance-testing.rst +++ b/docs/gettinginvolved/acceptance-testing.rst @@ -351,17 +351,17 @@ CloudFormation Installer Tests ============================== There are tests for the Flocker CloudFormation installer. +These tests will get AWS credentials from an ``acceptance.yml`` file. +The ``acceptance.yml`` file format is described above. You can run them as follows: .. code-block:: console - CLOUDFORMATION_TEMPLATE_URL=https://s3.amazonaws.com/installer.downloads.clusterhq.com/flocker-cluster.cloudformation.json \ - KEY_PAIR= \ - ACCESS_KEY_ID= \ - SECRET_ACCESS_KEY= \ + ACCEPTANCE_YAML= \ + CLOUDFORMATION_TEMPLATE_URL= \ VOLUMEHUB_TOKEN= \ - trial flocker.acceptance.endtoend.test_installer + trial admin.test.test_installer This will create a new CloudFormation stack and perform the tests on it. @@ -377,4 +377,4 @@ Alternatively, you can perform the tests on an existing stack with the following AGENT_NODE2_IP= \ CLIENT_NODE_IP= \ CONTROL_NODE_IP= \ - trial flocker.acceptance.endtoend.test_installer + trial admin.test.test_installer diff --git a/flocker/acceptance/testtools.py b/flocker/acceptance/testtools.py index e3f7c62df8..301628b89c 100644 --- a/flocker/acceptance/testtools.py +++ b/flocker/acceptance/testtools.py @@ -37,6 +37,9 @@ ) from ..common import gather_deferreds, loop_until, timeout, retry_failure +from ..common.configuration import ( + extract_substructure, MissingConfigError, Optional +) from ..common.runner import download_file, run_ssh from ..control.httpapi import REST_API_PORT @@ -1335,3 +1338,47 @@ def assert_http_server(test, host, port, d = query_http_server(host, port, path) d.addCallback(test.assertEqual, expected_response) return d + + +def acceptance_yaml_for_test(test_case): + """ + Load configuration from a yaml file specified in an environment variable. + + Raises a SkipTest exception if the environment variable is not specified. + """ + _ENV_VAR = 'ACCEPTANCE_YAML' + filename = environ.get(_ENV_VAR) + if not filename: + test_case.skip( + 'Must set {} to an acceptance.yaml file (' + 'http://doc-dev.clusterhq.com/gettinginvolved/appendix.html#acceptance-testing-configuration' # noqa + ') plus additional keys in order to run this test.'.format( + _ENV_VAR)) + with open(filename) as f: + config = yaml.safe_load(f) + return config + + +def extract_substructure_for_test(test_case, substructure, config): + """ + Extract the keys from the config in substructure, which may be a nested + dictionary. + + Raises a ``unittest.SkipTest`` if the substructure is not found in the + configuration. + + This can be used to load credentials all at once for testing purposes. + """ + try: + return extract_substructure(config, substructure) + except MissingConfigError as e: + yaml.add_representer( + Optional, + lambda d, x: d.represent_scalar(u'tag:yaml.org,2002:str', repr(x))) + test_case.skip( + 'Skipping test: could not get configuration: {}\n\n' + 'In order to run this test, add ensure file at $ACCEPTANCE_YAML ' + 'has structure like:\n\n{}'.format( + e.message, + yaml.dump(substructure, default_flow_style=False)) + ) diff --git a/flocker/common/configuration.py b/flocker/common/configuration.py new file mode 100644 index 0000000000..6572d430c8 --- /dev/null +++ b/flocker/common/configuration.py @@ -0,0 +1,76 @@ +# Copyright ClusterHQ Inc. See LICENSE file for details. + +""" +Helpers for loading and parsing YAML configuration. +""" + + +class MissingConfigError(Exception): + """ + Error that is raised to indicate that some required configuration key was + not specified. + """ + pass + + +class Optional(object): + """ + Object for configuring optional configuration values. + """ + + def __init__(self, default, description): + self.default = default + self.description = description + + def __repr__(self): + return "{} (optional default={})".format(self.description, + repr(self.default)) + + +def _is_optional(substructure): + """ + Determines if a substructure is an optional part of the configuration. + """ + if type(substructure) == Optional: + return True + if type(substructure) is dict: + for value in substructure.values(): + if not _is_optional(value): + return False + return True + return False + + +def extract_substructure(base, substructure): + """ + Assuming that substructure is a possibly nested dictionary, return a new + dictionary with the same keys (and subkeys) as substructure, but extract + the leaf values from base. + + This is used to extract and verify a configuration from a yaml blob. + """ + if (type(substructure) is not dict and + type(base) is not dict): + return base + if type(base) is not dict: + raise MissingConfigError( + "Found non-dict value {} when expecting a sub-configuration " + "{}.".format(repr(base), repr(substructure))) + if type(substructure) is not dict: + raise MissingConfigError( + "Found dict value {} when expecting a simple configuration value " + "{}.".format(repr(base), repr(substructure))) + try: + subdict = [] + for key, value in substructure.iteritems(): + if type(value) is Optional: + base_val = base.get(key, value.default) + elif _is_optional(value): + base_val = base.get(key, {}) + else: + base_val = base[key] + subdict.append((key, extract_substructure(base_val, value))) + return dict(subdict) + except KeyError as e: + raise MissingConfigError( + "Missing key {} in configuration".format(e.args[0]))