diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 0b0c28a9..bdd0b99a 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -116,3 +116,6 @@ - Fix bug in attach from 2.5.9 2.5.11: - Fix bug in attach from 2.5.10 +2.5.12: + - Handle resources that don't provide iface. + - Add iface to lambda invoke. diff --git a/cloudify_aws/common/decorators.py b/cloudify_aws/common/decorators.py index 9eb1a064..f4536f85 100644 --- a/cloudify_aws/common/decorators.py +++ b/cloudify_aws/common/decorators.py @@ -154,7 +154,7 @@ def wrapper_outer(function): '''Outer function''' def wrapper_inner(*argc, **kwargs): ctx = kwargs.get('ctx') - iface = kwargs.get('iface') + iface = kwargs['iface'] resource_config = kwargs.get('resource_config') # Create a copy of the resource config for clean manipulation. diff --git a/cloudify_aws/common/tests/test_iface_requirement.py b/cloudify_aws/common/tests/test_iface_requirement.py new file mode 100644 index 00000000..c85c46a5 --- /dev/null +++ b/cloudify_aws/common/tests/test_iface_requirement.py @@ -0,0 +1,194 @@ +# Copyright (c) 2021 Cloudify Platform Ltd. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import yaml +from mock import patch, MagicMock +from importlib import import_module + +from cloudify.state import current_ctx +from cloudify.exceptions import (OperationRetry, NonRecoverableError) + +from cloudify_aws.common.tests.test_base import TestBase + +REL_LIFE = 'cloudify.interfaces.relationship_lifecycle' + + +def get_callable(operation_mapping): + if not isinstance(operation_mapping, dict): + raise Exception( + 'Operation {op} is not dict.'.format(op=operation_mapping)) + elif 'implementation' not in operation_mapping: + return + elif operation_mapping['implementation'] == '~': + return + elif not operation_mapping['implementation']: + return + modules = operation_mapping['implementation'].split('.') + del modules[0] + func = modules.pop(-1) + import_path = '.'.join(modules) + module = import_module(import_path) + return module and getattr(module, func, None) + + +class testIfaceRequirement(TestBase): + + @staticmethod + def get_plugin_yaml(): + plugin_yaml_path = os.path.abspath( + os.path.join( + os.path.dirname(__file__), + '..', '..', '..', 'plugin.yaml')) + plugin_yaml_file = open(plugin_yaml_path, 'r') + return yaml.load(plugin_yaml_file, Loader=yaml.FullLoader) + + @staticmethod + def get_node_type_operations(plugin_yaml): + operations = [] + for _, node in plugin_yaml['node_types'].items(): + try: + op_list = node['interfaces']['cloudify.interfaces.lifecycle'] + except KeyError: + continue + for _, op in op_list.items(): + module = get_callable(op) + operations.append(module) + return operations + + @staticmethod + def get_relationships_operations(plugin_yaml): + operations = [] + for _, rel in plugin_yaml['relationships'].items(): + op_list = rel.get('source_interfaces', {}).get(REL_LIFE, {}) + op_list.update(rel.get('target_interfaces', {}).get(REL_LIFE, {})) + for _, op in op_list.items(): + module = get_callable(op) + operations.append(module) + return operations + + def get_op_ctx(self, operation): + mock_group = MagicMock() + mock_group.type_hierarchy = [ + 'cloudify.relationships.depends_on', + 'cloudify.relationships.contained_in' + ] + mock_group.target.instance.runtime_properties = { + 'aws_resource_id': 'aws_id', + 'aws_resource_arn': 'foo', + 'ListenerArn': 'arn:aws:foo', + 'resource_config': {} + } + mock_group.target.node.type_hierarchy = [ + 'cloudify.nodes.Root', + 'cloudify.nodes.aws.autoscaling.Group', + 'cloudify.nodes.aws.SNS.Topic', + 'cloudify.nodes.aws.elb.Classic.LoadBalancer', + 'cloudify.nodes.aws.elb.LoadBalancer', + 'cloudify.nodes.aws.elb.Listener', + 'cloudify.nodes.aws.ec2.Vpc', + 'cloudify.nodes.aws.s3.Bucket', + 'cloudify.nodes.aws.ec2.NetworkACL', + 'cloudify.nodes.aws.ec2.RouteTable', + 'cloudify.nodes.aws.efs.FileSystem', + 'cloudify.nodes.aws.ec2.Subnet', + 'cloudify.nodes.aws.kms.CustomerMasterKey', + 'cloudify.nodes.aws.ecs.Cluster' + ] + _ctx = self.get_mock_ctx( + operation, test_relationships=[mock_group]) + _ctx.instance.runtime_properties['aws_resource_id'] = 'foo' + _ctx.instance.runtime_properties['LoadBalancerName'] = 'foo' + _ctx.instance.runtime_properties['PolicyName'] = 'foo' + _ctx.instance.runtime_properties['rule_number'] = 'foo' + _ctx.instance.runtime_properties['egress'] = 'foo' + _ctx.instance.runtime_properties['network_acl_id'] = 'foo' + _ctx.instance.runtime_properties['vpc_id'] = 'foo' + _ctx.instance.runtime_properties['association_ids'] = \ + ['foo'] + _ctx.instance.runtime_properties['destination_cidr_block'] = \ + 'foo' + _ctx.instance.runtime_properties['AutoScalingGroupName'] = \ + 'foo' + _ctx.instance.runtime_properties['KeyId'] = 'foo' + _ctx.instance.runtime_properties['instances'] = ['foo'] + _ctx.instance.runtime_properties['resource_config'] = { + 'HostedZoneId': 'foo', + 'ChangeBatch': { + 'Changes': [{'ResourceRecordSet': 'foo'}] + }, + 'Endpoint': 'arn:aws:foo', + 'Key': 'foo', + 'GroupName': 'foo', + 'KeyName': 'foo', + 'DhcpConfigurations': ['foo'], + 'Type': 'foo', + 'DestinationCidrBlock': 'foo', + 'Targets': [{'Id': 'foo'}], + 'KeyId': 'foo', + } + _ctx.node.properties['client_config'] = \ + {'region_name': 'eu-west-1'} + _ctx.node.properties['resource_config'] = {'kwargs': {}} + _ctx.node.properties['log_create_response'] = False + _ctx.node.properties['create_secret'] = False + _ctx.node.properties['store_kube_config_in_runtime'] = \ + False + return _ctx + + def perform_operation(self, operation_callable, args, kwargs): + try: + operation_callable(*args, **kwargs) + except NonRecoverableError as e: + if 'unexpected status' in str(e): + return + elif 'Found no AMIs matching provided filters' in str(e): + return + raise + except OperationRetry as e: + if 'pending state' in str(e): + return + elif 'Updating Autoscaling Group' in str(e): + return + elif 'Waiting for Instance' in str(e): + return + raise + except AttributeError as e: + if "no attribute 'node'" not in str(e): + raise + kwargs['ctx'] = self.get_mock_relationship_ctx( + operation_callable, + test_source=self.get_op_ctx(operation_callable), + test_target=self.get_op_ctx(operation_callable)) + self.perform_operation(operation_callable, args, kwargs) + + @patch('cloudify_aws.common.connection.Boto3Connection') + @patch('cloudify_aws.common.decorators._wait_for_status') + @patch('cloudify_aws.common.AWSResourceBase.make_client_call') + @patch('cloudify_aws.common.connection.Boto3Connection.client') + @patch('cloudify.context.CloudifyContext._verify_in_relationship_context') + def test_iface_requirement(self, _, __, ___, ____, _____): + plugin_yaml = self.get_plugin_yaml() + operations = self.get_node_type_operations(plugin_yaml) + \ + self.get_relationships_operations(plugin_yaml) + for operation in operations: + if operation: + _ctx = self.get_op_ctx(operation) + current_ctx.set(ctx=_ctx) + args = tuple() + kwargs = dict( + ctx=_ctx, + resource_config={}, + force_delete=False) + self.perform_operation(operation, args, kwargs) diff --git a/cloudify_aws/ec2/resources/subnet.py b/cloudify_aws/ec2/resources/subnet.py index 6f237e03..6441a060 100644 --- a/cloudify_aws/ec2/resources/subnet.py +++ b/cloudify_aws/ec2/resources/subnet.py @@ -96,7 +96,7 @@ def modify_subnet_attribute(self, params=None): @decorators.aws_resource(EC2Subnet, resource_type=RESOURCE_TYPE) -def prepare(ctx, iface, resource_config, **_): +def prepare(ctx, resource_config, **_): '''Prepares an AWS EC2 Subnet''' # Save the parameters ctx.instance.runtime_properties['resource_config'] = resource_config diff --git a/cloudify_aws/ec2/tests/test_subnet.py b/cloudify_aws/ec2/tests/test_subnet.py index e97822d1..30aaa617 100644 --- a/cloudify_aws/ec2/tests/test_subnet.py +++ b/cloudify_aws/ec2/tests/test_subnet.py @@ -103,9 +103,9 @@ def test_class_delete(self): def test_prepare(self): ctx = self.get_mock_ctx("Subnet") config = {SUBNET_ID: 'subnet', CIDR_BLOCK: 'cidr_block'} - iface = MagicMock() - iface.create = self.mock_return(config) - subnet.prepare(ctx, iface, config) + # iface = MagicMock() + # iface.create = self.mock_return(config) + subnet.prepare(ctx, config) self.assertEqual(ctx.instance.runtime_properties['resource_config'], config) diff --git a/cloudify_aws/elb/resources/classic/listener.py b/cloudify_aws/elb/resources/classic/listener.py index f8849dba..7cc325bc 100644 --- a/cloudify_aws/elb/resources/classic/listener.py +++ b/cloudify_aws/elb/resources/classic/listener.py @@ -122,6 +122,6 @@ def delete(ctx, iface, resource_config, **_): if not lb: lb = ctx.instance.runtime_properties[LB_NAME] - ports = [listener.get(LB_PORT, None) - for listener in params.get(LISTENERS)] - iface.delete({LB_NAME: lb, LB_PORTS: ports}) + for listener in params.get(LISTENERS, []): + ports = listener.get(LB_PORT) + iface.delete({LB_NAME: lb, LB_PORTS: ports}) diff --git a/cloudify_aws/elb/resources/classic/load_balancer.py b/cloudify_aws/elb/resources/classic/load_balancer.py index e57d47d4..1dd7cc70 100644 --- a/cloudify_aws/elb/resources/classic/load_balancer.py +++ b/cloudify_aws/elb/resources/classic/load_balancer.py @@ -190,7 +190,7 @@ def start(ctx, iface, resource_config, **_): @decorators.aws_resource(ELBClassicLoadBalancer, RESOURCE_TYPE, ignore_properties=True) -def delete(iface, resource_config, **_): +def delete(ctx, iface, resource_config, **_): """Deletes an AWS ELB classic load balancer""" # Create a copy of the resource config for clean manipulation. @@ -204,7 +204,7 @@ def delete(iface, resource_config, **_): iface.delete(params) -@decorators.aws_relationship(None, RESOURCE_TYPE) +@decorators.aws_relationship(ELBClassicLoadBalancer, RESOURCE_TYPE) def assoc(ctx, **_): """associate instance with ELB classic LB""" instance_id = \ @@ -230,7 +230,7 @@ def assoc(ctx, **_): instance_id, lb)) -@decorators.aws_relationship(None, RESOURCE_TYPE) +@decorators.aws_relationship(ELBClassicLoadBalancer, RESOURCE_TYPE) def disassoc(ctx, **_): """disassociate instance with ELB classic LB""" instance_id = \ diff --git a/cloudify_aws/elb/resources/target_group.py b/cloudify_aws/elb/resources/target_group.py index 72130a0a..cf57caeb 100644 --- a/cloudify_aws/elb/resources/target_group.py +++ b/cloudify_aws/elb/resources/target_group.py @@ -144,8 +144,7 @@ def delete(iface, resource_config, **_): iface.delete(resource_config) -@decorators.aws_resource(ELBTargetGroup, - RESOURCE_TYPE) +@decorators.aws_resource(ELBTargetGroup, RESOURCE_TYPE) def modify(ctx, iface, resource_config, **_): '''modify an AWS ELB target group attributes''' # Build API params diff --git a/cloudify_aws/iam/resources/access_key.py b/cloudify_aws/iam/resources/access_key.py index 64dd3514..42851314 100644 --- a/cloudify_aws/iam/resources/access_key.py +++ b/cloudify_aws/iam/resources/access_key.py @@ -23,7 +23,7 @@ RESOURCE_TYPE = 'IAM User Access Key' -@decorators.aws_resource(resource_type=RESOURCE_TYPE) +@decorators.aws_resource(IAMUser, RESOURCE_TYPE) def configure(ctx, resource_config, **_): '''Configures an AWS IAM Access Key''' # Save the parameters @@ -31,7 +31,7 @@ def configure(ctx, resource_config, **_): utils.clean_params(resource_config) -@decorators.aws_relationship(resource_type=RESOURCE_TYPE) +@decorators.aws_relationship(IAMUser, RESOURCE_TYPE) def attach_to(ctx, resource_config, **_): '''Attaches an IAM Access Key to something else''' rtprops = ctx.source.instance.runtime_properties @@ -49,7 +49,7 @@ def attach_to(ctx, resource_config, **_): resp['SecretAccessKey'] -@decorators.aws_relationship(resource_type=RESOURCE_TYPE) +@decorators.aws_relationship(IAMUser, RESOURCE_TYPE) def detach_from(ctx, resource_config, **_): '''Detaches an IAM Access Key from something else''' if utils.is_node_type(ctx.target.node, @@ -58,7 +58,8 @@ def detach_from(ctx, resource_config, **_): node=ctx.source.node, instance=ctx.source.instance, raise_on_missing=True) - IAMUser(ctx.target.node, logger=ctx.logger, + IAMUser(ctx.target.node, + logger=ctx.logger, resource_id=utils.get_resource_id( node=ctx.target.node, instance=ctx.target.instance, diff --git a/cloudify_aws/iam/resources/login_profile.py b/cloudify_aws/iam/resources/login_profile.py index 72c14504..305ead56 100644 --- a/cloudify_aws/iam/resources/login_profile.py +++ b/cloudify_aws/iam/resources/login_profile.py @@ -23,7 +23,7 @@ RESOURCE_TYPE = 'IAM User Login Profile' -@decorators.aws_resource(resource_type=RESOURCE_TYPE) +@decorators.aws_resource(IAMUser, RESOURCE_TYPE) def configure(ctx, resource_config, **_): '''Configures an AWS IAM Login Profile''' # Save the parameters @@ -31,7 +31,7 @@ def configure(ctx, resource_config, **_): utils.clean_params(resource_config) -@decorators.aws_relationship(resource_type=RESOURCE_TYPE) +@decorators.aws_relationship(IAMUser, RESOURCE_TYPE) def attach_to(ctx, resource_config, **_): '''Attaches an IAM Login Profile to something else''' rtprops = ctx.source.instance.runtime_properties @@ -46,7 +46,7 @@ def attach_to(ctx, resource_config, **_): raise_on_missing=True)).create_login_profile(params) -@decorators.aws_relationship(resource_type=RESOURCE_TYPE) +@decorators.aws_relationship(IAMUser, RESOURCE_TYPE) def detach_from(ctx, resource_config, **_): '''Detaches an IAM Login Profile from something else''' if utils.is_node_type(ctx.target.node, diff --git a/cloudify_aws/iam/tests/test_access_key.py b/cloudify_aws/iam/tests/test_access_key.py index e3a11d0b..b27863a6 100644 --- a/cloudify_aws/iam/tests/test_access_key.py +++ b/cloudify_aws/iam/tests/test_access_key.py @@ -73,7 +73,6 @@ def test_attach_to_User(self): 'SecretAccessKey': 'aws_secret_access_key' } }) - access_key.attach_to( ctx=_ctx, resource_config=None, iface=None ) diff --git a/cloudify_aws/iam/tests/test_login_profile.py b/cloudify_aws/iam/tests/test_login_profile.py index 3ddb9e54..7a5372b2 100644 --- a/cloudify_aws/iam/tests/test_login_profile.py +++ b/cloudify_aws/iam/tests/test_login_profile.py @@ -59,7 +59,7 @@ def test_configure(self): type_name='iam', type_class=login_profile) - def test_attach_to_User(self): + def test_attach_to_user(self): _source_ctx, _target_ctx, _ctx = self._create_common_relationships( 'test_attach_to', LOGIN_PROFILE_TH, @@ -86,7 +86,7 @@ def test_attach_to_User(self): } ) - def test_detach_from_User(self): + def test_detach_from_user(self): _source_ctx, _target_ctx, _ctx = self._create_common_relationships( 'test_detach_from', LOGIN_PROFILE_TH, diff --git a/cloudify_aws/lambda_serverless/resources/invoke.py b/cloudify_aws/lambda_serverless/resources/invoke.py index ecedcb8e..421d9dac 100644 --- a/cloudify_aws/lambda_serverless/resources/invoke.py +++ b/cloudify_aws/lambda_serverless/resources/invoke.py @@ -24,14 +24,14 @@ RESOURCE_TYPE = 'Lambda Function Invocation' -@decorators.aws_resource(resource_type=RESOURCE_TYPE) +@decorators.aws_resource(LambdaFunction, RESOURCE_TYPE) def configure(ctx, resource_config, **_): '''Configures an AWS Lambda Invoke''' # Save the parameters ctx.instance.runtime_properties['resource_config'] = resource_config -@decorators.aws_relationship(resource_type=RESOURCE_TYPE) +@decorators.aws_relationship(LambdaFunction, RESOURCE_TYPE) def attach_to(ctx, resource_config, **_): '''Attaches an Lambda Invoke to something else''' rtprops = ctx.source.instance.runtime_properties @@ -40,18 +40,20 @@ def attach_to(ctx, resource_config, **_): ctx.source.node.properties.get('resource_encoding') if utils.is_node_type(ctx.target.node, 'cloudify.nodes.aws.lambda.Function'): - ctx.source.instance.runtime_properties['output'] = LambdaFunction( + lambda_fn = LambdaFunction( ctx.target.node, logger=ctx.logger, resource_encoding=resource_encoding, resource_id=utils.get_resource_id( node=ctx.target.node, instance=ctx.target.instance, - raise_on_missing=True)).invoke( - resource_config or rtprops.get('resource_config')) + raise_on_missing=True)) + result = lambda_fn.invoke(resource_config or rtprops.get( + 'resource_config')) + ctx.source.instance.runtime_properties['output'] = result -@decorators.aws_relationship(resource_type=RESOURCE_TYPE) -def detach_from(ctx, resource_config, **_): +@decorators.aws_relationship(LambdaFunction, RESOURCE_TYPE) +def detach_from(ctx, **_): '''Detaches an Lambda Invoke from something else''' props = ctx.target.instance.runtime_properties function_name = props.get('aws_resource_id') diff --git a/cloudify_aws/lambda_serverless/tests/test_invoke.py b/cloudify_aws/lambda_serverless/tests/test_invoke.py index a9b4494a..b336627e 100644 --- a/cloudify_aws/lambda_serverless/tests/test_invoke.py +++ b/cloudify_aws/lambda_serverless/tests/test_invoke.py @@ -87,7 +87,8 @@ def test_attach_to(self): relation_ctx = self._get_relationship_context(SUBNET_GROUP_F) with patch(LAMBDA_PATH) as mock, patch(INVOKE_PATH + 'utils') as utils: utils.is_node_type = MagicMock(return_value=True) - invoke.attach_to(ctx=relation_ctx, resource_config=True) + invoke.attach_to( + ctx=relation_ctx, resource_config=True) self.assertTrue(mock.called) output = relation_ctx.source.instance.runtime_properties['output'] self.assertIsInstance(output, MagicMock) diff --git a/cloudify_aws/sns/resources/subscription.py b/cloudify_aws/sns/resources/subscription.py index 5a918337..5085f97f 100644 --- a/cloudify_aws/sns/resources/subscription.py +++ b/cloudify_aws/sns/resources/subscription.py @@ -104,7 +104,6 @@ def create(ctx, iface, resource_config, **_): # Create a copy of the resource config for clean manipulation. params = \ dict() if not resource_config else resource_config.copy() - topic_arn = params.get(TOPIC_ARN) # Add the required TopicArn parameter. if not topic_arn: diff --git a/plugin.yaml b/plugin.yaml index ee388784..520bc9e9 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -2,9 +2,9 @@ plugins: aws: executor: central_deployment_agent - source: https://github.com/cloudify-cosmo/cloudify-aws-plugin/archive/2.5.11.zip + source: https://github.com/cloudify-cosmo/cloudify-aws-plugin/archive/2.5.12.zip package_name: cloudify-aws-plugin - package_version: '2.5.11' + package_version: '2.5.12' data_types: diff --git a/test-requirements.txt b/test-requirements.txt index 6cc7d156..fa67c02c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,6 +6,6 @@ nose-cov==1.6 tox==3.15.0 pylint==1.9.5 # for integration and platform tests -awscli==1.18.51 +# awscli==1.18.51 pytest==4.6.3 - +pyyaml>=4.2b4