From fa5a6520e9a5d8bbc13b03465e6b141c452583b1 Mon Sep 17 00:00:00 2001 From: 345thomas Date: Thu, 1 Jun 2017 18:21:57 +0100 Subject: [PATCH 1/4] waitfor step --- pypyraws/contextargs.py | 59 ++ pypyraws/errors.py | 14 + pypyraws/steps/client.py | 53 +- pypyraws/steps/waitfor.py | 191 +++++ tests/unit/pypyraws/contextargs_test.py | 172 +++++ tests/unit/pypyraws/errors_test.py | 33 + tests/unit/pypyraws/steps/client_test.py | 114 +-- tests/unit/pypyraws/steps/waitfor_test.py | 835 ++++++++++++++++++++++ 8 files changed, 1314 insertions(+), 157 deletions(-) create mode 100644 pypyraws/contextargs.py create mode 100644 pypyraws/errors.py create mode 100644 pypyraws/steps/waitfor.py create mode 100644 tests/unit/pypyraws/contextargs_test.py create mode 100644 tests/unit/pypyraws/errors_test.py create mode 100644 tests/unit/pypyraws/steps/waitfor_test.py diff --git a/pypyraws/contextargs.py b/pypyraws/contextargs.py new file mode 100644 index 0000000..e3b4bb2 --- /dev/null +++ b/pypyraws/contextargs.py @@ -0,0 +1,59 @@ +"""Prepare aws arguments from the pypyr context.""" +from pypyr.errors import KeyInContextHasNoValueError, KeyNotInContextError + + +def get_awsclient_args(context, calling_module_name): + """Get required args from context for awsClientIn type steps. + + Args: + context: pypyr.context.Context. + calling_module_name: string. This is just to make a friendly error msg + should something go wrong. + + Returns: + tuple(client_in, service_name, method_name) + + Raises: + pypyr.errors.KeyNotInContextError: Required key missing in context. + pypyr.errors.KeyInContextHasNoValueError: Required key exists but is + empty or None. + """ + try: + client_in = context['awsClientIn'] + service_name = client_in['serviceName'] + method_name = client_in['methodName'] + except KeyError as err: + raise KeyNotInContextError( + f"awsClientIn missing required key for {calling_module_name}: " + f"{err}" + ) from err + + if not (service_name and service_name.strip()): + raise KeyInContextHasNoValueError( + f'serviceName required in awsClientIn for {calling_module_name}') + + if not (method_name and method_name.strip()): + raise KeyInContextHasNoValueError( + f'methodName required in awsClientIn for {calling_module_name}') + + return client_in, service_name, method_name + + +def get_formatted_iterable(input_dict, field_name, context): + """Format inputdict's field_name field against context. + + Args: + input_dict: dict. Dictionary containing dict to format. + field_name: str. Points at field in input_dict to format. + context: pypyr.context.Context. Substitutes string expressions from + this. + + Returns: + dict: Formatted dictionary that was at input_dict['field_name'] + Returns None if input_dict['field_name'] doesn't exist. + """ + output = input_dict.get(field_name, None) + if output is not None: + output = context.get_formatted_iterable(output) + + return output diff --git a/pypyraws/errors.py b/pypyraws/errors.py new file mode 100644 index 0000000..a9d383d --- /dev/null +++ b/pypyraws/errors.py @@ -0,0 +1,14 @@ +"""Custom exceptions for pypyraws. + +All pypyraws specific exceptions derive from pypyr root Error. +""" + +from pypyr.errors import PlugInError + + +class Error(PlugInError): + """Base class for all pypyraws exceptions.""" + + +class WaitTimeOut(Error): + """Aws resource that did not finish processing within wait limit.""" diff --git a/pypyraws/steps/client.py b/pypyraws/steps/client.py index d59855a..af65c10 100644 --- a/pypyraws/steps/client.py +++ b/pypyraws/steps/client.py @@ -1,7 +1,7 @@ """pypyr step that runs any boto3 low-level client method.""" import logging import pypyraws.aws.service -from pypyr.errors import KeyInContextHasNoValueError, KeyNotInContextError +import pypyraws.contextargs as contextargs # pypyr logger means the log level will be set correctly and output formatted. @@ -41,15 +41,16 @@ def run_step(context): None. """ logger.debug("started") - client_in, service_name, method_name = get_service_args(context) + client_in, service_name, method_name = contextargs.get_awsclient_args( + context, __name__) - client_args = client_in.get('clientArgs', None) - if client_args is not None: - client_args = context.get_formatted_iterable(client_args) + client_args = contextargs.get_formatted_iterable(input_dict=client_in, + field_name='clientArgs', + context=context) - method_args = client_in.get('methodArgs', None) - if method_args is not None: - method_args = context.get_formatted_iterable(method_args) + method_args = contextargs.get_formatted_iterable(input_dict=client_in, + field_name='methodArgs', + context=context) context['awsClientOut'] = pypyraws.aws.service.operation_exec( service_name=context.get_formatted_string(service_name), @@ -61,39 +62,3 @@ def run_step(context): logger.info(f"Executed {method_name} on aws {service_name}.") logger.debug("done") - - -def get_service_args(context): - """Gets required args from context for this step. - - Args: - context - dict. context. - - Returns: - tuple(client_in, service_name, method_name) - - Raises: - pypyr.errors.KeyNotInContextError: Required key missing in context. - pypyr.errors.KeyInContextHasNoValueError: Required key exists but is - empty or None. - """ - try: - client_in = context['awsClientIn'] - service_name = client_in['serviceName'] - method_name = client_in['methodName'] - except KeyError as err: - raise KeyNotInContextError( - "awsClientIn missing required key for pypyraws.steps.client: " - f"{err}" - ) from err - - # of course, if They went and made it a bool and True this will pass. - if not (service_name and service_name.strip()): - raise KeyInContextHasNoValueError( - 'serviceName required in awsClientIn for pypyraws.steps.client') - - if not (method_name and method_name.strip()): - raise KeyInContextHasNoValueError( - 'methodName required in awsClientIn for pypyraws.steps.client') - - return client_in, service_name, method_name diff --git a/pypyraws/steps/waitfor.py b/pypyraws/steps/waitfor.py new file mode 100644 index 0000000..edbf818 --- /dev/null +++ b/pypyraws/steps/waitfor.py @@ -0,0 +1,191 @@ +"""pypyr step that creates a custom waiter for any aws client operation.""" +import logging +from pypyr.utils.poll import wait_until_true +import pypyraws.aws.service +import pypyraws.contextargs as contextargs +from pypyraws.errors import WaitTimeOut + +# pypyr logger means the log level will be set correctly and output formatted. +logger = logging.getLogger(__name__) + + +def run_step(context): + """Custom waiter for any aws client operation. + + All of the awsWaitFor descendant values support {key} + string interpolation, except waitForField. + + Args: + context: + Dictionary. Mandatory. + Requires the following context keys in context: + - awsWaitFor. dict. mandatory. Contains keys: + - awsClientIn. dict. mandatory. This is the same as for the + pypyraws.steps.client in step. Contains keys: + - serviceName: mandatory. String for service name. + Available services here: + http://boto3.readthedocs.io/en/latest/reference/services/ + - methodName: mandatory. String. Name of method to + execute. + - clientArgs: optional. Dict. kwargs for the boto client + ctor. + - methodArgs: optional. Dict. kwargs for the client + method call + - waitForField: mandatory. string. format expression for + field name to check in awsClient response. + - toBe: mandatory. string. string. Stop waiting when + waitForField equals this value. + - pollInterval: optional. int. In seconds. Default to 30. + - maxAttempts: optional. int. Default 10. + - errorOnWaitTimeout: optional. Default True. Throws error + if maxAttempts exhausted without + reaching toBe value. If false, + step completes without raising + error. + + Returns: None + Adds key to context: + - awsWaitForTimedOut: bool. Adds key with value True if + errorOnWaitTimeout=False and max_attempts exhausted without + reaching toBe. If steps completes successfully and waitForField's + value becomes toBe, awsWaitForTimedOut == False. + + Raises: + pypyr.errors.KeyNotInContextError: awsWaitFor missing in context. + pypyr.errors.KeyInContextHasNoValueError: awsWaitFor exists but is + None. + pypyr.errors.WaitTimeOut: maxAttempts exceeded without waitForField + changing to toBe. + """ + logger.debug("started") + context.assert_key_has_value('awsWaitFor', __name__) + wait_for = context['awsWaitFor'] + + client_in, service_name, method_name = contextargs.get_awsclient_args( + wait_for, __name__) + + service_name = context.get_formatted_string(service_name) + method_name = context.get_formatted_string(method_name) + + client_args = contextargs.get_formatted_iterable(input_dict=client_in, + field_name='clientArgs', + context=context) + + method_args = contextargs.get_formatted_iterable(input_dict=client_in, + field_name='methodArgs', + context=context) + + (wait_for_field, + to_be, + poll_interval, + max_attempts, + error_on_wait_timeout) = get_poll_args(wait_for, context) + + wait_response = wait_until_true(interval=poll_interval, + max_attempts=max_attempts)( + execute_aws_client_method)( + service_name=service_name, + method_name=method_name, + client_args=client_args, + method_args=method_args, + wait_for_field=wait_for_field, + to_be=to_be + ) + + if wait_response: + context['awsWaitForTimedOut'] = False + logger.info(f"aws {service_name} {method_name} returned {to_be}. " + "Pipeline will now continue.") + else: + if error_on_wait_timeout: + context['awsWaitForTimedOut'] = True + logger.error(f"aws {service_name} {method_name} did not return " + f"{to_be} within {max_attempts}. errorOnWaitTimeout " + "is True, throwing error") + raise WaitTimeOut(f"aws {service_name} {method_name} did not " + f"return {to_be} within {max_attempts} retries.") + else: + context['awsWaitForTimedOut'] = True + logger.warn(f"aws {service_name} {method_name} did NOT return " + f" {to_be}. errorOnWaitTimeout is False, so pipeline " + "will continue regardless.") + + logger.debug("done") + + +def execute_aws_client_method(service_name, + method_name, + client_args, + method_args, + wait_for_field, + to_be): + """Execute method_name on service_name. + + Args: + service_name: string. Name of aws service. + method_name: method to execute. + client_args: aws client constructor args. + method_args: method args + wait_for_field: look for this field in the aws response + to_be: return True if wait_for_field's value equals this. + + Return: + True if value of wait_for_field == to_be, False if not. + """ + logger.debug("started") + response = pypyraws.aws.service.operation_exec( + service_name=service_name, + method_name=method_name, + client_args=client_args, + operation_args=method_args) + + wait_for_this_value = wait_for_field.format(**response) + logger.info(f"{wait_for_field} in aws response is: {wait_for_this_value}") + if wait_for_this_value == str(to_be): + logger.debug("Required status reached. The wait is so over.") + logger.debug("done") + return True + else: + logger.debug("Required status not reached, keep waiting.") + logger.debug("done") + return False + + +def get_poll_args(waitfor_dict, context): + """Gets polling arguments from waitfor_dict. + + Args: + waitfor_dict: The awsWaitFor dict + context: the pypyr context + + Returns: + tuple(wait_for_field, + to_be, + poll_interval, + max_attempts, + error_on_wait_timeout) + """ + logger.debug("started") + wait_for_field = waitfor_dict['waitForField'] + + to_be = context.get_formatted_string(str(waitfor_dict['toBe'])) + + poll_interval = context.get_formatted_as_type( + waitfor_dict.get('pollInterval', 30), + out_type=float) + + max_attempts = context.get_formatted_as_type( + waitfor_dict.get('maxAttempts', 10), + out_type=int) + + error_on_wait_timeout = context.get_formatted_as_type( + waitfor_dict.get('errorOnWaitTimeout', True), + out_type=bool) + + logger.debug("done") + + return (wait_for_field, + to_be, + poll_interval, + max_attempts, + error_on_wait_timeout) diff --git a/tests/unit/pypyraws/contextargs_test.py b/tests/unit/pypyraws/contextargs_test.py new file mode 100644 index 0000000..4b44176 --- /dev/null +++ b/tests/unit/pypyraws/contextargs_test.py @@ -0,0 +1,172 @@ +"""contextargs.py unit tests.""" +from pypyr.context import Context +from pypyr.errors import KeyInContextHasNoValueError, KeyNotInContextError +import pypyraws.contextargs as contextargs +import pytest + +# ---------------------------- get_awsclient_args-----------------------------# + + +def test_get_awsclient_args_pass(): + """get_awsclient_args pass""" + context = Context({ + 'k1': 'v1', + 'awsClientIn': { + 'serviceName': 'service name', + 'methodName': 'method_name', + 'arbKey': 'arb_value' + }}) + client_in, service_name, method_name = contextargs.get_awsclient_args( + context, 'pypyraws.steps.client') + + assert client_in == { + 'serviceName': 'service name', + 'methodName': 'method_name', + 'arbKey': 'arb_value' + } + assert service_name == 'service name' + assert method_name == 'method_name' + + +def test_get_awsclient_args_missing_awsclientin(): + """Missing awsClientIn raises""" + context = Context({'k1': 'v1'}) + + with pytest.raises(KeyNotInContextError) as err_info: + client_in, service_name, method_name = contextargs.get_awsclient_args( + context, 'pypyraws.steps.client') + + assert repr(err_info.value) == ( + "KeyNotInContextError(\'awsClientIn not found in the pypyr " + "context.',)") + + +def test_get_awsclient_args_missing_servicename(): + """Missing serviceName raises""" + context = Context({ + 'k1': 'v1', + 'awsClientIn': { + 'methodName': 'method_name', + 'arbKey': 'arb_value' + }}) + + with pytest.raises(KeyNotInContextError) as err_info: + client_in, service_name, method_name = contextargs.get_awsclient_args( + context, 'pypyraws.steps.client') + + assert repr(err_info.value) == ( + "KeyNotInContextError(\"awsClientIn missing required key for " + "pypyraws.steps.client: 'serviceName'\",)") + + +def test_get_awsclient_args_missing_methodname(): + """Missing methodName raises""" + context = Context({ + 'k1': 'v1', + 'awsClientIn': { + 'serviceName': 'service name', + 'arbKey': 'arb_value' + }}) + + with pytest.raises(KeyNotInContextError) as err_info: + client_in, service_name, method_name = contextargs.get_awsclient_args( + context, 'pypyraws.steps.client') + + assert repr(err_info.value) == ( + "KeyNotInContextError(\"awsClientIn missing required key for " + "pypyraws.steps.client: 'methodName'\",)") + + +def test_get_awsclient_args_servicename_empty(): + """Empty serviceName raises""" + context = Context({ + 'k1': 'v1', + 'awsClientIn': { + 'serviceName': '', + 'methodName': 'method_name', + 'arbKey': 'arb_value' + }}) + + with pytest.raises(KeyInContextHasNoValueError) as err_info: + client_in, service_name, method_name = contextargs.get_awsclient_args( + context, 'pypyraws.steps.client') + + assert repr(err_info.value) == ( + "KeyInContextHasNoValueError('serviceName required in awsClientIn " + "for pypyraws.steps.client',)") + + +def test_get_awsclient_args_methodname_empty(): + """Whitespace serviceName raises""" + context = Context({ + 'k1': 'v1', + 'awsClientIn': { + 'serviceName': 'service name', + 'methodName': ' ', + 'arbKey': 'arb_value' + }}) + + with pytest.raises(KeyInContextHasNoValueError) as err_info: + client_in, service_name, method_name = contextargs.get_awsclient_args( + context, 'pypyraws.steps.client') + + assert repr(err_info.value) == ( + "KeyInContextHasNoValueError('methodName required in awsClientIn " + "for pypyraws.steps.client',)") + +# ---------------------------- get_awsclient_args-----------------------------# + +# ---------------------------- get_formatted_iterable-------------------------# + + +def test_getformatted_iterable_pass(): + """get_formatted_iterable passes""" + context = Context({ + 'k1': 'v1', + 'k2': 'v2', + 'awsClientIn': { + 'serviceName': '', + 'methodName': 'method_name', + 'clientArgs': { + 'cak1': 'cak2', + '{k1}': '{k2}' + }, + 'arbKey': 'arb_value' + + }}) + + result = contextargs.get_formatted_iterable( + input_dict=context['awsClientIn'], + field_name='clientArgs', + context=context) + + assert result == { + 'cak1': 'cak2', + 'v1': 'v2' + } + + +def test_getformatted_iterable_none_pass(): + """get_formatted_iterable none returns none""" + context = Context({ + 'k1': 'v1', + 'k2': 'v2', + 'awsClientIn': { + 'serviceName': '', + 'methodName': 'method_name', + 'clientArgs': { + 'cak1': 'cak2', + '{k1}': '{k2}' + }, + 'arbKey': 'arb_value' + + }}) + + result = contextargs.get_formatted_iterable( + input_dict=context['awsClientIn'], + field_name='doesntexist', + context=context) + + assert result is None + +# ---------------------------- get_formatted_iterable-------------------------# diff --git a/tests/unit/pypyraws/errors_test.py b/tests/unit/pypyraws/errors_test.py new file mode 100644 index 0000000..687d947 --- /dev/null +++ b/tests/unit/pypyraws/errors_test.py @@ -0,0 +1,33 @@ +"""errors.py unit tests.""" +from pypyr.errors import Error as PypyrError +from pypyr.errors import PlugInError +from pypyraws.errors import Error as PypyrAwsError +from pypyraws.errors import WaitTimeOut +import pytest + + +def test_base_error_raises(): + """Pypyr root Error raises with correct message.""" + with pytest.raises(PypyrAwsError) as err_info: + raise PypyrAwsError("this is error text right here") + + assert repr(err_info.value) == ("Error('this is error text " + "right here',)") + + +def test_wait_timeout_raises(): + """Aws WaitTimeOut error raises with correct message.""" + with pytest.raises(WaitTimeOut) as err_info: + raise WaitTimeOut("this is error text right here") + + assert repr(err_info.value) == ("WaitTimeOut('this is error " + "text right here',)") + + +def test_wait_timeout_inheritance(): + """WaitTimeOut should inherit all the way up to pypyr Error.""" + # confirm subclassed from pypyr root error + err = WaitTimeOut() + assert isinstance(err, PypyrAwsError) + assert isinstance(err, PlugInError) + assert isinstance(err, PypyrError) diff --git a/tests/unit/pypyraws/steps/client_test.py b/tests/unit/pypyraws/steps/client_test.py index a7eade8..ff52e48 100644 --- a/tests/unit/pypyraws/steps/client_test.py +++ b/tests/unit/pypyraws/steps/client_test.py @@ -1,7 +1,7 @@ """client.py unit tests.""" from unittest.mock import patch from pypyr.context import Context -from pypyr.errors import KeyInContextHasNoValueError, KeyNotInContextError +from pypyr.errors import KeyNotInContextError import pypyraws.steps.client as client_step import pytest @@ -295,115 +295,3 @@ def test_aws_client_substitute_no_method_args(mock_service): operation_args=None ) # ---------------------------- substitutions --------------------------------# -# -# ---------------------------- get_service_args------------------------------# - - -def test_get_service_args_pass(): - """get_service_args pass""" - context = Context({ - 'k1': 'v1', - 'awsClientIn': { - 'serviceName': 'service name', - 'methodName': 'method_name', - 'arbKey': 'arb_value' - }}) - client_in, service_name, method_name = client_step.get_service_args( - context) - - assert client_in == { - 'serviceName': 'service name', - 'methodName': 'method_name', - 'arbKey': 'arb_value' - } - assert service_name == 'service name' - assert method_name == 'method_name' - - -def test_get_service_args_missing_awsclientin(): - """Missing awsClientIn raises""" - context = Context({'k1': 'v1'}) - - with pytest.raises(KeyNotInContextError) as err_info: - client_in, service_name, method_name = client_step.get_service_args( - context) - - assert repr(err_info.value) == ( - "KeyNotInContextError(\'awsClientIn not found in the pypyr " - "context.',)") - - -def test_get_service_args_missing_servicename(): - """Missing serviceName raises""" - context = Context({ - 'k1': 'v1', - 'awsClientIn': { - 'methodName': 'method_name', - 'arbKey': 'arb_value' - }}) - - with pytest.raises(KeyNotInContextError) as err_info: - client_in, service_name, method_name = client_step.get_service_args( - context) - - assert repr(err_info.value) == ( - "KeyNotInContextError(\"awsClientIn missing required key for " - "pypyraws.steps.client: 'serviceName'\",)") - - -def test_get_service_args_missing_methodname(): - """Missing methodName raises""" - context = Context({ - 'k1': 'v1', - 'awsClientIn': { - 'serviceName': 'service name', - 'arbKey': 'arb_value' - }}) - - with pytest.raises(KeyNotInContextError) as err_info: - client_in, service_name, method_name = client_step.get_service_args( - context) - - assert repr(err_info.value) == ( - "KeyNotInContextError(\"awsClientIn missing required key for " - "pypyraws.steps.client: 'methodName'\",)") - - -def test_get_service_args_servicename_empty(): - """Empty serviceName raises""" - context = Context({ - 'k1': 'v1', - 'awsClientIn': { - 'serviceName': '', - 'methodName': 'method_name', - 'arbKey': 'arb_value' - }}) - - with pytest.raises(KeyInContextHasNoValueError) as err_info: - client_in, service_name, method_name = client_step.get_service_args( - context) - - assert repr(err_info.value) == ( - "KeyInContextHasNoValueError('serviceName required in awsClientIn " - "for pypyraws.steps.client',)") - - -def test_get_service_args_methodname_empty(): - """Whitespace serviceName raises""" - context = Context({ - 'k1': 'v1', - 'awsClientIn': { - 'serviceName': 'service name', - 'methodName': ' ', - 'arbKey': 'arb_value' - }}) - - with pytest.raises(KeyInContextHasNoValueError) as err_info: - client_in, service_name, method_name = client_step.get_service_args( - context) - - assert repr(err_info.value) == ( - "KeyInContextHasNoValueError('methodName required in awsClientIn " - "for pypyraws.steps.client',)") - -# ---------------------------- get_service_args------------------------------# diff --git a/tests/unit/pypyraws/steps/waitfor_test.py b/tests/unit/pypyraws/steps/waitfor_test.py new file mode 100644 index 0000000..a1074b3 --- /dev/null +++ b/tests/unit/pypyraws/steps/waitfor_test.py @@ -0,0 +1,835 @@ +"""waitfor.py unit tests.""" +import logging +from pypyr.context import Context +from pypyr.errors import KeyNotInContextError +from pypyraws.errors import WaitTimeOut +import pypyraws.steps.waitfor as waitfor_step +import pytest +from unittest.mock import call, patch + +# ---------------------------- run_step -------------------------------------# + + +def test_waitfor_missing_awswaitfor(): + """Missing awsWaitFor raises""" + context = Context({'k1': 'v1'}) + + with pytest.raises(KeyNotInContextError) as err_info: + waitfor_step.run_step(context) + + assert repr(err_info.value) == ( + 'KeyNotInContextError("context[\'awsWaitFor\'] doesn\'t exist. It ' + 'must exist for pypyraws.steps.waitfor.",)') + + +@patch('pypyraws.aws.service.operation_exec', return_value={'rk1': 'rv1', + 'rk2': 'rv2'}) +@patch('time.sleep') +def test_waitfor_pass_1st_time_no_client_args(mock_sleep, mock_service): + """Successful run with no client args""" + context = Context({ + 'k1': 'v1', + 'awsWaitFor': { + 'awsClientIn': { + 'serviceName': 'service name', + 'methodName': 'method_name', + 'arbKey': 'arb_value' + }, + 'waitForField': '{rk2}', + 'toBe': 'rv2' + }}) + + logger = logging.getLogger('pypyraws.steps.waitfor') + with patch.object(logger, 'info') as mock_logger_info: + waitfor_step.run_step(context) + + assert mock_logger_info.mock_calls == [ + call('{rk2} in aws response is: rv2'), + call('aws service name method_name returned rv2. ' + 'Pipeline will now continue.')] + + assert len(context) == 3 + assert not context['awsWaitForTimedOut'] + assert context['k1'] == 'v1' + assert context['awsWaitFor']['awsClientIn'] == { + 'serviceName': 'service name', + 'methodName': 'method_name', + 'arbKey': 'arb_value' + } + + mock_sleep.assert_not_called() + mock_service.assert_called_once_with(service_name='service name', + method_name='method_name', + client_args=None, + operation_args=None, + ) + + +@patch('pypyraws.aws.service.operation_exec', return_value={'rk1': 'rv1', + 'rk2': 'rv2'}) +@patch('time.sleep') +def test_waitfor_fail_no_client_args(mock_sleep, mock_service): + """Fail run with no client args""" + context = Context({ + 'k1': 'v1', + 'awsWaitFor': { + 'awsClientIn': { + 'serviceName': 'service name', + 'methodName': 'method_name', + 'arbKey': 'arb_value' + }, + 'waitForField': '{rk2}', + 'toBe': 'xxx' + }}) + + with pytest.raises(WaitTimeOut) as err_info: + logger = logging.getLogger('pypyraws.steps.waitfor') + with patch.object(logger, 'error') as mock_logger_error: + waitfor_step.run_step(context) + + assert repr(err_info.value) == ("WaitTimeOut('aws service name " + "method_name did not return xxx within " + "10 retries.',)") + + mock_logger_error.assert_called_once_with( + 'aws service name method_name did not return xxx within 10. ' + 'errorOnWaitTimeout is True, throwing error') + + assert len(context) == 3 + assert context['awsWaitForTimedOut'] + assert context['k1'] == 'v1' + assert context['awsWaitFor']['awsClientIn'] == { + 'serviceName': 'service name', + 'methodName': 'method_name', + 'arbKey': 'arb_value' + } + + mock_sleep.call_count == 10 + mock_sleep.assert_called_with(30) + mock_service.assert_called_with(service_name='service name', + method_name='method_name', + client_args=None, + operation_args=None, + ) + + +@patch('pypyraws.aws.service.operation_exec', return_value={'rk1': 'rv1', + 'rk2': 'rv2'}) +@patch('time.sleep') +def test_waitfor_fail_no_client_args_no_throw(mock_sleep, mock_service): + """Fail run with no client args and no error thrown""" + context = Context({ + 'k1': 'v1', + 'awsWaitFor': { + 'awsClientIn': { + 'serviceName': 'service name', + 'methodName': 'method_name', + 'arbKey': 'arb_value' + }, + 'waitForField': '{rk2}', + 'toBe': 'xxx', + 'errorOnWaitTimeout': False + }}) + + logger = logging.getLogger('pypyraws.steps.waitfor') + with patch.object(logger, 'warn') as mock_logger_warn: + waitfor_step.run_step(context) + + mock_logger_warn.assert_called_once_with( + 'aws service name method_name did NOT return xxx. errorOnWaitTimeout ' + 'is False, so pipeline will continue regardless.') + + assert len(context) == 3 + assert context['k1'] == 'v1' + assert context['awsWaitFor']['awsClientIn'] == { + 'serviceName': 'service name', + 'methodName': 'method_name', + 'arbKey': 'arb_value' + } + + assert context['awsWaitForTimedOut'] + + mock_sleep.call_count == 10 + mock_sleep.assert_called_with(30) + mock_service.assert_called_with(service_name='service name', + method_name='method_name', + client_args=None, + operation_args=None, + ) + + +@patch('pypyraws.aws.service.operation_exec') +@patch('time.sleep') +def test_waitfor_pass_client_args(mock_sleep, mock_service): + """Successful run with client args pass on 3.""" + mock_service.side_effect = [ + {'rk1': 'rv1', + 'rk2': 'rv2'}, # 1 + {'rk1': 'rv1', + 'rk2': 'rv2'}, # 2 + {'rk1': 'rv1', + 'rk2': 'xxx'}, # 3 + {'rk1': 'rv1', + 'rk2': 'rv2'} # 4 + ] + + context = Context({ + 'k1': 'v1', + 'awsWaitFor': { + 'awsClientIn': { + 'serviceName': 'service name', + 'methodName': 'method_name', + 'arbKey': 'arb_value', + 'clientArgs': {'ck1': 'cv1', 'ck2': 'cv2'}}, + 'waitForField': '{rk2}', + 'toBe': 'xxx' + }}) + waitfor_step.run_step(context) + + assert len(context) == 3 + assert context['k1'] == 'v1' + assert context['awsWaitFor']['awsClientIn'] == { + 'serviceName': 'service name', + 'methodName': 'method_name', + 'arbKey': 'arb_value', + 'clientArgs': {'ck1': 'cv1', 'ck2': 'cv2'} + } + assert not context['awsWaitForTimedOut'] + + mock_service.assert_called_with(service_name='service name', + method_name='method_name', + client_args={'ck1': 'cv1', + 'ck2': 'cv2'}, + operation_args=None, + ) + mock_sleep.call_count == 3 + mock_sleep.assert_called_with(30) + + +@patch('pypyraws.aws.service.operation_exec', return_value={'rk1': 'rv1', + 'rk2': 'rv2'}) +@patch('time.sleep') +def test_waitfor_pass_method_args(mock_sleep, mock_service): + """Successful run with method args pass on 2 with int""" + mock_service.side_effect = [ + {'rk1': 'rv1', + 'rk2': 'rv2'}, # 1 + {'rk1': 'rv1', + 'rk2': 123}, # 2 + {'rk1': 'rv1', + 'rk2': 'xxx'}, # 3 + ] + + context = Context({ + 'k1': 'v1', + 'awsWaitFor': { + 'awsClientIn': { + 'serviceName': 'service name', + 'methodName': 'method_name', + 'arbKey': 'arb_value', + 'methodArgs': {'mk1': 'mv1', 'mk2': 'mv2'} + }, + 'waitForField': '{rk2}', + 'toBe': 123 + }}) + waitfor_step.run_step(context) + + assert len(context) == 3 + assert not context['awsWaitForTimedOut'] + assert context['k1'] == 'v1' + assert context['awsWaitFor']['awsClientIn'] == { + 'serviceName': 'service name', + 'methodName': 'method_name', + 'arbKey': 'arb_value', + 'methodArgs': {'mk1': 'mv1', 'mk2': 'mv2'} + } + + mock_service.assert_called_with(service_name='service name', + method_name='method_name', + client_args=None, + operation_args={'mk1': 'mv1', + 'mk2': 'mv2'}, + ) + mock_sleep.call_count == 2 + mock_sleep.assert_called_with(30) + + +@patch('pypyraws.aws.service.operation_exec') +@patch('time.sleep') +def test_waitfor_pass_all_args(mock_sleep, mock_service): + """Successful run with client and method args pass on #2.""" + mock_service.side_effect = [ + {'rk1': 'rv1', + 'rk2': 'rv2'}, # 1 + {'rk1': 'rv1', + 'rk2': False}, # 2 + {'rk1': 'rv1', + 'rk2': 'rv2'}, # 3 + ] + + context = Context({ + 'k1': 'v1', + 'awsWaitFor': { + 'awsClientIn': { + 'serviceName': 'service name', + 'methodName': 'method_name', + 'arbKey': 'arb_value', + 'clientArgs': {'ck1': 'cv1', 'ck2': 'cv2'}, + 'methodArgs': {'mk1': 'mv1', 'mk2': 'mv2'}}, + 'waitForField': '{rk2}', + 'toBe': False, + 'pollInterval': 99 + }}) + waitfor_step.run_step(context) + + assert len(context) == 3 + assert context['k1'] == 'v1' + assert context['awsWaitFor']['awsClientIn'] == { + 'serviceName': 'service name', + 'methodName': 'method_name', + 'arbKey': 'arb_value', + 'clientArgs': {'ck1': 'cv1', 'ck2': 'cv2'}, + 'methodArgs': {'mk1': 'mv1', 'mk2': 'mv2'} + } + assert not context['awsWaitForTimedOut'] + mock_sleep.assert_called_once_with(99) + mock_service.assert_called_with(service_name='service name', + method_name='method_name', + client_args={'ck1': 'cv1', + 'ck2': 'cv2'}, + operation_args={'mk1': 'mv1', + 'mk2': 'mv2'}, + ) + +# ---------------------------- run_step -------------------------------------# + +# ---------------------------- substitutions --------------------------------# + + +@patch('pypyraws.aws.service.operation_exec') +@patch('time.sleep') +def test_waitfor_substitute_all_args(mock_sleep, mock_service): + """Successful substitution run with client and method args""" + mock_service.side_effect = [ + {'rk1': 'rv1', + 'rk2': True}, + {'rk1': 'rv1', + 'rk2': False} + ] + context = Context({ + 'k1': 'v1', + 'k2': 'v2', + 'k3': 'v3', + 'k4': 'v4', + 'k5': 'v5', + 'k6': 'v6', + 'k7': 'v7', + 'k8': False, + 'k9': 99, + 'k10': 77, + 'k11': False, + 'awsWaitFor': { + 'awsClientIn': { + 'serviceName': 'service name {k1}', + 'methodName': 'method_name {k2}', + 'arbKey': 'arb_value {k3}', + 'clientArgs': {'ck1{k4}': 'cv1{k5}', 'ck2': 'cv2'}, + 'methodArgs': {'mk1': 'mv1', 'mk2{k6}': 'mv2{k7}'}}, + 'waitForField': '{rk2}', + 'toBe': '{k8}', + 'pollInterval': '{k9}', + 'maxAttempts': '{k10}', + 'errorOnWaitTimeout': '{k11}' + }}) + waitfor_step.run_step(context) + + assert len(context) == 13 + assert context['k1'] == 'v1' + assert context['awsWaitFor']['awsClientIn'] == { + 'serviceName': 'service name {k1}', + 'methodName': 'method_name {k2}', + 'arbKey': 'arb_value {k3}', + 'clientArgs': {'ck1{k4}': 'cv1{k5}', 'ck2': 'cv2'}, + 'methodArgs': {'mk1': 'mv1', 'mk2{k6}': 'mv2{k7}'} + } + assert not context['awsWaitForTimedOut'] + mock_sleep.assert_called_once_with(99) + mock_service.assert_called_with(service_name='service name v1', + method_name='method_name v2', + client_args={'ck1v4': 'cv1v5', + 'ck2': 'cv2'}, + operation_args={'mk1': 'mv1', + 'mk2v6': 'mv2v7'}, + ) + + +@patch('pypyraws.aws.service.operation_exec', return_value={'rk1': 'rv1', + 'rk2': 123}) +@patch('time.sleep') +def test_waitfor_substitute_no_client_meth_args(mock_sleep, mock_service): + """Successful substitution run with no client and no method args""" + context = Context({ + 'k1': 'v1', + 'k2': 'v2', + 'k3': 'v3', + 'k4': 'v4', + 'k5': 'v5', + 'k6': 'v6', + 'k7': 'v7', + 'k8': 66, + 'k9': 1, + 'k10': False, + 'awsWaitFor': { + 'awsClientIn': { + 'serviceName': 'service name {k1}', + 'methodName': 'method_name {k2}', + 'arbKey': 'arb_value {k3}' + }, + 'waitForField': '{rk2}', + 'toBe': 123, + 'pollInterval': '{k8}', + 'maxAttempts': '{k9}', + 'errorOnWaitTimeout': '{k10}' + }}) + waitfor_step.run_step(context) + + assert len(context) == 12 + assert context['k1'] == 'v1' + assert context['awsWaitFor']['awsClientIn'] == { + 'serviceName': 'service name {k1}', + 'methodName': 'method_name {k2}', + 'arbKey': 'arb_value {k3}' + } + assert not context['awsWaitForTimedOut'] + mock_sleep.assert_not_called() + mock_service.assert_called_once_with(service_name='service name v1', + method_name='method_name v2', + client_args=None, + operation_args=None + ) + + +@patch('pypyraws.aws.service.operation_exec', return_value={'rk1': 'rv1', + 'rk2': 'rv2'}) +@patch('time.sleep') +def test_aws_waitfor_substitute_no_client_args(mock_sleep, mock_service): + """Successful run with no client but method args""" + context = Context({ + 'k1': 'v1', + 'k2': 'v2', + 'k3': 'v3', + 'k4': 'v4', + 'k5': 'v5', + 'k6': 'v6', + 'k7': 'v7', + 'k8': 66, + 'k9': 1, + 'k10': False, + 'awsWaitFor': { + 'awsClientIn': { + 'serviceName': 'service name {k1}', + 'methodName': 'method_name {k2}', + 'arbKey': 'arb_value {k3}', + 'methodArgs': {'mk1': 'mv1', 'mk2{k6}': 'mv2{k7}'}}, + 'waitForField': '{rk2}', + 'toBe': 123, + 'pollInterval': '{k8}', + 'maxAttempts': '{k9}', + 'errorOnWaitTimeout': '{k10}' + }}) + waitfor_step.run_step(context) + + assert len(context) == 12 + assert context['k1'] == 'v1' + assert context['awsWaitFor']['awsClientIn'] == { + 'serviceName': 'service name {k1}', + 'methodName': 'method_name {k2}', + 'arbKey': 'arb_value {k3}', + 'methodArgs': {'mk1': 'mv1', 'mk2{k6}': 'mv2{k7}'} + } + assert context['awsWaitForTimedOut'] + mock_sleep.assert_not_called() + mock_service.assert_called_once_with(service_name='service name v1', + method_name='method_name v2', + client_args=None, + operation_args={'mk1': 'mv1', + 'mk2v6': 'mv2v7'}, + ) + + +@patch('pypyraws.aws.service.operation_exec', return_value={'rk1': 'rv1', + 'rk2': 123.4}) +@patch('time.sleep') +def test_aws_waitfor_substitute_no_method_args(mock_sleep, mock_service): + """Successful run with client but no method args""" + context = Context({ + 'k1': 'v1', + 'k2': 'v2', + 'k3': 'v3', + 'k4': 'v4', + 'k5': 'v5', + 'k6': 'v6', + 'k7': 'v7', + 'awsWaitFor': { + 'awsClientIn': { + 'serviceName': 'service name {k1}', + 'methodName': 'method_name {k2}', + 'arbKey': 'arb_value {k3}', + 'clientArgs': {'ck1{k4}': 'cv1{k5}', 'ck2': 'cv2'}}, + 'waitForField': '{rk2}', + 'toBe': 123.4, + 'pollInterval': 1, + 'maxAttempts': 1, + }}) + waitfor_step.run_step(context) + + assert len(context) == 9 + assert context['k1'] == 'v1' + assert context['awsWaitFor']['awsClientIn'] == { + 'serviceName': 'service name {k1}', + 'methodName': 'method_name {k2}', + 'arbKey': 'arb_value {k3}', + 'clientArgs': {'ck1{k4}': 'cv1{k5}', 'ck2': 'cv2'} + } + assert not context['awsWaitForTimedOut'] + mock_sleep.assert_not_called() + mock_service.assert_called_once_with(service_name='service name v1', + method_name='method_name v2', + client_args={'ck1v4': 'cv1v5', + 'ck2': 'cv2'}, + operation_args=None + ) +# ---------------------------- substitutions -------------------------------- + +# ----------------------execute_aws_client_method --------------------------- + + +@patch('pypyraws.aws.service.operation_exec', return_value={'rk1': 'rv1', + 'rk2': [ + 123.4, + 123, + 'string here', + True, + {'rks1': 'rks2' + } + ]}) +def test_execute_awsclientmethod_parse_response_top_string(mock_service): + """Successful response parsing""" + assert waitfor_step.execute_aws_client_method(service_name='service name', + method_name='method name', + client_args={ + 'ck1': 'cv1', + 'ck2': 'cv2'}, + method_args={ + 'mk1': 'mv1', + 'mk2': 'mv2'}, + wait_for_field='{rk1}', + to_be='rv1') + + mock_service.assert_called_once_with(service_name='service name', + method_name='method name', + client_args={'ck1': 'cv1', + 'ck2': 'cv2'}, + operation_args={'mk1': 'mv1', + 'mk2': 'mv2'} + ) + + +@patch('pypyraws.aws.service.operation_exec', return_value={'rk1': 'rv1', + 'rk2': [ + 123.4, + 123, + 'string here', + True, + {'rks1': + 'rks2' + } + ]}) +def test_execute_awsclientmethod_parse_response_path_float(mock_service): + """Successful response parsing to complex path with float type.""" + assert waitfor_step.execute_aws_client_method(service_name='service name', + method_name='method name', + client_args={ + 'ck1': 'cv1', + 'ck2': 'cv2'}, + method_args={ + 'mk1': 'mv1', + 'mk2': 'mv2'}, + wait_for_field='{rk2[0]}', + to_be=123.4) + + mock_service.assert_called_once_with(service_name='service name', + method_name='method name', + client_args={'ck1': 'cv1', + 'ck2': 'cv2'}, + operation_args={'mk1': 'mv1', + 'mk2': 'mv2'}) + + +@patch('pypyraws.aws.service.operation_exec', return_value={'rk1': 'rv1', + 'rk2': [ + 123.4, + 123, + 'string here', + True, + {'rks1': + 'rks2' + } + ]}) +def test_execute_awsclientmethod_parse_response_path_int(mock_service): + """Successful response parsing to a complex path with int type""" + assert waitfor_step.execute_aws_client_method(service_name='service name', + method_name='method name', + client_args={ + 'ck1': 'cv1', + 'ck2': 'cv2'}, + method_args={ + 'mk1': 'mv1', + 'mk2': 'mv2'}, + wait_for_field='{rk2[1]}', + to_be=123) + + mock_service.assert_called_once_with(service_name='service name', + method_name='method name', + client_args={'ck1': 'cv1', + 'ck2': 'cv2'}, + operation_args={'mk1': 'mv1', + 'mk2': 'mv2'}) + + +@patch('pypyraws.aws.service.operation_exec', return_value={'rk1': 'rv1', + 'rk2': [ + 123.4, + 123, + 'string here', + True, + {'rks1': + 'rks2' + } + ]}) +def test_execute_awsclientmethod_parse_response_path_bool_true(mock_service): + """Successful response parsing to a complex path with bool type True""" + assert waitfor_step.execute_aws_client_method(service_name='service name', + method_name='method name', + client_args={ + 'ck1': 'cv1', + 'ck2': 'cv2'}, + method_args={ + 'mk1': 'mv1', + 'mk2': 'mv2'}, + wait_for_field='{rk2[3]}', + to_be=True) + + mock_service.assert_called_once_with(service_name='service name', + method_name='method name', + client_args={'ck1': 'cv1', + 'ck2': 'cv2'}, + operation_args={'mk1': 'mv1', + 'mk2': 'mv2'}) + + +@patch('pypyraws.aws.service.operation_exec', return_value={'rk1': 'rv1', + 'rk2': [ + 123.4, + 123, + 'string here', + False, + {'rks1': + 'rks2' + } + ]}) +def test_execute_awsclientmethod_parse_response_path_bool_false(mock_service): + """Successful response parsing to a complex path with bool type False.""" + assert waitfor_step.execute_aws_client_method(service_name='service name', + method_name='method name', + client_args={ + 'ck1': 'cv1', + 'ck2': 'cv2'}, + method_args={ + 'mk1': 'mv1', + 'mk2': 'mv2'}, + wait_for_field='{rk2[3]}', + to_be=False) + + mock_service.assert_called_once_with(service_name='service name', + method_name='method name', + client_args={'ck1': 'cv1', + 'ck2': 'cv2'}, + operation_args={'mk1': 'mv1', + 'mk2': 'mv2'}) + + +@patch('pypyraws.aws.service.operation_exec', return_value={'rk1': 'rv1', + 'rk2': [ + 123.4, + 123, + 'string here', + False, + {'rks1': + 'rks2' + } + ]}) +def test_execute_awsclientmethod_parse_false_path_bool_false(mock_service): + """Fail response parsing to a complex path with bool type False.""" + assert not waitfor_step.execute_aws_client_method( + service_name='service name', + method_name='method name', + client_args={ + 'ck1': 'cv1', + 'ck2': 'cv2'}, + method_args={ + 'mk1': 'mv1', + 'mk2': 'mv2'}, + wait_for_field='{rk2[3]}', + to_be=True) + + mock_service.assert_called_once_with(service_name='service name', + method_name='method name', + client_args={'ck1': 'cv1', + 'ck2': 'cv2'}, + operation_args={'mk1': 'mv1', + 'mk2': 'mv2'}) + + +@patch('pypyraws.aws.service.operation_exec', return_value={'rk1': 'rv1', + 'rk2': [ + 123.4, + 123, + 'string here', + True, + {'rks1': + 'rks2' + } + ]}) +def test_execute_awsclientmethod_parse_false_path_bool_true(mock_service): + """Fail response parsing to a complex path with bool type False.""" + assert not waitfor_step.execute_aws_client_method( + service_name='service name', + method_name='method name', + client_args={ + 'ck1': 'cv1', + 'ck2': 'cv2'}, + method_args={ + 'mk1': 'mv1', + 'mk2': 'mv2'}, + wait_for_field='{rk2[3]}', + to_be=False) + + mock_service.assert_called_once_with(service_name='service name', + method_name='method name', + client_args={'ck1': 'cv1', + 'ck2': 'cv2'}, + operation_args={'mk1': 'mv1', + 'mk2': 'mv2'}) + + +@patch('pypyraws.aws.service.operation_exec', return_value={'rk1': 'rv1', + 'rk2': [ + 123.4, + 123, + 'string here', + True, + {'rks1': 'rkv2' + } + ]}) +def test_execute_awsclientmethod_parse_response_path_dict(mock_service): + """Successful response parsing""" + assert waitfor_step.execute_aws_client_method( + service_name='service name', + method_name='method name', + client_args={ + 'ck1': 'cv1', + 'ck2': 'cv2'}, + method_args={ + 'mk1': 'mv1', + 'mk2': 'mv2'}, + wait_for_field='{rk2[4][rks1]}', + to_be='rkv2') + + mock_service.assert_called_once_with(service_name='service name', + method_name='method name', + client_args={'ck1': 'cv1', + 'ck2': 'cv2'}, + operation_args={'mk1': 'mv1', + 'mk2': 'mv2'} + ) +# ----------------------execute_aws_client_method ------------------------ + +# ----------------------get_poll_args ------------------------------------ + + +def test_get_poll_args_defaults(): + """All defaults assigned""" + context = Context({ + 'waitFor': { + 'waitForField': 'field name', + 'toBe': 'value here'} + }) + + waitfor_dict = context['waitFor'] + (wait_for_field, + to_be, + poll_interval, + max_attempts, + error_on_wait_timeout) = waitfor_step.get_poll_args(waitfor_dict, context) + + assert wait_for_field == 'field name' + assert to_be == 'value here' + assert poll_interval == 30 + assert max_attempts == 10 + assert error_on_wait_timeout + + +def test_get_poll_args_all(): + """All args assigned.""" + context = Context({ + 'waitFor': { + 'waitForField': 'field name', + 'toBe': 'value here', + 'pollInterval': 99, + 'maxAttempts': 66, + 'errorOnWaitTimeout': False} + }) + + waitfor_dict = context['waitFor'] + (wait_for_field, + to_be, + poll_interval, + max_attempts, + error_on_wait_timeout) = waitfor_step.get_poll_args(waitfor_dict, context) + + assert wait_for_field == 'field name' + assert to_be == 'value here' + assert poll_interval == 99 + assert max_attempts == 66 + assert not error_on_wait_timeout + + +def test_get_poll_args_substitutions(): + """All args assigned and substituted.""" + context = Context({ + 'k1': 123.4, + 'k2': 99, + 'k3': 66, + 'k4': False, + 'waitFor': { + 'waitForField': 'field name', + 'toBe': '{k1}', + 'pollInterval': '{k2}', + 'maxAttempts': '{k3}', + 'errorOnWaitTimeout': '{k4}'} + }) + + waitfor_dict = context['waitFor'] + (wait_for_field, + to_be, + poll_interval, + max_attempts, + error_on_wait_timeout) = waitfor_step.get_poll_args(waitfor_dict, context) + + assert wait_for_field == 'field name' + assert to_be == '123.4' + assert poll_interval == 99 + assert max_attempts == 66 + assert not error_on_wait_timeout +# ----------------------get_poll_args ------------------------------------ From 35e08d220dd54c29d3736432b77ddbda7b998419 Mon Sep 17 00:00:00 2001 From: 345thomas Date: Thu, 1 Jun 2017 19:43:24 +0100 Subject: [PATCH 2/4] README updates for waitfor --- README.rst | 68 ++++++++++++++++++++++++++++++++++++++- pypyraws/steps/waitfor.py | 2 +- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index e2aaee7..a3e9251 100644 --- a/README.rst +++ b/README.rst @@ -76,7 +76,11 @@ steps | `pypyraws.steps.s3fetchyaml`_ | Fetch a yaml file from s3 into the pypyr | s3Fetch (dict) | | | context. | | +-------------------------------+-------------------------------------------------+------------------------------+ -| `pypyraws.steps.wait`_ | Wait for an aws client method to complete. | awsWaitIn (dict) | +| `pypyraws.steps.wait`_ | Wait for an aws client waiter method to | awsWaitIn (dict) | +| | complete. | | ++-------------------------------+-------------------------------------------------+------------------------------+ +| `pypyraws.steps.waitfor`_ | Wait for any aws client method to complete, | awsWaitFor (dict) | +| | even when it doesn't have an official waiter. | | +-------------------------------+-------------------------------------------------+------------------------------+ pypyraws.steps.client @@ -98,6 +102,9 @@ support all the methods you need. You can actually pretty much just grab the json as written from the excellent AWS help docs, paste it into some json that pypyr consumes and tadaaa! +Alternatively, grab the samples from the boto3 python documentation to include +in some yaml - the python dictionary structures map to yaml without too much +faff. Supported AWS services ---------------------- @@ -361,6 +368,65 @@ The input context requires: The *awsWaitIn* context supports text `Substitutions`_. +pypyraws.steps.waitfor +====================== +Custom waiter for any aws client operation. Where `pypyraws.steps.wait`_ uses +the official AWS waiters from the low-level client api, this step allows you to +execute *any* aws low-level client method and wait for a specified field in +the response to become the value you want it to be. + +This is especially handy for things like Beanstalk, because Elastic Beanstalk +does not have Waiters for environment creation. + +The input context looks like this: + +.. code-block:: yaml + + awsWaitFor: + awsClientIn: # required. awsClientIn allows the same arguments as pypyraws.steps.client. + serviceName: elasticbeanstalk + methodName: describe_environments + methodArgs: + ApplicationName: my wonderful beanstalk default application + EnvironmentNames: + - my-wonderful-environment + VersionLabel: v0.1 + waitForField: '{Environments[0][Status]}' # required. format expression for field name to check in awsClient response + toBe: Ready # required. Stop waiting when waitForField equals this value + pollInterval: 30 # optional. Seconds to wait between polling attempts. Defaults to 30 if not specified. + maxAttempts: 10 # optional. Defaults to 10 if not specified. + errorOnWaitTimeout: True # optional. Defaults to True if not specified. Stop processing if maxAttempts exhausted without reaching toBe value. + +See `pypyraws.steps.client`_ for a full listing of available arguments under +*awsClientIn*. + +If ``errorOnWaitTimeout`` is True and ``max_attempts`` exhaust before reaching +the desired target state, pypyr will stop processing with a +``pypyraws.errors.WaitTimeOut`` error. + +Once this step completes it adds ``awsWaitForTimedOut`` to the pypyr context. +This is a boolean value with values: + ++--------------------------+---------------------------------------------------+ +| ``awsWaitForTimedOut`` | Description | ++--------------------------+---------------------------------------------------+ +| True | ``errorOnWaitTimeout=False`` and ``max_attempts`` | +| | exhausted without reaching ``toBe``. | ++--------------------------+---------------------------------------------------+ +| False | ``waitForField``'s value becomes ``toBe`` within | +| | ``max_attempts``. | ++--------------------------+---------------------------------------------------+ + + +The *awsWaitFor* context supports text `Substitutions`_. Do note that while +``waitForField`` uses substitution style format strings, the substitutions are +made against the response object that returns from the aws client call specified +in *awsClientIn*, and not from the pypyr context itself. + +See a worked example for an `elastic beanstalk custom waiter for environmment +creation here +`__. + ************* Substitutions ************* diff --git a/pypyraws/steps/waitfor.py b/pypyraws/steps/waitfor.py index edbf818..540a1bb 100644 --- a/pypyraws/steps/waitfor.py +++ b/pypyraws/steps/waitfor.py @@ -54,7 +54,7 @@ def run_step(context): pypyr.errors.KeyNotInContextError: awsWaitFor missing in context. pypyr.errors.KeyInContextHasNoValueError: awsWaitFor exists but is None. - pypyr.errors.WaitTimeOut: maxAttempts exceeded without waitForField + pypyraws.errors.WaitTimeOut: maxAttempts exceeded without waitForField changing to toBe. """ logger.debug("started") From 2b8b46fd99a816e9231c272e86df49222709586f Mon Sep 17 00:00:00 2001 From: 345thomas Date: Thu, 1 Jun 2017 19:49:59 +0100 Subject: [PATCH 3/4] README formatting update --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a3e9251..fcd8324 100644 --- a/README.rst +++ b/README.rst @@ -408,7 +408,7 @@ Once this step completes it adds ``awsWaitForTimedOut`` to the pypyr context. This is a boolean value with values: +--------------------------+---------------------------------------------------+ -| ``awsWaitForTimedOut`` | Description | +| awsWaitForTimedOut | Description | +--------------------------+---------------------------------------------------+ | True | ``errorOnWaitTimeout=False`` and ``max_attempts`` | | | exhausted without reaching ``toBe``. | From 1e72a2fce7a6039f36e60f3f8926fccfc3321dd4 Mon Sep 17 00:00:00 2001 From: 345thomas Date: Fri, 2 Jun 2017 15:47:14 +0100 Subject: [PATCH 4/4] reword 'regardless' --- pypyraws/steps/waitfor.py | 2 +- tests/unit/pypyraws/steps/waitfor_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pypyraws/steps/waitfor.py b/pypyraws/steps/waitfor.py index 540a1bb..835c34f 100644 --- a/pypyraws/steps/waitfor.py +++ b/pypyraws/steps/waitfor.py @@ -108,7 +108,7 @@ def run_step(context): context['awsWaitForTimedOut'] = True logger.warn(f"aws {service_name} {method_name} did NOT return " f" {to_be}. errorOnWaitTimeout is False, so pipeline " - "will continue regardless.") + "will proceed to the next step anyway.") logger.debug("done") diff --git a/tests/unit/pypyraws/steps/waitfor_test.py b/tests/unit/pypyraws/steps/waitfor_test.py index a1074b3..74beabd 100644 --- a/tests/unit/pypyraws/steps/waitfor_test.py +++ b/tests/unit/pypyraws/steps/waitfor_test.py @@ -137,7 +137,7 @@ def test_waitfor_fail_no_client_args_no_throw(mock_sleep, mock_service): mock_logger_warn.assert_called_once_with( 'aws service name method_name did NOT return xxx. errorOnWaitTimeout ' - 'is False, so pipeline will continue regardless.') + 'is False, so pipeline will proceed to the next step anyway.') assert len(context) == 3 assert context['k1'] == 'v1'