Merge pull request #5 from pypyr/dev

waitfor step - wait for any aws client method

Author: thomas

README.rst

@@ -76,7 +76,11 @@ steps
 | `pypyraws.steps.s3fetchyaml`_ | Fetch a yaml file from s3 into the pypyr        | s3Fetch (dict)               |
 |                               | context.                                        |                              |
 +-------------------------------+-------------------------------------------------+------------------------------+
-| `pypyraws.steps.wait`_        | Wait for an aws client method to complete.      | awsWaitIn (dict)             |
+| `pypyraws.steps.wait`_        | Wait for an aws client waiter method to         | awsWaitIn (dict)             |
+|                               | complete.                                       |                              |
++-------------------------------+-------------------------------------------------+------------------------------+
+| `pypyraws.steps.waitfor`_     | Wait for any aws client method to complete,     | awsWaitFor (dict)            |
+|                               | even when it doesn't have an official waiter.   |                              |
 +-------------------------------+-------------------------------------------------+------------------------------+
 
 pypyraws.steps.client
@@ -98,6 +102,9 @@ support all the methods you need.
 
 You can actually pretty much just grab the json as written from the excellent
 AWS help docs, paste it into some json that pypyr consumes and tadaaa!
+Alternatively, grab the samples from the boto3 python documentation to include
+in some yaml - the python dictionary structures map to yaml without too much
+faff.
 
 Supported AWS services
 ----------------------
@@ -361,6 +368,65 @@ The input context requires:
 
 The *awsWaitIn* context supports text `Substitutions`_.
 
+pypyraws.steps.waitfor
+======================
+Custom waiter for any aws client operation. Where `pypyraws.steps.wait`_ uses
+the official AWS waiters from the low-level client api, this step allows you to
+execute *any* aws low-level client method and wait for a specified field in
+the response to become the value you want it to be.
+
+This is especially handy for things like Beanstalk, because Elastic Beanstalk
+does not have Waiters for environment creation.
+
+The input context looks like this:
+
+.. code-block:: yaml
+
+  awsWaitFor:
+    awsClientIn: # required. awsClientIn allows the same arguments as pypyraws.steps.client.
+      serviceName: elasticbeanstalk
+      methodName: describe_environments
+      methodArgs:
+          ApplicationName: my wonderful beanstalk default application
+          EnvironmentNames:
+            - my-wonderful-environment
+          VersionLabel: v0.1
+    waitForField: '{Environments[0][Status]}' # required. format expression for field name to check in awsClient response
+    toBe: Ready # required. Stop waiting when waitForField equals this value
+    pollInterval: 30 # optional. Seconds to wait between polling attempts. Defaults to 30 if not specified.
+    maxAttempts: 10 # optional. Defaults to 10 if not specified.
+    errorOnWaitTimeout: True # optional. Defaults to True if not specified. Stop processing if maxAttempts exhausted without reaching toBe value.
+
+See `pypyraws.steps.client`_ for a full listing of available arguments under
+*awsClientIn*.
+
+If ``errorOnWaitTimeout`` is True and ``max_attempts`` exhaust before reaching
+the desired target state, pypyr will stop processing with a
+``pypyraws.errors.WaitTimeOut`` error.
+
+Once this step completes it adds ``awsWaitForTimedOut`` to the pypyr context.
+This is a boolean value with values:
+
++--------------------------+---------------------------------------------------+
+| awsWaitForTimedOut       | Description                                       |
++--------------------------+---------------------------------------------------+
+| True                     | ``errorOnWaitTimeout=False`` and ``max_attempts`` |
+|                          | exhausted without reaching ``toBe``.              |
++--------------------------+---------------------------------------------------+
+| False                    | ``waitForField``'s value becomes ``toBe`` within  |
+|                          | ``max_attempts``.                                 |
++--------------------------+---------------------------------------------------+
+
+
+The *awsWaitFor* context supports text `Substitutions`_. Do note that while
+``waitForField`` uses substitution style format strings, the substitutions are
+made against the response object that returns from the aws client call specified
+in *awsClientIn*, and not from the pypyr context itself.
+
+See a worked example for an `elastic beanstalk custom waiter for environmment
+creation here
+<https://github.com/pypyr/pypyr-example/blob/master/pipelines/aws-beanstalk-waitfor.yaml>`__.
+
 *************
 Substitutions
 *************

pypyraws/contextargs.py

@@ -0,0 +1,59 @@
+"""Prepare aws arguments from the pypyr context."""
+from pypyr.errors import KeyInContextHasNoValueError, KeyNotInContextError
+
+
+def get_awsclient_args(context, calling_module_name):
+    """Get required args from context for awsClientIn type steps.
+
+    Args:
+        context: pypyr.context.Context.
+        calling_module_name: string. This is just to make a friendly error msg
+                             should something go wrong.
+
+    Returns:
+        tuple(client_in, service_name, method_name)
+
+    Raises:
+        pypyr.errors.KeyNotInContextError: Required key missing in context.
+        pypyr.errors.KeyInContextHasNoValueError: Required key exists but is
+                                                  empty or None.
+    """
+    try:
+        client_in = context['awsClientIn']
+        service_name = client_in['serviceName']
+        method_name = client_in['methodName']
+    except KeyError as err:
+        raise KeyNotInContextError(
+            f"awsClientIn missing required key for {calling_module_name}: "
+            f"{err}"
+        ) from err
+
+    if not (service_name and service_name.strip()):
+        raise KeyInContextHasNoValueError(
+            f'serviceName required in awsClientIn for {calling_module_name}')
+
+    if not (method_name and method_name.strip()):
+        raise KeyInContextHasNoValueError(
+            f'methodName required in awsClientIn for {calling_module_name}')
+
+    return client_in, service_name, method_name
+
+
+def get_formatted_iterable(input_dict, field_name, context):
+    """Format inputdict's field_name field against context.
+
+    Args:
+        input_dict: dict. Dictionary containing dict to format.
+        field_name: str. Points at field in input_dict to format.
+        context: pypyr.context.Context. Substitutes string expressions from
+                 this.
+
+    Returns:
+        dict: Formatted dictionary that was at input_dict['field_name']
+              Returns None if input_dict['field_name'] doesn't exist.
+    """
+    output = input_dict.get(field_name, None)
+    if output is not None:
+        output = context.get_formatted_iterable(output)
+
+    return output

pypyraws/errors.py

@@ -0,0 +1,14 @@
+"""Custom exceptions for pypyraws.
+
+All pypyraws specific exceptions derive from pypyr root Error.
+"""
+
+from pypyr.errors import PlugInError
+
+
+class Error(PlugInError):
+    """Base class for all pypyraws exceptions."""
+
+
+class WaitTimeOut(Error):
+    """Aws resource that did not finish processing within wait limit."""

pypyraws/steps/client.py

@@ -1,7 +1,7 @@
 """pypyr step that runs any boto3 low-level client method."""
 import logging
 import pypyraws.aws.service
-from pypyr.errors import KeyInContextHasNoValueError, KeyNotInContextError
+import pypyraws.contextargs as contextargs
 
 
 # pypyr logger means the log level will be set correctly and output formatted.
@@ -41,15 +41,16 @@ def run_step(context):
                                                   None.
     """
     logger.debug("started")
-    client_in, service_name, method_name = get_service_args(context)
+    client_in, service_name, method_name = contextargs.get_awsclient_args(
+        context, __name__)
 
-    client_args = client_in.get('clientArgs', None)
-    if client_args is not None:
-        client_args = context.get_formatted_iterable(client_args)
+    client_args = contextargs.get_formatted_iterable(input_dict=client_in,
+                                                     field_name='clientArgs',
+                                                     context=context)
 
-    method_args = client_in.get('methodArgs', None)
-    if method_args is not None:
-        method_args = context.get_formatted_iterable(method_args)
+    method_args = contextargs.get_formatted_iterable(input_dict=client_in,
+                                                     field_name='methodArgs',
+                                                     context=context)
 
     context['awsClientOut'] = pypyraws.aws.service.operation_exec(
         service_name=context.get_formatted_string(service_name),
@@ -61,39 +62,3 @@ def run_step(context):
     logger.info(f"Executed {method_name} on aws {service_name}.")
 
     logger.debug("done")
-
-
-def get_service_args(context):
-    """Gets required args from context for this step.
-
-    Args:
-        context - dict. context.
-
-    Returns:
-        tuple(client_in, service_name, method_name)
-
-    Raises:
-        pypyr.errors.KeyNotInContextError: Required key missing in context.
-        pypyr.errors.KeyInContextHasNoValueError: Required key exists but is
-                                                  empty or None.
-    """
-    try:
-        client_in = context['awsClientIn']
-        service_name = client_in['serviceName']
-        method_name = client_in['methodName']
-    except KeyError as err:
-        raise KeyNotInContextError(
-            "awsClientIn missing required key for pypyraws.steps.client: "
-            f"{err}"
-        ) from err
-
-    # of course, if They went and made it a bool and True this will pass.
-    if not (service_name and service_name.strip()):
-        raise KeyInContextHasNoValueError(
-            'serviceName required in awsClientIn for pypyraws.steps.client')
-
-    if not (method_name and method_name.strip()):
-        raise KeyInContextHasNoValueError(
-            'methodName required in awsClientIn for pypyraws.steps.client')
-
-    return client_in, service_name, method_name