Handle Container Creation Failure in local invoke, `local start-api…

…` and `local start-lambda` (#6719)

* Refactor to run sam validate before any sam local logic

* Formatting

* refactor sam validate

* raise docker image creation error

* fix lint errors

* update type hint

* handle container creation error in local apigw service

* reformat

* handle errors in local api invoke and local lambda invoke

* move exception

* handle error in local invoke

* remove unnecessary integration tests

* fix formatting

* use try/except to run lint

---------

Co-authored-by: Jared Bentvelsen <[email protected]>
Co-authored-by: Wing Fung Lau <[email protected]>

samcli/commands/local/cli_common/invoke_context.py

@@ -11,10 +11,11 @@
 from typing import Any, Dict, List, Optional, TextIO, Tuple, Type, cast
 
 from samcli.commands._utils.template import TemplateFailedParsingException, TemplateNotFoundException
-from samcli.commands.exceptions import ContainersInitializationException
+from samcli.commands.exceptions import ContainersInitializationException, UserException
 from samcli.commands.local.cli_common.user_exceptions import DebugContextException, InvokeContextException
 from samcli.commands.local.lib.debug_context import DebugContext
 from samcli.commands.local.lib.local_lambda import LocalLambdaRunner
+from samcli.lib.lint import get_lint_matches
 from samcli.lib.providers.provider import Function, Stack
 from samcli.lib.providers.sam_function_provider import RefreshableSamFunctionProvider, SamFunctionProvider
 from samcli.lib.providers.sam_stack_provider import SamLocalStackProvider
@@ -100,6 +101,7 @@ def __init__(
         container_host_interface: Optional[str] = None,
         add_host: Optional[dict] = None,
         invoke_images: Optional[str] = None,
+        verbose: bool = False,
     ) -> None:
         """
         Initialize the context
@@ -154,6 +156,8 @@ def __init__(
             Optional. Docker extra hosts support from --add-host parameters
         invoke_images dict
             Optional. A dictionary that defines the custom invoke image URI of each function
+        verbose bool
+            Set template validation to verbose mode
         """
         self._template_file = template_file
         self._function_identifier = function_identifier
@@ -178,6 +182,7 @@ def __init__(
         self._aws_region = aws_region
         self._aws_profile = aws_profile
         self._shutdown = shutdown
+        self._verbose = verbose
 
         self._container_host = container_host
         self._container_host_interface = container_host_interface
@@ -220,6 +225,13 @@ def __enter__(self) -> "InvokeContext":
 
         self._stacks = self._get_stacks()
 
+        try:
+            _, matches_output = get_lint_matches(self._template_file, self._verbose, self._aws_region)
+            if matches_output:
+                LOG.warning("Lint Error found, containter creation might fail: %s", matches_output)
+        except UserException as e:
+            LOG.warning("Non blocking error found when trying to validate template: %s", e)
+
         _function_providers_class: Dict[ContainersMode, Type[SamFunctionProvider]] = {
             ContainersMode.WARM: RefreshableSamFunctionProvider,
             ContainersMode.COLD: SamFunctionProvider,

samcli/commands/local/invoke/cli.py

@@ -16,7 +16,11 @@
 from samcli.commands.local.lib.exceptions import InvalidIntermediateImageError
 from samcli.lib.telemetry.metric import track_command
 from samcli.lib.utils.version_checker import check_newer_version
-from samcli.local.docker.exceptions import ContainerNotStartableException, PortAlreadyInUse
+from samcli.local.docker.exceptions import (
+    ContainerNotStartableException,
+    DockerContainerCreationFailedException,
+    PortAlreadyInUse,
+)
 
 LOG = logging.getLogger(__name__)
 
@@ -200,6 +204,7 @@ def do_cli(  # pylint: disable=R0914
             container_host_interface=container_host_interface,
             add_host=add_host,
             invoke_images=processed_invoke_images,
+            ctx=ctx,
         ) as context:
             # Invoke the function
             context.local_lambda_runner.invoke(
@@ -220,7 +225,7 @@ def do_cli(  # pylint: disable=R0914
         PortAlreadyInUse,
     ) as ex:
         raise UserException(str(ex), wrapped_from=ex.__class__.__name__) from ex
-    except DockerImagePullFailedException as ex:
+    except (DockerImagePullFailedException, DockerContainerCreationFailedException) as ex:
         raise UserException(str(ex), wrapped_from=ex.__class__.__name__) from ex
     except ContainerNotStartableException as ex:
         raise UserException(str(ex), wrapped_from=ex.__class__.__name__) from ex

samcli/commands/local/start_api/cli.py

@@ -253,6 +253,7 @@ def do_cli(  # pylint: disable=R0914
             container_host_interface=container_host_interface,
             invoke_images=processed_invoke_images,
             add_host=add_host,
+            ctx=ctx,
         ) as invoke_context:
             ssl_context = (ssl_cert_file, ssl_key_file) if ssl_cert_file else None
             service = LocalApiService(

samcli/commands/local/start_lambda/cli.py

@@ -205,6 +205,7 @@ def do_cli(  # pylint: disable=R0914
             container_host_interface=container_host_interface,
             add_host=add_host,
             invoke_images=processed_invoke_images,
+            ctx=ctx,
         ) as invoke_context:
             service = LocalLambdaService(lambda_invoke_context=invoke_context, port=port, host=host)
             service.start()

samcli/commands/validate/validate.py

@@ -142,49 +142,15 @@ def _lint(ctx: Context, template: str) -> None:
         Path to the template file
 
     """
+    from samcli.lib.lint import get_lint_matches
 
-    import logging
-
-    import cfnlint.core  # type: ignore
-
-    from samcli.commands.exceptions import UserException
-
-    cfn_lint_logger = logging.getLogger("cfnlint")
-    cfn_lint_logger.propagate = False
     EventTracker.track_event("UsedFeature", "CFNLint")
+    matches, matches_output = get_lint_matches(template, ctx.debug, ctx.region)
+    if not matches:
+        click.secho("{} is a valid SAM Template".format(template), fg="green")
+        return
 
-    try:
-        lint_args = [template]
-        if ctx.debug:
-            lint_args.append("--debug")
-        if ctx.region:
-            lint_args.append("--region")
-            lint_args.append(ctx.region)
-
-        (args, filenames, formatter) = cfnlint.core.get_args_filenames(lint_args)
-        cfn_lint_logger.setLevel(logging.WARNING)
-        matches = list(cfnlint.core.get_matches(filenames, args))
-        if not matches:
-            click.secho("{} is a valid SAM Template".format(template), fg="green")
-            return
-
-        rules = cfnlint.core.get_used_rules()
-        matches_output = formatter.print_matches(matches, rules, filenames)
-
-        if matches_output:
-            click.secho(matches_output)
-
-        raise LinterRuleMatchedException(
-            "Linting failed. At least one linting rule was matched to the provided template."
-        )
+    if matches_output:
+        click.secho(matches_output)
 
-    except cfnlint.core.InvalidRegionException as e:
-        raise UserException(
-            "AWS Region was not found. Please configure your region through the --region option",
-            wrapped_from=e.__class__.__name__,
-        ) from e
-    except cfnlint.core.CfnLintExitException as lint_error:
-        raise UserException(
-            lint_error,
-            wrapped_from=lint_error.__class__.__name__,
-        ) from lint_error
+    raise LinterRuleMatchedException("Linting failed. At least one linting rule was matched to the provided template.")

samcli/lib/lint.py

@@ -0,0 +1,60 @@
+from typing import List, Optional, Tuple
+
+
+def get_lint_matches(template: str, debug: Optional[bool] = None, region: Optional[str] = None) -> Tuple[List, str]:
+    """
+    Parses provided SAM template and maps errors from CloudFormation template back to SAM template.
+
+    Cfn-lint loggers are added to the SAM cli logging hierarchy which at the root logger
+    configures with INFO level logging and a different formatting. This exposes and duplicates
+    some cfn-lint logs that are not typically shown to customers. Explicitly setting the level to
+    WARNING and propagate to be False remediates these issues.
+
+    Parameters
+    -----------
+    template
+        Path to the template file
+    debug
+        Enable debug mode
+    region
+        AWS region to run against
+
+    """
+
+    import logging
+
+    import cfnlint.core  # type: ignore
+
+    from samcli.commands.exceptions import UserException
+
+    cfn_lint_logger = logging.getLogger("cfnlint")
+    cfn_lint_logger.propagate = False
+
+    try:
+        lint_args = [template]
+        if debug:
+            lint_args.append("--debug")
+        if region:
+            lint_args.append("--region")
+            lint_args.append(region)
+
+        (args, filenames, formatter) = cfnlint.core.get_args_filenames(lint_args)
+        cfn_lint_logger.setLevel(logging.WARNING)
+        matches = list(cfnlint.core.get_matches(filenames, args))
+        if not matches:
+            return matches, ""
+
+        rules = cfnlint.core.get_used_rules()
+        matches_output = formatter.print_matches(matches, rules, filenames)
+        return matches, matches_output
+
+    except cfnlint.core.InvalidRegionException as e:
+        raise UserException(
+            "AWS Region was not found. Please configure your region through the --region option",
+            wrapped_from=e.__class__.__name__,
+        ) from e
+    except cfnlint.core.CfnLintExitException as lint_error:
+        raise UserException(
+            lint_error,
+            wrapped_from=lint_error.__class__.__name__,
+        ) from lint_error

samcli/local/apigw/local_apigw_service.py

@@ -30,6 +30,7 @@
 from samcli.local.apigw.path_converter import PathConverter
 from samcli.local.apigw.route import Route
 from samcli.local.apigw.service_error_responses import ServiceErrorResponses
+from samcli.local.docker.exceptions import DockerContainerCreationFailedException
 from samcli.local.events.api_event import (
     ContextHTTP,
     ContextIdentity,
@@ -730,6 +731,8 @@ def _request_handler(self, **kwargs):
             )
         except LambdaResponseParseException:
             endpoint_service_error = ServiceErrorResponses.lambda_body_failure_response()
+        except DockerContainerCreationFailedException as ex:
+            endpoint_service_error = ServiceErrorResponses.container_creation_failed(ex.message)
 
         if endpoint_service_error:
             return endpoint_service_error

samcli/local/apigw/service_error_responses.py

@@ -101,3 +101,13 @@ def route_not_found(*args):
         """
         response_data = jsonify(ServiceErrorResponses._MISSING_AUTHENTICATION)
         return make_response(response_data, ServiceErrorResponses.HTTP_STATUS_CODE_403)
+
+    @staticmethod
+    def container_creation_failed(message):
+        """
+        Constuct a Flask Response for when container creation fails for a Lambda Function
+
+        :return: a Flask Response
+        """
+        response_data = jsonify({"message": message})
+        return make_response(response_data, ServiceErrorResponses.HTTP_STATUS_CODE_501)

samcli/local/docker/container.py

@@ -17,14 +17,23 @@
 
 import docker
 import requests
-from docker.errors import NotFound as DockerNetworkNotFound
+from docker.errors import (
+    APIError as DockerAPIError,
+)
+from docker.errors import (
+    NotFound as DockerNetworkNotFound,
+)
 
 from samcli.lib.constants import DOCKER_MIN_API_VERSION
 from samcli.lib.utils.retry import retry
 from samcli.lib.utils.stream_writer import StreamWriter
 from samcli.lib.utils.tar import extract_tarfile
 from samcli.local.docker.effective_user import ROOT_USER_ID, EffectiveUser
-from samcli.local.docker.exceptions import ContainerNotStartableException, PortAlreadyInUse
+from samcli.local.docker.exceptions import (
+    ContainerNotStartableException,
+    DockerContainerCreationFailedException,
+    PortAlreadyInUse,
+)
 from samcli.local.docker.utils import NoFreePortsError, find_free_port, to_posix_path
 
 LOG = logging.getLogger(__name__)
@@ -227,7 +236,12 @@ def create(self):
         if self._extra_hosts:
             kwargs["extra_hosts"] = self._extra_hosts
 
-        real_container = self.docker_client.containers.create(self._image, **kwargs)
+        try:
+            real_container = self.docker_client.containers.create(self._image, **kwargs)
+        except DockerAPIError as ex:
+            raise DockerContainerCreationFailedException(
+                f"Container creation failed: {ex.explanation}, check template for potential issue"
+            )
         self.id = real_container.id
 
         self._logs_thread = None

samcli/local/docker/exceptions.py

@@ -37,3 +37,9 @@ class InvalidRuntimeException(UserException):
     """
     Raised when an invalid runtime is specified for a Lambda Function
     """
+
+
+class DockerContainerCreationFailedException(UserException):
+    """
+    Docker Container Creation failed. It could be due to invalid template
+    """

samcli/local/lambda_service/lambda_error_responses.py

@@ -20,6 +20,7 @@ class LambdaErrorResponses:
     InvalidRequestContentException = ("InvalidRequestContent", 400)
 
     NotImplementedException = ("NotImplemented", 501)
+    ContainerCreationFailed = ("ContainerCreationFailed", 501)
 
     PathNotFoundException = ("PathNotFoundLocally", 404)
 
@@ -204,6 +205,31 @@ def generic_method_not_allowed(*args):
             exception_tuple[1],
         )
 
+    @staticmethod
+    def container_creation_failed(message):
+        """
+        Creates a Container Creation Failed response
+
+        Parameters
+        ----------
+        args list
+            List of arguments Flask passes to the method
+
+        Returns
+        -------
+        Flask.Response
+            A response object representing the ContainerCreationFailed Error
+        """
+        error_type, status_code = LambdaErrorResponses.ContainerCreationFailed
+        return BaseLocalService.service_response(
+            LambdaErrorResponses._construct_error_response_body(
+                LambdaErrorResponses.LOCAL_SERVICE_ERROR,
+                message,
+            ),
+            LambdaErrorResponses._construct_headers(error_type),
+            status_code,
+        )
+
     @staticmethod
     def _construct_error_response_body(error_type, error_message):
         """

samcli/local/lambda_service/local_lambda_invoke_service.py

@@ -9,6 +9,7 @@
 
 from samcli.commands.local.lib.exceptions import UnsupportedInlineCodeError
 from samcli.lib.utils.stream_writer import StreamWriter
+from samcli.local.docker.exceptions import DockerContainerCreationFailedException
 from samcli.local.lambdafn.exceptions import FunctionNotFound
 from samcli.local.services.base_local_service import BaseLocalService, LambdaOutputParser
 
@@ -178,6 +179,8 @@ def _invoke_request_handler(self, function_name):
             return LambdaErrorResponses.not_implemented_locally(
                 "Inline code is not supported for sam local commands. Please write your code in a separate file."
             )
+        except DockerContainerCreationFailedException as ex:
+            return LambdaErrorResponses.container_creation_failed(ex.message)
 
         lambda_response, is_lambda_user_error_response = LambdaOutputParser.get_lambda_output(
             stdout_stream_string, stdout_stream_bytes

samcli/local/lambdafn/runtime.py

@@ -16,7 +16,7 @@
 from samcli.lib.utils.packagetype import ZIP
 from samcli.local.docker.container import Container
 from samcli.local.docker.container_analyzer import ContainerAnalyzer
-from samcli.local.docker.exceptions import ContainerFailureError
+from samcli.local.docker.exceptions import ContainerFailureError, DockerContainerCreationFailedException
 from samcli.local.docker.lambda_container import LambdaContainer
 
 from ...lib.providers.provider import LayerVersion
@@ -112,6 +112,10 @@ def create(
             self._container_manager.create(container)
             return container
 
+        except DockerContainerCreationFailedException:
+            LOG.warning("Failed to create container for function %s", function_config.full_path)
+            raise
+
         except KeyboardInterrupt:
             LOG.debug("Ctrl+C was pressed. Aborting container creation")
             raise