feat: sam Commands Understand Local File Paths for `AWS::Serverless::…

…Function.ImageUri` (#6930)

* feat: sam Commands Understand Local File Paths for `ImageUri`

As a summary, sam has learned to load an an image from a local archive
before proceeding with the `build`, `package`, and `deploy` commands.

When running `sam build` with an `ImageUri` pointing to a local file,
sam will load that archive into an image, then write the ID of the
image to the `ImageUri` property of the built template. ID works the
same as a tag for the Docker API, so business continues as usual from
here. The reason behind writing ID is that a loaded image could be
associated with multiple tags, and selecting one arbtrarily leads to
difficulties in the deploy command.

The package and deploy commands have three kinds of value for
`ImageUri` to consider. First, a value of the form
`{repo}:{tag}`. This functions as it always has. Second, an image
ID (in the form `sha256:{digest}`) which is probably the output of
`sam build`. In this case, the tag translation uses the name of the
as its input. Otherwise, they'd all end up with names starting with
"sha256". Last, a local file. In this case, it proceeds as it does in
the build command: Load the archive into an image first, then pass the
resource name into tag translation.

See: #6909

* Reword Explanatory Comment

See: #6909

* Correct Old Typo in String Parameter

See: #6909

* Take a Swing at Unit Tests

See: #6909

* Cover Another Test Case

See: #6909

* Take a Swing at Integration Tests

Also, genericize unit tests a little.

See: #6909

* Now Add Package Integration Tests

* And Deploy Integration Tests

* Point to Correct File, Remove Unused Import

---------

Co-authored-by: jysheng123 <[email protected]>
Co-authored-by: sidhujus <[email protected]>

samcli/commands/_utils/template.py

@@ -162,7 +162,11 @@ def _update_relative_paths(template_dict, original_root, new_root):
                 resource_type in [AWS_SERVERLESS_FUNCTION, AWS_LAMBDA_FUNCTION]
                 and properties.get("PackageType", ZIP) == IMAGE
             ):
-                continue
+                if not properties.get("ImageUri"):
+                    continue
+                resolved_image_archive_path = _resolve_relative_to(properties.get("ImageUri"), original_root, new_root)
+                if not resolved_image_archive_path or not pathlib.Path(resolved_image_archive_path).is_file():
+                    continue
 
             # SAM GraphQLApi has many instances of CODE_ARTIFACT_PROPERTY and all of them must be updated
             if resource_type == AWS_SERVERLESS_GRAPHQLAPI and path_prop_name == graphql_api.CODE_ARTIFACT_PROPERTY:

samcli/lib/build/app_builder.py

@@ -7,6 +7,7 @@
 import logging
 import os
 import pathlib
+from pathlib import Path
 from typing import Dict, List, NamedTuple, Optional, cast
 
 import docker
@@ -250,6 +251,7 @@ def _get_build_graph(
             function_build_details = FunctionBuildDefinition(
                 function.runtime,
                 function.codeuri,
+                function.imageuri,
                 function.packagetype,
                 function.architecture,
                 function.metadata,
@@ -460,6 +462,16 @@ def _stream_lambda_image_build_logs(self, build_logs: List[Dict[str, str]], func
         except LogStreamError as ex:
             raise DockerBuildFailed(msg=f"{function_name} failed to build: {str(ex)}") from ex
 
+    def _load_lambda_image(self, image_archive_path: str) -> str:
+        try:
+            with open(image_archive_path, mode="rb") as image_archive:
+                [image, *rest] = self._docker_client.images.load(image_archive)
+                if len(rest) != 0:
+                    raise DockerBuildFailed("Archive must represent a single image")
+                return f"{image.id}"
+        except (docker.errors.APIError, OSError) as ex:
+            raise DockerBuildFailed(msg=str(ex)) from ex
+
     def _build_layer(
         self,
         layer_name: str,
@@ -599,6 +611,7 @@ def _build_function(  # pylint: disable=R1710
         self,
         function_name: str,
         codeuri: str,
+        imageuri: Optional[str],
         packagetype: str,
         runtime: str,
         architecture: str,
@@ -619,6 +632,9 @@ def _build_function(  # pylint: disable=R1710
             Name or LogicalId of the function
         codeuri : str
             Path to where the code lives
+        imageuri : str
+            Location of the Lambda Image which is of the form {image}:{tag}, sha256:{digest},
+            or a path to a local archive
         packagetype : str
             The package type, 'Zip' or 'Image', see samcli/lib/utils/packagetype.py
         runtime : str
@@ -646,6 +662,8 @@ def _build_function(  # pylint: disable=R1710
             Path to the location where built artifacts are available
         """
         if packagetype == IMAGE:
+            if imageuri and Path(imageuri).is_file():  # something exists at this path and what exists is a file
+                return self._load_lambda_image(imageuri)  # should be an image archive – load it instead of building it
             # pylint: disable=fixme
             # FIXME: _build_lambda_image assumes metadata is not None, we need to throw an exception here
             return self._build_lambda_image(

samcli/lib/build/build_graph.py

@@ -107,6 +107,7 @@ def _toml_table_to_function_build_definition(uuid: str, toml_table: tomlkit.api.
     function_build_definition = FunctionBuildDefinition(
         toml_table.get(RUNTIME_FIELD),
         toml_table.get(CODE_URI_FIELD),
+        None,
         toml_table.get(PACKAGETYPE_FIELD, ZIP),
         toml_table.get(ARCHITECTURE_FIELD, X86_64),
         dict(toml_table.get(METADATA_FIELD, {})),
@@ -584,6 +585,7 @@ def __init__(
         self,
         runtime: Optional[str],
         codeuri: Optional[str],
+        imageuri: Optional[str],
         packagetype: str,
         architecture: str,
         metadata: Optional[Dict],
@@ -595,6 +597,7 @@ def __init__(
         super().__init__(source_hash, manifest_hash, env_vars, architecture)
         self.runtime = runtime
         self.codeuri = codeuri
+        self.imageuri = imageuri
         self.packagetype = packagetype
         self.handler = handler
 
@@ -688,6 +691,7 @@ def __eq__(self, other: Any) -> bool:
         return (
             self.runtime == other.runtime
             and self.codeuri == other.codeuri
+            and self.imageuri == other.imageuri
             and self.packagetype == other.packagetype
             and self.metadata == other.metadata
             and self.env_vars == other.env_vars

samcli/lib/build/build_strategy.py

@@ -128,7 +128,9 @@ def __init__(
         self,
         build_graph: BuildGraph,
         build_dir: str,
-        build_function: Callable[[str, str, str, str, str, Optional[str], str, dict, dict, Optional[str], bool], str],
+        build_function: Callable[
+            [str, str, Optional[str], str, str, str, Optional[str], str, dict, dict, Optional[str], bool], str
+        ],
         build_layer: Callable[[str, str, str, List[str], str, str, dict, Optional[str], bool, Optional[Dict]], str],
         cached: bool = False,
     ) -> None:
@@ -166,6 +168,7 @@ def build_single_function_definition(self, build_definition: FunctionBuildDefini
         result = self._build_function(
             build_definition.get_function_name(),
             build_definition.codeuri,  # type: ignore
+            build_definition.imageuri,
             build_definition.packagetype,
             build_definition.runtime,  # type: ignore
             build_definition.architecture,

samcli/lib/package/ecr_uploader.py

@@ -5,6 +5,7 @@
 import base64
 import logging
 from io import StringIO
+from pathlib import Path
 from typing import Dict
 
 import botocore
@@ -76,10 +77,26 @@ def upload(self, image, resource_name):
         if not self.login_session_active:
             self.login()
             self.login_session_active = True
+
+        # Sometimes the `resource_name` is used as the `image` parameter to `tag_translation`.
+        # This is because these two cases (directly from an archive or by ID) are effectively
+        # anonymous, so the best identifier available in scope is the resource name.
         try:
-            docker_img = self.docker_client.images.get(image)
+            if Path(image).is_file():
+                with open(image, mode="rb") as image_archive:
+                    [docker_img, *rest] = self.docker_client.images.load(image_archive)
+                    if len(rest) != 0:
+                        raise DockerPushFailedError("Archive must represent a single image")
+                    _tag = tag_translation(resource_name, docker_image_id=docker_img.id, gen_tag=self.tag)
+            else:
+                # If it's not a file, it's gotta be a {repo}:{tag} or a sha256:{digest}
+                docker_img = self.docker_client.images.get(image)
+                _tag = tag_translation(
+                    resource_name if image == docker_img.id else image,
+                    docker_image_id=docker_img.id,
+                    gen_tag=self.tag,
+                )
 
-            _tag = tag_translation(image, docker_image_id=docker_img.id, gen_tag=self.tag)
             repository = (
                 self.ecr_repo
                 if not self.ecr_repo_multi or not isinstance(self.ecr_repo_multi, dict)

samcli/lib/package/utils.py

@@ -125,6 +125,10 @@ def upload_local_image_artifacts(resource_id, resource_dict, property_name, pare
         LOG.debug("Property %s of %s is already an ECR URL", property_name, resource_id)
         return image_path
 
+    possible_image_archive_path = make_abs_path(parent_dir, image_path)
+    if is_local_file(possible_image_archive_path):
+        image_path = possible_image_archive_path
+
     return uploader.upload(image_path, resource_id)
 
 

samcli/lib/providers/provider.py

@@ -9,6 +9,7 @@
 import posixpath
 from collections import namedtuple
 from enum import Enum
+from pathlib import Path
 from typing import Any, Dict, Iterator, List, NamedTuple, Optional, Set, Union, cast
 
 from samcli.commands.local.cli_common.user_exceptions import (
@@ -953,6 +954,7 @@ def get_function_build_info(
     packagetype: str,
     inlinecode: Optional[str],
     codeuri: Optional[str],
+    imageuri: Optional[str],
     metadata: Optional[Dict],
 ) -> FunctionBuildInfo:
     """
@@ -974,8 +976,9 @@ def get_function_build_info(
         metadata = metadata or {}
         dockerfile = cast(str, metadata.get("Dockerfile", ""))
         docker_context = cast(str, metadata.get("DockerContext", ""))
-
-        if not dockerfile or not docker_context:
+        buildable = dockerfile and docker_context
+        loadable = imageuri and Path(imageuri).is_file()
+        if not buildable and not loadable:
             LOG.debug(
                 "Skip Building %s function, as it is missing either Dockerfile or DockerContext "
                 "metadata properties.",

samcli/lib/providers/sam_function_provider.py

@@ -3,6 +3,7 @@
 """
 
 import logging
+from pathlib import Path
 from typing import Any, Dict, Iterator, List, Optional, cast
 
 from samtranslator.policy_template_processor.exceptions import TemplateNotFoundException
@@ -446,12 +447,18 @@ def _build_function_configuration(
             LOG.debug("--base-dir is not presented, adjusting uri %s relative to %s", codeuri, stack.location)
             codeuri = SamLocalStackProvider.normalize_resource_path(stack.location, codeuri)
 
+        if imageuri and codeuri != ".":
+            normalized_image_uri = SamLocalStackProvider.normalize_resource_path(stack.location, imageuri)
+            if Path(normalized_image_uri).is_file():
+                LOG.debug("--base-dir is not presented, adjusting uri %s relative to %s", codeuri, stack.location)
+                imageuri = normalized_image_uri
+
         package_type = resource_properties.get("PackageType", ZIP)
         if package_type == ZIP and not resource_properties.get("Handler"):
             raise MissingFunctionHandlerException(f"Could not find handler for function: {name}")
 
         function_build_info = get_function_build_info(
-            get_full_path(stack.stack_path, function_id), package_type, inlinecode, codeuri, metadata
+            get_full_path(stack.stack_path, function_id), package_type, inlinecode, codeuri, imageuri, metadata
         )
 
         return Function(

samcli/local/docker/lambda_container.py

@@ -65,7 +65,8 @@ def __init__(
         runtime str
             Name of the Lambda runtime
         imageuri str
-            Name of the Lambda Image which is of the form {image}:{tag}
+            Location of the Lambda Image which is of the form {image}:{tag}, sha256:{digest},
+            or a path to a local archive
         handler str
             Handler of the function to run
         packagetype str
@@ -240,7 +241,8 @@ def _get_image(
         packagetype : str
             Package type for the lambda function which is either zip or image.
         image : str
-            Name of the Lambda Image which is of the form {image}:{tag}
+            Location of the Lambda Image which is of the form {image}:{tag}, sha256:{digest},
+            or a path to a local archive
         layers : List[str]
             List of layers
         architecture : str

samcli/local/lambdafn/config.py

@@ -46,7 +46,8 @@ def __init__(
         handler : str
             Handler method
         imageuri : str
-            Name of the Lambda Image which is of the form {image}:{tag}
+            Location of the Lambda Image which is of the form {image}:{tag}, sha256:{digest},
+            or a path to a local archive
         imageconfig : str
             Image configuration which can be used set to entrypoint, command and working dir for the container.
         packagetype : str

tests/integration/buildcmd/build_integ_base.py

@@ -219,6 +219,7 @@ def _verify_image_build_artifact(self, template_path, image_function_logical_id,
     def _verify_resource_property(self, template_path, logical_id, property, expected_value):
         with open(template_path, "r") as fp:
             template_dict = yaml_parse(fp.read())
+
             self.assertEqual(
                 expected_value, jmespath.search(f"Resources.{logical_id}.Properties.{property}", template_dict)
             )

tests/integration/buildcmd/test_build_cmd.py

@@ -86,6 +86,41 @@ def test_with_invalid_dockerfile_definition(self):
         self.assertIn("COPY requires at least two arguments", command_result.stderr.decode())
 
 
+@skipIf(SKIP_DOCKER_TESTS, SKIP_DOCKER_MESSAGE)
+class TestLoadingImagesFromArchive(BuildIntegBase):
+    template = "template_loadable_image.yaml"
+
+    FUNCTION_LOGICAL_ID = "ImageFunction"
+
+    def test_load_not_an_archive_passthrough(self):
+        overrides = {"ImageUri": "./load_image_archive/this_file_does_not_exist.tar.gz"}
+        cmdlist = self.get_command_list(parameter_overrides=overrides)
+        command_result = run_command(cmdlist, cwd=self.working_dir)
+
+        self.assertEqual(command_result.process.returncode, 0)
+
+    def test_bad_image_archive_fails(self):
+        overrides = {"ImageUri": "./load_image_archive/error.tar.gz"}
+        cmdlist = self.get_command_list(parameter_overrides=overrides)
+        command_result = run_command(cmdlist, cwd=self.working_dir)
+
+        self.assertEqual(command_result.process.returncode, 1)
+        self.assertIn("unexpected EOF", command_result.stderr.decode())
+
+    def test_load_success(self):
+        overrides = {"ImageUri": "./load_image_archive/archive.tar.gz"}
+        cmdlist = self.get_command_list(parameter_overrides=overrides)
+        command_result = run_command(cmdlist, cwd=self.working_dir)
+
+        self.assertEqual(command_result.process.returncode, 0)
+        self._verify_image_build_artifact(
+            self.built_template,
+            self.FUNCTION_LOGICAL_ID,
+            "ImageUri",
+            "sha256:81d2ff8422e3a78dc0c1eff53d8e46f5666a801b17b5607a920860c2d234f9d0",
+        )
+
+
 @skipIf(
     # Hits public ECR pull limitation, move it to canary tests
     (not RUN_BY_CANARY and not CI_OVERRIDE),

tests/integration/deploy/test_deploy_command.py

@@ -134,6 +134,60 @@ def test_no_package_and_deploy_with_s3_bucket_all_args(self, template_file):
         deploy_process_execute = self.run_command(deploy_command_list)
         self.assertEqual(deploy_process_execute.process.returncode, 0)
 
+    @parameterized.expand(["template-image-load.yaml"])
+    def test_deploy_directly_from_image_archive(self, template_file):
+        template_path = self.test_data_path.joinpath(os.path.join("load-image-archive", template_file))
+
+        stack_name = self._method_to_stack_name(self.id())
+        self.stacks.append({"name": stack_name})
+
+        # Package and Deploy in one go without confirming change set.
+        deploy_command_list = self.get_deploy_command_list(
+            template_file=template_path,
+            stack_name=stack_name,
+            capabilities="CAPABILITY_IAM",
+            s3_prefix=self.s3_prefix,
+            s3_bucket=self.s3_bucket.name,
+            image_repository=self.ecr_repo_name,
+            force_upload=True,
+            notification_arns=self.sns_arn,
+            parameter_overrides="Parameter=Clarity",
+            kms_key_id=self.kms_key,
+            no_execute_changeset=False,
+            tags="integ=true clarity=yes foo_bar=baz",
+            confirm_changeset=False,
+        )
+
+        deploy_process_execute = self.run_command(deploy_command_list)
+        self.assertEqual(deploy_process_execute.process.returncode, 0)
+
+    @parameterized.expand(["template-image-load-fail.yaml"])
+    def test_deploy_directly_from_image_archive_but_error_fail(self, template_file):
+        template_path = self.test_data_path.joinpath(os.path.join("load-image-archive", template_file))
+
+        stack_name = self._method_to_stack_name(self.id())
+        self.stacks.append({"name": stack_name})
+
+        # Package and Deploy in one go without confirming change set.
+        deploy_command_list = self.get_deploy_command_list(
+            template_file=template_path,
+            stack_name=stack_name,
+            capabilities="CAPABILITY_IAM",
+            s3_prefix=self.s3_prefix,
+            s3_bucket=self.s3_bucket.name,
+            image_repository=self.ecr_repo_name,
+            force_upload=True,
+            notification_arns=self.sns_arn,
+            parameter_overrides="Parameter=Clarity",
+            kms_key_id=self.kms_key,
+            no_execute_changeset=False,
+            tags="integ=true clarity=yes foo_bar=baz",
+            confirm_changeset=False,
+        )
+
+        deploy_process_execute = self.run_command(deploy_command_list)
+        self.assertEqual(deploy_process_execute.process.returncode, 1)
+
     @parameterized.expand(
         [
             "aws-serverless-function-image.yaml",