From e0bc44db64c9e81ea25521ed5cbd8fc8d2245c9c Mon Sep 17 00:00:00 2001 From: Chris/0 Date: Wed, 1 May 2024 12:38:43 -0400 Subject: [PATCH] feat: sam Commands Understand Local File Paths for `AWS::Serverless::Function.ImageUri` (#6930) * feat: sam Commands Understand Local File Paths for `ImageUri` As a summary, sam has learned to load an an image from a local archive before proceeding with the `build`, `package`, and `deploy` commands. When running `sam build` with an `ImageUri` pointing to a local file, sam will load that archive into an image, then write the ID of the image to the `ImageUri` property of the built template. ID works the same as a tag for the Docker API, so business continues as usual from here. The reason behind writing ID is that a loaded image could be associated with multiple tags, and selecting one arbtrarily leads to difficulties in the deploy command. The package and deploy commands have three kinds of value for `ImageUri` to consider. First, a value of the form `{repo}:{tag}`. This functions as it always has. Second, an image ID (in the form `sha256:{digest}`) which is probably the output of `sam build`. In this case, the tag translation uses the name of the as its input. Otherwise, they'd all end up with names starting with "sha256". Last, a local file. In this case, it proceeds as it does in the build command: Load the archive into an image first, then pass the resource name into tag translation. See: #6909 * Reword Explanatory Comment See: #6909 * Correct Old Typo in String Parameter See: #6909 * Take a Swing at Unit Tests See: #6909 * Cover Another Test Case See: #6909 * Take a Swing at Integration Tests Also, genericize unit tests a little. See: #6909 * Now Add Package Integration Tests * And Deploy Integration Tests * Point to Correct File, Remove Unused Import --------- Co-authored-by: jysheng123 <141280295+jysheng123@users.noreply.github.com> Co-authored-by: sidhujus <105385029+sidhujus@users.noreply.github.com> --- samcli/commands/_utils/template.py | 6 +- samcli/lib/build/app_builder.py | 18 +++ samcli/lib/build/build_graph.py | 4 + samcli/lib/build/build_strategy.py | 5 +- samcli/lib/package/ecr_uploader.py | 21 ++- samcli/lib/package/utils.py | 4 + samcli/lib/providers/provider.py | 7 +- samcli/lib/providers/sam_function_provider.py | 9 +- samcli/local/docker/lambda_container.py | 6 +- samcli/local/lambdafn/config.py | 3 +- .../integration/buildcmd/build_integ_base.py | 1 + tests/integration/buildcmd/test_build_cmd.py | 35 +++++ .../integration/deploy/test_deploy_command.py | 54 +++++++ .../package/test_package_command_image.py | 34 +++- .../load_image_archive/archive.tar.gz | Bin 0 -> 1158 bytes .../buildcmd/load_image_archive/error.tar.gz | Bin 0 -> 54 bytes .../buildcmd/template_loadable_image.yaml | 15 ++ .../package/load-image-archive/archive.tar.gz | Bin 0 -> 1158 bytes .../package/load-image-archive/error.tar.gz | Bin 0 -> 54 bytes .../template-image-load-fail.yaml | 22 +++ .../template-image-load.yaml | 22 +++ .../commands/buildcmd/test_build_context.py | 4 +- .../unit/lib/build_module/test_app_builder.py | 147 ++++++++++++++++-- .../unit/lib/build_module/test_build_graph.py | 76 ++++++--- .../lib/build_module/test_build_strategy.py | 20 ++- tests/unit/lib/package/test_ecr_uploader.py | 122 ++++++++++++++- 26 files changed, 577 insertions(+), 58 deletions(-) create mode 100644 tests/integration/testdata/buildcmd/load_image_archive/archive.tar.gz create mode 100644 tests/integration/testdata/buildcmd/load_image_archive/error.tar.gz create mode 100644 tests/integration/testdata/buildcmd/template_loadable_image.yaml create mode 100644 tests/integration/testdata/package/load-image-archive/archive.tar.gz create mode 100644 tests/integration/testdata/package/load-image-archive/error.tar.gz create mode 100644 tests/integration/testdata/package/load-image-archive/template-image-load-fail.yaml create mode 100644 tests/integration/testdata/package/load-image-archive/template-image-load.yaml diff --git a/samcli/commands/_utils/template.py b/samcli/commands/_utils/template.py index 622900d7e5..aa85fe3350 100644 --- a/samcli/commands/_utils/template.py +++ b/samcli/commands/_utils/template.py @@ -162,7 +162,11 @@ def _update_relative_paths(template_dict, original_root, new_root): resource_type in [AWS_SERVERLESS_FUNCTION, AWS_LAMBDA_FUNCTION] and properties.get("PackageType", ZIP) == IMAGE ): - continue + if not properties.get("ImageUri"): + continue + resolved_image_archive_path = _resolve_relative_to(properties.get("ImageUri"), original_root, new_root) + if not resolved_image_archive_path or not pathlib.Path(resolved_image_archive_path).is_file(): + continue # SAM GraphQLApi has many instances of CODE_ARTIFACT_PROPERTY and all of them must be updated if resource_type == AWS_SERVERLESS_GRAPHQLAPI and path_prop_name == graphql_api.CODE_ARTIFACT_PROPERTY: diff --git a/samcli/lib/build/app_builder.py b/samcli/lib/build/app_builder.py index a256c4627d..e7ab322144 100644 --- a/samcli/lib/build/app_builder.py +++ b/samcli/lib/build/app_builder.py @@ -7,6 +7,7 @@ import logging import os import pathlib +from pathlib import Path from typing import Dict, List, NamedTuple, Optional, cast import docker @@ -250,6 +251,7 @@ def _get_build_graph( function_build_details = FunctionBuildDefinition( function.runtime, function.codeuri, + function.imageuri, function.packagetype, function.architecture, function.metadata, @@ -460,6 +462,16 @@ def _stream_lambda_image_build_logs(self, build_logs: List[Dict[str, str]], func except LogStreamError as ex: raise DockerBuildFailed(msg=f"{function_name} failed to build: {str(ex)}") from ex + def _load_lambda_image(self, image_archive_path: str) -> str: + try: + with open(image_archive_path, mode="rb") as image_archive: + [image, *rest] = self._docker_client.images.load(image_archive) + if len(rest) != 0: + raise DockerBuildFailed("Archive must represent a single image") + return f"{image.id}" + except (docker.errors.APIError, OSError) as ex: + raise DockerBuildFailed(msg=str(ex)) from ex + def _build_layer( self, layer_name: str, @@ -599,6 +611,7 @@ def _build_function( # pylint: disable=R1710 self, function_name: str, codeuri: str, + imageuri: Optional[str], packagetype: str, runtime: str, architecture: str, @@ -619,6 +632,9 @@ def _build_function( # pylint: disable=R1710 Name or LogicalId of the function codeuri : str Path to where the code lives + imageuri : str + Location of the Lambda Image which is of the form {image}:{tag}, sha256:{digest}, + or a path to a local archive packagetype : str The package type, 'Zip' or 'Image', see samcli/lib/utils/packagetype.py runtime : str @@ -646,6 +662,8 @@ def _build_function( # pylint: disable=R1710 Path to the location where built artifacts are available """ if packagetype == IMAGE: + if imageuri and Path(imageuri).is_file(): # something exists at this path and what exists is a file + return self._load_lambda_image(imageuri) # should be an image archive – load it instead of building it # pylint: disable=fixme # FIXME: _build_lambda_image assumes metadata is not None, we need to throw an exception here return self._build_lambda_image( diff --git a/samcli/lib/build/build_graph.py b/samcli/lib/build/build_graph.py index a8121dd3dc..78085ad4d3 100644 --- a/samcli/lib/build/build_graph.py +++ b/samcli/lib/build/build_graph.py @@ -107,6 +107,7 @@ def _toml_table_to_function_build_definition(uuid: str, toml_table: tomlkit.api. function_build_definition = FunctionBuildDefinition( toml_table.get(RUNTIME_FIELD), toml_table.get(CODE_URI_FIELD), + None, toml_table.get(PACKAGETYPE_FIELD, ZIP), toml_table.get(ARCHITECTURE_FIELD, X86_64), dict(toml_table.get(METADATA_FIELD, {})), @@ -584,6 +585,7 @@ def __init__( self, runtime: Optional[str], codeuri: Optional[str], + imageuri: Optional[str], packagetype: str, architecture: str, metadata: Optional[Dict], @@ -595,6 +597,7 @@ def __init__( super().__init__(source_hash, manifest_hash, env_vars, architecture) self.runtime = runtime self.codeuri = codeuri + self.imageuri = imageuri self.packagetype = packagetype self.handler = handler @@ -688,6 +691,7 @@ def __eq__(self, other: Any) -> bool: return ( self.runtime == other.runtime and self.codeuri == other.codeuri + and self.imageuri == other.imageuri and self.packagetype == other.packagetype and self.metadata == other.metadata and self.env_vars == other.env_vars diff --git a/samcli/lib/build/build_strategy.py b/samcli/lib/build/build_strategy.py index b1317e9311..dbbf03a64b 100644 --- a/samcli/lib/build/build_strategy.py +++ b/samcli/lib/build/build_strategy.py @@ -128,7 +128,9 @@ def __init__( self, build_graph: BuildGraph, build_dir: str, - build_function: Callable[[str, str, str, str, str, Optional[str], str, dict, dict, Optional[str], bool], str], + build_function: Callable[ + [str, str, Optional[str], str, str, str, Optional[str], str, dict, dict, Optional[str], bool], str + ], build_layer: Callable[[str, str, str, List[str], str, str, dict, Optional[str], bool, Optional[Dict]], str], cached: bool = False, ) -> None: @@ -166,6 +168,7 @@ def build_single_function_definition(self, build_definition: FunctionBuildDefini result = self._build_function( build_definition.get_function_name(), build_definition.codeuri, # type: ignore + build_definition.imageuri, build_definition.packagetype, build_definition.runtime, # type: ignore build_definition.architecture, diff --git a/samcli/lib/package/ecr_uploader.py b/samcli/lib/package/ecr_uploader.py index 6414ccf071..22b26f1377 100644 --- a/samcli/lib/package/ecr_uploader.py +++ b/samcli/lib/package/ecr_uploader.py @@ -5,6 +5,7 @@ import base64 import logging from io import StringIO +from pathlib import Path from typing import Dict import botocore @@ -76,10 +77,26 @@ def upload(self, image, resource_name): if not self.login_session_active: self.login() self.login_session_active = True + + # Sometimes the `resource_name` is used as the `image` parameter to `tag_translation`. + # This is because these two cases (directly from an archive or by ID) are effectively + # anonymous, so the best identifier available in scope is the resource name. try: - docker_img = self.docker_client.images.get(image) + if Path(image).is_file(): + with open(image, mode="rb") as image_archive: + [docker_img, *rest] = self.docker_client.images.load(image_archive) + if len(rest) != 0: + raise DockerPushFailedError("Archive must represent a single image") + _tag = tag_translation(resource_name, docker_image_id=docker_img.id, gen_tag=self.tag) + else: + # If it's not a file, it's gotta be a {repo}:{tag} or a sha256:{digest} + docker_img = self.docker_client.images.get(image) + _tag = tag_translation( + resource_name if image == docker_img.id else image, + docker_image_id=docker_img.id, + gen_tag=self.tag, + ) - _tag = tag_translation(image, docker_image_id=docker_img.id, gen_tag=self.tag) repository = ( self.ecr_repo if not self.ecr_repo_multi or not isinstance(self.ecr_repo_multi, dict) diff --git a/samcli/lib/package/utils.py b/samcli/lib/package/utils.py index ef0405060a..c8c37aa7b6 100644 --- a/samcli/lib/package/utils.py +++ b/samcli/lib/package/utils.py @@ -125,6 +125,10 @@ def upload_local_image_artifacts(resource_id, resource_dict, property_name, pare LOG.debug("Property %s of %s is already an ECR URL", property_name, resource_id) return image_path + possible_image_archive_path = make_abs_path(parent_dir, image_path) + if is_local_file(possible_image_archive_path): + image_path = possible_image_archive_path + return uploader.upload(image_path, resource_id) diff --git a/samcli/lib/providers/provider.py b/samcli/lib/providers/provider.py index d14f3fa4b0..e7d5aa1e83 100644 --- a/samcli/lib/providers/provider.py +++ b/samcli/lib/providers/provider.py @@ -9,6 +9,7 @@ import posixpath from collections import namedtuple from enum import Enum +from pathlib import Path from typing import Any, Dict, Iterator, List, NamedTuple, Optional, Set, Union, cast from samcli.commands.local.cli_common.user_exceptions import ( @@ -953,6 +954,7 @@ def get_function_build_info( packagetype: str, inlinecode: Optional[str], codeuri: Optional[str], + imageuri: Optional[str], metadata: Optional[Dict], ) -> FunctionBuildInfo: """ @@ -974,8 +976,9 @@ def get_function_build_info( metadata = metadata or {} dockerfile = cast(str, metadata.get("Dockerfile", "")) docker_context = cast(str, metadata.get("DockerContext", "")) - - if not dockerfile or not docker_context: + buildable = dockerfile and docker_context + loadable = imageuri and Path(imageuri).is_file() + if not buildable and not loadable: LOG.debug( "Skip Building %s function, as it is missing either Dockerfile or DockerContext " "metadata properties.", diff --git a/samcli/lib/providers/sam_function_provider.py b/samcli/lib/providers/sam_function_provider.py index da0c64eaca..830d6c7c28 100644 --- a/samcli/lib/providers/sam_function_provider.py +++ b/samcli/lib/providers/sam_function_provider.py @@ -3,6 +3,7 @@ """ import logging +from pathlib import Path from typing import Any, Dict, Iterator, List, Optional, cast from samtranslator.policy_template_processor.exceptions import TemplateNotFoundException @@ -446,12 +447,18 @@ def _build_function_configuration( LOG.debug("--base-dir is not presented, adjusting uri %s relative to %s", codeuri, stack.location) codeuri = SamLocalStackProvider.normalize_resource_path(stack.location, codeuri) + if imageuri and codeuri != ".": + normalized_image_uri = SamLocalStackProvider.normalize_resource_path(stack.location, imageuri) + if Path(normalized_image_uri).is_file(): + LOG.debug("--base-dir is not presented, adjusting uri %s relative to %s", codeuri, stack.location) + imageuri = normalized_image_uri + package_type = resource_properties.get("PackageType", ZIP) if package_type == ZIP and not resource_properties.get("Handler"): raise MissingFunctionHandlerException(f"Could not find handler for function: {name}") function_build_info = get_function_build_info( - get_full_path(stack.stack_path, function_id), package_type, inlinecode, codeuri, metadata + get_full_path(stack.stack_path, function_id), package_type, inlinecode, codeuri, imageuri, metadata ) return Function( diff --git a/samcli/local/docker/lambda_container.py b/samcli/local/docker/lambda_container.py index 609d5bfbb3..d59b670920 100644 --- a/samcli/local/docker/lambda_container.py +++ b/samcli/local/docker/lambda_container.py @@ -65,7 +65,8 @@ def __init__( runtime str Name of the Lambda runtime imageuri str - Name of the Lambda Image which is of the form {image}:{tag} + Location of the Lambda Image which is of the form {image}:{tag}, sha256:{digest}, + or a path to a local archive handler str Handler of the function to run packagetype str @@ -240,7 +241,8 @@ def _get_image( packagetype : str Package type for the lambda function which is either zip or image. image : str - Name of the Lambda Image which is of the form {image}:{tag} + Location of the Lambda Image which is of the form {image}:{tag}, sha256:{digest}, + or a path to a local archive layers : List[str] List of layers architecture : str diff --git a/samcli/local/lambdafn/config.py b/samcli/local/lambdafn/config.py index cadbfbd9d3..1d940aed2d 100644 --- a/samcli/local/lambdafn/config.py +++ b/samcli/local/lambdafn/config.py @@ -46,7 +46,8 @@ def __init__( handler : str Handler method imageuri : str - Name of the Lambda Image which is of the form {image}:{tag} + Location of the Lambda Image which is of the form {image}:{tag}, sha256:{digest}, + or a path to a local archive imageconfig : str Image configuration which can be used set to entrypoint, command and working dir for the container. packagetype : str diff --git a/tests/integration/buildcmd/build_integ_base.py b/tests/integration/buildcmd/build_integ_base.py index 88aec2d868..184015d67c 100644 --- a/tests/integration/buildcmd/build_integ_base.py +++ b/tests/integration/buildcmd/build_integ_base.py @@ -219,6 +219,7 @@ def _verify_image_build_artifact(self, template_path, image_function_logical_id, def _verify_resource_property(self, template_path, logical_id, property, expected_value): with open(template_path, "r") as fp: template_dict = yaml_parse(fp.read()) + self.assertEqual( expected_value, jmespath.search(f"Resources.{logical_id}.Properties.{property}", template_dict) ) diff --git a/tests/integration/buildcmd/test_build_cmd.py b/tests/integration/buildcmd/test_build_cmd.py index 8d1bbe1b44..4133ccdf17 100644 --- a/tests/integration/buildcmd/test_build_cmd.py +++ b/tests/integration/buildcmd/test_build_cmd.py @@ -86,6 +86,41 @@ def test_with_invalid_dockerfile_definition(self): self.assertIn("COPY requires at least two arguments", command_result.stderr.decode()) +@skipIf(SKIP_DOCKER_TESTS, SKIP_DOCKER_MESSAGE) +class TestLoadingImagesFromArchive(BuildIntegBase): + template = "template_loadable_image.yaml" + + FUNCTION_LOGICAL_ID = "ImageFunction" + + def test_load_not_an_archive_passthrough(self): + overrides = {"ImageUri": "./load_image_archive/this_file_does_not_exist.tar.gz"} + cmdlist = self.get_command_list(parameter_overrides=overrides) + command_result = run_command(cmdlist, cwd=self.working_dir) + + self.assertEqual(command_result.process.returncode, 0) + + def test_bad_image_archive_fails(self): + overrides = {"ImageUri": "./load_image_archive/error.tar.gz"} + cmdlist = self.get_command_list(parameter_overrides=overrides) + command_result = run_command(cmdlist, cwd=self.working_dir) + + self.assertEqual(command_result.process.returncode, 1) + self.assertIn("unexpected EOF", command_result.stderr.decode()) + + def test_load_success(self): + overrides = {"ImageUri": "./load_image_archive/archive.tar.gz"} + cmdlist = self.get_command_list(parameter_overrides=overrides) + command_result = run_command(cmdlist, cwd=self.working_dir) + + self.assertEqual(command_result.process.returncode, 0) + self._verify_image_build_artifact( + self.built_template, + self.FUNCTION_LOGICAL_ID, + "ImageUri", + "sha256:81d2ff8422e3a78dc0c1eff53d8e46f5666a801b17b5607a920860c2d234f9d0", + ) + + @skipIf( # Hits public ECR pull limitation, move it to canary tests (not RUN_BY_CANARY and not CI_OVERRIDE), diff --git a/tests/integration/deploy/test_deploy_command.py b/tests/integration/deploy/test_deploy_command.py index 2bde5bbbf1..180df87df1 100644 --- a/tests/integration/deploy/test_deploy_command.py +++ b/tests/integration/deploy/test_deploy_command.py @@ -134,6 +134,60 @@ def test_no_package_and_deploy_with_s3_bucket_all_args(self, template_file): deploy_process_execute = self.run_command(deploy_command_list) self.assertEqual(deploy_process_execute.process.returncode, 0) + @parameterized.expand(["template-image-load.yaml"]) + def test_deploy_directly_from_image_archive(self, template_file): + template_path = self.test_data_path.joinpath(os.path.join("load-image-archive", template_file)) + + stack_name = self._method_to_stack_name(self.id()) + self.stacks.append({"name": stack_name}) + + # Package and Deploy in one go without confirming change set. + deploy_command_list = self.get_deploy_command_list( + template_file=template_path, + stack_name=stack_name, + capabilities="CAPABILITY_IAM", + s3_prefix=self.s3_prefix, + s3_bucket=self.s3_bucket.name, + image_repository=self.ecr_repo_name, + force_upload=True, + notification_arns=self.sns_arn, + parameter_overrides="Parameter=Clarity", + kms_key_id=self.kms_key, + no_execute_changeset=False, + tags="integ=true clarity=yes foo_bar=baz", + confirm_changeset=False, + ) + + deploy_process_execute = self.run_command(deploy_command_list) + self.assertEqual(deploy_process_execute.process.returncode, 0) + + @parameterized.expand(["template-image-load-fail.yaml"]) + def test_deploy_directly_from_image_archive_but_error_fail(self, template_file): + template_path = self.test_data_path.joinpath(os.path.join("load-image-archive", template_file)) + + stack_name = self._method_to_stack_name(self.id()) + self.stacks.append({"name": stack_name}) + + # Package and Deploy in one go without confirming change set. + deploy_command_list = self.get_deploy_command_list( + template_file=template_path, + stack_name=stack_name, + capabilities="CAPABILITY_IAM", + s3_prefix=self.s3_prefix, + s3_bucket=self.s3_bucket.name, + image_repository=self.ecr_repo_name, + force_upload=True, + notification_arns=self.sns_arn, + parameter_overrides="Parameter=Clarity", + kms_key_id=self.kms_key, + no_execute_changeset=False, + tags="integ=true clarity=yes foo_bar=baz", + confirm_changeset=False, + ) + + deploy_process_execute = self.run_command(deploy_command_list) + self.assertEqual(deploy_process_execute.process.returncode, 1) + @parameterized.expand( [ "aws-serverless-function-image.yaml", diff --git a/tests/integration/package/test_package_command_image.py b/tests/integration/package/test_package_command_image.py index 8c5b36649d..5e8405d922 100644 --- a/tests/integration/package/test_package_command_image.py +++ b/tests/integration/package/test_package_command_image.py @@ -139,8 +139,8 @@ def test_package_template_with_image_repositories_nested_stack(self, resource_id except TimeoutExpired: process.kill() raise - process_stderr = stderr.strip() + self.assertIn(f"{self.ecr_repo_name}", process_stderr.decode("utf-8")) self.assertEqual(0, process.returncode) @@ -271,3 +271,35 @@ def test_package_with_deep_nested_template_image(self): # check string like this: # ...python-ce689abb4f0d-3.9-slim: digest:... self.assertRegex(process_stderr, rf"{image}-.+-{tag}: digest:") + + @parameterized.expand(["template-image-load.yaml"]) + def test_package_with_loadable_image_archive(self, template_file): + template_path = self.test_data_path.joinpath(os.path.join("load-image-archive", template_file)) + command_list = PackageIntegBase.get_command_list(image_repository=self.ecr_repo_name, template=template_path) + + process = Popen(command_list, stderr=PIPE) + try: + _, stderr = process.communicate(timeout=TIMEOUT) + except TimeoutExpired: + process.kill() + raise + process_stderr = stderr.strip() + + self.assertEqual(0, process.returncode) + self.assertIn(f"{self.ecr_repo_name}", process_stderr.decode("utf-8")) + + @parameterized.expand(["template-image-load-fail.yaml"]) + def test_package_with_nonloadable_image_archive(self, template_file): + template_path = self.test_data_path.joinpath(os.path.join("load-image-archive", template_file)) + command_list = PackageIntegBase.get_command_list(image_repository=self.ecr_repo_name, template=template_path) + + process = Popen(command_list, stderr=PIPE) + try: + _, stderr = process.communicate(timeout=TIMEOUT) + except TimeoutExpired: + process.kill() + raise + process_stderr = stderr.strip() + + self.assertEqual(1, process.returncode) + self.assertIn("unexpected EOF", process_stderr.decode("utf-8")) diff --git a/tests/integration/testdata/buildcmd/load_image_archive/archive.tar.gz b/tests/integration/testdata/buildcmd/load_image_archive/archive.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..a1e1571714f4b843d9400d755c121b4d1f85dacc GIT binary patch literal 1158 zcmV;11bO=(iwFQNIv{2M1MOK`Z`(E$&hvhS!9C8Ad8cGx4_!K-0}7;Avkt=$1Qag` zS6vKA4w?r3?>mx{ICf(=7u#J|eGf7z-g)>Pen(ETGOb4kI|T+PA-Z3{>Tj1OToR0# zz?>Xlf*Df>1F_p_%`vSTUBOgd+Ry6Zz7eFI{p$-&h1}IKc#{2v1p5;P)qDHz0ZaB5 zlAFXQil)SJYzQTaNM#c&nbX=bsi~(Ldy1)#V<@2{HnHSHV5+ENYsP5zLvp^({_G3> zQ^FNL80=*ezjOP$e9~O)c#IA9U*zv*u|F6jE{ZBFJ{r_HYjdW!$LgP#` ztsELdHkTX?ktquw&e8Z9{aRe2@!#m(tCKfBkEV4s%1WcNQJscjd@p~OT@-YQc@`e% zbcp^etB;{Le;q0qY6Qa^7r#tHX3@BqX4wpet(XKDx?>%te1_5~`!Dg|Nrk=ll-WcpL!?N((ZF_M7^OyYAvpJnV!;SYB|KchgfmKW%Pc2! z%@~{bFO^)iNG9T%+1txLkjL(;3PFjp-2O&IWxlKeu`^$%4@hl#MP68#m>fDCZw&UEMGIm2L2OH$ZGsASkM2x;2MSY z!c@`|hY3j~ch<%v#tM#Inlj^zj0uCwGa)p!B25#5MZ&@UG)a|Kkg=4ar{CN_{ziww zRc8yg@usYsLgy{VpvLPmhYE}QAGNFIsaH*-&Bgo1f~pvMoz?E%yK>rm`O;O@-OH2a z>Mngeam8{!UGBIr?(R(|fLeVrL`S*3P5#yremU~de+0}+ zd;aeOK-Hrgxq-I&EfARWyPK3J`n+CT$~Vyz+UmcKt&HfpeqT;2_XCLk+VlIiI|6qiV1P)3 YBmCW};vPME^!WedUql36{{Sig07CmkD*ylh literal 0 HcmV?d00001 diff --git a/tests/integration/testdata/buildcmd/load_image_archive/error.tar.gz b/tests/integration/testdata/buildcmd/load_image_archive/error.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..4e4460da5e6c4b6705068ae9eee92f161170ca18 GIT binary patch literal 54 zcmb2|=3qz_lSpG={v6_QCiKK<{Xma1!6(l6`uK->=y;twq2qluP~)7w)~b*vObiwR KvI~?L7#IK`yAt&P literal 0 HcmV?d00001 diff --git a/tests/integration/testdata/buildcmd/template_loadable_image.yaml b/tests/integration/testdata/buildcmd/template_loadable_image.yaml new file mode 100644 index 0000000000..6fe74e7de6 --- /dev/null +++ b/tests/integration/testdata/buildcmd/template_loadable_image.yaml @@ -0,0 +1,15 @@ +AWSTemplateFormatVersion : '2010-09-09' +Transform: AWS::Serverless-2016-10-31 + +Parameters: + ImageUri: + Type: String + +Resources: + + ImageFunction: + Type: AWS::Serverless::Function + Properties: + PackageType: Image + ImageUri: !Ref ImageUri + Timeout: 600 diff --git a/tests/integration/testdata/package/load-image-archive/archive.tar.gz b/tests/integration/testdata/package/load-image-archive/archive.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..a1e1571714f4b843d9400d755c121b4d1f85dacc GIT binary patch literal 1158 zcmV;11bO=(iwFQNIv{2M1MOK`Z`(E$&hvhS!9C8Ad8cGx4_!K-0}7;Avkt=$1Qag` zS6vKA4w?r3?>mx{ICf(=7u#J|eGf7z-g)>Pen(ETGOb4kI|T+PA-Z3{>Tj1OToR0# zz?>Xlf*Df>1F_p_%`vSTUBOgd+Ry6Zz7eFI{p$-&h1}IKc#{2v1p5;P)qDHz0ZaB5 zlAFXQil)SJYzQTaNM#c&nbX=bsi~(Ldy1)#V<@2{HnHSHV5+ENYsP5zLvp^({_G3> zQ^FNL80=*ezjOP$e9~O)c#IA9U*zv*u|F6jE{ZBFJ{r_HYjdW!$LgP#` ztsELdHkTX?ktquw&e8Z9{aRe2@!#m(tCKfBkEV4s%1WcNQJscjd@p~OT@-YQc@`e% zbcp^etB;{Le;q0qY6Qa^7r#tHX3@BqX4wpet(XKDx?>%te1_5~`!Dg|Nrk=ll-WcpL!?N((ZF_M7^OyYAvpJnV!;SYB|KchgfmKW%Pc2! z%@~{bFO^)iNG9T%+1txLkjL(;3PFjp-2O&IWxlKeu`^$%4@hl#MP68#m>fDCZw&UEMGIm2L2OH$ZGsASkM2x;2MSY z!c@`|hY3j~ch<%v#tM#Inlj^zj0uCwGa)p!B25#5MZ&@UG)a|Kkg=4ar{CN_{ziww zRc8yg@usYsLgy{VpvLPmhYE}QAGNFIsaH*-&Bgo1f~pvMoz?E%yK>rm`O;O@-OH2a z>Mngeam8{!UGBIr?(R(|fLeVrL`S*3P5#yremU~de+0}+ zd;aeOK-Hrgxq-I&EfARWyPK3J`n+CT$~Vyz+UmcKt&HfpeqT;2_XCLk+VlIiI|6qiV1P)3 YBmCW};vPME^!WedUql36{{Sig07CmkD*ylh literal 0 HcmV?d00001 diff --git a/tests/integration/testdata/package/load-image-archive/error.tar.gz b/tests/integration/testdata/package/load-image-archive/error.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..4e4460da5e6c4b6705068ae9eee92f161170ca18 GIT binary patch literal 54 zcmb2|=3qz_lSpG={v6_QCiKK<{Xma1!6(l6`uK->=y;twq2qluP~)7w)~b*vObiwR KvI~?L7#IK`yAt&P literal 0 HcmV?d00001 diff --git a/tests/integration/testdata/package/load-image-archive/template-image-load-fail.yaml b/tests/integration/testdata/package/load-image-archive/template-image-load-fail.yaml new file mode 100644 index 0000000000..1774b16832 --- /dev/null +++ b/tests/integration/testdata/package/load-image-archive/template-image-load-fail.yaml @@ -0,0 +1,22 @@ +AWSTemplateFormatVersion : '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: A hello world image application. + +Parameters: + Parameter: + Type: String + Default: Sample + Description: A custom parameter + +Resources: + Hello: + Type: AWS::Serverless::Function + Properties: + PackageType: Image + ImageUri: ./error.tar.gz + Events: + HelloWorld: + Type: Api + Properties: + Path: /hello + Method: get diff --git a/tests/integration/testdata/package/load-image-archive/template-image-load.yaml b/tests/integration/testdata/package/load-image-archive/template-image-load.yaml new file mode 100644 index 0000000000..940c2536fd --- /dev/null +++ b/tests/integration/testdata/package/load-image-archive/template-image-load.yaml @@ -0,0 +1,22 @@ +AWSTemplateFormatVersion : '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: A hello world image application. + +Parameters: + Parameter: + Type: String + Default: Sample + Description: A custom parameter + +Resources: + Hello: + Type: AWS::Serverless::Function + Properties: + PackageType: Image + ImageUri: ./archive.tar.gz + Events: + HelloWorld: + Type: Api + Properties: + Path: /hello + Method: get diff --git a/tests/unit/commands/buildcmd/test_build_context.py b/tests/unit/commands/buildcmd/test_build_context.py index 5d48139d4b..a69b83bdac 100644 --- a/tests/unit/commands/buildcmd/test_build_context.py +++ b/tests/unit/commands/buildcmd/test_build_context.py @@ -76,7 +76,9 @@ def get_function( function_url_config=None, stack_path="", runtime_management_config=None, - function_build_info=get_function_build_info("stack/function", packagetype, inlinecode, codeuri, metadata), + function_build_info=get_function_build_info( + "stack/function", packagetype, inlinecode, codeuri, imageuri, metadata + ), ) diff --git a/tests/unit/lib/build_module/test_app_builder.py b/tests/unit/lib/build_module/test_app_builder.py index f621effe40..0e59234e37 100644 --- a/tests/unit/lib/build_module/test_app_builder.py +++ b/tests/unit/lib/build_module/test_app_builder.py @@ -5,8 +5,10 @@ import docker import json +from uuid import uuid4 + from unittest import TestCase -from unittest.mock import Mock, MagicMock, call, patch, ANY +from unittest.mock import Mock, MagicMock, call, mock_open, patch, ANY from pathlib import Path, WindowsPath from parameterized import parameterized @@ -56,11 +58,12 @@ def setUp(self): self.imageFunc1.get_build_dir = Mock() self.imageFunc1.inlinecode = None self.imageFunc1.architectures = [X86_64] + self.imageFunc1.packagetype = IMAGE + self.imageFunc1.imageuri = "imageuri" self.layer1 = Mock() self.layer2 = Mock() - self.imageFunc1.packagetype = IMAGE self.layer1.build_method = "build_method" self.layer1.name = "layer_name1" self.layer1.full_path = os.path.join("StackJ", "layer_name1") @@ -131,6 +134,7 @@ def build_layer_return( call( self.func1.name, self.func1.codeuri, + ANY, ZIP, self.func1.runtime, self.func1.architecture, @@ -144,6 +148,7 @@ def build_layer_return( call( self.func2.name, self.func2.codeuri, + ANY, ZIP, self.func2.runtime, self.func2.architecture, @@ -157,6 +162,7 @@ def build_layer_return( call( self.imageFunc1.name, self.imageFunc1.codeuri, + self.imageFunc1.imageuri, IMAGE, self.imageFunc1.runtime, self.imageFunc1.architecture, @@ -203,7 +209,7 @@ def build_layer_return( @patch("samcli.lib.build.build_graph.BuildGraph._write") def test_should_use_function_or_layer_get_build_dir_to_determine_artifact_dir(self, persist_mock): def get_func_call_with_artifact_dir(artifact_dir): - return call(ANY, ANY, ANY, ANY, ANY, ANY, artifact_dir, ANY, ANY, ANY, True) + return call(ANY, ANY, ANY, ANY, ANY, ANY, ANY, artifact_dir, ANY, ANY, ANY, True) def get_layer_call_with_artifact_dir(artifact_dir): return call(ANY, ANY, ANY, ANY, ANY, artifact_dir, ANY, ANY, True, ANY) @@ -296,6 +302,7 @@ def test_should_run_build_for_only_unique_builds(self, persist_mock, read_mock, call( function1_1.name, function1_1.codeuri, + ANY, ZIP, function1_1.runtime, function1_1.architectures[0], @@ -309,6 +316,7 @@ def test_should_run_build_for_only_unique_builds(self, persist_mock, read_mock, call( function2.name, function2.codeuri, + ANY, ZIP, function2.runtime, function1_1.architectures[0], @@ -480,6 +488,7 @@ def test_deprecated_runtimes(self, runtime): self.builder._build_function( function_name="function_name", codeuri="code_uri", + imageuri=None, packagetype=ZIP, runtime=runtime, architecture="architecture", @@ -527,7 +536,18 @@ def test_must_not_use_dep_layer_for_non_cached(self): builder.build() builder._build_function.assert_called_with( - "name", "codeuri", ZIP, "runtime", X86_64, "handler", str(Path("builddir/name")), {}, {}, None, True + "name", + "codeuri", + "imageuri", + ZIP, + "runtime", + X86_64, + "handler", + str(Path("builddir/name")), + {}, + {}, + None, + True, ) @@ -1716,10 +1736,61 @@ def test_can_raise_build_error(self): self.builder._build_lambda_image("Name", {}, X86_64) +class TestApplicationBuilder_load_lambda_image_function(TestCase): + def setUp(self): + self.docker_client_mock = Mock() + self.builder = ApplicationBuilder( + Mock(), + "/build/dir", + "/base/dir", + "/cached/dir", + stream_writer=Mock(), + docker_client=self.docker_client_mock, + ) + + @patch("builtins.open", new_callable=mock_open) + def test_loads_image_archive(self, mock_open): + id = f"sha256:{uuid4().hex}" + + self.docker_client_mock.images.load.return_value = [Mock(id=id)] + + image = self.builder._load_lambda_image("./path/to/archive.tar.gz") + self.assertEqual(id, image) + + @patch("builtins.open", new_callable=mock_open) + def test_archive_must_represent_a_single_image(self, mock_open): + self.docker_client_mock.images.load.return_value = [ + Mock(id=f"sha256:{uuid4().hex}"), + Mock(id=f"sha256:{uuid4().hex}"), + ] + + with self.assertRaises(DockerBuildFailed) as ex: + self.builder._load_lambda_image("./path/to/archive.tar.gz") + self.assertIn("single", str(ex.exception)) + + @patch("builtins.open", side_effect=OSError) + def test_image_archive_does_not_exist(self, mock_open): + with self.assertRaises(DockerBuildFailed): + self.builder._load_lambda_image("./path/to/nowhere.tar.gz") + + @patch("builtins.open", new_callable=mock_open) + def test_docker_api_error(self, mock_open): + self.docker_client_mock.images.load.side_effect = docker.errors.APIError("failed to dial") + + with self.assertRaises(DockerBuildFailed): + self.builder._load_lambda_image("./path/to/archive.tar.gz") + + class TestApplicationBuilder_build_function(TestCase): def setUp(self): + self.docker_client_mock = Mock() self.builder = ApplicationBuilder( - Mock(), "/build/dir", "/base/dir", "cachedir", stream_writer=StreamWriter(sys.stderr) + Mock(), + "/build/dir", + "/base/dir", + "cachedir", + stream_writer=StreamWriter(sys.stderr), + docker_client=self.docker_client_mock, ) @patch("samcli.lib.build.app_builder.get_workflow_config") @@ -1744,7 +1815,7 @@ def test_must_build_in_process(self, osutils_mock, get_workflow_config_mock): artifacts_dir = str(Path("/build/dir/function_full_path")) manifest_path = str(Path(os.path.join(code_dir, config_mock.manifest_name)).resolve()) - self.builder._build_function(function_name, codeuri, ZIP, runtime, architecture, handler, artifacts_dir) + self.builder._build_function(function_name, codeuri, None, ZIP, runtime, architecture, handler, artifacts_dir) self.builder._build_function_in_process.assert_called_with( config_mock, @@ -1807,7 +1878,9 @@ def test_must_custom_build_function_with_working_dir_metadata_in_process( get_build_options = ApplicationBuilder._get_build_options ApplicationBuilder._get_build_options = get_build_options_mock builder._build_function_in_process = build_function_in_process_mock - builder._build_function(function_name, codeuri, ZIP, runtime, architecture, handler, artifacts_dir, metadata) + builder._build_function( + function_name, codeuri, None, ZIP, runtime, architecture, handler, artifacts_dir, metadata + ) ApplicationBuilder._get_build_options = get_build_options @@ -1883,7 +1956,9 @@ def test_must_custom_build_function_with_custom_makefile_and_custom_project_root get_build_options = ApplicationBuilder._get_build_options ApplicationBuilder._get_build_options = get_build_options_mock builder._build_function_in_process = build_function_in_process_mock - builder._build_function(function_name, codeuri, ZIP, runtime, architecture, handler, artifacts_dir, metadata) + builder._build_function( + function_name, codeuri, None, ZIP, runtime, architecture, handler, artifacts_dir, metadata + ) ApplicationBuilder._get_build_options = get_build_options @@ -1961,7 +2036,9 @@ def test_must_custom_build_function_with_all_metadata_sutom_paths_properties_in_ get_build_options = ApplicationBuilder._get_build_options ApplicationBuilder._get_build_options = get_build_options_mock builder._build_function_in_process = build_function_in_process_mock - builder._build_function(function_name, codeuri, ZIP, runtime, architecture, handler, artifacts_dir, metadata) + builder._build_function( + function_name, codeuri, None, ZIP, runtime, architecture, handler, artifacts_dir, metadata + ) ApplicationBuilder._get_build_options = get_build_options @@ -2037,7 +2114,9 @@ def test_must_custom_build_function_with_only_context_path_metadata_in_process( get_build_options = ApplicationBuilder._get_build_options ApplicationBuilder._get_build_options = get_build_options_mock builder._build_function_in_process = build_function_in_process_mock - builder._build_function(function_name, codeuri, ZIP, runtime, architecture, handler, artifacts_dir, metadata) + builder._build_function( + function_name, codeuri, None, ZIP, runtime, architecture, handler, artifacts_dir, metadata + ) ApplicationBuilder._get_build_options = get_build_options @@ -2112,7 +2191,9 @@ def test_must_custom_build_function_with_only_project_root_dir_metadata_in_proce get_build_options = ApplicationBuilder._get_build_options ApplicationBuilder._get_build_options = get_build_options_mock builder._build_function_in_process = build_function_in_process_mock - builder._build_function(function_name, codeuri, ZIP, runtime, architecture, handler, artifacts_dir, metadata) + builder._build_function( + function_name, codeuri, None, ZIP, runtime, architecture, handler, artifacts_dir, metadata + ) ApplicationBuilder._get_build_options = get_build_options @@ -2183,7 +2264,9 @@ def test_must_custom_build_function_with_empty_metadata_in_process(self, osutils get_build_options = ApplicationBuilder._get_build_options ApplicationBuilder._get_build_options = get_build_options_mock builder._build_function_in_process = build_function_in_process_mock - builder._build_function(function_name, codeuri, ZIP, runtime, architecture, handler, artifacts_dir, metadata) + builder._build_function( + function_name, codeuri, None, ZIP, runtime, architecture, handler, artifacts_dir, metadata + ) ApplicationBuilder._get_build_options = get_build_options @@ -2237,6 +2320,7 @@ def test_must_build_in_process_with_metadata(self, osutils_mock, get_workflow_co self.builder._build_function( function_name, codeuri, + None, packagetype, runtime, architecture, @@ -2294,6 +2378,7 @@ def test_must_build_in_process_with_metadata_and_metadata_as_options( self.builder._build_function( function_name, codeuri, + None, packagetype, runtime, architecture, @@ -2344,7 +2429,9 @@ def test_must_build_in_container(self, osutils_mock, get_workflow_config_mock): # Settting the container manager will make us use the container self.builder._container_manager = Mock() - self.builder._build_function(function_name, codeuri, packagetype, runtime, architecture, handler, artifacts_dir) + self.builder._build_function( + function_name, codeuri, None, packagetype, runtime, architecture, handler, artifacts_dir + ) self.builder._build_function_on_container.assert_called_with( config_mock, @@ -2387,6 +2474,7 @@ def test_must_build_in_container_with_env_vars(self, osutils_mock, get_workflow_ self.builder._build_function( function_name, codeuri, + None, packagetype, runtime, architecture, @@ -2436,7 +2524,15 @@ def test_must_build_in_container_with_custom_specified_build_image(self, osutils self.builder._container_manager = Mock() self.builder._build_images = build_images self.builder._build_function( - function_name, codeuri, packagetype, runtime, architecture, handler, artifacts_dir, container_env_vars=None + function_name, + codeuri, + None, + packagetype, + runtime, + architecture, + handler, + artifacts_dir, + container_env_vars=None, ) self.builder._build_function_on_container.assert_called_with( @@ -2480,7 +2576,15 @@ def test_must_build_in_container_with_custom_default_build_image(self, osutils_m self.builder._container_manager = Mock() self.builder._build_images = build_images self.builder._build_function( - function_name, codeuri, packagetype, runtime, architecture, handler, artifacts_dir, container_env_vars=None + function_name, + codeuri, + None, + packagetype, + runtime, + architecture, + handler, + artifacts_dir, + container_env_vars=None, ) self.builder._build_function_on_container.assert_called_with( @@ -2496,6 +2600,19 @@ def test_must_build_in_container_with_custom_default_build_image(self, osutils_m specified_workflow=None, ) + @parameterized.expand([X86_64, ARM64]) + @patch.object(Path, "is_file", return_value=True) + @patch("builtins.open", new_callable=mock_open) + def test_loads_if_path_exists(self, mock_open, mock_is_file, architecture): + id = f"sha256:{uuid4().hex}" + function_name = "function_name" + imageuri = str(Path("./path/to/archive.tar.gz")) + + self.docker_client_mock.images.load.return_value = [Mock(id=id)] + + image = self.builder._build_function(function_name, None, imageuri, IMAGE, None, architecture, None, None) + self.assertEqual(id, image) + class TestApplicationBuilder_build_function_in_process(TestCase): def setUp(self): diff --git a/tests/unit/lib/build_module/test_build_graph.py b/tests/unit/lib/build_module/test_build_graph.py index 06f01fe35e..898e7a965d 100644 --- a/tests/unit/lib/build_module/test_build_graph.py +++ b/tests/unit/lib/build_module/test_build_graph.py @@ -111,6 +111,7 @@ def test_function_build_definition_to_toml_table(self): build_definition = FunctionBuildDefinition( "runtime", "codeuri", + None, ZIP, X86_64, {"key": "value"}, @@ -209,7 +210,7 @@ def test_toml_table_to_layer_build_definition(self): self.assertEqual(build_definition.architecture, toml_table[ARCHITECTURE_FIELD]) def test_minimal_function_build_definition_to_toml_table(self): - build_definition = FunctionBuildDefinition("runtime", "codeuri", ZIP, X86_64, {"key": "value"}, "handler") + build_definition = FunctionBuildDefinition("runtime", "codeuri", None, ZIP, X86_64, {"key": "value"}, "handler") build_definition.add_function(generate_function()) toml_table = _function_build_definition_to_toml_table(build_definition) @@ -354,6 +355,7 @@ def test_should_instantiate_first_time_and_update(self): function_build_definition1 = FunctionBuildDefinition( TestBuildGraph.RUNTIME, TestBuildGraph.CODEURI, + None, TestBuildGraph.ZIP, TestBuildGraph.ARCHITECTURE_FIELD, TestBuildGraph.METADATA, @@ -442,6 +444,7 @@ def test_functions_should_be_added_existing_build_graph(self): build_definition1 = FunctionBuildDefinition( TestBuildGraph.RUNTIME, TestBuildGraph.CODEURI, + None, TestBuildGraph.ZIP, TestBuildGraph.ARCHITECTURE_FIELD, TestBuildGraph.METADATA, @@ -466,6 +469,7 @@ def test_functions_should_be_added_existing_build_graph(self): build_definition2 = FunctionBuildDefinition( "another_runtime", "another_codeuri", + None, TestBuildGraph.ZIP, ARM64, None, @@ -565,6 +569,7 @@ def test_compare_hash_changes_should_succeed(self): build_definition = FunctionBuildDefinition( TestBuildGraph.RUNTIME, TestBuildGraph.CODEURI, + None, TestBuildGraph.ZIP, TestBuildGraph.ARCHITECTURE_FIELD, TestBuildGraph.METADATA, @@ -576,6 +581,7 @@ def test_compare_hash_changes_should_succeed(self): updated_definition = FunctionBuildDefinition( TestBuildGraph.RUNTIME, TestBuildGraph.CODEURI, + None, TestBuildGraph.ZIP, TestBuildGraph.ARCHITECTURE_FIELD, TestBuildGraph.METADATA, @@ -628,10 +634,10 @@ def test_compare_hash_changes_should_preserve_download_dependencies( self, old_manifest, new_manifest, download_dependencies ): updated_definition = FunctionBuildDefinition( - "runtime", "codeuri", ZIP, X86_64, {}, "app.handler", manifest_hash=old_manifest + "runtime", "codeuri", None, ZIP, X86_64, {}, "app.handler", manifest_hash=old_manifest ) existing_definition = FunctionBuildDefinition( - "runtime", "codeuri", ZIP, X86_64, {}, "app.handler", manifest_hash=new_manifest + "runtime", "codeuri", None, ZIP, X86_64, {}, "app.handler", manifest_hash=new_manifest ) BuildGraph._compare_hash_changes([updated_definition], [existing_definition]) self.assertEqual(existing_definition.download_dependencies, download_dependencies) @@ -688,7 +694,16 @@ def test_get_function_build_definition_with_logical_id(self): class TestBuildDefinition(TestCase): def test_single_function_should_return_function_and_handler_name(self): build_definition = FunctionBuildDefinition( - "runtime", "codeuri", ZIP, X86_64, {}, "handler", "source_hash", "manifest_hash", {"env_vars": "value"} + "runtime", + "codeuri", + None, + ZIP, + X86_64, + {}, + "handler", + "source_hash", + "manifest_hash", + {"env_vars": "value"}, ) build_definition.add_function(generate_function()) self.assertEqual(build_definition.get_handler_name(), "handler") @@ -696,7 +711,16 @@ def test_single_function_should_return_function_and_handler_name(self): def test_no_function_should_raise_exception(self): build_definition = FunctionBuildDefinition( - "runtime", "codeuri", ZIP, X86_64, {}, "handler", "source_hash", "manifest_hash", {"env_vars": "value"} + "runtime", + "codeuri", + None, + ZIP, + X86_64, + {}, + "handler", + "source_hash", + "manifest_hash", + {"env_vars": "value"}, ) self.assertRaises(InvalidBuildGraphException, build_definition.get_handler_name) @@ -704,10 +728,10 @@ def test_no_function_should_raise_exception(self): def test_same_runtime_codeuri_metadata_should_reflect_as_same_object(self): build_definition1 = FunctionBuildDefinition( - "runtime", "codeuri", ZIP, ARM64, {"key": "value"}, "handler", "source_hash", "manifest_hash" + "runtime", "codeuri", None, ZIP, ARM64, {"key": "value"}, "handler", "source_hash", "manifest_hash" ) build_definition2 = FunctionBuildDefinition( - "runtime", "codeuri", ZIP, ARM64, {"key": "value"}, "handler", "source_hash", "manifest_hash" + "runtime", "codeuri", None, ZIP, ARM64, {"key": "value"}, "handler", "source_hash", "manifest_hash" ) self.assertEqual(build_definition1, build_definition2) @@ -716,6 +740,7 @@ def test_skip_sam_related_metadata_should_reflect_as_same_object(self): build_definition1 = FunctionBuildDefinition( "runtime", "codeuri", + None, ZIP, ARM64, {"key": "value", "SamResourceId": "resourceId1", "SamNormalized": True}, @@ -726,6 +751,7 @@ def test_skip_sam_related_metadata_should_reflect_as_same_object(self): build_definition2 = FunctionBuildDefinition( "runtime", "codeuri", + None, ZIP, ARM64, {"key": "value", "SamResourceId": "resourceId2", "SamNormalized": True}, @@ -740,6 +766,7 @@ def test_same_env_vars_reflect_as_same_object(self): build_definition1 = FunctionBuildDefinition( "runtime", "codeuri", + None, ZIP, X86_64, {"key": "value"}, @@ -751,6 +778,7 @@ def test_same_env_vars_reflect_as_same_object(self): build_definition2 = FunctionBuildDefinition( "runtime", "codeuri", + None, ZIP, X86_64, {"key": "value"}, @@ -810,17 +838,17 @@ def test_same_env_vars_reflect_as_same_object(self): def test_different_runtime_codeuri_metadata_should_not_reflect_as_same_object( self, runtime1, codeuri1, metadata1, source_hash_1, runtime2, codeuri2, metadata2, source_hash_2 ): - build_definition1 = FunctionBuildDefinition(runtime1, codeuri1, ZIP, ARM64, metadata1, source_hash_1) - build_definition2 = FunctionBuildDefinition(runtime2, codeuri2, ZIP, ARM64, metadata2, source_hash_2) + build_definition1 = FunctionBuildDefinition(runtime1, codeuri1, None, ZIP, ARM64, metadata1, source_hash_1) + build_definition2 = FunctionBuildDefinition(runtime2, codeuri2, None, ZIP, ARM64, metadata2, source_hash_2) self.assertNotEqual(build_definition1, build_definition2) def test_different_architecture_should_not_reflect_as_same_object(self): build_definition1 = FunctionBuildDefinition( - "runtime", "codeuri", ZIP, X86_64, {"key": "value"}, "handler", "source_md5", {"env_vars": "value"} + "runtime", "codeuri", None, ZIP, X86_64, {"key": "value"}, "handler", "source_md5", {"env_vars": "value"} ) build_definition2 = FunctionBuildDefinition( - "runtime", "codeuri", ZIP, ARM64, {"key": "value"}, "handler", "source_md5", {"env_vars": "value"} + "runtime", "codeuri", None, ZIP, ARM64, {"key": "value"}, "handler", "source_md5", {"env_vars": "value"} ) self.assertNotEqual(build_definition1, build_definition2) @@ -829,6 +857,7 @@ def test_different_env_vars_should_not_reflect_as_same_object(self): build_definition1 = FunctionBuildDefinition( "runtime", "codeuri", + None, ZIP, ARM64, {"key": "value"}, @@ -840,6 +869,7 @@ def test_different_env_vars_should_not_reflect_as_same_object(self): build_definition2 = FunctionBuildDefinition( "runtime", "codeuri", + None, ZIP, ARM64, {"key": "value"}, @@ -853,13 +883,13 @@ def test_different_env_vars_should_not_reflect_as_same_object(self): def test_euqality_with_another_object(self): build_definition = FunctionBuildDefinition( - "runtime", "codeuri", ZIP, X86_64, None, "source_hash", "manifest_hash" + "runtime", "codeuri", None, ZIP, X86_64, None, "source_hash", "manifest_hash" ) self.assertNotEqual(build_definition, {}) def test_str_representation(self): build_definition = FunctionBuildDefinition( - "runtime", "codeuri", ZIP, ARM64, None, "handler", "source_hash", "manifest_hash" + "runtime", "codeuri", None, ZIP, ARM64, None, "handler", "source_hash", "manifest_hash" ) self.assertEqual( str(build_definition), @@ -870,13 +900,13 @@ def test_esbuild_definitions_equal_objects_independent_build_method(self): build_graph = BuildGraph("build/path") metadata = {"BuildMethod": "esbuild"} build_definition1 = FunctionBuildDefinition( - "runtime", "codeuri", ZIP, ARM64, metadata, "handler", "source_hash", "manifest_hash" + "runtime", "codeuri", None, ZIP, ARM64, metadata, "handler", "source_hash", "manifest_hash" ) function1 = generate_function( runtime=TestBuildGraph.RUNTIME, codeuri=TestBuildGraph.CODEURI, metadata=metadata, handler="handler-1" ) build_definition2 = FunctionBuildDefinition( - "runtime", "codeuri", ZIP, ARM64, metadata, "app.handler", "source_hash", "manifest_hash" + "runtime", "codeuri", None, ZIP, ARM64, metadata, "app.handler", "source_hash", "manifest_hash" ) function2 = generate_function( runtime=TestBuildGraph.RUNTIME, codeuri=TestBuildGraph.CODEURI, metadata=metadata, handler="handler-2" @@ -895,13 +925,13 @@ def test_independent_build_definitions_equal_objects_one_esbuild_build_method(se build_graph = BuildGraph("build/path") metadata = {"BuildMethod": "esbuild"} build_definition1 = FunctionBuildDefinition( - "runtime", "codeuri", ZIP, ARM64, metadata, "handler", "source_hash", "manifest_hash" + "runtime", "codeuri", None, ZIP, ARM64, metadata, "handler", "source_hash", "manifest_hash" ) function1 = generate_function( runtime=TestBuildGraph.RUNTIME, codeuri=TestBuildGraph.CODEURI, metadata=metadata, handler="handler-1" ) build_definition2 = FunctionBuildDefinition( - "runtime", "codeuri", ZIP, ARM64, {}, "handler", "source_hash", "manifest_hash" + "runtime", "codeuri", None, ZIP, ARM64, {}, "handler", "source_hash", "manifest_hash" ) function2 = generate_function( runtime=TestBuildGraph.RUNTIME, codeuri=TestBuildGraph.CODEURI, metadata={}, handler="handler-2" @@ -920,13 +950,13 @@ def test_two_esbuild_methods_same_handler(self): build_graph = BuildGraph("build/path") metadata = {"BuildMethod": "esbuild"} build_definition1 = FunctionBuildDefinition( - "runtime", "codeuri", ZIP, ARM64, metadata, "handler", "source_hash", "manifest_hash" + "runtime", "codeuri", None, ZIP, ARM64, metadata, "handler", "source_hash", "manifest_hash" ) function1 = generate_function( runtime=TestBuildGraph.RUNTIME, codeuri=TestBuildGraph.CODEURI, metadata=metadata, handler="handler" ) build_definition2 = FunctionBuildDefinition( - "runtime", "codeuri", ZIP, ARM64, metadata, "handler", "source_hash", "manifest_hash" + "runtime", "codeuri", None, ZIP, ARM64, metadata, "handler", "source_hash", "manifest_hash" ) function2 = generate_function( runtime=TestBuildGraph.RUNTIME, codeuri=TestBuildGraph.CODEURI, metadata={}, handler="handler" @@ -946,7 +976,7 @@ def test_build_folder_with_multiple_functions(self, build_improvements_22_enable patched_is_experimental.return_value = build_improvements_22_enabled build_graph = BuildGraph("build/path") build_definition = FunctionBuildDefinition( - "runtime", "codeuri", ZIP, ARM64, {}, "handler", "source_hash", "manifest_hash" + "runtime", "codeuri", None, ZIP, ARM64, {}, "handler", "source_hash", "manifest_hash" ) function1 = generate_function(runtime=TestBuildGraph.RUNTIME, codeuri=TestBuildGraph.CODEURI, handler="handler") function2 = generate_function(runtime=TestBuildGraph.RUNTIME, codeuri=TestBuildGraph.CODEURI, handler="handler") @@ -965,7 +995,7 @@ def test_build_folder_with_multiple_functions(self, build_improvements_22_enable def test_deepcopy_build_definition(self): build_definition = FunctionBuildDefinition( - "runtime", "codeuri", ZIP, ARM64, {}, "handler", "source_hash", "manifest_hash" + "runtime", "codeuri", None, ZIP, ARM64, {}, "handler", "source_hash", "manifest_hash" ) function1 = generate_function(runtime="runtime", codeuri="codeuri", handler="handler") function2 = generate_function(runtime="runtime", codeuri="codeuri", handler="handler") @@ -981,13 +1011,13 @@ def test_go_runtime_different_handlers_are_not_equal(self): build_graph = BuildGraph("build/path") metadata = {} build_definition1 = FunctionBuildDefinition( - "go1.x", "codeuri", ZIP, ARM64, metadata, "handler", "source_hash", "manifest_hash" + "go1.x", "codeuri", None, ZIP, ARM64, metadata, "handler", "source_hash", "manifest_hash" ) function1 = generate_function( runtime="go1.x", codeuri=TestBuildGraph.CODEURI, metadata=metadata, handler="handler" ) build_definition2 = FunctionBuildDefinition( - "go1.x", "codeuri", ZIP, ARM64, metadata, "handler.new", "source_hash", "manifest_hash" + "go1.x", "codeuri", None, ZIP, ARM64, metadata, "handler.new", "source_hash", "manifest_hash" ) function2 = generate_function( runtime="go1.x", codeuri=TestBuildGraph.CODEURI, metadata=metadata, handler="handler.new" diff --git a/tests/unit/lib/build_module/test_build_strategy.py b/tests/unit/lib/build_module/test_build_strategy.py index 27ea179a77..c0ecedb125 100644 --- a/tests/unit/lib/build_module/test_build_strategy.py +++ b/tests/unit/lib/build_module/test_build_strategy.py @@ -44,8 +44,12 @@ def setUp(self): self.function2.get_build_dir = Mock() self.function2.full_path = "function2" - self.function_build_definition1 = FunctionBuildDefinition("runtime", "codeuri", ZIP, X86_64, {}, "handler") - self.function_build_definition2 = FunctionBuildDefinition("runtime2", "codeuri", ZIP, X86_64, {}, "handler") + self.function_build_definition1 = FunctionBuildDefinition( + "runtime", "codeuri", None, ZIP, X86_64, {}, "handler" + ) + self.function_build_definition2 = FunctionBuildDefinition( + "runtime2", "codeuri", None, ZIP, X86_64, {}, "handler" + ) self.function_build_definition1.add_function(self.function1_1) self.function_build_definition1.add_function(self.function1_2) @@ -218,6 +222,7 @@ def test_build_layers_and_functions(self, mock_copy_tree): call( self.function_build_definition1.get_function_name(), self.function_build_definition1.codeuri, + self.function_build_definition1.imageuri, ZIP, self.function_build_definition1.runtime, self.function_build_definition1.architecture, @@ -231,6 +236,7 @@ def test_build_layers_and_functions(self, mock_copy_tree): call( self.function_build_definition2.get_function_name(), self.function_build_definition2.codeuri, + self.function_build_definition2.imageuri, ZIP, self.function_build_definition2.runtime, self.function_build_definition2.architecture, @@ -323,7 +329,7 @@ def test_build_single_function_definition_image_functions_with_same_metadata(sel function2.full_path = "Function2" function2.packagetype = IMAGE build_definition = FunctionBuildDefinition( - "3.12", "codeuri", IMAGE, X86_64, {}, "handler", env_vars={"FOO": "BAR"} + "3.12", "codeuri", "imageuri", IMAGE, X86_64, {}, "handler", env_vars={"FOO": "BAR"} ) # since they have the same metadata, they are put into the same build_definition. build_definition.functions = [function1, function2] @@ -678,7 +684,7 @@ def test_assert_incremental_build_function(self, patched_manifest_hash, patched_ self.build_strategy.build() self.build_function.assert_called_with( - ANY, ANY, ANY, ANY, ANY, ANY, ANY, ANY, ANY, dependency_dir, download_dependencies + ANY, ANY, ANY, ANY, ANY, ANY, ANY, ANY, ANY, ANY, dependency_dir, download_dependencies ) @parameterized.expand( @@ -739,7 +745,7 @@ def setUp(self) -> None: ] ) def test_will_call_incremental_build_strategy(self, mocked_read, mocked_write, runtime): - build_definition = FunctionBuildDefinition(runtime, "codeuri", "packate_type", X86_64, {}, "handler") + build_definition = FunctionBuildDefinition(runtime, "codeuri", None, "package_type", X86_64, {}, "handler") self.build_graph.put_function_build_definition(build_definition, Mock(full_path="function_full_path")) with patch.object( self.build_strategy, "_incremental_build_strategy" @@ -759,7 +765,7 @@ def test_will_call_incremental_build_strategy(self, mocked_read, mocked_write, r ] ) def test_will_call_cached_build_strategy(self, mocked_read, mocked_write, runtime): - build_definition = FunctionBuildDefinition(runtime, "codeuri", "packate_type", X86_64, {}, "handler") + build_definition = FunctionBuildDefinition(runtime, "codeuri", None, "package_type", X86_64, {}, "handler") self.build_graph.put_function_build_definition(build_definition, Mock(full_path="function_full_path")) with patch.object( self.build_strategy, "_incremental_build_strategy" @@ -833,7 +839,7 @@ def test_wrapper_with_or_without_container(self, mocked_read, mocked_write, runt use_container, ) - build_definition = FunctionBuildDefinition(runtime, "codeuri", "packate_type", X86_64, {}, "handler") + build_definition = FunctionBuildDefinition(runtime, "codeuri", None, "package_type", X86_64, {}, "handler") self.build_graph.put_function_build_definition(build_definition, Mock(full_path="function_full_path")) with patch.object( build_strategy, "_incremental_build_strategy" diff --git a/tests/unit/lib/package/test_ecr_uploader.py b/tests/unit/lib/package/test_ecr_uploader.py index c688c67dff..94bb7c8127 100644 --- a/tests/unit/lib/package/test_ecr_uploader.py +++ b/tests/unit/lib/package/test_ecr_uploader.py @@ -1,5 +1,10 @@ +import docker + from unittest import TestCase -from unittest.mock import MagicMock, patch, call +from unittest.mock import MagicMock, Mock, call, mock_open, patch + +from pathlib import Path +from uuid import uuid4 from botocore.exceptions import ClientError from docker.errors import APIError, BuildError @@ -14,6 +19,7 @@ DeleteArtifactFailedError, ) from samcli.lib.package.ecr_uploader import ECRUploader +from samcli.lib.package.image_utils import SHA_CHECKSUM_TRUNCATION_LENGTH from samcli.lib.utils.stream_writer import StreamWriter @@ -194,6 +200,120 @@ def test_upload_failure_while_streaming(self): with self.assertRaises(DockerPushFailedError): ecr_uploader.upload(image, resource_name="HelloWorldFunction") + @patch.object(Path, "is_file", return_value=True) + @patch("builtins.open", new_callable=mock_open) + def test_upload_from_image_archive(self, mock_open, mock_is_file): + resource_name = "HelloWorldFunction" + digest = uuid4().hex + id = f"sha256:{digest}" + image = "./path/to/archive.tar.gz" + + self.docker_client.images.load.return_value = [Mock(id=id)] + self.docker_client.api.push.return_value.__iter__.return_value = iter( + [ + {"status": "Pushing to xyz"}, + {"id": "1", "status": "Preparing", "progress": ""}, + {"id": "2", "status": "Preparing", "progress": ""}, + {"id": "3", "status": "Preparing", "progress": ""}, + {"id": "1", "status": "Pushing", "progress": "[====> ]"}, + {"id": "3", "status": "Pushing", "progress": "[====> ]"}, + {"id": "2", "status": "Pushing", "progress": "[====> ]"}, + {"id": "3", "status": "Pushed", "progress": "[========>]"}, + {"id": "1", "status": "Pushed", "progress": "[========>]"}, + {"id": "2", "status": "Pushed", "progress": "[========>]"}, + {"status": f"image {resource_name} pushed digest: {digest}"}, + {}, + ] + ) + + ecr_uploader = ECRUploader( + docker_client=self.docker_client, + ecr_client=self.ecr_client, + ecr_repo=self.ecr_repo, + ecr_repo_multi=self.ecr_repo_multi, + tag=self.tag, + ) + ecr_uploader.login = MagicMock() + tag = ecr_uploader.upload(image, resource_name=resource_name) + self.assertEqual(f"{self.ecr_repo}:{resource_name}-{digest[:SHA_CHECKSUM_TRUNCATION_LENGTH]}-{self.tag}", tag) + + @patch.object(Path, "is_file", return_value=False) + def test_upload_from_digest(self, mock_is_file): + resource_name = "HelloWorldFunction" + digest = uuid4().hex + id = f"sha256:{digest}" + image = id + + self.docker_client.images.get.return_value = Mock(id=id) + self.docker_client.api.push.return_value.__iter__.return_value = iter( + [ + {"status": "Pushing to xyz"}, + {"id": "1", "status": "Preparing", "progress": ""}, + {"id": "2", "status": "Preparing", "progress": ""}, + {"id": "3", "status": "Preparing", "progress": ""}, + {"id": "1", "status": "Pushing", "progress": "[====> ]"}, + {"id": "3", "status": "Pushing", "progress": "[====> ]"}, + {"id": "2", "status": "Pushing", "progress": "[====> ]"}, + {"id": "3", "status": "Pushed", "progress": "[========>]"}, + {"id": "1", "status": "Pushed", "progress": "[========>]"}, + {"id": "2", "status": "Pushed", "progress": "[========>]"}, + {"status": f"image {resource_name} pushed digest: {digest}"}, + {}, + ] + ) + + ecr_uploader = ECRUploader( + docker_client=self.docker_client, + ecr_client=self.ecr_client, + ecr_repo=self.ecr_repo, + ecr_repo_multi=self.ecr_repo_multi, + tag=self.tag, + ) + ecr_uploader.login = MagicMock() + tag = ecr_uploader.upload(image, resource_name=resource_name) + self.assertEqual(f"{self.ecr_repo}:{resource_name}-{digest[:SHA_CHECKSUM_TRUNCATION_LENGTH]}-{self.tag}", tag) + + @patch.object(Path, "is_file", return_value=True) + @patch("builtins.open", new_callable=mock_open) + def test_upload_failure_if_archive_represents_multiple_images(self, mock_open, mock_is_file): + resource_name = "HelloWorldFunction" + image = "./path/to/archive.tar.gz" + + self.docker_client.images.load.return_value = [Mock(), Mock()] + + ecr_uploader = ECRUploader( + docker_client=self.docker_client, + ecr_client=self.ecr_client, + ecr_repo=self.ecr_repo, + ecr_repo_multi=self.ecr_repo_multi, + tag=self.tag, + ) + ecr_uploader.login = MagicMock() + + with self.assertRaises(DockerPushFailedError): + ecr_uploader.upload(image, resource_name=resource_name) + + @patch.object(Path, "is_file", return_value=False) + def test_upload_failure_if_image_archive_does_not_exist(self, mock_is_file): + resource_name = "HelloWorldFunction" + image = "./path/to/archive.tar.gz" + + # this error is raised because we ask the docker service for an image + # with an id or tag which resembles a file path (as `image` above) + self.docker_client.images.get.side_effect = docker.errors.ImageNotFound("no such image") + + ecr_uploader = ECRUploader( + docker_client=self.docker_client, + ecr_client=self.ecr_client, + ecr_repo=self.ecr_repo, + ecr_repo_multi=self.ecr_repo_multi, + tag=self.tag, + ) + ecr_uploader.login = MagicMock() + + with self.assertRaises(DockerPushFailedError): + ecr_uploader.upload(image, resource_name=resource_name) + @patch("samcli.lib.package.ecr_uploader.click.echo") def test_delete_artifact_successful(self, patched_click_echo): ecr_uploader = ECRUploader(