From c4c3576cd5367f41970755f8b05283d3b7ae5f62 Mon Sep 17 00:00:00 2001 From: kiarash kiani Date: Thu, 1 Aug 2024 11:07:33 +0200 Subject: [PATCH 01/16] refactor: rename bucket module name to object_storage --- src/damavand/cloud/aws/bucket.py | 2 +- src/damavand/resource/__init__.py | 2 +- src/damavand/resource/{bucket.py => object_storage.py} | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) rename src/damavand/resource/{bucket.py => object_storage.py} (74%) diff --git a/src/damavand/cloud/aws/bucket.py b/src/damavand/cloud/aws/bucket.py index 736ccbb..966723a 100644 --- a/src/damavand/cloud/aws/bucket.py +++ b/src/damavand/cloud/aws/bucket.py @@ -39,7 +39,7 @@ def provision(self): ) @runtime - def add_object(self, object: bytes, path: str): + def write(self, object: bytes, path: str): try: self.__s3_client.put_object( Body=object, diff --git a/src/damavand/resource/__init__.py b/src/damavand/resource/__init__.py index 02fe23d..f3f7d21 100644 --- a/src/damavand/resource/__init__.py +++ b/src/damavand/resource/__init__.py @@ -1,5 +1,5 @@ from .resource import BaseResource, runtime, buildtime -from .bucket import BaseObjectStorage +from .object_storage import BaseObjectStorage all = [ BaseResource, diff --git a/src/damavand/resource/bucket.py b/src/damavand/resource/object_storage.py similarity index 74% rename from src/damavand/resource/bucket.py rename to src/damavand/resource/object_storage.py index 5afef0d..7e0de55 100644 --- a/src/damavand/resource/bucket.py +++ b/src/damavand/resource/object_storage.py @@ -16,14 +16,14 @@ def __init__( def provision(self): raise NotImplementedError - def get_object(self, path: str) -> bytes: + def read(self, path: str) -> bytes: raise NotImplementedError - def add_object(self, object: bytes, path: str): + def write(self, object: bytes, path: str): raise NotImplementedError - def remove_object(self, path: str): + def delete(self, path: str): raise NotImplementedError - def list_objects(self) -> list[str]: + def list(self) -> list[str]: raise NotImplementedError From b16b72cf649f01cf936498444b49bf2da39178b8 Mon Sep 17 00:00:00 2001 From: kiarash kiani Date: Thu, 1 Aug 2024 12:32:29 +0200 Subject: [PATCH 02/16] feat(aws, object_storage): add read method --- src/damavand/cloud/aws/bucket.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/damavand/cloud/aws/bucket.py b/src/damavand/cloud/aws/bucket.py index 966723a..828f2ed 100644 --- a/src/damavand/cloud/aws/bucket.py +++ b/src/damavand/cloud/aws/bucket.py @@ -1,4 +1,5 @@ import boto3 +import io import logging from botocore.exceptions import ClientError from typing import Optional @@ -49,6 +50,19 @@ def write(self, object: bytes, path: str): except ClientError as e: logger.error(f"Failed to add object to bucket `{self.name}`: {e}") + @runtime + def read(self, path: str) -> bytes: + try: + buffer = io.BytesIO() + self.__s3_client.download_fileobj( + Bucket=self.name, Key=path, Fileobj=buffer + ) + + return buffer.getvalue() + except ClientError as e: + logger.error(f"Failed to read object at `{path}`: {e}") + raise RuntimeError(e) + @buildtime def to_pulumi(self) -> PulumiResource: if self._pulumi_object is None: From 4f072b91ca822a7cf25fb942c255be94c5d76c63 Mon Sep 17 00:00:00 2001 From: kiarash kiani Date: Thu, 1 Aug 2024 12:33:46 +0200 Subject: [PATCH 03/16] refactor: using custom error for build time errors --- src/damavand/cloud/aws/bucket.py | 3 ++- src/damavand/errors.py | 2 ++ tests/clouds/aws/test_aws_bucket.py | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 src/damavand/errors.py diff --git a/src/damavand/cloud/aws/bucket.py b/src/damavand/cloud/aws/bucket.py index 828f2ed..cf3eff7 100644 --- a/src/damavand/cloud/aws/bucket.py +++ b/src/damavand/cloud/aws/bucket.py @@ -6,6 +6,7 @@ from pulumi_aws import s3 from pulumi import Resource as PulumiResource +from damavand.errors import BuildtimeError from damavand.resource import BaseObjectStorage from damavand.resource.resource import buildtime, runtime @@ -66,7 +67,7 @@ def read(self, path: str) -> bytes: @buildtime def to_pulumi(self) -> PulumiResource: if self._pulumi_object is None: - raise ValueError( + raise BuildtimeError( "Resource not provisioned yet. Call `provision` method first." ) diff --git a/src/damavand/errors.py b/src/damavand/errors.py new file mode 100644 index 0000000..f938223 --- /dev/null +++ b/src/damavand/errors.py @@ -0,0 +1,2 @@ +class BuildtimeError(Exception): + pass diff --git a/tests/clouds/aws/test_aws_bucket.py b/tests/clouds/aws/test_aws_bucket.py index 5da50b9..e1163bf 100644 --- a/tests/clouds/aws/test_aws_bucket.py +++ b/tests/clouds/aws/test_aws_bucket.py @@ -3,6 +3,7 @@ from pulumi_aws import s3 from damavand.cloud.aws import AwsBucket +from damavand.errors import BuildtimeError @pytest.fixture @@ -13,7 +14,7 @@ def bucket(): def test_to_pulumi_raise_before_provision(monkeypatch: MonkeyPatch, bucket: AwsBucket): monkeypatch.setattr("damavand.utils.is_building", lambda: True) - with pytest.raises(ValueError): + with pytest.raises(BuildtimeError): bucket.to_pulumi() From fbce7e8bf52c0445f986abcd34e2892d752cb946 Mon Sep 17 00:00:00 2001 From: kiarash kiani Date: Thu, 1 Aug 2024 12:34:43 +0200 Subject: [PATCH 04/16] feat: output running mode as warning log --- src/damavand/core.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/damavand/core.py b/src/damavand/core.py index 1b94c74..4470448 100644 --- a/src/damavand/core.py +++ b/src/damavand/core.py @@ -73,6 +73,9 @@ def from_azure_provider(app_name: str, **kwargs) -> "CloudConnection": def __init__(self, resource_factory: ResourceFactory) -> None: self.resource_factory = resource_factory + logger.warning( + f"Running in {'build' if utils.is_building() else 'runtime'} mode" + ) def synth(self): self.resource_factory.provision_all_resources() From 8ea1c2364a897a28d007e4bff24c15b463b2bdaa Mon Sep 17 00:00:00 2001 From: kiarash kiani Date: Thu, 1 Aug 2024 12:38:03 +0200 Subject: [PATCH 05/16] feat(object_storage): add delete method --- src/damavand/cloud/aws/bucket.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/damavand/cloud/aws/bucket.py b/src/damavand/cloud/aws/bucket.py index cf3eff7..c67ce49 100644 --- a/src/damavand/cloud/aws/bucket.py +++ b/src/damavand/cloud/aws/bucket.py @@ -50,6 +50,7 @@ def write(self, object: bytes, path: str): ) except ClientError as e: logger.error(f"Failed to add object to bucket `{self.name}`: {e}") + raise RuntimeError(e) @runtime def read(self, path: str) -> bytes: @@ -64,6 +65,14 @@ def read(self, path: str) -> bytes: logger.error(f"Failed to read object at `{path}`: {e}") raise RuntimeError(e) + @runtime + def delete(self, path: str): + try: + self.__s3_client.delete_object(Bucket=self.name, Key=path) + except ClientError as e: + logger.error(f"Failed to delete object at `{path}`: {e}") + raise RuntimeError(e) + @buildtime def to_pulumi(self) -> PulumiResource: if self._pulumi_object is None: From 2ebbd75076d25a6b21d1afe28744151d82098ae0 Mon Sep 17 00:00:00 2001 From: kiarash kiani Date: Thu, 1 Aug 2024 12:54:10 +0200 Subject: [PATCH 06/16] feat(object_storage): add list objects method --- src/damavand/cloud/aws/bucket.py | 18 +++++++++++++++++- src/damavand/resource/object_storage.py | 9 +++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/damavand/cloud/aws/bucket.py b/src/damavand/cloud/aws/bucket.py index c67ce49..96953a7 100644 --- a/src/damavand/cloud/aws/bucket.py +++ b/src/damavand/cloud/aws/bucket.py @@ -2,7 +2,7 @@ import io import logging from botocore.exceptions import ClientError -from typing import Optional +from typing import Iterable, Optional from pulumi_aws import s3 from pulumi import Resource as PulumiResource @@ -73,6 +73,22 @@ def delete(self, path: str): logger.error(f"Failed to delete object at `{path}`: {e}") raise RuntimeError(e) + @runtime + def list(self) -> Iterable[str]: + """ + Return an iterable of object keys in the bucket. + + __ATTENTION__: This method is expensive for large buckets as it request multiple times to fetch all objects. + """ + try: + paginator = self.__s3_client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=self.name): + for obj in page.get("Contents", []): + yield obj["Key"] + except ClientError as e: + logger.error(f"Failed to list objects in storage `{self.name}`: {e}") + raise RuntimeError(e) + @buildtime def to_pulumi(self) -> PulumiResource: if self._pulumi_object is None: diff --git a/src/damavand/resource/object_storage.py b/src/damavand/resource/object_storage.py index 7e0de55..b13e97d 100644 --- a/src/damavand/resource/object_storage.py +++ b/src/damavand/resource/object_storage.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Iterable, Optional from damavand.resource import BaseResource @@ -25,5 +25,10 @@ def write(self, object: bytes, path: str): def delete(self, path: str): raise NotImplementedError - def list(self) -> list[str]: + def list(self) -> Iterable[str]: + """ + Return an iterable of object keys in the bucket. + + __ATTENTION__: This method is expensive for large buckets as it request multiple times to fetch all objects. + """ raise NotImplementedError From 6a730f00e7223c0ddcc5ae32b04f94621d4f2c64 Mon Sep 17 00:00:00 2001 From: kiarash kiani Date: Thu, 1 Aug 2024 13:06:53 +0200 Subject: [PATCH 07/16] feat(object_storage): method to validate existance of an object --- pdm.lock | 184 ++++++++++++++++++++++-- pyproject.toml | 1 + src/damavand/cloud/aws/bucket.py | 14 ++ src/damavand/resource/object_storage.py | 9 +- 4 files changed, 198 insertions(+), 10 deletions(-) diff --git a/pdm.lock b/pdm.lock index 94f753f..e826490 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:b347302c460bc6355044ae6f50be35f18ab39ab50069310266128ad12df0c98f" +content_hash = "sha256:319b79bd0a69400564a82932d3df91f7badbf4ac1d4ebf92539c94ba9e208af9" [[metadata.targets]] requires_python = ">=3.11.0" @@ -61,7 +61,7 @@ name = "boto3" version = "1.34.149" requires_python = ">=3.8" summary = "The AWS SDK for Python" -groups = ["default"] +groups = ["default", "dev"] dependencies = [ "botocore<1.35.0,>=1.34.149", "jmespath<2.0.0,>=0.7.1", @@ -93,7 +93,7 @@ name = "botocore" version = "1.34.149" requires_python = ">=3.8" summary = "Low-level, data-driven core of boto 3." -groups = ["default"] +groups = ["default", "dev"] dependencies = [ "jmespath<2.0.0,>=0.7.1", "python-dateutil<3.0.0,>=2.1", @@ -120,6 +120,52 @@ files = [ {file = "botocore_stubs-1.34.149.tar.gz", hash = "sha256:0d7b41d97206a5957ebc95e33f7a865d338c4404535aaa5a1d32cd3fa5d1ea88"}, ] +[[package]] +name = "certifi" +version = "2024.7.4" +requires_python = ">=3.6" +summary = "Python package for providing Mozilla's CA Bundle." +groups = ["dev"] +files = [ + {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, + {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, +] + +[[package]] +name = "cffi" +version = "1.16.0" +requires_python = ">=3.8" +summary = "Foreign Function Interface for Python calling C code." +groups = ["dev"] +marker = "platform_python_implementation != \"PyPy\"" +dependencies = [ + "pycparser", +] +files = [ + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + [[package]] name = "charset-normalizer" version = "3.3.2" @@ -277,6 +323,37 @@ files = [ {file = "coverage-7.6.0.tar.gz", hash = "sha256:289cc803fa1dc901f84701ac10c9ee873619320f2f9aff38794db4a4a0268d51"}, ] +[[package]] +name = "cryptography" +version = "43.0.0" +requires_python = ">=3.7" +summary = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +groups = ["dev"] +dependencies = [ + "cffi>=1.12; platform_python_implementation != \"PyPy\"", +] +files = [ + {file = "cryptography-43.0.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:64c3f16e2a4fc51c0d06af28441881f98c5d91009b8caaff40cf3548089e9c74"}, + {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3dcdedae5c7710b9f97ac6bba7e1052b95c7083c9d0e9df96e02a1932e777895"}, + {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d9a1eca329405219b605fac09ecfc09ac09e595d6def650a437523fcd08dd22"}, + {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ea9e57f8ea880eeea38ab5abf9fbe39f923544d7884228ec67d666abd60f5a47"}, + {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9a8d6802e0825767476f62aafed40532bd435e8a5f7d23bd8b4f5fd04cc80ecf"}, + {file = "cryptography-43.0.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:cc70b4b581f28d0a254d006f26949245e3657d40d8857066c2ae22a61222ef55"}, + {file = "cryptography-43.0.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4a997df8c1c2aae1e1e5ac49c2e4f610ad037fc5a3aadc7b64e39dea42249431"}, + {file = "cryptography-43.0.0-cp37-abi3-win32.whl", hash = "sha256:6e2b11c55d260d03a8cf29ac9b5e0608d35f08077d8c087be96287f43af3ccdc"}, + {file = "cryptography-43.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:31e44a986ceccec3d0498e16f3d27b2ee5fdf69ce2ab89b52eaad1d2f33d8778"}, + {file = "cryptography-43.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:7b3f5fe74a5ca32d4d0f302ffe6680fcc5c28f8ef0dc0ae8f40c0f3a1b4fca66"}, + {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac1955ce000cb29ab40def14fd1bbfa7af2017cca696ee696925615cafd0dce5"}, + {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:299d3da8e00b7e2b54bb02ef58d73cd5f55fb31f33ebbf33bd00d9aa6807df7e"}, + {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ee0c405832ade84d4de74b9029bedb7b31200600fa524d218fc29bfa371e97f5"}, + {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:cb013933d4c127349b3948aa8aaf2f12c0353ad0eccd715ca789c8a0f671646f"}, + {file = "cryptography-43.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fdcb265de28585de5b859ae13e3846a8e805268a823a12a4da2597f1f5afc9f0"}, + {file = "cryptography-43.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2905ccf93a8a2a416f3ec01b1a7911c3fe4073ef35640e7ee5296754e30b762b"}, + {file = "cryptography-43.0.0-cp39-abi3-win32.whl", hash = "sha256:47ca71115e545954e6c1d207dd13461ab81f4eccfcb1345eac874828b5e3eaaf"}, + {file = "cryptography-43.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:0663585d02f76929792470451a5ba64424acc3cd5227b03921dab0e2f27b1709"}, + {file = "cryptography-43.0.0.tar.gz", hash = "sha256:b88075ada2d51aa9f18283532c9f60e72170041bba88d7f37e49cbb10275299e"}, +] + [[package]] name = "decli" version = "0.6.2" @@ -360,6 +437,17 @@ files = [ {file = "grpcio-1.60.1.tar.gz", hash = "sha256:dd1d3a8d1d2e50ad9b59e10aa7f07c7d1be2b367f3f2d33c5fade96ed5460962"}, ] +[[package]] +name = "idna" +version = "3.7" +requires_python = ">=3.5" +summary = "Internationalized Domain Names in Applications (IDNA)" +groups = ["dev"] +files = [ + {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, + {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, +] + [[package]] name = "iniconfig" version = "2.0.0" @@ -401,7 +489,7 @@ name = "jmespath" version = "1.0.1" requires_python = ">=3.7" summary = "JSON Matching Expressions" -groups = ["default"] +groups = ["default", "dev"] files = [ {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, @@ -462,6 +550,28 @@ files = [ {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, ] +[[package]] +name = "moto" +version = "5.0.11" +requires_python = ">=3.8" +summary = "" +groups = ["dev"] +dependencies = [ + "Jinja2>=2.10.1", + "boto3>=1.9.201", + "botocore>=1.14.0", + "cryptography>=3.3.1", + "python-dateutil<3.0.0,>=2.1", + "requests>=2.5", + "responses>=0.15.0", + "werkzeug!=2.2.0,!=2.2.1,>=0.5", + "xmltodict", +] +files = [ + {file = "moto-5.0.11-py2.py3-none-any.whl", hash = "sha256:bdba9bec0afcde9f99b58c5271d6458dbfcda0a0a1e9beaecd808d2591db65ea"}, + {file = "moto-5.0.11.tar.gz", hash = "sha256:606b641f4c6ef69f28a84147d6d6806d052011e7ae7b0fe46ae8858e7a27a0a3"}, +] + [[package]] name = "mypy" version = "1.11.0" @@ -662,6 +772,18 @@ files = [ {file = "pulumi_azure_native-2.51.0.tar.gz", hash = "sha256:f0c0f4005a5c9d15ff2a9cb28fa557b00b47cd0b77f26337db166d95ee0b18c7"}, ] +[[package]] +name = "pycparser" +version = "2.22" +requires_python = ">=3.8" +summary = "C parser in Python" +groups = ["dev"] +marker = "platform_python_implementation != \"PyPy\"" +files = [ + {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, + {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, +] + [[package]] name = "pygments" version = "2.18.0" @@ -772,7 +894,7 @@ name = "python-dateutil" version = "2.9.0.post0" requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" summary = "Extensions to the standard Python datetime module" -groups = ["default"] +groups = ["default", "dev"] dependencies = [ "six>=1.5", ] @@ -820,6 +942,39 @@ files = [ {file = "questionary-2.0.1.tar.gz", hash = "sha256:bcce898bf3dbb446ff62830c86c5c6fb9a22a54146f0f5597d3da43b10d8fc8b"}, ] +[[package]] +name = "requests" +version = "2.32.3" +requires_python = ">=3.8" +summary = "Python HTTP for Humans." +groups = ["dev"] +dependencies = [ + "certifi>=2017.4.17", + "charset-normalizer<4,>=2", + "idna<4,>=2.5", + "urllib3<3,>=1.21.1", +] +files = [ + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, +] + +[[package]] +name = "responses" +version = "0.25.3" +requires_python = ">=3.8" +summary = "A utility library for mocking out the `requests` Python library." +groups = ["dev"] +dependencies = [ + "pyyaml", + "requests<3.0,>=2.30.0", + "urllib3<3.0,>=1.25.10", +] +files = [ + {file = "responses-0.25.3-py3-none-any.whl", hash = "sha256:521efcbc82081ab8daa588e08f7e8a64ce79b91c39f6e62199b19159bea7dbcb"}, + {file = "responses-0.25.3.tar.gz", hash = "sha256:617b9247abd9ae28313d57a75880422d55ec63c29d33d629697590a034358dba"}, +] + [[package]] name = "rich" version = "13.7.1" @@ -841,7 +996,7 @@ name = "s3transfer" version = "0.10.2" requires_python = ">=3.8" summary = "An Amazon S3 Transfer Manager" -groups = ["default"] +groups = ["default", "dev"] dependencies = [ "botocore<2.0a.0,>=1.33.2", ] @@ -866,7 +1021,7 @@ name = "six" version = "1.16.0" requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" summary = "Python 2 and 3 compatibility utilities" -groups = ["default"] +groups = ["default", "dev"] files = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, @@ -932,7 +1087,7 @@ name = "urllib3" version = "2.2.2" requires_python = ">=3.8" summary = "HTTP library with thread-safe connection pooling, file post, and more." -groups = ["default"] +groups = ["default", "dev"] files = [ {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, @@ -956,7 +1111,7 @@ name = "werkzeug" version = "3.0.3" requires_python = ">=3.8" summary = "The comprehensive WSGI web application library." -groups = ["default"] +groups = ["default", "dev"] dependencies = [ "MarkupSafe>=2.1.1", ] @@ -978,3 +1133,14 @@ files = [ {file = "wmctrl-0.5-py2.py3-none-any.whl", hash = "sha256:ae695c1863a314c899e7cf113f07c0da02a394b968c4772e1936219d9234ddd7"}, {file = "wmctrl-0.5.tar.gz", hash = "sha256:7839a36b6fe9e2d6fd22304e5dc372dbced2116ba41283ea938b2da57f53e962"}, ] + +[[package]] +name = "xmltodict" +version = "0.13.0" +requires_python = ">=3.4" +summary = "Makes working with XML feel like you are working with JSON" +groups = ["dev"] +files = [ + {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, + {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, +] diff --git a/pyproject.toml b/pyproject.toml index f2dbfd7..98f30ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dev = [ "pytest>=8.3.2", "pytest-coverage>=0.0", "pyright>=1.1.374", + "moto>=5.0.11", ] [tool.commitizen] version = "1.0.0" diff --git a/src/damavand/cloud/aws/bucket.py b/src/damavand/cloud/aws/bucket.py index 96953a7..7192c4e 100644 --- a/src/damavand/cloud/aws/bucket.py +++ b/src/damavand/cloud/aws/bucket.py @@ -89,6 +89,20 @@ def list(self) -> Iterable[str]: logger.error(f"Failed to list objects in storage `{self.name}`: {e}") raise RuntimeError(e) + @runtime + def exist(self, path: str) -> bool: + """Check if an object exists in the bucket.""" + try: + self.__s3_client.head_object(Bucket=self.name, Key=path) + return True + except ClientError as e: + error_code = e.response.get("Error", {}).get("Code") + if error_code == "404": + return False + else: + logger.error(f"Failed to check object existence at `{path}`: {e}") + raise RuntimeError(e) + @buildtime def to_pulumi(self) -> PulumiResource: if self._pulumi_object is None: diff --git a/src/damavand/resource/object_storage.py b/src/damavand/resource/object_storage.py index b13e97d..881afdf 100644 --- a/src/damavand/resource/object_storage.py +++ b/src/damavand/resource/object_storage.py @@ -17,18 +17,25 @@ def provision(self): raise NotImplementedError def read(self, path: str) -> bytes: + """Read an object from the storage.""" raise NotImplementedError def write(self, object: bytes, path: str): + """Write an object""" raise NotImplementedError def delete(self, path: str): + """Delete an object from the storage.""" raise NotImplementedError def list(self) -> Iterable[str]: """ Return an iterable of object keys in the bucket. - __ATTENTION__: This method is expensive for large buckets as it request multiple times to fetch all objects. + __ATTENTION__: This method is expensive for large storages as it request multiple times to fetch all objects. """ raise NotImplementedError + + def exist(self, path: str) -> bool: + """Check if an object exists in the storage.""" + raise NotImplementedError From 9308986d7719e5ef26b099b74b47dea77e3d8bf3 Mon Sep 17 00:00:00 2001 From: kiarash kiani Date: Thu, 1 Aug 2024 13:33:51 +0200 Subject: [PATCH 08/16] fix(object_storage): align runtime region with buildtime region --- src/damavand/cloud/aws/bucket.py | 3 ++- src/damavand/core.py | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/damavand/cloud/aws/bucket.py b/src/damavand/cloud/aws/bucket.py index 7192c4e..db8e9c9 100644 --- a/src/damavand/cloud/aws/bucket.py +++ b/src/damavand/cloud/aws/bucket.py @@ -18,12 +18,13 @@ class AwsBucket(BaseObjectStorage): def __init__( self, name, + region: str, id_: Optional[str] = None, tags: dict[str, str] = {}, **kwargs, ) -> None: super().__init__(name, id_, tags, **kwargs) - self.__s3_client = boto3.client("s3") + self.__s3_client = boto3.client("s3", region_name=region) @buildtime def provision(self): diff --git a/src/damavand/core.py b/src/damavand/core.py index 4470448..2649e90 100644 --- a/src/damavand/core.py +++ b/src/damavand/core.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Optional +from typing import Any, Optional, cast from rich.console import Console import pulumi_aws as aws import pulumi_azure_native as azurerm @@ -34,7 +34,12 @@ def provision_all_resources(self) -> None: def new_object_storage(self, name: str, tags: dict, **kwargs) -> BaseObjectStorage: match self.provider: case AwsProvider(): - resource = AwsBucket(name, tags=tags, **kwargs) + resource = AwsBucket( + name, + region=cast(str, self.provider.region), + tags=tags, + **kwargs, + ) self._resources.append(resource) return resource case AzurermProvider(): From 85d6c56a9b5b942b02725410647aa2341dbc40ac Mon Sep 17 00:00:00 2001 From: kiarash kiani Date: Thu, 1 Aug 2024 13:37:33 +0200 Subject: [PATCH 09/16] chore(object_storage): add tests --- tests/clouds/aws/test_aws_bucket.py | 66 ++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/tests/clouds/aws/test_aws_bucket.py b/tests/clouds/aws/test_aws_bucket.py index e1163bf..fa52c7d 100644 --- a/tests/clouds/aws/test_aws_bucket.py +++ b/tests/clouds/aws/test_aws_bucket.py @@ -1,5 +1,7 @@ +import boto3 import pytest from _pytest.monkeypatch import MonkeyPatch +from moto import mock_aws from pulumi_aws import s3 from damavand.cloud.aws import AwsBucket @@ -7,8 +9,15 @@ @pytest.fixture +@mock_aws def bucket(): - return AwsBucket("test-bucket") + return AwsBucket("test-bucket", region="us-east-1") + + +@pytest.fixture +@mock_aws +def conn(): + return boto3.resource("s3", region_name="us-east-1") def test_to_pulumi_raise_before_provision(monkeypatch: MonkeyPatch, bucket: AwsBucket): @@ -23,3 +32,58 @@ def test_to_pulumi_return_pulumi_s3_bucket(monkeypatch: MonkeyPatch, bucket: Aws bucket.provision() assert isinstance(bucket.to_pulumi(), s3.Bucket) + + +@mock_aws +def test_write(bucket: AwsBucket, conn): + conn.create_bucket(Bucket=bucket.name) + + bucket.write(b"Hello, World!", "test.txt") + + obj = conn.Object(bucket.name, "test.txt") + assert obj.get()["Body"].read() == b"Hello, World!" + + +@mock_aws +def test_read(bucket: AwsBucket, conn): + conn.create_bucket(Bucket=bucket.name) + + obj = conn.Object(bucket.name, "test.txt") + obj.put(Body=b"Hello, World!") + + assert bucket.read("test.txt") == b"Hello, World!" + + +@mock_aws +def test_delete(bucket: AwsBucket, conn): + conn.create_bucket(Bucket=bucket.name) + + obj = conn.Object(bucket.name, "test.txt") + obj.put(Body=b"Hello, World!") + + bucket.delete("test.txt") + + with pytest.raises(conn.meta.client.exceptions.ClientError): + obj.get() + + +@mock_aws +def test_exist(bucket: AwsBucket, conn): + conn.create_bucket(Bucket=bucket.name) + + obj = conn.Object(bucket.name, "test.txt") + obj.put(Body=b"Hello, World!") + + assert bucket.exist("test.txt") + assert not bucket.exist("not-exist.txt") + + +@mock_aws +def test_list(bucket: AwsBucket, conn): + conn.create_bucket(Bucket=bucket.name) + + conn.Object(bucket.name, "test1.txt").put(Body=b"Hello, World!") + conn.Object(bucket.name, "test2.txt").put(Body=b"Hello, World!") + conn.Object(bucket.name, "test3.txt").put(Body=b"Hello, World!") + + assert set(bucket.list()) == {"test1.txt", "test2.txt", "test3.txt"} From 82735c3cdadaaf7fb2d9a1cb2518c504e08ddc5f Mon Sep 17 00:00:00 2001 From: kiarash kiani Date: Thu, 1 Aug 2024 14:37:49 +0200 Subject: [PATCH 10/16] fix(resource_factory): validate if aws provider has a region --- src/damavand/core.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/damavand/core.py b/src/damavand/core.py index 2649e90..bc5cf21 100644 --- a/src/damavand/core.py +++ b/src/damavand/core.py @@ -21,6 +21,9 @@ def __init__( provider: CloudProvider, resources: list[BaseResource] = [], ) -> None: + if isinstance(provider, AwsProvider) and not provider.region: + raise ValueError("AWS provider must have a region set.") + self.app_name = app_name self.provider = provider self._resources = resources From 233eeddd4026bf8b375cefccb33c5f8b922964ce Mon Sep 17 00:00:00 2001 From: kiarash kiani Date: Thu, 1 Aug 2024 17:34:53 +0200 Subject: [PATCH 11/16] feat(object_storage): improved the error handlings --- src/damavand/cloud/aws/bucket.py | 60 ++++++++++++++++++++--------- src/damavand/errors.py | 30 ++++++++++++++- src/damavand/utils.py | 6 +++ tests/clouds/aws/test_aws_bucket.py | 4 +- 4 files changed, 78 insertions(+), 22 deletions(-) diff --git a/src/damavand/cloud/aws/bucket.py b/src/damavand/cloud/aws/bucket.py index db8e9c9..1d617a5 100644 --- a/src/damavand/cloud/aws/bucket.py +++ b/src/damavand/cloud/aws/bucket.py @@ -6,9 +6,15 @@ from pulumi_aws import s3 from pulumi import Resource as PulumiResource -from damavand.errors import BuildtimeError +from damavand import utils from damavand.resource import BaseObjectStorage from damavand.resource.resource import buildtime, runtime +from damavand.errors import ( + CallResourceBeforeProvision, + RuntimeException, + ObjectNotFound, + ResourceAccessDenied, +) logger = logging.getLogger(__name__) @@ -50,8 +56,12 @@ def write(self, object: bytes, path: str): Key=path, ) except ClientError as e: - logger.error(f"Failed to add object to bucket `{self.name}`: {e}") - raise RuntimeError(e) + match utils.error_code_from_boto3(e): + case "AccessDenied": + raise ResourceAccessDenied(name=self.name) + case _: + logger.exception("Failed to write the object to AWS.") + raise RuntimeException() @runtime def read(self, path: str) -> bytes: @@ -63,16 +73,26 @@ def read(self, path: str) -> bytes: return buffer.getvalue() except ClientError as e: - logger.error(f"Failed to read object at `{path}`: {e}") - raise RuntimeError(e) + match utils.error_code_from_boto3(e): + case "AccessDenied": + raise ResourceAccessDenied(name=self.name) + case "NoSuchKey": + raise ObjectNotFound(name=path) + case _: + logger.exception("Failed to read the object from AWS") + raise RuntimeException() @runtime def delete(self, path: str): try: self.__s3_client.delete_object(Bucket=self.name, Key=path) except ClientError as e: - logger.error(f"Failed to delete object at `{path}`: {e}") - raise RuntimeError(e) + match utils.error_code_from_boto3(e): + case "AccessDenied": + raise ResourceAccessDenied(name=self.name) + case _: + logger.exception("Failed to delete the object from AWS.") + raise RuntimeException() @runtime def list(self) -> Iterable[str]: @@ -87,8 +107,12 @@ def list(self) -> Iterable[str]: for obj in page.get("Contents", []): yield obj["Key"] except ClientError as e: - logger.error(f"Failed to list objects in storage `{self.name}`: {e}") - raise RuntimeError(e) + match utils.error_code_from_boto3(e): + case "AccessDenied": + raise ResourceAccessDenied(name=self.name) + case _: + logger.exception("Failed to list objects from AWS.") + raise RuntimeException() @runtime def exist(self, path: str) -> bool: @@ -97,18 +121,18 @@ def exist(self, path: str) -> bool: self.__s3_client.head_object(Bucket=self.name, Key=path) return True except ClientError as e: - error_code = e.response.get("Error", {}).get("Code") - if error_code == "404": - return False - else: - logger.error(f"Failed to check object existence at `{path}`: {e}") - raise RuntimeError(e) + match utils.error_code_from_boto3(e): + case "NoSuchKey": + return False + case "AccessDenied": + raise ResourceAccessDenied(name=self.name) + case _: + logger.exception("Failed to check the object existence in AWS.") + raise RuntimeException() @buildtime def to_pulumi(self) -> PulumiResource: if self._pulumi_object is None: - raise BuildtimeError( - "Resource not provisioned yet. Call `provision` method first." - ) + raise CallResourceBeforeProvision() return self._pulumi_object diff --git a/src/damavand/errors.py b/src/damavand/errors.py index f938223..a83cb06 100644 --- a/src/damavand/errors.py +++ b/src/damavand/errors.py @@ -1,2 +1,28 @@ -class BuildtimeError(Exception): - pass +class DamavandException(Exception): + fmt = "An unknown error occurred." + + def __init__(self, **kwargs: str) -> None: + msg = self.fmt.format(**kwargs) + super().__init__(msg) + + +# Buildtime exceptions +class BuildtimeException(DamavandException): + fmt = "An unknown error occurred during buildtime." + + +class CallResourceBeforeProvision(BuildtimeException): + fmt = "Resource called before provision. Call `provision` method first." + + +# Runtime exceptions +class RuntimeException(DamavandException): + fmt = "An unknown error occurred happend during runtime." + + +class ResourceAccessDenied(RuntimeException): + fmt = "Access to resource `{name}` is denied." + + +class ObjectNotFound(RuntimeException): + fmt = "Object `{name}` not found in the storage." diff --git a/src/damavand/utils.py b/src/damavand/utils.py index 6b2893c..4c747c9 100644 --- a/src/damavand/utils.py +++ b/src/damavand/utils.py @@ -5,3 +5,9 @@ def is_building(): """Check if the application is being built or run""" return os.environ.get("MODE", "RUN") == "BUILD" + + +def error_code_from_boto3(e): + """Extract error code from boto3 exception""" + + return e.response.get("Error", {}).get("Code") diff --git a/tests/clouds/aws/test_aws_bucket.py b/tests/clouds/aws/test_aws_bucket.py index fa52c7d..490aca7 100644 --- a/tests/clouds/aws/test_aws_bucket.py +++ b/tests/clouds/aws/test_aws_bucket.py @@ -5,7 +5,7 @@ from pulumi_aws import s3 from damavand.cloud.aws import AwsBucket -from damavand.errors import BuildtimeError +from damavand.errors import CallResourceBeforeProvision @pytest.fixture @@ -23,7 +23,7 @@ def conn(): def test_to_pulumi_raise_before_provision(monkeypatch: MonkeyPatch, bucket: AwsBucket): monkeypatch.setattr("damavand.utils.is_building", lambda: True) - with pytest.raises(BuildtimeError): + with pytest.raises(CallResourceBeforeProvision): bucket.to_pulumi() From 17c918dd59052d19036e290eeadca6d0286526d1 Mon Sep 17 00:00:00 2001 From: kiarash kiani Date: Fri, 2 Aug 2024 09:16:47 +0200 Subject: [PATCH 12/16] fix(object_storage): not catching boto errors with http codes --- src/damavand/cloud/aws/bucket.py | 14 +++++++------- tests/clouds/aws/test_aws_bucket.py | 10 +++++++++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/damavand/cloud/aws/bucket.py b/src/damavand/cloud/aws/bucket.py index 1d617a5..9734720 100644 --- a/src/damavand/cloud/aws/bucket.py +++ b/src/damavand/cloud/aws/bucket.py @@ -57,7 +57,7 @@ def write(self, object: bytes, path: str): ) except ClientError as e: match utils.error_code_from_boto3(e): - case "AccessDenied": + case "AccessDenied" | "403": raise ResourceAccessDenied(name=self.name) case _: logger.exception("Failed to write the object to AWS.") @@ -74,9 +74,9 @@ def read(self, path: str) -> bytes: return buffer.getvalue() except ClientError as e: match utils.error_code_from_boto3(e): - case "AccessDenied": + case "AccessDenied" | "403": raise ResourceAccessDenied(name=self.name) - case "NoSuchKey": + case "NoSuchKey" | "404": raise ObjectNotFound(name=path) case _: logger.exception("Failed to read the object from AWS") @@ -88,7 +88,7 @@ def delete(self, path: str): self.__s3_client.delete_object(Bucket=self.name, Key=path) except ClientError as e: match utils.error_code_from_boto3(e): - case "AccessDenied": + case "AccessDenied" | "403": raise ResourceAccessDenied(name=self.name) case _: logger.exception("Failed to delete the object from AWS.") @@ -108,7 +108,7 @@ def list(self) -> Iterable[str]: yield obj["Key"] except ClientError as e: match utils.error_code_from_boto3(e): - case "AccessDenied": + case "AccessDenied" | "403": raise ResourceAccessDenied(name=self.name) case _: logger.exception("Failed to list objects from AWS.") @@ -122,9 +122,9 @@ def exist(self, path: str) -> bool: return True except ClientError as e: match utils.error_code_from_boto3(e): - case "NoSuchKey": + case "NoSuchKey" | "404": return False - case "AccessDenied": + case "AccessDenied" | "403": raise ResourceAccessDenied(name=self.name) case _: logger.exception("Failed to check the object existence in AWS.") diff --git a/tests/clouds/aws/test_aws_bucket.py b/tests/clouds/aws/test_aws_bucket.py index 490aca7..44c5b7b 100644 --- a/tests/clouds/aws/test_aws_bucket.py +++ b/tests/clouds/aws/test_aws_bucket.py @@ -5,7 +5,7 @@ from pulumi_aws import s3 from damavand.cloud.aws import AwsBucket -from damavand.errors import CallResourceBeforeProvision +from damavand.errors import CallResourceBeforeProvision, ObjectNotFound @pytest.fixture @@ -54,6 +54,14 @@ def test_read(bucket: AwsBucket, conn): assert bucket.read("test.txt") == b"Hello, World!" +@mock_aws +def test_read_not_exist(bucket: AwsBucket, conn): + conn.create_bucket(Bucket=bucket.name) + + with pytest.raises(ObjectNotFound): + bucket.read("test-not-exist.txt") + + @mock_aws def test_delete(bucket: AwsBucket, conn): conn.create_bucket(Bucket=bucket.name) From fdf5b23baac6da80f45bb3e1968d211ad71e1a8d Mon Sep 17 00:00:00 2001 From: kiarash kiani Date: Fri, 2 Aug 2024 11:12:00 +0200 Subject: [PATCH 13/16] fix(object_storage): not having access to original boto3 ClientError All the exceptions wrapping boto3 ClientError with set the ClientError as their root cause --- src/damavand/cloud/aws/bucket.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/src/damavand/cloud/aws/bucket.py b/src/damavand/cloud/aws/bucket.py index 9734720..27633d3 100644 --- a/src/damavand/cloud/aws/bucket.py +++ b/src/damavand/cloud/aws/bucket.py @@ -58,10 +58,9 @@ def write(self, object: bytes, path: str): except ClientError as e: match utils.error_code_from_boto3(e): case "AccessDenied" | "403": - raise ResourceAccessDenied(name=self.name) + raise ResourceAccessDenied(name=self.name) from e case _: - logger.exception("Failed to write the object to AWS.") - raise RuntimeException() + raise RuntimeException() from e @runtime def read(self, path: str) -> bytes: @@ -75,12 +74,11 @@ def read(self, path: str) -> bytes: except ClientError as e: match utils.error_code_from_boto3(e): case "AccessDenied" | "403": - raise ResourceAccessDenied(name=self.name) + raise ResourceAccessDenied(name=self.name) from e case "NoSuchKey" | "404": - raise ObjectNotFound(name=path) + raise ObjectNotFound(name=path) from e case _: - logger.exception("Failed to read the object from AWS") - raise RuntimeException() + raise RuntimeException() from e @runtime def delete(self, path: str): @@ -89,10 +87,9 @@ def delete(self, path: str): except ClientError as e: match utils.error_code_from_boto3(e): case "AccessDenied" | "403": - raise ResourceAccessDenied(name=self.name) + raise ResourceAccessDenied(name=self.name) from e case _: - logger.exception("Failed to delete the object from AWS.") - raise RuntimeException() + raise RuntimeException() from e @runtime def list(self) -> Iterable[str]: @@ -109,10 +106,9 @@ def list(self) -> Iterable[str]: except ClientError as e: match utils.error_code_from_boto3(e): case "AccessDenied" | "403": - raise ResourceAccessDenied(name=self.name) + raise ResourceAccessDenied(name=self.name) from e case _: - logger.exception("Failed to list objects from AWS.") - raise RuntimeException() + raise RuntimeException() from e @runtime def exist(self, path: str) -> bool: @@ -125,10 +121,9 @@ def exist(self, path: str) -> bool: case "NoSuchKey" | "404": return False case "AccessDenied" | "403": - raise ResourceAccessDenied(name=self.name) + raise ResourceAccessDenied(name=self.name) from e case _: - logger.exception("Failed to check the object existence in AWS.") - raise RuntimeException() + raise RuntimeException() from e @buildtime def to_pulumi(self) -> PulumiResource: From 2d9d52c56792f271a2f065c525bd43be3b894957 Mon Sep 17 00:00:00 2001 From: kiarash kiani Date: Fri, 2 Aug 2024 15:11:03 +0200 Subject: [PATCH 14/16] feat(errors): option to provide custom message to an exception --- src/damavand/errors.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/damavand/errors.py b/src/damavand/errors.py index a83cb06..c3cb8e3 100644 --- a/src/damavand/errors.py +++ b/src/damavand/errors.py @@ -1,9 +1,15 @@ +from typing import Optional + + class DamavandException(Exception): fmt = "An unknown error occurred." - def __init__(self, **kwargs: str) -> None: - msg = self.fmt.format(**kwargs) - super().__init__(msg) + def __init__(self, msg: Optional[str] = None, **kwargs: str) -> None: + if msg is None: + default_msg = self.fmt.format(**kwargs) + super().__init__(default_msg) + else: + super().__init__(msg) # Buildtime exceptions From e62f95bd8fad36c964d9b70b522f282e75e79a32 Mon Sep 17 00:00:00 2001 From: kiarash kiani Date: Fri, 2 Aug 2024 15:13:37 +0200 Subject: [PATCH 15/16] fix(provider): setted properties were not directly available providers arguments were not directly available due to design of pulumi. This solution creates defautl wapper around pulumi providers to keep contexts. --- src/damavand/cloud/provider.py | 55 ++++++++++++++++++++++++++++++++-- src/damavand/core.py | 43 ++++++++++++++++---------- 2 files changed, 80 insertions(+), 18 deletions(-) diff --git a/src/damavand/cloud/provider.py b/src/damavand/cloud/provider.py index 61a9896..38359b0 100644 --- a/src/damavand/cloud/provider.py +++ b/src/damavand/cloud/provider.py @@ -1,6 +1,55 @@ -from typing import Union -from pulumi_azure_native import Provider as AzurermProvider -from pulumi_aws import Provider as AwsProvider +from typing import Optional, Union +from pulumi_azure_native import Provider as PulumiAzurermProvider +from pulumi_aws import Provider as PulumiAwsProvider +from pulumi import ResourceOptions + + +class BaseProvider: + def __init__(self, app_name: str) -> None: + self.app_name = app_name + + +class AwsProvider(BaseProvider, PulumiAwsProvider): + def __init__( + self, + app_name: str, + region: str, + opts: Optional[ResourceOptions] = None, + **kwargs, + ) -> None: + """Create a new AWS provider instance. For available options, see: https://www.pulumi.com/registry/packages/aws/api-docs/provider/#inputs""" + + BaseProvider.__init__( + self, + app_name=app_name, + ) + PulumiAwsProvider.__init__( + self, + resource_name=f"{app_name}-provider", + opts=opts, + region=region, + **kwargs, + ) + + self.__region = region + + @property + def enforced_region(self) -> str: + """Region that enforced to all AWS operations""" + return self.__region + + +class AzurermProvider(BaseProvider, PulumiAzurermProvider): + def __init__(self, app_name: str, **kwargs) -> None: + BaseProvider.__init__( + self, + app_name=app_name, + ) + PulumiAzurermProvider.__init__( + self, + resource_name=f"{app_name}-provider", + **kwargs, + ) CloudProvider = Union[ diff --git a/src/damavand/core.py b/src/damavand/core.py index bc5cf21..1f428e9 100644 --- a/src/damavand/core.py +++ b/src/damavand/core.py @@ -1,8 +1,6 @@ import logging -from typing import Any, Optional, cast +from typing import Any, Optional from rich.console import Console -import pulumi_aws as aws -import pulumi_azure_native as azurerm from damavand import utils from damavand.resource import BaseResource, BaseObjectStorage @@ -21,9 +19,6 @@ def __init__( provider: CloudProvider, resources: list[BaseResource] = [], ) -> None: - if isinstance(provider, AwsProvider) and not provider.region: - raise ValueError("AWS provider must have a region set.") - self.app_name = app_name self.provider = provider self._resources = resources @@ -39,8 +34,8 @@ def new_object_storage(self, name: str, tags: dict, **kwargs) -> BaseObjectStora case AwsProvider(): resource = AwsBucket( name, - region=cast(str, self.provider.region), - tags=tags, + region=self.provider.enforced_region, + tags={**self.all_tags, **tags}, **kwargs, ) self._resources.append(resource) @@ -53,31 +48,49 @@ def new_object_storage(self, name: str, tags: dict, **kwargs) -> BaseObjectStora class CloudConnection: @staticmethod - def from_aws_provider(app_name: str, region: str, **kwargs) -> "CloudConnection": + def from_aws_provider( + app_name: str, region: str, tags: dict[str, str], **kwargs + ) -> "CloudConnection": """ Create a connection for AWS provider. Check `AwsProvider` class for more information about the available arguments. """ provider = AwsProvider( - f"{app_name}-provider", - args=aws.ProviderArgs(region=region, **kwargs), + app_name=app_name, + region=region, + **kwargs, ) - return CloudConnection(ResourceFactory(app_name, provider)) + return CloudConnection( + ResourceFactory( + app_name=app_name, + provider=provider, + tags=tags, + ) + ) @staticmethod - def from_azure_provider(app_name: str, **kwargs) -> "CloudConnection": + def from_azure_provider( + app_name: str, tags: dict[str, str], **kwargs + ) -> "CloudConnection": """ Create a connection for Azure provider. Check `AzurermProvider` class for more information about the available arguments. """ provider = AzurermProvider( - f"{app_name}-provider", args=azurerm.ProviderArgs(**kwargs) + app_name=app_name, ) - return CloudConnection(ResourceFactory(app_name, provider)) + return CloudConnection( + ResourceFactory( + app_name=app_name, + provider=provider, + tags=tags, + **kwargs, + ) + ) def __init__(self, resource_factory: ResourceFactory) -> None: self.resource_factory = resource_factory From b88f6e65e098b16f755d370753bbc2da0df37785 Mon Sep 17 00:00:00 2001 From: kiarash kiani Date: Fri, 2 Aug 2024 15:15:54 +0200 Subject: [PATCH 16/16] feat(tags): add support for default and user/global tags --- src/damavand/core.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/damavand/core.py b/src/damavand/core.py index 1f428e9..4d1fb4a 100644 --- a/src/damavand/core.py +++ b/src/damavand/core.py @@ -18,10 +18,31 @@ def __init__( app_name: str, provider: CloudProvider, resources: list[BaseResource] = [], + tags: dict[str, str] = {}, ) -> None: self.app_name = app_name self.provider = provider self._resources = resources + self._user_defined_tags = tags + + @property + def default_tags(self) -> dict[str, str]: + return { + "application": self.app_name, + "environment": "development", + "iac_optimizer": "damavand", + } + + @property + def user_defined_tags(self) -> dict[str, str]: + return self._user_defined_tags + + @property + def all_tags(self) -> dict[str, str]: + return { + **self.default_tags, + **self.user_defined_tags, + } def provision_all_resources(self) -> None: """Provision all resources in the factory"""