diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 1ac3e13..9503ad9 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -45,15 +45,15 @@ jobs: - uses: actions/checkout@v4 - name: Run Ruff syntax checks - uses: chartboost/ruff-action@v1 + uses: astral-sh/ruff-action@v3 with: src: './tb_pulumi' - name: Run Ruff linter - uses: chartboost/ruff-action@v1 + uses: astral-sh/ruff-action@v3 with: src: './tb_pulumi' - args: 'format --check' + args: 'format --check --diff' # Attempt to build the documentation. Fail if there are errors or warnings. build_docs: diff --git a/docs/monitors.rst b/docs/monitors.rst index 5585d2a..fc5b6c0 100644 --- a/docs/monitors.rst +++ b/docs/monitors.rst @@ -4,17 +4,19 @@ When you use a ``ThunderbirdPulumiProject`` and add ``ThunderbirdComponentResour resources in an internal mapping correlating the name of the module to a collection of its resources. These resources can have complex structures with nested lists, dicts, and ``ThunderbirdComponentResource`` s. The project's :py:meth:`tb_pulumi.ThunderbirdPulumiProject.flatten` function returns these as a flat list of unlabeled Pulumi -``Resource`` s. +``Resource`` s and ``Output`` s. The ``monitoring`` module contains two base classes intended to provide common interfaces to building monitoring -patterns. The first is a ``MonitoringGroup``. This is little more than a ``ThunderbirdComponentResource`` that contains -a config dictionary. The purpose is to contain the resources involved in a monitoring solution. That is, alarms and a -notification setup. - -You should extend this class such that the resources returned by ``flatten`` are iterated over. If your module -understands that a resource it comes across can be monitored, the class should create alarms via an extension of the -second class, ``AlarmGroup``. This base class should be extended such that it creates alarms for a specific single -resource. For example, a single load balancer might have many different metrics being monitored. +patterns. The first is a :py:class:`tb_pulumi.monitoring.MonitoringGroup`. This is a +:py:class:`tb_pulumi.ThunderbirdComponentResource` which stores a configuration of overrides internally. It also +recursively unpacks and resolves any ``Output`` s in the resource stack. The purpose is twofold: + +- To contain and enumerate the resources which can be monitored that exist within the stack. +- To define the monitoring setup itself, including a method of notification. + +The second is a :py:class:`tb_pulumi.monitoring.AlarmGroup`. This class represents an overridable set of alarms for a +single resource. ``MonitoringGroup`` s must map resource types to ``AlarmGroup`` types that handle those resources in +their ``monitor`` functions. As an example, take a look at :py:class:`tb_pulumi.cloudwatch.CloudWatchMonitoringGroup`, a ``MonitoringGroup`` extension that uses AWS CloudWatch to alarm on metrics produced by AWS resources. It creates an diff --git a/pyproject.toml b/pyproject.toml index 01c0e41..4f69e94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,10 +19,6 @@ dev = [ "sphinx" ] -# Ruff -[tool.ruff] -line-length = 120 - # Exclude a variety of commonly ignored directories. exclude = [ ".eggs", @@ -36,26 +32,3 @@ exclude = [ # Always generate Python 3.12-compatible code. target-version = "py312" - -[tool.ruff.format] -# Prefer single quotes over double quotes. -quote-style = "single" - -[tool.ruff.lint] -# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default. -select = ["E", "F"] -ignore = [] - -# Allow autofix for all enabled rules (when `--fix`) is provided. -fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"] -unfixable = [] - -# Allow unused variables when underscore-prefixed. -dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" - -[tool.ruff.lint.flake8-quotes] -inline-quotes = "single" - -[tool.ruff.lint.mccabe] -# Unlike Flake8, default to a complexity level of 10. -max-complexity = 10 diff --git a/requirements.txt b/requirements.txt index 42a2efa..ca85253 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ boto3>=1.34,<2.0 cryptography>=43.0.0,<44.0 pulumi>=3.130.0,<4.0.0 -pulumi-aws==6.57.0 +pulumi-aws==6.65.0 pulumi-random>=4.16,<5.0 # pyyaml is also a requirement, but is installed for us by pulumi diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..941bf5c --- /dev/null +++ b/ruff.toml @@ -0,0 +1,23 @@ +line-length = 120 + +[format] +quote-style = "single" + +[lint] +# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default. +select = ["E", "F"] +ignore = [] + +# Allow autofix for all enabled rules (when `--fix`) is provided. +fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"] +unfixable = [] + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +[lint.flake8-quotes] +inline-quotes = "single" + +[lint.mccabe] +# Unlike Flake8, default to a complexity level of 10. +max-complexity = 10 \ No newline at end of file diff --git a/tb_pulumi/__init__.py b/tb_pulumi/__init__.py index 8a0d680..994c49b 100644 --- a/tb_pulumi/__init__.py +++ b/tb_pulumi/__init__.py @@ -108,6 +108,11 @@ class ThunderbirdComponentResource(pulumi.ComponentResource): :param project: The project this resource belongs to. :type project: :py:class:`tb_pulumi.ThunderbirdPulumiProject` + :param exclude_from_project: When ``True`` , this prevents this component resource from being registered directly + with the project. This does not prevent the component resource from being discovered by the project's + ``flatten`` function, provided that it is nested within some resource that is not excluded from the project. + :type exclude_from_project: bool, optional + :param opts: Additional ``pulumi.ResourceOptions`` to apply to this resource. Defaults to None. :type opts: pulumi.ResourceOptions, optional @@ -121,16 +126,17 @@ def __init__( pulumi_type: str, name: str, project: ThunderbirdPulumiProject, + exclude_from_project: bool = False, opts: pulumi.ResourceOptions = None, tags: dict = {}, ): self.name: str = name #: Identifier for this set of resources. self.project: ThunderbirdPulumiProject = project #: Project this resource is a member of. + self.exclude_from_project = exclude_from_project if self.protect_resources: pulumi.info( - f'Resource protection has been enabled on {name}. ' - 'To disable, export TBPULUMI_DISABLE_PROTECTION=True' + f'Resource protection has been enabled on {name}. To disable, export TBPULUMI_DISABLE_PROTECTION=True' ) # Merge provided opts with defaults before calling superconstructor @@ -145,19 +151,22 @@ def __init__( def finish(self, outputs: dict[str, Any], resources: dict[str, pulumi.Resource | list[pulumi.Resource]]): """Registers the provided ``outputs`` as Pulumi outputs for the module. Also stores the mapping of ``resources`` - internally as the ``resources`` member where they it be acted on collectively by a ``ThunderbirdPulumiProject``. - Any implementation of this class should call this function at the end of its ``__init__`` function to ensure its - state is properly represented. + internally as the ``resources`` member where they can be acted on collectively by a + ``ThunderbirdPulumiProject``. Any implementation of this class should call this function at the end of its + ``__init__`` function to ensure its state is properly represented. - Values in ``resources`` should be either a Resource or derivative (such as a ThunderbirdComponentResource) or a - list of such. + Values in ``resources`` should be either a Resource or derivative (such as a ThunderbirdComponentResource). + Alternatively, supply a list or dict of such. """ - # Register outputs both with the ThunderbirdPulumiProject and Pulumi itself + # Register resources internally; register outputs with Pulumi self.resources = resources - self.project.resources[self.name] = self.resources self.register_outputs(outputs) + # Register resources within the project if not excluded + if not self.exclude_from_project: + self.project.resources[self.name] = self.resources + @property def protect_resources(self) -> bool: """Determines whether resources should have protection against changes enabled based on the project's @@ -217,7 +226,7 @@ def env_var_is_true(name: str) -> bool: return env_var_matches(name, ['t', 'true', 'yes'], False) -def flatten(item: dict | list | ThunderbirdComponentResource | pulumi.Resource) -> set[pulumi.Resource]: +def flatten(item: dict | list | ThunderbirdComponentResource | pulumi.Output | pulumi.Resource) -> set[pulumi.Resource]: """Recursively traverses a nested collection of Pulumi ``Resource`` s, converting them into a flat set which can be more easily iterated over. @@ -236,13 +245,11 @@ def flatten(item: dict | list | ThunderbirdComponentResource | pulumi.Resource) if type(item) is list: to_flatten = item elif type(item) is dict: - to_flatten = [value for _, value in item.items()] + to_flatten = item.values() elif isinstance(item, ThunderbirdComponentResource): - to_flatten = [value for _, value in item.resources.items()] - elif isinstance(item, pulumi.Resource): + to_flatten = item.resources.values() + elif isinstance(item, pulumi.Resource) or isinstance(item, pulumi.Output): return [item] - else: - pass if to_flatten is not None: for item in to_flatten: diff --git a/tb_pulumi/cloudfront.py b/tb_pulumi/cloudfront.py index 4bcad5d..841bd0a 100644 --- a/tb_pulumi/cloudfront.py +++ b/tb_pulumi/cloudfront.py @@ -52,6 +52,9 @@ class CloudFrontS3Service(tb_pulumi.ThunderbirdComponentResource): :param opts: Additional pulumi.ResourceOptions to apply to these resources. Defaults to None. :type opts: pulumi.ResourceOptions, optional + + :param kwargs: Any other keyword arguments which will be passed as inputs to the ``ThunderbirdComponentResource`` + resource. """ def __init__( @@ -68,7 +71,7 @@ def __init__( opts: pulumi.ResourceOptions = None, **kwargs, ): - super().__init__('tb:cloudfront:CloudFrontS3Service', name=name, project=project, opts=opts) + super().__init__('tb:cloudfront:CloudFrontS3Service', name=name, project=project, opts=opts, **kwargs) # The function supports supplying the bucket policy at this time, but we have to have the CF distro built first. # For this reason, we build the bucket without the policy and attach the policy later on. diff --git a/tb_pulumi/cloudwatch.py b/tb_pulumi/cloudwatch.py index 1ce298d..4323735 100644 --- a/tb_pulumi/cloudwatch.py +++ b/tb_pulumi/cloudwatch.py @@ -39,32 +39,44 @@ def __init__( notify_emails: list[str] = [], opts: pulumi.ResourceOptions = None, ): - super().__init__( - pulumi_type='tb:cloudwatch:CloudWatchMonitoringGroup', name=name, project=project, opts=opts, config=config - ) - - supported_types = { - aws.lb.load_balancer.LoadBalancer: AlbAlarmGroup, + type_map = { + aws.lb.load_balancer.LoadBalancer: LoadBalancerAlarmGroup, aws.alb.target_group.TargetGroup: AlbTargetGroupAlarmGroup, aws.cloudfront.Distribution: CloudFrontDistributionAlarmGroup, aws.cloudfront.Function: CloudFrontFunctionAlarmGroup, aws.ecs.Service: EcsServiceAlarmGroup, } - supported_resources = [ - resource for resource in self.project.flatten() if type(resource) in supported_types.keys() - ] + + self.notify_emails = notify_emails + + super().__init__( + pulumi_type='tb:cloudwatch:CloudWatchMonitoringGroup', + name=name, + project=project, + type_map=type_map, + opts=opts, + config=config, + ) + + def monitor(self, outputs): + """This function gets called only after all outputs in the project have been resolved into values. It constructs + all monitors for the resources in this project. + + :param outputs: A list of resolved outputs discovered in the project. + :type outputs: list + """ sns_topic = aws.sns.Topic( - f'{name}-topic', name=f'{self.project.name_prefix}-alarms', opts=pulumi.ResourceOptions(parent=self) + f'{self.name}-topic', name=f'{self.project.name_prefix}-alarms', opts=pulumi.ResourceOptions(parent=self) ) # API details on SNS topic subscriptions can be found here: # https://docs.aws.amazon.com/sns/latest/api/API_Subscribe.html subscriptions = [] - for idx, email in enumerate(notify_emails): + for idx, email in enumerate(self.notify_emails): subscriptions.append( aws.sns.TopicSubscription( - f'{name}-snssub-{idx}', + f'{self.name}-snssub-{idx}', protocol='email', endpoint=email, topic=sns_topic.arn, @@ -76,12 +88,12 @@ def __init__( # The next two lines are useful for debugging monitoring setups since that logic depends largely on obscure # class names. These will show all resources and their classes in a project as well as a filtered list of # those resources correctly detected by the logic above. - # pulumi.info(f'All resources: {'\n'.join([str(res.__class__) for res in self.project.flatten()])}') + # pulumi.info(f'All resources: {'\n'.join([f'{res._name}: {str(res.__class__)}' for res in self.project.flatten()])}') # noqa: E501 # pulumi.info(f'Supported resources: {supported_resources}') - for res in supported_resources: + for res in self.supported_resources: shortname = res._name.replace(f'{self.project.name_prefix}-', '') # Make this name shorter, less redundant - alarms[res._name] = supported_types[type(res)]( - name=f'{name}-{shortname}', + alarms[res._name] = self.type_map[type(res)]( + name=f'{self.name}-{shortname}', project=self.project, resource=res, monitoring_group=self, @@ -94,6 +106,60 @@ def __init__( ) +class LoadBalancerAlarmGroup(tb_pulumi.monitoring.AlarmGroup): + """In AWS, a load balancer can have a handful of types: ``application`` , ``gateway`` , or ``network`` . The metrics + emitted by the load balancer - and therefore the kinds of alarms we can build - depend on which type it is. However, + all types are represented by the same class, ``aws.lb.load_balancer.LoadBalancer`` . This necessitates a class for + disambiguation. The ``load_balancer_type`` is an Output, so here we wait until we can determine that type, then + build the appropriate AlarmGroup class for the resource. + + :param name: The name of the alarm group resource. + :type name: str + + :param monitoring_group: The ``MonitoringGroup`` that this ``AlarmGroup`` belongs to. + :type monitoring_group: MonitoringGroup + + :param project: The ``ThunderbirdPulumiProject`` whose resources are being monitored. + :type project: tb_pulumi.ThunderbirdPulumiProject + + :param resource: The Pulumi ``Resource`` object this ``AlarmGroup`` is building alarms for. + :type resource: pulumi.Resource + + :param opts: Additional ``pulumi.ResourceOptions`` to apply to this resource. Defaults to None. + :type opts: pulumi.ResourceOptions, optional + """ + + def __init__( + self, + name: str, + project: tb_pulumi.ThunderbirdPulumiProject, + resource: aws.lb.load_balancer.LoadBalancer, + monitoring_group: CloudWatchMonitoringGroup, + opts: pulumi.ResourceOptions = None, + **kwargs, + ): + # Internalize the data so we can access it later when we know what LB type we're dealing with + self.name = name + self.project = project + self.resource = resource + self.monitoring_group = monitoring_group + self.opts = opts + self.kwargs = kwargs + + resource.load_balancer_type.apply(lambda lb_type: self.__build_alarm_group(lb_type)) + + def __build_alarm_group(self, lb_type: str): + if lb_type == 'application': + self.alarm_group = AlbAlarmGroup( + name=self.name, + project=self.project, + resource=self.resource, + monitoring_group=self.monitoring_group, + opts=self.opts, + **self.kwargs, + ) + + class AlbAlarmGroup(tb_pulumi.monitoring.AlarmGroup): """A set of alarms for Application Load Balancers. Contains the following configurable alarms: @@ -157,13 +223,13 @@ def __init__( alb_5xx = pulumi.Output.all(res_name=resource.name, res_suffix=resource.arn_suffix).apply( lambda outputs: aws.cloudwatch.MetricAlarm( f'{self.name}-alb5xx', - name=f'{outputs['res_name']}-alb5xx', + name=f'{outputs["res_name"]}-alb5xx', alarm_actions=[monitoring_group.resources['sns_topic'].arn], comparison_operator='GreaterThanOrEqualToThreshold', dimensions={'LoadBalancer': outputs['res_suffix']}, metric_name='HTTPCode_ELB_5XX_Count', namespace='AWS/ApplicationELB', - alarm_description=f'Elevated 5xx errors on ALB {outputs['res_name']}', + alarm_description=f'Elevated 5xx errors on ALB {outputs["res_name"]}', tags=alb_5xx_tags, opts=pulumi.ResourceOptions( parent=self, depends_on=[resource, monitoring_group.resources['sns_topic']] @@ -185,13 +251,13 @@ def __init__( target_5xx = pulumi.Output.all(res_name=resource.name, res_suffix=resource.arn_suffix).apply( lambda outputs: aws.cloudwatch.MetricAlarm( f'{self.name}-target5xx', - name=f'{outputs['res_name']}-target5xx', + name=f'{outputs["res_name"]}-target5xx', alarm_actions=[monitoring_group.resources['sns_topic'].arn], comparison_operator='GreaterThanOrEqualToThreshold', dimensions={'LoadBalancer': outputs['res_suffix']}, metric_name='HTTPCode_ELB_5XX_Count', namespace='AWS/ApplicationELB', - alarm_description=f'Elevated 5xx errors on the targets of ALB {outputs['res_name']}', + alarm_description=f'Elevated 5xx errors on the targets of ALB {outputs["res_name"]}', tags=target_5xx_tags, opts=pulumi.ResourceOptions( parent=self, depends_on=[resource, monitoring_group.resources['sns_topic']] @@ -213,13 +279,13 @@ def __init__( response_time = pulumi.Output.all(res_name=resource.name, res_suffix=resource.arn_suffix).apply( lambda outputs: aws.cloudwatch.MetricAlarm( f'{self.name}-responsetime', - name=f'{outputs['res_name']}-responsetime', + name=f'{outputs["res_name"]}-responsetime', alarm_actions=[monitoring_group.resources['sns_topic'].arn], comparison_operator='GreaterThanOrEqualToThreshold', dimensions={'LoadBalancer': outputs['res_suffix']}, metric_name='TargetResponseTime', namespace='AWS/ApplicationELB', - alarm_description=f'Average response time is over {response_time_opts['threshold']} second(s) for {response_time_opts['period']} seconds', # noqa: E501 + alarm_description=f'Average response time is over {response_time_opts["threshold"]} second(s) for {response_time_opts["period"]} seconds', # noqa: E501 tags=response_time_tags, opts=pulumi.ResourceOptions( parent=self, depends_on=[resource, monitoring_group.resources['sns_topic']] @@ -335,14 +401,14 @@ def __unhealthy_hosts_metric_alarm( return pulumi.Output.all(tg_suffix=tg_suffix, lb_suffix=lb_suffix).apply( lambda outputs: aws.cloudwatch.MetricAlarm( f'{self.name}-unhealthy-hosts', - name=f'{outputs['tg_suffix'].split('/')[1]}-{outputs['lb_suffix'].split('/')[1]}-unhealthy-hosts', + name=f'{outputs["tg_suffix"].split("/")[1]}-{outputs["lb_suffix"].split("/")[1]}-unhealthy-hosts', alarm_actions=[self.monitoring_group.resources['sns_topic'].arn], comparison_operator='GreaterThanOrEqualToThreshold', dimensions={'TargetGroup': outputs['tg_suffix'], 'LoadBalancer': outputs['lb_suffix']}, metric_name='UnHealthyHostCount', namespace='AWS/ApplicationELB', - alarm_description=f'{outputs['tg_suffix'].split('/')[1]} has detected unhealthy hosts in load balancer ' - f'{outputs['lb_suffix'].split('/')[1]}', + alarm_description=f'{outputs["tg_suffix"].split("/")[1]} has detected unhealthy hosts in load balancer ' + f'{outputs["lb_suffix"].split("/")[1]}', tags=tags, opts=pulumi.ResourceOptions( parent=self, depends_on=[target_group, self.monitoring_group.resources['sns_topic']] @@ -408,14 +474,14 @@ def __init__( distro_4xx = pulumi.Output.all(res_id=resource.id, res_comment=resource.comment).apply( lambda outputs: aws.cloudwatch.MetricAlarm( f'{self.name}-4xx', - name=f'{self.project.name_prefix}-cfdistro-{outputs['res_id']}-4xx', + name=f'{self.project.name_prefix}-cfdistro-{outputs["res_id"]}-4xx', alarm_actions=[monitoring_group.resources['sns_topic'].arn], comparison_operator='GreaterThanOrEqualToThreshold', dimensions={'DistributionId': outputs['res_id']}, metric_name='4xxErrorRate', namespace='AWS/CloudFront', - alarm_description=f'4xx error rate for CloudFront Distribution "{outputs['res_comment']}" exceeds ' - f'{distro_4xx_opts['threshold']} on average over {distro_4xx_opts['period']} seconds.', + alarm_description=f'4xx error rate for CloudFront Distribution "{outputs["res_comment"]}" exceeds ' + f'{distro_4xx_opts["threshold"]} on average over {distro_4xx_opts["period"]} seconds.', tags=distro_4xx_tags, opts=pulumi.ResourceOptions( parent=self, depends_on=[resource, monitoring_group.resources['sns_topic']] @@ -493,7 +559,7 @@ def __init__( metric_name='FunctionComputeUtilization', namespace='AWS/CloudFront', alarm_description=f'CPU utilization on CloudFront Function {res_name} exceeds ' - f'{cpu_utilization_opts['threshold']}.', + f'{cpu_utilization_opts["threshold"]}.', tags=cpu_utilization_tags, opts=pulumi.ResourceOptions( parent=self, depends_on=[resource, monitoring_group.resources['sns_topic']] @@ -566,7 +632,7 @@ def __init__( cpu_utilization = pulumi.Output.all(res_name=resource.name, cluster_arn=resource.cluster).apply( lambda outputs: aws.cloudwatch.MetricAlarm( f'{self.name}-cpu', - name=f'{outputs['res_name']}-cpu', + name=f'{outputs["res_name"]}-cpu', alarm_actions=[monitoring_group.resources['sns_topic'].arn], comparison_operator='GreaterThanOrEqualToThreshold', # There is no direct way to get the Cluster name from a Service, but we can get the ARN, which has the @@ -574,8 +640,8 @@ def __init__( dimensions={'ClusterName': outputs['cluster_arn'].split('/')[-1], 'ServiceName': outputs['res_name']}, metric_name='CPUUtilization', namespace='AWS/ECS', - alarm_description=f'CPU utilization on the {outputs['res_name']} cluster exceeds ' - f'{cpu_utilization_opts['threshold']}%', + alarm_description=f'CPU utilization on the {outputs["res_name"]} cluster exceeds ' + f'{cpu_utilization_opts["threshold"]}%', tags=cpu_utilization_tags, opts=pulumi.ResourceOptions( parent=self, depends_on=[resource, monitoring_group.resources['sns_topic']] @@ -599,7 +665,7 @@ def __init__( memory_utilization = pulumi.Output.all(res_name=resource.name, cluster_arn=resource.cluster).apply( lambda outputs: aws.cloudwatch.MetricAlarm( f'{self.name}-memory', - name=f'{outputs['res_name']}-memory', + name=f'{outputs["res_name"]}-memory', alarm_actions=[monitoring_group.resources['sns_topic'].arn], comparison_operator='GreaterThanOrEqualToThreshold', # There is no direct way to get the Cluster name from a Service, but we can get the ARN, which has the @@ -607,8 +673,8 @@ def __init__( dimensions={'ClusterName': outputs['cluster_arn'].split('/')[-1], 'ServiceName': outputs['res_name']}, metric_name='MemoryUtilization', namespace='AWS/ECS', - alarm_description=f'Memory utilization on the {outputs['res_name']} cluster exceeds ' - f'{memory_utilization_opts['threshold']}%', + alarm_description=f'Memory utilization on the {outputs["res_name"]} cluster exceeds ' + f'{memory_utilization_opts["threshold"]}%', tags=memory_utilization_tags, opts=pulumi.ResourceOptions( parent=self, depends_on=[resource, monitoring_group.resources['sns_topic']] diff --git a/tb_pulumi/ec2.py b/tb_pulumi/ec2.py index 4a90b41..f623483 100644 --- a/tb_pulumi/ec2.py +++ b/tb_pulumi/ec2.py @@ -50,6 +50,10 @@ class NetworkLoadBalancer(tb_pulumi.ThunderbirdComponentResource): :param opts: Additional pulumi.ResourceOptions to apply to these resources. Defaults to None. :type opts: pulumi.ResourceOptions, optional + :param tags: Key/value pairs to merge with the default tags which get applied to all resources in this group. + Defaults to {}. + :type tags: dict, optional + :param kwargs: Any other keyword arguments which will be passed as inputs to the LoadBalancer resource. A full listing of options is found `here `_. @@ -62,14 +66,23 @@ def __init__( listener_port: int, subnets: list[str], target_port: int, + exclude_from_project: bool = False, ingress_cidrs: list[str] = None, internal: bool = True, ips: list[str] = [], security_group_description: str = None, opts: pulumi.ResourceOptions = None, + tags: dict = {}, **kwargs, ): - super().__init__('tb:ec2:NetworkLoadBalancer', name, project, opts=opts) + super().__init__( + 'tb:ec2:NetworkLoadBalancer', + name=name, + project=project, + exclude_from_project=exclude_from_project, + opts=opts, + tags=tags, + ) # The primary_subnet is just the first subnet listed, used for determining VPC placement primary_subnet = subnets[0] @@ -77,8 +90,9 @@ def __init__( # Build a security group that allows ingress on our listener port security_group_with_rules = tb_pulumi.network.SecurityGroupWithRules( f'{name}-sg', - project, + project=project, vpc_id=primary_subnet.vpc_id, + exclude_from_project=True, rules={ 'ingress': [ { @@ -241,6 +255,7 @@ def __init__( f'{name}-sg', project, vpc_id=vpc_id, + exclude_from_project=True, rules={ 'ingress': [ { @@ -368,6 +383,7 @@ def __init__( project, secret_name=priv_secret, secret_value=self.resources['private_key'], + exclude_from_project=True, opts=pulumi.ResourceOptions(parent=self, depends_on=[private_key]), ) public_key_secret = tb_pulumi.secrets.SecretsManagerSecret( @@ -375,6 +391,7 @@ def __init__( project, secret_name=pub_secret, secret_value=self.resources['public_key'], + exclude_from_project=True, opts=pulumi.ResourceOptions(parent=self, depends_on=[public_key]), ) else: diff --git a/tb_pulumi/fargate.py b/tb_pulumi/fargate.py index 8514cc5..0fbbccf 100644 --- a/tb_pulumi/fargate.py +++ b/tb_pulumi/fargate.py @@ -259,6 +259,7 @@ def __init__( fsalb_name, project, subnets=subnets, + exclude_from_project=True, internal=internal, security_groups=security_groups, services=services, diff --git a/tb_pulumi/monitoring.py b/tb_pulumi/monitoring.py index 7d790d5..a7cd669 100644 --- a/tb_pulumi/monitoring.py +++ b/tb_pulumi/monitoring.py @@ -3,6 +3,7 @@ import pulumi import tb_pulumi +from abc import abstractclassmethod from functools import cached_property @@ -11,7 +12,7 @@ class MonitoringGroup(tb_pulumi.ThunderbirdComponentResource): be extended to provide specific monitoring solutions for the resources contained in the specified ``project``. :param pulumi_type: The "type" string (commonly referred to in docs as just ``t``) of the component as described - by `Pulumi's docs `_. + by `Pulumi's type docs `_. :type pulumi_type: str :param name: The name of the ``MonitoringGroup`` resource. @@ -20,25 +21,37 @@ class MonitoringGroup(tb_pulumi.ThunderbirdComponentResource): :param project: The ``ThunderbirdPulumiProject`` to build monitoring resources for. :type project: tb_pulumi.ThunderbirdPulumiProject - :param config: A configuration dictionary. The specific format and content of this dictionary should be defined by - classes extending this class. The dictionary should be configured in roughly the following way: - :: + :param type_map: A dict where the keys are ``pulumi.Resource`` derivatives representing types of resources this + monitoring group recognizes, and where the values are ``tb_pulumi.monitoring.AlarmGroup`` derivatives which + actually declare those monitors. For example, an ``aws.cloudfront.Distribution`` key might map to a + ``tb_pulumi.cloudwatch.CloudFrontDistributionAlarmGroup`` value. + :type type_map: dict[type, type] + + :param config: A configuration dictionary. The specific format and content of this dictionary is defined in part by + classes extending this class. However, the dictionary should be configured in the following broad way, with + downstream monitoring groups defining the specifics of the monitor configs: + + .. code-block:: javascript + :linenos: { "alarms": { "name-of-the-resource-being-monitored": { "monitor_name": { "enabled": False + // Downstream monitoring groups tell you what else goes right here } } } } - ``"alarms"`` should be a dictionary defining override settings for alarms. Its keys should be the names of - Pulumi resources being monitored, and their values should also be dictionaries. The alarm group will define - some alarms which can be tweaked. Refer to their documentation for details. All alarms should respond to a - boolean ``"enabled"`` value such that the alarm will not be created if this is ``False``. Beyond that, configure - each alarm as described in its alarm group documentation. Defaults to {}. + This config defines override settings for alarms whose default configurations are insufficient for a specific + use case. Since each resource can have multiple alarms associated with it, the ``"alarm"`` dict's keys should be + the names of Pulumi resources being monitored. Their values should also be dictionaries, and those keys should + be the names of the alarms as defined in the documentation for those alarm groups. + + All alarms should respond to a boolean ``"enabled"`` value such that the alarm will not be created if this is + ``False``. Beyond that, configure each alarm as described in its alarm group documentation. Defaults to {}. :type config: dict, optional :param opts: Additional ``pulumi.ResourceOptions`` to apply to this resource. Defaults to None. @@ -50,11 +63,110 @@ def __init__( pulumi_type: str, name: str, project: tb_pulumi.ThunderbirdPulumiProject, + type_map: dict, config: dict = {}, opts: pulumi.ResourceOptions = None, ): super().__init__(pulumi_type=pulumi_type, name=name, project=project, opts=opts) - self.config = config + self.config: dict = config + self.type_map: dict = type_map + + # Start with a list of all resources; sort them out into known and unknown things + _all_contents = self.project.flatten() + + #: All Pulumi Outputs in the project + self.all_outputs = [res for res in _all_contents if isinstance(res, pulumi.Output)] + + #: All items in the project which are already-resolved pulumi resources + self.all_resources = [res for res in _all_contents if not isinstance(res, pulumi.Output)] + + #: All resources in the project which have an entry in the type_map; this may contain Outputs, which will be + #: resolved by the time :py:meth:`tb_pulumi.monitoring.MonitoringGroup.monitor` is invoked. + self.supported_resources = [] + + def __parse_resource_item( + item: list | dict | pulumi.Output | pulumi.Resource | tb_pulumi.ThunderbirdComponentResource, + ): + """Not all items in a project's ``resources`` dict are actually Pulumi Resources. Sometimes we build + resources downstream of a Pulumi Output, which makes those resources (as they are known to the project) + actually Outputs and not recognizable resource types. We can only detect what kind of thing those Outputs + really are by asking from within code called inside an output's `apply` function. This necessitates an + unpacking process on this end of things to recursively resolve those Outputs into Resources that we can + build alarms around. + + This function processes and recursively "unpacks" an ``item`` , which could be any of the following things: + + - A Pulumi Resource that we may or may not be able to monitor. + - A ``tb_pulumi.ThunderbirdComponentResource`` that potentially contains other Resources or Outputs + in a potentially nested structure. + - A Pulumi Output that could represent either of the above things, or could be a collection of a + combination of those things. + - A list of any of the above items. + - A dict where the values could be any of the above items. + + Given one of these things, this function determines what kind of item it's dealing with and responds + appropriately to unpack and resolve the item. The function doesn't return any value, but instead manipulates + the internal resource listing directly, resulting in an ``all_resources`` list that includes the unpacked + and resolved Outputs. + + It is important to note that this listing is **eventually resolved**. Because this function deals in Pulumi + Outputs, the ``all_resources`` list **will still contain Outputs, even after running this function!** + However, ``all_resources`` will contain valid, resolved values when accessed from within a function that + relies upon the application of every item in ``all_outputs``, such as the ``monitor`` function. This is why + we build all monitoring resources inside of the ``monitor`` function. + """ + + if type(item) is list: + for i in item: + __parse_resource_item(i) + elif type(item) is dict: + for i in item.values(): + __parse_resource_item(i) + elif isinstance(item, tb_pulumi.ThunderbirdComponentResource): + __parse_resource_item(item.resources) + elif isinstance(item, pulumi.Resource): + self.all_resources.append(item) + elif isinstance(item, pulumi.Output): + item.apply(__parse_resource_item) + + # Expand and resolve all outputs using the above parsing function + for output in self.all_outputs: + __parse_resource_item(output) + + # When all outputs are applied, trigger the `on_apply` event. + pulumi.Output.all(*self.all_outputs).apply(lambda outputs: self.__on_apply(outputs)) + + def __on_apply(self, outputs): + """This function gets called only after all outputs in the project have been resolved into values. This + function should be considered to be a post-apply stage of the ``__init__`` function. + + :param outputs: A list of resolved outputs discovered in the project. + :type outputs: list + """ + + # From this side of an apply, we can see the resource types and look for ones we know about + self.supported_resources = [res for res in self.all_resources if type(res) in self.type_map.keys()] + + # Call downstream monitoring setups + self.monitor(outputs) + + @abstractclassmethod + def monitor(self, outputs): + """This function gets called after all of a project's outputs have been recursively unpacked and resolved, and + after this class's post-apply construction has completed. + + This is an abstract method which must be implemented by an inheriting class. That function should construct all + monitors for the supported resources in this project within this function. This function is essentially a + hand-off to an implementing class, an indicator that the project has been successfully parsed, and monitors can + now be built. + + Because this works as an extension of ``__init__``, the ``finish`` call for downstream monitoring groups should + be made from within this function instead of the constructor. + + :param outputs: A list of resolved outputs discovered in the project. + :type outputs: list + """ + raise NotImplementedError() class AlarmGroup(tb_pulumi.ThunderbirdComponentResource): diff --git a/tb_pulumi/network.py b/tb_pulumi/network.py index 8c168eb..454b9b2 100644 --- a/tb_pulumi/network.py +++ b/tb_pulumi/network.py @@ -179,6 +179,7 @@ def __init__( f'{name}-endpoint-sg', project, vpc_id=vpc.id, + exclude_from_project=True, rules={ 'ingress': [ { @@ -322,7 +323,7 @@ def __init__( rule.update({'type': 'ingress', 'security_group_id': sg.id}) ingress_rules.append( aws.ec2.SecurityGroupRule( - f'{name}-ingress-{rule['to_port']}', + f'{name}-ingress-{rule["to_port"]}', opts=pulumi.ResourceOptions(parent=self, depends_on=[sg]), **rule, ) @@ -333,7 +334,7 @@ def __init__( rule.update({'type': 'egress', 'security_group_id': sg.id}) egress_rules.append( aws.ec2.SecurityGroupRule( - f'{name}-egress-{rule['to_port']}', + f'{name}-egress-{rule["to_port"]}', opts=pulumi.ResourceOptions(parent=self, depends_on=[sg]), **rule, ) diff --git a/tb_pulumi/rds.py b/tb_pulumi/rds.py index 11b205c..758eb40 100644 --- a/tb_pulumi/rds.py +++ b/tb_pulumi/rds.py @@ -7,6 +7,7 @@ import tb_pulumi import tb_pulumi.ec2 import tb_pulumi.network +import tb_pulumi.secrets from tb_pulumi.constants import SERVICE_PORTS @@ -83,6 +84,11 @@ class RdsDatabaseGroup(tb_pulumi.ThunderbirdComponentResource): '15.7' :type engine_version: str, optional + :param exclude_from_project: When ``True`` , this prevents this component resource from being registered directly + with the project. This does not prevent the component resource from being discovered by the project's + ``flatten`` function, provided that it is nested within some resource that is not excluded from the project. + :type exclude_from_project: bool, optional + :param instance_class: One of the database sizes listed `in these docs `_. Defaults to 'db.t3.micro'. @@ -137,6 +143,11 @@ class RdsDatabaseGroup(tb_pulumi.ThunderbirdComponentResource): :param port: Specify a non-default listening port. Defaults to None. :type port: int, optional + :param secret_recovery_window_in_days: Number of days to retain the database_url secret after it has been deleted. + Set this to zero in testing environments to avoid issues during stack rebuilds. Defaults to None (which causes + AWS to default to 7 days). + :type secret_recovery_window_in_days: int, optional + :param sg_cidrs: A list of CIDRs from which ingress should be allowed. If this is left to the default value, a sensible default will be selected. If `internal` is True, this will allow access from the `vpc_cidr`. Otherwise, traffic will be allowed from anywhere. Defaults to None. @@ -150,6 +161,10 @@ class RdsDatabaseGroup(tb_pulumi.ThunderbirdComponentResource): Defaults to 'gp3'. :type storage_type: str, optional + :param tags: Key/value pairs to merge with the default tags which get applied to all resources in this group. + Defaults to {}. + :type tags: dict, optional + :param opts: Additional pulumi.ResourceOptions to apply to these resources. Defaults to None. :type opts: pulumi.ResourceOptions, optional @@ -179,6 +194,7 @@ def __init__( enabled_instance_cloudwatch_logs_exports: list[str] = [], engine: str = 'postgres', engine_version: str = '15.7', + exclude_from_project: bool = False, instance_class: str = 'db.t3.micro', internal: bool = True, jumphost_public_key: str = None, @@ -191,13 +207,17 @@ def __init__( parameter_group_family: str = 'postgres15', performance_insights_enabled: bool = False, port: int = None, + secret_recovery_window_in_days: int = None, sg_cidrs: list[str] = None, skip_final_snapshot: bool = False, storage_type: str = 'gp3', + tags: dict = {}, opts: pulumi.ResourceOptions = None, **kwargs, ): - super().__init__('tb:rds:RdsDatabaseGroup', name, project, opts=opts) + super().__init__( + 'tb:rds:RdsDatabaseGroup', name, project, exclude_from_project=exclude_from_project, opts=opts, tags=tags + ) # Generate a random password password = pulumi_random.RandomPassword( @@ -214,16 +234,14 @@ def __init__( # Store the password in Secrets Manager secret_fullname = f'{self.project.project}/{self.project.stack}/{name}/root_password' - secret = aws.secretsmanager.Secret( + secret = tb_pulumi.secrets.SecretsManagerSecret( f'{name}-secret', - name=secret_fullname, - opts=pulumi.ResourceOptions(parent=self), - ) - secret_version = aws.secretsmanager.SecretVersion( - f'{name}-secretversion', - secret_id=secret.id, - secret_string=password.result, - opts=pulumi.ResourceOptions(parent=self, depends_on=[secret, password]), + project=project, + exclude_from_project=True, + secret_name=secret_fullname, + secret_value=password.result, + recovery_window_in_days=secret_recovery_window_in_days, + opts=pulumi.ResourceOptions(parent=self, depends_on=[password]), ) # If no ingress CIDRs have been defined, find a reasonable default @@ -243,6 +261,7 @@ def __init__( f'{name}-sg', project, vpc_id=vpc_id, + exclude_from_project=True, rules={ 'ingress': [ { @@ -393,7 +412,29 @@ def __init__( port = SERVICE_PORTS.get(engine, 5432) inst_addrs = [instance.address for instance in instances] load_balancer = pulumi.Output.all(*inst_addrs).apply( - lambda addresses: self.__load_balancer(name, project, port, subnets, vpc_cidr, instances, *addresses) + lambda addresses: tb_pulumi.ec2.NetworkLoadBalancer( + f'{name}-nlb', + project=project, + exclude_from_project=True, + listener_port=port, + subnets=subnets, + target_port=port, + ingress_cidrs=[vpc_cidr], + internal=True, + ips=[socket.gethostbyname(addr) for addr in addresses], + security_group_description=f'Allow database traffic for {name}', + opts=pulumi.ResourceOptions(parent=self, depends_on=[*instances, *subnets]), + ) + ) + + ssm_param_db_read_host = load_balancer.apply( + lambda lb: aws.ssm.Parameter( + f'{name}-ssm-dbreadhost', + name=f'/{self.project.project}/{self.project.stack}/db-read-host', + type=aws.ssm.ParameterType.STRING, + value=lb.resources['nlb'].dns_name, + opts=pulumi.ResourceOptions(depends_on=[load_balancer]), + ) ) if build_jumphost: @@ -401,6 +442,7 @@ def __init__( f'{name}-jumphost', project, subnets[0].id, + exclude_from_project=True, kms_key_id=key.arn, public_key=jumphost_public_key, source_cidrs=jumphost_source_cidrs, @@ -419,42 +461,19 @@ def __init__( 'instances': instances, 'jumphost': jumphost if build_jumphost else None, 'key': key, - 'load_balancer': load_balancer['nlb'], + 'load_balancer': load_balancer, 'parameter_group': parameter_group, 'password': password, 'secret': secret, - 'secret_version': secret_version, 'security_group': security_group_with_rules, 'ssm_param_db_name': ssm_param_db_name, 'ssm_param_db_write_host': ssm_param_db_write_host, 'ssm_param_port': ssm_param_port, - 'ssm_param_read_host': load_balancer['ssm_param_read_host'], + 'ssm_param_read_host': ssm_param_db_read_host, 'subnet_group': subnet_group, }, ) - def __load_balancer(self, name, project, port, subnets, vpc_cidr, instances, *addresses): - # Build a load balancer - nlb = tb_pulumi.ec2.NetworkLoadBalancer( - f'{name}-nlb', - project, - port, - subnets, - port, - ingress_cidrs=[vpc_cidr], - internal=True, - ips=[socket.gethostbyname(addr) for addr in addresses], - security_group_description=f'Allow database traffic for {name}', - opts=pulumi.ResourceOptions(parent=self, depends_on=[*instances, *subnets]), - ) - ssm_param_read_host = self.__ssm_param( - f'{name}-ssm-dbreadhost', - f'/{self.project.project}/{self.project.stack}/db-read-host', - nlb.resources['nlb'].dns_name.apply(lambda dns_name: dns_name), - depends_on=[nlb], - ) - return {'nlb': nlb, 'ssm_param_read_host': ssm_param_read_host} - def __ssm_param(self, name, param_name, value, depends_on: list[pulumi.Output] = None): """Build an SSM Parameter.""" return aws.ssm.Parameter( diff --git a/tb_pulumi/secrets.py b/tb_pulumi/secrets.py index 712e360..133bea2 100644 --- a/tb_pulumi/secrets.py +++ b/tb_pulumi/secrets.py @@ -17,6 +17,11 @@ class SecretsManagerSecret(tb_pulumi.ThunderbirdComponentResource): :param project: The ThunderbirdPulumiProject to add these resources to. :type project: ThunderbirdPulumiProject + :param exclude_from_project: When ``True`` , this prevents this component resource from being registered directly + with the project. This does not prevent the component resource from being discovered by the project's + ``flatten`` function, provided that it is nested within some resource that is not excluded from the project. + :type exclude_from_project: bool, optional + :param secret_name: A slash ("/") delimited name for the secret in AWS. The last segment of this will be used as the "short name" for abbreviated references. :type name: str @@ -28,6 +33,10 @@ class SecretsManagerSecret(tb_pulumi.ThunderbirdComponentResource): :param opts: Additional pulumi.ResourceOptions to apply to these resources. Defaults to None. :type opts: pulumi.ResourceOptions, optional + :param tags: Key/value pairs to merge with the default tags which get applied to all resources in this group. + Defaults to {}. + :type tags: dict, optional + :param kwargs: Any other keyword arguments which will be passed as inputs to the ``aws.secretsmanager.Secret`` resource. """ @@ -38,13 +47,22 @@ def __init__( project: tb_pulumi.ThunderbirdPulumiProject, secret_name: str, secret_value: typing.Any, + exclude_from_project: bool = False, opts: pulumi.ResourceOptions = None, + tags: dict = {}, **kwargs, ): - super().__init__('tb:secrets:SecretsManagerSecret', name, project, opts=opts) + super().__init__( + 'tb:secrets:SecretsManagerSecret', + name, + project, + exclude_from_project=exclude_from_project, + opts=opts, + tags=tags, + ) secret = aws.secretsmanager.Secret( - f'{name}-secret', opts=pulumi.ResourceOptions(parent=self), name=secret_name, **kwargs + f'{name}-secret', opts=pulumi.ResourceOptions(parent=self), name=secret_name, tags=self.tags, **kwargs ) version = aws.secretsmanager.SecretVersion( @@ -87,9 +105,10 @@ def __init__( project: tb_pulumi.ThunderbirdPulumiProject, secret_names: list[str] = [], opts: pulumi.ResourceOptions = None, + tags: dict = {}, **kwargs, ): - super().__init__('tb:secrets:PulumiSecretsManager', name, project, opts=opts) + super().__init__('tb:secrets:PulumiSecretsManager', name, project, opts=opts, tags=tags) secrets = [] # First build the secrets @@ -104,7 +123,9 @@ def __init__( project=self.project, secret_name=secret_fullname, secret_value=secret_string, + exclude_from_project=True, opts=pulumi.ResourceOptions(parent=self), + tags=self.tags, **kwargs, ) secrets.append(secret)