Fix a number of issues related to project resource traversal

.github/workflows/validate.yml

@@ -45,15 +45,15 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Run Ruff syntax checks
-        uses: chartboost/ruff-action@v1
+        uses: astral-sh/ruff-action@v3
         with:
           src: './tb_pulumi'
 
       - name: Run Ruff linter
-        uses: chartboost/ruff-action@v1
+        uses: astral-sh/ruff-action@v3
         with:
           src: './tb_pulumi'
-          args: 'format --check'
+          args: 'format --check --diff'
 
   # Attempt to build the documentation. Fail if there are errors or warnings.
   build_docs:

docs/monitors.rst

@@ -4,17 +4,19 @@ When you use a ``ThunderbirdPulumiProject`` and add ``ThunderbirdComponentResour
 resources in an internal mapping correlating the name of the module to a collection of its resources. These resources
 can have complex structures with nested lists, dicts, and ``ThunderbirdComponentResource`` s. The project's
 :py:meth:`tb_pulumi.ThunderbirdPulumiProject.flatten` function returns these as a flat list of unlabeled Pulumi
-``Resource`` s.
+``Resource`` s and ``Output`` s.
 
 The ``monitoring`` module contains two base classes intended to provide common interfaces to building monitoring
-patterns. The first is a ``MonitoringGroup``. This is little more than a ``ThunderbirdComponentResource`` that contains
-a config dictionary. The purpose is to contain the resources involved in a monitoring solution. That is, alarms and a
-notification setup.
-
-You should extend this class such that the resources returned by ``flatten`` are iterated over. If your module
-understands that a resource it comes across can be monitored, the class should create alarms via an extension of the
-second class, ``AlarmGroup``. This base class should be extended such that it creates alarms for a specific single
-resource. For example, a single load balancer might have many different metrics being monitored.
+patterns. The first is a :py:class:`tb_pulumi.monitoring.MonitoringGroup`. This is a
+:py:class:`tb_pulumi.ThunderbirdComponentResource` which stores a configuration of overrides internally. It also
+recursively unpacks and resolves any ``Output`` s in the resource stack. The purpose is twofold:
+
+- To contain and enumerate the resources which can be monitored that exist within the stack.
+- To define the monitoring setup itself, including a method of notification.
+
+The second is a :py:class:`tb_pulumi.monitoring.AlarmGroup`. This class represents an overridable set of alarms for a
+single resource. ``MonitoringGroup`` s must map resource types to ``AlarmGroup`` types that handle those resources in
+their ``monitor`` functions.
 
 As an example, take a look at :py:class:`tb_pulumi.cloudwatch.CloudWatchMonitoringGroup`, a ``MonitoringGroup``
 extension that uses AWS CloudWatch to alarm on metrics produced by AWS resources. It creates an

pyproject.toml

@@ -19,10 +19,6 @@ dev = [
     "sphinx"
 ]
 
-# Ruff
-[tool.ruff]
-line-length = 120
-
 # Exclude a variety of commonly ignored directories.
 exclude = [
     ".eggs",
@@ -36,26 +32,3 @@ exclude = [
 
 # Always generate Python 3.12-compatible code.
 target-version = "py312"
-
-[tool.ruff.format]
-# Prefer single quotes over double quotes.
-quote-style = "single"
-
-[tool.ruff.lint]
-# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
-select = ["E", "F"]
-ignore = []
-
-# Allow autofix for all enabled rules (when `--fix`) is provided.
-fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
-unfixable = []
-
-# Allow unused variables when underscore-prefixed.
-dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
-
-[tool.ruff.lint.flake8-quotes]
-inline-quotes = "single"
-
-[tool.ruff.lint.mccabe]
-# Unlike Flake8, default to a complexity level of 10.
-max-complexity = 10

requirements.txt

@@ -1,6 +1,6 @@
 boto3>=1.34,<2.0
 cryptography>=43.0.0,<44.0
 pulumi>=3.130.0,<4.0.0
-pulumi-aws==6.57.0
+pulumi-aws==6.65.0
 pulumi-random>=4.16,<5.0
 # pyyaml is also a requirement, but is installed for us by pulumi

ruff.toml

@@ -0,0 +1,23 @@
+line-length = 120
+
+[format]
+quote-style = "single"
+
+[lint]
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+select = ["E", "F"]
+ignore = []
+
+# Allow autofix for all enabled rules (when `--fix`) is provided.
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+unfixable = []
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+[lint.flake8-quotes]
+inline-quotes = "single"
+
+[lint.mccabe]
+# Unlike Flake8, default to a complexity level of 10.
+max-complexity = 10

tb_pulumi/__init__.py

@@ -108,6 +108,11 @@ class ThunderbirdComponentResource(pulumi.ComponentResource):
     :param project: The project this resource belongs to.
     :type project: :py:class:`tb_pulumi.ThunderbirdPulumiProject`
 
+    :param exclude_from_project: When ``True`` , this prevents this component resource from being registered directly
+        with the project. This does not prevent the component resource from being discovered by the project's
+        ``flatten`` function, provided that it is nested within some resource that is not excluded from the project.
+    :type exclude_from_project: bool, optional
+
     :param opts: Additional ``pulumi.ResourceOptions`` to apply to this resource. Defaults to None.
     :type opts: pulumi.ResourceOptions, optional
 
@@ -121,16 +126,17 @@ def __init__(
         pulumi_type: str,
         name: str,
         project: ThunderbirdPulumiProject,
+        exclude_from_project: bool = False,
         opts: pulumi.ResourceOptions = None,
         tags: dict = {},
     ):
         self.name: str = name  #: Identifier for this set of resources.
         self.project: ThunderbirdPulumiProject = project  #: Project this resource is a member of.
+        self.exclude_from_project = exclude_from_project
 
         if self.protect_resources:
             pulumi.info(
-                f'Resource protection has been enabled on {name}. '
-                'To disable, export TBPULUMI_DISABLE_PROTECTION=True'
+                f'Resource protection has been enabled on {name}. To disable, export TBPULUMI_DISABLE_PROTECTION=True'
             )
 
         # Merge provided opts with defaults before calling superconstructor
@@ -145,19 +151,22 @@ def __init__(
 
     def finish(self, outputs: dict[str, Any], resources: dict[str, pulumi.Resource | list[pulumi.Resource]]):
         """Registers the provided ``outputs`` as Pulumi outputs for the module. Also stores the mapping of ``resources``
-        internally as the ``resources`` member where they it be acted on collectively by a ``ThunderbirdPulumiProject``.
-        Any implementation of this class should call this function at the end of its ``__init__`` function to ensure its
-        state is properly represented.
+        internally as the ``resources`` member where they can be acted on collectively by a
+        ``ThunderbirdPulumiProject``. Any implementation of this class should call this function at the end of its
+        ``__init__`` function to ensure its state is properly represented.
 
-        Values in ``resources`` should be either a Resource or derivative (such as a ThunderbirdComponentResource) or a
-        list of such.
+        Values in ``resources`` should be either a Resource or derivative (such as a ThunderbirdComponentResource).
+        Alternatively, supply a list or dict of such.
         """
 
-        # Register outputs both with the ThunderbirdPulumiProject and Pulumi itself
+        # Register resources internally; register outputs with Pulumi
         self.resources = resources
-        self.project.resources[self.name] = self.resources
         self.register_outputs(outputs)
 
+        # Register resources within the project if not excluded
+        if not self.exclude_from_project:
+            self.project.resources[self.name] = self.resources
+
     @property
     def protect_resources(self) -> bool:
         """Determines whether resources should have protection against changes enabled based on the project's
@@ -217,7 +226,7 @@ def env_var_is_true(name: str) -> bool:
     return env_var_matches(name, ['t', 'true', 'yes'], False)
 
 
-def flatten(item: dict | list | ThunderbirdComponentResource | pulumi.Resource) -> set[pulumi.Resource]:
+def flatten(item: dict | list | ThunderbirdComponentResource | pulumi.Output | pulumi.Resource) -> set[pulumi.Resource]:
     """Recursively traverses a nested collection of Pulumi ``Resource`` s, converting them into a flat set which can be
     more easily iterated over.
 
@@ -236,13 +245,11 @@ def flatten(item: dict | list | ThunderbirdComponentResource | pulumi.Resource)
     if type(item) is list:
         to_flatten = item
     elif type(item) is dict:
-        to_flatten = [value for _, value in item.items()]
+        to_flatten = item.values()
     elif isinstance(item, ThunderbirdComponentResource):
-        to_flatten = [value for _, value in item.resources.items()]
-    elif isinstance(item, pulumi.Resource):
+        to_flatten = item.resources.values()
+    elif isinstance(item, pulumi.Resource) or isinstance(item, pulumi.Output):
         return [item]
-    else:
-        pass
 
     if to_flatten is not None:
         for item in to_flatten:

tb_pulumi/cloudfront.py

@@ -52,6 +52,9 @@ class CloudFrontS3Service(tb_pulumi.ThunderbirdComponentResource):
 
     :param opts: Additional pulumi.ResourceOptions to apply to these resources. Defaults to None.
     :type opts: pulumi.ResourceOptions, optional
+
+    :param kwargs: Any other keyword arguments which will be passed as inputs to the ``ThunderbirdComponentResource``
+        resource.
     """
 
     def __init__(
@@ -68,7 +71,7 @@ def __init__(
         opts: pulumi.ResourceOptions = None,
         **kwargs,
     ):
-        super().__init__('tb:cloudfront:CloudFrontS3Service', name=name, project=project, opts=opts)
+        super().__init__('tb:cloudfront:CloudFrontS3Service', name=name, project=project, opts=opts, **kwargs)
 
         # The function supports supplying the bucket policy at this time, but we have to have the CF distro built first.
         # For this reason, we build the bucket without the policy and attach the policy later on.