Merge pull request #3973 from opsmill/bab-merge-stable-into-develop

Merge stable into develop

.github/workflows/update-compose-file.yml

@@ -29,6 +29,8 @@ jobs:
     steps:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
+        with:
+          token: ${{ secrets.GH_INFRAHUB_BOT_TOKEN }}
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -37,15 +39,13 @@ jobs:
         run: |
           pipx install poetry
           poetry config virtualenvs.prefer-active-python true
-          pip install invoke toml
       - name: "Install Package"
         run: "poetry install --all-extras"
 
       - name: "Update Docker Env variable in docker-compose.yml file"
-        run: "invoke dev.gen-config-env -u"
+        run: "poetry run invoke dev.gen-config-env -u"
       - name: "Update Infrahub Image Version in docker-compose.yml file"
-        run: "invoke dev.update-docker-compose"
-
+        run: "poetry run invoke dev.update-docker-compose"
       - name: Commit docker-compose.yml
         uses: github-actions-x/[email protected]
         with:

backend/infrahub/api/schema.py

@@ -23,7 +23,7 @@
 from infrahub.core.schema_manager import SchemaBranch, SchemaNamespace, SchemaUpdateValidationResult  # noqa: TCH001
 from infrahub.core.validators.checker import schema_validators_checker
 from infrahub.database import InfrahubDatabase  # noqa: TCH001
-from infrahub.exceptions import MigrationError, PermissionDeniedError
+from infrahub.exceptions import MigrationError
 from infrahub.log import get_logger
 from infrahub.message_bus import Meta, messages
 from infrahub.services import services
@@ -85,7 +85,7 @@ class SchemaLoadAPI(SchemaRoot):
     version: str
 
 
-class SchemasLoadAPI(SchemaRoot):
+class SchemasLoadAPI(BaseModel):
     schemas: list[SchemaLoadAPI]
 
 
@@ -244,7 +244,7 @@ async def load_schema(
         errors += schema.validate_namespaces()
 
     if errors:
-        raise PermissionDeniedError(", ".join(errors))
+        raise SchemaNotValidError(message=", ".join(errors))
 
     async with lock.registry.global_schema_lock():
         branch_schema = registry.schema.get_schema_branch(name=branch.name)
@@ -330,7 +330,7 @@ async def check_schema(
         errors += schema.validate_namespaces()
 
     if errors:
-        raise PermissionDeniedError(", ".join(errors))
+        raise SchemaNotValidError(message=", ".join(errors))
 
     branch_schema = registry.schema.get_schema_branch(name=branch.name)
 

backend/infrahub/core/manager.py

@@ -276,6 +276,7 @@ async def query_peers(
         at: Optional[Union[Timestamp, str]] = None,
         branch: Optional[Union[Branch, str]] = None,
         branch_agnostic: bool = False,
+        fetch_peers: bool = False,
     ) -> list[Relationship]:
         branch = await registry.get_branch(branch=branch, db=db)
         at = Timestamp(at)
@@ -308,16 +309,26 @@ async def query_peers(
             if display_label_fields:
                 fields = deep_merge_dict(dicta=fields, dictb=display_label_fields)
 
-        return [
-            await Relationship(schema=schema, branch=branch, at=at, node_id=peer.source_id).load(
+        if fetch_peers:
+            peer_ids = [peer.peer_id for peer in peers_info]
+            peer_nodes = await cls.get_many(
+                db=db, ids=peer_ids, fields=fields, at=at, branch=branch, branch_agnostic=branch_agnostic
+            )
+
+        results = []
+        for peer in peers_info:
+            result = await Relationship(schema=schema, branch=branch, at=at, node_id=peer.source_id).load(
                 db=db,
                 id=peer.rel_node_id,
                 db_id=peer.rel_node_db_id,
                 updated_at=peer.updated_at,
                 data=peer,
             )
-            for peer in peers_info
-        ]
+            if fetch_peers:
+                await result.set_peer(value=peer_nodes[peer.peer_id])
+            results.append(result)
+
+        return results
 
     @classmethod
     async def count_hierarchy(

backend/infrahub/graphql/resolver.py

@@ -79,6 +79,7 @@ async def default_resolver(*args, **kwargs):
             at=context.at,
             branch=context.branch,
             branch_agnostic=node_rel.branch is BranchSupportType.AGNOSTIC,
+            fetch_peers=True,
         )
 
         if node_rel.cardinality == "many":
@@ -134,6 +135,7 @@ async def single_relationship_resolver(parent: dict, info: GraphQLResolveInfo, *
             at=context.at,
             branch=context.branch,
             branch_agnostic=node_rel.branch is BranchSupportType.AGNOSTIC,
+            fetch_peers=True,
         )
 
         if not objs:
@@ -231,6 +233,7 @@ async def many_relationship_resolver(
             at=context.at,
             branch=context.branch,
             branch_agnostic=node_rel.branch is BranchSupportType.AGNOSTIC,
+            fetch_peers=True,
         )
 
         if not objs:

backend/infrahub/message_bus/messages/__init__.py

@@ -22,6 +22,12 @@
 from .git_repository_merge import GitRepositoryMerge
 from .git_repository_read_only_add import GitRepositoryAddReadOnly
 from .git_repository_read_only_pull import GitRepositoryPullReadOnly
+from .proposed_change.request_proposedchange_dataintegrity import RequestProposedChangeDataIntegrity
+from .proposed_change.request_proposedchange_refreshartifacts import RequestProposedChangeRefreshArtifacts
+from .proposed_change.request_proposedchange_repositorychecks import RequestProposedChangeRepositoryChecks
+from .proposed_change.request_proposedchange_rungenerators import RequestProposedChangeRunGenerators
+from .proposed_change.request_proposedchange_runtests import RequestProposedChangeRunTests
+from .proposed_change.request_proposedchange_schemaintegrity import RequestProposedChangeSchemaIntegrity
 from .refresh_registry_branches import RefreshRegistryBranches
 from .refresh_registry_rebasedbranch import RefreshRegistryRebasedBranch
 from .refresh_webhook_configuration import RefreshWebhookConfiguration
@@ -35,13 +41,7 @@
 from .request_git_sync import RequestGitSync
 from .request_graphqlquerygroup_update import RequestGraphQLQueryGroupUpdate
 from .request_proposed_change_cancel import RequestProposedChangeCancel
-from .request_proposedchange_dataintegrity import RequestProposedChangeDataIntegrity
 from .request_proposedchange_pipeline import RequestProposedChangePipeline
-from .request_proposedchange_refreshartifacts import RequestProposedChangeRefreshArtifacts
-from .request_proposedchange_repositorychecks import RequestProposedChangeRepositoryChecks
-from .request_proposedchange_rungenerators import RequestProposedChangeRunGenerators
-from .request_proposedchange_runtests import RequestProposedChangeRunTests
-from .request_proposedchange_schemaintegrity import RequestProposedChangeSchemaIntegrity
 from .request_repository_checks import RequestRepositoryChecks
 from .request_repository_userchecks import RequestRepositoryUserChecks
 from .schema_migration_path import SchemaMigrationPath, SchemaMigrationPathResponse

backend/infrahub/message_bus/messages/request_proposedchange_refreshartifacts.py → backend/infrahub/message_bus/messages/proposed_change/base_with_diff.py

@@ -4,7 +4,7 @@
 from infrahub.message_bus.types import ProposedChangeBranchDiff
 
 
-class RequestProposedChangeRefreshArtifacts(InfrahubMessage):
+class BaseProposedChangeWithDiffMessage(InfrahubMessage):
     """Sent trigger the refresh of artifacts that are impacted by the proposed change."""
 
     model_config = ConfigDict(arbitrary_types_allowed=True)

backend/infrahub/message_bus/messages/proposed_change/request_proposedchange_dataintegrity.py

@@ -0,0 +1,5 @@
+from .base_with_diff import BaseProposedChangeWithDiffMessage
+
+
+class RequestProposedChangeDataIntegrity(BaseProposedChangeWithDiffMessage):
+    """Sent trigger data integrity checks for a proposed change"""

backend/infrahub/message_bus/messages/proposed_change/request_proposedchange_refreshartifacts.py

@@ -0,0 +1,5 @@
+from .base_with_diff import BaseProposedChangeWithDiffMessage
+
+
+class RequestProposedChangeRefreshArtifacts(BaseProposedChangeWithDiffMessage):
+    """Sent trigger the refresh of artifacts that are impacted by the proposed change."""

backend/infrahub/message_bus/messages/proposed_change/request_proposedchange_repositorychecks.py

@@ -0,0 +1,5 @@
+from .base_with_diff import BaseProposedChangeWithDiffMessage
+
+
+class RequestProposedChangeRepositoryChecks(BaseProposedChangeWithDiffMessage):
+    """Sent when a proposed change is created to trigger additional checks"""

backend/infrahub/message_bus/messages/proposed_change/request_proposedchange_rungenerators.py

@@ -0,0 +1,12 @@
+from pydantic import Field
+
+from .base_with_diff import BaseProposedChangeWithDiffMessage
+
+
+class RequestProposedChangeRunGenerators(BaseProposedChangeWithDiffMessage):
+    """Sent trigger the generators that are impacted by the proposed change to run."""
+
+    refresh_artifacts: bool = Field(..., description="Whether to regenerate artifacts after the generators are run")
+    do_repository_checks: bool = Field(
+        ..., description="Whether to run repository and user checks after the generators are run"
+    )

backend/infrahub/message_bus/messages/proposed_change/request_proposedchange_runtests.py

@@ -0,0 +1,5 @@
+from .base_with_diff import BaseProposedChangeWithDiffMessage
+
+
+class RequestProposedChangeRunTests(BaseProposedChangeWithDiffMessage):
+    """Sent trigger to run tests (smoke, units, integrations) for a proposed change."""

backend/infrahub/message_bus/messages/proposed_change/request_proposedchange_schemaintegrity.py

@@ -0,0 +1,5 @@
+from .base_with_diff import BaseProposedChangeWithDiffMessage
+
+
+class RequestProposedChangeSchemaIntegrity(BaseProposedChangeWithDiffMessage):
+    """Sent trigger schema integrity checks for a proposed change"""

backend/infrahub/message_bus/operations/requests/proposed_change.py

@@ -145,7 +145,7 @@ async def pipeline(message: messages.RequestProposedChangePipeline, service: Inf
         branch_diff = ProposedChangeBranchDiff(diff_summary=diff_summary, repositories=repositories)
         await _populate_subscribers(branch_diff=branch_diff, service=service, branch=message.source_branch)
 
-        if message.check_type in [CheckType.ALL, CheckType.ARTIFACT]:
+        if message.check_type is CheckType.ARTIFACT:
             await task_report.info("Adding Refresh Artifact job", proposed_change=message.proposed_change)
             events.append(
                 messages.RequestProposedChangeRefreshArtifacts(
@@ -166,6 +166,8 @@ async def pipeline(message: messages.RequestProposedChangePipeline, service: Inf
                     source_branch_sync_with_git=message.source_branch_sync_with_git,
                     destination_branch=message.destination_branch,
                     branch_diff=branch_diff,
+                    refresh_artifacts=message.check_type is CheckType.ALL,
+                    do_repository_checks=message.check_type is CheckType.ALL,
                 )
             )
 
@@ -183,7 +185,7 @@ async def pipeline(message: messages.RequestProposedChangePipeline, service: Inf
                 )
             )
 
-        if message.check_type in [CheckType.ALL, CheckType.REPOSITORY, CheckType.USER]:
+        if message.check_type in [CheckType.REPOSITORY, CheckType.USER]:
             await task_report.info("Adding Repository Check job", proposed_change=message.proposed_change)
             events.append(
                 messages.RequestProposedChangeRepositoryChecks(
@@ -458,6 +460,35 @@ async def run_generators(message: messages.RequestProposedChangeRunGenerators, s
                 msg.assign_meta(parent=message)
                 await service.send(message=msg)
 
+    next_messages: list[InfrahubMessage] = []
+    if message.refresh_artifacts:
+        await task_report.info("Adding Refresh Artifact job", proposed_change=message.proposed_change)
+        next_messages.append(
+            messages.RequestProposedChangeRefreshArtifacts(
+                proposed_change=message.proposed_change,
+                source_branch=message.source_branch,
+                source_branch_sync_with_git=message.source_branch_sync_with_git,
+                destination_branch=message.destination_branch,
+                branch_diff=message.branch_diff,
+            )
+        )
+
+    if message.do_repository_checks:
+        await task_report.info("Adding Repository Check job", proposed_change=message.proposed_change)
+        next_messages.append(
+            messages.RequestProposedChangeRepositoryChecks(
+                proposed_change=message.proposed_change,
+                source_branch=message.source_branch,
+                source_branch_sync_with_git=message.source_branch_sync_with_git,
+                destination_branch=message.destination_branch,
+                branch_diff=message.branch_diff,
+            )
+        )
+
+    for next_msg in next_messages:
+        next_msg.assign_meta(parent=message)
+        await service.send(message=next_msg)
+
 
 GATHER_ARTIFACT_DEFINITIONS = """
 query GatherArtifactDefinitions {

backend/tests/helpers/cli.py

@@ -0,0 +1,6 @@
+import re
+
+
+def remove_ansi_color(text: str) -> str:
+    ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
+    return ansi_escape.sub("", text)