Prefetch package artifacts and remote artifacts

closes #1148

CHANGES/1148.feature

@@ -0,0 +1 @@
+Improved performance by prefetching relevant Artifacts and RemoteArtifacts during publishing, reducing the number of database calls.

pulp_deb/app/serializers/content_serializers.py

@@ -466,7 +466,7 @@ def from822(cls, data, **kwargs):
         package_fields["custom_fields"] = custom_fields
         return cls(data=package_fields, **kwargs)
 
-    def to822(self, component=""):
+    def to822(self, component="", remote_artifact_dict=None):
         """Create deb822.Package object from model."""
         ret = deb822.Packages()
 
@@ -483,7 +483,10 @@ def to822(self, component=""):
             artifact = self.instance._artifacts.get()
             artifact.touch()  # Orphan cleanup protection until we are done!
         else:
-            artifact = RemoteArtifact.objects.filter(sha256=self.instance.sha256).first()
+            if remote_artifact_dict is not None:
+                artifact = remote_artifact_dict.get(self.instance.sha256)
+            else:
+                artifact = RemoteArtifact.objects.filter(sha256=self.instance.sha256).first()
 
         if artifact:
             ret["MD5sum"] = artifact.md5 if artifact.md5 else None

pulp_deb/app/tasks/publishing.py

@@ -18,6 +18,7 @@
 from pulpcore.plugin.models import (
     PublishedArtifact,
     PublishedMetadata,
+    RemoteArtifact,
     RepositoryVersion,
 )
 
@@ -147,8 +148,11 @@ def publish(
 
                 packages = Package.objects.filter(
                     pk__in=repo_version.content.order_by("-pulp_created")
-                )
-                release_helper.components[component].add_packages(packages)
+                ).prefetch_related("contentartifact_set", "_artifacts")
+                sha256_values = [package.sha256 for package in packages if package.sha256]
+                remote_artifacts = RemoteArtifact.objects.filter(sha256__in=sha256_values)
+                remote_artifact_dict = {artifact.sha256: artifact for artifact in remote_artifacts}
+                release_helper.components[component].add_packages(packages, remote_artifact_dict)
 
                 source_packages = SourcePackage.objects.filter(
                     pk__in=repo_version.content.order_by("-pulp_created"),
@@ -248,12 +252,21 @@ def publish(
                     )
 
                     for component in components:
-                        packages = [
-                            prc.package
-                            for prc in package_release_components
-                            if prc.release_component.component == component
-                        ]
-                        release_helper.components[component].add_packages(packages)
+                        packages = Package.objects.filter(
+                            pk__in=[
+                                prc.package.pk
+                                for prc in package_release_components
+                                if prc.release_component.component == component
+                            ]
+                        ).prefetch_related("contentartifact_set", "_artifacts")
+                        sha256_values = [package.sha256 for package in packages if package.sha256]
+                        remote_artifacts = RemoteArtifact.objects.filter(sha256__in=sha256_values)
+                        remote_artifact_dict = {
+                            artifact.sha256: artifact for artifact in remote_artifacts
+                        }
+                        release_helper.components[component].add_packages(
+                            packages, remote_artifact_dict
+                        )
 
                         source_packages = [
                             drc.source_package
@@ -311,7 +324,7 @@ def __init__(self, parent, component):
             source_index_path,
         )
 
-    def add_packages(self, packages):
+    def add_packages(self, packages, remote_artifact_dict):
         published_artifacts = []
         package_data = []
 
@@ -335,7 +348,7 @@ def add_packages(self, packages):
         for package, architecture in package_data:
             package_serializer = Package822Serializer(package, context={"request": None})
             try:
-                package_serializer.to822(self.component).dump(
+                package_serializer.to822(self.component, remote_artifact_dict).dump(
                     self.package_index_files[architecture][0]
                 )
             except KeyError: