Add mirror option to optimize sync

[noissue]

pulp_deb/app/tasks/synchronizing.py

@@ -76,6 +76,56 @@
 log = logging.getLogger(__name__)
 
 
+def get_distribution_release_file_artifact_set_sha256(distribution, remote):
+    log.info(_('Downloading Release file for distribution: "{}"').format(distribution))
+    if distribution[-1] == "/":
+        release_file_dir = distribution.strip("/")
+    else:
+        release_file_dir = os.path.join("dists", distribution)
+
+    release_file_info_serialized = {}
+    base_url = os.path.join(remote.url, release_file_dir)
+    for filename in ReleaseFile.SUPPORTED_ARTIFACTS:
+        url = os.path.join(base_url, filename)
+        downloader = remote.get_downloader(url=url)
+        try:
+            result = downloader.fetch()
+        except FileNotFoundError:
+            continue
+        sha256 = result.artifact_attributes["sha256"]
+        release_file_info_serialized[filename] = sha256
+
+    hash_string = ""
+    for filename, sha256 in release_file_info_serialized.items():
+        hash_string = hash_string + filename + "," + sha256 + "\n"
+
+    return hashlib.sha256(hash_string.encode("utf-8")).hexdigest()
+
+
+def get_previous_release_file(previous_version, distribution):
+    previous_release_file_qs = previous_version.get_content(
+        ReleaseFile.objects.filter(distribution=distribution)
+    )
+    if previous_release_file_qs.count() > 1:
+        message = "Previous ReleaseFile count: {}. There should only be one."
+        raise Exception(message.format(previous_release_file_qs.count()))
+    return previous_release_file_qs.first()
+
+
+def gen_remote_options(remote):
+    return {
+        "distributions": remote.distributions,
+        "components": remote.components,
+        "architectures": remote.architectures,
+        "policy": remote.policy,
+        "sync_sources": remote.sync_sources,
+        "sync_udebs": remote.sync_udebs,
+        "sync_installer": remote.sync_installer,
+        "gpgkey": remote.gpgkey,
+        "ignore_missing_package_indices": remote.ignore_missing_package_indices,
+    }
+
+
 class NoReleaseFile(Exception):
     """
     Exception to signal, that no file representing a release is present.
@@ -185,6 +235,37 @@ def synchronize(remote_pk, repository_pk, mirror, optimize):
     if not remote.url:
         raise ValueError(_("A remote must have a url specified to synchronize."))
 
+    skip_count = []
+    for dist in remote.distributions.split():
+        artifact_set_sha256 = get_distribution_release_file_artifact_set_sha256(dist, remote)
+        previous_release_file = get_previous_release_file(previous_repo_version, dist)
+        if (
+            previous_release_file
+            and previous_release_file.artifact_set_sha256 == artifact_set_sha256
+        ):
+            skip_count.append(True)
+        else:
+            skip_count.append(False)
+
+    remote_options = gen_remote_options(remote)
+    if not previous_repo_version.info:
+        diff_remote_options = False
+    elif previous_repo_version.info["remote_options"] == remote_options:
+        diff_remote_options = False
+    else:
+        diff_remote_options = True
+
+    should_skip = all(skip_count) and not diff_remote_options
+
+    if should_skip:
+        log.info("No change in ReleaseFiles detected. Skipping sync.")
+        with ProgressReport(
+            message="Skipping sync (no changes for any ReleaseFile)",
+            code="sync.complete_skip.was_skipped",
+        ) as pb:
+            asyncio.run(pb.aincrement())
+        return
+
     first_stage = DebFirstStage(remote, optimize, mirror, previous_repo_version)
     DebDeclarativeVersion(first_stage, repository, mirror=mirror).create()
 
@@ -564,12 +645,6 @@ def __init__(self, remote, optimize, mirror, previous_repo_version, *args, **kwa
             elif mirror and not previous_sync_info["sync_options"]["mirror"]:
                 log.info(_("Setting optimize=False since this sync switches to mirror=True."))
                 self.optimize = False
-            # TODO: https://github.com/pulp/pulp_deb/issues/631
-            if mirror:
-                log.info(_("Falling back to optimize=False behaviour since mirror=True is set!"))
-                log.info(_("See https://github.com/pulp/pulp_deb/issues/631 for more information."))
-                self.optimize = False
-                self.sync_info["sync_options"]["optimize"] = False
 
     async def run(self):
         """

pulp_deb/tests/functional/api/test_sync.py

@@ -16,6 +16,7 @@
     DEB_INSTALLER_SOURCE_FIXTURE_SUMMARY,
     DEB_REPORT_CODE_SKIP_PACKAGE,
     DEB_REPORT_CODE_SKIP_RELEASE,
+    DEB_REPORT_CODE_SKIP_COMPLETE,
     DEB_SIGNING_KEY,
 )
 from pulp_deb.tests.functional.utils import get_counts_from_content_summary
@@ -47,7 +48,7 @@ def test_sync(
 
     # Verify latest `repository_version` is 1 and sync was not skipped
     assert repo.latest_version_href.endswith("/1/")
-    assert not is_sync_skipped(task, DEB_REPORT_CODE_SKIP_RELEASE)
+    assert not is_sync_skipped(task, DEB_REPORT_CODE_SKIP_COMPLETE)
 
     # Verify that the repo content and added content matches the summary
     assert get_counts_from_content_summary(summary.present) == fixture_summary
@@ -60,7 +61,7 @@ def test_sync(
 
     # Verify that the latest `repository_version` is still 1 and sync was skipped
     assert repo.latest_version_href.endswith("/1/")
-    assert is_sync_skipped(task_skip, DEB_REPORT_CODE_SKIP_RELEASE)
+    assert is_sync_skipped(task_skip, DEB_REPORT_CODE_SKIP_COMPLETE)
 
     # Verify that the repo content still matches the summary
     assert get_counts_from_content_summary(summary.present) == fixture_summary
@@ -122,7 +123,7 @@ def test_sync_missing_package_indices(
         (
             DEB_FIXTURE_STANDARD_REPOSITORY_NAME,
             {"distributions": "no_dist"},
-            ["Could not find a Release file at"],
+            ["Not Found"],
         ),
         (
             DEB_FIXTURE_INVALID_REPOSITORY_NAME,
@@ -311,9 +312,7 @@ def test_sync_optimize_skip_unchanged_package_index(
 
 
 @pytest.mark.parallel
-def test_sync_optimize_switch_to_no_mirror(
-    deb_init_and_sync,
-):
+def test_sync_optimize_switch_to_no_mirror(deb_init_and_sync):
     """
     Test that when syncing a repo with mirror=True, and then re-syncing that repo with
     mirror=False, optimize=True, the releases will be skipped by optimize mode.
@@ -322,13 +321,30 @@ def test_sync_optimize_switch_to_no_mirror(
     sync_args = {"mirror": True}
     repo, remote, task = deb_init_and_sync(sync_args=sync_args, return_task=True)
     assert repo.latest_version_href.endswith("/1/")
-    assert not is_sync_skipped(task, DEB_REPORT_CODE_SKIP_RELEASE)
+    assert not is_sync_skipped(task, DEB_REPORT_CODE_SKIP_COMPLETE)
     sync_args = {"optimize": True, "mirror": False}
     repo, _, task = deb_init_and_sync(
         repository=repo, remote=remote, sync_args=sync_args, return_task=True
     )
     assert repo.latest_version_href.endswith("/1/")
-    assert is_sync_skipped(task, DEB_REPORT_CODE_SKIP_RELEASE)
+    assert is_sync_skipped(task, DEB_REPORT_CODE_SKIP_COMPLETE)
+
+
+@pytest.mark.parallel
+def test_sync_optimize_with_mirror_enabled(deb_init_and_sync):
+    """Test if enabling mirror sync option will skip syncing (optimize) on resync."""
+
+    sync_args = {"mirror": True}
+    repo, remote, task = deb_init_and_sync(sync_args=sync_args, return_task=True)
+    assert repo.latest_version_href.endswith("/1/")
+    assert not is_sync_skipped(task, DEB_REPORT_CODE_SKIP_COMPLETE)
+
+    # resync
+    repo, _, task = deb_init_and_sync(
+        repository=repo, remote=remote, sync_args=sync_args, return_task=True
+    )
+    assert repo.latest_version_href.endswith("/1/")
+    assert is_sync_skipped(task, DEB_REPORT_CODE_SKIP_COMPLETE)
 
 
 def test_sync_orphan_cleanup_fail(

pulp_deb/tests/functional/constants.py

@@ -213,6 +213,7 @@ def _clean_dict(d):
 
 DEB_FIXTURE_PACKAGE_COUNT = DEB_FIXTURE_SUMMARY.get(DEB_PACKAGE_NAME, 0)
 
+DEB_REPORT_CODE_SKIP_COMPLETE = "sync.complete_skip.was_skipped"
 DEB_REPORT_CODE_SKIP_RELEASE = "sync.release_file.was_skipped"
 DEB_REPORT_CODE_SKIP_PACKAGE = "sync.package_index.was_skipped"