diff --git a/.github/workflows/scripts/post_before_script.sh b/.github/workflows/scripts/post_before_script.sh new file mode 100644 index 0000000..ab9a2b9 --- /dev/null +++ b/.github/workflows/scripts/post_before_script.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +# Install gem client before running the test +SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "stream" "generate-bindings" "lowerbounds") +if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then + cmd_prefix dnf install -yq gem +fi diff --git a/CHANGES/94.feature b/CHANGES/94.feature new file mode 100644 index 0000000..8369856 --- /dev/null +++ b/CHANGES/94.feature @@ -0,0 +1 @@ +Added support for pull-through caching. Add a remote to a distribution to enable this feature. diff --git a/pulp_gem/app/models.py b/pulp_gem/app/models.py index 5440536..afc29e1 100644 --- a/pulp_gem/app/models.py +++ b/pulp_gem/app/models.py @@ -1,8 +1,11 @@ from logging import getLogger -from django.db import models +from django.db import models, utils +from pathlib import PurePath +from tempfile import NamedTemporaryFile from pulpcore.plugin.models import ( + Artifact, Content, Publication, Distribution, @@ -10,6 +13,8 @@ Repository, ) +from pulp_gem.specs import analyse_gem + log = getLogger(__name__) @@ -42,6 +47,28 @@ def gemspec_path(self): """The path for this gem's gemspec for the content app.""" return f"quick/Marshal.4.8/{self.name}-{self.version}.gemspec.rz" + @staticmethod + def init_from_artifact_and_relative_path(artifact, relative_path): + """""" + name, version, spec_data = analyse_gem(artifact.file) + content = GemContent(name=name, version=version) + relative_path = content.relative_path + + with NamedTemporaryFile(mode="wb", dir=".", delete=False) as temp_file: + temp_file.write(spec_data) + temp_file.flush() + + spec_artifact = Artifact.init_and_validate(temp_file.name) + try: + spec_artifact.save() + except utils.IntegrityError: + spec_artifact = Artifact.objects.get(spec_artifact.q()) + spec_artifact.touch() + spec_relative_path = content.gemspec_path + + artifacts = {relative_path: artifact, spec_relative_path: spec_artifact} + return content, artifacts + class Meta: default_related_name = "%(app_label)s_%(model_name)s" unique_together = ("name", "version") @@ -76,6 +103,20 @@ class GemRemote(Remote): TYPE = "gem" + def get_remote_artifact_content_type(self, relative_path=None): + """ + Return a modified GemContent class that has a reference to this remote. + + This will ensure that GemContent.init_from_artifact_and_relative_path can properly create + the Remote Artifact for the second Artifact it needs whether that be the gem file or the + gemspec. + """ + if relative_path: + path = PurePath(relative_path) + if path.match("gems/*.gem"): + return GemContent + return None + class Meta: default_related_name = "%(app_label)s_%(model_name)s" diff --git a/pulp_gem/app/serializers.py b/pulp_gem/app/serializers.py index 83eb978..67b0208 100644 --- a/pulp_gem/app/serializers.py +++ b/pulp_gem/app/serializers.py @@ -195,7 +195,14 @@ class GemDistributionSerializer(DistributionSerializer): queryset=Publication.objects.exclude(complete=False), allow_null=True, ) + remote = DetailRelatedField( + required=False, + help_text=_("Remote that can be used to fetch content when using pull-through caching."), + view_name_pattern=r"remotes(-.*/.*)?-detail", + queryset=Remote.objects.all(), + allow_null=True, + ) class Meta: - fields = DistributionSerializer.Meta.fields + ("publication",) + fields = DistributionSerializer.Meta.fields + ("publication", "remote") model = GemDistribution diff --git a/pulp_gem/tests/functional/api/test_full_mirror.py b/pulp_gem/tests/functional/api/test_full_mirror.py new file mode 100644 index 0000000..7693c91 --- /dev/null +++ b/pulp_gem/tests/functional/api/test_full_mirror.py @@ -0,0 +1,77 @@ +import pytest +import subprocess +from aiohttp.client_exceptions import ClientResponseError + + +def test_pull_through_metadata( + gem_remote_factory, + gem_distribution_factory, + gem_content_api_client, + artifacts_api_client, + http_get, +): + """ + Test that pull-through caching can retrieve metadata files upstream, but does not save them. + """ + artifacts_before = artifacts_api_client.list().count + content_before = gem_content_api_client.list().count + + # Choose a remote source that supports all the different gem repository metadata formats + remote = gem_remote_factory(url="https://rubygems.org") + distribution = gem_distribution_factory(remote=remote.pulp_href) + + urls = [ + "info/pulp_file_client", + "specs.4.8", + "latest_specs.4.8", + "prerelease_specs.4.8", + "quick/Marshal.4.8/pulp_file_client-1.14.0.gemspec.rz", + ] + for path in urls: + url = distribution.base_url + path + assert http_get(url) + + artifacts_after = artifacts_api_client.list().count + content_after = gem_content_api_client.list().count + + assert artifacts_before == artifacts_after + assert content_before == content_after + + with pytest.raises(ClientResponseError) as e: + http_get(distribution.base_url + "NOT_A_VALID_LINK") + + assert e.value.status == 404 + + +def test_pull_through_install( + gem_remote_factory, gem_distribution_factory, gem_content_api_client, delete_orphans_pre +): + """ + Test that gem clients can install from a distribution with pull-through caching. + """ + out = subprocess.run(("which", "gem")) + if out.returncode != 0: + pytest.skip("gem not installed on test machine") + content_before = gem_content_api_client.list().count + + remote = gem_remote_factory(url="https://rubygems.org") + distribution = gem_distribution_factory(remote=remote.pulp_href) + + remote2 = gem_remote_factory() + distribution2 = gem_distribution_factory(remote=remote2.pulp_href) + + for dis, gem in zip((distribution, distribution2), ("a", "beryl")): + cmd = ["gem", "i", "--remote", "--clear-sources", "-s", dis.base_url, gem, "-v", "0.1.0"] + + out = subprocess.run(cmd, stdout=subprocess.PIPE) + assert f"Successfully installed {gem}-0.1.0" in out.stdout.decode("utf-8") + + r = gem_content_api_client.list(name=gem, version="0.1.0") + assert r.count == 1 + assert r.results[0].name == gem + assert r.results[0].version == "0.1.0" + + subprocess.run(("gem", "uninstall", gem, "-v", "0.1.0")) + + content_after = gem_content_api_client.list().count + assert content_before + 2 == content_after diff --git a/requirements.txt b/requirements.txt index 853a334..1ab8680 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -pulpcore>=3.25.0,<3.40 +pulpcore>=3.28.0,<3.40 rubymarshal==1.0.3