Implement compact_index and add digest to gem

fixes pulp#96

CHANGES/96.feature

@@ -0,0 +1 @@
+Implement new compact_index format. Add checksum and dependency information to gem content.

CHANGES/96.removal

@@ -0,0 +1,3 @@
+Disable synching without compact index format. Existing on-demand content will be broken after this release.
+
+TODO: Provide a data repair command.

pulp_gem/app/migrations/0006_gemremote_excludes_gemremote_includes_and_more.py

@@ -0,0 +1,58 @@
+# Generated by Django 4.2.1 on 2023-06-14 14:53
+
+import django.contrib.postgres.fields.hstore
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0107_distribution_hidden"),
+        ("gem", "0005_rename_gemcontent_shallowgemcontent"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="gemremote",
+            name="excludes",
+            field=django.contrib.postgres.fields.hstore.HStoreField(null=True),
+        ),
+        migrations.AddField(
+            model_name="gemremote",
+            name="includes",
+            field=django.contrib.postgres.fields.hstore.HStoreField(null=True),
+        ),
+        migrations.AddField(
+            model_name="gemremote",
+            name="prereleases",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.CreateModel(
+            name="GemContent",
+            fields=[
+                (
+                    "content_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="core.content",
+                    ),
+                ),
+                ("name", models.TextField()),
+                ("version", models.TextField()),
+                ("checksum", models.CharField(db_index=True, max_length=64)),
+                ("dependencies", django.contrib.postgres.fields.hstore.HStoreField(default=dict)),
+                ("required_ruby_version", models.TextField(null=True)),
+                ("required_rubygems_version", models.TextField(null=True)),
+                ("prerelease", models.BooleanField(default=False)),
+            ],
+            options={
+                "default_related_name": "%(app_label)s_%(model_name)s",
+                "unique_together": {("name", "version", "checksum")},
+            },
+            bases=("core.content",),
+        ),
+    ]

pulp_gem/app/models.py

@@ -1,5 +1,6 @@
 from logging import getLogger
 
+from django.contrib.postgres.fields import HStoreField
 from django.db import models
 
 from pulpcore.plugin.models import (
@@ -46,7 +47,45 @@ class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
         unique_together = ("name", "version")
 
-GemContent = ShallowGemContent
+
+class GemContent(Content):
+    """
+    The "gem" content type.
+
+    Content of this type represents a ruby gem file
+    with its spec data.
+
+    Fields:
+        name (str): The name of the gem.
+        version (str): The version of the gem.
+
+    """
+
+    TYPE = "gem"
+    repo_key_fields = ("name", "version")
+
+    name = models.TextField(blank=False, null=False)
+    version = models.TextField(blank=False, null=False)
+    checksum = models.CharField(max_length=64, null=False, db_index=True)
+    prerelease = models.BooleanField(default=False)
+    dependencies = HStoreField(default=dict)
+    required_ruby_version = models.TextField(null=True)
+    required_rubygems_version = models.TextField(null=True)
+
+    @property
+    def relative_path(self):
+        """The relative path this gem is stored under for the content app."""
+        return f"gems/{self.name}-{self.version}.gem"
+
+    @property
+    def gemspec_path(self):
+        """The path for this gem's gemspec for the content app."""
+        return f"quick/Marshal.4.8/{self.name}-{self.version}.gemspec.rz"
+
+    class Meta:
+        default_related_name = "%(app_label)s_%(model_name)s"
+        unique_together = ("name", "version", "checksum")
+
 
 class GemDistribution(Distribution):
     """
@@ -77,6 +116,10 @@ class GemRemote(Remote):
 
     TYPE = "gem"
 
+    prereleases = models.BooleanField(default=False)
+    includes = HStoreField(null=True)
+    excludes = HStoreField(null=True)
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
 
@@ -87,7 +130,7 @@ class GemRepository(Repository):
     """
 
     TYPE = "gem"
-    CONTENT_TYPES = [ShallowGemContent]
+    CONTENT_TYPES = [GemContent, ShallowGemContent]
     REMOTE_TYPES = [GemRemote]
 
     class Meta:

pulp_gem/app/serializers.py

@@ -3,9 +3,11 @@
 import os
 
 from rest_framework.serializers import (
+    BooleanField,
     CharField,
     ChoiceField,
     FileField,
+    HStoreField,
     HyperlinkedRelatedField,
     ValidationError,
 )
@@ -72,6 +74,14 @@ class GemContentSerializer(MultipleArtifactContentSerializer):
     )
     name = CharField(help_text=_("Name of the gem"), read_only=True)
     version = CharField(help_text=_("Version of the gem"), read_only=True)
+    prerelease = BooleanField(help_text=_("Whether the gem is a prerelease"), read_only=True)
+    dependencies = HStoreField(read_only=True)
+    required_ruby_version = CharField(
+        help_text=_("Required ruby version of the gem"), read_only=True
+    )
+    required_rubygems_version = CharField(
+        help_text=_("Required rubygems version of the gem"), read_only=True
+    )
 
     def __init__(self, *args, **kwargs):
         """Initializer for GemContentSerializer."""
@@ -98,23 +108,25 @@ def deferred_validate(self, data):
         """Validate the GemContent data (deferred)."""
         artifact = data.pop("artifact")
 
-        name, version, spec_data = analyse_gem(artifact.file)
-        relative_path = os.path.join("gems", name + "-" + version + ".gem")
+        gem_info, spec_data = analyse_gem(artifact.file)
+        relative_path = os.path.join("gems", gem_info["name"] + "-" + gem_info["version"] + ".gem")
 
         spec_artifact = _artifact_from_data(spec_data)
-        spec_relative_path = os.path.join("quick/Marshal.4.8", name + "-" + version + ".gemspec.rz")
+        spec_relative_path = os.path.join(
+            "quick/Marshal.4.8", gem_info["name"] + "-" + gem_info["version"] + ".gemspec.rz"
+        )
 
-        data["name"] = name
-        data["version"] = version
+        data.update(gem_info)
         data["artifacts"] = {relative_path: artifact, spec_relative_path: spec_artifact}
+        data["checksum"] = artifact.sha256
 
         # Validate uniqueness
-        content = GemContent.objects.filter(name=name, version=version)
+        content = GemContent.objects.filter(checksum=data["checksum"])
         if content.exists():
             raise ValidationError(
-                _(
-                    "There is already a gem content with name '{name}' and version '{version}'."
-                ).format(name=name, version=version)
+                _("There is already a gem content with that artifact.").format(
+                    name=name, version=version
+                )
             )
 
         return data
@@ -142,6 +154,10 @@ class Meta:
             "repository",
             "name",
             "version",
+            "prerelease",
+            "dependencies",
+            "required_ruby_version",
+            "required_rubygems_version",
         )
         model = GemContent