Implement compact_index and add digest to gem

* Rename GemContent to ShallowGemContent
    This is to move the pre GA release model out of the way for the new one
    with a proper checksum. Old content should continue to serve as is.
    Added a datarepair command to migrate pre GA gems.
* Add filtering to remotes for sync
    Filters for includes and excludes with version constraints as well as
    filtering for preleleases were added.

fixes pulp#96

.github/workflows/scripts/install.sh

@@ -84,7 +84,7 @@ if [ "$TEST" = "s3" ]; then
   sed -i -e '$a s3_test: true\
 minio_access_key: "'$MINIO_ACCESS_KEY'"\
 minio_secret_key: "'$MINIO_SECRET_KEY'"\
-pulp_scenario_settings: null\
+pulp_scenario_settings: {"allowed_content_checksums": ["md5", "sha224", "sha256", "sha384", "sha512"]}\
 ' vars/main.yaml
   export PULP_API_ROOT="/rerouted/djnd/"
 fi

CHANGES/96.feature

@@ -0,0 +1,4 @@
+Implemented new synching and publishing the compact index format.
+Rubymarshal and quick index will still be generated when publishing, but synching is exclusive to the new format.
+Added checksum and dependency information to gem content.
+Added ``prereleases`` and ``includes`` / ``excludes`` filter to remotes.

CHANGES/96.removal

@@ -0,0 +1,3 @@
+Disabled synching without compact index format.
+Existing content will still be downloadable.
+There is a ``pulpcore-manager datarepair-shallow-gems`` command that will reindex content to the new format given their artifacts are persisted.

pulp_gem/app/management/commands/datarepair-shallow-gems.py

@@ -0,0 +1,60 @@
+from gettext import gettext as _
+
+from django.core.management import BaseCommand
+
+from pulpcore.plugin.util import get_url
+from pulpcore.plugin.models import RepositoryContent
+from pulp_gem.app.models import ShallowGemContent
+from pulp_gem.app.serializers import GemContentSerializer
+
+
+def replace_content(old_content, new_content):
+    """Exchange all occurances of `old_content` in repository versions with `new_content`."""
+    RepositoryContent.objects.filter(content_id=old_content.pk).update(content_id=new_content.pk)
+
+
+class Command(BaseCommand):
+    """
+    Django management command for migrating shallow gems.
+    """
+
+    help = "This script migrates the pre GA generated gem content if artifacts are available."
+
+    def add_arguments(self, parser):
+        """Set up arguments."""
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help=_("Don't modify anything, just collect results."),
+        )
+
+    def handle(self, *args, **options):
+        dry_run = options["dry_run"]
+        failed_gems = 0
+        migrated_gems = 0
+
+        shallow_gem_qs = ShallowGemContent.objects.all()
+        count = shallow_gem_qs.count()
+        print(f"Shallow Gems count: {count}")
+        if count == 0:
+            return
+
+        for sgem in shallow_gem_qs:
+            try:
+                artifact = sgem.contentartifact_set.get(relative_path=sgem.relative_path).artifact
+                serializer = GemContentSerializer(data={"artifact": get_url(artifact)})
+                serializer.is_valid(raise_exception=True)
+                assert serializer.validated_data["name"] == sgem.name
+                assert serializer.validated_data["version"] == sgem.version
+                if not dry_run:
+                    gem = serializer.create(serializer.validated_data)
+                    replace_content(sgem, gem)
+                    sgem.delete()
+            except Exception as e:
+                failed_gems += 1
+                print(f"Failed to migrate gem '{sgem.name}' '{sgem.version}': {e}")
+            else:
+                migrated_gems += 1
+
+        print(f"Successfully migrated gems: {migrated_gems}")
+        print(f"Gems failed to migrate: {failed_gems}")

pulp_gem/app/migrations/0005_rename_gemcontent_shallowgemcontent.py

@@ -0,0 +1,27 @@
+# Generated by Django 4.2.1 on 2023-06-14 14:37
+
+from django.db import migrations
+
+
+def rename_gem_up(apps, schema_editor):
+    Content = apps.get_model("core", "Content")
+    Content.objects.filter(pulp_type="gem.gem").update(pulp_type="gem.shallow-gem")
+
+
+def rename_gem_down(apps, schema_editor):
+    Content = apps.get_model("core", "Content")
+    Content.objects.filter(pulp_type="gem.shallow-gem").update(pulp_type="gem.gem")
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("gem", "0004_alter_gemcontent_content_ptr_and_more"),
+    ]
+
+    operations = [
+        migrations.RenameModel(
+            old_name="GemContent",
+            new_name="ShallowGemContent",
+        ),
+        migrations.RunPython(code=rename_gem_up, reverse_code=rename_gem_down, elidable=True),
+    ]

pulp_gem/app/migrations/0006_gemremote_excludes_gemremote_includes_and_more.py

@@ -0,0 +1,57 @@
+# Generated by Django 4.2.1 on 2023-06-14 14:53
+
+import django.contrib.postgres.fields.hstore
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("gem", "0005_rename_gemcontent_shallowgemcontent"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="gemremote",
+            name="excludes",
+            field=django.contrib.postgres.fields.hstore.HStoreField(null=True),
+        ),
+        migrations.AddField(
+            model_name="gemremote",
+            name="includes",
+            field=django.contrib.postgres.fields.hstore.HStoreField(null=True),
+        ),
+        migrations.AddField(
+            model_name="gemremote",
+            name="prereleases",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.CreateModel(
+            name="GemContent",
+            fields=[
+                (
+                    "content_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="core.content",
+                    ),
+                ),
+                ("name", models.TextField()),
+                ("version", models.TextField()),
+                ("checksum", models.CharField(db_index=True, max_length=64)),
+                ("dependencies", django.contrib.postgres.fields.hstore.HStoreField(default=dict)),
+                ("required_ruby_version", models.TextField(null=True)),
+                ("required_rubygems_version", models.TextField(null=True)),
+                ("prerelease", models.BooleanField(default=False)),
+            ],
+            options={
+                "default_related_name": "%(app_label)s_%(model_name)s",
+                "unique_together": {("checksum",)},
+            },
+            bases=("core.content",),
+        ),
+    ]