diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index e6c3f8b..c8fb5d4 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -84,7 +84,7 @@ if [ "$TEST" = "s3" ]; then sed -i -e '$a s3_test: true\ minio_access_key: "'$MINIO_ACCESS_KEY'"\ minio_secret_key: "'$MINIO_SECRET_KEY'"\ -pulp_scenario_settings: null\ +pulp_scenario_settings: {"allowed_content_checksums": ["md5", "sha224", "sha256", "sha384", "sha512"]}\ ' vars/main.yaml export PULP_API_ROOT="/rerouted/djnd/" fi diff --git a/CHANGES/96.feature b/CHANGES/96.feature new file mode 100644 index 0000000..1b582b0 --- /dev/null +++ b/CHANGES/96.feature @@ -0,0 +1,4 @@ +Implemented new synching and publishing the compact index format. +Rubymarshal and quick index will still be generated when publishing, but synching is exclusive to the new format. +Added checksum and dependency information to gem content. +Added ``prereleases`` and ``includes`` / ``excludes`` filter to remotes. diff --git a/CHANGES/96.removal b/CHANGES/96.removal new file mode 100644 index 0000000..4446875 --- /dev/null +++ b/CHANGES/96.removal @@ -0,0 +1,3 @@ +Disabled synching without compact index format. +Existing content will still be downloadable. +There is a ``pulpcore-manager datarepair-shallow-gems`` command that will reindex content to the new format given their artifacts are persisted. diff --git a/README.rst b/README.rst deleted file mode 100644 index 3e28fdd..0000000 --- a/README.rst +++ /dev/null @@ -1,137 +0,0 @@ -.. image:: https://travis-ci.org/ATIX-AG/pulp_gem.svg?branch=master - :target: https://travis-ci.org/ATIX-AG/pulp_gem - -``pulp_gem`` Plugin -=================== - -This is the ``pulp_gem`` Plugin for `Pulp Project -3.0+ `__. This plugin adds importers and distributors -for rubygems. - -All REST API examples below use `httpie `__ to perform the requests. -The ``httpie`` commands below assume that the user executing the commands has a ``.netrc`` file -in the home directory. The ``.netrc`` should have the following configuration: - -.. code-block:: - - machine localhost - login admin - password admin - -If you configured the ``admin`` user with a different password, adjust the configuration -accordingly. If you prefer to specify the username and password with each request, please see -``httpie`` documentation on how to do that. - -This documentation makes use of the `jq library `_ -to parse the json received from requests, in order to get the unique urls generated -when objects are created. To follow this documentation as-is please install the jq -library with: - -``$ sudo dnf install jq`` - -Install ``pulpcore`` --------------------- - -Follow the `installation -instructions `__ -provided with pulpcore. - -Install ``pulp-gem`` from source --------------------------------- - -1) sudo -u pulp -i -2) source ~/pulpvenv/bin/activate -3) git clone https://github.com/ATIX-AG/pulp_gem -4) cd pulp\_gem -5) python setup.py develop -6) django-admin makemigrations pulp\_gem -7) django-admin migrate pulp\_gem -8) django-admin runserver 24817 -9) gunicorn pulpcore.content:server --bind 'localhost:24816' --worker-class 'aiohttp.GunicornWebWorker' -w 2 -10) sudo systemctl restart pulpcore-resource-manager -11) sudo systemctl restart pulpcore-worker@1 -12) sudo systemctl restart pulpcore-worker@2 - -Install ``pulp-gem`` From PyPI ------------------------------- - -1) sudo -u pulp -i -2) source ~/pulpvenv/bin/activate -3) pip install pulp-gem -4) django-admin makemigrations pulp\_gem -5) django-admin migrate pulp\_gem -6) django-admin runserver 24817 -7) gunicorn pulpcore.content:server --bind 'localhost:24816' --worker-class 'aiohttp.GunicornWebWorker' -w 2 -8) sudo systemctl restart pulpcore-resource-manager -9) sudo systemctl restart pulpcore-worker@1 -10) sudo systemctl restart pulpcore-worker@2 - -Create a repository ``foo`` ---------------------------- - -``$ http POST http://localhost:24817/pulp/api/v3/repositories/ name=foo`` - -.. code:: json - - { - "pulp_href": "/pulp/api/v3/repositories/1/", - "...": "..." - } - -``$ export REPO_HREF=$(http :24817/pulp/api/v3/repositories/ | jq -r '.results[] | select(.name == "foo") | .pulp_href')`` - -Add a remote ------------- - -``$ http POST http://localhost:24817/pulp/api/v3/remotes/gem/ name='bar' url='https://rubygems.org/' policy='streamed'`` - -.. code:: json - - { - "pulp_href": "/pulp/api/v3/remotes/gem/1/", - "..." : "..." - } - -``$ export REMOTE_HREF=$(http :24817/pulp/api/v3/remotes/gem/ | jq -r '.results[] | select(.name == "bar") | .pulp_href')`` - -Sync repository ``foo`` using remote ``bar`` --------------------------------------------- - -``$ http POST ':24817'${REMOTE_HREF}'sync/' repository=$REPO_HREF`` - -Upload ``foo-0.0.1.gem`` to Pulp --------------------------------- - -Create an Artifact by uploading the gemfile to Pulp. - -``$ http --form POST http://localhost:24817/pulp/api/v3/artifacts/ file@./foo-0.0.1.gem`` - -.. code:: json - - { - "pulp_href": "/pulp/api/v3/artifacts/1/", - "...": "..." - } - -You need to upload the corresponding ``foo-0.0.1.gemspec.rz`` in the same way. - -Create ``gem`` content from an Artifact ---------------------------------------- - -``$ http POST http://localhost:24817/pulp/api/v3/content/gem/gems/ _artifact="/pulp/api/v3/artifacts/1/"`` - -.. code:: json - - { - "pulp_href": "/pulp/api/v3/content/gem/gems/1/", - "_artifacts": { - "gems/foo-0.0.1.gem":"/pulp/api/v3/artifacts/1/", - "quick/Marshal.4.8/foo-0.0.1.gemspec.rz":"/pulp/api/v3/artifacts/2/" - }, - "name": "foo", - "notes": {}, - "type": "gem", - "version": "0.0.1" - } - -``$ export CONTENT_HREF=$(http :24817/pulp/api/v3/content/gem/gems/ | jq -r '.results[] | select(.name == "foo") | .pulp_href')`` diff --git a/docs/_static/api.json b/docs/_static/api.json deleted file mode 100644 index 141fb81..0000000 --- a/docs/_static/api.json +++ /dev/null @@ -1 +0,0 @@ -{"swagger": "2.0", "info": {"title": "Pulp 3 API", "license": {"name": "GPLv2+"}, "logo": {"url": "https://pulp.plan.io/attachments/download/517478/pulp_logo_word_rectangle.svg"}, "version": "v3"}, "host": "localhost:24817", "schemes": ["http"], "basePath": "/", "consumes": ["application/json"], "produces": ["application/json"], "securityDefinitions": {"Basic": {"type": "basic"}}, "security": [{"Basic": []}], "paths": {"/pulp/api/v3/content/gem/gems/": {"get": {"operationId": "content_gem_gems_list", "summary": "List gem contents", "description": "A ViewSet for GemContent.", "parameters": [{"name": "ordering", "in": "query", "description": "Which field to use when ordering the results.", "required": false, "type": "string"}, {"name": "name", "in": "query", "description": "Filter results where name matches value", "required": false, "type": "string"}, {"name": "version", "in": "query", "description": "Filter results where version matches value", "required": false, "type": "string"}, {"name": "repository_version", "in": "query", "description": "Repository Version referenced by HREF", "required": false, "type": "string"}, {"name": "repository_version_added", "in": "query", "description": "Repository Version referenced by HREF", "required": false, "type": "string"}, {"name": "repository_version_removed", "in": "query", "description": "Repository Version referenced by HREF", "required": false, "type": "string"}, {"name": "limit", "in": "query", "description": "Number of results to return per page.", "required": false, "type": "integer"}, {"name": "offset", "in": "query", "description": "The initial index from which to return the results.", "required": false, "type": "integer"}, {"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"required": ["count", "results"], "type": "object", "properties": {"count": {"type": "integer"}, "next": {"type": "string", "format": "uri", "x-nullable": true}, "previous": {"type": "string", "format": "uri", "x-nullable": true}, "results": {"type": "array", "items": {"$ref": "#/definitions/gem.GemContentRead"}}}}}}, "tags": ["content: gems"]}, "post": {"operationId": "content_gem_gems_create", "summary": "Create a gem content", "description": "Trigger an asynchronous task to create content,optionally create new repository version.", "parameters": [{"name": "artifact", "in": "formData", "description": "Artifact file representing the physical content", "required": false, "type": "string", "format": "uri"}, {"name": "file", "in": "formData", "description": "An uploaded file that should be turned into the artifact of the content unit.", "required": false, "type": "file"}, {"name": "repository", "in": "formData", "description": "A URI of a repository the new content unit should be associated with.", "required": false, "type": "string", "format": "uri"}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "consumes": ["multipart/form-data", "application/x-www-form-urlencoded"], "tags": ["content: gems"]}, "parameters": []}, "{gem_content_href}": {"get": {"operationId": "content_gem_gems_read", "summary": "Inspect a gem content", "description": "A ViewSet for GemContent.", "parameters": [{"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"$ref": "#/definitions/gem.GemContentRead"}}}, "tags": ["content: gems"]}, "parameters": [{"name": "gem_content_href", "in": "path", "description": "URI of Gem Content. e.g.: /pulp/api/v3/content/gem/gems/1/", "required": true, "type": "string"}]}, "/pulp/api/v3/distributions/gem/gem/": {"get": {"operationId": "distributions_gem_gem_list", "summary": "List gem distributions", "description": "ViewSet for GemDistributions.", "parameters": [{"name": "ordering", "in": "query", "description": "Which field to use when ordering the results.", "required": false, "type": "string"}, {"name": "name", "in": "query", "description": "", "required": false, "type": "string"}, {"name": "name__in", "in": "query", "description": "Filter results where name is in a comma-separated list of values", "required": false, "type": "string"}, {"name": "base_path", "in": "query", "description": "", "required": false, "type": "string"}, {"name": "base_path__contains", "in": "query", "description": "Filter results where base_path contains value", "required": false, "type": "string"}, {"name": "base_path__icontains", "in": "query", "description": "Filter results where base_path contains value", "required": false, "type": "string"}, {"name": "base_path__in", "in": "query", "description": "Filter results where base_path is in a comma-separated list of values", "required": false, "type": "string"}, {"name": "limit", "in": "query", "description": "Number of results to return per page.", "required": false, "type": "integer"}, {"name": "offset", "in": "query", "description": "The initial index from which to return the results.", "required": false, "type": "integer"}, {"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"required": ["count", "results"], "type": "object", "properties": {"count": {"type": "integer"}, "next": {"type": "string", "format": "uri", "x-nullable": true}, "previous": {"type": "string", "format": "uri", "x-nullable": true}, "results": {"type": "array", "items": {"$ref": "#/definitions/gem.GemDistributionRead"}}}}}}, "tags": ["distributions: gem"]}, "post": {"operationId": "distributions_gem_gem_create", "summary": "Create a gem distribution", "description": "Trigger an asynchronous create task", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/gem.GemDistribution"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["distributions: gem"]}, "parameters": []}, "{gem_distribution_href}": {"get": {"operationId": "distributions_gem_gem_read", "summary": "Inspect a gem distribution", "description": "ViewSet for GemDistributions.", "parameters": [{"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"$ref": "#/definitions/gem.GemDistributionRead"}}}, "tags": ["distributions: gem"]}, "put": {"operationId": "distributions_gem_gem_update", "summary": "Update a gem distribution", "description": "Trigger an asynchronous update task", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/gem.GemDistribution"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["distributions: gem"]}, "patch": {"operationId": "distributions_gem_gem_partial_update", "summary": "Partially update a gem distribution", "description": "Trigger an asynchronous partial update task", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/gem.GemDistribution"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["distributions: gem"]}, "delete": {"operationId": "distributions_gem_gem_delete", "summary": "Delete a gem distribution", "description": "Trigger an asynchronous delete task", "parameters": [], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["distributions: gem"]}, "parameters": [{"name": "gem_distribution_href", "in": "path", "description": "URI of Gem Distribution. e.g.: /pulp/api/v3/distributions/gem/gem/1/", "required": true, "type": "string"}]}, "/pulp/api/v3/publications/gem/gem/": {"get": {"operationId": "publications_gem_gem_list", "summary": "List gem publications", "description": "A ViewSet for GemPublication.", "parameters": [{"name": "ordering", "in": "query", "description": "Which field to use when ordering the results.", "required": false, "type": "string"}, {"name": "repository_version", "in": "query", "description": "Repository Version referenced by HREF", "required": false, "type": "string"}, {"name": "pulp_created__lt", "in": "query", "description": "Filter results where pulp_created is less than value", "required": false, "type": "string"}, {"name": "pulp_created__lte", "in": "query", "description": "Filter results where pulp_created is less than or equal to value", "required": false, "type": "string"}, {"name": "pulp_created__gt", "in": "query", "description": "Filter results where pulp_created is greater than value", "required": false, "type": "string"}, {"name": "pulp_created__gte", "in": "query", "description": "Filter results where pulp_created is greater than or equal to value", "required": false, "type": "string"}, {"name": "pulp_created__range", "in": "query", "description": "Filter results where pulp_created is between two comma separated values", "required": false, "type": "string"}, {"name": "pulp_created", "in": "query", "description": "ISO 8601 formatted dates are supported", "required": false, "type": "string"}, {"name": "limit", "in": "query", "description": "Number of results to return per page.", "required": false, "type": "integer"}, {"name": "offset", "in": "query", "description": "The initial index from which to return the results.", "required": false, "type": "integer"}, {"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"required": ["count", "results"], "type": "object", "properties": {"count": {"type": "integer"}, "next": {"type": "string", "format": "uri", "x-nullable": true}, "previous": {"type": "string", "format": "uri", "x-nullable": true}, "results": {"type": "array", "items": {"$ref": "#/definitions/gem.GemPublicationRead"}}}}}}, "tags": ["publications: gem"]}, "post": {"operationId": "publications_gem_gem_create", "summary": "Create a gem publication", "description": "Trigger an asynchronous task to publish gem content", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/gem.GemPublication"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["publications: gem"]}, "parameters": []}, "{gem_publication_href}": {"get": {"operationId": "publications_gem_gem_read", "summary": "Inspect a gem publication", "description": "A ViewSet for GemPublication.", "parameters": [{"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"$ref": "#/definitions/gem.GemPublicationRead"}}}, "tags": ["publications: gem"]}, "delete": {"operationId": "publications_gem_gem_delete", "summary": "Delete a gem publication", "description": "A ViewSet for GemPublication.", "parameters": [], "responses": {"204": {"description": ""}}, "tags": ["publications: gem"]}, "parameters": [{"name": "gem_publication_href", "in": "path", "description": "URI of Gem Publication. e.g.: /pulp/api/v3/publications/gem/gem/1/", "required": true, "type": "string"}]}, "/pulp/api/v3/remotes/gem/gem/": {"get": {"operationId": "remotes_gem_gem_list", "summary": "List gem remotes", "description": "A ViewSet for GemRemote.", "parameters": [{"name": "ordering", "in": "query", "description": "Which field to use when ordering the results.", "required": false, "type": "string"}, {"name": "name", "in": "query", "description": "", "required": false, "type": "string"}, {"name": "name__in", "in": "query", "description": "Filter results where name is in a comma-separated list of values", "required": false, "type": "string"}, {"name": "pulp_last_updated__lt", "in": "query", "description": "Filter results where pulp_last_updated is less than value", "required": false, "type": "string"}, {"name": "pulp_last_updated__lte", "in": "query", "description": "Filter results where pulp_last_updated is less than or equal to value", "required": false, "type": "string"}, {"name": "pulp_last_updated__gt", "in": "query", "description": "Filter results where pulp_last_updated is greater than value", "required": false, "type": "string"}, {"name": "pulp_last_updated__gte", "in": "query", "description": "Filter results where pulp_last_updated is greater than or equal to value", "required": false, "type": "string"}, {"name": "pulp_last_updated__range", "in": "query", "description": "Filter results where pulp_last_updated is between two comma separated values", "required": false, "type": "string"}, {"name": "pulp_last_updated", "in": "query", "description": "ISO 8601 formatted dates are supported", "required": false, "type": "string"}, {"name": "limit", "in": "query", "description": "Number of results to return per page.", "required": false, "type": "integer"}, {"name": "offset", "in": "query", "description": "The initial index from which to return the results.", "required": false, "type": "integer"}, {"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"required": ["count", "results"], "type": "object", "properties": {"count": {"type": "integer"}, "next": {"type": "string", "format": "uri", "x-nullable": true}, "previous": {"type": "string", "format": "uri", "x-nullable": true}, "results": {"type": "array", "items": {"$ref": "#/definitions/gem.GemRemoteRead"}}}}}}, "tags": ["remotes: gem"]}, "post": {"operationId": "remotes_gem_gem_create", "summary": "Create a gem remote", "description": "A ViewSet for GemRemote.", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/gem.GemRemote"}}], "responses": {"201": {"description": "", "schema": {"$ref": "#/definitions/gem.GemRemoteRead"}}}, "tags": ["remotes: gem"]}, "parameters": []}, "{gem_remote_href}": {"get": {"operationId": "remotes_gem_gem_read", "summary": "Inspect a gem remote", "description": "A ViewSet for GemRemote.", "parameters": [{"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"$ref": "#/definitions/gem.GemRemoteRead"}}}, "tags": ["remotes: gem"]}, "put": {"operationId": "remotes_gem_gem_update", "summary": "Update a gem remote", "description": "Trigger an asynchronous update task", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/gem.GemRemote"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["remotes: gem"]}, "patch": {"operationId": "remotes_gem_gem_partial_update", "summary": "Partially update a gem remote", "description": "Trigger an asynchronous partial update task", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/gem.GemRemote"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["remotes: gem"]}, "delete": {"operationId": "remotes_gem_gem_delete", "summary": "Delete a gem remote", "description": "Trigger an asynchronous delete task", "parameters": [], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["remotes: gem"]}, "parameters": [{"name": "gem_remote_href", "in": "path", "description": "URI of Gem Remote. e.g.: /pulp/api/v3/remotes/gem/gem/1/", "required": true, "type": "string"}]}, "{gem_remote_href}sync/": {"post": {"operationId": "remotes_gem_gem_sync", "description": "Trigger an asynchronous task to sync gem content", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/RepositorySyncURL"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["remotes: gem"]}, "parameters": [{"name": "gem_remote_href", "in": "path", "description": "URI of Gem Remote. e.g.: /pulp/api/v3/remotes/gem/gem/1/", "required": true, "type": "string"}]}, "/pulp/api/v3/repositories/gem/gem/": {"get": {"operationId": "repositories_gem_gem_list", "summary": "List gem repositorys", "description": "A ViewSet for GemRepository.\nSimilar to the PackageViewSet above, define endpoint_name,\nqueryset and serializer, at a minimum.", "parameters": [{"name": "ordering", "in": "query", "description": "Which field to use when ordering the results.", "required": false, "type": "string"}, {"name": "name", "in": "query", "description": "", "required": false, "type": "string"}, {"name": "name__in", "in": "query", "description": "Filter results where name is in a comma-separated list of values", "required": false, "type": "string"}, {"name": "limit", "in": "query", "description": "Number of results to return per page.", "required": false, "type": "integer"}, {"name": "offset", "in": "query", "description": "The initial index from which to return the results.", "required": false, "type": "integer"}, {"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"required": ["count", "results"], "type": "object", "properties": {"count": {"type": "integer"}, "next": {"type": "string", "format": "uri", "x-nullable": true}, "previous": {"type": "string", "format": "uri", "x-nullable": true}, "results": {"type": "array", "items": {"$ref": "#/definitions/gem.GemRepositoryRead"}}}}}}, "tags": ["repositories: gem"]}, "post": {"operationId": "repositories_gem_gem_create", "summary": "Create a gem repository", "description": "A ViewSet for GemRepository.\nSimilar to the PackageViewSet above, define endpoint_name,\nqueryset and serializer, at a minimum.", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/gem.GemRepository"}}], "responses": {"201": {"description": "", "schema": {"$ref": "#/definitions/gem.GemRepositoryRead"}}}, "tags": ["repositories: gem"]}, "parameters": []}, "{gem_repository_href}": {"get": {"operationId": "repositories_gem_gem_read", "summary": "Inspect a gem repository", "description": "A ViewSet for GemRepository.\nSimilar to the PackageViewSet above, define endpoint_name,\nqueryset and serializer, at a minimum.", "parameters": [{"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"$ref": "#/definitions/gem.GemRepositoryRead"}}}, "tags": ["repositories: gem"]}, "put": {"operationId": "repositories_gem_gem_update", "summary": "Update a gem repository", "description": "Trigger an asynchronous update task", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/gem.GemRepository"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["repositories: gem"]}, "patch": {"operationId": "repositories_gem_gem_partial_update", "summary": "Partially update a gem repository", "description": "Trigger an asynchronous partial update task", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/gem.GemRepository"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["repositories: gem"]}, "delete": {"operationId": "repositories_gem_gem_delete", "summary": "Delete a gem repository", "description": "Trigger an asynchronous delete task", "parameters": [], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["repositories: gem"]}, "parameters": [{"name": "gem_repository_href", "in": "path", "description": "URI of Gem Repository. e.g.: /pulp/api/v3/repositories/gem/gem/1/", "required": true, "type": "string"}]}, "{gem_repository_href}modify/": {"post": {"operationId": "repositories_gem_gem_modify", "summary": "Modify Repository Content", "description": "Trigger an asynchronous task to create a new repository version.", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/RepositoryAddRemoveContent"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["repositories: gem"]}, "parameters": [{"name": "gem_repository_href", "in": "path", "description": "URI of Gem Repository. e.g.: /pulp/api/v3/repositories/gem/gem/1/", "required": true, "type": "string"}]}, "{gem_repository_href}sync/": {"post": {"operationId": "repositories_gem_gem_sync", "summary": "Sync from remote", "description": "Trigger an asynchronous task to sync content.", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/RepositorySyncURL"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["repositories: gem"]}, "parameters": [{"name": "gem_repository_href", "in": "path", "description": "URI of Gem Repository. e.g.: /pulp/api/v3/repositories/gem/gem/1/", "required": true, "type": "string"}]}, "{gem_repository_href}versions/": {"get": {"operationId": "repositories_gem_gem_versions_list", "summary": "List repository versions", "description": "A ViewSet for a GemRepositoryVersion represents a single Gem repository version.", "parameters": [{"name": "ordering", "in": "query", "description": "Which field to use when ordering the results.", "required": false, "type": "string"}, {"name": "number", "in": "query", "description": "", "required": false, "type": "number"}, {"name": "number__lt", "in": "query", "description": "Filter results where number is less than value", "required": false, "type": "number"}, {"name": "number__lte", "in": "query", "description": "Filter results where number is less than or equal to value", "required": false, "type": "number"}, {"name": "number__gt", "in": "query", "description": "Filter results where number is greater than value", "required": false, "type": "number"}, {"name": "number__gte", "in": "query", "description": "Filter results where number is greater than or equal to value", "required": false, "type": "number"}, {"name": "number__range", "in": "query", "description": "Filter results where number is between two comma separated values", "required": false, "type": "number"}, {"name": "pulp_created__lt", "in": "query", "description": "Filter results where pulp_created is less than value", "required": false, "type": "string"}, {"name": "pulp_created__lte", "in": "query", "description": "Filter results where pulp_created is less than or equal to value", "required": false, "type": "string"}, {"name": "pulp_created__gt", "in": "query", "description": "Filter results where pulp_created is greater than value", "required": false, "type": "string"}, {"name": "pulp_created__gte", "in": "query", "description": "Filter results where pulp_created is greater than or equal to value", "required": false, "type": "string"}, {"name": "pulp_created__range", "in": "query", "description": "Filter results where pulp_created is between two comma separated values", "required": false, "type": "string"}, {"name": "content", "in": "query", "description": "Content Unit referenced by HREF", "required": false, "type": "string"}, {"name": "pulp_created", "in": "query", "description": "ISO 8601 formatted dates are supported", "required": false, "type": "string"}, {"name": "limit", "in": "query", "description": "Number of results to return per page.", "required": false, "type": "integer"}, {"name": "offset", "in": "query", "description": "The initial index from which to return the results.", "required": false, "type": "integer"}, {"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"required": ["count", "results"], "type": "object", "properties": {"count": {"type": "integer"}, "next": {"type": "string", "format": "uri", "x-nullable": true}, "previous": {"type": "string", "format": "uri", "x-nullable": true}, "results": {"type": "array", "items": {"$ref": "#/definitions/RepositoryVersionRead"}}}}}}, "tags": ["repositories: gem versions"]}, "parameters": [{"name": "gem_repository_href", "in": "path", "description": "URI of Gem Repository. e.g.: /pulp/api/v3/repositories/gem/gem/1/", "required": true, "type": "string"}]}, "{gem_repository_version_href}": {"get": {"operationId": "repositories_gem_gem_versions_read", "summary": "Inspect a repository version", "description": "A ViewSet for a GemRepositoryVersion represents a single Gem repository version.", "parameters": [{"name": "fields", "in": "query", "description": "A list of fields to include in the response.", "required": false, "type": "string"}, {"name": "exclude_fields", "in": "query", "description": "A list of fields to exclude from the response.", "required": false, "type": "string"}], "responses": {"200": {"description": "", "schema": {"$ref": "#/definitions/RepositoryVersionRead"}}}, "tags": ["repositories: gem versions"]}, "delete": {"operationId": "repositories_gem_gem_versions_delete", "summary": "Delete a repository version", "description": "Trigger an asynchronous task to delete a repositroy version.", "parameters": [], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["repositories: gem versions"]}, "parameters": [{"name": "gem_repository_version_href", "in": "path", "description": "URI of Repository Version. e.g.: /pulp/api/v3/repositories/gem/gem/1/versions/1/", "required": true, "type": "string"}]}, "{gem_repository_version_href}repair/": {"post": {"operationId": "repositories_gem_gem_versions_repair", "description": "Trigger an asynchronous task to repair a repositroy version.", "parameters": [{"name": "data", "in": "body", "required": true, "schema": {"$ref": "#/definitions/RepositoryVersion"}}], "responses": {"202": {"description": "", "schema": {"$ref": "#/definitions/AsyncOperationResponse"}}}, "tags": ["repositories: gem versions"]}, "parameters": [{"name": "gem_repository_version_href", "in": "path", "description": "URI of Repository Version. e.g.: /pulp/api/v3/repositories/gem/gem/1/versions/1/", "required": true, "type": "string"}]}}, "definitions": {"gem.GemContentRead": {"type": "object", "properties": {"pulp_href": {"title": "Pulp href", "type": "string", "format": "uri", "readOnly": true}, "pulp_created": {"title": "Pulp created", "description": "Timestamp of creation.", "type": "string", "format": "date-time", "readOnly": true}, "artifacts": {"title": "Artifacts", "description": "A dict mapping relative paths inside the Content to the correspondingArtifact URLs. E.g.: {'relative/path': '/artifacts/1/'", "type": "object", "additionalProperties": {"type": "string"}, "readOnly": true}, "name": {"title": "Name", "description": "Name of the gem", "type": "string", "readOnly": true, "minLength": 1}, "version": {"title": "Version", "description": "Version of the gem", "type": "string", "readOnly": true, "minLength": 1}}}, "AsyncOperationResponse": {"required": ["task"], "type": "object", "properties": {"task": {"title": "Task", "description": "The href of the task.", "type": "string", "format": "uri"}}}, "gem.GemDistributionRead": {"required": ["base_path", "name"], "type": "object", "properties": {"pulp_href": {"title": "Pulp href", "type": "string", "format": "uri", "readOnly": true}, "pulp_created": {"title": "Pulp created", "description": "Timestamp of creation.", "type": "string", "format": "date-time", "readOnly": true}, "base_path": {"title": "Base path", "description": "The base (relative) path component of the published url. Avoid paths that overlap with other distribution base paths (e.g. \"foo\" and \"foo/bar\")", "type": "string", "minLength": 1}, "base_url": {"title": "Base url", "description": "The URL for accessing the publication as defined by this distribution.", "type": "string", "readOnly": true, "minLength": 1}, "content_guard": {"title": "Content guard", "description": "An optional content-guard.", "type": "string", "format": "uri", "x-nullable": true}, "name": {"title": "Name", "description": "A unique name. Ex, `rawhide` and `stable`.", "type": "string", "minLength": 1}, "publication": {"title": "Publication", "description": "Publication to be served", "type": "string", "format": "uri", "x-nullable": true}}}, "gem.GemDistribution": {"required": ["base_path", "name"], "type": "object", "properties": {"pulp_href": {"title": "Pulp href", "type": "string", "format": "uri", "readOnly": true}, "pulp_created": {"title": "Pulp created", "description": "Timestamp of creation.", "type": "string", "format": "date-time", "readOnly": true}, "base_path": {"title": "Base path", "description": "The base (relative) path component of the published url. Avoid paths that overlap with other distribution base paths (e.g. \"foo\" and \"foo/bar\")", "type": "string", "minLength": 1}, "base_url": {"title": "Base url", "description": "The URL for accessing the publication as defined by this distribution.", "type": "string", "readOnly": true, "minLength": 1}, "content_guard": {"title": "Content guard", "description": "An optional content-guard.", "type": "string", "format": "uri", "x-nullable": true}, "name": {"title": "Name", "description": "A unique name. Ex, `rawhide` and `stable`.", "type": "string", "minLength": 1}, "publication": {"title": "Publication", "description": "Publication to be served", "type": "string", "format": "uri", "x-nullable": true}}}, "gem.GemPublicationRead": {"type": "object", "properties": {"pulp_href": {"title": "Pulp href", "type": "string", "format": "uri", "readOnly": true}, "pulp_created": {"title": "Pulp created", "description": "Timestamp of creation.", "type": "string", "format": "date-time", "readOnly": true}, "repository_version": {"title": "Repository version", "type": "string", "format": "uri"}, "repository": {"title": "Repository", "description": "A URI of the repository to be published.", "type": "string", "format": "uri"}}}, "gem.GemPublication": {"type": "object", "properties": {"pulp_href": {"title": "Pulp href", "type": "string", "format": "uri", "readOnly": true}, "pulp_created": {"title": "Pulp created", "description": "Timestamp of creation.", "type": "string", "format": "date-time", "readOnly": true}, "repository_version": {"title": "Repository version", "type": "string", "format": "uri"}, "repository": {"title": "Repository", "description": "A URI of the repository to be published.", "type": "string", "format": "uri"}}}, "gem.GemRemoteRead": {"required": ["name", "url"], "type": "object", "properties": {"pulp_href": {"title": "Pulp href", "type": "string", "format": "uri", "readOnly": true}, "pulp_created": {"title": "Pulp created", "description": "Timestamp of creation.", "type": "string", "format": "date-time", "readOnly": true}, "name": {"title": "Name", "description": "A unique name for this remote.", "type": "string", "minLength": 1}, "url": {"title": "Url", "description": "The URL of an external content source.", "type": "string", "minLength": 1}, "ca_cert": {"title": "Ca cert", "description": "A string containing the PEM encoded CA certificate used to validate the server certificate presented by the remote server. All new line characters must be escaped.", "type": "string", "minLength": 1, "x-nullable": true}, "client_cert": {"title": "Client cert", "description": "A string containing the PEM encoded client certificate used for authentication. All new line characters must be escaped.", "type": "string", "minLength": 1, "x-nullable": true}, "client_key": {"title": "Client key", "description": "A PEM encoded private key used for authentication.", "type": "string", "minLength": 1, "x-nullable": true}, "tls_validation": {"title": "Tls validation", "description": "If True, TLS peer validation must be performed.", "type": "boolean"}, "proxy_url": {"title": "Proxy url", "description": "The proxy URL. Format: scheme://user:password@host:port", "type": "string", "minLength": 1, "x-nullable": true}, "username": {"title": "Username", "description": "The username to be used for authentication when syncing.", "type": "string", "minLength": 1, "x-nullable": true}, "password": {"title": "Password", "description": "The password to be used for authentication when syncing.", "type": "string", "minLength": 1, "x-nullable": true}, "pulp_last_updated": {"title": "Pulp last updated", "description": "Timestamp of the most recent update of the remote.", "type": "string", "format": "date-time", "readOnly": true}, "download_concurrency": {"title": "Download concurrency", "description": "Total number of simultaneous connections.", "type": "integer", "minimum": 1}, "policy": {"title": "Policy", "description": "The policy to use when downloading content. The possible values include: 'immediate', 'on_demand', and 'streamed'. 'immediate' is the default.", "type": "string", "enum": ["immediate", "on_demand", "streamed"], "default": "immediate"}}}, "gem.GemRemote": {"required": ["name", "url"], "type": "object", "properties": {"pulp_href": {"title": "Pulp href", "type": "string", "format": "uri", "readOnly": true}, "pulp_created": {"title": "Pulp created", "description": "Timestamp of creation.", "type": "string", "format": "date-time", "readOnly": true}, "name": {"title": "Name", "description": "A unique name for this remote.", "type": "string", "minLength": 1}, "url": {"title": "Url", "description": "The URL of an external content source.", "type": "string", "minLength": 1}, "ca_cert": {"title": "Ca cert", "description": "A string containing the PEM encoded CA certificate used to validate the server certificate presented by the remote server. All new line characters must be escaped.", "type": "string", "minLength": 1, "x-nullable": true}, "client_cert": {"title": "Client cert", "description": "A string containing the PEM encoded client certificate used for authentication. All new line characters must be escaped.", "type": "string", "minLength": 1, "x-nullable": true}, "client_key": {"title": "Client key", "description": "A PEM encoded private key used for authentication.", "type": "string", "minLength": 1, "x-nullable": true}, "tls_validation": {"title": "Tls validation", "description": "If True, TLS peer validation must be performed.", "type": "boolean"}, "proxy_url": {"title": "Proxy url", "description": "The proxy URL. Format: scheme://user:password@host:port", "type": "string", "minLength": 1, "x-nullable": true}, "username": {"title": "Username", "description": "The username to be used for authentication when syncing.", "type": "string", "minLength": 1, "x-nullable": true}, "password": {"title": "Password", "description": "The password to be used for authentication when syncing.", "type": "string", "minLength": 1, "x-nullable": true}, "pulp_last_updated": {"title": "Pulp last updated", "description": "Timestamp of the most recent update of the remote.", "type": "string", "format": "date-time", "readOnly": true}, "download_concurrency": {"title": "Download concurrency", "description": "Total number of simultaneous connections.", "type": "integer", "minimum": 1}, "policy": {"title": "Policy", "description": "The policy to use when downloading content. The possible values include: 'immediate', 'on_demand', and 'streamed'. 'immediate' is the default.", "type": "string", "enum": ["immediate", "on_demand", "streamed"], "default": "immediate"}}}, "RepositorySyncURL": {"required": ["remote"], "type": "object", "properties": {"remote": {"title": "Remote", "description": "A URI of the repository to be synchronized.", "type": "string", "format": "uri"}, "mirror": {"title": "Mirror", "description": "If ``True``, synchronization will remove all content that is not present in the remote repository. If ``False``, sync will be additive only.", "type": "boolean", "default": false}}}, "gem.GemRepositoryRead": {"required": ["name"], "type": "object", "properties": {"pulp_href": {"title": "Pulp href", "type": "string", "format": "uri", "readOnly": true}, "pulp_created": {"title": "Pulp created", "description": "Timestamp of creation.", "type": "string", "format": "date-time", "readOnly": true}, "versions_href": {"title": "Versions href", "type": "string", "format": "uri", "readOnly": true}, "latest_version_href": {"title": "Latest version href", "type": "string", "format": "uri", "readOnly": true}, "name": {"title": "Name", "description": "A unique name for this repository.", "type": "string", "minLength": 1}, "description": {"title": "Description", "description": "An optional description.", "type": "string", "minLength": 1, "x-nullable": true}}}, "gem.GemRepository": {"required": ["name"], "type": "object", "properties": {"pulp_href": {"title": "Pulp href", "type": "string", "format": "uri", "readOnly": true}, "pulp_created": {"title": "Pulp created", "description": "Timestamp of creation.", "type": "string", "format": "date-time", "readOnly": true}, "versions_href": {"title": "Versions href", "type": "string", "format": "uri", "readOnly": true}, "latest_version_href": {"title": "Latest version href", "type": "string", "format": "uri", "readOnly": true}, "name": {"title": "Name", "description": "A unique name for this repository.", "type": "string", "minLength": 1}, "description": {"title": "Description", "description": "An optional description.", "type": "string", "minLength": 1, "x-nullable": true}}}, "RepositoryAddRemoveContent": {"type": "object", "properties": {"add_content_units": {"description": "A list of content units to add to a new repository version. This content is added after remove_content_units are removed.", "type": "array", "items": {"type": "string"}}, "remove_content_units": {"description": "A list of content units to remove from the latest repository version. You may also specify '*' as an entry to remove all content. This content is removed before add_content_units are added.", "type": "array", "items": {"type": "string"}}, "base_version": {"title": "Base version", "description": "A repository version whose content will be used as the initial set of content for the new repository version", "type": "string", "format": "uri"}}}, "ContentSummary": {"title": "Content summary", "description": "Various count summaries of the content in the version and the HREF to view them.", "required": ["added", "removed", "present"], "type": "object", "properties": {"added": {"title": "Added", "type": "object", "additionalProperties": {"type": "object", "additionalProperties": {"type": "string"}}}, "removed": {"title": "Removed", "type": "object", "additionalProperties": {"type": "object", "additionalProperties": {"type": "string"}}}, "present": {"title": "Present", "type": "object", "additionalProperties": {"type": "object", "additionalProperties": {"type": "string"}}}}}, "RepositoryVersionRead": {"type": "object", "properties": {"pulp_href": {"title": "Pulp href", "type": "string", "format": "uri", "readOnly": true}, "pulp_created": {"title": "Pulp created", "description": "Timestamp of creation.", "type": "string", "format": "date-time", "readOnly": true}, "number": {"title": "Number", "type": "integer", "readOnly": true}, "base_version": {"title": "Base version", "description": "A repository version whose content was used as the initial set of content for this repository version", "type": "string", "format": "uri"}, "content_summary": {"$ref": "#/definitions/ContentSummary"}}}, "RepositoryVersion": {"type": "object", "properties": {"pulp_href": {"title": "Pulp href", "type": "string", "format": "uri", "readOnly": true}, "pulp_created": {"title": "Pulp created", "description": "Timestamp of creation.", "type": "string", "format": "date-time", "readOnly": true}, "number": {"title": "Number", "type": "integer", "readOnly": true}, "base_version": {"title": "Base version", "description": "A repository version whose content was used as the initial set of content for this repository version", "type": "string", "format": "uri"}, "content_summary": {"$ref": "#/definitions/ContentSummary"}}}}, "tags": [{"name": "content: gems", "x-displayName": "Content: Gems"}, {"name": "distributions: gem", "x-displayName": "Distributions: Gem"}, {"name": "publications: gem", "x-displayName": "Publications: Gem"}, {"name": "remotes: gem", "x-displayName": "Remotes: Gem"}, {"name": "repositories: gem", "x-displayName": "Repositories: Gem"}, {"name": "repositories: gem versions", "x-displayName": "Repositories: Gem Versions"}]} \ No newline at end of file diff --git a/pulp_gem/app/management/__init__.py b/pulp_gem/app/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pulp_gem/app/management/commands/__init__.py b/pulp_gem/app/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pulp_gem/app/management/commands/datarepair-shallow-gems.py b/pulp_gem/app/management/commands/datarepair-shallow-gems.py new file mode 100644 index 0000000..0f43e52 --- /dev/null +++ b/pulp_gem/app/management/commands/datarepair-shallow-gems.py @@ -0,0 +1,60 @@ +from gettext import gettext as _ + +from django.core.management import BaseCommand + +from pulpcore.plugin.util import get_url +from pulpcore.plugin.models import RepositoryContent +from pulp_gem.app.models import ShallowGemContent +from pulp_gem.app.serializers import GemContentSerializer + + +def replace_content(old_content, new_content): + """Exchange all occurances of `old_content` in repository versions with `new_content`.""" + RepositoryContent.objects.filter(content_id=old_content.pk).update(content_id=new_content.pk) + + +class Command(BaseCommand): + """ + Django management command for migrating shallow gems. + """ + + help = "This script migrates the pre GA generated gem content if artifacts are available." + + def add_arguments(self, parser): + """Set up arguments.""" + parser.add_argument( + "--dry-run", + action="store_true", + help=_("Don't modify anything, just collect results."), + ) + + def handle(self, *args, **options): + dry_run = options["dry_run"] + failed_gems = 0 + migrated_gems = 0 + + shallow_gem_qs = ShallowGemContent.objects.all() + count = shallow_gem_qs.count() + print(f"Shallow Gems count: {count}") + if count == 0: + return + + for sgem in shallow_gem_qs: + try: + artifact = sgem.contentartifact_set.get(relative_path=sgem.relative_path).artifact + serializer = GemContentSerializer(data={"artifact": get_url(artifact)}) + serializer.is_valid(raise_exception=True) + assert serializer.validated_data["name"] == sgem.name + assert serializer.validated_data["version"] == sgem.version + if not dry_run: + gem = serializer.create(serializer.validated_data) + replace_content(sgem, gem) + sgem.delete() + except Exception as e: + failed_gems += 1 + print(f"Failed to migrate gem '{sgem.name}' '{sgem.version}': {e}") + else: + migrated_gems += 1 + + print(f"Successfully migrated gems: {migrated_gems}") + print(f"Gems failed to migrate: {failed_gems}") diff --git a/pulp_gem/app/migrations/0005_rename_gemcontent_shallowgemcontent.py b/pulp_gem/app/migrations/0005_rename_gemcontent_shallowgemcontent.py new file mode 100644 index 0000000..8d9ab2b --- /dev/null +++ b/pulp_gem/app/migrations/0005_rename_gemcontent_shallowgemcontent.py @@ -0,0 +1,27 @@ +# Generated by Django 4.2.1 on 2023-06-14 14:37 + +from django.db import migrations + + +def rename_gem_up(apps, schema_editor): + Content = apps.get_model("core", "Content") + Content.objects.filter(pulp_type="gem.gem").update(pulp_type="gem.shallow-gem") + + +def rename_gem_down(apps, schema_editor): + Content = apps.get_model("core", "Content") + Content.objects.filter(pulp_type="gem.shallow-gem").update(pulp_type="gem.gem") + + +class Migration(migrations.Migration): + dependencies = [ + ("gem", "0004_alter_gemcontent_content_ptr_and_more"), + ] + + operations = [ + migrations.RenameModel( + old_name="GemContent", + new_name="ShallowGemContent", + ), + migrations.RunPython(code=rename_gem_up, reverse_code=rename_gem_down, elidable=True), + ] diff --git a/pulp_gem/app/migrations/0006_gemremote_excludes_gemremote_includes_and_more.py b/pulp_gem/app/migrations/0006_gemremote_excludes_gemremote_includes_and_more.py new file mode 100644 index 0000000..1753f9a --- /dev/null +++ b/pulp_gem/app/migrations/0006_gemremote_excludes_gemremote_includes_and_more.py @@ -0,0 +1,57 @@ +# Generated by Django 4.2.1 on 2023-06-14 14:53 + +import django.contrib.postgres.fields.hstore +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("gem", "0005_rename_gemcontent_shallowgemcontent"), + ] + + operations = [ + migrations.AddField( + model_name="gemremote", + name="excludes", + field=django.contrib.postgres.fields.hstore.HStoreField(null=True), + ), + migrations.AddField( + model_name="gemremote", + name="includes", + field=django.contrib.postgres.fields.hstore.HStoreField(null=True), + ), + migrations.AddField( + model_name="gemremote", + name="prereleases", + field=models.BooleanField(default=False), + ), + migrations.CreateModel( + name="GemContent", + fields=[ + ( + "content_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="core.content", + ), + ), + ("name", models.TextField()), + ("version", models.TextField()), + ("checksum", models.CharField(db_index=True, max_length=64)), + ("dependencies", django.contrib.postgres.fields.hstore.HStoreField(default=dict)), + ("required_ruby_version", models.TextField(null=True)), + ("required_rubygems_version", models.TextField(null=True)), + ("prerelease", models.BooleanField(default=False)), + ], + options={ + "default_related_name": "%(app_label)s_%(model_name)s", + "unique_together": {("checksum",)}, + }, + bases=("core.content",), + ), + ] diff --git a/pulp_gem/app/models.py b/pulp_gem/app/models.py index afc29e1..0ec753f 100644 --- a/pulp_gem/app/models.py +++ b/pulp_gem/app/models.py @@ -1,11 +1,11 @@ from logging import getLogger -from django.db import models, utils from pathlib import PurePath from tempfile import NamedTemporaryFile +from django.contrib.postgres.fields import HStoreField +from django.db import models from pulpcore.plugin.models import ( - Artifact, Content, Publication, Distribution, @@ -19,6 +19,39 @@ log = getLogger(__name__) +class ShallowGemContent(Content): + """ + The "shallow-gem" content type. + + Content of this type represents a ruby gem file with its spec data. + This is the old deprecated format that is only carried around for legacy installations. + + Fields: + name (str): The name of the gem. + version (str): The version of the gem. + + """ + + TYPE = "shallow-gem" + + name = models.TextField(blank=False, null=False) + version = models.TextField(blank=False, null=False) + + @property + def relative_path(self): + """The relative path this gem is stored under for the content app.""" + return f"gems/{self.name}-{self.version}.gem" + + @property + def gemspec_path(self): + """The path for this gem's gemspec for the content app.""" + return f"quick/Marshal.4.8/{self.name}-{self.version}.gemspec.rz" + + class Meta: + default_related_name = "%(app_label)s_%(model_name)s" + unique_together = ("name", "version") + + class GemContent(Content): """ The "gem" content type. @@ -33,9 +66,15 @@ class GemContent(Content): """ TYPE = "gem" + repo_key_fields = ("name", "version") name = models.TextField(blank=False, null=False) version = models.TextField(blank=False, null=False) + checksum = models.CharField(max_length=64, null=False, db_index=True) + prerelease = models.BooleanField(default=False) + dependencies = HStoreField(default=dict) + required_ruby_version = models.TextField(null=True) + required_rubygems_version = models.TextField(null=True) @property def relative_path(self): @@ -49,29 +88,23 @@ def gemspec_path(self): @staticmethod def init_from_artifact_and_relative_path(artifact, relative_path): - """""" - name, version, spec_data = analyse_gem(artifact.file) - content = GemContent(name=name, version=version) + gem_info, spec_data = analyse_gem(artifact.file) + content = GemContent(**gem_info) relative_path = content.relative_path with NamedTemporaryFile(mode="wb", dir=".", delete=False) as temp_file: temp_file.write(spec_data) temp_file.flush() - spec_artifact = Artifact.init_and_validate(temp_file.name) - try: - spec_artifact.save() - except utils.IntegrityError: - spec_artifact = Artifact.objects.get(spec_artifact.q()) - spec_artifact.touch() spec_relative_path = content.gemspec_path - artifacts = {relative_path: artifact, spec_relative_path: spec_artifact} + # Spec artifact will be on-demand at this point + artifacts = {relative_path: artifact, spec_relative_path: None} return content, artifacts class Meta: default_related_name = "%(app_label)s_%(model_name)s" - unique_together = ("name", "version") + unique_together = ("checksum",) class GemDistribution(Distribution): @@ -103,6 +136,10 @@ class GemRemote(Remote): TYPE = "gem" + prereleases = models.BooleanField(default=False) + includes = HStoreField(null=True) + excludes = HStoreField(null=True) + def get_remote_artifact_content_type(self, relative_path=None): """ Return a modified GemContent class that has a reference to this remote. @@ -127,7 +164,7 @@ class GemRepository(Repository): """ TYPE = "gem" - CONTENT_TYPES = [GemContent] + CONTENT_TYPES = [GemContent, ShallowGemContent] REMOTE_TYPES = [GemRemote] class Meta: diff --git a/pulp_gem/app/serializers.py b/pulp_gem/app/serializers.py index 67b0208..c31143c 100644 --- a/pulp_gem/app/serializers.py +++ b/pulp_gem/app/serializers.py @@ -1,19 +1,21 @@ from gettext import gettext as _ +import tempfile +import hashlib import os +from django.db import IntegrityError from rest_framework.serializers import ( + BooleanField, CharField, ChoiceField, FileField, - HyperlinkedRelatedField, + HStoreField, ValidationError, ) -from pulpcore.plugin.files import PulpTemporaryUploadedFile -from pulpcore.plugin.models import Artifact, Publication, Remote, Repository, RepositoryVersion +from pulpcore.plugin.models import Artifact, Publication, Remote, Repository from pulpcore.plugin.serializers import ( - ArtifactSerializer, DetailRelatedField, MultipleArtifactContentSerializer, PublicationSerializer, @@ -35,15 +37,17 @@ def _artifact_from_data(raw_data): - tmpfile = PulpTemporaryUploadedFile( - "tmpfile", "application/octet-stream", len(raw_data), "", "" - ) - tmpfile.write(raw_data) + sha256 = hashlib.sha256(raw_data).hexdigest() + artifact = Artifact.objects.filter(sha256=sha256).first() + if artifact: + return artifact - artifact_serializer = ArtifactSerializer(data={"file": tmpfile}) - artifact_serializer.is_valid(raise_exception=True) + with tempfile.NamedTemporaryFile("wb", dir=".", delete=False) as tmpfile: + tmpfile.write(raw_data) - return artifact_serializer.save() + artifact = Artifact.init_and_validate(tmpfile.name, expected_digests={"sha256": sha256}) + artifact.save() + return artifact class GemContentSerializer(MultipleArtifactContentSerializer): @@ -63,15 +67,23 @@ class GemContentSerializer(MultipleArtifactContentSerializer): required=False, write_only=True, ) - repository = HyperlinkedRelatedField( + repository = DetailRelatedField( help_text=_("A URI of a repository the new content unit should be associated with."), required=False, write_only=True, + view_name_pattern=r"repositories(-.*/.*)-detail", queryset=Repository.objects.all(), - view_name="repositories-detail", ) name = CharField(help_text=_("Name of the gem"), read_only=True) version = CharField(help_text=_("Version of the gem"), read_only=True) + prerelease = BooleanField(help_text=_("Whether the gem is a prerelease"), read_only=True) + dependencies = HStoreField(read_only=True) + required_ruby_version = CharField( + help_text=_("Required ruby version of the gem"), read_only=True + ) + required_rubygems_version = CharField( + help_text=_("Required rubygems version of the gem"), read_only=True + ) def __init__(self, *args, **kwargs): """Initializer for GemContentSerializer.""" @@ -98,40 +110,44 @@ def deferred_validate(self, data): """Validate the GemContent data (deferred).""" artifact = data.pop("artifact") - name, version, spec_data = analyse_gem(artifact.file) - relative_path = os.path.join("gems", name + "-" + version + ".gem") + gem_info, spec_data = analyse_gem(artifact.file) + relative_path = os.path.join("gems", gem_info["name"] + "-" + gem_info["version"] + ".gem") spec_artifact = _artifact_from_data(spec_data) - spec_relative_path = os.path.join("quick/Marshal.4.8", name + "-" + version + ".gemspec.rz") + spec_relative_path = os.path.join( + "quick/Marshal.4.8", gem_info["name"] + "-" + gem_info["version"] + ".gemspec.rz" + ) - data["name"] = name - data["version"] = version + data.update(gem_info) data["artifacts"] = {relative_path: artifact, spec_relative_path: spec_artifact} - - # Validate uniqueness - content = GemContent.objects.filter(name=name, version=version) - if content.exists(): - raise ValidationError( - _( - "There is already a gem content with name '{name}' and version '{version}'." - ).format(name=name, version=version) - ) + data["checksum"] = artifact.sha256 return data + def retrieve(self, validated_data): + return GemContent.objects.filter(checksum=validated_data["checksum"]).first() + def create(self, validated_data): """Save the GemContent unit. This must be used inside a task that locks on the Artifact and if given, the repository. """ repository = validated_data.pop("repository", None) - content = super().create(validated_data) + content = self.retrieve(validated_data) + if content is None: + try: + content = super().create(validated_data) + except IntegrityError: + content = self.retrieve(validated_data) + if content is None: + raise if repository: + repository.cast() content_to_add = self.Meta.model.objects.filter(pk=content.pk) # create new repo version with uploaded package - with RepositoryVersion.create(repository) as new_version: + with repository.new_version() as new_version: new_version.add_content(content_to_add) return content @@ -142,6 +158,10 @@ class Meta: "repository", "name", "version", + "prerelease", + "dependencies", + "required_ruby_version", + "required_rubygems_version", ) model = GemContent @@ -157,9 +177,12 @@ class GemRemoteSerializer(RemoteSerializer): choices=Remote.POLICY_CHOICES, default=Remote.IMMEDIATE, ) + prereleases = BooleanField(default=False) + includes = HStoreField(required=False, allow_null=True) + excludes = HStoreField(required=False, allow_null=True) class Meta: - fields = RemoteSerializer.Meta.fields + fields = RemoteSerializer.Meta.fields + ("prereleases", "includes", "excludes") model = GemRemote diff --git a/pulp_gem/app/tasks/publishing.py b/pulp_gem/app/tasks/publishing.py index f1878d5..e25c6f0 100644 --- a/pulp_gem/app/tasks/publishing.py +++ b/pulp_gem/app/tasks/publishing.py @@ -1,16 +1,25 @@ -import logging -import re +import datetime import gzip +import hashlib +import logging +import os import shutil from gettext import gettext as _ from packaging import version +from django.conf import settings from django.core.files import File +from django.db import transaction from jinja2 import Template from pathlib import Path -from pulpcore.plugin.models import RepositoryVersion, PublishedMetadata +from pulpcore.plugin.models import ( + ContentArtifact, + RepositoryVersion, + PublishedArtifact, + PublishedMetadata, +) from pulp_gem.app.models import GemContent, GemPublication from pulp_gem.specs import write_specs, Key @@ -43,14 +52,36 @@ def _publish_specs(specs, relative_path, publication): with open(relative_path, "rb") as f_in: with gzip.open(relative_path + ".gz", "wb") as f_out: shutil.copyfileobj(f_in, f_out) - specs_metadata = PublishedMetadata.create_from_file( + PublishedMetadata.create_from_file( publication=publication, file=File(open(relative_path, "rb")) ) - specs_metadata.save() - specs_metadata_gz = PublishedMetadata.create_from_file( + PublishedMetadata.create_from_file( publication=publication, file=File(open(relative_path + ".gz", "rb")) ) - specs_metadata_gz.save() + + +def _publish_compact_index(lines, relative_path, publication, timestamp=False, with_list=False): + with open(relative_path, "w") as fp: + if timestamp: + timestamp = datetime.datetime.utcnow().isoformat(timespec="seconds") + fp.write(f"created_at: {timestamp}Z\n") + fp.write("---\n") + for line in lines: + fp.write(line + "\n") + metadata = PublishedMetadata.create_from_file( + publication=publication, file=File(open(relative_path, "rb")) + ) + if with_list: + with transaction.atomic(): + list_path = relative_path + ".list" + pm = PublishedMetadata.objects.create(relative_path=list_path, publication=publication) + ca = ContentArtifact.objects.create( + relative_path=list_path, content=pm, artifact=metadata._artifacts.first() + ) + PublishedArtifact.objects.create( + relative_path=list_path, content_artifact=ca, publication=publication + ) + return metadata def _create_index(publication, path="", links=None): @@ -63,10 +94,9 @@ def _create_index(publication, path="", links=None): with open(index_path, "w") as index: index.write(template.render(links=links, path=path)) - index_metadata = PublishedMetadata.create_from_file( + return PublishedMetadata.create_from_file( relative_path=index_path, publication=publication, file=File(open(index_path, "rb")) ) - index_metadata.save() def publish(repository_version_pk): @@ -96,13 +126,13 @@ def publish(repository_version_pk): .order_by("-pulp_created") .iterator() ): - if re.fullmatch(r"[0-9.]*", content.version): + if content.prerelease: + prerelease_specs.append(Key(content.name, content.version)) + else: specs.append(Key(content.name, content.version)) old_ver = latest_versions.get(content.name) if old_ver is None or version.parse(old_ver) < version.parse(content.version): latest_versions[content.name] = content.version - else: - prerelease_specs.append(Key(content.name, content.version)) gems.append(content.relative_path) gemspecs.append(content.gemspec_path) latest_specs = [Key(name, ver) for name, ver in latest_versions.items()] @@ -110,6 +140,37 @@ def publish(repository_version_pk): _publish_specs(specs, "specs.4.8", publication) _publish_specs(latest_specs, "latest_specs.4.8", publication) _publish_specs(prerelease_specs, "prerelease_specs.4.8", publication) + + # compact_index + gems_qs = GemContent.objects.filter(pk__in=publication.repository_version.content) + names_qs = gems_qs.order_by("name").values_list("name", flat=True).distinct() + _publish_compact_index(names_qs, "names", publication, with_list=True) + + versions_lines = [] + os.mkdir("info") + for name in names_qs: + lines = [] + for gem in gems_qs.filter(name=name): + deps = ",".join((f"{key}:{value}" for key, value in gem.dependencies.items())) + line = f"{gem.version} {deps}|checksum:{gem.checksum}" + if gem.required_ruby_version: + line += f",ruby:{gem.required_ruby_version}" + if gem.required_rubygems_version: + line += f",rubygems:{gem.required_rubygems_version}" + lines.append(line) + info_metadata = _publish_compact_index(lines, f"info/{name}", publication) + versions = ",".join(gems_qs.filter(name=name).values_list("version", flat=True)) + if "md5" in settings.ALLOWED_CONTENT_CHECKSUMS: + md5_sum = info_metadata._artifacts.first().md5 + else: + artifact = info_metadata._artifacts.first() + artifact.file.seek(0) + md5_sum = hashlib.md5(artifact.file.read()).hexdigest() + versions_lines.append(f"{name} {versions} {md5_sum}") + _publish_compact_index( + versions_lines, "versions", publication, timestamp=True, with_list=True + ) + _create_index( publication, path="", @@ -119,10 +180,16 @@ def publish(repository_version_pk): "specs.4.8", "latest_specs.4.8", "prerelease_specs.4.8", + "names", + "names.list", + "versions", + "versions.list", + "info/", ], ) _create_index(publication, path="gems/", links=gems) - _create_index(publication, path="quick/", links=[]) + _create_index(publication, path="quick/", links=["quick/Marshal.4.8/"]) _create_index(publication, path="quick/Marshal.4.8/", links=gemspecs) + _create_index(publication, path="info/", links=(f"info/{name}" for name in names_qs)) log.info(_("Publication: {publication} created").format(publication=publication.pk)) diff --git a/pulp_gem/app/tasks/synchronizing.py b/pulp_gem/app/tasks/synchronizing.py index f0017bb..235b885 100644 --- a/pulp_gem/app/tasks/synchronizing.py +++ b/pulp_gem/app/tasks/synchronizing.py @@ -1,10 +1,9 @@ import logging -import os from gettext import gettext as _ -from urllib.parse import urlparse, urlunparse +from urllib.parse import urljoin -from asgiref.sync import sync_to_async +from django.conf import settings from pulpcore.plugin.models import Artifact, ProgressReport, Remote, Repository from pulpcore.plugin.stages import ( @@ -12,50 +11,22 @@ DeclarativeContent, DeclarativeVersion, Stage, - ArtifactDownloader, - ArtifactSaver, - QueryExistingContents, - ContentSaver, - RemoteArtifactSaver, ) from pulp_gem.app.models import GemContent, GemRemote -from pulp_gem.specs import read_specs +from pulp_gem.specs import ( + NAME_REGEX, + VERSION_REGEX, + PRERELEASE_VERSION_REGEX, + read_versions, + read_info, + ruby_ver_includes, +) log = logging.getLogger(__name__) -class UpdateExistingContentArtifacts(Stage): - """ - Stage to update declarative_artifacts from existing content. - - A Stages API stage that sets existing - :class:`~pulpcore.plugin.models.Artifact` in - :class:`~pulpcore.plugin.stages.DeclarativeArtifact` instances from - :class:`~pulpcore.plugin.stages.DeclarativeContent` units if the respective - :class:`~pulpcore.plugin.models.Content` is already existing. - """ - - async def run(self): - """ - The coroutine for this stage. - - Returns: - The coroutine for this stage. - - """ - async for d_content in self.items(): - if d_content.content.pk is not None: - for d_artifact in d_content.d_artifacts: - content_artifact = await sync_to_async( - d_content.content.contentartifact_set.select_related("artifact").get - )(relative_path=d_artifact.relative_path) - if content_artifact.artifact is not None: - d_artifact.artifact = content_artifact.artifact - await self.put(d_content) - - def synchronize(remote_pk, repository_pk, mirror=False): """ Create a new version of the repository that is synchronized with the remote as specified. @@ -76,7 +47,7 @@ def synchronize(remote_pk, repository_pk, mirror=False): raise ValueError(_("A remote must have a url specified to synchronize.")) first_stage = GemFirstStage(remote) - dv = GemDeclarativeVersion(first_stage, repository, mirror=mirror) + dv = DeclarativeVersion(first_stage, repository, mirror=mirror) dv.create() @@ -102,72 +73,81 @@ async def run(self): # Interpret policy to download Artifacts or not deferred_download = self.remote.policy != Remote.IMMEDIATE - async with ProgressReport(message="Downloading Metadata") as progress: - parsed_url = urlparse(self.remote.url) - root_dir = parsed_url.path - specs_path = os.path.join(root_dir, "specs.4.8.gz") - specs_url = urlunparse(parsed_url._replace(path=specs_path)) - downloader = self.remote.get_downloader(url=specs_url) - result = await downloader.run() - await progress.aincrement() - - async with ProgressReport(message="Parsing Metadata") as progress: - for key in read_specs(result.path): - relative_path = os.path.join("gems", key.name + "-" + key.version + ".gem") - path = os.path.join(root_dir, relative_path) - url = urlunparse(parsed_url._replace(path=path)) - - spec_relative_path = os.path.join( - "quick/Marshal.4.8", key.name + "-" + key.version + ".gemspec.rz" - ) - spec_path = os.path.join(root_dir, spec_relative_path) - spec_url = urlunparse(parsed_url._replace(path=spec_path)) - gem = GemContent(name=key.name, version=key.version) - da_gem = DeclarativeArtifact( - artifact=Artifact(), - url=url, - relative_path=relative_path, - remote=self.remote, - deferred_download=deferred_download, - ) - da_spec = DeclarativeArtifact( - artifact=Artifact(), - url=spec_url, - relative_path=spec_relative_path, - remote=self.remote, - deferred_download=deferred_download, - ) - dc = DeclarativeContent(content=gem, d_artifacts=[da_gem, da_spec]) - await progress.aincrement() - await self.put(dc) - - -class GemDeclarativeVersion(DeclarativeVersion): - """ - Custom implementation of Declarative version. - """ - - def pipeline_stages(self, new_version): - """ - Build the list of pipeline stages feeding into the ContentUnitAssociation stage. - - This is overwritten to create a custom pipeline. - - Args: - new_version (:class:`~pulpcore.plugin.models.RepositoryVersion`): The - new repository version that is going to be built. - - Returns: - list: List of :class:`~pulpcore.plugin.stages.Stage` instances - - """ - pipeline = [ - self.first_stage, - QueryExistingContents(), - UpdateExistingContentArtifacts(), - ArtifactDownloader(), - ArtifactSaver(), - ContentSaver(), - RemoteArtifactSaver(), - ] - return pipeline + async with ProgressReport( + message="Downloading versions list", total=1 + ) as pr_download_versions: + versions_url = urljoin(self.remote.url, "versions") + versions_downloader = self.remote.get_downloader(url=versions_url) + versions_result = await versions_downloader.run() + await pr_download_versions.aincrement() + + async with ProgressReport(message="Parsing versions list") as pr_parse_versions: + async with ProgressReport(message="Parsing versions info") as pr_parse_info: + async for name, versions, md5_sum in read_versions(versions_result.path): + await pr_parse_versions.aincrement() + if not NAME_REGEX.match(name): + log.warn(f"Skipping invalid gem name: '{name}'.") + continue + if not self.remote.prereleases: + versions = [version for version in versions if VERSION_REGEX.match(version)] + else: + versions = [ + version + for version in versions + if PRERELEASE_VERSION_REGEX.match(version) + ] + if self.remote.includes: + if name not in self.remote.includes: + continue + version_requirements = self.remote.includes[name] + if version_requirements is not None: + versions = [ + version + for version in versions + if ruby_ver_includes(version_requirements, version) + ] + if self.remote.excludes: + if name in self.remote.excludes: + version_requirements = self.remote.excludes[name] + if version_requirements is None: + continue + versions = [ + version + for version in versions + if not ruby_ver_includes(version_requirements, version) + ] + if not versions: + continue + info_url = urljoin(urljoin(self.remote.url, "info/"), name) + if "md5" in settings.ALLOWED_CONTENT_CHECKSUMS: + extra_kwargs = {"expected_digests": {"md5": md5_sum}} + else: + extra_kwargs = {} + log.warn("Checksum of info file for '{}' could not be validated.", name) + info_downloader = self.remote.get_downloader(url=info_url, **extra_kwargs) + info_result = await info_downloader.run() + async for gem_info in read_info(info_result.path, versions): + gem_info["name"] = name + gem = GemContent(**gem_info) + gem_path = gem.relative_path + gem_url = urljoin(self.remote.url, gem_path) + gemspec_path = gem.gemspec_path + gemspec_url = urljoin(self.remote.url, gemspec_path) + + da_gem = DeclarativeArtifact( + artifact=Artifact(sha256=gem_info["checksum"]), + url=gem_url, + relative_path=gem_path, + remote=self.remote, + deferred_download=deferred_download, + ) + da_gemspec = DeclarativeArtifact( + artifact=Artifact(), + url=gemspec_url, + relative_path=gemspec_path, + remote=self.remote, + deferred_download=deferred_download, + ) + dc = DeclarativeContent(content=gem, d_artifacts=[da_gem, da_gemspec]) + await pr_parse_info.aincrement() + await self.put(dc) diff --git a/pulp_gem/specs.py b/pulp_gem/specs.py index aa3c00d..f9a6db8 100644 --- a/pulp_gem/specs.py +++ b/pulp_gem/specs.py @@ -1,8 +1,11 @@ from collections import namedtuple +import aiofiles import zlib import gzip +import re import yaml +from itertools import zip_longest from tarfile import TarFile import rubymarshal.classes @@ -10,10 +13,142 @@ import rubymarshal.reader +NAME_REGEX = re.compile(r"[\w\.-]+") +VERSION_REGEX = re.compile(r"\d+(?:\.\d+)*") +PRERELEASE_VERSION_REGEX = NAME_REGEX + # Natural key. Key = namedtuple("Key", ("name", "version")) +def _ver_tokens(version): + numeric = True + value = "" + for char in version: + if char >= "0" and char <= "9": + if not numeric: + if value: + yield value + value = "" + numeric = True + value += char + elif char == ".": + yield value + value = "" + numeric = True + else: + if numeric: + if value: + yield value + value = "" + numeric = False + value += char + yield value + + +def ruby_ver_cmp(ver1, ver2): + # https://docs.ruby-lang.org/en/2.4.0/Gem/Version.html + for part1, part2 in zip_longest(_ver_tokens(ver1), _ver_tokens(ver2), fillvalue="0"): + try: + val1 = [int(part1), ""] + except ValueError: + val1 = [-1, part1] + try: + val2 = [int(part2), ""] + except ValueError: + val2 = [-1, part2] + if val1 > val2: + return 1 + if val1 < val2: + return -1 + return 0 + + +def ruby_ver_includes(requirements, version): + for requirement in requirements.split("&"): + op, ver = requirement.split(" ", maxsplit=1) + cmp = ruby_ver_cmp(version, ver) + if op == "=" and cmp != 0: + return False + elif op == "<" and cmp != -1: + return False + elif op == "<=" and cmp == 1: + return False + elif op == ">" and cmp != 1: + return False + elif op == ">=" and cmp == -1: + return False + return True + + +async def read_versions(relative_path): + # File starts with: + # created_at: + # --- + async with aiofiles.open(relative_path, mode="r") as fp: + results = {} + preamble = True + async for line in fp: + line = line.strip() + if line == "---": + preamble = False + continue + if preamble: + continue + name, versions, md5_sum = line.split(" ", maxsplit=2) + versions = versions.split(",") + entry = results.get(name) or ([], "") + results[name] = (entry[0] + versions, md5_sum) + for name, (versions, md5_sum) in results.items(): + # Sanitize name + if not NAME_REGEX.match(name): + raise ValueError(f"Invalid gem name: {name}") + yield name, versions, md5_sum + + +async def read_info(relative_path, versions): + # File starts with: + # --- + async with aiofiles.open(relative_path, mode="r") as fp: + preamble = True + async for line in fp: + line = line.strip() + if line == "---": + preamble = False + continue + if preamble: + continue + gem_info = {} + front, back = line.split("|") + version, dependencies = front.split(" ", maxsplit=1) + if version not in versions: + continue + # Sanitize version + if VERSION_REGEX.match(version): + gem_info["prerelease"] = False + elif PRERELEASE_VERSION_REGEX.match(version): + gem_info["prerelease"] = True + else: + raise ValueError(f"Invalid version string: {version}") + gem_info["version"] = version + dependencies = dependencies.strip() + if dependencies: + gem_info["dependencies"] = dict( + (item.split(":", maxsplit=1) for item in dependencies.split(",")) + ) + for stmt in back.split(","): + key, value = stmt.split(":") + if key == "checksum": + gem_info["checksum"] = value + elif key == "ruby": + gem_info["required_ruby_version"] = value + elif key == "rubygems": + gem_info["required_rubygems_version"] = value + else: + raise ValueError(f"Invalid requirement: {stmt}") + yield gem_info + + def read_specs(relative_path): """ Read rubygem specs from file. @@ -46,12 +181,20 @@ def write_specs(keys, relative_path): rubymarshal.writer.write(fd, specs) +class RubyMarshalYamlLoader(yaml.SafeLoader): + pass + + def _yaml_ruby_constructor(loader, suffix, node): value = loader.construct_mapping(node) return rubymarshal.classes.UsrMarshal(suffix, value) -yaml.add_multi_constructor("!ruby/object:", _yaml_ruby_constructor, Loader=yaml.SafeLoader) +yaml.add_multi_constructor("!ruby/object:", _yaml_ruby_constructor, Loader=RubyMarshalYamlLoader) + + +def _collapse_requirement(data): + return "&".join([f"{req[0]} {req[1].values['version']}" for req in data.values["requirements"]]) def analyse_gem(file_obj): @@ -60,8 +203,30 @@ def analyse_gem(file_obj): """ with TarFile(fileobj=file_obj) as archive: with archive.extractfile("metadata.gz") as md_file: - data = yaml.safe_load(gzip.decompress(md_file.read())) + data = yaml.load(gzip.decompress(md_file.read()), Loader=RubyMarshalYamlLoader) + gem_info = { + "name": data.values["name"], + "version": data.values["version"].values["version"], + } + # Sanitize name + if not NAME_REGEX.match(gem_info["name"]): + raise ValueError(f"Invalid gem name: {gem_info['name']}") + # Sanitize version + if VERSION_REGEX.match(gem_info["version"]): + gem_info["prerelease"] = False + elif PRERELEASE_VERSION_REGEX.match(gem_info["version"]): + gem_info["prerelease"] = True + else: + raise ValueError(f"Invalid version string: {gem_info['version']}") + for key in ("required_ruby_version", "required_rubygems_version"): + if (requirement := data.values.get(key)) is not None: + gem_info[key] = _collapse_requirement(requirement) + if (dependencies := data.values.get("dependencies")) is not None: + gem_info["dependencies"] = { + dep.values["name"]: _collapse_requirement(dep.values["requirement"]) + for dep in dependencies + } # Workaroud del data.values["date"] zdata = zlib.compress(rubymarshal.writer.writes(data)) - return data.values["name"], data.values["version"].values["version"], zdata + return gem_info, zdata diff --git a/pulp_gem/tests/functional/constants.py b/pulp_gem/tests/functional/constants.py index 223d126..09f43ce 100644 --- a/pulp_gem/tests/functional/constants.py +++ b/pulp_gem/tests/functional/constants.py @@ -29,7 +29,7 @@ GEM_FIXTURE_URL = urljoin(PULP_FIXTURES_BASE_URL, "gem/") """The URL to a gem repository.""" -GEM_FIXTURE_COUNT = 4 +GEM_FIXTURE_COUNT = 6 """The number of content units available at :data:`GEM_FIXTURE_URL`.""" # This is 4 stable gems. There are also 2 prerelease gems not currently synced. diff --git a/pulp_gem/tests/unit/test_serializers.py b/pulp_gem/tests/unit/test_serializers.py index 9ca8aa1..0be5b0c 100644 --- a/pulp_gem/tests/unit/test_serializers.py +++ b/pulp_gem/tests/unit/test_serializers.py @@ -7,6 +7,19 @@ from pulpcore.plugin.models import Artifact +CHECKSUM_LEN = { + "md5": 32, + "sha1": 40, + "sha224": 56, + "sha256": 64, + "sha384": 96, + "sha512": 128, +} + + +def _checksums(char): + return {name: char * CHECKSUM_LEN[name] for name in settings.ALLOWED_CONTENT_CHECKSUMS} + class TestGemContentSerializer(TestCase): """Test GemContentSerializer.""" @@ -14,20 +27,14 @@ class TestGemContentSerializer(TestCase): def setUp(self): """Set up the GemContentSerializer tests.""" self.artifact = Artifact.objects.create( - sha224="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - sha256="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - sha384="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", # noqa - sha512="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", # noqa size=1024, file=SimpleUploadedFile("test_filename_a", b"test content_a"), + **_checksums("a"), ) self.artifact2 = Artifact.objects.create( - sha224="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - sha256="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - sha384="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", # noqa - sha512="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", # noqa size=1024, file=SimpleUploadedFile("test_filename_b", b"test content_b"), + **_checksums("b"), ) @patch("pulp_gem.app.serializers._artifact_from_data") @@ -35,11 +42,11 @@ def setUp(self): def test_valid_data(self, ANALYZE_GEM, _ARTIFACT_FROM_DATA): """Test that the GemContentSerializer accepts valid data.""" # Preparation - ANALYZE_GEM.return_value = ("testname", "1.2.3-test", "---\n...") + ANALYZE_GEM.return_value = ({"name": "testname", "version": "1.2.3-test"}, "---\n...") _ARTIFACT_FROM_DATA.return_value = self.artifact2 data = {"artifact": "{}artifacts/{}/".format(settings.V3_API_ROOT, self.artifact.pk)} serializer = GemContentSerializer(data=data) - self.assertTrue(serializer.is_valid()) + assert serializer.is_valid() # Verification ANALYZE_GEM.called_once_with(self.artifact) _ARTIFACT_FROM_DATA.called_once_with("---\n...") @@ -47,14 +54,14 @@ def test_valid_data(self, ANALYZE_GEM, _ARTIFACT_FROM_DATA): @patch("pulp_gem.app.serializers._artifact_from_data") @patch("pulp_gem.app.serializers.analyse_gem") def test_duplicate_data(self, ANALYZE_GEM, _ARTIFACT_FROM_DATA): - """Test that the GemContentSerializer does not accept data.""" + """Test that the GemContentSerializer does accept duplicate data.""" # Preparation - ANALYZE_GEM.return_value = ("testname", "1.2.3-test", "---\n...") + ANALYZE_GEM.return_value = ({"name": "testname", "version": "1.2.3-test"}, "---\n...") _ARTIFACT_FROM_DATA.return_value = self.artifact2 data = {"artifact": "{}artifacts/{}/".format(settings.V3_API_ROOT, self.artifact.pk)} serializer = GemContentSerializer(data=data) - self.assertTrue(serializer.is_valid()) + assert serializer.is_valid() serializer.save() # Test serializer = GemContentSerializer(data=data) - self.assertFalse(serializer.is_valid()) + assert serializer.is_valid() diff --git a/pulp_gem/tests/unit/test_spec.py b/pulp_gem/tests/unit/test_spec.py new file mode 100644 index 0000000..19178af --- /dev/null +++ b/pulp_gem/tests/unit/test_spec.py @@ -0,0 +1,22 @@ +from pulp_gem.specs import ruby_ver_cmp, ruby_ver_includes + + +def test_version_cmp(): + assert ruby_ver_cmp("0.0.0", "0") == 0 + assert ruby_ver_cmp("0", "0.0.0") == 0 + assert ruby_ver_cmp("1.0.0", "0") == 1 + assert ruby_ver_cmp("0", "1.0.0") == -1 + assert ruby_ver_cmp("1a", "1.a") == 0 + assert ruby_ver_cmp("1.0", "1.a") == 1 + assert ruby_ver_cmp("1.0", "1.0a") == 1 + assert ruby_ver_cmp("1.0a2", "1.0.a.1") == 1 + assert ruby_ver_cmp("1.0b1", "1.0.a.2") == 1 + + +def test_version_includes(): + assert ruby_ver_includes(">= 1&< 3", "1.0.0") + assert ruby_ver_includes(">= 1&< 3", "2.0.0") + assert not ruby_ver_includes(">= 1&< 3", "3.0.0") + assert ruby_ver_includes(">= 1&< 3", "1.5.a0") + assert ruby_ver_includes(">= 1&< 3", "3.0.0a5") + assert not ruby_ver_includes(">= 1&< 3", "3.0.1a5") diff --git a/setup.py b/setup.py index fac7227..8b704e6 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open("requirements.txt") as requirements: requirements = requirements.readlines() -with open("README.rst") as f: +with open("README.md") as f: long_description = f.read() setup( @@ -14,9 +14,9 @@ description="Gemfile plugin for the Pulp Project", long_description=long_description, license="GPLv2+", - author="Matthias Dellweg", - author_email="dellweg@atix.de", - url="https://github.com/ATIX-AG/pulp_gem", + author="Pulp Project Developers", + author_email="pulp-dev@redhat.com", + url="https://pulpproject.org/", python_requires=">=3.8", install_requires=requirements, include_package_data=True, @@ -32,4 +32,9 @@ "Programming Language :: Python :: 3.7", ), entry_points={"pulpcore.plugin": ["pulp_gem = pulp_gem:default_app_config"]}, + project_urls={ + "Documentation": "https://docs.pulpproject.org/pulp_gem/", + "Source": "https://github.com/pulp/pulp_gem", + "Tracker": "https://github.com/pulp/pulp_gem/issues", + }, ) diff --git a/template_config.yml b/template_config.yml index 4b8d46b..b16c0d5 100644 --- a/template_config.yml +++ b/template_config.yml @@ -48,7 +48,13 @@ pulp_scheme: https pulp_settings: null pulp_settings_azure: null pulp_settings_gcp: null -pulp_settings_s3: null +pulp_settings_s3: + allowed_content_checksums: + - md5 + - sha224 + - sha256 + - sha384 + - sha512 pulp_settings_stream: null pulpprojectdotorg_key_id: null pydocstyle: true