feat(deps): include pyramid_openapi3 (#15553)

docker-compose.yml

@@ -102,7 +102,7 @@ services:
 
   web:
     image: warehouse:docker-compose
-    command: gunicorn --reload -b 0.0.0.0:8000 --access-logfile - --error-logfile - warehouse.wsgi:application
+    command: gunicorn --reload --reload-extra-file=warehouse/api/openapi.yaml -b 0.0.0.0:8000 --access-logfile - --error-logfile - warehouse.wsgi:application
     env_file: dev/environment
     pull_policy: never
     volumes: *base_volumes

requirements/main.in

@@ -47,6 +47,7 @@ pyramid>=2.0
 pymacaroons
 pyramid_jinja2>=2.5
 pyramid_mailer>=0.14.1
+pyramid_openapi3>=0.17.1
 pyramid_retry>=0.3
 pyramid_rpc>=0.7
 pyramid_services>=2.1

tests/unit/api/test_config.py

@@ -0,0 +1,86 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import orjson
+import pretend
+import pytest
+
+from warehouse.api import config
+
+
+def test_api_set_content_type():
+    response = pretend.stub()
+
+    @pretend.call_recorder
+    def view(context, request):
+        return response
+
+    info = pretend.stub(options={"api_version": "v1"})
+    wrapped_view = config._api_set_content_type(view, info)
+
+    context = pretend.stub()
+    request = pretend.stub(response=pretend.stub(content_type=None))
+
+    assert wrapped_view(context, request) is response
+    assert request.response.content_type == "application/vnd.pypi.v1+json"
+
+
+def test_api_set_content_type_no_api_version():
+    response = pretend.stub()
+
+    @pretend.call_recorder
+    def view(context, request):
+        return response
+
+    info = pretend.stub(options={})
+    wrapped_view = config._api_set_content_type(view, info)
+
+    context = pretend.stub()
+    request = pretend.stub(response=pretend.stub(content_type=None))
+
+    assert wrapped_view(context, request) is response
+    assert request.response.content_type is None
+
+
+@pytest.mark.parametrize("env_name", ["development", "production"])
+def test_includeme(monkeypatch, env_name):
+    # We use `str(Path(__file__).parent / 'openapi.yaml'` to get the path.
+    # In our test, monkeypatch to a known value.
+    monkeypatch.setattr(config, "__file__", "/mnt/dummy/config.py")
+
+    conf = pretend.stub(
+        add_view_deriver=pretend.call_recorder(
+            lambda deriver, over=None, under=None: None
+        ),
+        include=pretend.call_recorder(lambda x: None),
+        pyramid_openapi3_spec=pretend.call_recorder(lambda *a, **kw: None),
+        pyramid_openapi3_add_deserializer=pretend.call_recorder(lambda *a, **kw: None),
+        pyramid_openapi3_add_explorer=pretend.call_recorder(lambda *a, **kw: None),
+        registry=pretend.stub(settings={"warehouse.env": env_name}),
+    )
+
+    config.includeme(conf)
+
+    assert conf.add_view_deriver.calls == [pretend.call(config._api_set_content_type)]
+    assert conf.include.calls == [pretend.call("pyramid_openapi3")]
+    assert conf.pyramid_openapi3_spec.calls == [
+        pretend.call("/mnt/dummy/openapi.yaml", route="/api/openapi.yaml")
+    ]
+    assert conf.pyramid_openapi3_add_deserializer.calls == [
+        pretend.call("application/vnd.pypi.api-v0-danger+json", orjson.loads)
+    ]
+    if env_name == "development":
+        assert conf.pyramid_openapi3_add_explorer.calls == [
+            pretend.call(route="/api/explorer/")
+        ]
+    else:
+        assert not conf.pyramid_openapi3_add_explorer.calls

tests/unit/api/test_echo.py

@@ -24,31 +24,6 @@ def test_echo(self, pyramid_request, pyramid_user):
 
 
 class TestAPIProjectObservations:
-    def test_missing_fields(self, pyramid_request):
-        project = ProjectFactory.create()
-        pyramid_request.json_body = {}
-
-        with pytest.raises(HTTPBadRequest) as exc:
-            api_projects_observations(project, pyramid_request)
-
-        assert exc.value.json == {
-            "error": "missing required fields",
-            "missing": ["kind", "summary"],
-        }
-
-    def test_invalid_kind(self, pyramid_request):
-        project = ProjectFactory.create()
-        pyramid_request.json_body = {"kind": "invalid", "summary": "test"}
-
-        with pytest.raises(HTTPBadRequest) as exc:
-            api_projects_observations(project, pyramid_request)
-
-        assert exc.value.json == {
-            "error": "invalid kind",
-            "kind": "invalid",
-            "project": project.name,
-        }
-
     def test_malware_missing_inspector_url(self, pyramid_request):
         project = ProjectFactory.create()
         pyramid_request.json_body = {"kind": "is_malware", "summary": "test"}
@@ -62,35 +37,18 @@ def test_malware_missing_inspector_url(self, pyramid_request):
             "project": project.name,
         }
 
-    def test_malware_invalid_inspector_url(self, pyramid_request):
-        project = ProjectFactory.create()
-        pyramid_request.json_body = {
-            "kind": "is_malware",
-            "summary": "test",
-            "inspector_url": "invalid",
-        }
-
-        with pytest.raises(HTTPBadRequest) as exc:
-            api_projects_observations(project, pyramid_request)
-
-        assert exc.value.json == {
-            "error": "invalid inspector_url",
-            "inspector_url": "invalid",
-            "project": project.name,
-        }
-
     def test_valid_malware_observation(self, db_request, pyramid_user):
         project = ProjectFactory.create()
         db_request.json_body = {
             "kind": "is_malware",
             "summary": "test",
-            "inspector_url": "https://inspector.pypi.io/...",
+            "inspector_url": f"https://inspector.pypi.io/project/{project.name}/...",
         }
 
         response = api_projects_observations(project, db_request)
 
-        assert isinstance(response, HTTPAccepted)
-        assert response.json_body == {
+        assert db_request.response.status == HTTPAccepted().status
+        assert response == {
             "project": project.name,
             "thanks": "for the observation",
         }
@@ -104,8 +62,8 @@ def test_valid_spam_observation(self, db_request, pyramid_user):
 
         response = api_projects_observations(project, db_request)
 
-        assert isinstance(response, HTTPAccepted)
-        assert response.json_body == {
+        assert db_request.response.status == HTTPAccepted().status
+        assert response == {
             "project": project.name,
             "thanks": "for the observation",
         }

tests/unit/test_config.py

@@ -357,6 +357,7 @@ def __init__(self):
             pretend.call(".banners"),
             pretend.call(".admin"),
             pretend.call(".forklift"),
+            pretend.call(".api.config"),
             pretend.call(".utils.wsgi"),
             pretend.call(".sentry"),
             pretend.call(".csp"),

warehouse/api/config.py

@@ -0,0 +1,78 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Configuration for the warehouse API
+"""
+
+from __future__ import annotations
+
+import typing
+
+from pathlib import Path
+
+import orjson
+
+from warehouse.config import Environment
+
+if typing.TYPE_CHECKING:
+    from pyramid.config import Configurator
+
+
+def _api_set_content_type(view, info):
+    """
+    Set the content type based on API version parameter.
+
+    Use in a `@view_config` decorator like so:
+
+        @view_config(renderer="json", api_version="v1", ...)
+        def my_view(request):
+            return {"hello": "world"}
+
+    This will set the content type to `application/vnd.pypi.v1+json` and
+    pass to whatever `json` renderer is configured.
+    """
+    if api_version := info.options.get("api_version"):  # pragma: no cover
+
+        def wrapper(context, request):
+            request.response.content_type = f"application/vnd.pypi.{api_version}+json"
+            return view(context, request)
+
+        return wrapper
+    return view
+
+
+_api_set_content_type.options = ("api_version",)  # type: ignore[attr-defined]
+
+
+def includeme(config: Configurator) -> None:
+    config.add_view_deriver(_api_set_content_type)
+
+    # Set up OpenAPI
+    config.include("pyramid_openapi3")
+    config.pyramid_openapi3_spec(
+        str(Path(__file__).parent / "openapi.yaml"),
+        route="/api/openapi.yaml",
+    )
+    # We use vendor prefixes to indicate the API version, so we need to add
+    # deserializers for each version.
+    config.pyramid_openapi3_add_deserializer(
+        "application/vnd.pypi.api-v0-danger+json", orjson.loads
+    )
+    if config.registry.settings["warehouse.env"] == Environment.development:
+        # Set up the route for the OpenAPI Web UI
+        config.pyramid_openapi3_add_explorer(route="/api/explorer/")
+
+    # Helpful toggles for development.
+    # config.registry.settings["pyramid_openapi3.enable_endpoint_validation"] = False
+    # config.registry.settings["pyramid_openapi3.enable_request_validation"] = False
+    # config.registry.settings["pyramid_openapi3.enable_response_validation"] = False

warehouse/api/echo.py

@@ -11,9 +11,10 @@
 # limitations under the License.
 from __future__ import annotations
 
+import http
 import typing
 
-from pyramid.httpexceptions import HTTPAccepted, HTTPBadRequest
+from pyramid.httpexceptions import HTTPBadRequest
 from pyramid.view import view_config
 
 from warehouse.authnz import Permissions
@@ -51,7 +52,9 @@ def ...
 
     # Set defaults for API views
     kwargs.update(
+        api_version="api-v0-danger",
         accept="application/vnd.pypi.api-v0-danger+json",
+        openapi=True,
         renderer="json",
         require_csrf=False,
         # TODO: Can we apply a macaroon-based rate limiter here,
@@ -64,10 +67,7 @@ def _wrapper(wrapped):
     return _wrapper
 
 
-@api_v0_view_config(
-    route_name="api.echo",
-    permission=Permissions.APIEcho,
-)
+@api_v0_view_config(route_name="api.echo", permission=Permissions.APIEcho)
 def api_echo(request: Request):
     return {
         "username": request.user.username,
@@ -79,34 +79,15 @@ def api_echo(request: Request):
     permission=Permissions.APIObservationsAdd,
     require_methods=["POST"],
 )
-def api_projects_observations(
-    project: Project, request: Request
-) -> HTTPAccepted | HTTPBadRequest:
+def api_projects_observations(project: Project, request: Request) -> dict:
     data = request.json_body
 
-    # TODO: Are there better mechanisms for validating the payload?
-    #  Maybe adopt https://github.com/Pylons/pyramid_openapi3 - too big?
-    required_fields = {"kind", "summary"}
-    if not required_fields.issubset(data.keys()):
-        raise HTTPBadRequest(
-            json={
-                "error": "missing required fields",
-                "missing": sorted(list(required_fields - data.keys())),
-            },
-        )
-    try:
-        # get the correct mapping for the `kind` field
-        kind = OBSERVATION_KIND_MAP[data["kind"]]
-    except KeyError:
-        raise HTTPBadRequest(
-            json={
-                "error": "invalid kind",
-                "kind": data["kind"],
-                "project": project.name,
-            }
-        )
+    # We know that this is a valid observation kind, so we can use it directly
+    kind = OBSERVATION_KIND_MAP[data["kind"]]
 
-    # TODO: Another case of needing more complex validation
+    # One case of needing more complex validation that OpenAPI does not yet support.
+    # Here we express a dependency between fields, but the validity of the inspector_url
+    # is handled by the OpenAPI schema.
     if kind == ObservationKind.IsMalware:
         if "inspector_url" not in data:
             raise HTTPBadRequest(
@@ -116,16 +97,6 @@ def api_projects_observations(
                     "project": project.name,
                 },
             )
-        if "inspector_url" in data and not data["inspector_url"].startswith(
-            "https://inspector.pypi.io/"
-        ):
-            raise HTTPBadRequest(
-                json={
-                    "error": "invalid inspector_url",
-                    "inspector_url": data["inspector_url"],
-                    "project": project.name,
-                },
-            )
 
     project.record_observation(
         request=request,
@@ -135,10 +106,10 @@ def api_projects_observations(
         payload=data,
     )
 
-    return HTTPAccepted(
-        json={
-            # TODO: What should we return to the caller?
-            "project": project.name,
-            "thanks": "for the observation",
-        },
-    )
+    # Override the status code, instead returning Response which changes the renderer.
+    request.response.status = http.HTTPStatus.ACCEPTED
+    return {
+        # TODO: What should we return to the caller?
+        "project": project.name,
+        "thanks": "for the observation",
+    }