Merge pull request #6226 from readthedocs/proxito-stub

Initial stub of proxito

readthedocs/proxito/middleware.py

@@ -0,0 +1,97 @@
+"""
+Middleware for Proxito.
+
+This is used to take the request and map the host to the proper project slug.
+
+Additional processing is done to get the project from the URL in the ``views.py`` as well.
+"""
+import logging
+
+from django.conf import settings
+from django.http import HttpResponseBadRequest
+from django.shortcuts import render
+from django.utils.deprecation import MiddlewareMixin
+from django.utils.translation import ugettext_lazy as _
+
+from readthedocs.projects.models import Domain
+
+
+log = logging.getLogger(__name__)  # noqa
+
+
+def map_host_to_project_slug(request):
+    """
+    Take the request and map the host to the proper project slug.
+
+    We check, in order:
+
+    * The ``HTTP_X_RTD_SLUG`` host header for explicit Project mapping
+        - This sets ``request.rtdheader`` True
+    * The ``PUBLIC_DOMAIN`` where we can use the subdomain as the project name
+        - This sets ``request.subdomain`` True
+    * The hostname without port information, which maps to ``Domain`` objects
+        - This sets ``request.cname`` True
+    """
+
+    host = request.get_host().lower().split(':')[0]
+    public_domain = settings.PUBLIC_DOMAIN.lower().split(':')[0]
+    host_parts = host.split('.')
+    public_domain_parts = public_domain.split('.')
+
+    project_slug = None
+
+    # Explicit Project slug being passed in
+    if 'HTTP_X_RTD_SLUG' in request.META:
+        project_slug = request.META['HTTP_X_RTD_SLUG'].lower()
+        request.rtdheader = True
+
+    elif public_domain in host:
+        # Serve from the PUBLIC_DOMAIN, ensuring it looks like `foo.PUBLIC_DOMAIN`
+        if public_domain_parts == host_parts[1:]:
+            project_slug = host_parts[0]
+            request.subdomain = True
+            log.debug('Proxito Public Domain: host=%s', host)
+        else:
+            # TODO: This can catch some possibly valid domains (docs.readthedocs.io.com) for example
+            # But these feel like they might be phishing, etc. so let's block them for now.
+            log.warning('Weird variation on our hostname: host=%s', host)
+            return HttpResponseBadRequest(_('Invalid hostname'))
+
+    # Serve CNAMEs
+    else:
+        domain = Domain.objects.filter(domain=host).first()
+        if domain:
+            project_slug = domain.project.slug
+            request.cname = True
+            log.debug('Proxito CNAME: host=%s', host)
+        else:
+            # Some person is CNAMEing to us without configuring a domain - 404.
+            log.debug('CNAME 404: host=%s', host)
+            return render(
+                request, 'core/dns-404.html', context={'host': host}, status=404
+            )
+
+    log.debug('Proxito Project: slug=%s', project_slug)
+    return project_slug
+
+
+class ProxitoMiddleware(MiddlewareMixin):
+
+    """The actual middleware we'll be using in prod."""
+
+    def process_request(self, request):  # noqa
+        if any([not settings.USE_SUBDOMAIN, 'localhost' in request.get_host(),
+                'testserver' in request.get_host()]):
+            log.debug('Not processing Proxito middleware')
+            return None
+
+        ret = map_host_to_project_slug(request)
+
+        # Handle returning a response
+        if hasattr(ret, 'status_code'):
+            return ret
+
+        # Otherwise set the slug on the request
+        request.host_project_slug = request.slug = ret
+
+        return None

readthedocs/proxito/tests/base.py

@@ -0,0 +1,56 @@
+# Copied from .org
+
+
+import django_dynamic_fixture as fixture
+from django.contrib.auth.models import User
+from django.test import TestCase
+from django.test.utils import override_settings
+
+from readthedocs.projects.models import Project
+
+
+@override_settings(
+    PUBLIC_DOMAIN='dev.readthedocs.io',
+    ROOT_URLCONF='readthedocs.proxito.urls',
+    MIDDLEWARE=['readthedocs.proxito.middleware.ProxitoMiddleware'],
+    USE_SUBDOMAIN=True,
+)
+class BaseDocServing(TestCase):
+
+    def setUp(self):
+        self.eric = fixture.get(User, username='eric')
+        self.eric.set_password('eric')
+        self.eric.save()
+        self.project = fixture.get(
+            Project, slug='project', privacy_level='project',
+            version_privacy_level='project', users=[self.eric],
+            main_language_project=None,
+        )
+        self.subproject = fixture.get(
+            Project,
+            slug='subproject',
+            users=[self.eric],
+            main_language_project=None,
+        )
+        self.project.add_subproject(self.subproject)
+        self.translation = fixture.get(
+            Project,
+            language='es',
+            slug='translation',
+            users=[self.eric],
+            main_language_project=self.project,
+        )
+        self.subproject_translation = fixture.get(
+            Project,
+            language='es',
+            slug='subproject-translation',
+            users=[self.eric],
+            main_language_project=self.subproject,
+        )
+        self.subproject_alias = fixture.get(
+            Project,
+            language='en',
+            slug='subproject-alias',
+            users=[self.eric],
+        )
+        self.project.add_subproject(self.subproject_alias, alias='this-is-an-alias')

readthedocs/proxito/tests/test_full.py

@@ -0,0 +1,146 @@
+# Copied from .org
+
+import os
+
+import mock
+from django.conf import settings
+from django.http import HttpResponse
+from django.test.utils import override_settings
+
+from .base import BaseDocServing
+
+
+@override_settings(PYTHON_MEDIA=False)
+class TestFullDocServing(BaseDocServing):
+    # Test the full range of possible doc URL's
+
+    def test_subproject_serving(self):
+        url = '/projects/subproject/en/latest/awesome.html'
+        host = 'project.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(
+            resp['x-accel-redirect'], '/proxito/html/subproject/latest/awesome.html',
+        )
+
+    def test_subproject_single_version(self):
+        self.subproject.single_version = True
+        self.subproject.save()
+        url = '/projects/subproject/awesome.html'
+        host = 'project.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(
+            resp['x-accel-redirect'], '/proxito/html/subproject/latest/awesome.html',
+        )
+
+    def test_subproject_translation_serving(self):
+        url = '/projects/subproject/es/latest/awesome.html'
+        host = 'project.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(
+            resp['x-accel-redirect'], '/proxito/html/subproject-translation/latest/awesome.html',
+        )
+
+    def test_subproject_alias_serving(self):
+        url = '/projects/this-is-an-alias/en/latest/awesome.html'
+        host = 'project.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(
+            resp['x-accel-redirect'], '/proxito/html/subproject-alias/latest/awesome.html',
+        )
+
+    def test_translation_serving(self):
+        url = '/es/latest/awesome.html'
+        host = 'project.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(
+            resp['x-accel-redirect'], '/proxito/html/translation/latest/awesome.html',
+        )
+
+    def test_normal_serving(self):
+        url = '/en/latest/awesome.html'
+        host = 'project.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(
+            resp['x-accel-redirect'], '/proxito/html/project/latest/awesome.html',
+        )
+
+    def test_single_version_serving(self):
+        self.project.single_version = True
+        self.project.save()
+        url = '/awesome.html'
+        host = 'project.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(
+            resp['x-accel-redirect'], '/proxito/html/project/latest/awesome.html',
+        )
+
+    def test_single_version_serving_looks_like_normal(self):
+        self.project.single_version = True
+        self.project.save()
+        url = '/en/stable/awesome.html'
+        host = 'project.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(
+            resp['x-accel-redirect'], '/proxito/html/project/latest/en/stable/awesome.html',
+        )
+
+        # Invalid tests
+
+    def test_invalid_language_for_project_with_versions(self):
+        url = '/foo/latest/awesome.html'
+        host = 'project.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(resp.status_code, 404)
+
+    def test_invalid_translation_for_project_with_versions(self):
+        url = '/cs/latest/awesome.html'
+        host = 'project.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(resp.status_code, 404)
+
+    def test_invalid_subproject(self):
+        url = '/projects/doesnt-exist/foo.html'
+        host = 'project.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(resp.status_code, 404)
+
+    # https://github.com/readthedocs/readthedocs.org/pull/6226/files/596aa85a4886407f0eb65233ebf9c38ee3e8d485#r332445803
+    def test_valid_project_as_invalid_subproject(self):
+        url = '/projects/translation/es/latest/foo.html'
+        host = 'project.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(resp.status_code, 404)
+
+
+class TestDocServingBackends(BaseDocServing):
+    # Test that nginx and python backends both work
+
+    @override_settings(PYTHON_MEDIA=True)
+    def test_python_media_serving(self):
+        with mock.patch(
+                'readthedocs.proxito.views.serve', return_value=HttpResponse()) as serve_mock:
+            url = '/en/latest/awesome.html'
+            host = 'project.dev.readthedocs.io'
+            self.client.get(url, HTTP_HOST=host)
+            serve_mock.assert_called_with(
+                mock.ANY,
+                'html/project/latest/awesome.html',
+                os.path.join(settings.SITE_ROOT, 'media'),
+            )
+
+    @override_settings(PYTHON_MEDIA=False)
+    def test_nginx_media_serving(self):
+        resp = self.client.get('/en/latest/awesome.html', HTTP_HOST='project.dev.readthedocs.io')
+        self.assertEqual(resp.status_code, 200)
+        self.assertEqual(
+            resp['x-accel-redirect'], '/proxito/html/project/latest/awesome.html',
+        )
+
+    @override_settings(PYTHON_MEDIA=False)
+    def test_project_nginx_serving_unicode_filename(self):
+        resp = self.client.get('/en/latest/úñíčódé.html', HTTP_HOST='project.dev.readthedocs.io')
+        self.assertEqual(resp.status_code, 200)
+        self.assertEqual(
+            resp['x-accel-redirect'],
+            '/proxito/html/project/latest/%C3%BA%C3%B1%C3%AD%C4%8D%C3%B3d%C3%A9.html',
+        )

readthedocs/proxito/tests/test_middleware.py

@@ -0,0 +1,91 @@
+# Copied from test_middleware.py
+
+from django.test import TestCase
+from django.test.client import RequestFactory
+from django.test.utils import override_settings
+from django_dynamic_fixture import get
+
+from readthedocs.projects.models import Domain, Project
+from readthedocs.proxito.middleware import ProxitoMiddleware
+from readthedocs.rtd_tests.utils import create_user
+
+
+@override_settings(USE_SUBDOMAIN=True)
+@override_settings(PUBLIC_DOMAIN='dev.readthedocs.io')
+class MiddlewareTests(TestCase):
+
+    def setUp(self):
+        self.factory = RequestFactory()
+        self.middleware = ProxitoMiddleware()
+        self.url = '/'
+        self.owner = create_user(username='owner', password='test')
+        self.pip = get(
+            Project,
+            slug='pip',
+            users=[self.owner],
+            privacy_level='public'
+        )
+
+    def run_middleware(self, request):
+        return self.middleware.process_request(request)
+
+    def test_proper_cname(self):
+        domain = 'docs.random.com'
+        get(Domain, project=self.pip, domain=domain)
+        request = self.factory.get(self.url, HTTP_HOST=domain)
+        res = self.run_middleware(request)
+        self.assertIsNone(res)
+        self.assertEqual(request.cname, True)
+        self.assertEqual(request.host_project_slug, 'pip')
+
+    def test_proper_cname_uppercase(self):
+        get(Domain, project=self.pip, domain='docs.random.com')
+        request = self.factory.get(self.url, HTTP_HOST='docs.RANDOM.COM')
+        self.run_middleware(request)
+        self.assertEqual(request.cname, True)
+        self.assertEqual(request.host_project_slug, 'pip')
+
+    def test_invalid_cname(self):
+        self.assertFalse(Domain.objects.filter(domain='my.host.com').exists())
+        request = self.factory.get(self.url, HTTP_HOST='my.host.com')
+        r = self.run_middleware(request)
+        # We show the 404 error page
+        self.assertContains(r, 'my.host.com', status_code=404)
+
+    def test_proper_subdomain(self):
+        request = self.factory.get(self.url, HTTP_HOST='pip.dev.readthedocs.io')
+        self.run_middleware(request)
+        self.assertEqual(request.subdomain, True)
+        self.assertEqual(request.host_project_slug, 'pip')
+
+    @override_settings(PUBLIC_DOMAIN='foo.bar.readthedocs.io')
+    def test_subdomain_different_length(self):
+        request = self.factory.get(
+            self.url, HTTP_HOST='pip.foo.bar.readthedocs.io'
+        )
+        self.run_middleware(request)
+        self.assertEqual(request.subdomain, True)
+        self.assertEqual(request.host_project_slug, 'pip')
+
+    def test_request_header(self):
+        request = self.factory.get(
+            self.url, HTTP_HOST='some.random.com', HTTP_X_RTD_SLUG='pip'
+        )
+        self.run_middleware(request)
+        self.assertEqual(request.rtdheader, True)
+        self.assertEqual(request.host_project_slug, 'pip')
+
+    def test_request_header_uppercase(self):
+        request = self.factory.get(
+            self.url, HTTP_HOST='some.random.com', HTTP_X_RTD_SLUG='PIP'
+        )
+        self.run_middleware(request)
+
+        self.assertEqual(request.rtdheader, True)
+        self.assertEqual(request.host_project_slug, 'pip')
+
+    def test_long_bad_subdomain(self):
+        domain = 'www.pip.dev.readthedocs.io'
+        request = self.factory.get(self.url, HTTP_HOST=domain)
+        res = self.run_middleware(request)
+        self.assertEqual(res.status_code, 400)