[django] upgraded django to 4.1

Several other libraries have also been upgraded along with it.
Three code changes had to be made.

1. Django 4 wants the middlewares to be explicitly sync and async.
Rather than rewriting all of the middleware classes, I took the
shortcut of creating a new mixin class that supports only sync calls.

2. CSRF_TRUSTED_ORIGINS in django 4+ need the scheme part as well. We
now check and add `http` and `https` to the origins.

3. django-babel also needs a change every time we do a django upgrade.
This is mostly just changes around django version support.

desktop/core/base_requirements.txt

@@ -5,28 +5,28 @@ avro-python3==1.8.2
 Babel==2.9.1
 celery[redis]==5.4.0
 cffi==1.15.0
-channels==3.0.3
-channels-redis==3.2.0
+channels==4.0.0
+channels-redis==4.0.0
 configobj==5.0.9
 cx-Oracle==8.3.0
 django-auth-ldap==4.3.0
-Django==3.2.25
+Django==4.1.13
 daphne==3.0.2
 django-redis==5.4.0
 django-celery-beat==2.6.0
 django-celery-results==2.5.1
-django-cors-headers==3.7.0
+django-cors-headers==3.13.0
 django-crequest==2018.5.11
 django-debug-panel==0.8.3
-django-debug-toolbar==1.11.1
-django-extensions==3.1.3
+django-debug-toolbar==3.6.0
+django-extensions==3.2.1
 django-ipware==3.0.2
 django_opentracing==1.1.0
 django_prometheus==1.0.15
 django-webpack-loader==1.0.0
 djangomako==1.3.2
-djangorestframework-simplejwt==5.2.0
-djangorestframework==3.12.2
+djangorestframework-simplejwt==5.2.1
+djangorestframework==3.14.0
 future==0.18.3
 gcs-oauth2-boto-plugin==3.0
 greenlet==3.1.1

desktop/core/ext-py3/django-babel/.travis.yml

@@ -17,6 +17,7 @@ env:
   - TOX_ENV=py35-django30
   - TOX_ENV=py35-django31
   - TOX_ENV=py35-django32
+  - TOX_ENV=py38-django41
 matrix:
   include:
     - env: TOX_ENV=py36-django18
@@ -35,6 +36,8 @@ matrix:
       python: "3.6"
     - env: TOX_ENV=py36-django32
       python: "3.6"
+    - env: TOX_ENV=py38-django41
+      python: "3.8"
     - env: TOX_ENV=py36-djangomaster
       python: "3.6"
 install: pip install tox

desktop/core/ext-py3/django-babel/setup.py

@@ -24,7 +24,7 @@ def read(*parts):
     url='https://github.com/python-babel/django-babel/',
     packages=find_packages(exclude=('tests',)),
     install_requires=[
-        'django>=1.8,<3.3',
+        'django>=1.8,<4.2',
         'babel>=1.3',
     ],
     classifiers=[

desktop/core/ext-py3/django-babel/tox.ini

@@ -1,6 +1,6 @@
 [tox]
 envlist = py{27,34,35,36}-django{18,111},
-          py{35,36,37}-django{20,21,22,30,31,32},
+          py{35,36,37,38}-django{20,21,22,30,31,32,41},
           py{36,37}-django{master},
           lint, docs
 
@@ -19,6 +19,7 @@ deps =
     django30: Django>=3.0,<3.1
     django31: Django>=3.1,<3.2
     django32: Django>=3.2,<3.3
+    django41: Django>=4.1,<4.2
     djangomaster: https://github.com/django/django/archive/master.tar.gz#egg=Django
 commands = py.test {posargs}
 

desktop/core/src/desktop/middleware.py

@@ -161,7 +161,34 @@ def get_header(response):
 HUE_LB_HOSTS = [urlparse(hue_lb).netloc for hue_lb in HUE_LOAD_BALANCER.get()] if HUE_LOAD_BALANCER.get() else []
 
 
-class AjaxMiddleware(MiddlewareMixin):
+'''
+Django4MiddlewareAdapterMixin has been solely added to migrate
+Django3 middleware classes to Django4. Any new middleware
+classes should try not to use this and instead follow the
+standard practice at the time.
+'''
+class Django4MiddlewareAdapterMixin(MiddlewareMixin):
+
+  def __call__(self, request):
+    """Sync version of the middleware"""
+    if hasattr(self, 'process_request'):
+      self.process_request(request)
+
+    response = self.get_response(request)
+
+    if hasattr(self, 'process_response'):
+      response = self.process_response(request, response)
+
+    return response
+
+  async def __aenter__(self):
+    pass  # Provide async behavior if needed
+
+  async def __aexit__(self, exc_type, exc_value, traceback):
+    pass  # Handle async cleanup
+
+
+class AjaxMiddleware(Django4MiddlewareAdapterMixin):
   """
   Middleware that augments request to set request.ajax
   for either is_ajax() (looks at HTTP headers) or ?format=json
@@ -172,7 +199,7 @@ def process_request(self, request):
     return None
 
 
-class ExceptionMiddleware(MiddlewareMixin):
+class ExceptionMiddleware(Django4MiddlewareAdapterMixin):
   """
   If exceptions know how to render themselves, use that.
   """
@@ -202,7 +229,7 @@ def process_exception(self, request, exception):
     return None
 
 
-class ClusterMiddleware(MiddlewareMixin):
+class ClusterMiddleware(Django4MiddlewareAdapterMixin):
   """
   Manages setting request.fs and request.jt
   """
@@ -230,7 +257,7 @@ def process_view(self, request, view_func, view_args, view_kwargs):
     request.jt = None
 
 
-class NotificationMiddleware(MiddlewareMixin):
+class NotificationMiddleware(Django4MiddlewareAdapterMixin):
   """
   Manages setting request.info and request.error
   """
@@ -359,7 +386,7 @@ def _load_app_middleware(cls, app):
     return result
 
 
-class LoginAndPermissionMiddleware(MiddlewareMixin):
+class LoginAndPermissionMiddleware(Django4MiddlewareAdapterMixin):
   """
   Middleware that forces all views (except those that opt out) through authentication.
   """
@@ -497,7 +524,7 @@ def __str__(self):
     return json.dumps(self.kwargs)
 
 
-class AuditLoggingMiddleware(MiddlewareMixin):
+class AuditLoggingMiddleware(Django4MiddlewareAdapterMixin):
 
   def __init__(self, get_response):
     self.get_response = get_response
@@ -557,7 +584,7 @@ def _get_allowed(self, request, response=None):
     return allowed
 
 
-class ProxyMiddleware(MiddlewareMixin):
+class ProxyMiddleware(Django4MiddlewareAdapterMixin):
 
   def __init__(self, get_response):
     self.get_response = get_response
@@ -616,7 +643,7 @@ def clean_username(self, username, request):
     return username
 
 
-class SpnegoMiddleware(MiddlewareMixin):
+class SpnegoMiddleware(Django4MiddlewareAdapterMixin):
   """
   Based on the WSGI SPNEGO middlware class posted here:
   http://code.activestate.com/recipes/576992/
@@ -784,7 +811,7 @@ def clean_username(self, username, request):
     return username
 
 
-class HueRemoteUserMiddleware(RemoteUserMiddleware):
+class HueRemoteUserMiddleware(Django4MiddlewareAdapterMixin, RemoteUserMiddleware):
   """
   Middleware to delegate authentication to a proxy server. The proxy server
   will set an HTTP header (defaults to Remote-User) with the name of the
@@ -801,7 +828,7 @@ def __init__(self, get_response):
     self.header = AUTH.REMOTE_USER_HEADER.get()
 
 
-class EnsureSafeMethodMiddleware(MiddlewareMixin):
+class EnsureSafeMethodMiddleware(Django4MiddlewareAdapterMixin):
   """
   Middleware to white list configured HTTP request methods.
   """
@@ -810,7 +837,7 @@ def process_request(self, request):
       return HttpResponseNotAllowed(HTTP_ALLOWED_METHODS.get())
 
 
-class EnsureSafeRedirectURLMiddleware(MiddlewareMixin):
+class EnsureSafeRedirectURLMiddleware(Django4MiddlewareAdapterMixin):
   """
   Middleware to white list configured redirect URLs.
   """
@@ -838,7 +865,7 @@ def process_response(self, request, response):
       return response
 
 
-class MetricsMiddleware(MiddlewareMixin):
+class MetricsMiddleware(Django4MiddlewareAdapterMixin):
   """
   Middleware to track the number of active requests.
   """
@@ -863,7 +890,7 @@ def process_response(self, request, response):
     return response
 
 
-class ContentSecurityPolicyMiddleware(MiddlewareMixin):
+class ContentSecurityPolicyMiddleware(Django4MiddlewareAdapterMixin):
   def __init__(self, get_response):
     self.get_response = get_response
     self.secure_content_security_policy = SECURE_CONTENT_SECURITY_POLICY.get()
@@ -920,7 +947,7 @@ def process_response(self, request, response):
     return response
 
 
-class MimeTypeJSFileFixStreamingMiddleware(MiddlewareMixin):
+class MimeTypeJSFileFixStreamingMiddleware(Django4MiddlewareAdapterMixin):
   """
   Middleware to detect and fix ".js" mimetype. SLES 11SP4 as example OS which detect js file
   as "text/x-js" and if strict X-Content-Type-Options=nosniff is set then browser fails to
@@ -979,7 +1006,7 @@ def __call__(self, request):
     return self.get_response(request)
 
 
-class CacheControlMiddleware(MiddlewareMixin):
+class CacheControlMiddleware(Django4MiddlewareAdapterMixin):
   def __init__(self, get_response):
     self.get_response = get_response
     self.custom_cache_control = CUSTOM_CACHE_CONTROL.get()

desktop/core/src/desktop/settings.py

@@ -498,8 +498,14 @@
   else:
     TRUSTED_ORIGINS += desktop.conf.KNOX.KNOX_PROXYHOSTS.get()
 
+CSRF_TRUSTED_ORIGINS = []
 if TRUSTED_ORIGINS:
-  CSRF_TRUSTED_ORIGINS = TRUSTED_ORIGINS
+  for origin in TRUSTED_ORIGINS:
+    if 'http://' in origin or 'https://' in origin:
+      CSRF_TRUSTED_ORIGINS.append(origin)
+    else:
+      CSRF_TRUSTED_ORIGINS.append('http://%s' % origin)
+      CSRF_TRUSTED_ORIGINS.append('https://%s' % origin)
 
 SECURE_HSTS_SECONDS = desktop.conf.SECURE_HSTS_SECONDS.get()
 SECURE_HSTS_INCLUDE_SUBDOMAINS = desktop.conf.SECURE_HSTS_INCLUDE_SUBDOMAINS.get()