Upgrade to django 3.2

.github/workflows/deploytest.yml

@@ -82,4 +82,4 @@ jobs:
         sudo apt-get update -y
         sudo apt-get install -y gettext
     - name: Test Django makemessages
-      run: python contentcuration/manage.py makemessages
+      run: python contentcuration/manage.py makemessages --all

Makefile

@@ -65,9 +65,7 @@ i18n-extract-frontend:
 
 i18n-extract-backend:
 	# generate backend messages
-	cd contentcuration && python manage.py makemessages
-	# workaround for Django 1.11 makemessages spitting out an invalid English translation file
-	python bin/fix_django_messages.py
+	cd contentcuration && python manage.py makemessages --all
 
 i18n-extract: i18n-extract-frontend i18n-extract-backend
 

contentcuration/contentcuration/api.py

@@ -1,8 +1,6 @@
 """
 This module contains utility functions used by API endpoints.
 """
-from future import standard_library
-standard_library.install_aliases()
 import hashlib
 import logging
 import os

contentcuration/contentcuration/catalog_settings.py

@@ -1,7 +1,7 @@
 # flake8: noqa
 from .production_settings import *  # noqa
 
-LANGUAGES += (("ar", ugettext("Arabic")),)  # noqa
+LANGUAGES += (("ar", gettext("Arabic")),)  # noqa
 
 LIBRARY_MODE = True
 SITE_READ_ONLY = True

contentcuration/contentcuration/celery.py

@@ -7,7 +7,7 @@
 from contentcuration.utils.celery.app import CeleryApp
 
 # set the default Django settings module for the 'celery' program.
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'contentcuration.settings')
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "contentcuration.settings")
 
 app = CeleryApp('contentcuration')
 
@@ -19,4 +19,4 @@
 
 @app.task(bind=True)
 def debug_task(self):
-    print('Request: {0!r}'.format(self.request))
+    print("Request: {0!r}".format(self.request))

contentcuration/contentcuration/context_processors.py

@@ -1,6 +1,6 @@
 from django.conf import settings
-from django.core.urlresolvers import get_resolver
 from django.template.loader import render_to_string
+from django.urls import get_resolver
 from django.utils.html import mark_safe
 from django_js_reverse.core import _safe_json
 from django_js_reverse.core import generate_json
@@ -11,14 +11,18 @@
 
 
 def site_variables(request):
-    return {'INCIDENT': settings.INCIDENT,
-            'BETA_MODE': settings.BETA_MODE,
-            'DEPRECATED': "contentworkshop" in request.get_host(),
-            'STORAGE_BASE_URL': "{bucket}/{storage_root}/".format(bucket=settings.AWS_S3_BUCKET_NAME, storage_root=settings.STORAGE_ROOT),
-            'STORAGE_HOST': settings.AWS_S3_ENDPOINT_URL,
-            'DEBUG': settings.DEBUG,
-            'LANG_INFO': json_for_parse_from_data(language_globals()),
-            'LOGGED_IN': not request.user.is_anonymous()}
+    return {
+        "INCIDENT": settings.INCIDENT,
+        "BETA_MODE": settings.BETA_MODE,
+        "DEPRECATED": "contentworkshop" in request.get_host(),
+        "STORAGE_BASE_URL": "{bucket}/{storage_root}/".format(
+            bucket=settings.AWS_S3_BUCKET_NAME, storage_root=settings.STORAGE_ROOT
+        ),
+        "STORAGE_HOST": settings.AWS_S3_ENDPOINT_URL,
+        "DEBUG": settings.DEBUG,
+        "LANG_INFO": json_for_parse_from_data(language_globals()),
+        "LOGGED_IN": not request.user.is_anonymous,
+    }
 
 
 def url_tag(self):
@@ -37,7 +41,7 @@ def url_tag(self):
         context={"data": _safe_json(data), "js_name": "window.Urls"},
     )
     return {
-        'I18N_URLS': mark_safe(
+        "I18N_URLS": mark_safe(
             """<script type="text/javascript">"""
             # Minify the generated Javascript
             + jsmin(js)

contentcuration/contentcuration/db/models/manager.py

@@ -613,51 +613,3 @@ def _deep_copy(
         self._copy_associated_objects(source_copy_id_map)
 
         return new_nodes
-
-    def build_tree_nodes(self, data, target=None, position="last-child"):
-        """
-        vendored from:
-        https://github.com/django-mptt/django-mptt/blob/fe2b9cc8cfd8f4b764d294747dba2758147712eb/mptt/managers.py#L614
-        """
-        opts = self.model._mptt_meta
-        if target:
-            tree_id = target.tree_id
-            if position in ("left", "right"):
-                level = getattr(target, opts.level_attr)
-                if position == "left":
-                    cursor = getattr(target, opts.left_attr)
-                else:
-                    cursor = getattr(target, opts.right_attr) + 1
-            else:
-                level = getattr(target, opts.level_attr) + 1
-                if position == "first-child":
-                    cursor = getattr(target, opts.left_attr) + 1
-                else:
-                    cursor = getattr(target, opts.right_attr)
-        else:
-            tree_id = self._get_next_tree_id()
-            cursor = 1
-            level = 0
-
-        stack = []
-
-        def treeify(data, cursor=1, level=0):
-            data = dict(data)
-            children = data.pop("children", [])
-            node = self.model(**data)
-            stack.append(node)
-            setattr(node, opts.tree_id_attr, tree_id)
-            setattr(node, opts.level_attr, level)
-            setattr(node, opts.left_attr, cursor)
-            for child in children:
-                cursor = treeify(child, cursor=cursor + 1, level=level + 1)
-            cursor += 1
-            setattr(node, opts.right_attr, cursor)
-            return cursor
-
-        treeify(data, cursor=cursor, level=level)
-
-        if target:
-            self._create_space(2 * len(stack), cursor - 1, tree_id)
-
-        return stack

contentcuration/contentcuration/debug/middleware.py

@@ -4,34 +4,44 @@
 import debug_panel.urls
 from debug_panel.cache import cache
 from debug_panel.middleware import DebugPanelMiddleware
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 
 
 class CustomDebugPanelMiddleware(DebugPanelMiddleware):
     """
     Custom version to fix SQL escaping:
     https://github.com/recamshak/django-debug-panel/issues/17#issuecomment-366268893
     """
+
     def process_response(self, request, response):
         """
         Store the DebugToolbarMiddleware rendered toolbar into a cache store.
         The data stored in the cache are then reachable from an URL that is appened
         to the HTTP response header under the 'X-debug-data-url' key.
         """
-        toolbar = self.__class__.debug_toolbars.get(threading.current_thread().ident, None)
+        toolbar = self.__class__.debug_toolbars.get(
+            threading.current_thread().ident, None
+        )
 
         response = super(DebugPanelMiddleware, self).process_response(request, response)
 
         if toolbar:
             # for django-debug-toolbar >= 1.4
             for panel in reversed(toolbar.enabled_panels):
-                if hasattr(panel, 'generate_stats') and not panel.get_stats():  # PATCH HERE
+                if (
+                    hasattr(panel, "generate_stats") and not panel.get_stats()
+                ):  # PATCH HERE
                     panel.generate_stats(request, response)
 
             cache_key = "%f" % time.time()
             cache.set(cache_key, toolbar.render_toolbar())
 
-            response['X-debug-data-url'] = request.build_absolute_uri(
-                reverse('debug_data', urlconf=debug_panel.urls, kwargs={'cache_key': cache_key}))
+            response["X-debug-data-url"] = request.build_absolute_uri(
+                reverse(
+                    "debug_data",
+                    urlconf=debug_panel.urls,
+                    kwargs={"cache_key": cache_key},
+                )
+            )
 
         return response

contentcuration/contentcuration/decorators.py

@@ -9,14 +9,14 @@
 
 def browser_is_supported(function):
     def wrap(request, *args, **kwargs):
-        user_agent_string = request.META.get('HTTP_USER_AGENT') or ""
+        user_agent_string = request.META.get("HTTP_USER_AGENT") or ""
 
         # Check if the user agent string matches the Kubernetes agents, Google Health Check agents, or an accepted browser
         for expected_agent in ACCEPTED_BROWSERS:
             if expected_agent in user_agent_string:
                 return function(request, *args, **kwargs)
 
-        return render(request, 'unsupported_browser.html')
+        return render(request, "unsupported_browser.html")
 
     wrap.__doc__ = function.__doc__
     wrap.__name__ = function.__name__
@@ -25,10 +25,10 @@ def wrap(request, *args, **kwargs):
 
 def is_admin(function):
     def wrap(request, *args, **kwargs):
-        if not request.user.is_anonymous() and request.user.is_admin:
+        if not request.user.is_anonymous and request.user.is_admin:
             return function(request, *args, **kwargs)
 
-        return render(request, 'unauthorized.html', status=403)
+        return render(request, "unauthorized.html", status=403)
 
     wrap.__doc__ = function.__doc__
     wrap.__name__ = function.__name__
@@ -38,17 +38,19 @@ def wrap(request, *args, **kwargs):
 def can_access_channel(function):
     def wrap(request, *args, **kwargs):
         try:
-            channel = Channel.objects.get(pk=kwargs['channel_id'])
+            channel = Channel.objects.get(pk=kwargs["channel_id"])
         except ObjectDoesNotExist:
-            return render(request, 'channel_not_found.html')
-
-        if channel.public or \
-                channel.editors.filter(id=request.user.id).exists() or \
-                channel.viewers.filter(id=request.user.id).exists() or \
-                (not request.user.is_anonymous() and request.user.is_admin):
+            return render(request, "channel_not_found.html")
+
+        if (
+            channel.public
+            or channel.editors.filter(id=request.user.id).exists()
+            or channel.viewers.filter(id=request.user.id).exists()
+            or (not request.user.is_anonymous and request.user.is_admin)
+        ):
             return function(request, *args, **kwargs)
 
-        return render(request, 'channel_not_found.html', status=404)
+        return render(request, "channel_not_found.html", status=404)
 
     wrap.__doc__ = function.__doc__
     wrap.__name__ = function.__name__
@@ -58,14 +60,17 @@ def wrap(request, *args, **kwargs):
 def can_edit_channel(function):
     def wrap(request, *args, **kwargs):
         try:
-            channel = Channel.objects.get(pk=kwargs['channel_id'], deleted=False)
+            channel = Channel.objects.get(pk=kwargs["channel_id"], deleted=False)
 
-            if not channel.editors.filter(id=request.user.id).exists() and not request.user.is_admin:
-                return render(request, 'unauthorized.html', status=403)
+            if (
+                not channel.editors.filter(id=request.user.id).exists()
+                and not request.user.is_admin
+            ):
+                return render(request, "unauthorized.html", status=403)
 
             return function(request, *args, **kwargs)
         except ObjectDoesNotExist:
-            return render(request, 'channel_not_found.html')
+            return render(request, "channel_not_found.html")
 
     wrap.__doc__ = function.__doc__
     wrap.__name__ = function.__name__

contentcuration/contentcuration/management/commands/loadconstants.py

@@ -1,4 +1,5 @@
 from future import standard_library
+
 standard_library.install_aliases()
 from builtins import str
 from builtins import object
@@ -30,7 +31,8 @@ def generate_list(self):
                 "model": self.model,
                 "pk": self.id_field,
                 "fields": self.get_dict(constant),
-            } for constant in self.default_list
+            }
+            for constant in self.default_list
         ]
 
     def get_dict(self, constant):
@@ -160,14 +162,12 @@ def get_dict(self, constant):
 
 
 class EarlyExit(BaseException):
-
     def __init__(self, message, db_path):
         self.message = message
         self.db_path = db_path
 
 
 class Command(BaseCommand):
-
     def add_arguments(self, parser):
         pass
 
@@ -178,19 +178,31 @@ def handle(self, *args, **options):
                 current_model = ""
                 new_model_count = 0
                 for constant in constant_list:
-                    current_model = constant['model'].__name__
+                    current_model = constant["model"].__name__
                     if current_model in cache:
                         cache.delete(current_model)
-                    obj, isNew = constant['model'].objects.update_or_create(**{constant['pk']: constant['fields'][constant['pk']]})
+                    obj, isNew = constant["model"].objects.update_or_create(
+                        **{constant["pk"]: constant["fields"][constant["pk"]]}
+                    )
                     new_model_count += 1 if isNew else 0
-                    for attr, value in list(constant['fields'].items()):
-                        setattr(obj, attr, value)
+                    for attr, value in list(constant["fields"].items()):
+                        field = obj._meta.get_field(attr)
+                        if field.many_to_many:
+                            getattr(obj, attr).set(value)
+                        else:
+                            setattr(obj, attr, value)
 
                     obj.save()
-                self.stdout.write("{0}: {1} constants saved ({2} new)".format(str(current_model), len(constant_list), new_model_count))
+                self.stdout.write(
+                    "{0}: {1} constants saved ({2} new)".format(
+                        str(current_model), len(constant_list), new_model_count
+                    )
+                )
 
             # Create garbage node
-            garbage_node, _new = models.ContentNode.objects.get_or_create(pk=settings.ORPHANAGE_ROOT_ID, kind_id=content_kinds.TOPIC)
+            garbage_node, _new = models.ContentNode.objects.get_or_create(
+                pk=settings.ORPHANAGE_ROOT_ID, kind_id=content_kinds.TOPIC
+            )
             garbage_node.title = "Garbage Node Root"
             garbage_node.description = "This node as the default parent for nodes not associated with a channel"
             garbage_node.save()
@@ -201,7 +213,7 @@ def handle(self, *args, **options):
             logging.warning("DB is in read-only mode, skipping loadconstants")
 
         except EarlyExit as e:
-            logging.warning("Exited early due to {message}.".format(
-                message=e.message))
-            self.stdout.write("You can find your database in {path}".format(
-                path=e.db_path))
+            logging.warning("Exited early due to {message}.".format(message=e.message))
+            self.stdout.write(
+                "You can find your database in {path}".format(path=e.db_path)
+            )