[OLD] Release PR for Sprint ending 2021-03-29

Makefile

@@ -11,7 +11,7 @@ dummyusers:
 	cd contentcuration/ && python manage.py loaddata contentcuration/fixtures/admin_user_token.json
 
 prodceleryworkers:
-	cd contentcuration/ && celery -A contentcuration worker -l info --concurrency=3 --without-mingle --without-gossip
+	cd contentcuration/ && celery -A contentcuration worker -l info --concurrency=3 --task-events --without-mingle --without-gossip
 
 prodcelerydashboard:
 	# connect to the celery dashboard by visiting http://localhost:5555

babel.config.js

@@ -3,7 +3,7 @@ module.exports = {
     [
       '@babel/preset-env',
       {
-        useBuiltIns: 'entry',
+        useBuiltIns: 'usage',
         corejs: '3',
       },
     ],

contentcuration/contentcuration/apps.py

@@ -9,6 +9,6 @@ class ContentConfig(AppConfig):
 
     def ready(self):
         # see note in the celery_signals.py file for why we import here.
-        import contentcuration.utils.celery_signals
+        import contentcuration.utils.celery.signals  # noqa
         if settings.AWS_AUTO_CREATE_BUCKET:
             ensure_storage_bucket_public()

contentcuration/contentcuration/celery.py

@@ -2,20 +2,19 @@
 
 import os
 
-from celery import Celery
-from django.conf import settings
+import django
+
+from contentcuration.utils.celery.app import CeleryApp
 
 # set the default Django settings module for the 'celery' program.
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'contentcuration.settings')
 
-app = Celery('contentcuration')
+app = CeleryApp('contentcuration')
 
 # Using a string here means the worker will not have to
 # pickle the object when using Windows.
 app.config_from_object('django.conf:settings', namespace='CELERY')
-import django
 django.setup()
-app.autodiscover_tasks(lambda: settings.INSTALLED_APPS, force=True)
 
 
 @app.task(bind=True)

contentcuration/contentcuration/db/models/expressions.py

@@ -1,5 +1,7 @@
 from django.db.models import BooleanField
+from django.db.models import F
 from django.db.models import Q
+from django.db.models import Value
 from django.db.models.expressions import CombinedExpression
 from django.db.models.expressions import Func
 from django.db.models.sql.where import WhereNode
@@ -28,6 +30,20 @@ class BooleanComparison(CombinedExpression):
     output_field = BooleanField()
 
 
+class IsNull(BooleanComparison):
+    """
+    An expression that results in a Boolean value, useful for annotating
+    if a column IS or IS NOT NULL
+
+    Example:
+        IsNull('my_field_name') -> my_field_name IS NULL
+        IsNull('my_field_name', negate=True) -> my_field_name IS NOT NULL
+    """
+    def __init__(self, field_name, negate=False):
+        operator = 'IS NOT' if negate else 'IS'
+        super(IsNull, self).__init__(F(field_name), operator, Value(None))
+
+
 class Array(Func):
     """
     Create an array datatype within Postgres.

contentcuration/contentcuration/db/models/manager.py

@@ -14,8 +14,7 @@
 
 from contentcuration.db.advisory_lock import advisory_lock
 from contentcuration.db.models.query import CustomTreeQuerySet
-from contentcuration.utils.tasks import increment_progress
-from contentcuration.utils.tasks import set_total
+from contentcuration.utils.cache import ResourceSizeCache
 
 
 logging = logger.getLogger(__name__)
@@ -198,6 +197,7 @@ def move_node(self, node, target, position="last-child"):
         ``MPTTMeta.order_insertion_by``.  In most cases you should just
         move the node yourself by setting node.parent.
         """
+        old_parent = node.parent
         with self.lock_mptt(node.tree_id, target.tree_id):
             # Call _mptt_refresh to ensure that the mptt fields on
             # these nodes are up to date once we have acquired a lock
@@ -212,6 +212,14 @@ def move_node(self, node, target, position="last-child"):
         node_moved.send(
             sender=node.__class__, instance=node, target=target, position=position,
         )
+        # when moving to a new tree, like trash, we'll blanket reset the modified for the
+        # new root and the old root nodes
+        if old_parent.tree_id != target.tree_id:
+            for size_cache in [
+                ResourceSizeCache(target.get_root()),
+                ResourceSizeCache(old_parent.get_root())
+            ]:
+                size_cache.reset_modified(None)
 
     def get_source_attributes(self, source):
         """
@@ -330,14 +338,18 @@ def copy_node(
         excluded_descendants=None,
         can_edit_source_channel=None,
         batch_size=None,
+        progress_tracker=None
     ):
+        """
+        :type progress_tracker: contentcuration.utils.celery.ProgressTracker|None
+        """
         if batch_size is None:
             batch_size = BATCH_SIZE
         source_channel_id = node.get_channel_id()
 
         total_nodes = self._all_nodes_to_copy(node, excluded_descendants).count()
-
-        set_total(total_nodes)
+        if progress_tracker:
+            progress_tracker.set_total(total_nodes)
 
         return self._copy(
             node,
@@ -349,6 +361,7 @@ def copy_node(
             excluded_descendants,
             can_edit_source_channel,
             batch_size,
+            progress_tracker=progress_tracker,
         )
 
     def _copy(
@@ -362,9 +375,13 @@ def _copy(
         excluded_descendants,
         can_edit_source_channel,
         batch_size,
+        progress_tracker=None,
     ):
+        """
+        :type progress_tracker: contentcuration.utils.celery.ProgressTracker|None
+        """
         if node.rght - node.lft < batch_size:
-            return self._deep_copy(
+            copied_nodes = self._deep_copy(
                 node,
                 target,
                 position,
@@ -374,6 +391,9 @@ def _copy(
                 excluded_descendants,
                 can_edit_source_channel,
             )
+            if progress_tracker:
+                progress_tracker.increment(len(copied_nodes))
+            return copied_nodes
         else:
             node_copy = self._shallow_copy(
                 node,
@@ -384,6 +404,8 @@ def _copy(
                 mods,
                 can_edit_source_channel,
             )
+            if progress_tracker:
+                progress_tracker.increment()
             children = node.get_children().order_by("lft")
             if excluded_descendants:
                 children = children.exclude(node_id__in=excluded_descendants.keys())
@@ -398,6 +420,7 @@ def _copy(
                     excluded_descendants,
                     can_edit_source_channel,
                     batch_size,
+                    progress_tracker=progress_tracker,
                 )
             return [node_copy]
 
@@ -532,7 +555,6 @@ def _shallow_copy(
             node_copy.save(force_insert=True)
 
         self._copy_associated_objects({node.id: node_copy.id})
-        increment_progress(1)
         return node_copy
 
     def _deep_copy(
@@ -590,8 +612,6 @@ def _deep_copy(
 
         self._copy_associated_objects(source_copy_id_map)
 
-        increment_progress(len(nodes_to_copy))
-
         return new_nodes
 
     def build_tree_nodes(self, data, target=None, position="last-child"):

contentcuration/contentcuration/db/models/query.py

@@ -1,6 +1,110 @@
+from django.db import connections
+from django.db.models.expressions import Col
+from django.db.models.sql.compiler import SQLCompiler
+from django.db.models.sql.constants import INNER
+from django.db.models.sql.query import Query
 from django_cte import CTEQuerySet
+from django_cte import With as CTEWith
 from mptt.querysets import TreeQuerySet
 
 
+RIGHT_JOIN = 'RIGHT JOIN'
+
+
 class CustomTreeQuerySet(TreeQuerySet, CTEQuerySet):
     pass
+
+
+class With(CTEWith):
+    """
+    Custom CTE class which allows more join types than just INNER and LOUTER (LEFT)
+    """
+    def join(self, model_or_queryset, *filter_q, **filter_kw):
+        """
+        Slight hack to allow more join types
+        """
+        join_type = filter_kw.get('_join_type', INNER)
+        queryset = super(With, self).join(model_or_queryset, *filter_q, **filter_kw)
+
+        # the underlying Django code forces the join type into INNER or a LEFT OUTER join
+        alias, _ = queryset.query.table_alias(self.name)
+        join = queryset.query.alias_map[alias]
+        if join.join_type != join_type:
+            join.join_type = join_type
+        return queryset
+
+
+class WithValues(With):
+    """
+    Allows the creation of a CTE that holds a VALUES list
+
+    @see https://www.postgresql.org/docs/9.6/queries-values.html
+    """
+    def __init__(self, fields, values_list, name="cte"):
+        super(WithValues, self).__init__(None, name=name)
+        self.query = WithValuesQuery(self)
+        self.fields = fields
+        self.values_list = values_list
+
+    def _resolve_ref(self, name):
+        """
+        Gets called when a column reference is accessed via the CTE instance `.col.name`
+        """
+        if name not in self.fields:
+            raise RuntimeError("No field with name `{}`".format(name))
+
+        field = self.fields.get(name)
+        field.set_attributes_from_name(name)
+        return Col(self.name, field, output_field=field)
+
+
+class WithValuesSQLCompiler(SQLCompiler):
+    TEMPLATE = "SELECT * FROM (VALUES {values_statement}) AS {cte_name}({fields_statement})"
+
+    def as_sql(self, with_limits=True, with_col_aliases=False):
+        """
+        Ideally this would return something like:
+            WITH t_cte(fieldA, fieldB) AS (VALUES (), ...)
+        But django_cte doesn't give us a way to do that, so we do this instead:
+            WITH t_cte AS (SELECT * FROM (VALUES (), ...) AS _t_cte(fieldA, fieldB)))
+
+        :return: A tuple of SQL and parameters
+        """
+        value_parameters = ", ".join(["%s"] * len(self.cte.fields))
+        values_statement = ", ".join(["({})".format(value_parameters)] * len(self.cte.values_list))
+        fields_statement = ", ".join([self.connection.ops.quote_name(field) for field in list(self.cte.fields)])
+        sql = self.TEMPLATE.format(
+            values_statement=values_statement,
+            cte_name="_{}".format(self.cte.name),
+            fields_statement=fields_statement
+        )
+        return sql, list(sum(self.cte.values_list, ()))
+
+    @property
+    def cte(self):
+        """
+        :rtype: WithValues
+        """
+        return self.query.cte
+
+
+class WithValuesQuery(Query):
+    """
+    Dedicated query class for creating a CTE
+
+    Note: this does inherit from Query, which we're not passing a Model instance so not all Query
+    functionality is intended to work
+    """
+    def __init__(self, cte):
+        super(WithValuesQuery, self).__init__(None)
+        self.cte = cte
+
+    def get_compiler(self, using=None, connection=None):
+        """
+        This code is modeled after Query.get_compiler()
+        """
+        if using is None and connection is None:
+            raise ValueError("Need either using or connection")
+        if using:
+            connection = connections[using]
+        return WithValuesSQLCompiler(self, connection, using)

contentcuration/contentcuration/dev_urls.py

@@ -1,6 +1,7 @@
 from django.conf import settings
 from django.conf.urls import include
 from django.conf.urls import url
+from django.contrib import admin
 from django.http.response import HttpResponseRedirect
 from drf_yasg import openapi
 from drf_yasg.views import get_schema_view
@@ -31,6 +32,7 @@ def webpack_redirect_view(request):
 
 urlpatterns = urlpatterns + [
     url(r"^__open-in-editor/", webpack_redirect_view),
+    url(r'^admin/', include(admin.site.urls)),
     url(
         r"^swagger(?P<format>\.json|\.yaml)$",
         schema_view.without_ui(cache_timeout=0),

contentcuration/contentcuration/frontend/channelEdit/components/ContentNodeListItem/index.vue

@@ -263,7 +263,7 @@
       ...mapActions('task', ['deleteTask']),
       handleTileClick(e) {
         // Ensures that clicking an icon button is not treated the same as clicking the card
-        if (e.target.tagName !== 'svg' && !this.copying) {
+        if (e.target && e.target.tagName !== 'svg' && !this.copying) {
           this.isTopic ? this.$emit('topicChevronClick') : this.$emit('infoClick');
         }
       },