From 484ab75531df78ee0a469c3ed7d69696ecea663a Mon Sep 17 00:00:00 2001 From: Aron Asor Date: Wed, 13 Jan 2021 08:18:09 -0800 Subject: [PATCH 01/48] first pass of set_storage_used cronjob --- ...set-storage-used-mgmt-command-cronjob.yaml | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 k8s/templates/set-storage-used-mgmt-command-cronjob.yaml diff --git a/k8s/templates/set-storage-used-mgmt-command-cronjob.yaml b/k8s/templates/set-storage-used-mgmt-command-cronjob.yaml new file mode 100644 index 0000000000..cd30ba6f2f --- /dev/null +++ b/k8s/templates/set-storage-used-mgmt-command-cronjob.yaml @@ -0,0 +1,78 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "studio.fullname" . }}-set-storage-used-job-config + labels: + tier: job + app: {{ template "studio.fullname" . }} + chart: {{ .Chart.Name }} + release: {{ .Release.Name }} +data: + DJANGO_LOG_FILE: /var/log/django.log + DATA_DB_HOST: {{ template "cloudsql-proxy.fullname" . }} + DATA_DB_PORT: "5432" + MPLBACKEND: PS + RUN_MODE: k8s + RELEASE_COMMIT_SHA: {{ .Values.studioApp.releaseCommit | default "" }} + BRANCH_ENVIRONMENT: {{ .Release.Name }} + AWS_BUCKET_NAME: {{ .Values.studioApp.gcs.bucketName }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "studio.fullname" . }}-set-storage-used-job-secrets + labels: + app: {{ template "studio.fullname" . }} + chart: {{ .Chart.Name }} + release: {{ .Release.Name }} +type: Opaque +data: + DATA_DB_USER: {{ index .Values "cloudsql-proxy" "credentials" "username" | b64enc }} + DATA_DB_PASS: {{ index .Values "cloudsql-proxy" "credentials" "password" | b64enc }} + DATA_DB_NAME: {{ index .Values "cloudsql-proxy" "credentials" "dbname" | b64enc }} + SENTRY_DSN_KEY: {{ .Values.sentry.dsnKey | b64enc }} +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: set-storage-used-cronjob + labels: + tier: job + chart: {{ .Chart.Name }} + release: {{ .Release.Name }} +spec: + schedule: "@midnight" + jobTemplate: + spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: app + image: {{ .Values.studioApp.imageName }} + command: + - python + - contentcuration/manage.py + - set_storage_used + env: + - name: DJANGO_SETTINGS_MODULE + value: contentcuration.production_settings + envFrom: + - configMapRef: + name: {{ template "studio.fullname" . }}-set-storage-used-job-config + - secretRef: + name: {{ template "studio.fullname" . }}-set-storage-used-job-secrets + resources: + requests: + cpu: 0.5 + memory: 1Gi + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: full-gcp-access-scope + operator: In + values: + - "true" From aa8d7331471db742b988c1d3bbf6fc391de04fa9 Mon Sep 17 00:00:00 2001 From: Aron Asor Date: Wed, 13 Jan 2021 08:58:36 -0800 Subject: [PATCH 02/48] add first pass of mark_incomplete management command --- .../mark-incomplete-mgmt-command-cronjob.yaml | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 k8s/templates/mark-incomplete-mgmt-command-cronjob.yaml diff --git a/k8s/templates/mark-incomplete-mgmt-command-cronjob.yaml b/k8s/templates/mark-incomplete-mgmt-command-cronjob.yaml new file mode 100644 index 0000000000..ad36b8b0e4 --- /dev/null +++ b/k8s/templates/mark-incomplete-mgmt-command-cronjob.yaml @@ -0,0 +1,78 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "studio.fullname" . }}-mark-incomplete-job-config + labels: + tier: job + app: {{ template "studio.fullname" . }} + chart: {{ .Chart.Name }} + release: {{ .Release.Name }} +data: + DJANGO_LOG_FILE: /var/log/django.log + DATA_DB_HOST: {{ template "cloudsql-proxy.fullname" . }} + DATA_DB_PORT: "5432" + MPLBACKEND: PS + RUN_MODE: k8s + RELEASE_COMMIT_SHA: {{ .Values.studioApp.releaseCommit | default "" }} + BRANCH_ENVIRONMENT: {{ .Release.Name }} + AWS_BUCKET_NAME: {{ .Values.studioApp.gcs.bucketName }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "studio.fullname" . }}-mark-incomplete-job-secrets + labels: + app: {{ template "studio.fullname" . }} + chart: {{ .Chart.Name }} + release: {{ .Release.Name }} +type: Opaque +data: + DATA_DB_USER: {{ index .Values "cloudsql-proxy" "credentials" "username" | b64enc }} + DATA_DB_PASS: {{ index .Values "cloudsql-proxy" "credentials" "password" | b64enc }} + DATA_DB_NAME: {{ index .Values "cloudsql-proxy" "credentials" "dbname" | b64enc }} + SENTRY_DSN_KEY: {{ .Values.sentry.dsnKey | b64enc }} +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: mark-incomplete-cronjob + labels: + tier: job + chart: {{ .Chart.Name }} + release: {{ .Release.Name }} +spec: + schedule: "00 12 10 */36 *" + jobTemplate: + spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: app + image: {{ .Values.studioApp.imageName }} + command: + - python + - contentcuration/manage.py + - mark_incomplete + env: + - name: DJANGO_SETTINGS_MODULE + value: contentcuration.production_settings + envFrom: + - configMapRef: + name: {{ template "studio.fullname" . }}-mark-incomplete-job-config + - secretRef: + name: {{ template "studio.fullname" . }}-mark-incomplete-job-secrets + resources: + requests: + cpu: 0.5 + memory: 1Gi + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: full-gcp-access-scope + operator: In + values: + - "true" From 77d9f6bb30ddca10b3ebe7a58427ee6c9e7439da Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Thu, 14 Jan 2021 07:52:59 -0800 Subject: [PATCH 03/48] Clean up and linearize migration history. --- .../migrations/0110_auto_20200117_1948.py | 21 ----- .../migrations/0111_auto_20200423_0124.py | 21 ----- .../migrations/0114_merge_20200806_1746.py | 16 ---- .../0115_index_contentnode_node_id_field.py | 6 +- .../migrations/0116_auto_20200828_2219.py | 21 ----- ...=> 0116_index_channel_contentnode_file.py} | 2 +- ...d_index.py => 0117_assessment_id_index.py} | 2 +- .../migrations/0117_auto_20200828_2220.py | 16 ---- .../migrations/0118_auto_20200904_1737.py | 16 ---- .../migrations/0118_relaunch_migrations.py | 89 +++++++++++++++++++ .../migrations/0120_auto_20200917_1912.py | 17 ---- .../migrations/0121_auto_20200917_1912.py | 19 ---- .../migrations/0123_auto_20200921_1536.py | 28 ------ .../migrations/0124_auto_20201117_2221.py | 21 ----- .../migrations/0124_auto_20201204_1645.py | 24 ----- .../migrations/0125_auto_20201202_2240.py | 21 ----- .../migrations/0126_merge_20201207_2127.py | 16 ---- .../migrations/0127_auto_20210107_1707.py | 20 ----- .../0127_invitation_status_fields.py | 31 ------- .../migrations/0128_merge_20210112_1828.py | 16 ---- .../migrations/0129_auto_20210113_1723.py | 33 ------- .../migrations/0129_merge_20210112_2327.py | 16 ---- .../migrations/0130_merge_20210113_1839.py | 16 ---- 23 files changed, 95 insertions(+), 393 deletions(-) delete mode 100644 contentcuration/contentcuration/migrations/0110_auto_20200117_1948.py delete mode 100644 contentcuration/contentcuration/migrations/0111_auto_20200423_0124.py delete mode 100644 contentcuration/contentcuration/migrations/0114_merge_20200806_1746.py delete mode 100644 contentcuration/contentcuration/migrations/0116_auto_20200828_2219.py rename contentcuration/contentcuration/migrations/{0119_auto_20200915_1839.py => 0116_index_channel_contentnode_file.py} (97%) rename contentcuration/contentcuration/migrations/{0122_assessment_id_index.py => 0117_assessment_id_index.py} (95%) delete mode 100644 contentcuration/contentcuration/migrations/0117_auto_20200828_2220.py delete mode 100644 contentcuration/contentcuration/migrations/0118_auto_20200904_1737.py create mode 100644 contentcuration/contentcuration/migrations/0118_relaunch_migrations.py delete mode 100644 contentcuration/contentcuration/migrations/0120_auto_20200917_1912.py delete mode 100644 contentcuration/contentcuration/migrations/0121_auto_20200917_1912.py delete mode 100644 contentcuration/contentcuration/migrations/0123_auto_20200921_1536.py delete mode 100644 contentcuration/contentcuration/migrations/0124_auto_20201117_2221.py delete mode 100644 contentcuration/contentcuration/migrations/0124_auto_20201204_1645.py delete mode 100644 contentcuration/contentcuration/migrations/0125_auto_20201202_2240.py delete mode 100644 contentcuration/contentcuration/migrations/0126_merge_20201207_2127.py delete mode 100644 contentcuration/contentcuration/migrations/0127_auto_20210107_1707.py delete mode 100644 contentcuration/contentcuration/migrations/0127_invitation_status_fields.py delete mode 100644 contentcuration/contentcuration/migrations/0128_merge_20210112_1828.py delete mode 100644 contentcuration/contentcuration/migrations/0129_auto_20210113_1723.py delete mode 100644 contentcuration/contentcuration/migrations/0129_merge_20210112_2327.py delete mode 100644 contentcuration/contentcuration/migrations/0130_merge_20210113_1839.py diff --git a/contentcuration/contentcuration/migrations/0110_auto_20200117_1948.py b/contentcuration/contentcuration/migrations/0110_auto_20200117_1948.py deleted file mode 100644 index e8badd3820..0000000000 --- a/contentcuration/contentcuration/migrations/0110_auto_20200117_1948.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.20 on 2020-01-17 19:48 -from __future__ import unicode_literals - -from django.db import migrations -from django.db import models - - -class Migration(migrations.Migration): - - dependencies = [ - ('contentcuration', '0109_auto_20191202_1759'), - ] - - operations = [ - migrations.AlterField( - model_name='contentnode', - name='title', - field=models.CharField(blank=True, max_length=200), - ), - ] diff --git a/contentcuration/contentcuration/migrations/0111_auto_20200423_0124.py b/contentcuration/contentcuration/migrations/0111_auto_20200423_0124.py deleted file mode 100644 index 14fbf84379..0000000000 --- a/contentcuration/contentcuration/migrations/0111_auto_20200423_0124.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.20 on 2020-04-23 01:24 -from __future__ import unicode_literals - -from django.db import migrations -from django.db import models - - -class Migration(migrations.Migration): - - dependencies = [ - ('contentcuration', '0110_auto_20200117_1948'), - ] - - operations = [ - migrations.AlterField( - model_name='invitation', - name='first_name', - field=models.CharField(blank=True, max_length=100), - ), - ] diff --git a/contentcuration/contentcuration/migrations/0114_merge_20200806_1746.py b/contentcuration/contentcuration/migrations/0114_merge_20200806_1746.py deleted file mode 100644 index 499c585023..0000000000 --- a/contentcuration/contentcuration/migrations/0114_merge_20200806_1746.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2020-08-06 17:46 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('contentcuration', '0111_auto_20200423_0124'), - ('contentcuration', '0113_channel_tagline'), - ] - - operations = [ - ] diff --git a/contentcuration/contentcuration/migrations/0115_index_contentnode_node_id_field.py b/contentcuration/contentcuration/migrations/0115_index_contentnode_node_id_field.py index eb81f4320d..04dc54c739 100644 --- a/contentcuration/contentcuration/migrations/0115_index_contentnode_node_id_field.py +++ b/contentcuration/contentcuration/migrations/0115_index_contentnode_node_id_field.py @@ -2,15 +2,17 @@ # Generated by Django 1.11.29 on 2020-08-06 20:20 from __future__ import unicode_literals +from django.db import migrations +from django.db import models + import contentcuration.models -from django.db import migrations, models class Migration(migrations.Migration): atomic = False dependencies = [ - ("contentcuration", "0114_merge_20200806_1746"), + ("contentcuration", "0114_assessment_item_unique_keypair"), ] operations = [ diff --git a/contentcuration/contentcuration/migrations/0116_auto_20200828_2219.py b/contentcuration/contentcuration/migrations/0116_auto_20200828_2219.py deleted file mode 100644 index ed1416e249..0000000000 --- a/contentcuration/contentcuration/migrations/0116_auto_20200828_2219.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2020-08-28 22:19 -from __future__ import unicode_literals - -from django.db import migrations -from django.db import models - - -class Migration(migrations.Migration): - - dependencies = [ - ('contentcuration', '0115_index_contentnode_node_id_field'), - ] - - operations = [ - migrations.AddField( - model_name='contentnode', - name='complete', - field=models.NullBooleanField(), - ), - ] diff --git a/contentcuration/contentcuration/migrations/0119_auto_20200915_1839.py b/contentcuration/contentcuration/migrations/0116_index_channel_contentnode_file.py similarity index 97% rename from contentcuration/contentcuration/migrations/0119_auto_20200915_1839.py rename to contentcuration/contentcuration/migrations/0116_index_channel_contentnode_file.py index 33e4c59ee2..695ee570e5 100644 --- a/contentcuration/contentcuration/migrations/0119_auto_20200915_1839.py +++ b/contentcuration/contentcuration/migrations/0116_index_channel_contentnode_file.py @@ -11,7 +11,7 @@ class Migration(migrations.Migration): atomic = False dependencies = [ - ('contentcuration', '0118_auto_20200904_1737'), + ('contentcuration', '0115_index_contentnode_node_id_field'), ] operations = [ diff --git a/contentcuration/contentcuration/migrations/0122_assessment_id_index.py b/contentcuration/contentcuration/migrations/0117_assessment_id_index.py similarity index 95% rename from contentcuration/contentcuration/migrations/0122_assessment_id_index.py rename to contentcuration/contentcuration/migrations/0117_assessment_id_index.py index 6646c645cf..aee4e57372 100644 --- a/contentcuration/contentcuration/migrations/0122_assessment_id_index.py +++ b/contentcuration/contentcuration/migrations/0117_assessment_id_index.py @@ -12,7 +12,7 @@ class Migration(migrations.Migration): atomic = False dependencies = [ - ("contentcuration", "0121_auto_20200917_1912"), + ("contentcuration", "0116_index_channel_contentnode_file"), ] operations = [ diff --git a/contentcuration/contentcuration/migrations/0117_auto_20200828_2220.py b/contentcuration/contentcuration/migrations/0117_auto_20200828_2220.py deleted file mode 100644 index 68ca5aac1f..0000000000 --- a/contentcuration/contentcuration/migrations/0117_auto_20200828_2220.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2020-08-28 22:20 -from __future__ import unicode_literals - -from django.db import migrations -from django.db import models - - -class Migration(migrations.Migration): - - dependencies = [ - ('contentcuration', '0116_auto_20200828_2219'), - ] - - operations = [ - ] diff --git a/contentcuration/contentcuration/migrations/0118_auto_20200904_1737.py b/contentcuration/contentcuration/migrations/0118_auto_20200904_1737.py deleted file mode 100644 index 994cc87a16..0000000000 --- a/contentcuration/contentcuration/migrations/0118_auto_20200904_1737.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2020-09-04 17:37 -from __future__ import unicode_literals - -from django.db import migrations -from django.db import models - - -class Migration(migrations.Migration): - - dependencies = [ - ('contentcuration', '0117_auto_20200828_2220'), - ] - - operations = [ - ] diff --git a/contentcuration/contentcuration/migrations/0118_relaunch_migrations.py b/contentcuration/contentcuration/migrations/0118_relaunch_migrations.py new file mode 100644 index 0000000000..8d8c32ac1d --- /dev/null +++ b/contentcuration/contentcuration/migrations/0118_relaunch_migrations.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2021-01-14 15:50 +from __future__ import unicode_literals + +import django.db.models.deletion +import django.utils.timezone +from django.conf import settings +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ('contentcuration', '0117_assessment_id_index'), + ] + + operations = [ + migrations.AddField( + model_name='contentnode', + name='complete', + field=models.NullBooleanField(), + ), + migrations.AddField( + model_name='invitation', + name='accepted', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='invitation', + name='declined', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='invitation', + name='revoked', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='user', + name='disk_space_used', + field=models.FloatField(default=0, help_text='How many bytes a user has uploaded'), + ), + migrations.AlterField( + model_name='channel', + name='preferences', + field=models.TextField(default='{"license": null, "language": null, "author": null, "aggregator": null, "provider": null, "copyright_holder": null, "license_description": null, "mastery_model": "num_correct_in_a_row_5", "m_value": 5, "n_value": 5, "auto_derive_video_thumbnail": true, "auto_derive_audio_thumbnail": true, "auto_derive_document_thumbnail": true, "auto_derive_html5_thumbnail": true, "auto_derive_exercise_thumbnail": true, "auto_randomize_questions": true}'), + ), + migrations.AlterField( + model_name='contentnode', + name='created', + field=models.DateTimeField(default=django.utils.timezone.now, verbose_name='created'), + ), + migrations.AlterField( + model_name='contentnode', + name='title', + field=models.CharField(blank=True, max_length=200), + ), + migrations.AlterField( + model_name='contenttag', + name='channel', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='tags', to='contentcuration.Channel'), + ), + migrations.AlterField( + model_name='file', + name='uploaded_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='files', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='invitation', + name='channel', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='pending_editors', to='contentcuration.Channel'), + ), + migrations.AlterField( + model_name='invitation', + name='first_name', + field=models.CharField(blank=True, max_length=100), + ), + migrations.AlterField( + model_name='invitation', + name='sender', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='sent_by', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='user', + name='preferences', + field=models.TextField(default='{"license": null, "language": null, "author": null, "aggregator": null, "provider": null, "copyright_holder": null, "license_description": null, "mastery_model": "num_correct_in_a_row_5", "m_value": 5, "n_value": 5, "auto_derive_video_thumbnail": true, "auto_derive_audio_thumbnail": true, "auto_derive_document_thumbnail": true, "auto_derive_html5_thumbnail": true, "auto_derive_exercise_thumbnail": true, "auto_randomize_questions": true}'), + ), + ] diff --git a/contentcuration/contentcuration/migrations/0120_auto_20200917_1912.py b/contentcuration/contentcuration/migrations/0120_auto_20200917_1912.py deleted file mode 100644 index 07256fac03..0000000000 --- a/contentcuration/contentcuration/migrations/0120_auto_20200917_1912.py +++ /dev/null @@ -1,17 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2020-09-17 19:12 -from __future__ import unicode_literals - -from django.db import migrations -from django.db import models - - -class Migration(migrations.Migration): - - dependencies = [ - ('contentcuration', '0119_auto_20200915_1839'), - ] - - operations = [ - - ] diff --git a/contentcuration/contentcuration/migrations/0121_auto_20200917_1912.py b/contentcuration/contentcuration/migrations/0121_auto_20200917_1912.py deleted file mode 100644 index 5126568e47..0000000000 --- a/contentcuration/contentcuration/migrations/0121_auto_20200917_1912.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2020-09-17 19:12 -from __future__ import unicode_literals - -from django.db import migrations -from django.db import models - -import contentcuration.models - - -class Migration(migrations.Migration): - atomic = False - dependencies = [ - ("contentcuration", "0120_auto_20200917_1912"), - ] - - operations = [ - - ] diff --git a/contentcuration/contentcuration/migrations/0123_auto_20200921_1536.py b/contentcuration/contentcuration/migrations/0123_auto_20200921_1536.py deleted file mode 100644 index 724584f76f..0000000000 --- a/contentcuration/contentcuration/migrations/0123_auto_20200921_1536.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2020-09-21 15:36 -from __future__ import unicode_literals - -import django.db.models.deletion -from django.db import migrations -from django.db import models - - -class Migration(migrations.Migration): - - dependencies = [ - ("contentcuration", "0122_assessment_id_index"), - ] - - operations = [ - migrations.AlterField( - model_name="contenttag", - name="channel", - field=models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name="tags", - to="contentcuration.Channel", - ), - ), - ] diff --git a/contentcuration/contentcuration/migrations/0124_auto_20201117_2221.py b/contentcuration/contentcuration/migrations/0124_auto_20201117_2221.py deleted file mode 100644 index 40138fd132..0000000000 --- a/contentcuration/contentcuration/migrations/0124_auto_20201117_2221.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2020-11-17 22:21 -from __future__ import unicode_literals - -from django.db import migrations -from django.db import models - - -class Migration(migrations.Migration): - - dependencies = [ - ('contentcuration', '0123_auto_20200921_1536'), - ] - - operations = [ - migrations.AddField( - model_name='user', - name='disk_space_used', - field=models.FloatField(blank=True, help_text='How many bytes a user has uploaded', null=True), - ), - ] diff --git a/contentcuration/contentcuration/migrations/0124_auto_20201204_1645.py b/contentcuration/contentcuration/migrations/0124_auto_20201204_1645.py deleted file mode 100644 index cf9351e7a3..0000000000 --- a/contentcuration/contentcuration/migrations/0124_auto_20201204_1645.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2020-12-04 16:45 -from __future__ import unicode_literals - -import django.utils.timezone -from django.db import migrations -from django.db import models - - -class Migration(migrations.Migration): - - dependencies = [ - ("contentcuration", "0123_auto_20200921_1536"), - ] - - operations = [ - migrations.AlterField( - model_name="contentnode", - name="created", - field=models.DateTimeField( - default=django.utils.timezone.now, verbose_name="created" - ), - ), - ] diff --git a/contentcuration/contentcuration/migrations/0125_auto_20201202_2240.py b/contentcuration/contentcuration/migrations/0125_auto_20201202_2240.py deleted file mode 100644 index 25edcc4ca2..0000000000 --- a/contentcuration/contentcuration/migrations/0125_auto_20201202_2240.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2020-12-02 22:41 -from __future__ import unicode_literals - -from django.db import migrations -from django.db import models - - -class Migration(migrations.Migration): - - dependencies = [ - ('contentcuration', '0124_auto_20201117_2221'), - ] - - operations = [ - migrations.AlterField( - model_name='user', - name='disk_space_used', - field=models.FloatField(default=0, help_text='How many bytes a user has uploaded'), - ), - ] diff --git a/contentcuration/contentcuration/migrations/0126_merge_20201207_2127.py b/contentcuration/contentcuration/migrations/0126_merge_20201207_2127.py deleted file mode 100644 index debbbf8260..0000000000 --- a/contentcuration/contentcuration/migrations/0126_merge_20201207_2127.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2020-12-07 21:27 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('contentcuration', '0124_auto_20201204_1645'), - ('contentcuration', '0125_auto_20201202_2240'), - ] - - operations = [ - ] diff --git a/contentcuration/contentcuration/migrations/0127_auto_20210107_1707.py b/contentcuration/contentcuration/migrations/0127_auto_20210107_1707.py deleted file mode 100644 index 4d40ac62ec..0000000000 --- a/contentcuration/contentcuration/migrations/0127_auto_20210107_1707.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2021-01-07 17:07 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('contentcuration', '0126_merge_20201207_2127'), - ] - - operations = [ - migrations.AlterField( - model_name='contentnode', - name='complete', - field=models.NullBooleanField(), - ), - ] diff --git a/contentcuration/contentcuration/migrations/0127_invitation_status_fields.py b/contentcuration/contentcuration/migrations/0127_invitation_status_fields.py deleted file mode 100644 index f48140c0d2..0000000000 --- a/contentcuration/contentcuration/migrations/0127_invitation_status_fields.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2020-12-18 08:00 -from __future__ import unicode_literals - -from django.db import migrations -from django.db import models - - -class Migration(migrations.Migration): - - dependencies = [ - ('contentcuration', '0126_merge_20201207_2127'), - ] - - operations = [ - migrations.AddField( - model_name='invitation', - name='accepted', - field=models.BooleanField(default=False), - ), - migrations.AddField( - model_name='invitation', - name='declined', - field=models.BooleanField(default=False), - ), - migrations.AddField( - model_name='invitation', - name='revoked', - field=models.BooleanField(default=False), - ), - ] diff --git a/contentcuration/contentcuration/migrations/0128_merge_20210112_1828.py b/contentcuration/contentcuration/migrations/0128_merge_20210112_1828.py deleted file mode 100644 index f5eff1e224..0000000000 --- a/contentcuration/contentcuration/migrations/0128_merge_20210112_1828.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2021-01-12 18:28 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('contentcuration', '0127_auto_20210107_1707'), - ('contentcuration', '0114_assessment_item_unique_keypair'), - ] - - operations = [ - ] diff --git a/contentcuration/contentcuration/migrations/0129_auto_20210113_1723.py b/contentcuration/contentcuration/migrations/0129_auto_20210113_1723.py deleted file mode 100644 index 8e70efaa2b..0000000000 --- a/contentcuration/contentcuration/migrations/0129_auto_20210113_1723.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2021-01-13 17:23 -from __future__ import unicode_literals - -import django.db.models.deletion -from django.conf import settings -from django.db import migrations -from django.db import models - - -class Migration(migrations.Migration): - - dependencies = [ - ('contentcuration', '0128_merge_20210112_1828'), - ] - - operations = [ - migrations.AlterField( - model_name='file', - name='uploaded_by', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='files', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='invitation', - name='channel', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='pending_editors', to='contentcuration.Channel'), - ), - migrations.AlterField( - model_name='invitation', - name='sender', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='sent_by', to=settings.AUTH_USER_MODEL), - ), - ] diff --git a/contentcuration/contentcuration/migrations/0129_merge_20210112_2327.py b/contentcuration/contentcuration/migrations/0129_merge_20210112_2327.py deleted file mode 100644 index 50c2fceee9..0000000000 --- a/contentcuration/contentcuration/migrations/0129_merge_20210112_2327.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2021-01-12 23:27 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('contentcuration', '0128_merge_20210112_1828'), - ('contentcuration', '0127_invitation_status_fields'), - ] - - operations = [ - ] diff --git a/contentcuration/contentcuration/migrations/0130_merge_20210113_1839.py b/contentcuration/contentcuration/migrations/0130_merge_20210113_1839.py deleted file mode 100644 index 28baf1728e..0000000000 --- a/contentcuration/contentcuration/migrations/0130_merge_20210113_1839.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2021-01-13 18:39 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('contentcuration', '0129_auto_20210113_1723'), - ('contentcuration', '0129_merge_20210112_2327'), - ] - - operations = [ - ] From b99b3b4a98a646ce2a43443506d370e3961d66c0 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Thu, 14 Jan 2021 11:02:54 -0800 Subject: [PATCH 04/48] Add True setting for incomplete. Wrap in transaction. --- .../management/commands/mark_incomplete.py | 66 ++++++++++--------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/contentcuration/contentcuration/management/commands/mark_incomplete.py b/contentcuration/contentcuration/management/commands/mark_incomplete.py index be417d1c79..5903d6d2b5 100644 --- a/contentcuration/contentcuration/management/commands/mark_incomplete.py +++ b/contentcuration/contentcuration/management/commands/mark_incomplete.py @@ -2,6 +2,7 @@ import time from django.core.management.base import BaseCommand +from django.db import transaction from django.db.models import Exists from django.db.models import OuterRef from django.db.models import Q @@ -19,37 +20,42 @@ class Command(BaseCommand): def handle(self, *args, **options): start = time.time() - exercise_check_query = AssessmentItem.objects.filter(contentnode=OuterRef('id')) \ - .exclude(type=exercises.PERSEUS_QUESTION)\ - .filter( - Q(question='') | - Q(answers='[]') | - (~Q(type=exercises.INPUT_QUESTION) & ~Q(answers__iregex=r'"correct":\s*true')) # hack to check if no correct answers - ) - file_check_query = File.objects.filter(preset__supplementary=False, contentnode=OuterRef("id")) + # Wrap in a transaction so that other null incompletes are not accidentally + # marked as True. + with transaction.atomic(): + exercise_check_query = AssessmentItem.objects.filter(contentnode=OuterRef('id')) \ + .exclude(type=exercises.PERSEUS_QUESTION)\ + .filter( + Q(question='') | + Q(answers='[]') | + (~Q(type=exercises.INPUT_QUESTION) & ~Q(answers__iregex=r'"correct":\s*true')) # hack to check if no correct answers + ) + file_check_query = File.objects.filter(preset__supplementary=False, contentnode=OuterRef("id")) - invalid_nodes = ContentNode.objects.filter(complete__isnull=True).annotate( - has_files=Exists(file_check_query), - has_questions=Exists(AssessmentItem.objects.filter(contentnode=OuterRef("id"))), - invalid_exercise=Exists(exercise_check_query) - ).filter( - Q(title='') | - ~Q(kind_id=content_kinds.TOPIC) & ( - (~Q(kind_id=content_kinds.EXERCISE) & Q(has_files=False)) | - Q(license=None) | - (Q(license__is_custom=True) & (Q(license_description=None) | Q(license_description=''))) | - (Q(license__copyright_holder_required=True) & (Q(copyright_holder=None) | Q(copyright_holder=''))) - ) | - Q(kind_id=content_kinds.EXERCISE) & ( - Q(has_questions=False) | - Q(invalid_exercise=True) | - ~Q(extra_fields__has_key='type') | - Q(extra_fields__type=exercises.M_OF_N) & ( - ~Q(extra_fields__has_key='m') | ~Q(extra_fields__has_key='n') + invalid_nodes = ContentNode.objects.filter(complete__isnull=True).annotate( + has_files=Exists(file_check_query), + has_questions=Exists(AssessmentItem.objects.filter(contentnode=OuterRef("id"))), + invalid_exercise=Exists(exercise_check_query) + ).filter( + Q(title='') | + ~Q(kind_id=content_kinds.TOPIC) & ( + (~Q(kind_id=content_kinds.EXERCISE) & Q(has_files=False)) | + Q(license=None) | + (Q(license__is_custom=True) & (Q(license_description=None) | Q(license_description=''))) | + (Q(license__copyright_holder_required=True) & (Q(copyright_holder=None) | Q(copyright_holder=''))) + ) | + Q(kind_id=content_kinds.EXERCISE) & ( + Q(has_questions=False) | + Q(invalid_exercise=True) | + ~Q(extra_fields__has_key='type') | + Q(extra_fields__type=exercises.M_OF_N) & ( + ~Q(extra_fields__has_key='m') | ~Q(extra_fields__has_key='n') + ) ) - ) - ).values_list('id', flat=True) + ).values_list('id', flat=True) - # Getting an error on bulk update with the annotations, so query again - ContentNode.objects.filter(pk__in=invalid_nodes).update(complete=False) + # Getting an error on bulk update with the annotations, so query again + ContentNode.objects.filter(pk__in=invalid_nodes).update(complete=False) + # Update anything that has not been marked as false as true, by the law of the excluded middle + ContentNode.objects.filter(complete__isnull=True).update(complete=True) logging.info('mark_incomplete command completed in {}s'.format(time.time() - start)) From 9884b999f5f3f62ed46da3db945ec95523f3dcd3 Mon Sep 17 00:00:00 2001 From: Jordan Yoshihara Date: Thu, 14 Jan 2021 11:38:56 -0800 Subject: [PATCH 05/48] Added epub renderer --- .../views/files/ContentRenderer.vue | 22 +++++-- .../channelEdit/views/files/EpubRenderer.vue | 66 +++++++++++++++++++ .../channelEdit/views/files/FilePreview.vue | 5 -- 3 files changed, 84 insertions(+), 9 deletions(-) create mode 100644 contentcuration/contentcuration/frontend/channelEdit/views/files/EpubRenderer.vue diff --git a/contentcuration/contentcuration/frontend/channelEdit/views/files/ContentRenderer.vue b/contentcuration/contentcuration/frontend/channelEdit/views/files/ContentRenderer.vue index c59573d28c..c709eafa9f 100644 --- a/contentcuration/contentcuration/frontend/channelEdit/views/files/ContentRenderer.vue +++ b/contentcuration/contentcuration/frontend/channelEdit/views/files/ContentRenderer.vue @@ -42,6 +42,10 @@ @load="loading = false" > +
+ +
+ @@ -62,13 +66,16 @@ import uniqBy from 'lodash/uniqBy'; import sortBy from 'lodash/sortBy'; + import ePub from 'epubjs'; import { mapGetters } from 'vuex'; + import EpubRenderer from './EpubRenderer'; import FileStatus from 'shared/views/files/FileStatus'; export default { name: 'ContentRenderer', components: { FileStatus, + EpubRenderer, }, props: { fileId: { @@ -117,6 +124,9 @@ isPDF() { return this.file.file_format === 'pdf'; }, + isEpub() { + return this.file.file_format === 'epub'; + }, htmlPath() { return `/zipcontent/${this.file.checksum}.${this.file.file_format}`; }, @@ -145,14 +155,16 @@ video, audio, embed, - iframe { + iframe, + .epub { width: 100%; outline: none; } .v-card, .v-card > .layout, embed, - iframe { + iframe, + .epub { min-height: 200px; max-height: @max-height; } @@ -162,7 +174,8 @@ .message-card, video, embed, - iframe { + iframe, + .epub { border-color: var(--v-greyBorder-base) !important; border-style: solid; border-width: 1px; @@ -173,7 +186,8 @@ .v-card, audio, embed, - iframe { + iframe, + .epub { min-height: @max-height; } } diff --git a/contentcuration/contentcuration/frontend/channelEdit/views/files/EpubRenderer.vue b/contentcuration/contentcuration/frontend/channelEdit/views/files/EpubRenderer.vue new file mode 100644 index 0000000000..d93ee13470 --- /dev/null +++ b/contentcuration/contentcuration/frontend/channelEdit/views/files/EpubRenderer.vue @@ -0,0 +1,66 @@ + + + diff --git a/contentcuration/contentcuration/frontend/channelEdit/views/files/FilePreview.vue b/contentcuration/contentcuration/frontend/channelEdit/views/files/FilePreview.vue index ebc80cb467..e691bd991e 100644 --- a/contentcuration/contentcuration/frontend/channelEdit/views/files/FilePreview.vue +++ b/contentcuration/contentcuration/frontend/channelEdit/views/files/FilePreview.vue @@ -96,10 +96,6 @@ isAudio() { return this.file.file_format === 'mp3'; }, - isEPub() { - // TODO: Remove once epub previewer is available - return this.file.file_format === 'epub'; - }, isZip() { return this.file.file_format === 'zip'; }, @@ -110,7 +106,6 @@ this.file.url && this.isPreviewable && !this.isAudio && - !this.isEPub && !this.file.uploading ); }, From 04248676284faf31219ad1c160f46100a6c1c0bb Mon Sep 17 00:00:00 2001 From: Jordan Yoshihara Date: Thu, 14 Jan 2021 11:39:17 -0800 Subject: [PATCH 06/48] Removed unused import --- .../frontend/channelEdit/views/files/ContentRenderer.vue | 1 - 1 file changed, 1 deletion(-) diff --git a/contentcuration/contentcuration/frontend/channelEdit/views/files/ContentRenderer.vue b/contentcuration/contentcuration/frontend/channelEdit/views/files/ContentRenderer.vue index c709eafa9f..27a96905e2 100644 --- a/contentcuration/contentcuration/frontend/channelEdit/views/files/ContentRenderer.vue +++ b/contentcuration/contentcuration/frontend/channelEdit/views/files/ContentRenderer.vue @@ -66,7 +66,6 @@ import uniqBy from 'lodash/uniqBy'; import sortBy from 'lodash/sortBy'; - import ePub from 'epubjs'; import { mapGetters } from 'vuex'; import EpubRenderer from './EpubRenderer'; import FileStatus from 'shared/views/files/FileStatus'; From 71686828da11aee8ad9e4b3a9089c3c1d653d21b Mon Sep 17 00:00:00 2001 From: Jordan Yoshihara Date: Thu, 14 Jan 2021 11:43:00 -0800 Subject: [PATCH 07/48] Added comment --- .../frontend/channelEdit/views/files/EpubRenderer.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentcuration/contentcuration/frontend/channelEdit/views/files/EpubRenderer.vue b/contentcuration/contentcuration/frontend/channelEdit/views/files/EpubRenderer.vue index d93ee13470..4719341f9b 100644 --- a/contentcuration/contentcuration/frontend/channelEdit/views/files/EpubRenderer.vue +++ b/contentcuration/contentcuration/frontend/channelEdit/views/files/EpubRenderer.vue @@ -59,7 +59,7 @@ displayed.then(() => { this.$emit('load'); }); - }, 1000); + }, 1000); // There seems to be some lag for loading, so add delay to be safe }, }; From db5286b5d5d708ff00523b986e646eb03b6ecd9c Mon Sep 17 00:00:00 2001 From: Jordan Yoshihara Date: Thu, 14 Jan 2021 11:59:00 -0800 Subject: [PATCH 08/48] Made width a computed field --- .../frontend/channelEdit/views/files/EpubRenderer.vue | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/contentcuration/contentcuration/frontend/channelEdit/views/files/EpubRenderer.vue b/contentcuration/contentcuration/frontend/channelEdit/views/files/EpubRenderer.vue index 4719341f9b..961edef489 100644 --- a/contentcuration/contentcuration/frontend/channelEdit/views/files/EpubRenderer.vue +++ b/contentcuration/contentcuration/frontend/channelEdit/views/files/EpubRenderer.vue @@ -10,7 +10,7 @@ @click="rendition.prev()" /> -
+
{ this.book = ePub(this.src); this.rendition = this.book.renderTo(this.$refs.epub, { manager: 'continuous', flow: 'paginated', - width: 'calc(100% - 96px)', + width: this.width, height: '100%', }); const displayed = this.rendition.display(); From 7e876a5cb49f21e9abf6a119e4cdd7496e2cc441 Mon Sep 17 00:00:00 2001 From: Jordan Yoshihara Date: Thu, 14 Jan 2021 12:30:24 -0800 Subject: [PATCH 09/48] Fixed issue where creating a channel could open the sharing tab instead of details --- .../frontend/channelList/views/Channel/ChannelList.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentcuration/contentcuration/frontend/channelList/views/Channel/ChannelList.vue b/contentcuration/contentcuration/frontend/channelList/views/Channel/ChannelList.vue index 5e732d92d3..987f00cedf 100644 --- a/contentcuration/contentcuration/frontend/channelList/views/Channel/ChannelList.vue +++ b/contentcuration/contentcuration/frontend/channelList/views/Channel/ChannelList.vue @@ -111,7 +111,7 @@ this.createChannel().then(id => { this.$router.push({ name: RouterNames.CHANNEL_EDIT, - params: { channelId: id }, + params: { channelId: id, tab: 'edit' }, query: { last: this.$route.name }, }); }); From 1c8cf087953232fb09585739ab91f11907aedf48 Mon Sep 17 00:00:00 2001 From: Aron Asor Date: Thu, 14 Jan 2021 12:15:51 -0800 Subject: [PATCH 10/48] set a node affinity for workers This ensures we run the workers on a node that has full GCP access scope --- k8s/templates/studio-deployment.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/k8s/templates/studio-deployment.yaml b/k8s/templates/studio-deployment.yaml index 424f71f5eb..f6a74f36fa 100644 --- a/k8s/templates/studio-deployment.yaml +++ b/k8s/templates/studio-deployment.yaml @@ -146,3 +146,12 @@ spec: limits: cpu: 2 memory: 8Gi + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: full-gcp-access-scope + operator: In + values: + - "true" From 0961c22e9102eb5b8c6a7eba5607b57fbd72483e Mon Sep 17 00:00:00 2001 From: Aron Asor Date: Thu, 14 Jan 2021 12:29:38 -0800 Subject: [PATCH 11/48] set a default of 5 replicas for the frontend pods Temporary while I figure out how to convince helm not to overwrite the number of replicas --- k8s/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/k8s/values.yaml b/k8s/values.yaml index 9873f30140..e5c8cc9fe7 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -17,6 +17,7 @@ studioApp: imageName: "REPLACEME" postmarkApiKey: "REPLACEME" releaseCommit: "" + replicas: 5 appPort: 8081 gcs: bucketName: develop-studio-content From 288724e5ae057093d36b5e8cd457d7d1e32f42e2 Mon Sep 17 00:00:00 2001 From: Jordan Yoshihara Date: Thu, 14 Jan 2021 12:38:07 -0800 Subject: [PATCH 12/48] Added title for new channels --- .../frontend/shared/views/channel/ChannelModal.vue | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contentcuration/contentcuration/frontend/shared/views/channel/ChannelModal.vue b/contentcuration/contentcuration/frontend/shared/views/channel/ChannelModal.vue index faa2a43dea..6e0f822a68 100644 --- a/contentcuration/contentcuration/frontend/shared/views/channel/ChannelModal.vue +++ b/contentcuration/contentcuration/frontend/shared/views/channel/ChannelModal.vue @@ -295,7 +295,9 @@ } }, updateTitleForPage() { - if (this.$route.params.tab === 'edit') { + if (this.isNew) { + this.updateTabTitle(`${this.$tr('creatingHeader')} - ${this.channel.name}`); + } else if (this.$route.params.tab === 'edit') { this.updateTabTitle(`${this.$tr('editTab')} - ${this.channel.name}`); } else { this.updateTabTitle(`${this.$tr('shareTab')} - ${this.channel.name}`); From 4f5a99a11f65945b23915827511804faac5a00ce Mon Sep 17 00:00:00 2001 From: Jordan Yoshihara Date: Thu, 14 Jan 2021 12:39:52 -0800 Subject: [PATCH 13/48] fixed tab title --- .../frontend/shared/views/channel/ChannelModal.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentcuration/contentcuration/frontend/shared/views/channel/ChannelModal.vue b/contentcuration/contentcuration/frontend/shared/views/channel/ChannelModal.vue index 6e0f822a68..a81f652a50 100644 --- a/contentcuration/contentcuration/frontend/shared/views/channel/ChannelModal.vue +++ b/contentcuration/contentcuration/frontend/shared/views/channel/ChannelModal.vue @@ -296,7 +296,7 @@ }, updateTitleForPage() { if (this.isNew) { - this.updateTabTitle(`${this.$tr('creatingHeader')} - ${this.channel.name}`); + this.updateTabTitle(this.$tr('creatingHeader')); } else if (this.$route.params.tab === 'edit') { this.updateTabTitle(`${this.$tr('editTab')} - ${this.channel.name}`); } else { From 817896d5ed3eacea9674796bb4e4288765fa38c3 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Thu, 14 Jan 2021 13:13:47 -0800 Subject: [PATCH 14/48] Make node deletion an async task. --- contentcuration/contentcuration/tasks.py | 26 +++++++++++++++++ .../contentcuration/viewsets/contentnode.py | 28 +++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/contentcuration/contentcuration/tasks.py b/contentcuration/contentcuration/tasks.py index f42f5a00a2..ff73f719f0 100644 --- a/contentcuration/contentcuration/tasks.py +++ b/contentcuration/contentcuration/tasks.py @@ -60,6 +60,31 @@ # runs the management command 'exportchannel' async through celery +@task(bind=True, name="delete_node_task") +def delete_node_task( + self, + user_id, + channel_id, + node_id, +): + node = ContentNode.objects.get(id=node_id) + + deleted = False + attempts = 0 + try: + while not deleted and attempts < 10: + try: + node.delete() + deleted = True + except OperationalError as e: + if "deadlock detected" in e.args[0]: + pass + else: + raise + except Exception as e: + report_exception(e) + + @task(bind=True, name="move_nodes_task") def move_nodes_task( self, @@ -261,6 +286,7 @@ def sendcustomemails_task(subject, message, query): type_mapping = { "duplicate-nodes": {"task": duplicate_nodes_task, "progress_tracking": True}, "move-nodes": {"task": move_nodes_task, "progress_tracking": False}, + "delete-node": {"task": delete_node_task, "progress_tracking": False}, "export-channel": {"task": export_channel_task, "progress_tracking": True}, "sync-channel": {"task": sync_channel_task, "progress_tracking": True}, "get-node-diff": {"task": generatenodediff_task, "progress_tracking": False}, diff --git a/contentcuration/contentcuration/viewsets/contentnode.py b/contentcuration/contentcuration/viewsets/contentnode.py index 87841ef101..52fb7362c7 100644 --- a/contentcuration/contentcuration/viewsets/contentnode.py +++ b/contentcuration/contentcuration/viewsets/contentnode.py @@ -51,6 +51,7 @@ from contentcuration.viewsets.sync.constants import TASK_ID from contentcuration.viewsets.sync.utils import generate_delete_event from contentcuration.viewsets.sync.utils import generate_update_event +from contentcuration.viewsets.sync.utils import log_sync_exception channel_query = Channel.objects.filter(main_tree__tree_id=OuterRef("tree_id")) @@ -759,3 +760,30 @@ def copy( None, [generate_update_event(pk, CONTENTNODE, {TASK_ID: task_info.task_id})], ) + + def delete_from_changes(self, changes): + errors = [] + changes_to_return = [] + queryset = self.get_edit_queryset().order_by() + for change in changes: + try: + instance = queryset.get(**dict(self.values_from_key(change["key"]))) + + task_args = { + "user_id": self.request.user.id, + "channel_id": instance.channel_id, + "node_id": instance.id, + } + + task, task_info = create_async_task( + "delete-node", self.request.user, **task_args + ) + except ContentNode.DoesNotExist: + # If the object already doesn't exist, as far as the user is concerned + # job done! + pass + except Exception as e: + log_sync_exception(e) + change["errors"] = [str(e)] + errors.append(change) + return errors, changes_to_return From 3de8297e43d87b38f576ba1236e71e29f9f052f1 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Thu, 14 Jan 2021 13:15:38 -0800 Subject: [PATCH 15/48] Use channel methods for filtering queryset --- .../contentcuration/viewsets/task.py | 23 +------------------ 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/contentcuration/contentcuration/viewsets/task.py b/contentcuration/contentcuration/viewsets/task.py index eb19812ff3..263e29822a 100644 --- a/contentcuration/contentcuration/viewsets/task.py +++ b/contentcuration/contentcuration/viewsets/task.py @@ -1,7 +1,4 @@ from django.conf import settings -from django.db.models import Exists -from django.db.models import OuterRef -from django.db.models import Q from django_filters.rest_framework import DjangoFilterBackend from django_filters.rest_framework import UUIDFilter from rest_framework.permissions import IsAuthenticated @@ -9,7 +6,6 @@ from contentcuration.celery import app from contentcuration.models import Channel from contentcuration.models import Task -from contentcuration.models import User from contentcuration.viewsets.base import DestroyModelMixin from contentcuration.viewsets.base import ReadOnlyValuesViewset from contentcuration.viewsets.base import RequiredFilterSet @@ -19,24 +15,7 @@ class TaskFilter(RequiredFilterSet): channel = UUIDFilter(method="filter_channel") def filter_channel(self, queryset, name, value): - user_id = not self.request.user.is_anonymous() and self.request.user.id - user_email = not self.request.user.is_anonymous() and self.request.user.email - user_queryset = User.objects.filter(id=user_id) - channel_queryset = Channel.objects.annotate( - edit=Exists(user_queryset.filter(editable_channels=OuterRef("id"))), - view=Exists(user_queryset.filter(view_only_channels=OuterRef("id"))), - ) - channel_queryset = channel_queryset.filter( - Q(view=True) - | Q(edit=True) - | Q( - id__in=Channel.objects.filter(deleted=False) - .filter(Q(public=True) | Q(pending_editors__email=user_email)) - .values_list("id", flat=True) - .distinct() - ) - ) - + channel_queryset = Channel.filter_view_queryset(Channel.objects.all(), self.request.user) if channel_queryset.filter(id=value).exists(): return queryset.filter(metadata__affects__channel=value.hex) return queryset.none() From faae9b616824d57972c75e1428f4ace505f76e5b Mon Sep 17 00:00:00 2001 From: Jordan Yoshihara Date: Thu, 14 Jan 2021 14:16:01 -0800 Subject: [PATCH 16/48] Added pagination to channel lists --- .../frontend/channelList/constants.js | 2 + .../channelList/views/Channel/ChannelList.vue | 43 ++++++++++++------- .../Channel/__tests__/channelList.spec.js | 6 ++- .../frontend/shared/vuex/channel/actions.js | 11 +++-- .../frontend/shared/vuex/channel/index.js | 8 ++++ .../frontend/shared/vuex/channel/mutations.js | 20 +++++++++ .../contentcuration/viewsets/channel.py | 1 + 7 files changed, 72 insertions(+), 19 deletions(-) diff --git a/contentcuration/contentcuration/frontend/channelList/constants.js b/contentcuration/contentcuration/frontend/channelList/constants.js index 708aee5668..f2d9d8bbb3 100644 --- a/contentcuration/contentcuration/frontend/channelList/constants.js +++ b/contentcuration/contentcuration/frontend/channelList/constants.js @@ -34,3 +34,5 @@ export const ListTypeToRouteMapping = { }; export const RouteToListTypeMapping = invert(ListTypeToRouteMapping); + +export const CHANNEL_PAGE_SIZE = 25; diff --git a/contentcuration/contentcuration/frontend/channelList/views/Channel/ChannelList.vue b/contentcuration/contentcuration/frontend/channelList/views/Channel/ChannelList.vue index 5e732d92d3..e54749476c 100644 --- a/contentcuration/contentcuration/frontend/channelList/views/Channel/ChannelList.vue +++ b/contentcuration/contentcuration/frontend/channelList/views/Channel/ChannelList.vue @@ -35,6 +35,12 @@ fullWidth /> + + + @@ -46,11 +52,12 @@