diff --git a/contentcuration/contentcuration/apps.py b/contentcuration/contentcuration/apps.py index e448580629..f42624ce9e 100644 --- a/contentcuration/contentcuration/apps.py +++ b/contentcuration/contentcuration/apps.py @@ -1,7 +1,7 @@ from django.apps import AppConfig from django.conf import settings -from contentcuration.utils.minio_utils import ensure_storage_bucket_public +from contentcuration.utils.storage_common import is_gcs_backend class ContentConfig(AppConfig): @@ -10,5 +10,7 @@ class ContentConfig(AppConfig): def ready(self): # see note in the celery_signals.py file for why we import here. import contentcuration.utils.celery.signals # noqa - if settings.AWS_AUTO_CREATE_BUCKET: + + if settings.AWS_AUTO_CREATE_BUCKET and not is_gcs_backend(): + from contentcuration.utils.minio_utils import ensure_storage_bucket_public ensure_storage_bucket_public() diff --git a/contentcuration/contentcuration/management/commands/setup.py b/contentcuration/contentcuration/management/commands/setup.py index d7be84141a..5a41ed9e7b 100644 --- a/contentcuration/contentcuration/management/commands/setup.py +++ b/contentcuration/contentcuration/management/commands/setup.py @@ -23,8 +23,8 @@ from contentcuration.utils.db_tools import create_topic from contentcuration.utils.db_tools import create_user from contentcuration.utils.files import duplicate_file -from contentcuration.utils.minio_utils import ensure_storage_bucket_public from contentcuration.utils.publish import publish_channel +from contentcuration.utils.storage_common import is_gcs_backend logmodule.basicConfig() logging = logmodule.getLogger(__name__) @@ -56,7 +56,9 @@ def handle(self, *args, **options): sys.exit() # create the minio bucket - ensure_storage_bucket_public() + if not is_gcs_backend(): + from contentcuration.utils.minio_utils import ensure_storage_bucket_public + ensure_storage_bucket_public() # create the cache table try: diff --git a/contentcuration/contentcuration/utils/minio_utils.py b/contentcuration/contentcuration/utils/minio_utils.py index bb08e06f11..07d1bfe3cb 100644 --- a/contentcuration/contentcuration/utils/minio_utils.py +++ b/contentcuration/contentcuration/utils/minio_utils.py @@ -13,6 +13,8 @@ from minio.error import BucketAlreadyOwnedByYou from minio.error import ResponseError +from contentcuration.utils.storage_common import is_gcs_backend + logger = logging.getLogger(__name__) @@ -42,6 +44,12 @@ def stop_minio(p): def ensure_storage_bucket_public(bucket=None, will_sleep=True): + # GCS' S3 compatibility is broken, especially in bucket operations; + # skip bucket creation there and just bug Aron to create buckets with + # public-read access for you + if is_gcs_backend(): + logging.info("Skipping storage creation on googleapis") + return # If true, sleep for 5 seconds to wait for minio to start if will_sleep: @@ -53,14 +61,6 @@ def ensure_storage_bucket_public(bucket=None, will_sleep=True): bucketname = bucket host = urlparse(settings.AWS_S3_ENDPOINT_URL).netloc - - # GCS' S3 compatibility is broken, especially in bucket operations; - # skip bucket creation there and just bug Aron to create buckets with - # public-read access for you - if "storage.googleapis.com" in host: - logging.info("Skipping storage creation on googleapis") - return - c = minio.Minio( host, access_key=settings.AWS_ACCESS_KEY_ID, diff --git a/contentcuration/contentcuration/utils/storage_common.py b/contentcuration/contentcuration/utils/storage_common.py index b41b018511..9ce747fe1e 100644 --- a/contentcuration/contentcuration/utils/storage_common.py +++ b/contentcuration/contentcuration/utils/storage_common.py @@ -1,6 +1,7 @@ import mimetypes import os from datetime import timedelta +from urllib.parse import urlparse from django.conf import settings from django.core.files.storage import default_storage @@ -19,6 +20,15 @@ class UnknownStorageBackendError(Exception): pass +def is_gcs_backend(): + """ + Determines if storage is GCS backend, which if not we can assume it is minio + :return: A bool + """ + host = urlparse(settings.AWS_S3_ENDPOINT_URL).netloc + return "storage.googleapis.com" in host + + def determine_content_type(filename): """ Guesses the content type of a filename. Returns the mimetype of a file. diff --git a/requirements-dev.in b/requirements-dev.in index 21278b07f4..483fcda800 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -41,3 +41,4 @@ git+https://github.com/someshchaturvedi/customizable-django-profiler.git#customi tabulate==0.8.2 fonttools flower==0.9.4 +minio==3.0.3 diff --git a/requirements-dev.txt b/requirements-dev.txt index 87119990a1..799e0b63c7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -40,6 +40,7 @@ certifi==2020.12.5 # via # -c requirements.txt # geventhttpclient + # minio # requests cfgv==3.3.0 # via pre-commit @@ -194,6 +195,8 @@ mccabe==0.6.1 # via # flake8 # pylint +minio==3.0.3 + # via -r requirements-dev.in mixer==6.1.3 # via -r requirements-dev.in mock==4.0.3 @@ -301,6 +304,7 @@ pytz==2021.1 # celery # django # flower + # minio pyyaml==5.4.1 # via # aspy.yaml @@ -371,9 +375,10 @@ uritemplate==3.0.1 # -c requirements.txt # coreapi # drf-yasg -urllib3==1.26.4 +urllib3==1.26.5 # via # -c requirements.txt + # minio # requests vine==1.3.0 # via diff --git a/requirements.in b/requirements.in index 4d1fd30d63..29179dd55e 100644 --- a/requirements.in +++ b/requirements.in @@ -14,7 +14,6 @@ newrelic>=2.86.3.70 celery<5 redis pycountry==17.5.14 -minio==3.0.3 pathlib progressbar2==3.38.0 python-postmark==0.5.0 diff --git a/requirements.txt b/requirements.txt index ff199c333e..e1c11d4f6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,6 @@ celery==4.4.7 # via -r requirements.in certifi==2020.12.5 # via - # minio # requests # sentry-sdk cffi==1.14.5 @@ -168,8 +167,6 @@ kombu==4.6.11 # via celery le-utils==0.1.31 # via -r requirements.in -minio==3.0.3 - # via -r requirements.in newrelic==6.2.0.156 # via -r requirements.in oauth2client==4.1.3 @@ -232,7 +229,6 @@ pytz==2021.1 # django # django-postmark # google-api-core - # minio raven==6.10.0 # via -r requirements.in redis==3.5.3 @@ -278,10 +274,9 @@ typing-extensions==3.10.0.0 # via asgiref uritemplate==3.0.1 # via google-api-python-client -urllib3==1.26.4 +urllib3==1.26.5 # via # botocore - # minio # requests # sentry-sdk vine==1.3.0