Merge pull request #22569 from cpennington/static-files-s3

Static files s3

openedx/core/djangoapps/theming/finders.py

@@ -47,7 +47,7 @@ class ThemeFilesFinder(BaseFinder):
         themes = get_themes()
         for theme in themes:
             theme_storage = self.storage_class(
-                os.path.join(theme.path, self.source_dir),
+                location=os.path.join(theme.path, self.source_dir),
                 prefix=theme.theme_dir_name,
             )
 

openedx/core/djangoapps/theming/storage.py

@@ -28,7 +28,7 @@ from openedx.core.djangoapps.theming.helpers import (
 )
 
 
-class ThemeStorage(StaticFilesStorage):
+class ThemeMixin(object):
     """
     Comprehensive theme aware Static files storage.
     """
@@ -38,16 +38,10 @@ class ThemeStorage(StaticFilesStorage):
     # instead of "images/logo.png"
     prefix = None
 
-    def __init__(self, location=None, base_url=None, file_permissions_mode=None,
-                 directory_permissions_mode=None, prefix=None):
+    def __init__(self, **kwargs):
 
-        self.prefix = prefix
-        super(ThemeStorage, self).__init__(
-            location=location,
-            base_url=base_url,
-            file_permissions_mode=file_permissions_mode,
-            directory_permissions_mode=directory_permissions_mode,
-        )
+        self.prefix = kwargs.pop('prefix', None)
+        super(ThemeMixin, self).__init__(**kwargs)
 
     def url(self, name):
         """
@@ -76,7 +70,7 @@ class ThemeStorage(StaticFilesStorage):
         if prefix and self.themed(name, prefix):
             name = os.path.join(prefix, name)
 
-        return super(ThemeStorage, self).url(name)
+        return super(ThemeMixin, self).url(name)
 
     def themed(self, name, theme):
         """
@@ -112,6 +106,10 @@ class ThemeStorage(StaticFilesStorage):
             return self.exists(os.path.join(theme, name))
 
 
+class ThemeStorage(ThemeMixin, StaticFilesStorage):
+    pass
+
+
 class ThemeCachedFilesMixin(CachedFilesMixin):
     """
     Comprehensive theme aware CachedFilesMixin.

openedx/core/lib/django_require/staticstorage.py

@@ -2,12 +2,12 @@
 :class:`~django_require.staticstorage.OptimizedCachedRequireJsStorage`
 """
 
-
-from openedx.core.storage import PipelineForgivingStorage
+from openedx.core.storage import PipelineForgivingMixin
+from pipeline.storage import PipelineCachedStorage
 from require.storage import OptimizedFilesMixin
 
 
-class OptimizedCachedRequireJsStorage(OptimizedFilesMixin, PipelineForgivingStorage):
+class OptimizedCachedRequireJsStorage(OptimizedFilesMixin, PipelineForgivingMixin, PipelineCachedStorage):
     """
     Custom storage backend that is used by Django-require.
     """

openedx/core/storage.py

@@ -7,20 +7,20 @@ from django.contrib.staticfiles.storage import StaticFilesStorage
 from django.core.files.storage import get_storage_class, FileSystemStorage
 from django.utils.deconstruct import deconstructible
 from django.utils.lru_cache import lru_cache
-from pipeline.storage import NonPackagingMixin, PipelineCachedStorage
+from pipeline.storage import NonPackagingMixin
 from require.storage import OptimizedFilesMixin
-from storages.backends.s3boto import S3BotoStorage
+from storages.backends.s3boto3 import S3Boto3Storage
 
-from openedx.core.djangoapps.theming.storage import ThemeCachedFilesMixin, ThemePipelineMixin, ThemeStorage
+from openedx.core.djangoapps.theming.storage import ThemeCachedFilesMixin, ThemePipelineMixin, ThemeMixin
 
 
-class PipelineForgivingStorage(PipelineCachedStorage):
+class PipelineForgivingMixin(object):
     """
     An extension of the django-pipeline storage backend which forgives missing files.
     """
     def hashed_name(self, name, content=None, **kwargs):
         try:
-            out = super(PipelineForgivingStorage, self).hashed_name(name, content, **kwargs)
+            out = super(PipelineForgivingMixin, self).hashed_name(name, content, **kwargs)
         except ValueError:
             # This means that a file could not be found, and normally this would
             # cause a fatal error, which seems rather excessive given that
@@ -30,7 +30,7 @@ class PipelineForgivingStorage(PipelineCachedStorage):
 
     def stored_name(self, name):
         try:
-            out = super(PipelineForgivingStorage, self).stored_name(name)
+            out = super(PipelineForgivingMixin, self).stored_name(name)
         except ValueError:
             # This means that a file could not be found, and normally this would
             # cause a fatal error, which seems rather excessive given that
@@ -39,25 +39,33 @@ class PipelineForgivingStorage(PipelineCachedStorage):
         return out
 
 
-class ProductionStorage(
-        PipelineForgivingStorage,
+class ProductionMixin(
+        PipelineForgivingMixin,
         OptimizedFilesMixin,
         ThemePipelineMixin,
         ThemeCachedFilesMixin,
-        ThemeStorage,
-        StaticFilesStorage
+        ThemeMixin,
 ):
     """
-    This class combines Django's StaticFilesStorage class with several mixins
-    that provide additional functionality. We use this version on production.
+    This class combines several mixins that provide additional functionality, and
+    can be applied over an existing Storage.
+    We use this version on production.
     """
     pass
 
 
+class ProductionStorage(ProductionMixin, StaticFilesStorage):
+    pass
+
+
+class ProductionS3Storage(ProductionMixin, S3Boto3Storage):  # pylint: disable=abstract-method
+    pass
+
+
 class DevelopmentStorage(
         NonPackagingMixin,
         ThemePipelineMixin,
-        ThemeStorage,
+        ThemeMixin,
         StaticFilesStorage
 ):
     """
@@ -68,29 +76,6 @@ class DevelopmentStorage(
     pass
 
 
-class S3ReportStorage(S3BotoStorage):  # pylint: disable=abstract-method
-    """
-    Storage for reports.
-    """
-    def __init__(self, acl=None, bucket=None, custom_domain=None, **settings):
-        """
-        init method for S3ReportStorage, Note that we have added an extra key-word
-        argument named "custom_domain" and this argument should not be passed to the superclass's init.
-
-        Args:
-            acl: content policy for the uploads i.e. private, public etc.
-            bucket: Name of S3 bucket to use for storing and/or retrieving content
-            custom_domain: custom domain to use for generating file urls
-            **settings: additional settings to be passed in to S3BotoStorage,
-
-        Returns:
-
-        """
-        if custom_domain:
-            self.custom_domain = custom_domain
-        super(S3ReportStorage, self).__init__(acl=acl, bucket=bucket, **settings)
-
-
 @deconstructible
 class OverwriteStorage(FileSystemStorage):
     """