ignore MAC meta files on import and also remove any such files from the course

STUD-1725

common/lib/xmodule/xmodule/contentstore/mongo.py

@@ -13,6 +13,7 @@ import os
 import json
 from bson.son import SON
 from opaque_keys.edx.keys import AssetKey
+from xmodule.modulestore.django import ASSET_IGNORE_REGEX
 
 
 class MongoContentStore(ContentStore):
@@ -170,6 +171,26 @@ class MongoContentStore(ContentStore):
             course_key, start=start, maxresults=maxresults, get_thumbnails=False, sort=sort
         )
 
+    def remove_redundant_content_for_courses(self):
+        """
+        Finds and removes all redundant files (Mac OS metadata files with filename ".DS_Store"
+        or filename starts with "._") for all courses
+        """
+        assets_to_delete = 0
+        for prefix in ['_id', 'content_son']:
+            query = SON([
+                ('{}.tag'.format(prefix), XASSET_LOCATION_TAG),
+                ('{}.category'.format(prefix), 'asset'),
+                ('{}.name'.format(prefix), {'$regex': ASSET_IGNORE_REGEX}),
+            ])
+            items = self.fs_files.find(query)
+            assets_to_delete = assets_to_delete + items.count()
+            for asset in items:
+                self.fs.delete(asset[prefix])
+
+            self.fs_files.remove(query)
+        return assets_to_delete
+
     def _get_all_content_for_course(self, course_key, get_thumbnails=False, start=0, maxresults=-1, sort=None):
         '''
         Returns a list of all static assets for a course. The return format is a list of asset data dictionary elements.

common/lib/xmodule/xmodule/modulestore/django.py

@@ -8,6 +8,8 @@ from __future__ import absolute_import
 
 from importlib import import_module
 from django.conf import settings
+if not settings.configured:
+    settings.configure()
 from django.core.cache import get_cache, InvalidCacheBackendError
 import django.utils
 
@@ -25,6 +27,8 @@ try:
 except ImportError:
     HAS_REQUEST_CACHE = False
 
+ASSET_IGNORE_REGEX = getattr(settings, "ASSET_IGNORE_REGEX", r"(^\._.*$)|(^\.DS_Store$)|(^.*~$)")
+
 
 def load_function(path):
     """

common/lib/xmodule/xmodule/modulestore/xml_importer.py

@@ -3,6 +3,7 @@ import os
 import mimetypes
 from path import path
 import json
+import re
 
 from .xml import XMLModuleStore, ImportSystem, ParentTracker
 from xblock.runtime import KvsFieldData, DictKeyValueStore
@@ -15,6 +16,7 @@ from xmodule.errortracker import make_error_tracker
 from .store_utilities import rewrite_nonportable_content_links
 import xblock
 from xmodule.tabs import CourseTabList
+from xmodule.modulestore.django import ASSET_IGNORE_REGEX
 from xmodule.modulestore.exceptions import InvalidLocationError
 from xmodule.modulestore.mongo.base import MongoRevisionKey
 from xmodule.modulestore import ModuleStoreEnum
@@ -49,7 +51,7 @@ def import_static_content(
 
             content_path = os.path.join(dirname, filename)
 
-            if filename.endswith('~'):
+            if re.match(ASSET_IGNORE_REGEX, filename):
                 if verbose:
                     log.debug('skipping static content %s...', content_path)
                 continue

common/lib/xmodule/xmodule/tests/test_import_static.py

@@ -21,3 +21,21 @@ class IgnoredFilesTestCase(unittest.TestCase):
         self.assertIn("example.txt", name_val)
         self.assertNotIn("example.txt~", name_val)
         self.assertIn("GREEN", name_val["example.txt"])
+
+    def test_ignore_dot_underscore_static_files(self):
+        """
+        Test for ignored Mac OS metadata files (filename starts with "._")
+        """
+        course_dir = DATA_DIR / "dot-underscore"
+        course_id = SlashSeparatedCourseKey("edX", "dot-underscore", "2014_Fall")
+        content_store = Mock()
+        content_store.generate_thumbnail.return_value = ("content", "location")
+        import_static_content(course_dir, content_store, course_id)
+        saved_static_content = [call[0][0] for call in content_store.save.call_args_list]
+        name_val = {sc.name: sc.data for sc in saved_static_content}
+        self.assertIn("example.txt", name_val)
+        self.assertIn(".example.txt", name_val)
+        self.assertNotIn("._example.txt", name_val)
+        self.assertNotIn(".DS_Store", name_val)
+        self.assertIn("GREEN", name_val["example.txt"])
+        self.assertIn("BLUE", name_val[".example.txt"])

common/test/data/dot-underscore/README.md

@@ -0,0 +1,6 @@
+IGNORE MAC METADATA FILES
+
+This course simulates an import of a course from a Mac OS that has some unnessary 
+metadata files (filename starts with ._) in assets (static/._example.txt). These 
+files do not belong with the content so skip them on import and also do a 
+cleanup for such already added assets.

common/test/data/dot-underscore/about/index.html

@@ -0,0 +1 @@
+GREEN

common/test/data/dot-underscore/course.xml

@@ -0,0 +1 @@
+<course org="edX" course="dot-underscore" slug="2014_Fall"/>

common/test/data/dot-underscore/course/2014_Fall.xml

@@ -0,0 +1,2 @@
+<course>
+</course>

common/test/data/dot-underscore/static/.example.txt

@@ -0,0 +1 @@
+BLUE

common/test/data/dot-underscore/static/example.txt

@@ -0,0 +1 @@
+GREEN