From 6fca1fd959a6ea5ed9118f7dbb499d3cd401b590 Mon Sep 17 00:00:00 2001
From: David Baumgold <david@davidbaumgold.com>
Date: Mon, 26 Jan 2015 13:29:31 -0500
Subject: [PATCH 01/10] Tender is only accessible via HTTP, not HTTPS

---
 cms/templates/activation_invalid.html | 2 +-
 cms/templates/error.html              | 2 +-
 cms/templates/index.html              | 8 ++++----
 cms/templates/login.html              | 2 +-
 cms/templates/widgets/footer.html     | 2 +-
 cms/templates/widgets/sock.html       | 4 ++--
 6 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/cms/templates/activation_invalid.html b/cms/templates/activation_invalid.html
index b077d64bd7..07265f3077 100644
--- a/cms/templates/activation_invalid.html
+++ b/cms/templates/activation_invalid.html
@@ -28,7 +28,7 @@
       <ul class="list-actions">
         % if settings.TENDER_DOMAIN:
         <li class="action-item">
-          <a href="//${settings.TENDER_DOMAIN}/discussion/new" class="action action-primary show-tender">
+          <a href="http://${settings.TENDER_DOMAIN}/discussion/new" class="action action-primary show-tender">
             ${_('Contact {platform_name} Support').format(platform_name=settings.PLATFORM_NAME)}
           </a>
         </li>
diff --git a/cms/templates/error.html b/cms/templates/error.html
index 01f19e1b32..09bb830fa6 100644
--- a/cms/templates/error.html
+++ b/cms/templates/error.html
@@ -14,7 +14,7 @@
 from django.conf import settings
 
 if settings.TENDER_DOMAIN:
-  help_link_start = '<a href="//{domain}/discussion/new" class="show-tender" title="{title}">'.format(
+  help_link_start = '<a href="http://{domain}/discussion/new" class="show-tender" title="{title}">'.format(
     domain=settings.TENDER_DOMAIN,
     title=_("Use our feedback tool, Tender, to share your feedback")
   ),
diff --git a/cms/templates/index.html b/cms/templates/index.html
index adf126dc4b..218c005458 100644
--- a/cms/templates/index.html
+++ b/cms/templates/index.html
@@ -102,7 +102,7 @@
           </div>
         </form>
       </div>
-      
+
         %if libraries_enabled:
         <div class="wrapper-create-element wrapper-create-library">
           <form class="form-create create-library library-info" id="create-library-form" name="create-library-form">
@@ -464,7 +464,7 @@
           </li>
           % if settings.TENDER_DOMAIN:
             <li class="action-item">
-              <a href="//${settings.TENDER_DOMAIN}/discussion/new" class="show-tender" title="${_("Use our feedback tool, Tender, to request help")}">
+              <a href="http://${settings.TENDER_DOMAIN}/discussion/new" class="show-tender" title="${_("Use our feedback tool, Tender, to request help")}">
                 ${_("Request help with {studio_name}").format(studio_name=settings.STUDIO_NAME)}
               </a>
             </li>
@@ -497,7 +497,7 @@
 from django.conf import settings
 
 if settings.TENDER_DOMAIN:
-  help_link_start = '<a href="//{domain}/discussion/new" class="show-tender">'.format(
+  help_link_start = '<a href="http://{domain}/discussion/new" class="show-tender">'.format(
     domain=settings.TENDER_DOMAIN,
   )
   help_link_end = '</a>'
@@ -543,7 +543,7 @@ else:
         <ol class='list-actions'>
           % if settings.TENDER_DOMAIN:
             <li class="action-item">
-              <a href="//${settings.TENDER_DOMAIN}/discussion/new" class="show-tender" title="${_("Use our feedback tool, Tender, to request help")}">
+              <a href="http://${settings.TENDER_DOMAIN}/discussion/new" class="show-tender" title="${_("Use our feedback tool, Tender, to request help")}">
                 ${_("Request help with your {studio_name} account").format(studio_name=settings.STUDIO_NAME)}
               </a>
             </li>
diff --git a/cms/templates/login.html b/cms/templates/login.html
index 07d2c8d820..c0ab813ae0 100644
--- a/cms/templates/login.html
+++ b/cms/templates/login.html
@@ -52,7 +52,7 @@ from django.utils.translation import ugettext as _
         <div class="bit">
           <h3 class="title-3">${_("Need Help?")}</h3>
           <p>${_('Having trouble with your account? Use {link_start}our support center{link_end} to look over self help steps, find solutions others have found to the same problem, or let us know of your issue.').format(
-              link_start='<a href="//{domain}" rel="external">'.format(domain=settings.TENDER_DOMAIN),
+              link_start='<a href="http://{domain}" rel="external">'.format(domain=settings.TENDER_DOMAIN),
               link_end='</a>',
             )}</p>
         </div>
diff --git a/cms/templates/widgets/footer.html b/cms/templates/widgets/footer.html
index dfb4192371..14d0fc63a0 100644
--- a/cms/templates/widgets/footer.html
+++ b/cms/templates/widgets/footer.html
@@ -17,7 +17,7 @@
         </li>
         % if settings.TENDER_DOMAIN and user.is_authenticated():
         <li class="nav-item nav-peripheral-feedback">
-          <a href="//${settings.TENDER_DOMAIN}/discussion/new" class="show-tender" title="${_('Use our feedback tool, Tender, to share your feedback')}">${_("Contact Us")}</a>
+          <a href="http://${settings.TENDER_DOMAIN}/discussion/new" class="show-tender" title="${_('Use our feedback tool, Tender, to share your feedback')}">${_("Contact Us")}</a>
         </li>
         % endif
       </ol>
diff --git a/cms/templates/widgets/sock.html b/cms/templates/widgets/sock.html
index 755d0ca922..40e1dcfce8 100644
--- a/cms/templates/widgets/sock.html
+++ b/cms/templates/widgets/sock.html
@@ -31,7 +31,7 @@
 
         % if settings.TENDER_DOMAIN:
           <li class="action-item">
-            <a href="//${settings.TENDER_DOMAIN}" rel="external" class="action action-primary">${_("{studio_name} Author Support").format(studio_name=settings.STUDIO_NAME)}</a>
+            <a href="http://${settings.TENDER_DOMAIN}" rel="external" class="action action-primary">${_("{studio_name} Author Support").format(studio_name=settings.STUDIO_NAME)}</a>
             <span class="tip">${_("{studio_name} Author Support").format(studio_name=settings.STUDIO_NAME)}</span>
           </li>
         % endif
@@ -54,7 +54,7 @@
         <ul class="list-actions">
           <li class="action-item">
 
-            <a href="//${settings.TENDER_DOMAIN}/discussion/new" class="action action-primary show-tender" title="${_("Use our feedback tool, Tender, to share your feedback")}"><i class="icon fa fa-comments"></i>${_("Contact Us")}</a>
+            <a href="http://${settings.TENDER_DOMAIN}/discussion/new" class="action action-primary show-tender" title="${_("Use our feedback tool, Tender, to share your feedback")}"><i class="icon fa fa-comments"></i>${_("Contact Us")}</a>
           </li>
         </ul>
       </div>

From 715979d36b5feb29b6590bc1b38e747244685a04 Mon Sep 17 00:00:00 2001
From: Greg Price <gprice@edx.org>
Date: Mon, 26 Jan 2015 17:26:11 -0500
Subject: [PATCH 02/10] Fix bugs in CSV download of video upload data

This fixes two bugs:
1. All statuses were listed as "Unknown"
2. Non-ASCII characters in a header column translation resulted in a 500
---
 cms/djangoapps/contentstore/views/tests/test_videos.py | 3 +++
 cms/djangoapps/contentstore/views/videos.py            | 8 ++++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/cms/djangoapps/contentstore/views/tests/test_videos.py b/cms/djangoapps/contentstore/views/tests/test_videos.py
index 8f7e606574..0539ac7fc3 100644
--- a/cms/djangoapps/contentstore/views/tests/test_videos.py
+++ b/cms/djangoapps/contentstore/views/tests/test_videos.py
@@ -347,7 +347,10 @@ class VideoUrlsCsvTestCase(VideoUploadTestMixin, CourseTestCase):
             actual_video_ids.append(response_video["Video ID"])
             original_video = self._get_previous_upload(response_video["Video ID"])
             self.assertEqual(response_video["Name"], original_video["client_video_id"])
+            self.assertEqual(response_video["Duration"], str(original_video["duration"]))
+            dateutil.parser.parse(response_video["Date Added"])
             self.assertEqual(response_video["Video ID"], original_video["edx_video_id"])
+            self.assertEqual(response_video["Status"], StatusDisplayStrings.get(original_video["status"]))
             for profile in expected_profiles:
                 response_profile_url = response_video["{} URL".format(profile)]
                 original_encoded_for_profile = next(
diff --git a/cms/djangoapps/contentstore/views/videos.py b/cms/djangoapps/contentstore/views/videos.py
index eb7a852381..3ab7a962ce 100644
--- a/cms/djangoapps/contentstore/views/videos.py
+++ b/cms/djangoapps/contentstore/views/videos.py
@@ -153,7 +153,7 @@ def video_encodings_download(request, course_key_string):
                 (duration_col, duration_val),
                 (added_col, video["created"].isoformat()),
                 (video_id_col, video["edx_video_id"]),
-                (status_col, StatusDisplayStrings.get(video["status"])),
+                (status_col, video["status"]),
             ] +
             [
                 (get_profile_header(encoded_video["profile"]), encoded_video["url"])
@@ -177,7 +177,11 @@ def video_encodings_download(request, course_key_string):
     )
     writer = csv.DictWriter(
         response,
-        [name_col, duration_col, added_col, video_id_col, status_col] + profile_cols,
+        [
+            col_name.encode("utf-8")
+            for col_name
+            in [name_col, duration_col, added_col, video_id_col, status_col] + profile_cols
+        ],
         dialect=csv.excel
     )
     writer.writeheader()

From c29f3af1865b7cfecd5d14c69e3399706a292dee Mon Sep 17 00:00:00 2001
From: Calen Pennington <cale@edx.org>
Date: Wed, 21 Jan 2015 16:28:44 -0500
Subject: [PATCH 03/10] Replace SplitMongo thread_cache usage with use of
 request_cache

---
 .../xmodule/modulestore/split_mongo/split.py  | 27 ++++++++-----------
 1 file changed, 11 insertions(+), 16 deletions(-)

diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
index 866d916d9c..d3f4560ab2 100644
--- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
+++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
@@ -53,7 +53,6 @@ Representation:
         definition. Acts as a pseudo-object identifier.
 """
 import copy
-import threading
 import datetime
 import hashlib
 import logging
@@ -618,10 +617,6 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
         self.db_connection = MongoConnection(**doc_store_config)
         self.db = self.db_connection.database
 
-        # Code review question: How should I expire entries?
-        # _add_cache could use a lru mechanism to control the cache size?
-        self.thread_cache = threading.local()
-
         if default_class is not None:
             module_path, __, class_name = default_class.rpartition('.')
             class_ = getattr(import_module(module_path), class_name)
@@ -732,10 +727,10 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
         Find the descriptor cache for this course if it exists
         :param course_version_guid:
         """
-        if not hasattr(self.thread_cache, 'course_cache'):
-            self.thread_cache.course_cache = {}
-        system = self.thread_cache.course_cache
-        return system.get(course_version_guid)
+        if self.request_cache is None:
+            return None
+
+        return self.request_cache.data.setdefault('course_cache', {}).get(course_version_guid)
 
     def _add_cache(self, course_version_guid, system):
         """
@@ -743,9 +738,8 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
         :param course_version_guid:
         :param system:
         """
-        if not hasattr(self.thread_cache, 'course_cache'):
-            self.thread_cache.course_cache = {}
-        self.thread_cache.course_cache[course_version_guid] = system
+        if self.request_cache is not None:
+            self.request_cache.data.setdefault('course_cache', {})[course_version_guid] = system
         return system
 
     def _clear_cache(self, course_version_guid=None):
@@ -753,15 +747,16 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
         Should only be used by testing or something which implements transactional boundary semantics.
         :param course_version_guid: if provided, clear only this entry
         """
+        if self.request_cache is None:
+            return
+
         if course_version_guid:
-            if not hasattr(self.thread_cache, 'course_cache'):
-                self.thread_cache.course_cache = {}
             try:
-                del self.thread_cache.course_cache[course_version_guid]
+                del self.request_cache.data.setdefault('course_cache', {})[course_version_guid]
             except KeyError:
                 pass
         else:
-            self.thread_cache.course_cache = {}
+            self.request_cache.data['course_cache'] = {}
 
     def _lookup_course(self, course_key):
         '''

From d9dc82ba417891f1114b1c2885375d95d5c659cb Mon Sep 17 00:00:00 2001
From: Calen Pennington <cale@edx.org>
Date: Mon, 26 Jan 2015 14:05:32 -0500
Subject: [PATCH 04/10] Stringify ObjectIds before attempting to make a
 filename out of them

---
 .../modulestore/split_mongo/caching_descriptor_system.py        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py
index 73a950b355..775f23d18a 100644
--- a/common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py
+++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py
@@ -53,7 +53,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin):
         if course_entry.course_key.course:
             root = modulestore.fs_root / course_entry.course_key.org / course_entry.course_key.course / course_entry.course_key.run
         else:
-            root = modulestore.fs_root / course_entry.structure['_id']
+            root = modulestore.fs_root / str(course_entry.structure['_id'])
         root.makedirs_p()  # create directory if it doesn't exist
 
         id_manager = SplitMongoIdManager(self)

From 68325f1c2ee988892f64bd627acb0601444e58ea Mon Sep 17 00:00:00 2001
From: John Eskew <jeskew@edx.org>
Date: Mon, 26 Jan 2015 14:43:11 -0500
Subject: [PATCH 05/10] Add BlockData class to wrap all block data loaded from
 the Split modulestore, in order to separate serializable block data from
 out-of-band, non-saved, temporary metadata.

Eliminate the deepcopy causing performance problems with import/export.

https://openedx.atlassian.net/browse/PLAT-416
---
 .../modulestore/split_mongo/__init__.py       | 75 +++++++++++++++++++
 .../split_mongo/caching_descriptor_system.py  | 31 ++++----
 .../split_mongo/mongo_connection.py           | 13 ++--
 .../xmodule/modulestore/split_mongo/split.py  | 18 ++---
 4 files changed, 108 insertions(+), 29 deletions(-)

diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/__init__.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/__init__.py
index 06ae542506..6fd3e5a7c5 100644
--- a/common/lib/xmodule/xmodule/modulestore/split_mongo/__init__.py
+++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/__init__.py
@@ -21,3 +21,78 @@ class BlockKey(namedtuple('BlockKey', 'type id')):
 
 
 CourseEnvelope = namedtuple('CourseEnvelope', 'course_key structure')
+
+
+class BlockData(object):
+    """
+    Wrap the block data in an object instead of using a straight Python dictionary.
+    Allows the storing of meta-information about a structure that doesn't persist along with
+    the structure itself.
+    """
+    @contract(block_dict=dict)
+    def __init__(self, block_dict={}):  # pylint: disable=dangerous-default-value
+        # Has the definition been loaded?
+        self.definition_loaded = False
+        self.from_storable(block_dict)
+
+    def to_storable(self):
+        """
+        Serialize to a Mongo-storable format.
+        """
+        return {
+            'fields': self.fields,
+            'block_type': self.block_type,
+            'definition': self.definition,
+            'defaults': self.defaults,
+            'edit_info': self.edit_info
+        }
+
+    @contract(stored=dict)
+    def from_storable(self, stored):
+        """
+        De-serialize from Mongo-storable format to an object.
+        """
+        self.fields = stored.get('fields', {})
+        self.block_type = stored.get('block_type', None)
+        self.definition = stored.get('definition', None)
+        self.defaults = stored.get('defaults', {})
+        self.edit_info = stored.get('edit_info', {})
+
+    def get(self, key, *args, **kwargs):
+        """
+        Dict-like 'get' method. Raises AttributeError if requesting non-existent attribute and no default.
+        """
+        if len(args) > 0:
+            return getattr(self, key, args[0])
+        elif 'default' in kwargs:
+            return getattr(self, key, kwargs['default'])
+        else:
+            return getattr(self, key)
+
+    def __getitem__(self, key):
+        """
+        Dict-like '__getitem__'.
+        """
+        if not hasattr(self, key):
+            raise KeyError
+        else:
+            return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+    def __delitem__(self, key):
+        delattr(self, key)
+
+    def __iter__(self):
+        return self.__dict__.iterkeys()
+
+    def setdefault(self, key, default=None):
+        """
+        Dict-like 'setdefault'.
+        """
+        try:
+            return getattr(self, key)
+        except AttributeError:
+            setattr(self, key, default)
+            return default
diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py
index 775f23d18a..604be77601 100644
--- a/common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py
+++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py
@@ -117,10 +117,10 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin):
         if cached_module:
             return cached_module
 
-        json_data = self.get_module_data(block_key, course_key)
+        block_data = self.get_module_data(block_key, course_key)
 
-        class_ = self.load_block_type(json_data.get('block_type'))
-        block = self.xblock_from_json(class_, course_key, block_key, json_data, course_entry_override, **kwargs)
+        class_ = self.load_block_type(block_data.get('block_type'))
+        block = self.xblock_from_json(class_, course_key, block_key, block_data, course_entry_override, **kwargs)
         self.modulestore.cache_block(course_key, version_guid, block_key, block)
         return block
 
@@ -154,26 +154,27 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin):
     # pointing to the same structure, the access is likely to be chunky enough that the last known container
     # is the intended one when not given a course_entry_override; thus, the caching of the last branch/course id.
     @contract(block_key="BlockKey | None")
-    def xblock_from_json(
-        self, class_, course_key, block_key, json_data, course_entry_override=None, **kwargs
-    ):
+    def xblock_from_json(self, class_, course_key, block_key, block_data, course_entry_override=None, **kwargs):
+        """
+        Load and return block info.
+        """
         if course_entry_override is None:
             course_entry_override = self.course_entry
         else:
             # most recent retrieval is most likely the right one for next caller (see comment above fn)
             self.course_entry = CourseEnvelope(course_entry_override.course_key, self.course_entry.structure)
 
-        definition_id = json_data.get('definition')
+        definition_id = block_data.get('definition')
 
         # If no usage id is provided, generate an in-memory id
         if block_key is None:
-            block_key = BlockKey(json_data['block_type'], LocalId())
+            block_key = BlockKey(block_data['block_type'], LocalId())
 
         convert_fields = lambda field: self.modulestore.convert_references_to_keys(
             course_key, class_, field, self.course_entry.structure['blocks'],
         )
 
-        if definition_id is not None and not json_data.get('definition_loaded', False):
+        if definition_id is not None and not block_data['definition_loaded']:
             definition_loader = DefinitionLazyLoader(
                 self.modulestore,
                 course_key,
@@ -194,8 +195,8 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin):
             block_id=block_key.id,
         )
 
-        converted_fields = convert_fields(json_data.get('fields', {}))
-        converted_defaults = convert_fields(json_data.get('defaults', {}))
+        converted_fields = convert_fields(block_data.get('fields', {}))
+        converted_defaults = convert_fields(block_data.get('defaults', {}))
         if block_key in self._parent_map:
             parent_key = self._parent_map[block_key]
             parent = course_key.make_usage_key(parent_key.type, parent_key.id)
@@ -223,7 +224,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin):
         except Exception:
             log.warning("Failed to load descriptor", exc_info=True)
             return ErrorDescriptor.from_json(
-                json_data,
+                block_data,
                 self,
                 course_entry_override.course_key.make_usage_key(
                     block_type='error',
@@ -232,9 +233,9 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin):
                 error_msg=exc_info_to_str(sys.exc_info())
             )
 
-        edit_info = json_data.get('edit_info', {})
-        module._edited_by = edit_info.get('edited_by')
-        module._edited_on = edit_info.get('edited_on')
+        edit_info = block_data.get('edit_info', {})
+        module._edited_by = edit_info.get('edited_by')  # pylint: disable=protected-access
+        module._edited_on = edit_info.get('edited_on')  # pylint: disable=protected-access
         module.previous_version = edit_info.get('previous_version')
         module.update_version = edit_info.get('update_version')
         module.source_version = edit_info.get('source_version', None)
diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py
index 9535a5232d..382baa3f91 100644
--- a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py
+++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py
@@ -8,13 +8,16 @@ import pymongo
 # Import this just to export it
 from pymongo.errors import DuplicateKeyError  # pylint: disable=unused-import
 
-from contracts import check
+from contracts import check, new_contract
 from xmodule.exceptions import HeartbeatFailure
-from xmodule.modulestore.split_mongo import BlockKey
+from xmodule.modulestore.split_mongo import BlockKey, BlockData
 import datetime
 import pytz
 
 
+new_contract('BlockData', BlockData)
+
+
 def structure_from_mongo(structure):
     """
     Converts the 'blocks' key from a list [block_data] to a map
@@ -34,7 +37,7 @@ def structure_from_mongo(structure):
     for block in structure['blocks']:
         if 'children' in block['fields']:
             block['fields']['children'] = [BlockKey(*child) for child in block['fields']['children']]
-        new_blocks[BlockKey(block['block_type'], block.pop('block_id'))] = block
+        new_blocks[BlockKey(block['block_type'], block.pop('block_id'))] = BlockData(block)
     structure['blocks'] = new_blocks
 
     return structure
@@ -49,7 +52,7 @@ def structure_to_mongo(structure):
         directly into mongo.
     """
     check('BlockKey', structure['root'])
-    check('dict(BlockKey: dict)', structure['blocks'])
+    check('dict(BlockKey: BlockData)', structure['blocks'])
     for block in structure['blocks'].itervalues():
         if 'children' in block['fields']:
             check('list(BlockKey)', block['fields']['children'])
@@ -58,7 +61,7 @@ def structure_to_mongo(structure):
     new_structure['blocks'] = []
 
     for block_key, block in structure['blocks'].iteritems():
-        new_block = dict(block)
+        new_block = dict(block.to_storable())
         new_block.setdefault('block_type', block_key.type)
         new_block['block_id'] = block_key.id
         new_structure['blocks'].append(new_block)
diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
index d3f4560ab2..1162125300 100644
--- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
+++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
@@ -78,7 +78,7 @@ from xmodule.modulestore import (
 from ..exceptions import ItemNotFoundError
 from .caching_descriptor_system import CachingDescriptorSystem
 from xmodule.modulestore.split_mongo.mongo_connection import MongoConnection, DuplicateKeyError
-from xmodule.modulestore.split_mongo import BlockKey, CourseEnvelope
+from xmodule.modulestore.split_mongo import BlockKey, CourseEnvelope, BlockData
 from xmodule.error_module import ErrorDescriptor
 from collections import defaultdict
 from types import NoneType
@@ -676,8 +676,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
             new_module_data = {}
             for block_id in base_block_ids:
                 new_module_data = self.descendants(
-                    # copy or our changes like setting 'definition_loaded' will affect the active bulk operation data
-                    copy.deepcopy(system.course_entry.structure['blocks']),
+                    system.course_entry.structure['blocks'],
                     block_id,
                     depth,
                     new_module_data
@@ -2332,7 +2331,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
 
             return result
 
-    @contract(block_key=BlockKey, blocks='dict(BlockKey: dict)')
+    @contract(block_key=BlockKey, blocks='dict(BlockKey: BlockData)')
     def _remove_subtree(self, block_key, blocks):
         """
         Remove the subtree rooted at block_key
@@ -2912,6 +2911,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
                 self._delete_if_true_orphan(BlockKey(*child), structure)
             del structure['blocks'][orphan]
 
+    @contract(returns=BlockData)
     def _new_block(self, user_id, category, block_fields, definition_id, new_id, raw=False, block_defaults=None):
         """
         Create the core document structure for a block.
@@ -2936,20 +2936,20 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
         }
         if block_defaults:
             document['defaults'] = block_defaults
-        return document
+        return BlockData(document)
 
-    @contract(block_key=BlockKey)
+    @contract(block_key=BlockKey, returns='BlockData | None')
     def _get_block_from_structure(self, structure, block_key):
         """
-        Encodes the block id before retrieving it from the structure to ensure it can
+        Encodes the block key before retrieving it from the structure to ensure it can
         be a json dict key.
         """
         return structure['blocks'].get(block_key)
 
-    @contract(block_key=BlockKey)
+    @contract(block_key=BlockKey, content=BlockData)
     def _update_block_in_structure(self, structure, block_key, content):
         """
-        Encodes the block id before accessing it in the structure to ensure it can
+        Encodes the block key before accessing it in the structure to ensure it can
         be a json dict key.
         """
         structure['blocks'][block_key] = content

From e20b77b4f87375689dd81d366157690dcbe1727d Mon Sep 17 00:00:00 2001
From: Adam Palay <adam@edx.org>
Date: Wed, 28 Jan 2015 10:21:01 -0500
Subject: [PATCH 06/10] Revert "Fix tolerance broken"

This reverts commit 66b949b745b45b3a25ff32fdd23423ecaaff712c.
---
 common/lib/capa/capa/tests/test_util.py |  9 ---------
 common/lib/capa/capa/util.py            | 17 +----------------
 2 files changed, 1 insertion(+), 25 deletions(-)

diff --git a/common/lib/capa/capa/tests/test_util.py b/common/lib/capa/capa/tests/test_util.py
index ecb6ea02c9..f6b76e1c9e 100644
--- a/common/lib/capa/capa/tests/test_util.py
+++ b/common/lib/capa/capa/tests/test_util.py
@@ -81,15 +81,6 @@ class UtilTest(unittest.TestCase):
         self.assertFalse(result)
         result = compare_with_tolerance(infinity, infinity, '1.0', False)
         self.assertTrue(result)
-        # Test absolute tolerance for smaller values
-        result = compare_with_tolerance(100.01, 100.0, 0.01, False)
-        self.assertTrue(result)
-        result = compare_with_tolerance(100.001, 100.0, 0.001, False)
-        self.assertTrue(result)
-        result = compare_with_tolerance(100.01, 100.0, '0.01%', False)
-        self.assertTrue(result)
-        result = compare_with_tolerance(100.002, 100.0, 0.001, False)
-        self.assertFalse(result)
 
     def test_sanitize_html(self):
         """
diff --git a/common/lib/capa/capa/util.py b/common/lib/capa/capa/util.py
index ccbebd2a4b..e9197ad19e 100644
--- a/common/lib/capa/capa/util.py
+++ b/common/lib/capa/capa/util.py
@@ -5,7 +5,6 @@ import bleach
 
 from calc import evaluator
 from cmath import isinf
-from decimal import Decimal
 #-----------------------------------------------------------------------------
 #
 # Utility functions used in CAPA responsetypes
@@ -68,21 +67,7 @@ def compare_with_tolerance(student_complex, instructor_complex, tolerance=defaul
     else:
         # v1 and v2 are, in general, complex numbers:
         # there are some notes about backward compatibility issue: see responsetypes.get_staff_ans()).
-        decimal_places = None
-        # count the "decimal_places" for "student_complex". e.g, for
-        # "student_complex" value "152.3667" the "decimal_places" will be
-        # 4 as there are 4 digits "3667" after decimal
-        if isinstance(student_complex, float):
-            decimal_places = Decimal(str(student_complex)).as_tuple().exponent * -1   # pylint: disable=E1103
-
-        abs_value = abs(student_complex - instructor_complex)
-
-        # decimal_places could be NaN in some cases
-        if decimal_places and isinstance(decimal_places, int):
-            # abs_value contains 17 digits exponent value so
-            # round it up to "decimal_places"
-            abs_value = round(abs_value, decimal_places)
-        return abs_value <= tolerance
+        return abs(student_complex - instructor_complex) <= tolerance
 
 
 def contextualize_text(text, context):  # private

From 513f8cefd877b7ec8823de7add33f9b3027bc07e Mon Sep 17 00:00:00 2001
From: jsa <jsa@edx.org>
Date: Wed, 28 Jan 2015 03:30:25 -0500
Subject: [PATCH 07/10] reuse instances of OSFS in old mongo

---
 common/lib/xmodule/xmodule/modulestore/mongo/base.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/base.py b/common/lib/xmodule/xmodule/modulestore/mongo/base.py
index 9c791f472d..eaf808d18d 100644
--- a/common/lib/xmodule/xmodule/modulestore/mongo/base.py
+++ b/common/lib/xmodule/xmodule/modulestore/mongo/base.py
@@ -71,6 +71,9 @@ BLOCK_TYPES_WITH_CHILDREN = list(set(
 # Allow us to call _from_deprecated_(son|string) throughout the file
 # pylint: disable=protected-access
 
+# at module level, cache one instance of OSFS per filesystem root.
+_OSFS_INSTANCE = {}
+
 
 class MongoRevisionKey(object):
     """
@@ -836,10 +839,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
         location = Location._from_deprecated_son(item['location'], course_key.run)
         data_dir = getattr(item, 'data_dir', location.course)
         root = self.fs_root / data_dir
-
-        root.makedirs_p()  # create directory if it doesn't exist
-
-        resource_fs = OSFS(root)
+        resource_fs = _OSFS_INSTANCE.setdefault(root, OSFS(root, create=True))
 
         cached_metadata = {}
         if apply_cached_metadata:

From a1ffcc52594b1e7240501aab0ea145f1da3edb5a Mon Sep 17 00:00:00 2001
From: Syed Hassan Raza <shrhawk88@gmail.com>
Date: Tue, 13 Jan 2015 17:30:17 +0500
Subject: [PATCH 08/10] Disable referrer (TNL-1144)

---
 common/static/js/vendor/noreferrer.js | 10 ++++++++++
 lms/templates/footer-edx-new.html     | 14 ++++++++------
 2 files changed, 18 insertions(+), 6 deletions(-)
 create mode 100644 common/static/js/vendor/noreferrer.js

diff --git a/common/static/js/vendor/noreferrer.js b/common/static/js/vendor/noreferrer.js
new file mode 100644
index 0000000000..d853a0bfb4
--- /dev/null
+++ b/common/static/js/vendor/noreferrer.js
@@ -0,0 +1,10 @@
+/*
+ jquery.noreferrer.js, version 0.1.3
+ https://github.com/knu/noreferrer
+
+ Copyright (c) 2011 Akinori MUSHA
+ Licensed under the 2-clause BSD license.
+*/
+(function(){(function(){$.browser.webkit||$.event.add(window,"load",function(){$("a[href][rel~=noreferrer], area[href][rel~=noreferrer]").each(function(){var b,e,c,g,d,f,h;b=this;c=b.href;$.browser.opera?(b.href="http://www.google.com/url?q="+encodeURIComponent(c),b.title||(b.title="Go to "+c)):(d=!1,g=function(){b.href="javascript:void(0)"},f=function(){b.href=c},$(b).bind("mouseout mouseover focus blur",f).mousedown(function(a){a.which===2&&(d=!0)}).blur(function(){d=!1}).mouseup(function(a){if(!(a.which===
+2&&d))return!0;g();d=!1;setTimeout(function(){alert("Middle clicking on this link is disabled");f()},500);return!1}),e="<html><head><meta http-equiv='Refresh' content='0; URL="+$("<p/>").text(c).html()+"' /></head><body></body></html>",$.browser.msie?$(b).click(function(){var a;switch(a=this.target||"_self"){case "_self":case window.name:a=window;break;default:a=window.open(null,a)}a=a.document;a.clear();a.write(e);a.close();return!1}):(h="data:text/html;charset=utf-8,"+
+encodeURIComponent(e),$(b).click(function(){this.href=h;return!0})))})})})()}).call(this);
\ No newline at end of file
diff --git a/lms/templates/footer-edx-new.html b/lms/templates/footer-edx-new.html
index 9f207bc276..b0a9d6f091 100644
--- a/lms/templates/footer-edx-new.html
+++ b/lms/templates/footer-edx-new.html
@@ -2,7 +2,6 @@
 <%! from django.core.urlresolvers import reverse %>
 <%! from django.utils.translation import ugettext as _ %>
 <%namespace name='static' file='static_content.html'/>
-
 ## WARNING: These files are specific to edx.org and are not used in installations outside of that domain. Open edX users will want to use the file "footer.html" for any changes or overrides.
 <div class="wrapper wrapper-footer edx-footer edx-footer-new">
   <footer id="footer-global" class="footer-global" role="contentinfo" aria-label="Footer">
@@ -65,31 +64,31 @@
       <div class="footer-follow-links">
         ## Translators: This is the website name of www.twitter.com. Please
         ## translate this the way that Twitter advertises in your language.
-        <a href="${settings.PLATFORM_TWITTER_URL}" title="${_("Twitter")}">
+        <a href="${settings.PLATFORM_TWITTER_URL}" title="${_("Twitter")}" rel="noreferrer">
           <i class="icon fa fa-twitter element-invisible"></i>
           <span class="copy">${_("Twitter")}</span>
         </a>
         ## Translators: This is the website name of www.facebook.com. Please
         ## translate this the way that Facebook advertises in your language.
-        <a href="${settings.PLATFORM_FACEBOOK_ACCOUNT}" title="${_("Facebook")}">
+        <a href="${settings.PLATFORM_FACEBOOK_ACCOUNT}" title="${_("Facebook")}" rel="noreferrer">
           <i class="icon fa fa-facebook element-invisible"></i>
           <span class="copy">${_("Facebook")}</span>
         </a>
         ## Translators: This is the website name of www.meetup.com. Please
         ## translate this the way that Meetup advertises in your language.
-        <a href="${settings.PLATFORM_MEETUP_URL}" title="${_("Meetup")}">
+        <a href="${settings.PLATFORM_MEETUP_URL}" title="${_("Meetup")}" rel="noreferrer">
           <i class="icon fa fa-calendar element-invisible"></i>
           <span class="copy">${_("Meetup")}</span>
         </a>
         ## Translators: This is the website name of www.linked.com. Please
         ## translate this the way that LinkedIn advertises in your language.
-        <a href="${settings.PLATFORM_LINKEDIN_URL}" title="${_("LinkedIn")}">
+        <a href="${settings.PLATFORM_LINKEDIN_URL}" title="${_("LinkedIn")}" rel="noreferrer">
           <i class="icon fa fa-linkedin element-invisible"></i>
           <span class="copy">${_("LinkedIn")}</span>
         </a>
         ## Translators: This is the website name of plus.google.com. Please
         ## translate this the way that Google+ advertises in your language.
-        <a href="${settings.PLATFORM_GOOGLE_PLUS_URL}" title="${_("Google+")}">
+        <a href="${settings.PLATFORM_GOOGLE_PLUS_URL}" title="${_("Google+")}" rel="noreferrer">
           <i class="icon fa fa-google-plus element-invisible"></i>
           <span class="copy">${_("Google+")}</span>
         </a>
@@ -113,3 +112,6 @@
     </div>
   </footer>
 </div>
+
+<script type="text/javascript" src="/static/js/vendor/noreferrer.js" charset="utf-8"></script>
+

From c71a1833161d3e2c456b00f841d7b72e7d8998d4 Mon Sep 17 00:00:00 2001
From: Diana Huang <dkh@edx.org>
Date: Thu, 29 Jan 2015 11:41:18 -0500
Subject: [PATCH 09/10] Update embargo copy.

ECOM-963

update copy in tests
---
 common/djangoapps/embargo/tests/test_middleware.py | 2 +-
 lms/templates/static_templates/embargo.html        | 6 ++----
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/common/djangoapps/embargo/tests/test_middleware.py b/common/djangoapps/embargo/tests/test_middleware.py
index a02f4d31b8..10d03f0360 100644
--- a/common/djangoapps/embargo/tests/test_middleware.py
+++ b/common/djangoapps/embargo/tests/test_middleware.py
@@ -54,7 +54,7 @@ class EmbargoMiddlewareTests(ModuleStoreTestCase):
         CourseEnrollment.enroll(self.user, self.regular_course.id)
         CourseEnrollment.enroll(self.user, self.embargo_course.id)
         # Text from lms/templates/static_templates/embargo.html
-        self.embargo_text = "Unfortunately, at this time edX must comply with export controls, and we cannot allow you to access this particular course."
+        self.embargo_text = "Unfortunately, at this time edX must comply with export controls, and we cannot allow you to access this course."
 
         self.patcher = mock.patch.object(pygeoip.GeoIP, 'country_code_by_addr', self.mock_country_code_by_addr)
         self.patcher.start()
diff --git a/lms/templates/static_templates/embargo.html b/lms/templates/static_templates/embargo.html
index 016864b7cf..f10ba08e6c 100644
--- a/lms/templates/static_templates/embargo.html
+++ b/lms/templates/static_templates/embargo.html
@@ -6,11 +6,9 @@
 <section class="outside-app">
 <p>
 ${_("Our system indicates that you are trying to access this {platform_name} "
-    "course from a country currently subject to U.S. economic and trade "
+    "course from a country or region currently subject to U.S. economic and trade "
     "sanctions. Unfortunately, at this time {platform_name} must comply with "
-    "export controls, and we cannot allow you to access this particular course. "
-    "Feel free to browse our catalogue to find other courses you may be "
-    "interested in taking."
+    "export controls, and we cannot allow you to access this course."
   ).format(
     platform_name=settings.PLATFORM_NAME,
   )}

From d4d785bfd794986867eab2bac150530803632b30 Mon Sep 17 00:00:00 2001
From: Adam Palay <adam@edx.org>
Date: Thu, 29 Jan 2015 14:18:38 -0500
Subject: [PATCH 10/10] update codejail (TNL-1248) (optionally use a proxy
 process to spawn subprocesses)

---
 requirements/edx/github.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements/edx/github.txt b/requirements/edx/github.txt
index daadae99dc..9029c0f6a1 100644
--- a/requirements/edx/github.txt
+++ b/requirements/edx/github.txt
@@ -22,7 +22,7 @@ git+https://github.com/mitocw/django-cas.git@60a5b8e5a62e63e0d5d224a87f0b489201a
 
 # Our libraries:
 -e git+https://github.com/edx/XBlock.git@9c634481dfc85a17dcb3351ca232d7098a38e10e#egg=XBlock
--e git+https://github.com/edx/codejail.git@2b095e820ff752a108653bb39d518b122f7154db#egg=codejail
+-e git+https://github.com/edx/codejail.git@6b17c33a89bef0ac510926b1d7fea2748b73aadd#egg=codejail
 -e git+https://github.com/edx/js-test-tool.git@v0.1.6#egg=js_test_tool
 -e git+https://github.com/edx/event-tracking.git@0.1.0#egg=event-tracking
 -e git+https://github.com/edx/bok-choy.git@9fd05383dc434e0275c9874fd50ad14e519448bc#egg=bok_choy