From c21cf66569d13f9ac32e78587081c2c061dcf1c0 Mon Sep 17 00:00:00 2001
From: John Eskew <jeskew@edx.org>
Date: Tue, 14 Apr 2015 17:38:58 -0400
Subject: [PATCH 1/7] Only include published items in course export.

---
 .../xmodule/modulestore/xml_exporter.py       | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/common/lib/xmodule/xmodule/modulestore/xml_exporter.py b/common/lib/xmodule/xmodule/modulestore/xml_exporter.py
index a3d95a9b78..bbc7d5c995 100644
--- a/common/lib/xmodule/xmodule/modulestore/xml_exporter.py
+++ b/common/lib/xmodule/xmodule/modulestore/xml_exporter.py
@@ -153,20 +153,15 @@ class ExportManager(object):
         Perform the export given the parameters handed to this class at init.
         """
         with self.modulestore.bulk_operations(self.courselike_key):
-            # depth = None: Traverses down the entire course structure.
-            # lazy = False: Loads and caches all block definitions during traversal for fast access later
-            #               -and- to eliminate many round-trips to read individual definitions.
-            # Why these parameters? Because a course export needs to access all the course block information
-            # eventually. Accessing it all now at the beginning increases performance of the export.
-            fsm = OSFS(self.root_dir)
-            courselike = self.get_courselike()
-            export_fs = courselike.runtime.export_fs = fsm.makeopendir(self.target_dir)
-            root_courselike_dir = self.root_dir + '/' + self.target_dir
 
+            fsm = OSFS(self.root_dir)
             root = lxml.etree.Element('unknown')  # pylint: disable=no-member
 
             # export only the published content
             with self.modulestore.branch_setting(ModuleStoreEnum.Branch.published_only, self.courselike_key):
+                courselike = self.get_courselike()
+                export_fs = courselike.runtime.export_fs = fsm.makeopendir(self.target_dir)
+
                 # change all of the references inside the course to use the xml expected key type w/o version & branch
                 xml_centric_courselike_key = self.get_key()
                 adapt_references(courselike, xml_centric_courselike_key, export_fs)
@@ -176,6 +171,7 @@ class ExportManager(object):
             self.process_root(root, export_fs)
 
             # Process extra items-- drafts, assets, etc
+            root_courselike_dir = self.root_dir + '/' + self.target_dir
             self.process_extra(root, courselike, root_courselike_dir, xml_centric_courselike_key, export_fs)
 
             # Any last pass adjustments
@@ -192,6 +188,11 @@ class CourseExportManager(ExportManager):
         )
 
     def get_courselike(self):
+        # depth = None: Traverses down the entire course structure.
+        # lazy = False: Loads and caches all block definitions during traversal for fast access later
+        #               -and- to eliminate many round-trips to read individual definitions.
+        # Why these parameters? Because a course export needs to access all the course block information
+        # eventually. Accessing it all now at the beginning increases performance of the export.
         return self.modulestore.get_course(self.courselike_key, depth=None, lazy=False)
 
     def process_root(self, root, export_fs):

From 45ad00ae38bcba0d52c967af8014467358325a1b Mon Sep 17 00:00:00 2001
From: John Eskew <jeskew@edx.org>
Date: Tue, 14 Apr 2015 13:32:05 -0400
Subject: [PATCH 2/7] Add tests for more draft/published import/export. Include
 tests that mimic behavior in PLAT-452.

---
 .../tests/test_mixed_modulestore.py           | 358 +++++++++++++++++-
 1 file changed, 348 insertions(+), 10 deletions(-)

diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py
index a82711ccfb..548440cca2 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py
@@ -10,15 +10,19 @@ import itertools
 import mimetypes
 from unittest import skip
 from uuid import uuid4
+from contextlib import contextmanager
 
 # Mixed modulestore depends on django, so we'll manually configure some django settings
 # before importing the module
 # TODO remove this import and the configuration -- xmodule should not depend on django!
 from django.conf import settings
+# This import breaks this test file when run separately. Needs to be fixed! (PLAT-449)
 from mock_django import mock_signal_receiver
 from nose.plugins.attrib import attr
 import pymongo
 from pytz import UTC
+from shutil import rmtree
+from tempfile import mkdtemp
 
 from xmodule.x_module import XModuleMixin
 from xmodule.modulestore.edit_info import EditInfoMixin
@@ -27,6 +31,7 @@ from xmodule.modulestore.tests.test_cross_modulestore_import_export import Mongo
 from xmodule.contentstore.content import StaticContent
 from opaque_keys.edx.keys import CourseKey
 from xmodule.modulestore.xml_importer import import_course_from_xml
+from xmodule.modulestore.xml_exporter import export_course_to_xml
 from xmodule.modulestore.django import SignalHandler
 
 if not settings.configured:
@@ -49,9 +54,7 @@ from xmodule.tests import DATA_DIR, CourseComparisonTest
 log = logging.getLogger(__name__)
 
 
-@ddt.ddt
-@attr('mongo')
-class TestMixedModuleStore(CourseComparisonTest):
+class CommonMixedModuleStoreSetup(CourseComparisonTest):
     """
     Quasi-superclass which tests Location based apps against both split and mongo dbs (Locator and
     Location-based dbs)
@@ -126,7 +129,7 @@ class TestMixedModuleStore(CourseComparisonTest):
         """
         Set up the database for testing
         """
-        super(TestMixedModuleStore, self).setUp()
+        super(CommonMixedModuleStoreSetup, self).setUp()
 
         self.exclude_field(None, 'wiki_slug')
         self.exclude_field(None, 'xml_attributes')
@@ -241,6 +244,12 @@ class TestMixedModuleStore(CourseComparisonTest):
         """
         return self.course_locations[string].course_key
 
+    def _has_changes(self, location):
+        """
+        Helper function that loads the item before calling has_changes
+        """
+        return self.store.has_changes(self.store.get_item(location))
+
     # pylint: disable=dangerous-default-value
     def _initialize_mixed(self, mappings=MAPPINGS, contentstore=None):
         """
@@ -285,6 +294,13 @@ class TestMixedModuleStore(CourseComparisonTest):
         )
         self._create_course(self.course_locations[self.MONGO_COURSEID].course_key)
 
+
+@ddt.ddt
+@attr('mongo')
+class TestMixedModuleStore(CommonMixedModuleStoreSetup):
+    """
+    Tests of the MixedModulestore interface methods.
+    """
     @ddt.data('draft', 'split')
     def test_get_modulestore_type(self, default_ms):
         """
@@ -506,12 +522,6 @@ class TestMixedModuleStore(CourseComparisonTest):
         component = self.store.publish(component.location, self.user_id)
         self.assertFalse(self.store.has_changes(component))
 
-    def _has_changes(self, location):
-        """
-        Helper function that loads the item before calling has_changes
-        """
-        return self.store.has_changes(self.store.get_item(location))
-
     def setup_has_changes(self, default_ms):
         """
         Common set up for has_changes tests below.
@@ -2244,3 +2254,331 @@ class TestMixedModuleStore(CourseComparisonTest):
                         self.store.update_item(unit, self.user_id)
                         self.assertEqual(receiver.call_count, 0)
                     self.assertEqual(receiver.call_count, 0)
+
+
+@ddt.ddt
+@attr('mongo')
+class TestPublishOverExportImport(CommonMixedModuleStoreSetup):
+    """
+    Tests which publish (or don't publish) items - and then export/import the course,
+    checking the state of the imported items.
+    """
+    def setUp(self):
+        """
+        Set up the database for testing
+        """
+        super(TestPublishOverExportImport, self).setUp()
+
+        self.user_id = ModuleStoreEnum.UserID.test
+        self.export_dir = mkdtemp()
+        self.addCleanup(rmtree, self.export_dir, ignore_errors=True)
+
+    def _export_import_course_round_trip(self, modulestore, contentstore, source_course_key, export_dir):
+        """
+        Export the course from a modulestore and then re-import the course.
+        """
+        top_level_export_dir = 'exported_source_course'
+        export_course_to_xml(
+            modulestore,
+            contentstore,
+            source_course_key,
+            export_dir,
+            top_level_export_dir,
+        )
+
+        import_course_from_xml(
+            modulestore,
+            'test_user',
+            export_dir,
+            source_dirs=[top_level_export_dir],
+            static_content_store=contentstore,
+            target_id=source_course_key,
+            create_if_not_present=True,
+            raise_on_failure=True,
+        )
+
+    @contextmanager
+    def _build_store(self, default_ms):
+        """
+        Perform the modulestore-building and course creation steps for a mixed modulestore test.
+        """
+        with MongoContentstoreBuilder().build() as contentstore:
+            # initialize the mixed modulestore
+            self._initialize_mixed(contentstore=contentstore, mappings={})
+            with self.store.default_store(default_ms):
+                source_course_key = self.store.make_course_key("org.source", "course.source", "run.source")
+                self._create_course(source_course_key)
+                yield contentstore, source_course_key
+
+    @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
+    def test_draft_has_changes_before_export_and_after_import(self, default_ms):
+        """
+        Tests that an unpublished unit remains with no changes across export and re-import.
+        """
+        with self._build_store(default_ms) as (contentstore, source_course_key):
+
+            # Create a dummy component to test against and don't publish it.
+            draft_xblock = self.store.create_item(
+                self.user_id,
+                self.course.id,
+                'vertical',
+                block_id='test_vertical'
+            )
+            # Not yet published, so changes are present
+            self.assertTrue(self._has_changes(draft_xblock.location))
+
+            self._export_import_course_round_trip(
+                self.store, contentstore, source_course_key, self.export_dir
+            )
+
+            # Verify that the imported block still is a draft, i.e. has changes.
+            self.assertTrue(self._has_changes(draft_xblock.location))
+
+    @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
+    def test_published_has_changes_before_export_and_after_import(self, default_ms):
+        """
+        Tests that an published unit remains published across export and re-import.
+        """
+        with self._build_store(default_ms) as (contentstore, source_course_key):
+
+            # Create a dummy component to test against and publish it.
+            published_xblock = self.store.create_item(
+                self.user_id,
+                self.course.id,
+                'vertical',
+                block_id='test_vertical'
+            )
+            self.store.publish(published_xblock.location, self.user_id)
+
+            # Retrieve the published block and make sure it's published.
+            self.assertFalse(self._has_changes(published_xblock.location))
+
+            self._export_import_course_round_trip(
+                self.store, contentstore, source_course_key, self.export_dir
+            )
+
+            # Get the published xblock from the imported course.
+            # Verify that it still is published, i.e. has no changes.
+            self.assertFalse(self._has_changes(published_xblock.location))
+
+    @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
+    def test_changed_published_has_changes_before_export_and_after_import(self, default_ms):
+        """
+        Tests that an published unit with an unpublished draft remains published across export and re-import.
+        """
+        with self._build_store(default_ms) as (contentstore, source_course_key):
+
+            # Create a dummy component to test against and publish it.
+            published_xblock = self.store.create_item(
+                self.user_id,
+                self.course.id,
+                'vertical',
+                block_id='test_vertical'
+            )
+            self.store.publish(published_xblock.location, self.user_id)
+
+            # Retrieve the published block and make sure it's published.
+            self.assertFalse(self._has_changes(published_xblock.location))
+
+            updated_display_name = 'Changed Display Name'
+            component = self.store.get_item(published_xblock.location)
+            component.display_name = updated_display_name
+            component = self.store.update_item(component, self.user_id)
+            self.assertTrue(self.store.has_changes(component))
+
+            self._export_import_course_round_trip(
+                self.store, contentstore, source_course_key, self.export_dir
+            )
+
+            # Get the published xblock from the imported course.
+            # Verify that the published block still has a draft block, i.e. has changes.
+            self.assertTrue(self._has_changes(published_xblock.location))
+
+            # Verify that the changes in the draft vertical still exist.
+            with self.store.branch_setting(ModuleStoreEnum.Branch.draft_preferred, source_course_key):
+                component = self.store.get_item(published_xblock.location)
+                self.assertEqual(component.display_name, updated_display_name)
+
+    @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
+    def test_seq_with_unpublished_vertical_has_changes_before_export_and_after_import(self, default_ms):
+        """
+        Tests that an published unit with an unpublished draft remains published across export and re-import.
+        """
+        with self._build_store(default_ms) as (contentstore, source_course_key):
+
+            # create chapter
+            chapter = self.store.create_child(
+                self.user_id, self.course.location, 'chapter', block_id='section_one'
+            )
+            self.store.publish(chapter.location, self.user_id)
+
+            # create sequential
+            sequential = self.store.create_child(
+                self.user_id, chapter.location, 'sequential', block_id='subsection_one'
+            )
+            self.store.publish(sequential.location, self.user_id)
+
+            # create vertical - don't publish it!
+            vertical = self.store.create_child(
+                self.user_id, sequential.location, 'vertical', block_id='moon_unit'
+            )
+
+            # Retrieve the published block and make sure it's published.
+            # Chapter is published - but the changes in vertical below means it "has_changes".
+            self.assertTrue(self._has_changes(chapter.location))
+            # Sequential is published - but the changes in vertical below means it "has_changes".
+            self.assertTrue(self._has_changes(sequential.location))
+            # Vertical is unpublished - so it "has_changes".
+            self.assertTrue(self._has_changes(vertical.location))
+
+            self._export_import_course_round_trip(
+                self.store, contentstore, source_course_key, self.export_dir
+            )
+
+            # Get the published xblock from the imported course.
+            # Verify that the published block still has a draft block, i.e. has changes.
+            self.assertTrue(self._has_changes(chapter.location))
+            self.assertTrue(self._has_changes(sequential.location))
+            self.assertTrue(self._has_changes(vertical.location))
+
+    @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
+    def test_vertical_with_draft_and_published_unit_has_changes_before_export_and_after_import(self, default_ms):
+        """
+        Tests that an published unit with an unpublished draft remains published across export and re-import.
+        """
+        with self._build_store(default_ms) as (contentstore, source_course_key):
+
+            # create chapter
+            chapter = self.store.create_child(
+                self.user_id, self.course.location, 'chapter', block_id='section_one'
+            )
+            self.store.publish(chapter.location, self.user_id)
+
+            # create sequential
+            sequential = self.store.create_child(
+                self.user_id, chapter.location, 'sequential', block_id='subsection_one'
+            )
+            self.store.publish(sequential.location, self.user_id)
+
+            # create vertical
+            vertical = self.store.create_child(
+                self.user_id, sequential.location, 'vertical', block_id='moon_unit'
+            )
+            # Vertical has changes until it is actually published.
+            self.assertTrue(self._has_changes(vertical.location))
+            self.store.publish(vertical.location, self.user_id)
+            self.assertFalse(self._has_changes(vertical.location))
+
+            # create unit
+            unit = self.store.create_child(
+                self.user_id, vertical.location, 'html', block_id='html_unit'
+            )
+            # Vertical has a new child -and- unit is unpublished. So both have changes.
+            self.assertTrue(self._has_changes(vertical.location))
+            self.assertTrue(self._has_changes(unit.location))
+
+            # Publishing the vertical also publishes its unit child.
+            self.store.publish(vertical.location, self.user_id)
+            self.assertFalse(self._has_changes(vertical.location))
+            self.assertFalse(self._has_changes(unit.location))
+
+            # Publishing the unit separately has no effect on whether it has changes - it's already published.
+            self.store.publish(unit.location, self.user_id)
+            self.assertFalse(self._has_changes(vertical.location))
+            self.assertFalse(self._has_changes(unit.location))
+
+            # Retrieve the published block and make sure it's published.
+            self.store.publish(chapter.location, self.user_id)
+            self.assertFalse(self._has_changes(chapter.location))
+            self.assertFalse(self._has_changes(sequential.location))
+            self.assertFalse(self._has_changes(vertical.location))
+            self.assertFalse(self._has_changes(unit.location))
+
+            # Now make changes to the unit - but don't publish them.
+            component = self.store.get_item(unit.location)
+            updated_display_name = 'Changed Display Name'
+            component.display_name = updated_display_name
+            component = self.store.update_item(component, self.user_id)
+            self.assertTrue(self._has_changes(component.location))
+
+            # Export the course - then import the course export.
+            self._export_import_course_round_trip(
+                self.store, contentstore, source_course_key, self.export_dir
+            )
+
+            # Get the published xblock from the imported course.
+            # Verify that the published block still has a draft block, i.e. has changes.
+            self.assertTrue(self._has_changes(chapter.location))
+            self.assertTrue(self._has_changes(sequential.location))
+            self.assertTrue(self._has_changes(vertical.location))
+            self.assertTrue(self._has_changes(unit.location))
+
+            # Verify that the changes in the draft unit still exist.
+            with self.store.branch_setting(ModuleStoreEnum.Branch.draft_preferred, source_course_key):
+                component = self.store.get_item(unit.location)
+                self.assertEqual(component.display_name, updated_display_name)
+
+            # Verify that the draft changes don't exist in the published unit - it still uses the default name.
+            with self.store.branch_setting(ModuleStoreEnum.Branch.published_only, source_course_key):
+                component = self.store.get_item(unit.location)
+                self.assertEqual(component.display_name, 'Text')
+
+    @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
+    def test_vertical_with_published_unit_remains_published_before_export_and_after_import(self, default_ms):
+        """
+        Tests that an published unit remains published across export and re-import.
+        """
+        with self._build_store(default_ms) as (contentstore, source_course_key):
+
+            # create chapter
+            chapter = self.store.create_child(
+                self.user_id, self.course.location, 'chapter', block_id='section_one'
+            )
+            self.store.publish(chapter.location, self.user_id)
+
+            # create sequential
+            sequential = self.store.create_child(
+                self.user_id, chapter.location, 'sequential', block_id='subsection_one'
+            )
+            self.store.publish(sequential.location, self.user_id)
+
+            # create vertical
+            vertical = self.store.create_child(
+                self.user_id, sequential.location, 'vertical', block_id='moon_unit'
+            )
+            # Vertical has changes until it is actually published.
+            self.assertTrue(self._has_changes(vertical.location))
+            self.store.publish(vertical.location, self.user_id)
+            self.assertFalse(self._has_changes(vertical.location))
+
+            # create unit
+            unit = self.store.create_child(
+                self.user_id, vertical.location, 'html', block_id='html_unit'
+            )
+            # Now make changes to the unit.
+            updated_display_name = 'Changed Display Name'
+            unit.display_name = updated_display_name
+            unit = self.store.update_item(unit, self.user_id)
+            self.assertTrue(self._has_changes(unit.location))
+
+            # Publishing the vertical also publishes its unit child.
+            self.store.publish(vertical.location, self.user_id)
+            self.assertFalse(self._has_changes(vertical.location))
+            self.assertFalse(self._has_changes(unit.location))
+
+            # Export the course - then import the course export.
+            self._export_import_course_round_trip(
+                self.store, contentstore, source_course_key, self.export_dir
+            )
+
+            # Get the published xblock from the imported course.
+            # Verify that the published block still has a draft block, i.e. has changes.
+            self.assertFalse(self._has_changes(chapter.location))
+            self.assertFalse(self._has_changes(sequential.location))
+            self.assertFalse(self._has_changes(vertical.location))
+            self.assertFalse(self._has_changes(unit.location))
+
+            # Verify that the published changes exist in the published unit.
+            with self.store.branch_setting(ModuleStoreEnum.Branch.published_only, source_course_key):
+                component = self.store.get_item(unit.location)
+                self.assertEqual(component.display_name, updated_display_name)

From 61aa3c27895359e6ad5f447d03e48ad2b88406a6 Mon Sep 17 00:00:00 2001
From: John Eskew <jeskew@edx.org>
Date: Thu, 23 Apr 2015 14:01:33 -0400
Subject: [PATCH 3/7] If BlockKey already, then return.

---
 common/lib/xmodule/xmodule/modulestore/split_mongo/split.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
index adb33387f7..e0f7df1900 100644
--- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
+++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
@@ -2831,6 +2831,8 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
             # perhaps replace by fixing the views or Field Reference*.from_json to return a Key
             if isinstance(reference, basestring):
                 reference = BlockUsageLocator.from_string(reference)
+            elif isinstance(reference, BlockKey):
+                return reference
             return BlockKey.from_usage_key(reference)
 
         for field_name, value in fields.iteritems():

From 062ad95f79a71eba10d5d7078b98ac72c1812234 Mon Sep 17 00:00:00 2001
From: John Eskew <jeskew@edx.org>
Date: Thu, 23 Apr 2015 14:12:51 -0400
Subject: [PATCH 4/7] Change str to repr and change formatting.

---
 .../xmodule/xmodule/modulestore/__init__.py   | 43 +++++++++++--------
 1 file changed, 26 insertions(+), 17 deletions(-)

diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py
index ec57877ef1..6ed562ce34 100644
--- a/common/lib/xmodule/xmodule/modulestore/__init__.py
+++ b/common/lib/xmodule/xmodule/modulestore/__init__.py
@@ -352,16 +352,21 @@ class EditInfo(object):
         self.original_usage = edit_info.get('original_usage', None)
         self.original_usage_version = edit_info.get('original_usage_version', None)
 
-    def __str__(self):
-        return ("EditInfo(previous_version={0.previous_version}, "
-                "update_version={0.update_version}, "
-                "source_version={0.source_version}, "
-                "edited_on={0.edited_on}, "
-                "edited_by={0.edited_by}, "
-                "original_usage={0.original_usage}, "
-                "original_usage_version={0.original_usage_version}, "
-                "_subtree_edited_on={0._subtree_edited_on}, "
-                "_subtree_edited_by={0._subtree_edited_by})").format(self)
+    def __repr__(self):
+        # pylint: disable=bad-continuation, redundant-keyword-arg
+        return ("{classname}(previous_version={self.previous_version}, "
+                "update_version={self.update_version}, "
+                "source_version={source_version}, "
+                "edited_on={self.edited_on}, "
+                "edited_by={self.edited_by}, "
+                "original_usage={self.original_usage}, "
+                "original_usage_version={self.original_usage_version}, "
+                "_subtree_edited_on={self._subtree_edited_on}, "
+                "_subtree_edited_by={self._subtree_edited_by})").format(
+            self=self,
+            classname=self.__class__.__name__,
+            source_version="UNSET" if self.source_version is None else self.source_version,
+        )  # pylint: disable=bad-continuation
 
 
 class BlockData(object):
@@ -408,13 +413,17 @@ class BlockData(object):
         # EditInfo object containing all versioning/editing data.
         self.edit_info = EditInfo(**block_data.get('edit_info', {}))
 
-    def __str__(self):
-        return ("BlockData(fields={0.fields}, "
-                "block_type={0.block_type}, "
-                "definition={0.definition}, "
-                "definition_loaded={0.definition_loaded}, "
-                "defaults={0.defaults}, "
-                "edit_info={0.edit_info})").format(self)
+    def __repr__(self):
+        # pylint: disable=bad-continuation, redundant-keyword-arg
+        return ("{classname}(fields={self.fields}, "
+                "block_type={self.block_type}, "
+                "definition={self.definition}, "
+                "definition_loaded={self.definition_loaded}, "
+                "defaults={self.defaults}, "
+                "edit_info={self.edit_info})").format(
+            self=self,
+            classname=self.__class__.__name__,
+        )  # pylint: disable=bad-continuation
 
 
 new_contract('BlockData', BlockData)

From 1c6ab4cc6a75910bbd9124a28e83b456e66665f8 Mon Sep 17 00:00:00 2001
From: John Eskew <jeskew@edx.org>
Date: Tue, 28 Apr 2015 14:38:30 -0400
Subject: [PATCH 5/7] Add has_changes check for draft modules before exporting
 them. Only export draft modules that have changes w.r.t. the published branch
 should be exported to the /drafts directory in the course export.

---
 common/lib/xmodule/xmodule/modulestore/xml_exporter.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/common/lib/xmodule/xmodule/modulestore/xml_exporter.py b/common/lib/xmodule/xmodule/modulestore/xml_exporter.py
index bbc7d5c995..31b4f93313 100644
--- a/common/lib/xmodule/xmodule/modulestore/xml_exporter.py
+++ b/common/lib/xmodule/xmodule/modulestore/xml_exporter.py
@@ -42,6 +42,9 @@ def _export_drafts(modulestore, course_key, export_fs, xml_centric_course_key):
             qualifiers={'category': {'$nin': DIRECT_ONLY_CATEGORIES}},
             revision=ModuleStoreEnum.RevisionOption.draft_only
         )
+        # Check to see if the returned draft modules have changes w.r.t. the published module.
+        # Only modules with changes will be exported into the /drafts directory.
+        draft_modules = [module for module in draft_modules if modulestore.has_changes(module)]
 
         if draft_modules:
             draft_course_dir = export_fs.makeopendir(DRAFT_DIR)

From b290b305df9bac7e9c507497e03e8c7464a99c74 Mon Sep 17 00:00:00 2001
From: John Eskew <jeskew@edx.org>
Date: Wed, 29 Apr 2015 13:51:07 -0400
Subject: [PATCH 6/7] Reorganize code, change method name, & fix comments.
 Change xml file reading code when importing a course.

---
 .../modulestore/split_mongo/split_draft.py    |   7 +-
 .../modulestore/tests/test_xml_importer.py    |   8 +-
 .../xmodule/modulestore/xml_importer.py       | 186 ++++++++----------
 3 files changed, 92 insertions(+), 109 deletions(-)

diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py
index e69c172d73..2be612337b 100644
--- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py
+++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py
@@ -494,11 +494,14 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
                 block_id = self.DEFAULT_ROOT_LIBRARY_BLOCK_ID
             new_usage_key = course_key.make_usage_key(block_type, block_id)
 
+            # Only the course import process calls import_xblock(). If the branch setting is published_only,
+            # then the non-draft blocks are being imported.
             if self.get_branch_setting() == ModuleStoreEnum.Branch.published_only:
-                # override existing draft (PLAT-297, PLAT-299). NOTE: this has the effect of removing
-                # any local changes w/ the import.
+                # Override any existing drafts (PLAT-297, PLAT-299). This import/publish step removes
+                # any local changes during the course import.
                 draft_course = course_key.for_branch(ModuleStoreEnum.BranchName.draft)
                 with self.branch_setting(ModuleStoreEnum.Branch.draft_preferred, draft_course):
+                    # Importing the block and publishing the block links the draft & published blocks' version history.
                     draft_block = self.import_xblock(user_id, draft_course, block_type, block_id, fields, runtime)
                     return self.publish(draft_block.location.version_agnostic(), user_id, blacklist=EXCLUDE_ALL, **kwargs)
 
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py b/common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py
index 3adde6d8d2..f92af44ceb 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py
@@ -9,7 +9,7 @@ from xmodule.x_module import XModuleMixin
 from opaque_keys.edx.locations import Location
 from xmodule.modulestore import ModuleStoreEnum
 from xmodule.modulestore.inheritance import InheritanceMixin
-from xmodule.modulestore.xml_importer import _import_module_and_update_references
+from xmodule.modulestore.xml_importer import _update_and_import_module
 from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
 from xmodule.tests import DATA_DIR
@@ -144,7 +144,7 @@ class RemapNamespaceTest(ModuleStoreNoSettings):
 
         # Move to different runtime w/ different course id
         target_location_namespace = SlashSeparatedCourseKey("org", "course", "run")
-        new_version = _import_module_and_update_references(
+        new_version = _update_and_import_module(
             self.xblock,
             modulestore(),
             999,
@@ -181,7 +181,7 @@ class RemapNamespaceTest(ModuleStoreNoSettings):
 
         # Remap the namespace
         target_location_namespace = Location("org", "course", "run", "category", "stubxblock")
-        new_version = _import_module_and_update_references(
+        new_version = _update_and_import_module(
             self.xblock,
             modulestore(),
             999,
@@ -213,7 +213,7 @@ class RemapNamespaceTest(ModuleStoreNoSettings):
 
         # Remap the namespace
         target_location_namespace = Location("org", "course", "run", "category", "stubxblock")
-        new_version = _import_module_and_update_references(
+        new_version = _update_and_import_module(
             self.xblock,
             modulestore(),
             999,
diff --git a/common/lib/xmodule/xmodule/modulestore/xml_importer.py b/common/lib/xmodule/xmodule/modulestore/xml_importer.py
index fb6b7838a4..49c375802a 100644
--- a/common/lib/xmodule/xmodule/modulestore/xml_importer.py
+++ b/common/lib/xmodule/xmodule/modulestore/xml_importer.py
@@ -316,7 +316,7 @@ class ImportManager(object):
         log.debug('course data_dir=%s', source_courselike.data_dir)
 
         with self.store.branch_setting(ModuleStoreEnum.Branch.draft_preferred, dest_id):
-            course = _import_module_and_update_references(
+            course = _update_and_import_module(
                 source_courselike, self.store, self.user_id,
                 courselike_key,
                 dest_id,
@@ -381,7 +381,7 @@ class ImportManager(object):
                     if self.verbose:
                         log.debug('importing module location %s', child.location)
 
-                    _import_module_and_update_references(
+                    _update_and_import_module(
                         child,
                         self.store,
                         self.user_id,
@@ -399,7 +399,7 @@ class ImportManager(object):
             if self.verbose:
                 log.debug('importing module location %s', leftover)
 
-            _import_module_and_update_references(
+            _update_and_import_module(
                 self.xml_module_store.get_item(leftover),
                 self.store,
                 self.user_id,
@@ -620,13 +620,68 @@ def import_library_from_xml(*args, **kwargs):
     return list(manager.run_imports())
 
 
-def _import_module_and_update_references(
+def _update_and_import_module(
         module, store, user_id,
         source_course_id, dest_course_id,
         do_import_static=True, runtime=None):
-
+    """
+    Update all the module reference fields to the destination course id,
+    then import the module into the destination course.
+    """
     logging.debug(u'processing import of module %s...', unicode(module.location))
 
+    def _update_module_references(module, source_course_id, dest_course_id):
+        """
+        Move the module to a new course.
+        """
+        def _convert_ref_fields_to_new_namespace(reference):  # pylint: disable=invalid-name
+            """
+            Convert a reference to the new namespace, but only
+            if the original namespace matched the original course.
+
+            Otherwise, returns the input value.
+            """
+            assert isinstance(reference, UsageKey)
+            if source_course_id == reference.course_key:
+                return reference.map_into_course(dest_course_id)
+            else:
+                return reference
+
+        fields = {}
+        for field_name, field in module.fields.iteritems():
+            if field.scope != Scope.parent and field.is_set_on(module):
+                if isinstance(field, Reference):
+                    value = field.read_from(module)
+                    if value is None:
+                        fields[field_name] = None
+                    else:
+                        fields[field_name] = _convert_ref_fields_to_new_namespace(field.read_from(module))
+                elif isinstance(field, ReferenceList):
+                    references = field.read_from(module)
+                    fields[field_name] = [_convert_ref_fields_to_new_namespace(reference) for reference in references]
+                elif isinstance(field, ReferenceValueDict):
+                    reference_dict = field.read_from(module)
+                    fields[field_name] = {
+                        key: _convert_ref_fields_to_new_namespace(reference)
+                        for key, reference
+                        in reference_dict.iteritems()
+                    }
+                elif field_name == 'xml_attributes':
+                    value = field.read_from(module)
+                    # remove any export/import only xml_attributes
+                    # which are used to wire together draft imports
+                    if 'parent_url' in value:
+                        del value['parent_url']
+                    if 'parent_sequential_url' in value:
+                        del value['parent_sequential_url']
+
+                    if 'index_in_children_list' in value:
+                        del value['index_in_children_list']
+                    fields[field_name] = value
+                else:
+                    fields[field_name] = field.read_from(module)
+        return fields
+
     if do_import_static and 'data' in module.fields and isinstance(module.fields['data'], xblock.fields.String):
         # we want to convert all 'non-portable' links in the module_data
         # (if it is a string) to portable strings (e.g. /static/)
@@ -636,53 +691,7 @@ def _import_module_and_update_references(
             module.data
         )
 
-    # Move the module to a new course
-    def _convert_reference_fields_to_new_namespace(reference):
-        """
-        Convert a reference to the new namespace, but only
-        if the original namespace matched the original course.
-
-        Otherwise, returns the input value.
-        """
-        assert isinstance(reference, UsageKey)
-        if source_course_id == reference.course_key:
-            return reference.map_into_course(dest_course_id)
-        else:
-            return reference
-
-    fields = {}
-    for field_name, field in module.fields.iteritems():
-        if field.scope != Scope.parent and field.is_set_on(module):
-            if isinstance(field, Reference):
-                value = field.read_from(module)
-                if value is None:
-                    fields[field_name] = None
-                else:
-                    fields[field_name] = _convert_reference_fields_to_new_namespace(field.read_from(module))
-            elif isinstance(field, ReferenceList):
-                references = field.read_from(module)
-                fields[field_name] = [_convert_reference_fields_to_new_namespace(reference) for reference in references]
-            elif isinstance(field, ReferenceValueDict):
-                reference_dict = field.read_from(module)
-                fields[field_name] = {
-                    key: _convert_reference_fields_to_new_namespace(reference)
-                    for key, reference
-                    in reference_dict.iteritems()
-                }
-            elif field_name == 'xml_attributes':
-                value = field.read_from(module)
-                # remove any export/import only xml_attributes
-                # which are used to wire together draft imports
-                if 'parent_url' in value:
-                    del value['parent_url']
-                if 'parent_sequential_url' in value:
-                    del value['parent_sequential_url']
-
-                if 'index_in_children_list' in value:
-                    del value['index_in_children_list']
-                fields[field_name] = value
-            else:
-                fields[field_name] = field.read_from(module)
+    fields = _update_module_references(module, source_course_id, dest_course_id)
 
     return store.import_xblock(
         user_id, dest_course_id, module.location.category,
@@ -699,14 +708,13 @@ def _import_course_draft(
         target_id,
         mongo_runtime
 ):
-    '''
-    This will import all the content inside of the 'drafts' folder, if it exists
-    NOTE: This is not a full course import, basically in our current
-    application only verticals (and downwards) can be in draft.
-    Therefore, we need to use slightly different call points into
-    the import process_xml as we can't simply call XMLModuleStore() constructor
-    (like we do for importing public content)
-    '''
+    """
+    This method will import all the content inside of the 'drafts' folder, if content exists.
+    NOTE: This is not a full course import! In our current application, only verticals
+    (and blocks beneath) can be in draft. Therefore, different call points into the import
+    process_xml are used as the XMLModuleStore() constructor cannot simply be called
+    (as is done for importing public content).
+    """
     draft_dir = course_data_path + "/drafts"
     if not os.path.exists(draft_dir):
         return
@@ -720,7 +728,9 @@ def _import_course_draft(
     # Whether or not data_dir ends with a "/" differs in production vs. test.
     if not data_dir.endswith("/"):
         data_dir += "/"
+    # Remove absolute path, leaving relative <course_name>/drafts.
     draft_course_dir = draft_dir.replace(data_dir, '', 1)
+
     system = ImportSystem(
         xmlstore=xml_module_store,
         course_id=source_course_id,
@@ -761,7 +771,7 @@ def _import_course_draft(
                 parent.children.insert(index, non_draft_location)
                 store.update_item(parent, user_id)
 
-        _import_module_and_update_references(
+        _update_and_import_module(
             module, store, user_id,
             source_course_id,
             target_id,
@@ -770,52 +780,23 @@ def _import_course_draft(
         for child in module.get_children():
             _import_module(child)
 
-    # Now walk the /vertical directory.
+    # Now walk the /drafts directory.
     # Each file in the directory will be a draft copy of the vertical.
 
     # First it is necessary to order the draft items by their desired index in the child list,
     # since the order in which os.walk() returns the files is not guaranteed.
     drafts = []
-    for dirname, _dirnames, filenames in os.walk(draft_dir):
+    for rootdir, __, filenames in os.walk(draft_dir):
         for filename in filenames:
-            module_path = os.path.join(dirname, filename)
+            if filename.startswith('._'):
+                # Skip any OSX quarantine files, prefixed with a '._'.
+                continue
+            module_path = os.path.join(rootdir, filename)
             with open(module_path, 'r') as f:
                 try:
-                    # note, on local dev it seems like OSX will put
-                    # some extra files in the directory with "quarantine"
-                    # information. These files are binary files and will
-                    # throw exceptions when we try to parse the file
-                    # as an XML string. Let's make sure we're
-                    # dealing with a string before ingesting
-                    data = f.read()
+                    xml = f.read().decode('utf-8')
 
-                    try:
-                        xml = data.decode('utf-8')
-                    except UnicodeDecodeError, err:
-                        # seems like on OSX localdev, the OS is making
-                        # quarantine files in the unzip directory
-                        # when importing courses so if we blindly try to
-                        # enumerate through the directory, we'll try
-                        # to process a bunch of binary quarantine files
-                        # (which are prefixed with a '._' character which
-                        # will dump a bunch of exceptions to the output,
-                        # although they are harmless.
-                        #
-                        # Reading online docs there doesn't seem to be
-                        # a good means to detect a 'hidden' file that works
-                        # well across all OS environments. So for now, I'm using
-                        # OSX's utilization of a leading '.' in the filename
-                        # to indicate a system hidden file.
-                        #
-                        # Better yet would be a way to figure out if this is
-                        # a binary file, but I haven't found a good way
-                        # to do this yet.
-                        if filename.startswith('._'):
-                            continue
-                        # Not a 'hidden file', then re-raise exception
-                        raise err
-
-                    # process_xml call below recursively processes all descendants. If
+                    # The process_xml() call below recursively processes all descendants. If
                     # we call this on all verticals in a course with verticals nested below
                     # the unit level, we try to import the same content twice, causing naming conflicts.
                     # Therefore only process verticals at the unit level, assuming that any other
@@ -838,13 +819,12 @@ def _import_course_draft(
                         draft = draft_node_constructor(
                             module=descriptor, url=draft_url, parent_url=parent_url, index=index
                         )
-
                         drafts.append(draft)
 
                 except Exception:  # pylint: disable=broad-except
-                    logging.exception('Error while parsing course xml.')
+                    logging.exception('Error while parsing course drafts xml.')
 
-    # sort drafts by `index_in_children_list` attribute
+    # Sort drafts by `index_in_children_list` attribute.
     drafts.sort(key=lambda x: x.index)
 
     for draft in get_draft_subtree_roots(drafts):
@@ -864,11 +844,11 @@ def allowed_metadata_by_category(category):
 
 
 def check_module_metadata_editability(module):
-    '''
+    """
     Assert that there is no metadata within a particular module that
     we can't support editing. However we always allow 'display_name'
     and 'xml_attributes'
-    '''
+    """
     allowed = allowed_metadata_by_category(module.location.category)
     if '*' in allowed:
         # everything is allowed

From 8ca57f83d35b3e7716014ffedb363a5024ce2091 Mon Sep 17 00:00:00 2001
From: John Eskew <jeskew@edx.org>
Date: Wed, 29 Apr 2015 14:03:22 -0400
Subject: [PATCH 7/7]  Separate draft/published import into two separate bulk
 operations.  Add import_drafts abstract method in addition to
 import_children.

---
 .../xmodule/modulestore/xml_importer.py       | 46 +++++++++++++++++--
 1 file changed, 42 insertions(+), 4 deletions(-)

diff --git a/common/lib/xmodule/xmodule/modulestore/xml_importer.py b/common/lib/xmodule/xmodule/modulestore/xml_importer.py
index 49c375802a..ab8316cd1c 100644
--- a/common/lib/xmodule/xmodule/modulestore/xml_importer.py
+++ b/common/lib/xmodule/xmodule/modulestore/xml_importer.py
@@ -352,12 +352,19 @@ class ImportManager(object):
         raise NotImplementedError
 
     @abstractmethod
-    def import_children(self, source_courselike, courselike, courselike_key, data_path, dest_id):
+    def import_children(self, source_courselike, courselike, courselike_key, dest_id):
         """
         To be overloaded with a method that installs the child items into self.store.
         """
         raise NotImplementedError
 
+    @abstractmethod
+    def import_drafts(self, courselike, courselike_key, data_path, dest_id):
+        """
+        To be overloaded with a method that installs the draft items into self.store.
+        """
+        raise NotImplementedError
+
     def recursive_build(self, source_courselike, courselike, courselike_key, dest_id):
         """
         Recursively imports all child blocks from the temporary modulestore into the
@@ -419,8 +426,12 @@ class ImportManager(object):
                 dest_id, runtime = self.get_dest_id(courselike_key)
             except DuplicateCourseError:
                 continue
+
+            # This bulk operation wraps all the operations to populate the published branch.
             with self.store.bulk_operations(dest_id):
+                # Retrieve the course itself.
                 source_courselike, courselike, data_path = self.get_courselike(courselike_key, runtime, dest_id)
+
                 # Import all static pieces.
                 self.import_static(data_path, dest_id)
 
@@ -428,7 +439,17 @@ class ImportManager(object):
                 self.import_asset_metadata(data_path, dest_id)
 
                 # Import all children
-                self.import_children(source_courselike, courselike, courselike_key, data_path, dest_id)
+                self.import_children(source_courselike, courselike, courselike_key, dest_id)
+
+            # This bulk operation wraps all the operations to populate the draft branch with any items
+            # from the /drafts subdirectory.
+            # Drafts must be imported in a separate bulk operation from published items to import properly,
+            # due to the recursive_build() above creating a draft item for each course block
+            # and then publishing it.
+            with self.store.bulk_operations(dest_id):
+                # Import all draft items into the courselike.
+                courselike = self.import_drafts(courselike, courselike_key, data_path, dest_id)
+
             yield courselike
 
 
@@ -520,13 +541,19 @@ class CourseImportManager(ImportManager):
         if course.tabs is None or len(course.tabs) == 0:
             CourseTabList.initialize_default(course)
 
-    def import_children(self, source_courselike, courselike, courselike_key, data_path, dest_id):
+    def import_children(self, source_courselike, courselike, courselike_key, dest_id):
         """
         Imports all children into the desired store.
         """
+        # The branch setting of published_only forces an overwrite of all draft modules
+        # during the course import.
         with self.store.branch_setting(ModuleStoreEnum.Branch.published_only, dest_id):
             self.recursive_build(source_courselike, courselike, courselike_key, dest_id)
 
+    def import_drafts(self, courselike, courselike_key, data_path, dest_id):
+        """
+        Imports all drafts into the desired store.
+        """
         # Import any draft items
         with self.store.branch_setting(ModuleStoreEnum.Branch.draft_preferred, dest_id):
             _import_course_draft(
@@ -539,6 +566,11 @@ class CourseImportManager(ImportManager):
                 courselike.runtime
             )
 
+        # Importing the drafts potentially triggered a new structure version.
+        # If so, the HEAD version_guid of the passed-in courselike will be out-of-date.
+        # Fetch the course to return the most recent course version.
+        return self.store.get_course(courselike.id.replace(branch=None, version_guid=None))
+
 
 class LibraryImportManager(ImportManager):
     """
@@ -597,12 +629,18 @@ class LibraryImportManager(ImportManager):
         """
         pass
 
-    def import_children(self, source_courselike, courselike, courselike_key, data_path, dest_id):
+    def import_children(self, source_courselike, courselike, courselike_key, dest_id):
         """
         Imports all children into the desired store.
         """
         self.recursive_build(source_courselike, courselike, courselike_key, dest_id)
 
+    def import_drafts(self, courselike, courselike_key, data_path, dest_id):
+        """
+        Imports all drafts into the desired store.
+        """
+        return courselike
+
 
 def import_course_from_xml(*args, **kwargs):
     """