Merge pull request #12522 from edx/brian/cms-course-content-export

Define CMS management command for research course exports

cms/djangoapps/contentstore/management/commands/export_olx.py

@@ -0,0 +1,111 @@
+"""
+A Django command that exports a course to a tar.gz file.
+
+If <filename> is '-', it pipes the file to stdout.
+
+This is used by Analytics research exports to provide researchers
+with course content.
+
+At present, it differs from Studio exports in several ways:
+
+* It does not include static content.
+* The top-level directory in the resulting tarball is a "safe"
+  (i.e. ascii) version of the course_key, rather than the word "course".
+* It only supports the export of courses.  It does not export libraries.
+
+"""
+
+import os
+import re
+import shutil
+import tarfile
+from tempfile import mktemp, mkdtemp
+from textwrap import dedent
+
+from path import Path as path
+
+from django.core.management.base import BaseCommand, CommandError
+
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.xml_exporter import export_course_to_xml
+from opaque_keys import InvalidKeyError
+from opaque_keys.edx.keys import CourseKey
+
+
+class Command(BaseCommand):
+    """
+    Export a course to XML. The output is compressed as a tar.gz file.
+
+    """
+    help = dedent(__doc__).strip()
+
+    def add_arguments(self, parser):
+        parser.add_argument('course_id')
+        parser.add_argument('--output', default=None)
+
+    def handle(self, *args, **options):
+
+        course_id = options['course_id']
+        try:
+            course_key = CourseKey.from_string(course_id)
+        except InvalidKeyError:
+            raise CommandError("Unparsable course_id")
+        except IndexError:
+            raise CommandError("Insufficient arguments")
+
+        filename = options['output']
+        pipe_results = False
+        if filename is None:
+            filename = mktemp()
+            pipe_results = True
+
+        export_course_to_tarfile(course_key, filename)
+
+        results = self._get_results(filename) if pipe_results else None
+
+        self.stdout.write(results, ending="")
+
+    def _get_results(self, filename):
+        """Load results from file"""
+        with open(filename) as f:
+            results = f.read()
+            os.remove(filename)
+        return results
+
+
+def export_course_to_tarfile(course_key, filename):
+    """Exports a course into a tar.gz file"""
+    tmp_dir = mkdtemp()
+    try:
+        course_dir = export_course_to_directory(course_key, tmp_dir)
+        compress_directory(course_dir, filename)
+    finally:
+        shutil.rmtree(tmp_dir, ignore_errors=True)
+
+
+def export_course_to_directory(course_key, root_dir):
+    """Export course into a directory"""
+    store = modulestore()
+    course = store.get_course(course_key)
+    if course is None:
+        raise CommandError("Invalid course_id")
+
+    # The safest characters are A-Z, a-z, 0-9, <underscore>, <period> and <hyphen>.
+    # We represent the first four with \w.
+    # TODO: Once we support courses with unicode characters, we will need to revisit this.
+    replacement_char = u'-'
+    course_dir = replacement_char.join([course.id.org, course.id.course, course.id.run])
+    course_dir = re.sub(r'[^\w\.\-]', replacement_char, course_dir)
+
+    export_course_to_xml(store, None, course.id, root_dir, course_dir)
+
+    export_dir = path(root_dir) / course_dir
+    return export_dir
+
+
+def compress_directory(directory, filename):
+    """Compress a directory into a tar.gz file"""
+    mode = 'w:gz'
+    name = path(directory).name
+    with tarfile.open(filename, mode) as tar_file:
+        tar_file.add(directory, arcname=name)

cms/djangoapps/contentstore/management/commands/tests/test_export_olx.py

@@ -0,0 +1,94 @@
+"""
+Tests for exporting OLX content.
+"""
+
+import ddt
+from path import Path as path
+import shutil
+from StringIO import StringIO
+import tarfile
+from tempfile import mkdtemp
+import unittest
+
+from django.core.management import CommandError, call_command
+
+from xmodule.modulestore.tests.factories import CourseFactory
+from xmodule.modulestore import ModuleStoreEnum
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+from xmodule.modulestore.django import modulestore
+
+
+class TestArgParsingCourseExportOlx(unittest.TestCase):
+    """
+    Tests for parsing arguments for the `export_olx` management command
+    """
+    def test_no_args(self):
+        """
+        Test export command with no arguments
+        """
+        errstring = "Error: too few arguments"
+        with self.assertRaisesRegexp(CommandError, errstring):
+            call_command('export_olx')
+
+
+@ddt.ddt
+class TestCourseExportOlx(ModuleStoreTestCase):
+    """
+    Test exporting OLX content from a course or library.
+    """
+
+    def test_invalid_course_key(self):
+        """
+        Test export command with an invalid course key.
+        """
+        errstring = "Unparsable course_id"
+        with self.assertRaisesRegexp(CommandError, errstring):
+            call_command('export_olx', 'InvalidCourseID')
+
+    def test_course_key_not_found(self):
+        """
+        Test export command with a valid course key that doesn't exist.
+        """
+        errstring = "Invalid course_id"
+        with self.assertRaisesRegexp(CommandError, errstring):
+            call_command('export_olx', 'x/y/z')
+
+    def create_dummy_course(self, store_type):
+        """Create small course."""
+        course = CourseFactory.create(default_store=store_type)
+        self.assertTrue(
+            modulestore().has_course(course.id),
+            "Could not find course in {}".format(store_type)
+        )
+        return course.id
+
+    def check_export_file(self, tar_file, course_key):
+        """Check content of export file."""
+        names = tar_file.getnames()
+        dirname = "{0.org}-{0.course}-{0.run}".format(course_key)
+        self.assertIn(dirname, names)
+        # Check if some of the files are present, without being exhaustive.
+        self.assertIn("{}/about".format(dirname), names)
+        self.assertIn("{}/about/overview.html".format(dirname), names)
+        self.assertIn("{}/assets/assets.xml".format(dirname), names)
+        self.assertIn("{}/policies".format(dirname), names)
+
+    @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
+    def test_export_course(self, store_type):
+        test_course_key = self.create_dummy_course(store_type)
+        tmp_dir = path(mkdtemp())
+        self.addCleanup(shutil.rmtree, tmp_dir)
+        filename = tmp_dir / 'test.tar.gz'
+        call_command('export_olx', '--output', filename, unicode(test_course_key))
+        with tarfile.open(filename) as tar_file:
+            self.check_export_file(tar_file, test_course_key)
+
+    @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
+    def test_export_course_stdout(self, store_type):
+        test_course_key = self.create_dummy_course(store_type)
+        out = StringIO()
+        call_command('export_olx', unicode(test_course_key), stdout=out)
+        out.seek(0)
+        output = out.read()
+        with tarfile.open(fileobj=StringIO(output)) as tar_file:
+            self.check_export_file(tar_file, test_course_key)

lms/djangoapps/courseware/management/commands/export_course.py

@@ -3,6 +3,14 @@ A Django command that exports a course to a tar.gz file.
 
 If <filename> is '-', it pipes the file to stdout
 
+NOTE: This used to be used by Analytics research exports to provide
+researchers with course content.  It is now DEPRECATED, and
+functionality has moved to export_olx.py in
+cms/djangoapps/contentstore/management/commands.
+
+Note: when removing this file, also remove references to it
+from test_dump_course.
+
 """
 
 import os