diff --git a/cms/djangoapps/contentstore/tests/test_contentstore.py b/cms/djangoapps/contentstore/tests/test_contentstore.py index 8588932356..bc73b688f7 100644 --- a/cms/djangoapps/contentstore/tests/test_contentstore.py +++ b/cms/djangoapps/contentstore/tests/test_contentstore.py @@ -23,6 +23,7 @@ from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import CourseKey, UsageKey from opaque_keys.edx.locations import AssetLocation, CourseLocator from path import Path as path +from six import text_type from waffle.testutils import override_switch from contentstore.tests.utils import AjaxEnabledTestClient, CourseTestCase, get_url, parse_json @@ -117,7 +118,7 @@ class ImportRequiredTestCases(ContentStoreTestCase): Asset name in XML: "/invalid\\displayname/subs-esLhHcdKGWvKs.srt" """ content_store = contentstore() - expected_displayname = '_invalid_displayname_subs-esLhHcdKGWvKs.srt' + expected_displayname = u'_invalid_displayname_subs-esLhHcdKGWvKs.srt' import_course_from_xml( self.store, @@ -156,10 +157,10 @@ class ImportRequiredTestCases(ContentStoreTestCase): # Test course export does not fail root_dir = path(mkdtemp_clean()) print 'Exporting to tempdir = {0}'.format(root_dir) - export_course_to_xml(self.store, content_store, course.id, root_dir, 'test_export') + export_course_to_xml(self.store, content_store, course.id, root_dir, u'test_export') - filesystem = OSFS(root_dir / 'test_export/static') - exported_static_files = filesystem.listdir() + filesystem = OSFS(text_type(root_dir / 'test_export/static')) + exported_static_files = filesystem.listdir(u'/') # Verify that asset have been overwritten during export. self.assertEqual(len(exported_static_files), 1) @@ -236,18 +237,18 @@ class ImportRequiredTestCases(ContentStoreTestCase): self.assertIsNotNone(course_updates) # check that course which is imported has files 'updates.html' and 'updates.items.json' - filesystem = OSFS(data_dir + '/course_info_updates/info') - self.assertTrue(filesystem.exists('updates.html')) - self.assertTrue(filesystem.exists('updates.items.json')) + filesystem = OSFS(text_type(data_dir + '/course_info_updates/info')) + self.assertTrue(filesystem.exists(u'updates.html')) + self.assertTrue(filesystem.exists(u'updates.items.json')) # verify that course info update module has same data content as in data file from which it is imported # check 'data' field content - with filesystem.open('updates.html', 'r') as course_policy: + with filesystem.open(u'updates.html', 'r') as course_policy: on_disk = course_policy.read() self.assertEqual(course_updates.data, on_disk) # check 'items' field content - with filesystem.open('updates.items.json', 'r') as course_policy: + with filesystem.open(u'updates.items.json', 'r') as course_policy: on_disk = loads(course_policy.read()) self.assertEqual(course_updates.items, on_disk) @@ -255,19 +256,19 @@ class ImportRequiredTestCases(ContentStoreTestCase): # with same content as in course 'info' directory root_dir = path(mkdtemp_clean()) print 'Exporting to tempdir = {0}'.format(root_dir) - export_course_to_xml(self.store, content_store, course.id, root_dir, 'test_export') + export_course_to_xml(self.store, content_store, course.id, root_dir, u'test_export') # check that exported course has files 'updates.html' and 'updates.items.json' - filesystem = OSFS(root_dir / 'test_export/info') - self.assertTrue(filesystem.exists('updates.html')) - self.assertTrue(filesystem.exists('updates.items.json')) + filesystem = OSFS(text_type(root_dir / 'test_export/info')) + self.assertTrue(filesystem.exists(u'updates.html')) + self.assertTrue(filesystem.exists(u'updates.items.json')) # verify that exported course has same data content as in course_info_update module - with filesystem.open('updates.html', 'r') as grading_policy: + with filesystem.open(u'updates.html', 'r') as grading_policy: on_disk = grading_policy.read() self.assertEqual(on_disk, course_updates.data) - with filesystem.open('updates.items.json', 'r') as grading_policy: + with filesystem.open(u'updates.items.json', 'r') as grading_policy: on_disk = loads(grading_policy.read()) self.assertEqual(on_disk, course_updates.items) @@ -315,39 +316,39 @@ class ImportRequiredTestCases(ContentStoreTestCase): print 'Exporting to tempdir = {0}'.format(root_dir) # export out to a tempdir - export_course_to_xml(self.store, content_store, course_id, root_dir, 'test_export') + export_course_to_xml(self.store, content_store, course_id, root_dir, u'test_export') # check for static tabs - self.verify_content_existence(self.store, root_dir, course_id, 'tabs', 'static_tab', '.html') + self.verify_content_existence(self.store, root_dir, course_id, u'tabs', 'static_tab', '.html') # check for about content - self.verify_content_existence(self.store, root_dir, course_id, 'about', 'about', '.html') + self.verify_content_existence(self.store, root_dir, course_id, u'about', 'about', '.html') # assert that there is an html and video directory in drafts: draft_dir = OSFS(root_dir / 'test_export/drafts') - self.assertTrue(draft_dir.exists('html')) - self.assertTrue(draft_dir.exists('video')) + self.assertTrue(draft_dir.exists(u'html')) + self.assertTrue(draft_dir.exists(u'video')) # and assert that they contain the created modules - self.assertIn(self.DRAFT_HTML + ".xml", draft_dir.listdir('html')) - self.assertIn(self.DRAFT_VIDEO + ".xml", draft_dir.listdir('video')) + self.assertIn(self.DRAFT_HTML + ".xml", draft_dir.listdir(u'html')) + self.assertIn(self.DRAFT_VIDEO + ".xml", draft_dir.listdir(u'video')) # and assert the child of the orphaned draft wasn't exported - self.assertNotIn(self.ORPHAN_DRAFT_HTML + ".xml", draft_dir.listdir('html')) + self.assertNotIn(self.ORPHAN_DRAFT_HTML + ".xml", draft_dir.listdir(u'html')) # check for grading_policy.json filesystem = OSFS(root_dir / 'test_export/policies/2012_Fall') - self.assertTrue(filesystem.exists('grading_policy.json')) + self.assertTrue(filesystem.exists(u'grading_policy.json')) course = self.store.get_course(course_id) # compare what's on disk compared to what we have in our course - with filesystem.open('grading_policy.json', 'r') as grading_policy: + with filesystem.open(u'grading_policy.json', 'r') as grading_policy: on_disk = loads(grading_policy.read()) self.assertEqual(on_disk, course.grading_policy) # check for policy.json - self.assertTrue(filesystem.exists('policy.json')) + self.assertTrue(filesystem.exists(u'policy.json')) # compare what's on disk to what we have in the course module - with filesystem.open('policy.json', 'r') as course_policy: + with filesystem.open(u'policy.json', 'r') as course_policy: on_disk = loads(course_policy.read()) self.assertIn('course/2012_Fall', on_disk) self.assertEqual(on_disk['course/2012_Fall'], own_metadata(course)) @@ -417,7 +418,7 @@ class ImportRequiredTestCases(ContentStoreTestCase): print 'Exporting to tempdir = {0}'.format(root_dir) # export out to a tempdir - export_course_to_xml(self.store, content_store, course_id, root_dir, 'test_export') + export_course_to_xml(self.store, content_store, course_id, root_dir, u'test_export') shutil.rmtree(root_dir) @@ -443,7 +444,7 @@ class ImportRequiredTestCases(ContentStoreTestCase): print 'Exporting to tempdir = {0}'.format(root_dir) # export out to a tempdir - export_course_to_xml(self.store, content_store, course_id, root_dir, 'test_export') + export_course_to_xml(self.store, content_store, course_id, root_dir, u'test_export') shutil.rmtree(root_dir) @@ -496,7 +497,7 @@ class ImportRequiredTestCases(ContentStoreTestCase): # Export the course root_dir = path(mkdtemp_clean()) - export_course_to_xml(self.store, content_store, course_id, root_dir, 'test_roundtrip') + export_course_to_xml(self.store, content_store, course_id, root_dir, u'test_roundtrip') # Reimport and get the video back import_course_from_xml(self.store, self.user.id, root_dir) @@ -517,7 +518,7 @@ class ImportRequiredTestCases(ContentStoreTestCase): # Export the course root_dir = path(mkdtemp_clean()) - export_course_to_xml(self.store, content_store, course_id, root_dir, 'test_roundtrip') + export_course_to_xml(self.store, content_store, course_id, root_dir, u'test_roundtrip') # Reimport and get the video back import_course_from_xml(self.store, self.user.id, root_dir, create_if_not_present=True) @@ -541,7 +542,7 @@ class ImportRequiredTestCases(ContentStoreTestCase): root_dir = path(mkdtemp_clean()) print 'Exporting to tempdir = {0}'.format(root_dir) - export_course_to_xml(self.store, None, course_id, root_dir, 'test_export_no_content_store') + export_course_to_xml(self.store, None, course_id, root_dir, u'test_export_no_content_store') # Delete the course from module store and reimport it @@ -596,7 +597,7 @@ class ImportRequiredTestCases(ContentStoreTestCase): content_store, course_id, root_dir, - 'test_no_xml_attributes' + u'test_no_xml_attributes' ) @@ -695,7 +696,7 @@ class MiscCourseTests(ContentStoreTestCase): def test_export_on_invalid_displayname(self, invalid_displayname): """ Tests that assets with invalid 'displayname' does not cause export to fail """ content_store = contentstore() - exported_asset_name = '_Fake_asset_displayname' + exported_asset_name = u'_Fake_asset_displayname' # Create an asset with slash `invalid_displayname` ' asset_key = self.course.id.make_asset_key('asset', "fake_asset.txt") @@ -713,10 +714,10 @@ class MiscCourseTests(ContentStoreTestCase): # Now export the course to a tempdir and test that it contains assets. The export should pass root_dir = path(mkdtemp_clean()) print 'Exporting to tempdir = {0}'.format(root_dir) - export_course_to_xml(self.store, content_store, self.course.id, root_dir, 'test_export') + export_course_to_xml(self.store, content_store, self.course.id, root_dir, u'test_export') filesystem = OSFS(root_dir / 'test_export/static') - exported_static_files = filesystem.listdir() + exported_static_files = filesystem.listdir(u'/') # Verify that only single asset has been exported with the expected asset name. self.assertTrue(filesystem.exists(exported_asset_name)) @@ -737,13 +738,13 @@ class MiscCourseTests(ContentStoreTestCase): # Make an existing unit a draft self.store.convert_to_draft(self.problem.location, self.user.id) root_dir = path(mkdtemp_clean()) - export_course_to_xml(self.store, None, self.course.id, root_dir, 'test_export') + export_course_to_xml(self.store, None, self.course.id, root_dir, u'test_export') # Verify that problem is exported in the drafts. This is expected because we are # mocking get_item to for drafts. Expect no draft is exported. # Specifically get_item is used in `xmodule.modulestore.xml_exporter._export_drafts` export_draft_dir = OSFS(root_dir / 'test_export/drafts') - self.assertEqual(len(export_draft_dir.listdir()), 0) + self.assertEqual(len(export_draft_dir.listdir(u'/')), 0) # Remove tempdir shutil.rmtree(root_dir) @@ -751,7 +752,7 @@ class MiscCourseTests(ContentStoreTestCase): def test_assets_overwrite(self): """ Tests that assets will similar 'displayname' will be overwritten during export """ content_store = contentstore() - asset_displayname = 'Fake_asset.txt' + asset_displayname = u'Fake_asset.txt' # Create two assets with similar 'displayname' for i in range(2): @@ -773,11 +774,11 @@ class MiscCourseTests(ContentStoreTestCase): # Now export the course to a tempdir and test that it contains assets. root_dir = path(mkdtemp_clean()) print 'Exporting to tempdir = {0}'.format(root_dir) - export_course_to_xml(self.store, content_store, self.course.id, root_dir, 'test_export') + export_course_to_xml(self.store, content_store, self.course.id, root_dir, u'test_export') # Verify that asset have been overwritten during export. filesystem = OSFS(root_dir / 'test_export/static') - exported_static_files = filesystem.listdir() + exported_static_files = filesystem.listdir(u'/') self.assertTrue(filesystem.exists(asset_displayname)) self.assertEqual(len(exported_static_files), 1) @@ -2109,7 +2110,7 @@ class ContentLicenseTest(ContentStoreTestCase): root_dir = path(mkdtemp_clean()) self.course.license = "creative-commons: BY SA" self.store.update_item(self.course, None) - export_course_to_xml(self.store, content_store, self.course.id, root_dir, 'test_license') + export_course_to_xml(self.store, content_store, self.course.id, root_dir, u'test_license') fname = "{block}.xml".format(block=self.course.scope_ids.usage_id.block_id) run_file_path = root_dir / "test_license" / "course" / fname run_xml = etree.parse(run_file_path.open()) @@ -2122,7 +2123,7 @@ class ContentLicenseTest(ContentStoreTestCase): parent_location=self.course.location, category='video', license="all-rights-reserved" ) - export_course_to_xml(self.store, content_store, self.course.id, root_dir, 'test_license') + export_course_to_xml(self.store, content_store, self.course.id, root_dir, u'test_license') fname = "{block}.xml".format(block=video_descriptor.scope_ids.usage_id.block_id) video_file_path = root_dir / "test_license" / "video" / fname video_xml = etree.parse(video_file_path.open()) diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index fa8bfc4158..6c90a1cef3 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -611,7 +611,7 @@ class LoncapaProblem(object): """ includes = self.tree.findall('.//include') for inc in includes: - filename = inc.get('file') + filename = inc.get('file').decode('utf-8') if filename is not None: try: # open using LoncapaSystem OSFS filestore diff --git a/common/lib/capa/capa/tests/test_html_render.py b/common/lib/capa/capa/tests/test_html_render.py index fd9ccbfbe9..37c6d8bf59 100644 --- a/common/lib/capa/capa/tests/test_html_render.py +++ b/common/lib/capa/capa/tests/test_html_render.py @@ -43,8 +43,8 @@ class CapaHtmlRenderTest(unittest.TestCase): def test_include_html(self): # Create a test file to include self._create_test_file( - 'test_include.xml', - 'Test include' + u'test_include.xml', + u'Test include' ) # Generate some XML with an diff --git a/common/lib/xmodule/xmodule/assetstore/__init__.py b/common/lib/xmodule/xmodule/assetstore/__init__.py index 3960ea265a..c02d8d8c4c 100644 --- a/common/lib/xmodule/xmodule/assetstore/__init__.py +++ b/common/lib/xmodule/xmodule/assetstore/__init__.py @@ -45,10 +45,10 @@ class AssetMetadata(object): ASSET_XML_TAG = 'asset' # Top-level directory name in exported course XML which holds asset metadata. - EXPORTED_ASSET_DIR = 'assets' + EXPORTED_ASSET_DIR = u'assets' # Filename of all asset metadata exported as XML. - EXPORTED_ASSET_FILENAME = 'assets.xml' + EXPORTED_ASSET_FILENAME = u'assets.xml' @contract(asset_id='AssetKey', pathname='basestring|None', internal_name='basestring|None', diff --git a/common/lib/xmodule/xmodule/backcompat_module.py b/common/lib/xmodule/xmodule/backcompat_module.py index 71fcce8977..7b0fa690fa 100644 --- a/common/lib/xmodule/xmodule/backcompat_module.py +++ b/common/lib/xmodule/xmodule/backcompat_module.py @@ -24,7 +24,7 @@ def process_includes(fn): next_include = xml_object.find('include') while next_include is not None: system.error_tracker("WARNING: the tag is deprecated, and will go away.") - file = next_include.get('file') + file = next_include.get('file').decode('utf-8') parent = next_include.getparent() if file is None: diff --git a/common/lib/xmodule/xmodule/course_module.py b/common/lib/xmodule/xmodule/course_module.py index 6bc33cda24..97ea6d0ab9 100644 --- a/common/lib/xmodule/xmodule/course_module.py +++ b/common/lib/xmodule/xmodule/course_module.py @@ -1002,12 +1002,12 @@ class CourseDescriptor(CourseFields, SequenceDescriptor, LicenseMixin): policy_dir = None url_name = xml_obj.get('url_name', xml_obj.get('slug')) if url_name: - policy_dir = 'policies/' + url_name + policy_dir = u'policies/' + url_name # Try to load grading policy - paths = ['grading_policy.json'] + paths = [u'grading_policy.json'] if policy_dir: - paths = [policy_dir + '/grading_policy.json'] + paths + paths = [policy_dir + u'/grading_policy.json'] + paths try: policy = json.loads(cls.read_grading_policy(paths, system)) diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index 5f569e7fe1..5fe3aa92ff 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -7,7 +7,7 @@ import textwrap from datetime import datetime from django.conf import settings -from fs.errors import ResourceNotFoundError +from fs.errors import ResourceNotFound from lxml import etree from path import Path as path from pkg_resources import resource_string @@ -236,7 +236,7 @@ class HtmlDescriptor(HtmlBlock, XmlDescriptor, EditingDescriptor): # pylint: di ) base = path(pointer_path).dirname() # log.debug("base = {0}, base.dirname={1}, filename={2}".format(base, base.dirname(), filename)) - filepath = "{base}/{name}.html".format(base=base, name=filename) + filepath = u"{base}/{name}.html".format(base=base, name=filename) # log.debug("looking for html file for {0} at {1}".format(location, filepath)) # VS[compat] @@ -259,8 +259,8 @@ class HtmlDescriptor(HtmlBlock, XmlDescriptor, EditingDescriptor): # pylint: di break try: - with system.resources_fs.open(filepath) as infile: - html = infile.read().decode('utf-8') + with system.resources_fs.open(filepath, encoding='utf-8') as infile: + html = infile.read() # Log a warning if we can't parse the file, but don't error if not check_html(html) and len(html) > 0: msg = "Couldn't parse html in {0}, content = {1}".format(filepath, html) @@ -275,7 +275,7 @@ class HtmlDescriptor(HtmlBlock, XmlDescriptor, EditingDescriptor): # pylint: di return definition, [] - except (ResourceNotFoundError) as err: + except ResourceNotFound as err: msg = 'Unable to load file contents at path {0}: {1} '.format( filepath, err) # add more info and re-raise @@ -295,8 +295,8 @@ class HtmlDescriptor(HtmlBlock, XmlDescriptor, EditingDescriptor): # pylint: di pathname=pathname ) - resource_fs.makedir(os.path.dirname(filepath), recursive=True, allow_recreate=True) - with resource_fs.open(filepath, 'w') as filestream: + resource_fs.makedirs(os.path.dirname(filepath), recreate=True) + with resource_fs.open(filepath, 'wb') as filestream: html_data = self.data.encode('utf-8') filestream.write(html_data) diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py index 6fd1d96437..4bba5cfdca 100644 --- a/common/lib/xmodule/xmodule/modulestore/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/__init__.py @@ -34,8 +34,8 @@ new_contract('AssetKey', AssetKey) new_contract('AssetMetadata', AssetMetadata) new_contract('XBlock', XBlock) -LIBRARY_ROOT = 'library.xml' -COURSE_ROOT = 'course.xml' +LIBRARY_ROOT = u'library.xml' +COURSE_ROOT = u'course.xml' # List of names of computed fields on xmodules that are of type usage keys. # This list can be used to determine which fields need to be stripped of diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py b/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py index 3fce2798e3..01fa561a8b 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py @@ -39,7 +39,7 @@ COURSE_DATA_NAMES = ( 'split_test_module_draft', ) -EXPORTED_COURSE_DIR_NAME = 'exported_source_course' +EXPORTED_COURSE_DIR_NAME = u'exported_source_course' @ddt.ddt diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py index e1801de0ec..22119ae19f 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py @@ -572,7 +572,7 @@ class TestMongoModuleStore(TestMongoModuleStoreBase): root_dir = path(mkdtemp()) self.addCleanup(shutil.rmtree, root_dir) - export_course_to_xml(self.draft_store, self.content_store, course_key, root_dir, 'test_export') + export_course_to_xml(self.draft_store, self.content_store, course_key, root_dir, u'test_export') self.assertTrue(path(root_dir / 'test_export/static/images/course_image.jpg').isfile()) self.assertTrue(path(root_dir / 'test_export/static/images_course_image.jpg').isfile()) @@ -588,7 +588,7 @@ class TestMongoModuleStore(TestMongoModuleStoreBase): root_dir = path(mkdtemp()) self.addCleanup(shutil.rmtree, root_dir) - export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export') + export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, u'test_export') self.assertTrue(path(root_dir / 'test_export/static/just_a_test.jpg').isfile()) self.assertFalse(path(root_dir / 'test_export/static/images/course_image.jpg').isfile()) @@ -601,7 +601,7 @@ class TestMongoModuleStore(TestMongoModuleStoreBase): course = self.draft_store.get_course(CourseKey.from_string('edX/simple_with_draft/2012_Fall')) root_dir = path(mkdtemp()) self.addCleanup(shutil.rmtree, root_dir) - export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export') + export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, u'test_export') self.assertFalse(path(root_dir / 'test_export/static/images/course_image.jpg').isfile()) self.assertFalse(path(root_dir / 'test_export/static/images_course_image.jpg').isfile()) diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_publish.py b/common/lib/xmodule/xmodule/modulestore/tests/test_publish.py index d2c2df49ca..17cdebe83a 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_publish.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_publish.py @@ -497,8 +497,8 @@ class DraftPublishedOpBaseTestSetup(OLXFormatChecker, DraftPublishedOpTestCourse Setup base class for draft/published/OLX tests. """ - EXPORTED_COURSE_BEFORE_DIR_NAME = 'exported_course_before' - EXPORTED_COURSE_AFTER_DIR_NAME = 'exported_course_after_{}' + EXPORTED_COURSE_BEFORE_DIR_NAME = u'exported_course_before' + EXPORTED_COURSE_AFTER_DIR_NAME = u'exported_course_after_{}' def setUp(self): super(DraftPublishedOpBaseTestSetup, self).setUp() diff --git a/common/lib/xmodule/xmodule/modulestore/xml_exporter.py b/common/lib/xmodule/xmodule/modulestore/xml_exporter.py index 473236aefa..fb6fefeec2 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml_exporter.py +++ b/common/lib/xmodule/xmodule/modulestore/xml_exporter.py @@ -4,6 +4,7 @@ Methods for exporting course data to XML import logging from abc import abstractmethod +from six import text_type import lxml.etree from xblock.fields import Scope, Reference, ReferenceList, ReferenceValueDict from xmodule.contentstore.content import StaticContent @@ -42,7 +43,7 @@ def _export_drafts(modulestore, course_key, export_fs, xml_centric_course_key): # Only modules with changes will be exported into the /drafts directory. draft_modules = [module for module in draft_modules if modulestore.has_changes(module)] if draft_modules: - draft_course_dir = export_fs.makeopendir(DRAFT_DIR) + draft_course_dir = export_fs.makedir(DRAFT_DIR, recreate=True) # accumulate tuples of draft_modules and their parents in # this list: @@ -118,7 +119,7 @@ class ExportManager(object): self.contentstore = contentstore self.courselike_key = courselike_key self.root_dir = root_dir - self.target_dir = target_dir + self.target_dir = text_type(target_dir) @abstractmethod def get_key(self): @@ -160,7 +161,7 @@ class ExportManager(object): # export only the published content with self.modulestore.branch_setting(ModuleStoreEnum.Branch.published_only, self.courselike_key): courselike = self.get_courselike() - export_fs = courselike.runtime.export_fs = fsm.makeopendir(self.target_dir) + export_fs = courselike.runtime.export_fs = fsm.makedir(self.target_dir, recreate=True) # change all of the references inside the course to use the xml expected key type w/o version & branch xml_centric_courselike_key = self.get_key() @@ -196,8 +197,8 @@ class CourseExportManager(ExportManager): return self.modulestore.get_course(self.courselike_key, depth=None, lazy=False) def process_root(self, root, export_fs): - with export_fs.open('course.xml', 'w') as course_xml: - lxml.etree.ElementTree(root).write(course_xml) + with export_fs.open(u'course.xml', 'wb') as course_xml: + lxml.etree.ElementTree(root).write(course_xml, encoding='utf-8') def process_extra(self, root, courselike, root_courselike_dir, xml_centric_courselike_key, export_fs): # Export the modulestore's asset metadata. @@ -210,11 +211,11 @@ class CourseExportManager(ExportManager): # All asset types are exported using the "asset" tag - but their asset type is specified in each asset key. asset = lxml.etree.SubElement(asset_root, AssetMetadata.ASSET_XML_TAG) asset_md.to_xml(asset) - with OSFS(asset_dir).open(AssetMetadata.EXPORTED_ASSET_FILENAME, 'w') as asset_xml_file: - lxml.etree.ElementTree(asset_root).write(asset_xml_file) + with OSFS(asset_dir).open(AssetMetadata.EXPORTED_ASSET_FILENAME, 'wb') as asset_xml_file: + lxml.etree.ElementTree(asset_root).write(asset_xml_file, encoding='utf-8') # export the static assets - policies_dir = export_fs.makeopendir('policies') + policies_dir = export_fs.makedir('policies', recreate=True) if self.contentstore: self.contentstore.export_all_for_course( self.courselike_key, @@ -238,7 +239,7 @@ class CourseExportManager(ExportManager): output_dir = root_courselike_dir + '/static/images/' if not os.path.isdir(output_dir): os.makedirs(output_dir) - with OSFS(output_dir).open('course_image.jpg', 'wb') as course_image_file: + with OSFS(output_dir).open(u'course_image.jpg', 'wb') as course_image_file: course_image_file.write(course_image.data) # export the static tabs @@ -270,16 +271,17 @@ class CourseExportManager(ExportManager): # Use url_name for split mongo because course_run is not used when loading policies. course_policy_dir_name = courselike.url_name - course_run_policy_dir = policies_dir.makeopendir(course_policy_dir_name) + course_run_policy_dir = policies_dir.makedir(course_policy_dir_name, recreate=True) # export the grading policy - with course_run_policy_dir.open('grading_policy.json', 'w') as grading_policy: - grading_policy.write(dumps(courselike.grading_policy, cls=EdxJSONEncoder, sort_keys=True, indent=4)) + with course_run_policy_dir.open(u'grading_policy.json', 'wb') as grading_policy: + grading_policy.write(dumps(courselike.grading_policy, cls=EdxJSONEncoder, + sort_keys=True, indent=4).encode('utf-8')) # export all of the course metadata in policy.json - with course_run_policy_dir.open('policy.json', 'w') as course_policy: + with course_run_policy_dir.open(u'policy.json', 'wb') as course_policy: policy = {'course/' + courselike.location.name: own_metadata(courselike)} - course_policy.write(dumps(policy, cls=EdxJSONEncoder, sort_keys=True, indent=4)) + course_policy.write(dumps(policy, cls=EdxJSONEncoder, sort_keys=True, indent=4).encode('utf-8')) _export_drafts(self.modulestore, self.courselike_key, export_fs, xml_centric_courselike_key) @@ -315,7 +317,7 @@ class LibraryExportManager(ExportManager): to ease in duck typing during import. This may be expanded as a useful feature eventually. """ # export the static assets - export_fs.makeopendir('policies') + export_fs.makedir('policies', recreate=True) if self.contentstore: self.contentstore.export_all_for_course( @@ -332,7 +334,7 @@ class LibraryExportManager(ExportManager): called library.xml. """ # Create the Library.xml file, which acts as the index of all library contents. - xml_file = export_fs.open(LIBRARY_ROOT, 'w') + xml_file = export_fs.open(LIBRARY_ROOT, 'wb') xml_file.write(lxml.etree.tostring(root, pretty_print=True, encoding='utf-8')) xml_file.close() @@ -387,19 +389,20 @@ def _export_field_content(xblock_item, item_dir): for field_name in module_data: if field_name not in DEFAULT_CONTENT_FIELDS: # filename format: {dirname}.{field_name}.json - with item_dir.open('{0}.{1}.{2}'.format(xblock_item.location.name, field_name, 'json'), - 'w') as field_content_file: - field_content_file.write(dumps(module_data.get(field_name, {}), cls=EdxJSONEncoder, sort_keys=True, indent=4)) + with item_dir.open(u'{0}.{1}.{2}'.format(xblock_item.location.name, field_name, 'json'), + 'wb') as field_content_file: + field_content_file.write(dumps(module_data.get(field_name, {}), cls=EdxJSONEncoder, + sort_keys=True, indent=4).encode('utf-8')) def export_extra_content(export_fs, modulestore, source_course_key, dest_course_key, category_type, dirname, file_suffix=''): items = modulestore.get_items(source_course_key, qualifiers={'category': category_type}) if len(items) > 0: - item_dir = export_fs.makeopendir(dirname) + item_dir = export_fs.makedir(dirname, recreate=True) for item in items: adapt_references(item, dest_course_key, export_fs) - with item_dir.open(item.location.name + file_suffix, 'w') as item_file: + with item_dir.open(item.location.name + file_suffix, 'wb') as item_file: item_file.write(item.data.encode('utf8')) # export content fields other then metadata and data in json format in current directory diff --git a/common/lib/xmodule/xmodule/tests/test_export.py b/common/lib/xmodule/xmodule/tests/test_export.py index c43e3e4cb1..fa98a23f4c 100644 --- a/common/lib/xmodule/xmodule/tests/test_export.py +++ b/common/lib/xmodule/xmodule/tests/test_export.py @@ -71,14 +71,14 @@ class RoundTripTestCase(unittest.TestCase): @mock.patch('xmodule.video_module.video_module.edxval_api', None) @mock.patch('xmodule.course_module.requests.get') @ddt.data( - "toy", - "simple", - "conditional_and_poll", - "conditional", - "self_assessment", - "test_exam_registration", - "word_cloud", - "pure_xblock", + u"toy", + u"simple", + u"conditional_and_poll", + u"conditional", + u"self_assessment", + u"test_exam_registration", + u"word_cloud", + u"pure_xblock", ) @XBlock.register_temp_plugin(PureXBlock, 'pure') def test_export_roundtrip(self, course_dir, mock_get): @@ -107,12 +107,12 @@ class RoundTripTestCase(unittest.TestCase): # will still be there. print "Starting export" file_system = OSFS(root_dir) - initial_course.runtime.export_fs = file_system.makeopendir(course_dir) + initial_course.runtime.export_fs = file_system.makedir(course_dir, recreate=True) root = lxml.etree.Element('root') initial_course.add_xml_to_node(root) - with initial_course.runtime.export_fs.open('course.xml', 'w') as course_xml: - lxml.etree.ElementTree(root).write(course_xml) + with initial_course.runtime.export_fs.open('course.xml', 'wb') as course_xml: + lxml.etree.ElementTree(root).write(course_xml, encoding='utf-8') print "Starting second import" second_import = XMLModuleStore(root_dir, source_dirs=[course_dir], xblock_mixins=(XModuleMixin,)) diff --git a/common/lib/xmodule/xmodule/tests/test_import.py b/common/lib/xmodule/xmodule/tests/test_import.py index 52d4a9e630..9bed263d2d 100644 --- a/common/lib/xmodule/xmodule/tests/test_import.py +++ b/common/lib/xmodule/xmodule/tests/test_import.py @@ -219,7 +219,7 @@ class ImportTestCase(BaseCourseTestCase): self.assertEqual(node.attrib['org'], ORG) # Does the course still have unicorns? - with descriptor.runtime.export_fs.open('course/{url_name}.xml'.format(url_name=url_name)) as f: + with descriptor.runtime.export_fs.open(u'course/{url_name}.xml'.format(url_name=url_name)) as f: course_xml = etree.fromstring(f.read()) self.assertEqual(course_xml.attrib['unicorn'], unicorn_color) @@ -233,7 +233,7 @@ class ImportTestCase(BaseCourseTestCase): # Does the chapter tag now have a due attribute? # hardcoded path to child - with descriptor.runtime.export_fs.open('chapter/ch.xml') as f: + with descriptor.runtime.export_fs.open(u'chapter/ch.xml') as f: chapter_xml = etree.fromstring(f.read()) self.assertEqual(chapter_xml.tag, 'chapter') self.assertNotIn('due', chapter_xml.attrib) diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index 053b456cbc..b56847532f 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -475,8 +475,8 @@ class XmlParserMixin(object): # Write the definition to a file url_path = name_to_pathname(self.url_name) filepath = self._format_filepath(self.category, url_path) - self.runtime.export_fs.makedir(os.path.dirname(filepath), recursive=True, allow_recreate=True) - with self.runtime.export_fs.open(filepath, 'w') as fileobj: + self.runtime.export_fs.makedirs(os.path.dirname(filepath), recreate=True) + with self.runtime.export_fs.open(filepath, 'wb') as fileobj: ElementTree(xml_object).write(fileobj, pretty_print=True, encoding='utf-8') else: # Write all attributes from xml_object onto node diff --git a/lms/djangoapps/courseware/courses.py b/lms/djangoapps/courseware/courses.py index 0edf341c63..9351db60c0 100644 --- a/lms/djangoapps/courseware/courses.py +++ b/lms/djangoapps/courseware/courses.py @@ -25,7 +25,7 @@ from django.core.urlresolvers import reverse from django.http import Http404, QueryDict from enrollment.api import get_course_enrollment_details from edxmako.shortcuts import render_to_string -from fs.errors import ResourceNotFoundError +from fs.errors import ResourceNotFound from lms.djangoapps.courseware.courseware_access_exception import CoursewareAccessException from lms.djangoapps.courseware.exceptions import CourseAccessRedirect from opaque_keys.edx.keys import UsageKey @@ -207,13 +207,13 @@ def find_file(filesystem, dirs, filename): dirs: a list of path objects filename: a string - Returns d / filename if found in dir d, else raises ResourceNotFoundError. + Returns d / filename if found in dir d, else raises ResourceNotFound. """ for directory in dirs: filepath = path(directory) / filename if filesystem.exists(filepath): return filepath - raise ResourceNotFoundError(u"Could not find {0}".format(filename)) + raise ResourceNotFound(u"Could not find {0}".format(filename)) def get_course_about_section(request, course, section_key): @@ -419,7 +419,7 @@ def get_course_syllabus_section(course, section_key): course_id=course.id, static_asset_path=course.static_asset_path, ) - except ResourceNotFoundError: + except ResourceNotFound: log.exception( u"Missing syllabus section %s in course %s", section_key, course.location.to_deprecated_string() diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 9bf733d074..8f00520896 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -4,12 +4,15 @@ # * @edx/ospr - to check licensing # * @edx/devops - to check system requirements +appdirs==1.4.3 attrs==17.2.0 beautifulsoup4==4.1.3 beautifulsoup==3.2.1 bleach==1.4 html5lib==0.999 boto==2.39.0 +boto3==1.4.8 +botocore==1.8.17 celery==3.1.18 cryptography==1.9 cssselect==0.9.1 @@ -25,7 +28,7 @@ django-model-utils==3.0.0 django-mptt>=0.8.6,<0.9 django-oauth-toolkit==0.12.0 django-pipeline-forgiving==1.0.0 -django-pyfs==1.0.7 +django-pyfs==2.0 django-sekizai>=0.10 django-ses==0.8.4 django-simple-history==1.9.0 @@ -35,6 +38,7 @@ django-method-override==0.1.0 django-user-tasks==0.1.5 django-waffle==0.12.0 djangorestframework-jwt==1.11.0 +docutils==0.14 enum34==1.1.6 edx-ace==0.1.6 edx-ccx-keys==0.2.1 @@ -59,11 +63,15 @@ edxval==0.1.6 event-tracking==0.2.4 feedparser==5.1.3 firebase-token-generator==1.3.2 +fs==2.0.17 +fs-s3fs==0.1.5 +futures==3.2.0 ; python_version == "2.7" GitPython==0.3.2.RC1 glob2==0.3 gunicorn==0.17.4 help-tokens==1.0.3 httpretty==0.8.3 +jmespath==0.9.3 lazy==1.1 mako==1.0.2 Markdown>=2.6,<2.7 @@ -95,6 +103,7 @@ pysrt==0.4.7 PyYAML==3.12 requests-oauthlib==0.4.1 rules==1.1.1 +s3transfer==0.1.12 scipy==0.14.0 Shapely==1.2.16 singledispatch==3.4.0.2 diff --git a/requirements/edx/github.txt b/requirements/edx/github.txt index afad0a1104..b15af862fa 100644 --- a/requirements/edx/github.txt +++ b/requirements/edx/github.txt @@ -57,10 +57,6 @@ git+https://github.com/edx/nltk.git@2.0.6#egg=nltk==2.0.6 -e git+https://github.com/jazkarta/edx-jsme.git@690dbf75441fa91c7c4899df0b83d77f7deb5458#egg=edx-jsme git+https://github.com/mitodl/django-cas.git@afac57bc523f145ae826f4ed3d4fa8b2c86c5364#egg=django-cas==2.1.1 -e git+https://github.com/dgrtwo/ParsePy.git@7949b9f754d1445eff8e8f20d0e967b9a6420639#egg=parse_rest -# Master pyfs has a bug working with VPC auth. This is a fix. We should switch -# back to master when and if this fix is merged back. -# fs==0.4.0 -git+https://github.com/edx/pyfs.git@96e1922348bfe6d99201b9512a9ed946c87b7e0b#egg=fs==0.4.0 # The officially released version of django-debug-toolbar-mongo doesn't support DJDT 1.x. This commit does. git+https://github.com/hmarr/django-debug-toolbar-mongo.git@b0686a76f1ce3532088c4aee6e76b9abe61cc808#egg=django-debug-toolbar-mongo==0.1.10 @@ -91,7 +87,7 @@ git+https://github.com/edx/django-celery.git@756cb57aad765cb2b0d37372c1855b8f5f3 -e git+https://github.com/edx/django-splash.git@v0.2#egg=django-splash==0.2 -e git+https://github.com/edx/acid-block.git@e46f9cda8a03e121a00c7e347084d142d22ebfb7#egg=acid-xblock git+https://github.com/edx/edx-ora2.git@2.1.8#egg=ora2==2.1.8 -git+https://github.com/edx/RecommenderXBlock.git@0e744b393cf1f8b886fe77bc697e7d9d78d65cd6#egg=recommender-xblock==1.2 +git+https://github.com/edx/RecommenderXBlock.git@1.3#egg=recommender-xblock==1.3 git+https://github.com/solashirai/crowdsourcehinter.git@518605f0a95190949fe77bd39158450639e2e1dc#egg=crowdsourcehinter-xblock==0.1 -e git+https://github.com/edx/RateXBlock.git@367e19c0f6eac8a5f002fd0f1559555f8e74bfff#egg=rate-xblock -e git+https://github.com/edx/DoneXBlock.git@01a14f3bd80ae47dd08cdbbe2f88f3eb88d00fba#egg=done-xblock