Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PLAT-1861 Upgrade to pyfilesystem2

Browse Source
This commit is contained in:
Jeremy Bowman
2017-12-20 11:21:23 -05:00
parent e063a8bdd8
commit 65472399df
18 changed files with 122 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
cms/djangoapps/contentstore/tests/test_contentstore.py
View File

@@ -23,6 +23,7 @@ from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey, UsageKey
from opaque_keys.edx.locations import AssetLocation, CourseLocator
from path import Path as path
from six import text_type
from waffle.testutils import override_switch
from contentstore.tests.utils import AjaxEnabledTestClient, CourseTestCase, get_url, parse_json
@@ -117,7 +118,7 @@ class ImportRequiredTestCases(ContentStoreTestCase):
Asset name in XML: "/invalid\\displayname/subs-esLhHcdKGWvKs.srt"
"""
content_store = contentstore()
expected_displayname = '_invalid_displayname_subs-esLhHcdKGWvKs.srt'
expected_displayname = u'_invalid_displayname_subs-esLhHcdKGWvKs.srt'
import_course_from_xml(
self.store,
@@ -156,10 +157,10 @@ class ImportRequiredTestCases(ContentStoreTestCase):
# Test course export does not fail
root_dir = path(mkdtemp_clean())
print 'Exporting to tempdir = {0}'.format(root_dir)
export_course_to_xml(self.store, content_store, course.id, root_dir, 'test_export')
export_course_to_xml(self.store, content_store, course.id, root_dir, u'test_export')
filesystem = OSFS(root_dir / 'test_export/static')
exported_static_files = filesystem.listdir()
filesystem = OSFS(text_type(root_dir / 'test_export/static'))
exported_static_files = filesystem.listdir(u'/')
# Verify that asset have been overwritten during export.
self.assertEqual(len(exported_static_files), 1)
@@ -236,18 +237,18 @@ class ImportRequiredTestCases(ContentStoreTestCase):
self.assertIsNotNone(course_updates)
# check that course which is imported has files 'updates.html' and 'updates.items.json'
filesystem = OSFS(data_dir + '/course_info_updates/info')
self.assertTrue(filesystem.exists('updates.html'))
self.assertTrue(filesystem.exists('updates.items.json'))
filesystem = OSFS(text_type(data_dir + '/course_info_updates/info'))
self.assertTrue(filesystem.exists(u'updates.html'))
self.assertTrue(filesystem.exists(u'updates.items.json'))
# verify that course info update module has same data content as in data file from which it is imported
# check 'data' field content
with filesystem.open('updates.html', 'r') as course_policy:
with filesystem.open(u'updates.html', 'r') as course_policy:
on_disk = course_policy.read()
self.assertEqual(course_updates.data, on_disk)
# check 'items' field content
with filesystem.open('updates.items.json', 'r') as course_policy:
with filesystem.open(u'updates.items.json', 'r') as course_policy:
on_disk = loads(course_policy.read())
self.assertEqual(course_updates.items, on_disk)
@@ -255,19 +256,19 @@ class ImportRequiredTestCases(ContentStoreTestCase):
# with same content as in course 'info' directory
root_dir = path(mkdtemp_clean())
print 'Exporting to tempdir = {0}'.format(root_dir)
export_course_to_xml(self.store, content_store, course.id, root_dir, 'test_export')
export_course_to_xml(self.store, content_store, course.id, root_dir, u'test_export')
# check that exported course has files 'updates.html' and 'updates.items.json'
filesystem = OSFS(root_dir / 'test_export/info')
self.assertTrue(filesystem.exists('updates.html'))
self.assertTrue(filesystem.exists('updates.items.json'))
filesystem = OSFS(text_type(root_dir / 'test_export/info'))
self.assertTrue(filesystem.exists(u'updates.html'))
self.assertTrue(filesystem.exists(u'updates.items.json'))
# verify that exported course has same data content as in course_info_update module
with filesystem.open('updates.html', 'r') as grading_policy:
with filesystem.open(u'updates.html', 'r') as grading_policy:
on_disk = grading_policy.read()
self.assertEqual(on_disk, course_updates.data)
with filesystem.open('updates.items.json', 'r') as grading_policy:
with filesystem.open(u'updates.items.json', 'r') as grading_policy:
on_disk = loads(grading_policy.read())
self.assertEqual(on_disk, course_updates.items)
@@ -315,39 +316,39 @@ class ImportRequiredTestCases(ContentStoreTestCase):
print 'Exporting to tempdir = {0}'.format(root_dir)
# export out to a tempdir
export_course_to_xml(self.store, content_store, course_id, root_dir, 'test_export')
export_course_to_xml(self.store, content_store, course_id, root_dir, u'test_export')
# check for static tabs
self.verify_content_existence(self.store, root_dir, course_id, 'tabs', 'static_tab', '.html')
self.verify_content_existence(self.store, root_dir, course_id, u'tabs', 'static_tab', '.html')
# check for about content
self.verify_content_existence(self.store, root_dir, course_id, 'about', 'about', '.html')
self.verify_content_existence(self.store, root_dir, course_id, u'about', 'about', '.html')
# assert that there is an html and video directory in drafts:
draft_dir = OSFS(root_dir / 'test_export/drafts')
self.assertTrue(draft_dir.exists('html'))
self.assertTrue(draft_dir.exists('video'))
self.assertTrue(draft_dir.exists(u'html'))
self.assertTrue(draft_dir.exists(u'video'))
# and assert that they contain the created modules
self.assertIn(self.DRAFT_HTML + ".xml", draft_dir.listdir('html'))
self.assertIn(self.DRAFT_VIDEO + ".xml", draft_dir.listdir('video'))
self.assertIn(self.DRAFT_HTML + ".xml", draft_dir.listdir(u'html'))
self.assertIn(self.DRAFT_VIDEO + ".xml", draft_dir.listdir(u'video'))
# and assert the child of the orphaned draft wasn't exported
self.assertNotIn(self.ORPHAN_DRAFT_HTML + ".xml", draft_dir.listdir('html'))
self.assertNotIn(self.ORPHAN_DRAFT_HTML + ".xml", draft_dir.listdir(u'html'))
# check for grading_policy.json
filesystem = OSFS(root_dir / 'test_export/policies/2012_Fall')
self.assertTrue(filesystem.exists('grading_policy.json'))
self.assertTrue(filesystem.exists(u'grading_policy.json'))
course = self.store.get_course(course_id)
# compare what's on disk compared to what we have in our course
with filesystem.open('grading_policy.json', 'r') as grading_policy:
with filesystem.open(u'grading_policy.json', 'r') as grading_policy:
on_disk = loads(grading_policy.read())
self.assertEqual(on_disk, course.grading_policy)
# check for policy.json
self.assertTrue(filesystem.exists('policy.json'))
self.assertTrue(filesystem.exists(u'policy.json'))
# compare what's on disk to what we have in the course module
with filesystem.open('policy.json', 'r') as course_policy:
with filesystem.open(u'policy.json', 'r') as course_policy:
on_disk = loads(course_policy.read())
self.assertIn('course/2012_Fall', on_disk)
self.assertEqual(on_disk['course/2012_Fall'], own_metadata(course))
@@ -417,7 +418,7 @@ class ImportRequiredTestCases(ContentStoreTestCase):
print 'Exporting to tempdir = {0}'.format(root_dir)
# export out to a tempdir
export_course_to_xml(self.store, content_store, course_id, root_dir, 'test_export')
export_course_to_xml(self.store, content_store, course_id, root_dir, u'test_export')
shutil.rmtree(root_dir)
@@ -443,7 +444,7 @@ class ImportRequiredTestCases(ContentStoreTestCase):
print 'Exporting to tempdir = {0}'.format(root_dir)
# export out to a tempdir
export_course_to_xml(self.store, content_store, course_id, root_dir, 'test_export')
export_course_to_xml(self.store, content_store, course_id, root_dir, u'test_export')
shutil.rmtree(root_dir)
@@ -496,7 +497,7 @@ class ImportRequiredTestCases(ContentStoreTestCase):
# Export the course
root_dir = path(mkdtemp_clean())
export_course_to_xml(self.store, content_store, course_id, root_dir, 'test_roundtrip')
export_course_to_xml(self.store, content_store, course_id, root_dir, u'test_roundtrip')
# Reimport and get the video back
import_course_from_xml(self.store, self.user.id, root_dir)
@@ -517,7 +518,7 @@ class ImportRequiredTestCases(ContentStoreTestCase):
# Export the course
root_dir = path(mkdtemp_clean())
export_course_to_xml(self.store, content_store, course_id, root_dir, 'test_roundtrip')
export_course_to_xml(self.store, content_store, course_id, root_dir, u'test_roundtrip')
# Reimport and get the video back
import_course_from_xml(self.store, self.user.id, root_dir, create_if_not_present=True)
@@ -541,7 +542,7 @@ class ImportRequiredTestCases(ContentStoreTestCase):
root_dir = path(mkdtemp_clean())
print 'Exporting to tempdir = {0}'.format(root_dir)
export_course_to_xml(self.store, None, course_id, root_dir, 'test_export_no_content_store')
export_course_to_xml(self.store, None, course_id, root_dir, u'test_export_no_content_store')
# Delete the course from module store and reimport it
@@ -596,7 +597,7 @@ class ImportRequiredTestCases(ContentStoreTestCase):
content_store,
course_id,
root_dir,
'test_no_xml_attributes'
u'test_no_xml_attributes'
)
@@ -695,7 +696,7 @@ class MiscCourseTests(ContentStoreTestCase):
def test_export_on_invalid_displayname(self, invalid_displayname):
""" Tests that assets with invalid 'displayname' does not cause export to fail """
content_store = contentstore()
exported_asset_name = '_Fake_asset_displayname'
exported_asset_name = u'_Fake_asset_displayname'
# Create an asset with slash `invalid_displayname` '
asset_key = self.course.id.make_asset_key('asset', "fake_asset.txt")
@@ -713,10 +714,10 @@ class MiscCourseTests(ContentStoreTestCase):
# Now export the course to a tempdir and test that it contains assets. The export should pass
root_dir = path(mkdtemp_clean())
print 'Exporting to tempdir = {0}'.format(root_dir)
export_course_to_xml(self.store, content_store, self.course.id, root_dir, 'test_export')
export_course_to_xml(self.store, content_store, self.course.id, root_dir, u'test_export')
filesystem = OSFS(root_dir / 'test_export/static')
exported_static_files = filesystem.listdir()
exported_static_files = filesystem.listdir(u'/')
# Verify that only single asset has been exported with the expected asset name.
self.assertTrue(filesystem.exists(exported_asset_name))
@@ -737,13 +738,13 @@ class MiscCourseTests(ContentStoreTestCase):
# Make an existing unit a draft
self.store.convert_to_draft(self.problem.location, self.user.id)
root_dir = path(mkdtemp_clean())
export_course_to_xml(self.store, None, self.course.id, root_dir, 'test_export')
export_course_to_xml(self.store, None, self.course.id, root_dir, u'test_export')
# Verify that problem is exported in the drafts. This is expected because we are
# mocking get_item to for drafts. Expect no draft is exported.
# Specifically get_item is used in `xmodule.modulestore.xml_exporter._export_drafts`
export_draft_dir = OSFS(root_dir / 'test_export/drafts')
self.assertEqual(len(export_draft_dir.listdir()), 0)
self.assertEqual(len(export_draft_dir.listdir(u'/')), 0)
# Remove tempdir
shutil.rmtree(root_dir)
@@ -751,7 +752,7 @@ class MiscCourseTests(ContentStoreTestCase):
def test_assets_overwrite(self):
""" Tests that assets will similar 'displayname' will be overwritten during export """
content_store = contentstore()
asset_displayname = 'Fake_asset.txt'
asset_displayname = u'Fake_asset.txt'
# Create two assets with similar 'displayname'
for i in range(2):
@@ -773,11 +774,11 @@ class MiscCourseTests(ContentStoreTestCase):
# Now export the course to a tempdir and test that it contains assets.
root_dir = path(mkdtemp_clean())
print 'Exporting to tempdir = {0}'.format(root_dir)
export_course_to_xml(self.store, content_store, self.course.id, root_dir, 'test_export')
export_course_to_xml(self.store, content_store, self.course.id, root_dir, u'test_export')
# Verify that asset have been overwritten during export.
filesystem = OSFS(root_dir / 'test_export/static')
exported_static_files = filesystem.listdir()
exported_static_files = filesystem.listdir(u'/')
self.assertTrue(filesystem.exists(asset_displayname))
self.assertEqual(len(exported_static_files), 1)
@@ -2109,7 +2110,7 @@ class ContentLicenseTest(ContentStoreTestCase):
root_dir = path(mkdtemp_clean())
self.course.license = "creative-commons: BY SA"
self.store.update_item(self.course, None)
export_course_to_xml(self.store, content_store, self.course.id, root_dir, 'test_license')
export_course_to_xml(self.store, content_store, self.course.id, root_dir, u'test_license')
fname = "{block}.xml".format(block=self.course.scope_ids.usage_id.block_id)
run_file_path = root_dir / "test_license" / "course" / fname
run_xml = etree.parse(run_file_path.open())
@@ -2122,7 +2123,7 @@ class ContentLicenseTest(ContentStoreTestCase):
parent_location=self.course.location, category='video',
license="all-rights-reserved"
)
export_course_to_xml(self.store, content_store, self.course.id, root_dir, 'test_license')
export_course_to_xml(self.store, content_store, self.course.id, root_dir, u'test_license')
fname = "{block}.xml".format(block=video_descriptor.scope_ids.usage_id.block_id)
video_file_path = root_dir / "test_license" / "video" / fname
video_xml = etree.parse(video_file_path.open())

2
common/lib/capa/capa/capa_problem.py
View File

@@ -611,7 +611,7 @@ class LoncapaProblem(object):
"""
includes = self.tree.findall('.//include')
for inc in includes:
filename = inc.get('file')
filename = inc.get('file').decode('utf-8')
if filename is not None:
try:
# open using LoncapaSystem OSFS filestore

4
common/lib/capa/capa/tests/test_html_render.py
View File

@@ -43,8 +43,8 @@ class CapaHtmlRenderTest(unittest.TestCase):
def test_include_html(self):
# Create a test file to include
self._create_test_file(
'test_include.xml',
'<test>Test include</test>'
u'test_include.xml',
u'<test>Test include</test>'
)
# Generate some XML with an <include>

4
common/lib/xmodule/xmodule/assetstore/__init__.py
View File

@@ -45,10 +45,10 @@ class AssetMetadata(object):
ASSET_XML_TAG = 'asset'
# Top-level directory name in exported course XML which holds asset metadata.
EXPORTED_ASSET_DIR = 'assets'
EXPORTED_ASSET_DIR = u'assets'
# Filename of all asset metadata exported as XML.
EXPORTED_ASSET_FILENAME = 'assets.xml'
EXPORTED_ASSET_FILENAME = u'assets.xml'
@contract(asset_id='AssetKey',
pathname='basestring|None', internal_name='basestring|None',

2
common/lib/xmodule/xmodule/backcompat_module.py
View File

@@ -24,7 +24,7 @@ def process_includes(fn):
next_include = xml_object.find('include')
while next_include is not None:
system.error_tracker("WARNING: the <include> tag is deprecated, and will go away.")
file = next_include.get('file')
file = next_include.get('file').decode('utf-8')
parent = next_include.getparent()
if file is None:

6
common/lib/xmodule/xmodule/course_module.py
View File

@@ -1002,12 +1002,12 @@ class CourseDescriptor(CourseFields, SequenceDescriptor, LicenseMixin):
policy_dir = None
url_name = xml_obj.get('url_name', xml_obj.get('slug'))
if url_name:
policy_dir = 'policies/' + url_name
policy_dir = u'policies/' + url_name
# Try to load grading policy
paths = ['grading_policy.json']
paths = [u'grading_policy.json']
if policy_dir:
paths = [policy_dir + '/grading_policy.json'] + paths
paths = [policy_dir + u'/grading_policy.json'] + paths
try:
policy = json.loads(cls.read_grading_policy(paths, system))

14
common/lib/xmodule/xmodule/html_module.py
View File

@@ -7,7 +7,7 @@ import textwrap
from datetime import datetime
from django.conf import settings
from fs.errors import ResourceNotFoundError
from fs.errors import ResourceNotFound
from lxml import etree
from path import Path as path
from pkg_resources import resource_string
@@ -236,7 +236,7 @@ class HtmlDescriptor(HtmlBlock, XmlDescriptor, EditingDescriptor): # pylint: di
)
base = path(pointer_path).dirname()
# log.debug("base = {0}, base.dirname={1}, filename={2}".format(base, base.dirname(), filename))
filepath = "{base}/{name}.html".format(base=base, name=filename)
filepath = u"{base}/{name}.html".format(base=base, name=filename)
# log.debug("looking for html file for {0} at {1}".format(location, filepath))
# VS[compat]
@@ -259,8 +259,8 @@ class HtmlDescriptor(HtmlBlock, XmlDescriptor, EditingDescriptor): # pylint: di
break
try:
with system.resources_fs.open(filepath) as infile:
html = infile.read().decode('utf-8')
with system.resources_fs.open(filepath, encoding='utf-8') as infile:
html = infile.read()
# Log a warning if we can't parse the file, but don't error
if not check_html(html) and len(html) > 0:
msg = "Couldn't parse html in {0}, content = {1}".format(filepath, html)
@@ -275,7 +275,7 @@ class HtmlDescriptor(HtmlBlock, XmlDescriptor, EditingDescriptor): # pylint: di
return definition, []
except (ResourceNotFoundError) as err:
except ResourceNotFound as err:
msg = 'Unable to load file contents at path {0}: {1} '.format(
filepath, err)
# add more info and re-raise
@@ -295,8 +295,8 @@ class HtmlDescriptor(HtmlBlock, XmlDescriptor, EditingDescriptor): # pylint: di
pathname=pathname
)
resource_fs.makedir(os.path.dirname(filepath), recursive=True, allow_recreate=True)
with resource_fs.open(filepath, 'w') as filestream:
resource_fs.makedirs(os.path.dirname(filepath), recreate=True)
with resource_fs.open(filepath, 'wb') as filestream:
html_data = self.data.encode('utf-8')
filestream.write(html_data)

4
common/lib/xmodule/xmodule/modulestore/__init__.py
View File

@@ -34,8 +34,8 @@ new_contract('AssetKey', AssetKey)
new_contract('AssetMetadata', AssetMetadata)
new_contract('XBlock', XBlock)
LIBRARY_ROOT = 'library.xml'
COURSE_ROOT = 'course.xml'
LIBRARY_ROOT = u'library.xml'
COURSE_ROOT = u'course.xml'
# List of names of computed fields on xmodules that are of type usage keys.
# This list can be used to determine which fields need to be stripped of

2
common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py
View File

@@ -39,7 +39,7 @@ COURSE_DATA_NAMES = (
'split_test_module_draft',
)
EXPORTED_COURSE_DIR_NAME = 'exported_source_course'
EXPORTED_COURSE_DIR_NAME = u'exported_source_course'
@ddt.ddt

6
common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py
View File

@@ -572,7 +572,7 @@ class TestMongoModuleStore(TestMongoModuleStoreBase):
root_dir = path(mkdtemp())
self.addCleanup(shutil.rmtree, root_dir)
export_course_to_xml(self.draft_store, self.content_store, course_key, root_dir, 'test_export')
export_course_to_xml(self.draft_store, self.content_store, course_key, root_dir, u'test_export')
self.assertTrue(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
self.assertTrue(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
@@ -588,7 +588,7 @@ class TestMongoModuleStore(TestMongoModuleStoreBase):
root_dir = path(mkdtemp())
self.addCleanup(shutil.rmtree, root_dir)
export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export')
export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, u'test_export')
self.assertTrue(path(root_dir / 'test_export/static/just_a_test.jpg').isfile())
self.assertFalse(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
@@ -601,7 +601,7 @@ class TestMongoModuleStore(TestMongoModuleStoreBase):
course = self.draft_store.get_course(CourseKey.from_string('edX/simple_with_draft/2012_Fall'))
root_dir = path(mkdtemp())
self.addCleanup(shutil.rmtree, root_dir)
export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export')
export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, u'test_export')
self.assertFalse(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
self.assertFalse(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())

4
common/lib/xmodule/xmodule/modulestore/tests/test_publish.py
View File

@@ -497,8 +497,8 @@ class DraftPublishedOpBaseTestSetup(OLXFormatChecker, DraftPublishedOpTestCourse
Setup base class for draft/published/OLX tests.
"""
EXPORTED_COURSE_BEFORE_DIR_NAME = 'exported_course_before'
EXPORTED_COURSE_AFTER_DIR_NAME = 'exported_course_after_{}'
EXPORTED_COURSE_BEFORE_DIR_NAME = u'exported_course_before'
EXPORTED_COURSE_AFTER_DIR_NAME = u'exported_course_after_{}'
def setUp(self):
super(DraftPublishedOpBaseTestSetup, self).setUp()

45
common/lib/xmodule/xmodule/modulestore/xml_exporter.py
View File

@@ -4,6 +4,7 @@ Methods for exporting course data to XML
import logging
from abc import abstractmethod
from six import text_type
import lxml.etree
from xblock.fields import Scope, Reference, ReferenceList, ReferenceValueDict
from xmodule.contentstore.content import StaticContent
@@ -42,7 +43,7 @@ def _export_drafts(modulestore, course_key, export_fs, xml_centric_course_key):
# Only modules with changes will be exported into the /drafts directory.
draft_modules = [module for module in draft_modules if modulestore.has_changes(module)]
if draft_modules:
draft_course_dir = export_fs.makeopendir(DRAFT_DIR)
draft_course_dir = export_fs.makedir(DRAFT_DIR, recreate=True)
# accumulate tuples of draft_modules and their parents in
# this list:
@@ -118,7 +119,7 @@ class ExportManager(object):
self.contentstore = contentstore
self.courselike_key = courselike_key
self.root_dir = root_dir
self.target_dir = target_dir
self.target_dir = text_type(target_dir)
@abstractmethod
def get_key(self):
@@ -160,7 +161,7 @@ class ExportManager(object):
# export only the published content
with self.modulestore.branch_setting(ModuleStoreEnum.Branch.published_only, self.courselike_key):
courselike = self.get_courselike()
export_fs = courselike.runtime.export_fs = fsm.makeopendir(self.target_dir)
export_fs = courselike.runtime.export_fs = fsm.makedir(self.target_dir, recreate=True)
# change all of the references inside the course to use the xml expected key type w/o version & branch
xml_centric_courselike_key = self.get_key()
@@ -196,8 +197,8 @@ class CourseExportManager(ExportManager):
return self.modulestore.get_course(self.courselike_key, depth=None, lazy=False)
def process_root(self, root, export_fs):
with export_fs.open('course.xml', 'w') as course_xml:
lxml.etree.ElementTree(root).write(course_xml)
with export_fs.open(u'course.xml', 'wb') as course_xml:
lxml.etree.ElementTree(root).write(course_xml, encoding='utf-8')
def process_extra(self, root, courselike, root_courselike_dir, xml_centric_courselike_key, export_fs):
# Export the modulestore's asset metadata.
@@ -210,11 +211,11 @@ class CourseExportManager(ExportManager):
# All asset types are exported using the "asset" tag - but their asset type is specified in each asset key.
asset = lxml.etree.SubElement(asset_root, AssetMetadata.ASSET_XML_TAG)
asset_md.to_xml(asset)
with OSFS(asset_dir).open(AssetMetadata.EXPORTED_ASSET_FILENAME, 'w') as asset_xml_file:
lxml.etree.ElementTree(asset_root).write(asset_xml_file)
with OSFS(asset_dir).open(AssetMetadata.EXPORTED_ASSET_FILENAME, 'wb') as asset_xml_file:
lxml.etree.ElementTree(asset_root).write(asset_xml_file, encoding='utf-8')
# export the static assets
policies_dir = export_fs.makeopendir('policies')
policies_dir = export_fs.makedir('policies', recreate=True)
if self.contentstore:
self.contentstore.export_all_for_course(
self.courselike_key,
@@ -238,7 +239,7 @@ class CourseExportManager(ExportManager):
output_dir = root_courselike_dir + '/static/images/'
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
with OSFS(output_dir).open('course_image.jpg', 'wb') as course_image_file:
with OSFS(output_dir).open(u'course_image.jpg', 'wb') as course_image_file:
course_image_file.write(course_image.data)
# export the static tabs
@@ -270,16 +271,17 @@ class CourseExportManager(ExportManager):
# Use url_name for split mongo because course_run is not used when loading policies.
course_policy_dir_name = courselike.url_name
course_run_policy_dir = policies_dir.makeopendir(course_policy_dir_name)
course_run_policy_dir = policies_dir.makedir(course_policy_dir_name, recreate=True)
# export the grading policy
with course_run_policy_dir.open('grading_policy.json', 'w') as grading_policy:
grading_policy.write(dumps(courselike.grading_policy, cls=EdxJSONEncoder, sort_keys=True, indent=4))
with course_run_policy_dir.open(u'grading_policy.json', 'wb') as grading_policy:
grading_policy.write(dumps(courselike.grading_policy, cls=EdxJSONEncoder,
sort_keys=True, indent=4).encode('utf-8'))
# export all of the course metadata in policy.json
with course_run_policy_dir.open('policy.json', 'w') as course_policy:
with course_run_policy_dir.open(u'policy.json', 'wb') as course_policy:
policy = {'course/' + courselike.location.name: own_metadata(courselike)}
course_policy.write(dumps(policy, cls=EdxJSONEncoder, sort_keys=True, indent=4))
course_policy.write(dumps(policy, cls=EdxJSONEncoder, sort_keys=True, indent=4).encode('utf-8'))
_export_drafts(self.modulestore, self.courselike_key, export_fs, xml_centric_courselike_key)
@@ -315,7 +317,7 @@ class LibraryExportManager(ExportManager):
to ease in duck typing during import. This may be expanded as a useful feature eventually.
"""
# export the static assets
export_fs.makeopendir('policies')
export_fs.makedir('policies', recreate=True)
if self.contentstore:
self.contentstore.export_all_for_course(
@@ -332,7 +334,7 @@ class LibraryExportManager(ExportManager):
called library.xml.
"""
# Create the Library.xml file, which acts as the index of all library contents.
xml_file = export_fs.open(LIBRARY_ROOT, 'w')
xml_file = export_fs.open(LIBRARY_ROOT, 'wb')
xml_file.write(lxml.etree.tostring(root, pretty_print=True, encoding='utf-8'))
xml_file.close()
@@ -387,19 +389,20 @@ def _export_field_content(xblock_item, item_dir):
for field_name in module_data:
if field_name not in DEFAULT_CONTENT_FIELDS:
# filename format: {dirname}.{field_name}.json
with item_dir.open('{0}.{1}.{2}'.format(xblock_item.location.name, field_name, 'json'),
'w') as field_content_file:
field_content_file.write(dumps(module_data.get(field_name, {}), cls=EdxJSONEncoder, sort_keys=True, indent=4))
with item_dir.open(u'{0}.{1}.{2}'.format(xblock_item.location.name, field_name, 'json'),
'wb') as field_content_file:
field_content_file.write(dumps(module_data.get(field_name, {}), cls=EdxJSONEncoder,
sort_keys=True, indent=4).encode('utf-8'))
def export_extra_content(export_fs, modulestore, source_course_key, dest_course_key, category_type, dirname, file_suffix=''):
items = modulestore.get_items(source_course_key, qualifiers={'category': category_type})
if len(items) > 0:
item_dir = export_fs.makeopendir(dirname)
item_dir = export_fs.makedir(dirname, recreate=True)
for item in items:
adapt_references(item, dest_course_key, export_fs)
with item_dir.open(item.location.name + file_suffix, 'w') as item_file:
with item_dir.open(item.location.name + file_suffix, 'wb') as item_file:
item_file.write(item.data.encode('utf8'))
# export content fields other then metadata and data in json format in current directory

22
common/lib/xmodule/xmodule/tests/test_export.py
View File

@@ -71,14 +71,14 @@ class RoundTripTestCase(unittest.TestCase):
@mock.patch('xmodule.video_module.video_module.edxval_api', None)
@mock.patch('xmodule.course_module.requests.get')
@ddt.data(
"toy",
"simple",
"conditional_and_poll",
"conditional",
"self_assessment",
"test_exam_registration",
"word_cloud",
"pure_xblock",
u"toy",
u"simple",
u"conditional_and_poll",
u"conditional",
u"self_assessment",
u"test_exam_registration",
u"word_cloud",
u"pure_xblock",
)
@XBlock.register_temp_plugin(PureXBlock, 'pure')
def test_export_roundtrip(self, course_dir, mock_get):
@@ -107,12 +107,12 @@ class RoundTripTestCase(unittest.TestCase):
# will still be there.
print "Starting export"
file_system = OSFS(root_dir)
initial_course.runtime.export_fs = file_system.makeopendir(course_dir)
initial_course.runtime.export_fs = file_system.makedir(course_dir, recreate=True)
root = lxml.etree.Element('root')
initial_course.add_xml_to_node(root)
with initial_course.runtime.export_fs.open('course.xml', 'w') as course_xml:
lxml.etree.ElementTree(root).write(course_xml)
with initial_course.runtime.export_fs.open('course.xml', 'wb') as course_xml:
lxml.etree.ElementTree(root).write(course_xml, encoding='utf-8')
print "Starting second import"
second_import = XMLModuleStore(root_dir, source_dirs=[course_dir], xblock_mixins=(XModuleMixin,))

4
common/lib/xmodule/xmodule/tests/test_import.py
View File

@@ -219,7 +219,7 @@ class ImportTestCase(BaseCourseTestCase):
self.assertEqual(node.attrib['org'], ORG)
# Does the course still have unicorns?
with descriptor.runtime.export_fs.open('course/{url_name}.xml'.format(url_name=url_name)) as f:
with descriptor.runtime.export_fs.open(u'course/{url_name}.xml'.format(url_name=url_name)) as f:
course_xml = etree.fromstring(f.read())
self.assertEqual(course_xml.attrib['unicorn'], unicorn_color)
@@ -233,7 +233,7 @@ class ImportTestCase(BaseCourseTestCase):
# Does the chapter tag now have a due attribute?
# hardcoded path to child
with descriptor.runtime.export_fs.open('chapter/ch.xml') as f:
with descriptor.runtime.export_fs.open(u'chapter/ch.xml') as f:
chapter_xml = etree.fromstring(f.read())
self.assertEqual(chapter_xml.tag, 'chapter')
self.assertNotIn('due', chapter_xml.attrib)

4
common/lib/xmodule/xmodule/xml_module.py
View File

@@ -475,8 +475,8 @@ class XmlParserMixin(object):
# Write the definition to a file
url_path = name_to_pathname(self.url_name)
filepath = self._format_filepath(self.category, url_path)
self.runtime.export_fs.makedir(os.path.dirname(filepath), recursive=True, allow_recreate=True)
with self.runtime.export_fs.open(filepath, 'w') as fileobj:
self.runtime.export_fs.makedirs(os.path.dirname(filepath), recreate=True)
with self.runtime.export_fs.open(filepath, 'wb') as fileobj:
ElementTree(xml_object).write(fileobj, pretty_print=True, encoding='utf-8')
else:
# Write all attributes from xml_object onto node

8
lms/djangoapps/courseware/courses.py
View File

@@ -25,7 +25,7 @@ from django.core.urlresolvers import reverse
from django.http import Http404, QueryDict
from enrollment.api import get_course_enrollment_details
from edxmako.shortcuts import render_to_string
from fs.errors import ResourceNotFoundError
from fs.errors import ResourceNotFound
from lms.djangoapps.courseware.courseware_access_exception import CoursewareAccessException
from lms.djangoapps.courseware.exceptions import CourseAccessRedirect
from opaque_keys.edx.keys import UsageKey
@@ -207,13 +207,13 @@ def find_file(filesystem, dirs, filename):
dirs: a list of path objects
filename: a string
Returns d / filename if found in dir d, else raises ResourceNotFoundError.
Returns d / filename if found in dir d, else raises ResourceNotFound.
"""
for directory in dirs:
filepath = path(directory) / filename
if filesystem.exists(filepath):
return filepath
raise ResourceNotFoundError(u"Could not find {0}".format(filename))
raise ResourceNotFound(u"Could not find {0}".format(filename))
def get_course_about_section(request, course, section_key):
@@ -419,7 +419,7 @@ def get_course_syllabus_section(course, section_key):
course_id=course.id,
static_asset_path=course.static_asset_path,
)
except ResourceNotFoundError:
except ResourceNotFound:
log.exception(
u"Missing syllabus section %s in course %s",
section_key, course.location.to_deprecated_string()

11
requirements/edx/base.txt
View File

@@ -4,12 +4,15 @@
# * @edx/ospr - to check licensing
# * @edx/devops - to check system requirements
appdirs==1.4.3
attrs==17.2.0
beautifulsoup4==4.1.3
beautifulsoup==3.2.1
bleach==1.4
html5lib==0.999
boto==2.39.0
boto3==1.4.8
botocore==1.8.17
celery==3.1.18
cryptography==1.9
cssselect==0.9.1
@@ -25,7 +28,7 @@ django-model-utils==3.0.0
django-mptt>=0.8.6,<0.9
django-oauth-toolkit==0.12.0
django-pipeline-forgiving==1.0.0
django-pyfs==1.0.7
django-pyfs==2.0
django-sekizai>=0.10
django-ses==0.8.4
django-simple-history==1.9.0
@@ -35,6 +38,7 @@ django-method-override==0.1.0
django-user-tasks==0.1.5
django-waffle==0.12.0
djangorestframework-jwt==1.11.0
docutils==0.14
enum34==1.1.6
edx-ace==0.1.6
edx-ccx-keys==0.2.1
@@ -59,11 +63,15 @@ edxval==0.1.6
event-tracking==0.2.4
feedparser==5.1.3
firebase-token-generator==1.3.2
fs==2.0.17
fs-s3fs==0.1.5
futures==3.2.0 ; python_version == "2.7"
GitPython==0.3.2.RC1
glob2==0.3
gunicorn==0.17.4
help-tokens==1.0.3
httpretty==0.8.3
jmespath==0.9.3
lazy==1.1
mako==1.0.2
Markdown>=2.6,<2.7
@@ -95,6 +103,7 @@ pysrt==0.4.7
PyYAML==3.12
requests-oauthlib==0.4.1
rules==1.1.1
s3transfer==0.1.12
scipy==0.14.0
Shapely==1.2.16
singledispatch==3.4.0.2

6
requirements/edx/github.txt
View File

@@ -57,10 +57,6 @@ git+https://github.com/edx/nltk.git@2.0.6#egg=nltk==2.0.6
-e git+https://github.com/jazkarta/edx-jsme.git@690dbf75441fa91c7c4899df0b83d77f7deb5458#egg=edx-jsme
git+https://github.com/mitodl/django-cas.git@afac57bc523f145ae826f4ed3d4fa8b2c86c5364#egg=django-cas==2.1.1
-e git+https://github.com/dgrtwo/ParsePy.git@7949b9f754d1445eff8e8f20d0e967b9a6420639#egg=parse_rest
# Master pyfs has a bug working with VPC auth. This is a fix. We should switch
# back to master when and if this fix is merged back.
# fs==0.4.0
git+https://github.com/edx/pyfs.git@96e1922348bfe6d99201b9512a9ed946c87b7e0b#egg=fs==0.4.0
# The officially released version of django-debug-toolbar-mongo doesn't support DJDT 1.x. This commit does.
git+https://github.com/hmarr/django-debug-toolbar-mongo.git@b0686a76f1ce3532088c4aee6e76b9abe61cc808#egg=django-debug-toolbar-mongo==0.1.10
@@ -91,7 +87,7 @@ git+https://github.com/edx/django-celery.git@756cb57aad765cb2b0d37372c1855b8f5f3
-e git+https://github.com/edx/django-splash.git@v0.2#egg=django-splash==0.2
-e git+https://github.com/edx/acid-block.git@e46f9cda8a03e121a00c7e347084d142d22ebfb7#egg=acid-xblock
git+https://github.com/edx/edx-ora2.git@2.1.8#egg=ora2==2.1.8
git+https://github.com/edx/RecommenderXBlock.git@0e744b393cf1f8b886fe77bc697e7d9d78d65cd6#egg=recommender-xblock==1.2
git+https://github.com/edx/RecommenderXBlock.git@1.3#egg=recommender-xblock==1.3
git+https://github.com/solashirai/crowdsourcehinter.git@518605f0a95190949fe77bd39158450639e2e1dc#egg=crowdsourcehinter-xblock==0.1
-e git+https://github.com/edx/RateXBlock.git@367e19c0f6eac8a5f002fd0f1559555f8e74bfff#egg=rate-xblock
-e git+https://github.com/edx/DoneXBlock.git@01a14f3bd80ae47dd08cdbbe2f88f3eb88d00fba#egg=done-xblock