Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PLAT-1861 Upgrade to pyfilesystem2

Browse Source
This commit is contained in:
Jeremy Bowman
2017-12-20 11:21:23 -05:00
parent e063a8bdd8
commit 65472399df
18 changed files with 122 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
common/lib/capa/capa/capa_problem.py
View File

@@ -611,7 +611,7 @@ class LoncapaProblem(object):
"""
includes = self.tree.findall('.//include')
for inc in includes:
filename = inc.get('file')
filename = inc.get('file').decode('utf-8')
if filename is not None:
try:
# open using LoncapaSystem OSFS filestore

4
common/lib/capa/capa/tests/test_html_render.py
View File

@@ -43,8 +43,8 @@ class CapaHtmlRenderTest(unittest.TestCase):
def test_include_html(self):
# Create a test file to include
self._create_test_file(
'test_include.xml',
'<test>Test include</test>'
u'test_include.xml',
u'<test>Test include</test>'
)
# Generate some XML with an <include>

4
common/lib/xmodule/xmodule/assetstore/__init__.py
View File

@@ -45,10 +45,10 @@ class AssetMetadata(object):
ASSET_XML_TAG = 'asset'
# Top-level directory name in exported course XML which holds asset metadata.
EXPORTED_ASSET_DIR = 'assets'
EXPORTED_ASSET_DIR = u'assets'
# Filename of all asset metadata exported as XML.
EXPORTED_ASSET_FILENAME = 'assets.xml'
EXPORTED_ASSET_FILENAME = u'assets.xml'
@contract(asset_id='AssetKey',
pathname='basestring|None', internal_name='basestring|None',

2
common/lib/xmodule/xmodule/backcompat_module.py
View File

@@ -24,7 +24,7 @@ def process_includes(fn):
next_include = xml_object.find('include')
while next_include is not None:
system.error_tracker("WARNING: the <include> tag is deprecated, and will go away.")
file = next_include.get('file')
file = next_include.get('file').decode('utf-8')
parent = next_include.getparent()
if file is None:

6
common/lib/xmodule/xmodule/course_module.py
View File

@@ -1002,12 +1002,12 @@ class CourseDescriptor(CourseFields, SequenceDescriptor, LicenseMixin):
policy_dir = None
url_name = xml_obj.get('url_name', xml_obj.get('slug'))
if url_name:
policy_dir = 'policies/' + url_name
policy_dir = u'policies/' + url_name
# Try to load grading policy
paths = ['grading_policy.json']
paths = [u'grading_policy.json']
if policy_dir:
paths = [policy_dir + '/grading_policy.json'] + paths
paths = [policy_dir + u'/grading_policy.json'] + paths
try:
policy = json.loads(cls.read_grading_policy(paths, system))

14
common/lib/xmodule/xmodule/html_module.py
View File

@@ -7,7 +7,7 @@ import textwrap
from datetime import datetime
from django.conf import settings
from fs.errors import ResourceNotFoundError
from fs.errors import ResourceNotFound
from lxml import etree
from path import Path as path
from pkg_resources import resource_string
@@ -236,7 +236,7 @@ class HtmlDescriptor(HtmlBlock, XmlDescriptor, EditingDescriptor): # pylint: di
)
base = path(pointer_path).dirname()
# log.debug("base = {0}, base.dirname={1}, filename={2}".format(base, base.dirname(), filename))
filepath = "{base}/{name}.html".format(base=base, name=filename)
filepath = u"{base}/{name}.html".format(base=base, name=filename)
# log.debug("looking for html file for {0} at {1}".format(location, filepath))
# VS[compat]
@@ -259,8 +259,8 @@ class HtmlDescriptor(HtmlBlock, XmlDescriptor, EditingDescriptor): # pylint: di
break
try:
with system.resources_fs.open(filepath) as infile:
html = infile.read().decode('utf-8')
with system.resources_fs.open(filepath, encoding='utf-8') as infile:
html = infile.read()
# Log a warning if we can't parse the file, but don't error
if not check_html(html) and len(html) > 0:
msg = "Couldn't parse html in {0}, content = {1}".format(filepath, html)
@@ -275,7 +275,7 @@ class HtmlDescriptor(HtmlBlock, XmlDescriptor, EditingDescriptor): # pylint: di
return definition, []
except (ResourceNotFoundError) as err:
except ResourceNotFound as err:
msg = 'Unable to load file contents at path {0}: {1} '.format(
filepath, err)
# add more info and re-raise
@@ -295,8 +295,8 @@ class HtmlDescriptor(HtmlBlock, XmlDescriptor, EditingDescriptor): # pylint: di
pathname=pathname
)
resource_fs.makedir(os.path.dirname(filepath), recursive=True, allow_recreate=True)
with resource_fs.open(filepath, 'w') as filestream:
resource_fs.makedirs(os.path.dirname(filepath), recreate=True)
with resource_fs.open(filepath, 'wb') as filestream:
html_data = self.data.encode('utf-8')
filestream.write(html_data)

4
common/lib/xmodule/xmodule/modulestore/__init__.py
View File

@@ -34,8 +34,8 @@ new_contract('AssetKey', AssetKey)
new_contract('AssetMetadata', AssetMetadata)
new_contract('XBlock', XBlock)
LIBRARY_ROOT = 'library.xml'
COURSE_ROOT = 'course.xml'
LIBRARY_ROOT = u'library.xml'
COURSE_ROOT = u'course.xml'
# List of names of computed fields on xmodules that are of type usage keys.
# This list can be used to determine which fields need to be stripped of

2
common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py
View File

@@ -39,7 +39,7 @@ COURSE_DATA_NAMES = (
'split_test_module_draft',
)
EXPORTED_COURSE_DIR_NAME = 'exported_source_course'
EXPORTED_COURSE_DIR_NAME = u'exported_source_course'
@ddt.ddt

6
common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py
View File

@@ -572,7 +572,7 @@ class TestMongoModuleStore(TestMongoModuleStoreBase):
root_dir = path(mkdtemp())
self.addCleanup(shutil.rmtree, root_dir)
export_course_to_xml(self.draft_store, self.content_store, course_key, root_dir, 'test_export')
export_course_to_xml(self.draft_store, self.content_store, course_key, root_dir, u'test_export')
self.assertTrue(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
self.assertTrue(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
@@ -588,7 +588,7 @@ class TestMongoModuleStore(TestMongoModuleStoreBase):
root_dir = path(mkdtemp())
self.addCleanup(shutil.rmtree, root_dir)
export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export')
export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, u'test_export')
self.assertTrue(path(root_dir / 'test_export/static/just_a_test.jpg').isfile())
self.assertFalse(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
@@ -601,7 +601,7 @@ class TestMongoModuleStore(TestMongoModuleStoreBase):
course = self.draft_store.get_course(CourseKey.from_string('edX/simple_with_draft/2012_Fall'))
root_dir = path(mkdtemp())
self.addCleanup(shutil.rmtree, root_dir)
export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export')
export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, u'test_export')
self.assertFalse(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
self.assertFalse(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())

4
common/lib/xmodule/xmodule/modulestore/tests/test_publish.py
View File

@@ -497,8 +497,8 @@ class DraftPublishedOpBaseTestSetup(OLXFormatChecker, DraftPublishedOpTestCourse
Setup base class for draft/published/OLX tests.
"""
EXPORTED_COURSE_BEFORE_DIR_NAME = 'exported_course_before'
EXPORTED_COURSE_AFTER_DIR_NAME = 'exported_course_after_{}'
EXPORTED_COURSE_BEFORE_DIR_NAME = u'exported_course_before'
EXPORTED_COURSE_AFTER_DIR_NAME = u'exported_course_after_{}'
def setUp(self):
super(DraftPublishedOpBaseTestSetup, self).setUp()

45
common/lib/xmodule/xmodule/modulestore/xml_exporter.py
View File

@@ -4,6 +4,7 @@ Methods for exporting course data to XML
import logging
from abc import abstractmethod
from six import text_type
import lxml.etree
from xblock.fields import Scope, Reference, ReferenceList, ReferenceValueDict
from xmodule.contentstore.content import StaticContent
@@ -42,7 +43,7 @@ def _export_drafts(modulestore, course_key, export_fs, xml_centric_course_key):
# Only modules with changes will be exported into the /drafts directory.
draft_modules = [module for module in draft_modules if modulestore.has_changes(module)]
if draft_modules:
draft_course_dir = export_fs.makeopendir(DRAFT_DIR)
draft_course_dir = export_fs.makedir(DRAFT_DIR, recreate=True)
# accumulate tuples of draft_modules and their parents in
# this list:
@@ -118,7 +119,7 @@ class ExportManager(object):
self.contentstore = contentstore
self.courselike_key = courselike_key
self.root_dir = root_dir
self.target_dir = target_dir
self.target_dir = text_type(target_dir)
@abstractmethod
def get_key(self):
@@ -160,7 +161,7 @@ class ExportManager(object):
# export only the published content
with self.modulestore.branch_setting(ModuleStoreEnum.Branch.published_only, self.courselike_key):
courselike = self.get_courselike()
export_fs = courselike.runtime.export_fs = fsm.makeopendir(self.target_dir)
export_fs = courselike.runtime.export_fs = fsm.makedir(self.target_dir, recreate=True)
# change all of the references inside the course to use the xml expected key type w/o version & branch
xml_centric_courselike_key = self.get_key()
@@ -196,8 +197,8 @@ class CourseExportManager(ExportManager):
return self.modulestore.get_course(self.courselike_key, depth=None, lazy=False)
def process_root(self, root, export_fs):
with export_fs.open('course.xml', 'w') as course_xml:
lxml.etree.ElementTree(root).write(course_xml)
with export_fs.open(u'course.xml', 'wb') as course_xml:
lxml.etree.ElementTree(root).write(course_xml, encoding='utf-8')
def process_extra(self, root, courselike, root_courselike_dir, xml_centric_courselike_key, export_fs):
# Export the modulestore's asset metadata.
@@ -210,11 +211,11 @@ class CourseExportManager(ExportManager):
# All asset types are exported using the "asset" tag - but their asset type is specified in each asset key.
asset = lxml.etree.SubElement(asset_root, AssetMetadata.ASSET_XML_TAG)
asset_md.to_xml(asset)
with OSFS(asset_dir).open(AssetMetadata.EXPORTED_ASSET_FILENAME, 'w') as asset_xml_file:
lxml.etree.ElementTree(asset_root).write(asset_xml_file)
with OSFS(asset_dir).open(AssetMetadata.EXPORTED_ASSET_FILENAME, 'wb') as asset_xml_file:
lxml.etree.ElementTree(asset_root).write(asset_xml_file, encoding='utf-8')
# export the static assets
policies_dir = export_fs.makeopendir('policies')
policies_dir = export_fs.makedir('policies', recreate=True)
if self.contentstore:
self.contentstore.export_all_for_course(
self.courselike_key,
@@ -238,7 +239,7 @@ class CourseExportManager(ExportManager):
output_dir = root_courselike_dir + '/static/images/'
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
with OSFS(output_dir).open('course_image.jpg', 'wb') as course_image_file:
with OSFS(output_dir).open(u'course_image.jpg', 'wb') as course_image_file:
course_image_file.write(course_image.data)
# export the static tabs
@@ -270,16 +271,17 @@ class CourseExportManager(ExportManager):
# Use url_name for split mongo because course_run is not used when loading policies.
course_policy_dir_name = courselike.url_name
course_run_policy_dir = policies_dir.makeopendir(course_policy_dir_name)
course_run_policy_dir = policies_dir.makedir(course_policy_dir_name, recreate=True)
# export the grading policy
with course_run_policy_dir.open('grading_policy.json', 'w') as grading_policy:
grading_policy.write(dumps(courselike.grading_policy, cls=EdxJSONEncoder, sort_keys=True, indent=4))
with course_run_policy_dir.open(u'grading_policy.json', 'wb') as grading_policy:
grading_policy.write(dumps(courselike.grading_policy, cls=EdxJSONEncoder,
sort_keys=True, indent=4).encode('utf-8'))
# export all of the course metadata in policy.json
with course_run_policy_dir.open('policy.json', 'w') as course_policy:
with course_run_policy_dir.open(u'policy.json', 'wb') as course_policy:
policy = {'course/' + courselike.location.name: own_metadata(courselike)}
course_policy.write(dumps(policy, cls=EdxJSONEncoder, sort_keys=True, indent=4))
course_policy.write(dumps(policy, cls=EdxJSONEncoder, sort_keys=True, indent=4).encode('utf-8'))
_export_drafts(self.modulestore, self.courselike_key, export_fs, xml_centric_courselike_key)
@@ -315,7 +317,7 @@ class LibraryExportManager(ExportManager):
to ease in duck typing during import. This may be expanded as a useful feature eventually.
"""
# export the static assets
export_fs.makeopendir('policies')
export_fs.makedir('policies', recreate=True)
if self.contentstore:
self.contentstore.export_all_for_course(
@@ -332,7 +334,7 @@ class LibraryExportManager(ExportManager):
called library.xml.
"""
# Create the Library.xml file, which acts as the index of all library contents.
xml_file = export_fs.open(LIBRARY_ROOT, 'w')
xml_file = export_fs.open(LIBRARY_ROOT, 'wb')
xml_file.write(lxml.etree.tostring(root, pretty_print=True, encoding='utf-8'))
xml_file.close()
@@ -387,19 +389,20 @@ def _export_field_content(xblock_item, item_dir):
for field_name in module_data:
if field_name not in DEFAULT_CONTENT_FIELDS:
# filename format: {dirname}.{field_name}.json
with item_dir.open('{0}.{1}.{2}'.format(xblock_item.location.name, field_name, 'json'),
'w') as field_content_file:
field_content_file.write(dumps(module_data.get(field_name, {}), cls=EdxJSONEncoder, sort_keys=True, indent=4))
with item_dir.open(u'{0}.{1}.{2}'.format(xblock_item.location.name, field_name, 'json'),
'wb') as field_content_file:
field_content_file.write(dumps(module_data.get(field_name, {}), cls=EdxJSONEncoder,
sort_keys=True, indent=4).encode('utf-8'))
def export_extra_content(export_fs, modulestore, source_course_key, dest_course_key, category_type, dirname, file_suffix=''):
items = modulestore.get_items(source_course_key, qualifiers={'category': category_type})
if len(items) > 0:
item_dir = export_fs.makeopendir(dirname)
item_dir = export_fs.makedir(dirname, recreate=True)
for item in items:
adapt_references(item, dest_course_key, export_fs)
with item_dir.open(item.location.name + file_suffix, 'w') as item_file:
with item_dir.open(item.location.name + file_suffix, 'wb') as item_file:
item_file.write(item.data.encode('utf8'))
# export content fields other then metadata and data in json format in current directory

22
common/lib/xmodule/xmodule/tests/test_export.py
View File

@@ -71,14 +71,14 @@ class RoundTripTestCase(unittest.TestCase):
@mock.patch('xmodule.video_module.video_module.edxval_api', None)
@mock.patch('xmodule.course_module.requests.get')
@ddt.data(
"toy",
"simple",
"conditional_and_poll",
"conditional",
"self_assessment",
"test_exam_registration",
"word_cloud",
"pure_xblock",
u"toy",
u"simple",
u"conditional_and_poll",
u"conditional",
u"self_assessment",
u"test_exam_registration",
u"word_cloud",
u"pure_xblock",
)
@XBlock.register_temp_plugin(PureXBlock, 'pure')
def test_export_roundtrip(self, course_dir, mock_get):
@@ -107,12 +107,12 @@ class RoundTripTestCase(unittest.TestCase):
# will still be there.
print "Starting export"
file_system = OSFS(root_dir)
initial_course.runtime.export_fs = file_system.makeopendir(course_dir)
initial_course.runtime.export_fs = file_system.makedir(course_dir, recreate=True)
root = lxml.etree.Element('root')
initial_course.add_xml_to_node(root)
with initial_course.runtime.export_fs.open('course.xml', 'w') as course_xml:
lxml.etree.ElementTree(root).write(course_xml)
with initial_course.runtime.export_fs.open('course.xml', 'wb') as course_xml:
lxml.etree.ElementTree(root).write(course_xml, encoding='utf-8')
print "Starting second import"
second_import = XMLModuleStore(root_dir, source_dirs=[course_dir], xblock_mixins=(XModuleMixin,))

4
common/lib/xmodule/xmodule/tests/test_import.py
View File

@@ -219,7 +219,7 @@ class ImportTestCase(BaseCourseTestCase):
self.assertEqual(node.attrib['org'], ORG)
# Does the course still have unicorns?
with descriptor.runtime.export_fs.open('course/{url_name}.xml'.format(url_name=url_name)) as f:
with descriptor.runtime.export_fs.open(u'course/{url_name}.xml'.format(url_name=url_name)) as f:
course_xml = etree.fromstring(f.read())
self.assertEqual(course_xml.attrib['unicorn'], unicorn_color)
@@ -233,7 +233,7 @@ class ImportTestCase(BaseCourseTestCase):
# Does the chapter tag now have a due attribute?
# hardcoded path to child
with descriptor.runtime.export_fs.open('chapter/ch.xml') as f:
with descriptor.runtime.export_fs.open(u'chapter/ch.xml') as f:
chapter_xml = etree.fromstring(f.read())
self.assertEqual(chapter_xml.tag, 'chapter')
self.assertNotIn('due', chapter_xml.attrib)

4
common/lib/xmodule/xmodule/xml_module.py
View File

@@ -475,8 +475,8 @@ class XmlParserMixin(object):
# Write the definition to a file
url_path = name_to_pathname(self.url_name)
filepath = self._format_filepath(self.category, url_path)
self.runtime.export_fs.makedir(os.path.dirname(filepath), recursive=True, allow_recreate=True)
with self.runtime.export_fs.open(filepath, 'w') as fileobj:
self.runtime.export_fs.makedirs(os.path.dirname(filepath), recreate=True)
with self.runtime.export_fs.open(filepath, 'wb') as fileobj:
ElementTree(xml_object).write(fileobj, pretty_print=True, encoding='utf-8')
else:
# Write all attributes from xml_object onto node