Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

refactor: move xmodule folder to root

Browse Source 
- Moving xmodule folder to root as we're dissolving sub-projects of common folder in edx-platform
    - More info: https://openedx.atlassian.net/browse/BOM-2579
- -e common/lib/xmodule has been removed from the requirements as xmodule has itself become the part of edx-platform and not being installed through requirements
- The test files common/lib/xmodule/test_files/ have been removed as they are not being used anymore
This commit is contained in:
M Umar Khan
2022-05-19 21:40:48 +05:00
parent 26c8ec5c2a
commit a91df0c40f
487 changed files with 216 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files

657
xmodule/tests/test_import.py Normal file
View File

@@ -0,0 +1,657 @@
# lint-amnesty, pylint: disable=missing-module-docstring
import datetime
from tempfile import mkdtemp
from unittest.mock import Mock, patch
import ddt
from django.test import TestCase
from fs.osfs import OSFS
from lxml import etree
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator
from pytz import UTC
from xblock.core import XBlock
from xblock.fields import Integer, Scope, String
from xblock.runtime import DictKeyValueStore, KvsFieldData
from xmodule.fields import Date
from xmodule.modulestore import only_xmodules
from xmodule.modulestore.inheritance import InheritanceMixin, compute_inherited_metadata
from xmodule.modulestore.xml import ImportSystem, LibraryXMLModuleStore, XMLModuleStore
from xmodule.tests import DATA_DIR
from xmodule.x_module import XModuleMixin
from xmodule.xml_module import is_pointer_tag
ORG = 'test_org'
COURSE = 'test_course'
RUN = 'test_run'
class DummySystem(ImportSystem): # lint-amnesty, pylint: disable=abstract-method, missing-class-docstring
@patch('xmodule.modulestore.xml.OSFS', lambda dir: OSFS(mkdtemp()))
def __init__(self, load_error_modules, library=False):
if library:
xmlstore = LibraryXMLModuleStore("data_dir", source_dirs=[], load_error_modules=load_error_modules)
else:
xmlstore = XMLModuleStore("data_dir", source_dirs=[], load_error_modules=load_error_modules)
course_id = CourseKey.from_string('/'.join([ORG, COURSE, RUN]))
course_dir = "test_dir"
error_tracker = Mock()
super().__init__(
xmlstore=xmlstore,
course_id=course_id,
course_dir=course_dir,
error_tracker=error_tracker,
load_error_modules=load_error_modules,
mixins=(InheritanceMixin, XModuleMixin),
field_data=KvsFieldData(DictKeyValueStore()),
)
def render_template(self, _template, _context): # lint-amnesty, pylint: disable=method-hidden
raise Exception("Shouldn't be called")
class BaseCourseTestCase(TestCase):
'''Make sure module imports work properly, including for malformed inputs'''
@staticmethod
def get_system(load_error_modules=True, library=False):
'''Get a dummy system'''
return DummySystem(load_error_modules, library=library)
def get_course(self, name):
"""Get a test course by directory name. If there's more than one, error."""
print(f"Importing {name}")
modulestore = XMLModuleStore(
DATA_DIR,
source_dirs=[name],
xblock_mixins=(InheritanceMixin,),
xblock_select=only_xmodules,
)
courses = modulestore.get_courses()
assert len(courses) == 1
return courses[0]
class GenericXBlock(XBlock):
"""XBlock for testing pure xblock xml import"""
has_children = True
field1 = String(default="something", scope=Scope.user_state)
field2 = Integer(scope=Scope.user_state)
@ddt.ddt
class PureXBlockImportTest(BaseCourseTestCase):
"""
Tests of import pure XBlocks (not XModules) from xml
"""
def assert_xblocks_are_good(self, block):
"""Assert a number of conditions that must be true for `block` to be good."""
scope_ids = block.scope_ids
assert scope_ids.usage_id is not None
assert scope_ids.def_id is not None
for child_id in block.children:
child = block.runtime.get_block(child_id)
self.assert_xblocks_are_good(child)
@XBlock.register_temp_plugin(GenericXBlock)
@ddt.data(
"<genericxblock/>",
"<genericxblock field1='abc' field2='23' />",
"<genericxblock field1='abc' field2='23'><genericxblock/></genericxblock>",
)
@patch('xmodule.x_module.XModuleMixin.location')
def test_parsing_pure_xblock(self, xml, mock_location):
system = self.get_system(load_error_modules=False)
descriptor = system.process_xml(xml)
assert isinstance(descriptor, GenericXBlock)
self.assert_xblocks_are_good(descriptor)
assert not mock_location.called
class ImportTestCase(BaseCourseTestCase): # lint-amnesty, pylint: disable=missing-class-docstring
date = Date()
def test_fallback(self):
'''Check that malformed xml loads as an ErrorBlock.'''
# Use an exotic character to also flush out Unicode issues.
bad_xml = '''<sequential display_name="oops\N{SNOWMAN}"><video url="hi"></sequential>'''
system = self.get_system()
descriptor = system.process_xml(bad_xml)
assert descriptor.__class__.__name__ == 'ErrorBlockWithMixins'
def test_unique_url_names(self):
'''Check that each error gets its very own url_name'''
bad_xml = '''<sequential display_name="oops"><video url="hi"></sequential>'''
bad_xml2 = '''<sequential url_name="oops"><video url="hi"></sequential>'''
system = self.get_system()
descriptor1 = system.process_xml(bad_xml)
descriptor2 = system.process_xml(bad_xml2)
assert descriptor1.location != descriptor2.location
# Check that each vertical gets its very own url_name
bad_xml = '''<vertical display_name="abc"><problem url_name="exam1:2013_Spring:abc"/></vertical>'''
bad_xml2 = '''<vertical display_name="abc"><problem url_name="exam2:2013_Spring:abc"/></vertical>'''
descriptor1 = system.process_xml(bad_xml)
descriptor2 = system.process_xml(bad_xml2)
assert descriptor1.location != descriptor2.location
def test_reimport(self):
'''Make sure an already-exported error xml tag loads properly'''
self.maxDiff = None
bad_xml = '''<sequential display_name="oops"><video url="hi"></sequential>'''
system = self.get_system()
descriptor = system.process_xml(bad_xml)
node = etree.Element('unknown')
descriptor.add_xml_to_node(node)
re_import_descriptor = system.process_xml(etree.tostring(node))
assert re_import_descriptor.__class__.__name__ == 'ErrorBlockWithMixins'
assert descriptor.contents == re_import_descriptor.contents
assert descriptor.error_msg == re_import_descriptor.error_msg
def test_fixed_xml_tag(self):
"""Make sure a tag that's been fixed exports as the original tag type"""
# create a error tag with valid xml contents
root = etree.Element('error')
good_xml = '''<sequential display_name="fixed"><video url="hi"/></sequential>'''
root.text = good_xml
xml_str_in = etree.tostring(root)
# load it
system = self.get_system()
descriptor = system.process_xml(xml_str_in)
# export it
node = etree.Element('unknown')
descriptor.add_xml_to_node(node)
# Now make sure the exported xml is a sequential
assert node.tag == 'sequential'
def course_descriptor_inheritance_check(self, descriptor, from_date_string, unicorn_color, course_run=RUN):
"""
Checks to make sure that metadata inheritance on a course descriptor is respected.
"""
# pylint: disable=protected-access
print((descriptor, descriptor._field_data))
assert descriptor.due == ImportTestCase.date.from_json(from_date_string)
# Check that the child inherits due correctly
child = descriptor.get_children()[0]
assert child.due == ImportTestCase.date.from_json(from_date_string)
# need to convert v to canonical json b4 comparing
assert ImportTestCase.date.to_json(ImportTestCase.date.from_json(from_date_string)) ==\
child.xblock_kvs.inherited_settings['due']
# Now export and check things
file_system = OSFS(mkdtemp())
descriptor.runtime.export_fs = file_system.makedir('course', recreate=True)
node = etree.Element('unknown')
descriptor.add_xml_to_node(node)
# Check that the exported xml is just a pointer
print(("Exported xml:", etree.tostring(node)))
assert is_pointer_tag(node)
# but it's a special case course pointer
assert node.attrib['course'] == COURSE
assert node.attrib['org'] == ORG
# Does the course still have unicorns?
with descriptor.runtime.export_fs.open(f'course/{course_run}.xml') as f:
course_xml = etree.fromstring(f.read())
assert course_xml.attrib['unicorn'] == unicorn_color
# the course and org tags should be _only_ in the pointer
assert 'course' not in course_xml.attrib
assert 'org' not in course_xml.attrib
# did we successfully strip the url_name from the definition contents?
assert 'url_name' not in course_xml.attrib
# Does the chapter tag now have a due attribute?
# hardcoded path to child
with descriptor.runtime.export_fs.open('chapter/ch.xml') as f:
chapter_xml = etree.fromstring(f.read())
assert chapter_xml.tag == 'chapter'
assert 'due' not in chapter_xml.attrib
def test_metadata_import_export(self):
"""Two checks:
- unknown metadata is preserved across import-export
- inherited metadata doesn't leak to children.
"""
system = self.get_system()
from_date_string = 'March 20 17:00'
url_name = 'test1'
unicorn_color = 'purple'
start_xml = '''
<course org="{org}" course="{course}"
due="{due}" url_name="{url_name}" unicorn="{unicorn_color}">
<chapter url="hi" url_name="ch" display_name="CH">
<html url_name="h" display_name="H">Two houses, ...</html>
</chapter>
</course>'''.format(
due=from_date_string, org=ORG, course=COURSE, url_name=url_name, unicorn_color=unicorn_color
)
descriptor = system.process_xml(start_xml)
compute_inherited_metadata(descriptor)
self.course_descriptor_inheritance_check(descriptor, from_date_string, unicorn_color)
def test_library_metadata_import_export(self):
"""Two checks:
- unknown metadata is preserved across import-export
- inherited metadata doesn't leak to children.
"""
system = self.get_system(library=True)
from_date_string = 'March 26 17:00'
url_name = 'test2'
unicorn_color = 'rainbow'
start_xml = '''
<library org="TestOrg" library="TestLib" display_name="stuff">
<course org="{org}" course="{course}"
due="{due}" url_name="{url_name}" unicorn="{unicorn_color}">
<chapter url="hi" url_name="ch" display_name="CH">
<html url_name="h" display_name="H">Two houses, ...</html>
</chapter>
</course>
</library>'''.format(
due=from_date_string, org=ORG, course=COURSE, url_name=url_name, unicorn_color=unicorn_color
)
descriptor = system.process_xml(start_xml)
# pylint: disable=protected-access
original_unwrapped = descriptor._unwrapped_field_data
LibraryXMLModuleStore.patch_descriptor_kvs(descriptor)
# '_unwrapped_field_data' is reset in `patch_descriptor_kvs`
# pylint: disable=protected-access
assert original_unwrapped is not descriptor._unwrapped_field_data
compute_inherited_metadata(descriptor)
# Check the course module, since it has inheritance
descriptor = descriptor.get_children()[0]
self.course_descriptor_inheritance_check(descriptor, from_date_string, unicorn_color)
def test_metadata_no_inheritance(self):
"""
Checks that default value of None (for due) does not get marked as inherited when a
course is the root block.
"""
system = self.get_system()
url_name = 'test1'
start_xml = '''
<course org="{org}" course="{course}"
url_name="{url_name}" unicorn="purple">
<chapter url="hi" url_name="ch" display_name="CH">
<html url_name="h" display_name="H">Two houses, ...</html>
</chapter>
</course>'''.format(org=ORG, course=COURSE, url_name=url_name)
descriptor = system.process_xml(start_xml)
compute_inherited_metadata(descriptor)
self.course_descriptor_no_inheritance_check(descriptor)
def test_library_metadata_no_inheritance(self):
"""
Checks that the default value of None (for due) does not get marked as inherited when a
library is the root block.
"""
system = self.get_system()
url_name = 'test1'
start_xml = '''
<library org="TestOrg" library="TestLib" display_name="stuff">
<course org="{org}" course="{course}"
url_name="{url_name}" unicorn="purple">
<chapter url="hi" url_name="ch" display_name="CH">
<html url_name="h" display_name="H">Two houses, ...</html>
</chapter>
</course>
</library>'''.format(org=ORG, course=COURSE, url_name=url_name)
descriptor = system.process_xml(start_xml)
LibraryXMLModuleStore.patch_descriptor_kvs(descriptor)
compute_inherited_metadata(descriptor)
# Run the checks on the course node instead.
descriptor = descriptor.get_children()[0]
self.course_descriptor_no_inheritance_check(descriptor)
def course_descriptor_no_inheritance_check(self, descriptor):
"""
Verifies that a default value of None (for due) does not get marked as inherited.
"""
assert descriptor.due is None
# Check that the child does not inherit a value for due
child = descriptor.get_children()[0]
assert child.due is None
# Check that the child hasn't started yet
assert datetime.datetime.now(UTC) <= child.start
def override_metadata_check(self, descriptor, child, course_due, child_due):
"""
Verifies that due date can be overriden at child level.
"""
assert descriptor.due == ImportTestCase.date.from_json(course_due)
assert child.due == ImportTestCase.date.from_json(child_due)
# Test inherited metadata. Due does not appear here (because explicitly set on child).
assert ImportTestCase.date.to_json(ImportTestCase.date.from_json(course_due)) == child.xblock_kvs.inherited_settings['due'] # pylint: disable=line-too-long
def test_metadata_override_default(self):
"""
Checks that due date can be overriden at child level when a course is the root.
"""
system = self.get_system()
course_due = 'March 20 17:00'
child_due = 'April 10 00:00'
url_name = 'test1'
start_xml = '''
<course org="{org}" course="{course}"
due="{due}" url_name="{url_name}" unicorn="purple">
<chapter url="hi" url_name="ch" display_name="CH">
<html url_name="h" display_name="H">Two houses, ...</html>
</chapter>
</course>'''.format(due=course_due, org=ORG, course=COURSE, url_name=url_name)
descriptor = system.process_xml(start_xml)
child = descriptor.get_children()[0]
# pylint: disable=protected-access
child._field_data.set(child, 'due', child_due)
compute_inherited_metadata(descriptor)
self.override_metadata_check(descriptor, child, course_due, child_due)
def test_library_metadata_override_default(self):
"""
Checks that due date can be overriden at child level when a library is the root.
"""
system = self.get_system()
course_due = 'March 20 17:00'
child_due = 'April 10 00:00'
url_name = 'test1'
start_xml = '''
<library org="TestOrg" library="TestLib" display_name="stuff">
<course org="{org}" course="{course}"
due="{due}" url_name="{url_name}" unicorn="purple">
<chapter url="hi" url_name="ch" display_name="CH">
<html url_name="h" display_name="H">Two houses, ...</html>
</chapter>
</course>
</library>'''.format(due=course_due, org=ORG, course=COURSE, url_name=url_name)
descriptor = system.process_xml(start_xml)
LibraryXMLModuleStore.patch_descriptor_kvs(descriptor)
# Chapter is two levels down here.
child = descriptor.get_children()[0].get_children()[0]
# pylint: disable=protected-access
child._field_data.set(child, 'due', child_due)
compute_inherited_metadata(descriptor)
descriptor = descriptor.get_children()[0]
self.override_metadata_check(descriptor, child, course_due, child_due)
def test_is_pointer_tag(self):
"""
Check that is_pointer_tag works properly.
"""
yes = ["""<html url_name="blah"/>""",
"""<html url_name="blah"></html>""",
"""<html url_name="blah"> </html>""",
"""<problem url_name="blah"/>""",
"""<course org="HogwartsX" course="Mathemagics" url_name="3.14159"/>"""]
no = ["""<html url_name="blah" also="this"/>""",
"""<html url_name="blah">some text</html>""",
"""<problem url_name="blah"><sub>tree</sub></problem>""",
"""<course org="HogwartsX" course="Mathemagics" url_name="3.14159">
<chapter>3</chapter>
</course>
"""]
for xml_str in yes:
print(f"should be True for {xml_str}")
assert is_pointer_tag(etree.fromstring(xml_str))
for xml_str in no:
print(f"should be False for {xml_str}")
assert not is_pointer_tag(etree.fromstring(xml_str))
def test_metadata_inherit(self):
"""Make sure that metadata is inherited properly"""
print("Starting import")
course = self.get_course('toy')
def check_for_key(key, node, value):
"recursive check for presence of key"
print(f"Checking {str(node.location)}")
assert getattr(node, key) == value
for c in node.get_children():
check_for_key(key, c, value)
check_for_key('graceperiod', course, course.graceperiod)
def test_policy_loading(self):
"""Make sure that when two courses share content with the same
org and course names, policy applies to the right one."""
toy = self.get_course('toy')
two_toys = self.get_course('two_toys')
assert toy.url_name == '2012_Fall'
assert two_toys.url_name == 'TT_2012_Fall'
toy_ch = toy.get_children()[0]
two_toys_ch = two_toys.get_children()[0]
assert toy_ch.display_name == 'Overview'
assert two_toys_ch.display_name == 'Two Toy Overview'
# Also check that the grading policy loaded
assert two_toys.grade_cutoffs['C'] == 0.5999
# Also check that keys from policy are run through the
# appropriate attribute maps -- 'graded' should be True, not 'true'
assert toy.graded is True
def test_static_tabs_import(self):
"""Make sure that the static tabs are imported correctly"""
modulestore = XMLModuleStore(DATA_DIR, source_dirs=['toy'])
location_tab_syllabus = BlockUsageLocator(CourseLocator("edX", "toy", "2012_Fall", deprecated=True),
"static_tab", "syllabus", deprecated=True)
toy_tab_syllabus = modulestore.get_item(location_tab_syllabus)
assert toy_tab_syllabus.display_name == 'Syllabus'
assert toy_tab_syllabus.course_staff_only is False
location_tab_resources = BlockUsageLocator(CourseLocator("edX", "toy", "2012_Fall", deprecated=True),
"static_tab", "resources", deprecated=True)
toy_tab_resources = modulestore.get_item(location_tab_resources)
assert toy_tab_resources.display_name == 'Resources'
assert toy_tab_resources.course_staff_only is True
def test_definition_loading(self):
"""When two courses share the same org and course name and
both have a module with the same url_name, the definitions shouldn't clash.
TODO (vshnayder): once we have a CMS, this shouldn't
happen--locations should uniquely name definitions. But in
our imperfect XML world, it can (and likely will) happen."""
modulestore = XMLModuleStore(DATA_DIR, source_dirs=['toy', 'two_toys'])
location = BlockUsageLocator(CourseLocator("edX", "toy", "2012_Fall", deprecated=True),
"video", "Welcome", deprecated=True)
toy_video = modulestore.get_item(location)
location_two = BlockUsageLocator(CourseLocator("edX", "toy", "TT_2012_Fall", deprecated=True),
"video", "Welcome", deprecated=True)
two_toy_video = modulestore.get_item(location_two)
assert toy_video.youtube_id_1_0 == 'p2Q6BrNhdh8'
assert two_toy_video.youtube_id_1_0 == 'p2Q6BrNhdh9'
def test_colon_in_url_name(self):
"""Ensure that colons in url_names convert to file paths properly"""
print("Starting import")
# Not using get_courses because we need the modulestore object too afterward
modulestore = XMLModuleStore(DATA_DIR, source_dirs=['toy'])
courses = modulestore.get_courses()
assert len(courses) == 1
course = courses[0]
print("course errors:")
for (msg, err) in modulestore.get_course_errors(course.id):
print(msg)
print(err)
chapters = course.get_children()
assert len(chapters) == 5
ch2 = chapters[1]
assert ch2.url_name == 'secret:magic'
print("Ch2 location: ", ch2.location)
also_ch2 = modulestore.get_item(ch2.location)
assert ch2 == also_ch2
print("making sure html loaded")
loc = course.id.make_usage_key('html', 'secret:toylab')
html = modulestore.get_item(loc)
assert html.display_name == 'Toy lab'
def test_unicode(self):
"""Check that courses with unicode characters in filenames and in
org/course/name import properly. Currently, this means: (a) Having
files with unicode names does not prevent import; (b) if files are not
loaded because of unicode filenames, there are appropriate
exceptions/errors to that effect."""
print("Starting import")
modulestore = XMLModuleStore(DATA_DIR, source_dirs=['test_unicode'])
courses = modulestore.get_courses()
assert len(courses) == 1
course = courses[0]
print("course errors:")
# Expect to find an error/exception about characters in "®esources"
expect = "InvalidKeyError"
errors = modulestore.get_course_errors(course.id)
assert any(((expect in msg) or (expect in err)) for (msg, err) in errors)
chapters = course.get_children()
assert len(chapters) == 4
def test_url_name_mangling(self):
"""
Make sure that url_names are only mangled once.
"""
modulestore = XMLModuleStore(DATA_DIR, source_dirs=['toy'])
toy_id = CourseKey.from_string('edX/toy/2012_Fall')
course = modulestore.get_course(toy_id)
chapters = course.get_children()
ch1 = chapters[0]
sections = ch1.get_children()
assert len(sections) == 4
for i in (2, 3):
video = sections[i]
# Name should be 'video_{hash}'
print(f"video {i} url_name: {video.url_name}")
assert len(video.url_name) == (len('video_') + 12)
def test_poll_and_conditional_import(self):
modulestore = XMLModuleStore(DATA_DIR, source_dirs=['conditional_and_poll'])
course = modulestore.get_courses()[0]
chapters = course.get_children()
ch1 = chapters[0]
sections = ch1.get_children()
assert len(sections) == 1
conditional_location = course.id.make_usage_key('conditional', 'condone')
module = modulestore.get_item(conditional_location)
assert len(module.children) == 1
poll_location = course.id.make_usage_key('poll_question', 'first_poll')
module = modulestore.get_item(poll_location)
assert len(module.get_children()) == 0
assert module.voted is False
assert module.poll_answer == ''
assert module.poll_answers == {}
assert module.answers ==\
[{'text': 'Yes', 'id': 'Yes'}, {'text': 'No', 'id': 'No'}, {'text': "Don't know", 'id': 'Dont_know'}]
def test_error_on_import(self):
'''Check that when load_error_module is false, an exception is raised, rather than returning an ErrorBlock'''
bad_xml = '''<sequential display_name="oops"><video url="hi"></sequential>'''
system = self.get_system(False)
self.assertRaises(etree.XMLSyntaxError, system.process_xml, bad_xml)
def test_word_cloud_import(self):
modulestore = XMLModuleStore(DATA_DIR, source_dirs=['word_cloud'])
course = modulestore.get_courses()[0]
chapters = course.get_children()
ch1 = chapters[0]
sections = ch1.get_children()
assert len(sections) == 1
location = course.id.make_usage_key('word_cloud', 'cloud1')
module = modulestore.get_item(location)
assert len(module.get_children()) == 0
assert module.num_inputs == 5
assert module.num_top_words == 250
def test_cohort_config(self):
"""
Check that cohort config parsing works right.
Note: The cohort config on the CourseBlock is no longer used.
See openedx.core.djangoapps.course_groups.models.CourseCohortSettings.
"""
modulestore = XMLModuleStore(DATA_DIR, source_dirs=['toy'])
toy_id = CourseKey.from_string('edX/toy/2012_Fall')
course = modulestore.get_course(toy_id)
# No config -> False
assert not course.is_cohorted
# empty config -> False
course.cohort_config = {}
assert not course.is_cohorted
# false config -> False
course.cohort_config = {'cohorted': False}
assert not course.is_cohorted
# and finally...
course.cohort_config = {'cohorted': True}
assert course.is_cohorted