Merge branch 'feature/cale/cms-master' into feature/cale/cms-master-merge

Conflicts:
	requirements.txt
This commit is contained in:
Chris Dodge
2013-01-03 10:46:24 -05:00
123 changed files with 7234 additions and 585 deletions

View File

@@ -1145,7 +1145,13 @@ class CodeResponse(LoncapaResponse):
xml = self.xml
# TODO: XML can override external resource (grader/queue) URL
self.url = xml.get('url', None)
self.queue_name = xml.get('queuename', self.system.xqueue['default_queuename'])
# We do not support xqueue within Studio.
if self.system.xqueue is not None:
default_queuename = self.system.xqueue['default_queuename']
else:
default_queuename = None
self.queue_name = xml.get('queuename', default_queuename)
# VS[compat]:
# Check if XML uses the ExternalResponse format or the generic CodeResponse format
@@ -1234,6 +1240,13 @@ class CodeResponse(LoncapaResponse):
(err, self.answer_id, convert_files_to_filenames(student_answers)))
raise Exception(err)
# We do not support xqueue within Studio.
if self.system.xqueue is None:
cmap = CorrectMap()
cmap.set(self.answer_id, queuestate=None,
msg='Error checking problem: no external queueing server is configured.')
return cmap
# Prepare xqueue request
#------------------------------------------------------------

View File

@@ -12,6 +12,7 @@
<script type="text/javascript" src="<%= common_coffee_root %>/logger.js"></script>
<script type="text/javascript" src="<%= common_js_root %>/vendor/jquery.min.js"></script>
<script type="text/javascript" src="<%= common_js_root %>/vendor/jasmine-jquery.js"></script>
<script type="text/javascript" src="<%= common_js_root %>/vendor/CodeMirror/codemirror.js"></script>
<script type="text/javascript" src="<%= common_js_root %>/vendor/mathjax-MathJax-c9db6ac/MathJax.js"></script>
<script type="text/javascript">

View File

@@ -12,13 +12,16 @@ from .django import contentstore
from PIL import Image
class StaticContent(object):
def __init__(self, loc, name, content_type, data, last_modified_at=None, thumbnail_location=None):
def __init__(self, loc, name, content_type, data, last_modified_at=None, thumbnail_location=None, import_path=None):
self.location = loc
self.name = name #a display string which can be edited, and thus not part of the location which needs to be fixed
self.content_type = content_type
self.data = data
self.last_modified_at = last_modified_at
self.thumbnail_location = thumbnail_location
self.thumbnail_location = Location(thumbnail_location)
# optional information about where this file was imported from. This is needed to support import/export
# cycles
self.import_path = import_path
@property
def is_thumbnail(self):

View File

@@ -11,6 +11,8 @@ import logging
from .content import StaticContent, ContentStore
from xmodule.exceptions import NotFoundError
from fs.osfs import OSFS
import os
class MongoContentStore(ContentStore):
@@ -29,26 +31,50 @@ class MongoContentStore(ContentStore):
id = content.get_id()
# Seems like with the GridFS we can't update existing ID's we have to do a delete/add pair
if self.fs.exists({"_id" : id}):
self.fs.delete(id)
self.delete(id)
with self.fs.new_file(_id = id, filename=content.get_url_path(), content_type=content.content_type,
displayname=content.name, thumbnail_location=content.thumbnail_location) as fp:
displayname=content.name, thumbnail_location=content.thumbnail_location, import_path=content.import_path) as fp:
fp.write(content.data)
return content
def delete(self, id):
if self.fs.exists({"_id" : id}):
self.fs.delete(id)
def find(self, location):
id = StaticContent.get_id_from_location(location)
try:
with self.fs.get(id) as fp:
return StaticContent(location, fp.displayname, fp.content_type, fp.read(),
fp.uploadDate, thumbnail_location = fp.thumbnail_location if 'thumbnail_location' in fp else None)
fp.uploadDate, thumbnail_location = fp.thumbnail_location if hasattr(fp, 'thumbnail_location') else None,
import_path = fp.import_path if hasattr(fp, 'import_path') else None)
except NoFile:
raise NotFoundError()
def export(self, location, output_directory):
content = self.find(location)
if content.import_path is not None:
output_directory = output_directory + '/' + os.path.dirname(content.import_path)
if not os.path.exists(output_directory):
os.makedirs(output_directory)
disk_fs = OSFS(output_directory)
with disk_fs.open(content.name, 'wb') as asset_file:
asset_file.write(content.data)
def export_all_for_course(self, course_location, output_directory):
assets = self.get_all_content_for_course(course_location)
for asset in assets:
asset_location = Location(asset['_id'])
self.export(asset_location, output_directory)
def get_all_content_thumbnails_for_course(self, location):
return self._get_all_content_for_course(location, get_thumbnails = True)

View File

@@ -1,18 +1,17 @@
from fs.errors import ResourceNotFoundError
import logging
import json
from cStringIO import StringIO
from lxml import etree
from path import path # NOTE (THK): Only used for detecting presence of syllabus
import requests
import time
from cStringIO import StringIO
from xmodule.util.decorators import lazyproperty
from xmodule.graders import grader_from_conf
from xmodule.modulestore import Location
from xmodule.seq_module import SequenceDescriptor, SequenceModule
from xmodule.xml_module import XmlDescriptor
from xmodule.timeparse import parse_time, stringify_time
from xmodule.graders import grader_from_conf
from xmodule.util.decorators import lazyproperty
import json
import logging
import requests
import time
import copy
log = logging.getLogger(__name__)
@@ -92,10 +91,6 @@ class CourseDescriptor(SequenceDescriptor):
log.critical(msg)
system.error_tracker(msg)
self.enrollment_start = self._try_parse_time("enrollment_start")
self.enrollment_end = self._try_parse_time("enrollment_end")
self.end = self._try_parse_time("end")
# NOTE: relies on the modulestore to call set_grading_policy() right after
# init. (Modulestore is in charge of figuring out where to load the policy from)
@@ -105,19 +100,11 @@ class CourseDescriptor(SequenceDescriptor):
self.set_grading_policy(self.definition['data'].get('grading_policy', None))
def set_grading_policy(self, course_policy):
if course_policy is None:
course_policy = {}
def defaut_grading_policy(self):
"""
The JSON object can have the keys GRADER and GRADE_CUTOFFS. If either is
missing, it reverts to the default.
Return a dict which is a copy of the default grading policy
"""
default_policy_string = """
{
"GRADER" : [
default = {"GRADER" : [
{
"type" : "Homework",
"min_count" : 12,
@@ -129,37 +116,44 @@ class CourseDescriptor(SequenceDescriptor):
"type" : "Lab",
"min_count" : 12,
"drop_count" : 2,
"category" : "Labs",
"weight" : 0.15
},
{
"type" : "Midterm",
"name" : "Midterm Exam",
"type" : "Midterm Exam",
"short_label" : "Midterm",
"min_count" : 1,
"drop_count" : 0,
"weight" : 0.3
},
{
"type" : "Final",
"name" : "Final Exam",
"type" : "Final Exam",
"short_label" : "Final",
"min_count" : 1,
"drop_count" : 0,
"weight" : 0.4
}
],
"GRADE_CUTOFFS" : {
"A" : 0.87,
"B" : 0.7,
"C" : 0.6
}
}
"Pass" : 0.5
}}
return copy.deepcopy(default)
def set_grading_policy(self, course_policy):
"""
The JSON object can have the keys GRADER and GRADE_CUTOFFS. If either is
missing, it reverts to the default.
"""
if course_policy is None:
course_policy = {}
# Load the global settings as a dictionary
grading_policy = json.loads(default_policy_string)
grading_policy = self.defaut_grading_policy()
# Override any global settings with the course settings
grading_policy.update(course_policy)
# Here is where we should parse any configurations, so that we can fail early
grading_policy['RAW_GRADER'] = grading_policy['GRADER'] # used for cms access
grading_policy['GRADER'] = grader_from_conf(grading_policy['GRADER'])
self._grading_policy = grading_policy
@@ -168,8 +162,8 @@ class CourseDescriptor(SequenceDescriptor):
@classmethod
def read_grading_policy(cls, paths, system):
"""Load a grading policy from the specified paths, in order, if it exists."""
# Default to a blank policy
policy_str = '""'
# Default to a blank policy dict
policy_str = '{}'
for policy_path in paths:
if not system.resources_fs.exists(policy_path):
@@ -252,13 +246,53 @@ class CourseDescriptor(SequenceDescriptor):
def has_started(self):
return time.gmtime() > self.start
@property
def end(self):
return self._try_parse_time("end")
@end.setter
def end(self, value):
if isinstance(value, time.struct_time):
self.metadata['end'] = stringify_time(value)
@property
def enrollment_start(self):
return self._try_parse_time("enrollment_start")
@enrollment_start.setter
def enrollment_start(self, value):
if isinstance(value, time.struct_time):
self.metadata['enrollment_start'] = stringify_time(value)
@property
def enrollment_end(self):
return self._try_parse_time("enrollment_end")
@enrollment_end.setter
def enrollment_end(self, value):
if isinstance(value, time.struct_time):
self.metadata['enrollment_end'] = stringify_time(value)
@property
def grader(self):
return self._grading_policy['GRADER']
@property
def raw_grader(self):
return self._grading_policy['RAW_GRADER']
@raw_grader.setter
def raw_grader(self, value):
# NOTE WELL: this change will not update the processed graders. If we need that, this needs to call grader_from_conf
self._grading_policy['RAW_GRADER'] = value
self.definition['data'].setdefault('grading_policy',{})['GRADER'] = value
@property
def grade_cutoffs(self):
return self._grading_policy['GRADE_CUTOFFS']
@grade_cutoffs.setter
def grade_cutoffs(self, value):
self._grading_policy['GRADE_CUTOFFS'] = value
self.definition['data'].setdefault('grading_policy',{})['GRADE_CUTOFFS'] = value
@property
def lowest_passing_grade(self):

View File

@@ -1,6 +1,5 @@
import abc
import inspect
import json
import logging
import random
import sys
@@ -66,17 +65,27 @@ def grader_from_conf(conf):
for subgraderconf in conf:
subgraderconf = subgraderconf.copy()
weight = subgraderconf.pop("weight", 0)
# NOTE: 'name' used to exist in SingleSectionGrader. We are deprecating SingleSectionGrader
# and converting everything into an AssignmentFormatGrader by adding 'min_count' and
# 'drop_count'. AssignmentFormatGrader does not expect 'name', so if it appears
# in bad_args, go ahead remove it (this causes no errors). Eventually, SingleSectionGrader
# should be completely removed.
name = 'name'
try:
if 'min_count' in subgraderconf:
#This is an AssignmentFormatGrader
subgrader_class = AssignmentFormatGrader
elif 'name' in subgraderconf:
elif name in subgraderconf:
#This is an SingleSectionGrader
subgrader_class = SingleSectionGrader
else:
raise ValueError("Configuration has no appropriate grader class.")
bad_args = invalid_args(subgrader_class.__init__, subgraderconf)
# See note above concerning 'name'.
if bad_args.issuperset({name}):
bad_args = bad_args - {name}
del subgraderconf[name]
if len(bad_args) > 0:
log.warning("Invalid arguments for a subgrader: %s", bad_args)
for key in bad_args:

View File

@@ -159,7 +159,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
filepath = u'{category}/{pathname}.html'.format(category=self.category,
pathname=pathname)
resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
resource_fs.makedir(os.path.dirname(filepath), recursive=True, allow_recreate=True)
with resource_fs.open(filepath, 'w') as file:
file.write(self.definition['data'].encode('utf-8'))

View File

@@ -0,0 +1,8 @@
<section class="html-edit">
<div name="" class="edit-box" rows="8" cols="40">&lt;problem>
&lt;p>&lt;/p>
&lt;multiplechoiceresponse>
<pre>&lt;problem>
&lt;p>&lt;/p></pre>
<div><foo>bar</foo></div></div>
</section>

View File

@@ -0,0 +1,2 @@
*.js

View File

@@ -0,0 +1,17 @@
describe 'HTMLEditingDescriptor', ->
describe 'Read data from server, create Editor, and get data back out', ->
it 'Does not munge &lt', ->
# This is a test for Lighthouse #22,
# "html names are automatically converted to the symbols they describe"
# A better test would be a Selenium test to avoid duplicating the
# mako template structure in html-edit-formattingbug.html.
# However, we currently have no working Selenium tests.
loadFixtures 'html-edit-formattingbug.html'
@descriptor = new HTMLEditingDescriptor($('.html-edit'))
data = @descriptor.save().data
expect(data).toEqual("""&lt;problem&gt;
&lt;p&gt;&lt;/p&gt;
&lt;multiplechoiceresponse&gt;
<pre>&lt;problem&gt;
&lt;p&gt;&lt;/p&gt;</pre>
<div><foo>bar</foo></div>""")

View File

@@ -0,0 +1,2 @@
*.js

View File

@@ -1,11 +1,12 @@
class @HTMLEditingDescriptor
constructor: (@element) ->
@edit_box = CodeMirror.fromTextArea($(".edit-box", @element)[0], {
text = $(".edit-box", @element)[0];
replace_func = (elt) -> text.parentNode.replaceChild(elt, text)
@edit_box = CodeMirror(replace_func, {
value: text.innerHTML
mode: "text/html"
lineNumbers: true
lineWrapping: true
})
lineWrapping: true})
save: ->
data: @edit_box.getValue()

View File

@@ -153,7 +153,7 @@ class Location(_LocationBase):
def check(val, regexp):
if val is not None and regexp.search(val) is not None:
log.debug('invalid characters val="%s", list_="%s"' % (val, list_))
raise InvalidLocationError(location)
raise InvalidLocationError("Invalid characters in '%s'." % (val))
list_ = list(list_)
for val in list_[:4] + [list_[5]]:

View File

@@ -51,6 +51,9 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
self.modulestore = modulestore
self.module_data = module_data
self.default_class = default_class
# cdodge: other Systems have a course_id attribute defined. To keep things consistent, let's
# define an attribute here as well, even though it's None
self.course_id = None
def load_item(self, location):
location = Location(location)

View File

@@ -0,0 +1,130 @@
import logging
from xmodule.contentstore.content import StaticContent
from xmodule.modulestore import Location
from xmodule.modulestore.mongo import MongoModuleStore
def clone_course(modulestore, contentstore, source_location, dest_location, delete_original=False):
# first check to see if the modulestore is Mongo backed
if not isinstance(modulestore, MongoModuleStore):
raise Exception("Expected a MongoModuleStore in the runtime. Aborting....")
# check to see if the dest_location exists as an empty course
# we need an empty course because the app layers manage the permissions and users
if not modulestore.has_item(dest_location):
raise Exception("An empty course at {0} must have already been created. Aborting...".format(dest_location))
# verify that the dest_location really is an empty course, which means only one
dest_modules = modulestore.get_items([dest_location.tag, dest_location.org, dest_location.course, None, None, None])
if len(dest_modules) != 1:
raise Exception("Course at destination {0} is not an empty course. You can only clone into an empty course. Aborting...".format(dest_location))
# check to see if the source course is actually there
if not modulestore.has_item(source_location):
raise Exception("Cannot find a course at {0}. Aborting".format(source_location))
# Get all modules under this namespace which is (tag, org, course) tuple
modules = modulestore.get_items([source_location.tag, source_location.org, source_location.course, None, None, None])
for module in modules:
original_loc = Location(module.location)
if original_loc.category != 'course':
module.location = module.location._replace(tag = dest_location.tag, org = dest_location.org,
course = dest_location.course)
else:
# on the course module we also have to update the module name
module.location = module.location._replace(tag = dest_location.tag, org = dest_location.org,
course = dest_location.course, name=dest_location.name)
print "Cloning module {0} to {1}....".format(original_loc, module.location)
if 'data' in module.definition:
modulestore.update_item(module.location, module.definition['data'])
# repoint children
if 'children' in module.definition:
new_children = []
for child_loc_url in module.definition['children']:
child_loc = Location(child_loc_url)
child_loc = child_loc._replace(tag = dest_location.tag, org = dest_location.org,
course = dest_location.course)
new_children = new_children + [child_loc.url()]
modulestore.update_children(module.location, new_children)
# save metadata
modulestore.update_metadata(module.location, module.metadata)
# now iterate through all of the assets and clone them
# first the thumbnails
thumbs = contentstore.get_all_content_thumbnails_for_course(source_location)
for thumb in thumbs:
thumb_loc = Location(thumb["_id"])
content = contentstore.find(thumb_loc)
content.location = content.location._replace(org = dest_location.org,
course = dest_location.course)
print "Cloning thumbnail {0} to {1}".format(thumb_loc, content.location)
contentstore.save(content)
# now iterate through all of the assets, also updating the thumbnail pointer
assets = contentstore.get_all_content_for_course(source_location)
for asset in assets:
asset_loc = Location(asset["_id"])
content = contentstore.find(asset_loc)
content.location = content.location._replace(org = dest_location.org,
course = dest_location.course)
# be sure to update the pointer to the thumbnail
if content.thumbnail_location is not None:
content.thumbnail_location = content.thumbnail_location._replace(org = dest_location.org,
course = dest_location.course)
print "Cloning asset {0} to {1}".format(asset_loc, content.location)
contentstore.save(content)
return True
def delete_course(modulestore, contentstore, source_location):
# first check to see if the modulestore is Mongo backed
if not isinstance(modulestore, MongoModuleStore):
raise Exception("Expected a MongoModuleStore in the runtime. Aborting....")
# check to see if the source course is actually there
if not modulestore.has_item(source_location):
raise Exception("Cannot find a course at {0}. Aborting".format(source_location))
# first delete all of the thumbnails
thumbs = contentstore.get_all_content_thumbnails_for_course(source_location)
for thumb in thumbs:
thumb_loc = Location(thumb["_id"])
id = StaticContent.get_id_from_location(thumb_loc)
print "Deleting {0}...".format(id)
contentstore.delete(id)
# then delete all of the assets
assets = contentstore.get_all_content_for_course(source_location)
for asset in assets:
asset_loc = Location(asset["_id"])
id = StaticContent.get_id_from_location(asset_loc)
print "Deleting {0}...".format(id)
contentstore.delete(id)
# then delete all course modules
modules = modulestore.get_items([source_location.tag, source_location.org, source_location.course, None, None, None])
for module in modules:
if module.category != 'course': # save deleting the course module for last
print "Deleting {0}...".format(module.location)
modulestore.delete_item(module.location)
# finally delete the top-level course module itself
print "Deleting {0}...".format(source_location)
modulestore.delete_item(source_location)
return True

View File

@@ -0,0 +1,20 @@
import logging
from xmodule.modulestore import Location
from xmodule.modulestore.django import modulestore
from fs.osfs import OSFS
def export_to_xml(modulestore, contentstore, course_location, root_dir, course_dir):
course = modulestore.get_item(course_location)
fs = OSFS(root_dir)
export_fs = fs.makeopendir(course_dir)
xml = course.export_to_xml(export_fs)
with export_fs.open('course.xml', 'w') as course_xml:
course_xml.write(xml)
# export the static assets
contentstore.export_all_for_course(course_location, root_dir + '/' + course_dir + '/static/')

View File

@@ -32,7 +32,7 @@ def import_static_content(modules, course_loc, course_data_path, static_content_
with open(content_path, 'rb') as f:
data = f.read()
content = StaticContent(content_loc, filename, mime_type, data)
content = StaticContent(content_loc, filename, mime_type, data, import_path = fullname_with_subpath)
# first let's save a thumbnail so we can get back a thumbnail location
thumbnail_content = static_content_store.generate_thumbnail(content)
@@ -66,7 +66,7 @@ def verify_content_links(module, base_dir, static_content_store, link, remap_dic
with open(static_pathname, 'rb') as f:
data = f.read()
content = StaticContent(content_loc, filename, mime_type, data)
content = StaticContent(content_loc, filename, mime_type, data, import_path = path)
# first let's save a thumbnail so we can get back a thumbnail location
thumbnail_content = static_content_store.generate_thumbnail(content)

View File

@@ -1,5 +1,5 @@
---
metadata:
display_name: Empty
data: "<p>This is where you can add additional information about your course.</p>"
data: "<ol></ol>"
children: []

View File

@@ -1,6 +1,6 @@
---
metadata:
display_name: Formula Repsonse
display_name: Formula Response
rerandomize: never
showanswer: always
data: |

View File

@@ -25,7 +25,8 @@ class VideoModule(XModule):
resource_string(__name__, 'js/src/video/display.coffee')] +
[resource_string(__name__, 'js/src/video/display/' + filename)
for filename
in sorted(resource_listdir(__name__, 'js/src/video/display'))]}
in sorted(resource_listdir(__name__, 'js/src/video/display'))
if filename.endswith('.coffee')]}
css = {'scss': [resource_string(__name__, 'css/video/display.scss')]}
js_module_name = "Video"

View File

@@ -10,10 +10,11 @@ from collections import namedtuple
from pkg_resources import resource_listdir, resource_string, resource_isdir
from xmodule.modulestore import Location
from xmodule.timeparse import parse_time
from xmodule.timeparse import parse_time, stringify_time
from xmodule.contentstore.content import StaticContent, XASSET_SRCREF_PREFIX
from xmodule.modulestore.exceptions import ItemNotFoundError
import time
log = logging.getLogger('mitx.' + __name__)
@@ -519,6 +520,11 @@ class XModuleDescriptor(Plugin, HTMLSnippet, ResourceTemplates):
return None
return self._try_parse_time('start')
@start.setter
def start(self, value):
if isinstance(value, time.struct_time):
self.metadata['start'] = stringify_time(value)
@property
def own_metadata(self):
"""

View File

@@ -99,20 +99,30 @@ class XmlDescriptor(XModuleDescriptor):
metadata_to_strip = ('data_dir',
# cdodge: @TODO: We need to figure out a way to export out 'tabs' and 'grading_policy' which is on the course
'tabs', 'grading_policy',
'tabs', 'grading_policy', 'is_draft', 'published_by', 'published_date',
'discussion_blackouts',
# VS[compat] -- remove the below attrs once everything is in the CMS
'course', 'org', 'url_name', 'filename')
metadata_to_export_to_policy = ('discussion_topics')
# A dictionary mapping xml attribute names AttrMaps that describe how
# to import and export them
# Allow json to specify either the string "true", or the bool True. The string is preferred.
to_bool = lambda val: val == 'true' or val == True
from_bool = lambda val: str(val).lower()
bool_map = AttrMap(to_bool, from_bool)
to_int = lambda val: int(val)
from_int = lambda val: str(val)
int_map = AttrMap(to_int, from_int)
xml_attribute_map = {
# type conversion: want True/False in python, "true"/"false" in xml
'graded': bool_map,
'hide_progress_tab': bool_map,
'allow_anonymous': bool_map,
'allow_anonymous_to_peers': bool_map,
'weight':int_map
}
@@ -360,8 +370,9 @@ class XmlDescriptor(XModuleDescriptor):
# Add the non-inherited metadata
for attr in sorted(self.own_metadata):
# don't want e.g. data_dir
if attr not in self.metadata_to_strip:
if attr not in self.metadata_to_strip and attr not in self.metadata_to_export_to_policy:
val = val_for_xml(attr)
#logging.debug('location.category = {0}, attr = {1}'.format(self.location.category, attr))
xml_object.set(attr, val)
if self.export_to_file():