Merge branch 'feature/cale/cms-master' of github.com:MITx/mitx into feature/cdodge/export

Conflicts:
	common/lib/xmodule/xmodule/contentstore/content.py
	common/lib/xmodule/xmodule/contentstore/mongo.py

common/djangoapps/models/course_relative.py

@@ -0,0 +1,25 @@
+class CourseRelativeMember:
+    def __init__(self, location, idx):
+        self.course_location = location     # a Location obj
+        self.idx = idx                   # which milestone this represents. Hopefully persisted # so we don't have race conditions
+    
+### ??? If 2+ courses use the same textbook or other asset, should they point to the same db record?        
+class linked_asset(CourseRelativeMember):
+    """
+    Something uploaded to our asset lib which has a name/label and location. Here it's tracked by course and index, but
+    we could replace the label/url w/ a pointer to a real asset and keep the join info here.
+    """
+    def __init__(self, location, idx):
+        CourseRelativeMember.__init__(self, location, idx)
+        self.label = ""
+        self.url = None
+        
+class summary_detail_pair(CourseRelativeMember):
+    """
+    A short text with an arbitrary html descriptor used for paired label - details elements.
+    """
+    def __init__(self, location, idx):
+        CourseRelativeMember.__init__(self, location, idx)
+        self.summary = ""
+        self.detail = ""
+        

common/djangoapps/util/converters.py

@@ -0,0 +1,24 @@
+import time, datetime
+import re
+import calendar
+
+def time_to_date(time_obj):
+    """
+    Convert a time.time_struct to a true universal time (can pass to js Date constructor)
+    """
+    # TODO change to using the isoformat() function on datetime. js date can parse those
+    return calendar.timegm(time_obj) * 1000
+
+def jsdate_to_time(field):
+    """
+    Convert a universal time (iso format) or msec since epoch to a time obj
+    """
+    if field is None:
+        return field
+    elif isinstance(field, unicode) or isinstance(field, str):  # iso format but ignores time zone assuming it's Z
+        d=datetime.datetime(*map(int, re.split('[^\d]', field)[:6])) # stop after seconds. Debatable  
+        return d.utctimetuple()
+    elif isinstance(field, int) or isinstance(field, float):
+        return time.gmtime(field / 1000)
+    elif isinstance(field, time.struct_time):
+        return field

common/lib/capa/capa/responsetypes.py

@@ -1133,7 +1133,13 @@ class CodeResponse(LoncapaResponse):
         xml = self.xml
         # TODO: XML can override external resource (grader/queue) URL
         self.url = xml.get('url', None)
-        self.queue_name = xml.get('queuename', self.system.xqueue['default_queuename'])
+
+        # We do not support xqueue within Studio.
+        if self.system.xqueue is not None:
+            default_queuename = self.system.xqueue['default_queuename']
+        else:
+            default_queuename = None
+        self.queue_name = xml.get('queuename', default_queuename)
 
         # VS[compat]:
         #   Check if XML uses the ExternalResponse format or the generic CodeResponse format
@@ -1230,6 +1236,13 @@ class CodeResponse(LoncapaResponse):
                 (err, self.answer_id, convert_files_to_filenames(student_answers)))
             raise Exception(err)
 
+        # We do not support xqueue within Studio.
+        if self.system.xqueue is None:
+            cmap = CorrectMap()
+            cmap.set(self.answer_id, queuestate=None,
+                msg='Error checking problem: no external queueing server is configured.')
+            return cmap
+
         # Prepare xqueue request
         #------------------------------------------------------------
 

common/lib/xmodule/jasmine_test_runner.html.erb

@@ -12,6 +12,7 @@
   <script type="text/javascript" src="<%= common_coffee_root %>/logger.js"></script>
   <script type="text/javascript" src="<%= common_js_root %>/vendor/jquery.min.js"></script>
   <script type="text/javascript" src="<%= common_js_root %>/vendor/jasmine-jquery.js"></script>
+  <script type="text/javascript" src="<%= common_js_root %>/vendor/CodeMirror/codemirror.js"></script>
   <script type="text/javascript" src="<%= common_js_root %>/vendor/mathjax-MathJax-c9db6ac/MathJax.js"></script>
 
   <script type="text/javascript">

common/lib/xmodule/xmodule/contentstore/content.py

@@ -18,7 +18,7 @@ class StaticContent(object):
         self.content_type = content_type
         self.data = data
         self.last_modified_at = last_modified_at
-        self.thumbnail_location = thumbnail_location
+        self.thumbnail_location = Location(thumbnail_location)
         # optional information about where this file was imported from. This is needed to support import/export 
         # cycles
         self.import_path = import_path

common/lib/xmodule/xmodule/contentstore/mongo.py

@@ -31,8 +31,7 @@ class MongoContentStore(ContentStore):
         id = content.get_id()
 
         # Seems like with the GridFS we can't update existing ID's we have to do a delete/add pair
-        if self.fs.exists({"_id" : id}):
-            self.fs.delete(id)
+        self.delete(id)
 
         with self.fs.new_file(_id = id, filename=content.get_url_path(), content_type=content.content_type, 
             displayname=content.name, thumbnail_location=content.thumbnail_location, import_path=content.import_path) as fp:
@@ -41,13 +40,16 @@ class MongoContentStore(ContentStore):
         
         return content
         
-    
+    def delete(self, id):
+        if self.fs.exists({"_id" : id}):
+            self.fs.delete(id)
+
     def find(self, location):
         id = StaticContent.get_id_from_location(location)
         try:
             with self.fs.get(id) as fp:
                 return StaticContent(location, fp.displayname, fp.content_type, fp.read(), 
-                    fp.uploadDate, thumbnail_location = fp.thumbnail_location if 'thumbnail_location' in fp else None,
+                    fp.uploadDate, thumbnail_location = fp.thumbnail_location if hasattr(fp, 'thumbnail_location') else None,
                     import_path = fp.import_path if hasattr(fp, 'import_path') else None)
         except NoFile:
             raise NotFoundError()

common/lib/xmodule/xmodule/course_module.py

@@ -1,18 +1,17 @@
-from fs.errors import ResourceNotFoundError
-import logging
-import json
+from cStringIO import StringIO
 from lxml import etree
 from path import path # NOTE (THK): Only used for detecting presence of syllabus
-import requests
-import time
-from cStringIO import StringIO
-
-from xmodule.util.decorators import lazyproperty
+from xmodule.graders import grader_from_conf
 from xmodule.modulestore import Location
 from xmodule.seq_module import SequenceDescriptor, SequenceModule
-from xmodule.xml_module import XmlDescriptor
 from xmodule.timeparse import parse_time, stringify_time
-from xmodule.graders import grader_from_conf
+from xmodule.util.decorators import lazyproperty
+import json
+import logging
+import requests
+import time
+import copy
+
 
 log = logging.getLogger(__name__)
 
@@ -92,10 +91,6 @@ class CourseDescriptor(SequenceDescriptor):
             log.critical(msg)
             system.error_tracker(msg)
 
-        self.enrollment_start = self._try_parse_time("enrollment_start")
-        self.enrollment_end = self._try_parse_time("enrollment_end")
-        self.end = self._try_parse_time("end")
-
         # NOTE: relies on the modulestore to call set_grading_policy() right after
         # init.  (Modulestore is in charge of figuring out where to load the policy from)
 
@@ -105,19 +100,11 @@ class CourseDescriptor(SequenceDescriptor):
 
         self.set_grading_policy(self.definition['data'].get('grading_policy', None))
 
-
-    def set_grading_policy(self, course_policy):
-        if course_policy is None:
-            course_policy = {}
-
+    def defaut_grading_policy(self):
         """
-        The JSON object can have the keys GRADER and GRADE_CUTOFFS. If either is
-        missing, it reverts to the default.
+        Return a dict which is a copy of the default grading policy
         """
-
-        default_policy_string = """
-        {
-            "GRADER" : [
+        default = {"GRADER" : [
                 {
                     "type" : "Homework",
                     "min_count" : 12,
@@ -129,37 +116,44 @@ class CourseDescriptor(SequenceDescriptor):
                     "type" : "Lab",
                     "min_count" : 12,
                     "drop_count" : 2,
-                    "category" : "Labs",
                     "weight" : 0.15
                 },
                 {
-                    "type" : "Midterm",
-                    "name" : "Midterm Exam",
+                    "type" : "Midterm Exam",
                     "short_label" : "Midterm",
+                    "min_count" : 1,
+                    "drop_count" : 0,
                     "weight" : 0.3
                 },
                 {
-                    "type" : "Final",
-                    "name" : "Final Exam",
+                    "type" : "Final Exam",
                     "short_label" : "Final",
+                    "min_count" : 1,
+                    "drop_count" : 0,
                     "weight" : 0.4
                 }
             ],
             "GRADE_CUTOFFS" : {
-                "A" : 0.87,
-                "B" : 0.7,
-                "C" : 0.6
-            }
-        }
+                "Pass" : 0.5
+            }}
+        return copy.deepcopy(default)
+
+    def set_grading_policy(self, course_policy):
         """
+        The JSON object can have the keys GRADER and GRADE_CUTOFFS. If either is
+        missing, it reverts to the default.
+        """
+        if course_policy is None:
+            course_policy = {}
 
         # Load the global settings as a dictionary
-        grading_policy = json.loads(default_policy_string)
+        grading_policy = self.defaut_grading_policy()
 
         # Override any global settings with the course settings
         grading_policy.update(course_policy)
 
         # Here is where we should parse any configurations, so that we can fail early
+        grading_policy['RAW_GRADER'] = grading_policy['GRADER']  # used for cms access
         grading_policy['GRADER'] = grader_from_conf(grading_policy['GRADER'])
         self._grading_policy = grading_policy
 
@@ -168,8 +162,8 @@ class CourseDescriptor(SequenceDescriptor):
     @classmethod
     def read_grading_policy(cls, paths, system):
         """Load a grading policy from the specified paths, in order, if it exists."""
-        # Default to a blank policy
-        policy_str = '""'
+        # Default to a blank policy dict
+        policy_str = '{}'
 
         for policy_path in paths:
             if not system.resources_fs.exists(policy_path):
@@ -251,13 +245,53 @@ class CourseDescriptor(SequenceDescriptor):
     def has_started(self):
         return time.gmtime() > self.start
 
+    @property
+    def end(self):
+        return self._try_parse_time("end")
+    @end.setter
+    def end(self, value):
+        if isinstance(value, time.struct_time):
+            self.metadata['end'] = stringify_time(value)
+    @property
+    def enrollment_start(self):
+        return self._try_parse_time("enrollment_start")
+        
+    @enrollment_start.setter
+    def enrollment_start(self, value):
+        if isinstance(value, time.struct_time):
+            self.metadata['enrollment_start'] = stringify_time(value)
+    @property
+    def enrollment_end(self):        
+        return self._try_parse_time("enrollment_end")
+        
+    @enrollment_end.setter
+    def enrollment_end(self, value):
+        if isinstance(value, time.struct_time):
+            self.metadata['enrollment_end'] = stringify_time(value)
+        
     @property
     def grader(self):
         return self._grading_policy['GRADER']
+    
+    @property
+    def raw_grader(self):
+        return self._grading_policy['RAW_GRADER']
+    
+    @raw_grader.setter
+    def raw_grader(self, value):
+        # NOTE WELL: this change will not update the processed graders. If we need that, this needs to call grader_from_conf
+        self._grading_policy['RAW_GRADER'] = value
+        self.definition['data'].setdefault('grading_policy',{})['GRADER'] = value
 
     @property
     def grade_cutoffs(self):
         return self._grading_policy['GRADE_CUTOFFS']
+    
+    @grade_cutoffs.setter
+    def grade_cutoffs(self, value):
+        self._grading_policy['GRADE_CUTOFFS'] = value
+        self.definition['data'].setdefault('grading_policy',{})['GRADE_CUTOFFS'] = value
+    
 
     @property
     def tabs(self):

common/lib/xmodule/xmodule/js/fixtures/html-edit-formattingbug.html

@@ -0,0 +1,8 @@
+<section class="html-edit">
+    <div name="" class="edit-box" rows="8" cols="40">&lt;problem>
+&lt;p>&lt;/p>
+&lt;multiplechoiceresponse>
+<pre>&lt;problem>
+    &lt;p>&lt;/p></pre>
+<div><foo>bar</foo></div></div>
+</section>

common/lib/xmodule/xmodule/js/spec/.gitignore

@@ -0,0 +1,2 @@
+*.js
+

common/lib/xmodule/xmodule/js/spec/html/edit_spec.coffee

@@ -0,0 +1,17 @@
+describe 'HTMLEditingDescriptor', ->
+  describe 'Read data from server, create Editor, and get data back out', ->
+    it 'Does not munge &lt', ->
+#     This is a test for Lighthouse #22,
+#     "html names are automatically converted to the symbols they describe"
+#     A better test would be a Selenium test to avoid duplicating the
+#     mako template structure in html-edit-formattingbug.html.
+#     However, we currently have no working Selenium tests.
+      loadFixtures 'html-edit-formattingbug.html'
+      @descriptor = new HTMLEditingDescriptor($('.html-edit'))
+      data = @descriptor.save().data
+      expect(data).toEqual("""<problem>
+                              <p></p>
+                              <multiplechoiceresponse>
+                              <pre>&lt;problem&gt;
+                                  &lt;p&gt;&lt;/p&gt;</pre>
+                              <div><foo>bar</foo></div>""")

common/lib/xmodule/xmodule/js/src/.gitignore

@@ -0,0 +1,2 @@
+*.js
+

common/lib/xmodule/xmodule/js/src/html/edit.coffee

@@ -1,11 +1,12 @@
 class @HTMLEditingDescriptor
   constructor: (@element) ->
-    @edit_box = CodeMirror.fromTextArea($(".edit-box", @element)[0], {
+    text = $(".edit-box", @element)[0];
+    replace_func = (elt) -> text.parentNode.replaceChild(elt, text)
+    @edit_box = CodeMirror(replace_func, {
+      value: text.innerHTML
       mode: "text/html"
       lineNumbers: true
-      lineWrapping: true
-    })
+      lineWrapping: true})
 
   save: ->
     data: @edit_box.getValue()
-

common/lib/xmodule/xmodule/modulestore/mongo.py

@@ -51,6 +51,9 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
         self.modulestore = modulestore
         self.module_data = module_data
         self.default_class = default_class
+        # cdodge: other Systems have a course_id attribute defined. To keep things consistent, let's
+        # define an attribute here as well, even though it's None
+        self.course_id = None
 
     def load_item(self, location):
         location = Location(location)

common/lib/xmodule/xmodule/modulestore/store_utilities.py

@@ -0,0 +1,130 @@
+import logging
+from xmodule.contentstore.content import StaticContent
+from xmodule.modulestore import Location
+from xmodule.modulestore.mongo import MongoModuleStore
+
+def clone_course(modulestore, contentstore, source_location, dest_location, delete_original=False):
+  # first check to see if the modulestore is Mongo backed
+  if not isinstance(modulestore, MongoModuleStore):
+    raise Exception("Expected a MongoModuleStore in the runtime. Aborting....")
+
+  # check to see if the dest_location exists as an empty course
+  # we need an empty course because the app layers manage the permissions and users
+  if not modulestore.has_item(dest_location):
+    raise Exception("An empty course at {0} must have already been created. Aborting...".format(dest_location))
+
+  # verify that the dest_location really is an empty course, which means only one 
+  dest_modules = modulestore.get_items([dest_location.tag, dest_location.org, dest_location.course, None, None, None])
+
+  if len(dest_modules) != 1:
+    raise Exception("Course at destination {0} is not an empty course. You can only clone into an empty course. Aborting...".format(dest_location))
+
+  # check to see if the source course is actually there
+  if not modulestore.has_item(source_location):
+    raise Exception("Cannot find a course at {0}. Aborting".format(source_location))
+
+  # Get all modules under this namespace which is (tag, org, course) tuple
+
+  modules = modulestore.get_items([source_location.tag, source_location.org, source_location.course, None, None, None])
+
+  for module in modules:
+    original_loc = Location(module.location)
+
+    if original_loc.category != 'course':
+      module.location = module.location._replace(tag = dest_location.tag, org = dest_location.org, 
+        course = dest_location.course)
+    else:
+      # on the course module we also have to update the module name
+      module.location = module.location._replace(tag = dest_location.tag, org = dest_location.org, 
+        course = dest_location.course, name=dest_location.name)
+
+    print "Cloning module {0} to {1}....".format(original_loc, module.location)
+
+    if 'data' in module.definition:
+      modulestore.update_item(module.location, module.definition['data'])
+
+    # repoint children
+    if 'children' in module.definition:
+      new_children = []
+      for child_loc_url in module.definition['children']:
+        child_loc = Location(child_loc_url)
+        child_loc = child_loc._replace(tag = dest_location.tag, org = dest_location.org, 
+        course = dest_location.course)
+        new_children = new_children + [child_loc.url()]
+
+      modulestore.update_children(module.location, new_children)
+
+    # save metadata
+    modulestore.update_metadata(module.location, module.metadata)
+
+  # now iterate through all of the assets and clone them
+  # first the thumbnails
+  thumbs = contentstore.get_all_content_thumbnails_for_course(source_location)
+  for thumb in thumbs:
+    thumb_loc = Location(thumb["_id"])
+    content = contentstore.find(thumb_loc)
+    content.location = content.location._replace(org = dest_location.org, 
+        course = dest_location.course)
+
+    print "Cloning thumbnail {0} to {1}".format(thumb_loc, content.location)
+
+    contentstore.save(content)
+
+  # now iterate through all of the assets, also updating the thumbnail pointer
+
+  assets = contentstore.get_all_content_for_course(source_location)
+  for asset in assets:
+    asset_loc = Location(asset["_id"])
+    content = contentstore.find(asset_loc)
+    content.location = content.location._replace(org = dest_location.org, 
+        course = dest_location.course)
+
+    # be sure to update the pointer to the thumbnail
+    if content.thumbnail_location is not None:
+      content.thumbnail_location = content.thumbnail_location._replace(org = dest_location.org, 
+        course = dest_location.course)
+
+    print "Cloning asset {0} to {1}".format(asset_loc, content.location)
+
+    contentstore.save(content)
+
+  return True
+
+def delete_course(modulestore, contentstore, source_location):
+    # first check to see if the modulestore is Mongo backed
+  if not isinstance(modulestore, MongoModuleStore):
+    raise Exception("Expected a MongoModuleStore in the runtime. Aborting....")
+
+  # check to see if the source course is actually there
+  if not modulestore.has_item(source_location):
+    raise Exception("Cannot find a course at {0}. Aborting".format(source_location))
+
+  # first delete all of the thumbnails
+  thumbs = contentstore.get_all_content_thumbnails_for_course(source_location)
+  for thumb in thumbs:
+    thumb_loc = Location(thumb["_id"])
+    id = StaticContent.get_id_from_location(thumb_loc)
+    print "Deleting {0}...".format(id)
+    contentstore.delete(id)
+
+  # then delete all of the assets
+  assets = contentstore.get_all_content_for_course(source_location)
+  for asset in assets:
+    asset_loc = Location(asset["_id"])
+    id = StaticContent.get_id_from_location(asset_loc)
+    print "Deleting {0}...".format(id)
+    contentstore.delete(id)
+
+  # then delete all course modules
+  modules = modulestore.get_items([source_location.tag, source_location.org, source_location.course, None, None, None])
+
+  for module in modules:
+    if module.category != 'course': # save deleting the course module for last
+      print "Deleting {0}...".format(module.location)
+      modulestore.delete_item(module.location)
+
+  # finally delete the top-level course module itself
+  print "Deleting {0}...".format(source_location)
+  modulestore.delete_item(source_location)
+
+  return True

common/lib/xmodule/xmodule/templates/problem/forumularesponse.yaml

@@ -1,6 +1,6 @@
 ---
 metadata:
-    display_name: Formula Repsonse
+    display_name: Formula Response
     rerandomize: never
     showanswer: always
 data: |

common/lib/xmodule/xmodule/video_module.py

@@ -22,7 +22,8 @@ class VideoModule(XModule):
          resource_string(__name__, 'js/src/video/display.coffee')] +
         [resource_string(__name__, 'js/src/video/display/' + filename)
          for filename
-         in sorted(resource_listdir(__name__, 'js/src/video/display'))]}
+         in sorted(resource_listdir(__name__, 'js/src/video/display'))
+         if filename.endswith('.coffee')]}
     css = {'scss': [resource_string(__name__, 'css/video/display.scss')]}
     js_module_name = "Video"
 

common/lib/xmodule/xmodule/x_module.py

@@ -10,10 +10,11 @@ from collections import namedtuple
 from pkg_resources import resource_listdir, resource_string, resource_isdir
 
 from xmodule.modulestore import Location
-from xmodule.timeparse import parse_time
+from xmodule.timeparse import parse_time, stringify_time
 
 from xmodule.contentstore.content import StaticContent, XASSET_SRCREF_PREFIX
 from xmodule.modulestore.exceptions import ItemNotFoundError
+import time
 
 log = logging.getLogger('mitx.' + __name__)
 
@@ -494,6 +495,11 @@ class XModuleDescriptor(Plugin, HTMLSnippet, ResourceTemplates):
             return None
         return self._try_parse_time('start')
 
+    @start.setter
+    def start(self, value):
+        if isinstance(value, time.struct_time):
+            self.metadata['start'] = stringify_time(value)
+        
     @property
     def own_metadata(self):
         """

common/static/coffee/src/discussion/tooltip_manager.coffee

@@ -12,10 +12,10 @@ class @TooltipManager
       'click': @hideTooltip
 
   showTooltip: (e) =>
-    tooltipText = $(e.target).attr('data-tooltip')
+    $target = $(e.target).closest('[data-tooltip]')
+    tooltipText = $target.attr('data-tooltip')
     @$tooltip.html(tooltipText)
     @$body.append(@$tooltip)
-    $(e.target).children().css('pointer-events', 'none')
 
     tooltipCoords =
       x: e.pageX - (@$tooltip.outerWidth() / 2)
@@ -26,8 +26,8 @@ class @TooltipManager
     'top': tooltipCoords.y
 
     @tooltipTimer = setTimeout ()=>
-      @$tooltip.show().css('opacity', 1)
 
+      @$tooltip.show().css('opacity', 1)
       @tooltipTimer = setTimeout ()=>
         @hideTooltip()
       , 3000

common/test/data/graded/course.xml

@@ -1 +0,0 @@
-roots/2012_Fall.xml

common/test/data/graded/course.xml

@@ -0,0 +1 @@
+<course org="edX" course="graded" url_name="2012_Fall"/>

common/test/data/self_assessment/course.xml

@@ -1 +0,0 @@
-roots/2012_Fall.xml

common/test/data/self_assessment/course.xml

@@ -0,0 +1 @@
+<course org="edX" course="sa_test" url_name="2012_Fall"/>

common/test/data/test_start_date/course.xml

@@ -1 +0,0 @@
-roots/2012_Fall.xml

common/test/data/test_start_date/course.xml

@@ -0,0 +1 @@
+<course org="edX" course="test_start_date" url_name="2012_Fall"/>

common/test/data/toy/course.xml

@@ -1 +0,0 @@
-roots/2012_Fall.xml

common/test/data/toy/course.xml

@@ -0,0 +1 @@
+<course org="edX" course="toy" url_name="2012_Fall"/>