Merge pull request #84 from MITx/cms_reorg

Reorganization of course tree. 
Several minor comments, but they can be cleaned up after the merge.

common/lib/capa/capa_problem.py

@@ -1,5 +1,5 @@
 #
-# File:   courseware/capa/capa_problem.py
+# File:   capa/capa_problem.py
 #
 '''
 Main module which shows problems (of "capa" type).
@@ -31,7 +31,7 @@ from responsetypes import NumericalResponse, FormulaResponse, CustomResponse, Sc
 import calc
 import eia
 
-log = logging.getLogger("mitx.courseware")
+log = logging.getLogger(__name__)
 
 response_types = {'numericalresponse':NumericalResponse, 
                   'formularesponse':FormulaResponse,
@@ -231,9 +231,8 @@ class LoncapaProblem(object):
             code = unescape(code,XMLESC)
             try:
                 exec code in context, context		# use "context" for global context; thus defs in code are global within code
-            except Exception,err:
-                log.exception("[courseware.capa.capa_problem.extract_context] error %s" % err)
-                log.exception("in doing exec of this code: %s" % code)
+            except Exception:
+                log.exception("Error while execing code: " + code)
         return context
 
     def get_html(self):
@@ -273,9 +272,6 @@ class LoncapaProblem(object):
             else:
                 msg = ''
 
-            #if settings.DEBUG:
-            #    print "[courseware.capa.capa_problem.extract_html] msg = ",msg
-
             # do the rendering
             # This should be broken out into a helper function
             # that handles all input objects

common/lib/capa/inputtypes.py

@@ -25,9 +25,6 @@ Each input type takes the xml tree as 'element', the previous answer as 'value',
 import re
 import shlex # for splitting quoted strings
 
-from django.conf import settings
-
-from lxml.etree import Element
 from lxml import etree
 
 from mitxmako.shortcuts import render_to_string
@@ -166,8 +163,6 @@ def optioninput(element, value, status, msg=''):
 
     # osetdict = dict([('option_%s_%s' % (eid,x),oset[x]) for x in range(len(oset)) ])	# make dict with IDs
     osetdict = dict([(oset[x],oset[x]) for x in range(len(oset)) ])	# make dict with key,value same
-    if settings.DEBUG:
-        print '[courseware.capa.inputtypes.optioninput] osetdict=',osetdict
     
     context={'id':eid,
              'value':value,
@@ -383,7 +378,5 @@ def imageinput(element, value, status, msg=''):
         'state' : status,	# to change
         'msg': msg,			# to change
         }
-    if settings.DEBUG:
-        print '[courseware.capa.inputtypes.imageinput] context=',context
     html=render_to_string("imageinput.html", context)
     return etree.XML(html)

common/lib/capa/responsetypes.py

@@ -21,12 +21,11 @@ import abc
 
 # specific library imports
 from calc import evaluator, UndefinedVariable
-from django.conf import settings
 from util import contextualize_text
 from lxml import etree
 from lxml.html.soupparser import fromstring as fromstring_bs	# uses Beautiful Soup!!! FIXME?
 
-log = logging.getLogger("mitx.courseware")
+log = logging.getLogger(__name__)
 
 def compare_with_tolerance(v1, v2, tol):
     ''' Compare v1 to v2 with maximum tolerance tol
@@ -144,8 +143,6 @@ class OptionResponse(GenericResponse):
     def __init__(self, xml, context, system=None):
         self.xml = xml
         self.answer_fields = xml.findall('optioninput')
-        if settings.DEBUG:
-            print '[courseware.capa.responsetypes.OR.init] answer_fields=%s' % (self.answer_fields)
         self.context = context
 
     def get_score(self, student_answers):
@@ -274,7 +271,7 @@ def sympy_check2():
             # ie the comparison function is defined in the <script>...</script> stanza instead
             cfn = xml.get('cfn')
             if cfn:
-                if settings.DEBUG: log.info("[courseware.capa.responsetypes] cfn = %s" % cfn)
+                if settings.DEBUG: log.info("cfn = %s" % cfn)
                 if cfn in context:
                     self.code = context[cfn]
                 else:
@@ -779,8 +776,6 @@ class ImageResponse(GenericResponse):
                 correct_map[aid] = 'correct'
             else:
                 correct_map[aid] = 'incorrect'
-        if settings.DEBUG:
-            print "[capamodule.capa.responsetypes.imageinput] correct_map=",correct_map
         return correct_map
 
     def get_answers(self):

common/lib/xmodule/__init__.py

@@ -1,6 +1,3 @@
-import os
-import os.path
-
 import capa_module
 import html_module
 import schematic_module
@@ -9,8 +6,6 @@ import template_module
 import vertical_module
 import video_module
 
-from courseware import content_parser
-
 # Import all files in modules directory, excluding backups (# and . in name) 
 # and __init__
 #

common/lib/xmodule/capa_module.py

@@ -1,31 +1,42 @@
-import StringIO
 import datetime
 import dateutil
 import dateutil.parser
 import json
 import logging
-import math
-import numpy
-import os
-import random
-import scipy
-import struct
-import sys
 import traceback
+import re
 
+from datetime import timedelta
 from lxml import etree
 
 ## TODO: Abstract out from Django
 from mitxmako.shortcuts import render_to_string
 
 from x_module import XModule, XModuleDescriptor
-from courseware.capa.capa_problem import LoncapaProblem, StudentInputError
-import courseware.content_parser as content_parser
-from multicourse import multicourse_settings
-
+from capa.capa_problem import LoncapaProblem, StudentInputError
 log = logging.getLogger("mitx.courseware")
 
 #-----------------------------------------------------------------------------
+TIMEDELTA_REGEX = re.compile(r'^((?P<days>\d+?) day(?:s?))?(\s)?((?P<hours>\d+?) hour(?:s?))?(\s)?((?P<minutes>\d+?) minute(?:s)?)?(\s)?((?P<seconds>\d+?) second(?:s)?)?$')
+
+def item(l, default="", process=lambda x:x):
+    if len(l)==0:
+        return default
+    elif len(l)==1:
+        return process(l[0])
+    else:
+        raise Exception('Malformed XML')
+
+def parse_timedelta(time_str):
+    parts = TIMEDELTA_REGEX.match(time_str)
+    if not parts:
+        return
+    parts = parts.groupdict()
+    time_params = {}
+    for (name, param) in parts.iteritems():
+        if param:
+            time_params[name] = int(param)
+    return timedelta(**time_params)
 
 class ComplexEncoder(json.JSONEncoder):
     def default(self, obj):
@@ -141,11 +152,11 @@ class Module(XModule):
         
         dom2 = etree.fromstring(xml)
         
-        self.explanation="problems/"+content_parser.item(dom2.xpath('/problem/@explain'), default="closed")
-        # TODO: Should be converted to: self.explanation=content_parser.item(dom2.xpath('/problem/@explain'), default="closed")
-        self.explain_available=content_parser.item(dom2.xpath('/problem/@explain_available'))
+        self.explanation="problems/"+item(dom2.xpath('/problem/@explain'), default="closed")
+        # TODO: Should be converted to: self.explanation=item(dom2.xpath('/problem/@explain'), default="closed")
+        self.explain_available=item(dom2.xpath('/problem/@explain_available'))
 
-        display_due_date_string=content_parser.item(dom2.xpath('/problem/@due'))
+        display_due_date_string=item(dom2.xpath('/problem/@due'))
         if len(display_due_date_string)>0:
             self.display_due_date=dateutil.parser.parse(display_due_date_string)
             #log.debug("Parsed " + display_due_date_string + " to " + str(self.display_due_date))
@@ -153,27 +164,27 @@ class Module(XModule):
             self.display_due_date=None
         
         
-        grace_period_string = content_parser.item(dom2.xpath('/problem/@graceperiod'))
+        grace_period_string = item(dom2.xpath('/problem/@graceperiod'))
         if len(grace_period_string)>0 and self.display_due_date:
-            self.grace_period = content_parser.parse_timedelta(grace_period_string)
+            self.grace_period = parse_timedelta(grace_period_string)
             self.close_date = self.display_due_date + self.grace_period
             #log.debug("Then parsed " + grace_period_string + " to closing date" + str(self.close_date))
         else:
             self.grace_period = None
             self.close_date = self.display_due_date
             
-        self.max_attempts=content_parser.item(dom2.xpath('/problem/@attempts'))
+        self.max_attempts=item(dom2.xpath('/problem/@attempts'))
         if len(self.max_attempts)>0:
             self.max_attempts=int(self.max_attempts)
         else:
             self.max_attempts=None
 
-        self.show_answer=content_parser.item(dom2.xpath('/problem/@showanswer'))
+        self.show_answer=item(dom2.xpath('/problem/@showanswer'))
 
         if self.show_answer=="":
             self.show_answer="closed"
 
-        self.rerandomize=content_parser.item(dom2.xpath('/problem/@rerandomize'))
+        self.rerandomize=item(dom2.xpath('/problem/@rerandomize'))
         if self.rerandomize=="" or self.rerandomize=="always" or self.rerandomize=="true":
             self.rerandomize="always"
         elif self.rerandomize=="false" or self.rerandomize=="per_student":
@@ -188,10 +199,10 @@ class Module(XModule):
         if state!=None and 'attempts' in state:
             self.attempts=state['attempts']
 
-        # TODO: Should be: self.filename=content_parser.item(dom2.xpath('/problem/@filename')) 
-        self.filename= "problems/"+content_parser.item(dom2.xpath('/problem/@filename'))+".xml"
-        self.name=content_parser.item(dom2.xpath('/problem/@name'))
-        self.weight=content_parser.item(dom2.xpath('/problem/@weight'))
+        # TODO: Should be: self.filename=item(dom2.xpath('/problem/@filename')) 
+        self.filename= "problems/"+item(dom2.xpath('/problem/@filename'))+".xml"
+        self.name=item(dom2.xpath('/problem/@name'))
+        self.weight=item(dom2.xpath('/problem/@weight'))
         if self.rerandomize == 'never':
             seed = 1
         else:

common/lib/xmodule/schematic_module.py

@@ -1,9 +1,5 @@
 import json
 
-## TODO: Abstract out from Django
-from django.conf import settings
-from mitxmako.shortcuts import render_to_response, render_to_string
-
 from x_module import XModule, XModuleDescriptor
 
 class ModuleDescriptor(XModuleDescriptor):

common/lib/xmodule/template_module.py

@@ -1,7 +1,6 @@
 import json
-import os
 
-from mitxmako.shortcuts import render_to_response, render_to_string
+from mitxmako.shortcuts import render_to_string
 
 from x_module import XModule, XModuleDescriptor
 from lxml import etree
@@ -15,15 +14,7 @@ class Module(XModule):
 
     @classmethod
     def get_xml_tags(c):
-        ## TODO: Abstract out from filesystem and Django
-        ## HACK: For now, this lets us import without abstracting out
-        try: 
-            from django.conf import settings
-            tags = os.listdir(settings.DATA_DIR+'/custom_tags')
-        except: 
-            print "Could not open tags directory."
-            tags = []
-        return tags
+        return ['customtag']
 
     def get_html(self):
         return self.html
@@ -31,6 +22,6 @@ class Module(XModule):
     def __init__(self, system, xml, item_id, state=None):
         XModule.__init__(self, system, xml, item_id, state)
         xmltree = etree.fromstring(xml)
-        filename = xmltree.tag
+        filename = xmltree[0].text
         params = dict(xmltree.items())
         self.html = render_to_string(filename, params, namespace = 'custom_tags')

lms/djangoapps/courseware/content_parser.py

@@ -9,25 +9,20 @@ Does some caching (to be explained).
 
 import logging
 import os
-import re
 import sys
 import urllib
 
-from datetime import timedelta
 from lxml import etree
 from util.memcache import fasthash
 
-try: # This lets us do __name__ == ='__main__'
-    from django.conf import settings
+from django.conf import settings
 
-    from student.models import UserProfile
-    from student.models import UserTestGroup
-    from mitxmako.shortcuts import render_to_string
-    from util.cache import cache
-    from multicourse import multicourse_settings
-except: 
-    print "Could not import/content_parser"
-    settings = None 
+from student.models import UserProfile
+from student.models import UserTestGroup
+from mitxmako.shortcuts import render_to_string
+from util.cache import cache
+from multicourse import multicourse_settings
+import xmodule
 
 ''' This file will eventually form an abstraction layer between the
 course XML file and the rest of the system. 
@@ -40,24 +35,9 @@ class ContentException(Exception):
 
 log = logging.getLogger("mitx.courseware")
 
-
-timedelta_regex = re.compile(r'^((?P<days>\d+?) day(?:s?))?(\s)?((?P<hours>\d+?) hour(?:s?))?(\s)?((?P<minutes>\d+?) minute(?:s)?)?(\s)?((?P<seconds>\d+?) second(?:s)?)?$')
-
 def format_url_params(params):
     return [ urllib.quote(string.replace(' ','_')) for string in params ]
 
-def parse_timedelta(time_str):
-    parts = timedelta_regex.match(time_str)
-    if not parts:
-        return
-    parts = parts.groupdict()
-    time_params = {}
-    for (name, param) in parts.iteritems():
-        if param:
-            time_params[name] = int(param)
-    return timedelta(**time_params)
-
-
 def xpath(xml, query_string, **args):
     ''' Safe xpath query into an xml tree:
         * xml is the tree.
@@ -93,18 +73,9 @@ if __name__=='__main__':
     print xpath('<html><problem name="Bob"></problem></html>', '/{search}/problem[@name="{name}"]', 
                 search='html', name="Bob")
 
-def item(l, default="", process=lambda x:x):
-    if len(l)==0:
-        return default
-    elif len(l)==1:
-        return process(l[0])
-    else:
-        raise Exception('Malformed XML')
-
 def id_tag(course):
     ''' Tag all course elements with unique IDs '''
-    import courseware.modules
-    default_ids = courseware.modules.get_default_ids()
+    default_ids = xmodule.get_default_ids()
 
     # Tag elements with unique IDs
     elements = course.xpath("|".join(['//'+c for c in default_ids]))
@@ -166,11 +137,20 @@ def user_groups(user):
 
     # return [u.name for u in UserTestGroup.objects.raw("select * from auth_user, student_usertestgroup, student_usertestgroup_users where auth_user.id = student_usertestgroup_users.user_id and student_usertestgroup_users.usertestgroup_id = student_usertestgroup.id and auth_user.id = %s", [user.id])]
 
+def replace_custom_tags(tree):
+    tags = os.listdir(settings.DATA_DIR+'/custom_tags')
+    for tag in tags:
+        for element in tree.iter(tag):
+            element.tag = 'customtag'
+            impl = etree.SubElement(element, 'impl')
+            impl.text = tag
+
 def course_xml_process(tree):
     ''' Do basic pre-processing of an XML tree. Assign IDs to all
     items without. Propagate due dates, grace periods, etc. to child
     items. 
     '''
+    replace_custom_tags(tree)
     id_tag(tree)
     propogate_downward_tag(tree, "due")
     propogate_downward_tag(tree, "graded")

lms/djangoapps/courseware/grades.py

@@ -29,7 +29,7 @@ from courseware import graders
 from courseware.graders import Score
 from models import StudentModule
 import courseware.content_parser as content_parser
-import courseware.modules
+import xmodule
 
 _log = logging.getLogger("mitx.courseware")
 
@@ -197,7 +197,7 @@ def get_score(user, problem, cache, coursename=None):
         ## HACK 2: Backwards-compatibility: This should be written when a grade is saved, and removed from the system
         from module_render import I4xSystem
         system = I4xSystem(None, None, None, coursename=coursename)
-        total=float(courseware.modules.capa_module.Module(system, etree.tostring(problem), "id").max_score())
+        total=float(xmodule.capa_module.Module(system, etree.tostring(problem), "id").max_score())
         response.max_grade = total
         response.save()
         

lms/djangoapps/courseware/management/commands/check_course.py

@@ -8,7 +8,7 @@ from django.contrib.auth.models import User
 
 from courseware.content_parser import course_file
 import courseware.module_render
-import courseware.modules
+import xmodule
 
 class Command(BaseCommand):
     help = "Does basic validity tests on course.xml."
@@ -25,7 +25,7 @@ class Command(BaseCommand):
                 check = False
         print "Confirming all modules render. Nothing should print during this step. "
         for module in course.xpath('//problem|//html|//video|//vertical|//sequential|/tab'):
-            module_class = courseware.modules.modx_modules[module.tag]
+            module_class = xmodule.modx_modules[module.tag]
             # TODO: Abstract this out in render_module.py
             try: 
                 module_class(etree.tostring(module), 

lms/djangoapps/courseware/module_render.py

@@ -6,19 +6,18 @@ from lxml import etree
 from django.http import Http404
 from django.http import HttpResponse
 from django.shortcuts import redirect
-from django.template import Context
-from django.template import Context, loader
 
 from fs.osfs import OSFS
 
 from django.conf import settings
-from mitxmako.shortcuts import render_to_string
+from mitxmako.shortcuts import render_to_string, render_to_response
 
 from models import StudentModule
 from multicourse import multicourse_settings
+from util.views import accepts
 
-import courseware.modules
 import courseware.content_parser as content_parser
+import xmodule
 
 log = logging.getLogger("mitx.courseware")
 
@@ -90,7 +89,7 @@ def grade_histogram(module_id):
 
 def get_module(user, request, xml_module, module_object_preload, position=None):
     module_type=xml_module.tag
-    module_class=courseware.modules.get_module_class(module_type)
+    module_class=xmodule.get_module_class(module_type)
     module_id=xml_module.get('id') #module_class.id_attribute) or "" 
 
     # Grab state from database
@@ -231,7 +230,7 @@ def modx_dispatch(request, module=None, dispatch=None, id=None):
                        )
 
     try:
-        instance=courseware.modules.get_module_class(module)(system, 
+        instance=xmodule.get_module_class(module)(system, 
                                                              xml, 
                                                              id, 
                                                              state=oldstate)

lms/djangoapps/courseware/tests.py

@@ -10,9 +10,9 @@ import os
 
 import numpy
 
-import courseware.modules
-import courseware.capa.calc as calc
-import courseware.capa.capa_problem as lcp
+import xmodule
+import capa.calc as calc
+import capa.capa_problem as lcp
 import courseware.graders as graders
 from courseware.graders import Score, CourseGrader, WeightedSubsectionsGrader, SingleSectionGrader, AssignmentFormatGrader
 from courseware.grades import aggregate_scores
@@ -41,10 +41,10 @@ class ModelsTest(unittest.TestCase):
         pass
 
     def test_get_module_class(self):
-        vc = courseware.modules.get_module_class('video')
-        vc_str = "<class 'courseware.modules.video_module.Module'>"
+        vc = xmodule.get_module_class('video')
+        vc_str = "<class 'xmodule.video_module.Module'>"
         self.assertEqual(str(vc), vc_str)
-        video_id = courseware.modules.get_default_ids()['video']
+        video_id = xmodule.get_default_ids()['video']
         self.assertEqual(video_id, 'youtube')
 
     def test_calc(self):
@@ -110,7 +110,7 @@ class MultiChoiceTest(unittest.TestCase):
         self.assertEquals(test_lcp.grade_answers(false_answers)['1_2_1'], 'incorrect')
         
     def test_TF_grade(self):
-        truefalse_file =  os.getcwd()+"/djangoapps/courseware/test_files/truefalse.xml"
+        truefalse_file =  os.path.dirname(__file__)+"/test_files/truefalse.xml"
         test_lcp = lcp.LoncapaProblem(open(truefalse_file), '1', system=i4xs)
         correct_answers = {'1_2_1':['choice_foil2', 'choice_foil1']}
         self.assertEquals(test_lcp.grade_answers(correct_answers)['1_2_1'], 'correct')

lms/djangoapps/courseware/views.py

@@ -20,9 +20,9 @@ from module_render import render_x_module, make_track_function, I4xSystem
 from models import StudentModule
 from student.models import UserProfile
 from multicourse import multicourse_settings
+import xmodule
 
 import courseware.content_parser as content_parser
-import courseware.modules
 
 import courseware.grades as grades
 
@@ -288,4 +288,3 @@ def jump_to(request, probname=None):
         position = parent.index(pxml)+1	# position in sequence
         
     return index(request,course=coursename,chapter=chapter,section=section,position=position)
-

lms/djangoapps/multicourse/views.py

@@ -1,18 +1,6 @@
-import datetime
-import json
-import sys
-
 from django.conf import settings
-from django.contrib.auth.models import User
-from django.core.context_processors import csrf
-from django.core.mail import send_mail
-from django.http import Http404
-from django.http import HttpResponse
-from django.shortcuts import redirect
-from mitxmako.shortcuts import render_to_response, render_to_string
+from mitxmako.shortcuts import render_to_response
 
-import courseware.capa.calc
-import track.views
 from multicourse import multicourse_settings
 
 def mitxhome(request):