Splitting files into subdirectories in preparation for merge

djangoapps/courseware/capa/__init__.py

@@ -0,0 +1 @@
+

djangoapps/courseware/capa/calc.py

@@ -0,0 +1,220 @@
+import copy
+import logging
+import math
+import operator
+import re
+
+import numpy
+import scipy.constants
+
+from pyparsing import Word, alphas, nums, oneOf, Literal
+from pyparsing import ZeroOrMore, OneOrMore, StringStart
+from pyparsing import StringEnd, Optional, Forward
+from pyparsing import CaselessLiteral, Group, StringEnd
+from pyparsing import NoMatch, stringEnd, alphanums
+
+default_functions = {'sin' : numpy.sin, 
+                     'cos' : numpy.cos, 
+                     'tan' : numpy.tan, 
+                     'sqrt': numpy.sqrt,
+                     'log10':numpy.log10, 
+                     'log2':numpy.log2, 
+                     'ln': numpy.log,
+                     'arccos':numpy.arccos,
+                     'arcsin':numpy.arcsin,
+                     'arctan':numpy.arctan, 
+                     'abs':numpy.abs
+                     }
+default_variables = {'j':numpy.complex(0,1), 
+                     'e':numpy.e,
+                     'pi':numpy.pi, 
+                     'k':scipy.constants.k,
+                     'c':scipy.constants.c, 
+                     'T':298.15,
+                     'q':scipy.constants.e
+                     }
+
+log = logging.getLogger("mitx.courseware.capa")
+
+class UndefinedVariable(Exception):
+    def raiseself(self):
+        ''' Helper so we can use inside of a lambda '''
+        raise self
+
+
+general_whitespace = re.compile('[^\w]+')
+def check_variables(string, variables):
+    '''  Confirm the only variables in string are defined.  
+
+    Pyparsing uses a left-to-right parser, which makes the more
+    elegant approach pretty hopeless. 
+
+    achar = reduce(lambda a,b:a|b ,map(Literal,alphas)) # Any alphabetic character
+    undefined_variable = achar + Word(alphanums)
+    undefined_variable.setParseAction(lambda x:UndefinedVariable("".join(x)).raiseself())
+    varnames = varnames | undefined_variable'''
+    possible_variables = re.split(general_whitespace, string) # List of all alnums in string
+    bad_variables = list()
+    for v in possible_variables:
+        if len(v) == 0:
+            continue
+        if v[0] <= '9' and '0' <= 'v': # Skip things that begin with numbers
+            continue
+        if v not in variables: 
+            bad_variables.append(v)
+    if len(bad_variables)>0:
+        raise UndefinedVariable(' '.join(bad_variables))
+
+def evaluator(variables, functions, string, cs=False):
+    ''' Evaluate an expression. Variables are passed as a dictionary
+    from string to value. Unary functions are passed as a dictionary
+    from string to function. Variables must be floats.
+    cs: Case sensitive
+
+    TODO: Fix it so we can pass integers and complex numbers in variables dict
+    '''
+    # log.debug("variables: {0}".format(variables))
+    # log.debug("functions: {0}".format(functions))
+    # log.debug("string: {0}".format(string))
+
+    all_variables = copy.copy(default_variables)
+    all_variables.update(variables)
+    all_functions = copy.copy(default_functions)
+    all_functions.update(functions)
+    
+    if not cs: 
+        string_cs = string.lower()
+        for v in all_variables.keys():
+            all_variables[v.lower()]=all_variables[v]
+        for f in all_functions.keys():
+            all_functions[f.lower()]=all_functions[f]
+        CasedLiteral = CaselessLiteral 
+    else:
+        string_cs = string
+        CasedLiteral = Literal
+
+    check_variables(string_cs, set(all_variables.keys()+all_functions.keys()))
+
+    if string.strip() == "":
+        return float('nan')
+    ops = { "^" : operator.pow,
+            "*" : operator.mul,
+            "/" : operator.truediv,
+            "+" : operator.add,
+            "-" : operator.sub,
+            }
+    # We eliminated extreme ones, since they're rarely used, and potentially
+    # confusing. They may also conflict with variables if we ever allow e.g. 
+    # 5R instead of 5*R
+    suffixes={'%':0.01,'k':1e3,'M':1e6,'G':1e9,
+              'T':1e12,#'P':1e15,'E':1e18,'Z':1e21,'Y':1e24,
+              'c':1e-2,'m':1e-3,'u':1e-6,
+              'n':1e-9,'p':1e-12}#,'f':1e-15,'a':1e-18,'z':1e-21,'y':1e-24}
+    
+    def super_float(text):
+        ''' Like float, but with si extensions. 1k goes to 1000'''
+        if text[-1] in suffixes:
+            return float(text[:-1])*suffixes[text[-1]]
+        else:
+            return float(text)
+
+    def number_parse_action(x): # [ '7' ] ->  [ 7 ]
+        return [super_float("".join(x))]
+    def exp_parse_action(x): # [ 2 ^ 3 ^ 2 ] -> 512
+        x = [e for e in x if type(e) in [float, numpy.float64, numpy.complex]] # Ignore ^
+        x.reverse()
+        x=reduce(lambda a,b:b**a, x)
+        return x
+    def parallel(x): # Parallel resistors [ 1 2 ] => 2/3
+        if len(x) == 1:
+            return x[0]
+        if 0 in x:
+            return float('nan')
+        x = [1./e for e in x if type(e) == float] # Ignore ^
+        return 1./sum(x)
+    def sum_parse_action(x): # [ 1 + 2 - 3 ] -> 0
+        total = 0.0
+        op = ops['+']
+        for e in x:
+            if e in set('+-'):
+                op = ops[e]
+            else:
+                total=op(total, e)
+        return total
+    def prod_parse_action(x): # [ 1 * 2 / 3 ] => 0.66
+        prod = 1.0
+        op = ops['*']
+        for e in x:
+            if e in set('*/'):
+                op = ops[e]
+            else:
+                prod=op(prod, e)
+        return prod
+    def func_parse_action(x):
+        return [all_functions[x[0]](x[1])]
+
+    number_suffix=reduce(lambda a,b:a|b, map(Literal,suffixes.keys()), NoMatch()) # SI suffixes and percent
+    (dot,minus,plus,times,div,lpar,rpar,exp)=map(Literal,".-+*/()^")
+    
+    number_part=Word(nums)
+    inner_number = ( number_part+Optional("."+number_part) ) | ("."+number_part) # 0.33 or 7 or .34
+    number=Optional(minus | plus)+ inner_number + \
+        Optional(CaselessLiteral("E")+Optional("-")+number_part)+ \
+        Optional(number_suffix) # 0.33k or -17
+    number=number.setParseAction( number_parse_action ) # Convert to number
+    
+    # Predefine recursive variables
+    expr = Forward() 
+    factor = Forward()
+    
+    def sreduce(f, l):
+        ''' Same as reduce, but handle len 1 and len 0 lists sensibly '''
+        if len(l)==0:
+            return NoMatch()
+        if len(l)==1:
+            return l[0]
+        return reduce(f, l)
+
+    # Handle variables passed in. E.g. if we have {'R':0.5}, we make the substitution. 
+    # Special case for no variables because of how we understand PyParsing is put together
+    if len(all_variables)>0:
+        # We sort the list so that var names (like "e2") match before 
+        # mathematical constants (like "e"). This is kind of a hack.
+        all_variables_keys = sorted(all_variables.keys(), key=len, reverse=True)
+        varnames = sreduce(lambda x,y:x|y, map(lambda x: CasedLiteral(x), all_variables_keys))
+        varnames.setParseAction(lambda x:map(lambda y:all_variables[y], x))
+    else:
+        varnames=NoMatch()
+    # Same thing for functions. 
+    if len(all_functions)>0:
+        funcnames = sreduce(lambda x,y:x|y, map(lambda x: CasedLiteral(x), all_functions.keys()))
+        function = funcnames+lpar.suppress()+expr+rpar.suppress()
+        function.setParseAction(func_parse_action)
+    else:
+        function = NoMatch()
+
+    atom = number | function | varnames | lpar+expr+rpar
+    factor << (atom + ZeroOrMore(exp+atom)).setParseAction(exp_parse_action) # 7^6
+    paritem = factor + ZeroOrMore(Literal('||')+factor) # 5k || 4k
+    paritem=paritem.setParseAction(parallel)
+    term = paritem + ZeroOrMore((times|div)+paritem) # 7 * 5 / 4 - 3
+    term = term.setParseAction(prod_parse_action)
+    expr << Optional((plus|minus)) + term + ZeroOrMore((plus|minus)+term) # -5 + 4 - 3
+    expr=expr.setParseAction(sum_parse_action)
+    return (expr+stringEnd).parseString(string)[0]
+
+if __name__=='__main__':
+    variables={'R1':2.0, 'R3':4.0}
+    functions={'sin':numpy.sin, 'cos':numpy.cos}
+    print "X",evaluator(variables, functions, "10000||sin(7+5)-6k")
+    print "X",evaluator(variables, functions, "13")
+    print evaluator({'R1': 2.0, 'R3':4.0}, {}, "13")
+
+    print evaluator({'e1':1,'e2':1.0,'R3':7,'V0':5,'R5':15,'I1':1,'R4':6}, {},"e2")
+
+    print evaluator({'a': 2.2997471478310274, 'k': 9, 'm': 8, 'x': 0.66009498411213041}, {}, "5")
+    print evaluator({},{}, "-1")
+    print evaluator({},{}, "-(7+5)")
+    print evaluator({},{}, "-0.33")
+    print evaluator({},{}, "-.33")
+    print evaluator({},{}, "5+7 QWSEKO")

djangoapps/courseware/capa/capa_problem.py

@@ -0,0 +1,286 @@
+import copy
+import logging
+import math
+import numpy
+import os
+import random
+import re
+import scipy
+import struct
+
+from lxml import etree
+from lxml.etree import Element
+
+from mako.template import Template
+
+from util import contextualize_text
+from inputtypes import textline, schematic
+from responsetypes import numericalresponse, formularesponse, customresponse, schematicresponse, StudentInputError
+
+import calc
+import eia
+
+log = logging.getLogger("mitx.courseware")
+
+response_types = {'numericalresponse':numericalresponse, 
+                  'formularesponse':formularesponse,
+                  'customresponse':customresponse,
+                  'schematicresponse':schematicresponse}
+entry_types = ['textline', 'schematic']
+response_properties = ["responseparam", "answer"]
+# How to convert from original XML to HTML
+# We should do this with xlst later
+html_transforms = {'problem': {'tag':'div'},
+                   "numericalresponse": {'tag':'span'}, 
+                   "customresponse": {'tag':'span'}, 
+                   "schematicresponse": {'tag':'span'}, 
+                   "formularesponse": {'tag':'span'}, 
+                   "text": {'tag':'span'}}
+
+global_context={'random':random,
+                'numpy':numpy,
+                'math':math,
+                'scipy':scipy, 
+                'calc':calc, 
+                'eia':eia}
+
+# These should be removed from HTML output, including all subelements
+html_problem_semantics = ["responseparam", "answer", "script"]
+# These should be removed from HTML output, but keeping subelements
+html_skip = ["numericalresponse", "customresponse", "schematicresponse", "formularesponse", "text"]
+# These should be transformed
+html_special_response = {"textline":textline.render,
+                         "schematic":schematic.render}
+
+class LoncapaProblem(object):
+    def __init__(self, filename, id=None, state=None, seed=None):
+        ## Initialize class variables from state
+        self.seed = None
+        self.student_answers = dict()
+        self.correct_map = dict()
+        self.done = False
+        self.filename = filename
+
+        if seed != None:
+            self.seed = seed
+
+        if id:
+            self.problem_id = id
+        else:
+            print "NO ID"
+            raise Exception("This should never happen (183)")
+            #self.problem_id = filename
+
+        if state:
+            if 'seed' in state:
+                self.seed = state['seed']
+            if 'student_answers' in state:
+                self.student_answers = state['student_answers']
+            if 'correct_map' in state:
+                self.correct_map = state['correct_map']
+            if 'done' in state:
+                self.done = state['done']
+
+#        print self.seed
+
+        # TODO: Does this deplete the Linux entropy pool? Is this fast enough?
+        if not self.seed:
+            self.seed=struct.unpack('i', os.urandom(4))[0]
+
+#        print filename, self.seed, seed
+
+        ## Parse XML file
+        #log.debug(u"LoncapaProblem() opening file {0}".format(filename))
+        file_text = open(filename).read()
+        # Convert startouttext and endouttext to proper <text></text>
+        # TODO: Do with XML operations
+        file_text = re.sub("startouttext\s*/","text",file_text)
+        file_text = re.sub("endouttext\s*/","/text",file_text)
+        self.tree = etree.XML(file_text)
+
+        self.preprocess_problem(self.tree, correct_map=self.correct_map, answer_map = self.student_answers)
+        self.context = self.extract_context(self.tree, seed=self.seed)
+
+    def get_state(self):
+        ''' Stored per-user session data neeeded to: 
+            1) Recreate the problem
+            2) Populate any student answers. '''
+        return {'seed':self.seed, 
+                'student_answers':self.student_answers,
+                'correct_map':self.correct_map, 
+                'done':self.done}
+
+    def get_max_score(self):
+        sum = 0 
+        for et in entry_types: 
+            sum = sum + self.tree.xpath('count(//'+et+')')
+        return int(sum)
+
+    def get_score(self):
+        correct=0
+        for key in self.correct_map:
+            if self.correct_map[key] == u'correct':
+                correct += 1
+        if (not self.student_answers) or len(self.student_answers)==0:
+            return {'score':0,
+                    'total':self.get_max_score()}
+        else:
+            return {'score':correct,
+                    'total':self.get_max_score()}
+
+    def grade_answers(self, answers):
+        self.student_answers = answers
+        context=self.extract_context(self.tree)
+        self.correct_map = dict()
+        problems_simple = self.extract_problems(self.tree)
+        for response in problems_simple:
+            grader = response_types[response.tag](response, self.context)
+            results = grader.grade(answers)
+            self.correct_map.update(results)
+
+        return self.correct_map
+
+    def get_question_answers(self):
+        context=self.extract_context(self.tree)
+        answer_map = dict()
+        problems_simple = self.extract_problems(self.tree)
+        for response in problems_simple:
+            responder = response_types[response.tag](response, self.context)
+            results = responder.get_answers()
+            answer_map.update(results)
+
+        for entry in problems_simple.xpath("//"+"|//".join(response_properties+entry_types)):
+            answer = entry.get('correct_answer')
+            if answer:
+                answer_map[entry.get('id')] = contextualize_text(answer, self.context)
+
+        return answer_map
+
+    # ======= Private ========
+
+    def extract_context(self, tree, seed = struct.unpack('i', os.urandom(4))[0]):  # private
+        ''' Problem XML goes to Python execution context. Runs everything in script tags '''
+        random.seed(self.seed)
+        context = dict()
+        for script in tree.xpath('/problem/script'):
+            exec script.text in global_context, context
+        return context
+
+    def get_html(self):
+        return contextualize_text(etree.tostring(self.extract_html(self.tree)[0]), self.context)
+
+    def extract_html(self, problemtree):  # private
+        ''' Helper function for get_html. Recursively converts XML tree to HTML
+        '''
+        if problemtree.tag in html_problem_semantics:
+            return
+
+        if problemtree.tag in html_special_response:
+            status = "unsubmitted"
+            if problemtree.get('id') in self.correct_map:
+                status = self.correct_map[problemtree.get('id')]
+
+            value = ""
+            if self.student_answers and problemtree.get('id') in self.student_answers:
+                value = self.student_answers[problemtree.get('id')]
+
+            return html_special_response[problemtree.tag](problemtree, value, status) #TODO
+
+        tree=Element(problemtree.tag)
+        for item in problemtree:
+            subitems = self.extract_html(item)
+            if subitems: 
+                for subitem in subitems:
+                    tree.append(subitem)
+        for (key,value) in problemtree.items():
+            tree.set(key, value)
+
+        tree.text=problemtree.text
+        tree.tail=problemtree.tail
+
+        if problemtree.tag in html_transforms:
+            tree.tag=html_transforms[problemtree.tag]['tag']
+            # Reset attributes. Otherwise, we get metadata in HTML
+            # (e.g. answers) 
+            # TODO: We should remove and not zero them.
+            # I'm not sure how to do that quickly with lxml
+            for k in tree.keys():
+                tree.set(k,"")
+
+        # TODO: Fix. This loses Element().tail
+        #if problemtree.tag in html_skip:
+        #    return tree
+
+        return [tree]
+
+    def preprocess_problem(self, tree, correct_map=dict(), answer_map=dict()): # private
+        ''' Assign IDs to all the responses 
+        Assign sub-IDs to all entries (textline, schematic, etc.)
+        Annoted correctness and value
+        In-place transformation
+        '''
+        response_id = 1
+        for response in tree.xpath('//'+"|//".join(response_types)):
+            response_id_str=self.problem_id+"_"+str(response_id)
+            response.attrib['id']=response_id_str
+            if response_id not in correct_map:
+                correct = 'unsubmitted'
+            response.attrib['state'] = correct
+            response_id = response_id + 1
+            answer_id = 1
+            for entry in tree.xpath("|".join(['//'+response.tag+'[@id=$id]//'+x for x in entry_types]), 
+                                    id=response_id_str):
+                entry.attrib['response_id'] = str(response_id)
+                entry.attrib['answer_id'] = str(answer_id)
+                entry.attrib['id'] = "%s_%i_%i"%(self.problem_id, response_id, answer_id)
+                answer_id=answer_id+1
+
+    def extract_problems(self, problem_tree):
+        ''' Remove layout from the problem, and give a purified XML tree of just the problems '''
+        problem_tree=copy.deepcopy(problem_tree)
+        tree=Element('problem')
+        for response in problem_tree.xpath("//"+"|//".join(response_types)):
+            newresponse = copy.copy(response)
+            for e in newresponse: 
+                newresponse.remove(e)
+            # copy.copy is needed to make xpath work right. Otherwise, it starts at the root
+            # of the tree. We should figure out if there's some work-around
+            for e in copy.copy(response).xpath("//"+"|//".join(response_properties+entry_types)):
+                newresponse.append(e)
+                
+            tree.append(newresponse)
+        return tree
+
+if __name__=='__main__':
+    problem_id='simpleFormula'
+    filename = 'simpleFormula.xml'
+
+    problem_id='resistor'
+    filename = 'resistor.xml'
+
+    
+    lcp = LoncapaProblem(filename, problem_id)
+    
+    context = lcp.extract_context(lcp.tree)
+    problem = lcp.extract_problems(lcp.tree)
+    print lcp.grade_problems({'resistor_2_1':'1.0','resistor_3_1':'2.0'})
+    #print lcp.grade_problems({'simpleFormula_2_1':'3*x^3'})
+#numericalresponse(problem, context)
+    
+#print etree.tostring((lcp.tree))
+    print '============'
+    print
+#print etree.tostring(lcp.extract_problems(lcp.tree))
+    print lcp.get_html()
+#print extract_context(tree)
+    
+
+
+    # def handle_fr(self, element):
+    #     problem={"answer":self.contextualize_text(answer),
+    #              "type":"formularesponse",
+    #              "tolerance":evaluator({},{},self.contextualize_text(tolerance)),
+    #              "sample_range":dict(zip(variables, sranges)),
+    #              "samples_count": numsamples,
+    #              "id":id,
+    #     self.questions[self.lid]=problem        

djangoapps/courseware/capa/eia.py

@@ -0,0 +1,7 @@
+E6=[10,15,22,33,47,68]
+E12=[10,12,15,18,22,27,33,39,47,56,68,82]
+E24=[10,12,15,18,22,27,33,39,47,56,68,82,11,13,16,20,24,30,36,43,51,62,75,91]
+E48=[100,121,147,178,215,261,316,383,464,562,681,825,105,127,154,187,226,274,332,402,487,590,715,866,110,133,162,196,237,287,348,422,511,619,750,909,115,140,169,205,249,301,365,442,536,649,787,953]
+E96=[100,121,147,178,215,261,316,383,464,562,681,825,102,124,150,182,221,267,324,392,475,576,698,845,105,127,154,187,226,274,332,402,487,590,715,866,107,130,158,191,232,280,340,412,499,604,732,887,110,133,162,196,237,287,348,422,511,619,750,909,113,137,165,200,243,294,357,432,523,634,768,931,115,140,169,205,249,301,365,442,536,649,787,953,118,143,174,210,255,309,374,453,549,665,806,976]
+E192=[100,121,147,178,215,261,316,383,464,562,681,825,101,123,149,180,218,264,320,388,470,569,690,835,102,124,150,182,221,267,324,392,475,576,698,845,104,126,152,184,223,271,328,397,481,583,706,856,105,127,154,187,226,274,332,402,487,590,715,866,106,129,156,189,229,277,336,407,493,597,723,876,107,130,158,191,232,280,340,412,499,604,732,887,109,132,160,193,234,284,344,417,505,612,741,898,110,133,162,196,237,287,348,422,511,619,750,909,111,135,164,198,240,291,352,427,517,626,759,920,113,137,165,200,243,294,357,432,523,634,768,931,114,138,167,203,246,298,361,437,530,642,777,942,115,140,169,205,249,301,365,442,536,649,787,953,117,142,172,208,252,305,370,448,542,657,796,965,118,143,174,210,255,309,374,453,549,665,806,976,120,145,176,213,258,312,379,459,556,673,816,988]
+

djangoapps/courseware/capa/inputtypes.py

@@ -0,0 +1,39 @@
+from lxml.etree import Element
+from lxml import etree
+
+from mitxmako.shortcuts import render_to_response, render_to_string
+
+class textline(object):
+    @staticmethod
+    def render(element, value, state):
+        eid=element.get('id')
+        count = int(eid.split('_')[-2])-1 # HACK
+        context = {'id':eid, 'value':value, 'state':state, 'count':count}
+        html=render_to_string("textinput.html", context)
+        return etree.XML(html)
+
+class schematic(object):
+    @staticmethod
+    def render(element, value, state):
+        eid = element.get('id')
+        height = element.get('height')
+        width = element.get('width')
+        parts = element.get('parts')
+        analyses = element.get('analyses')
+        initial_value = element.get('initial_value')
+        submit_analyses = element.get('submit_analyses')
+        context = {
+            'id':eid,
+            'value':value,
+            'initial_value':initial_value,
+            'state':state,
+            'width':width,
+            'height':height,
+            'parts':parts,
+            'analyses':analyses,
+            'submit_analyses':submit_analyses,
+            }
+        html=render_to_string("schematicinput.html", context)
+        return etree.XML(html)
+
+

djangoapps/courseware/capa/responsetypes.py

@@ -0,0 +1,187 @@
+import json
+import math
+import numpy
+import random
+import scipy
+import traceback
+
+from calc import evaluator, UndefinedVariable
+from django.conf import settings
+from util import contextualize_text
+
+import calc
+import eia
+
+# TODO: Should be the same object as in capa_problem
+global_context={'random':random,
+                'numpy':numpy,
+                'math':math,
+                'scipy':scipy, 
+                'calc':calc, 
+                'eia':eia}
+
+
+def compare_with_tolerance(v1, v2, tol):
+    ''' Compare v1 to v2 with maximum tolerance tol
+    tol is relative if it ends in %; otherwise, it is absolute
+    '''
+    relative = "%" in tol
+    if relative: 
+        tolerance_rel = evaluator(dict(),dict(),tol[:-1]) * 0.01
+        tolerance = tolerance_rel * max(abs(v1), abs(v2))
+    else: 
+        tolerance = evaluator(dict(),dict(),tol)
+    return abs(v1-v2) <= tolerance
+
+class numericalresponse(object):
+    def __init__(self, xml, context):
+        self.xml = xml
+        self.correct_answer = contextualize_text(xml.get('answer'), context)
+        self.correct_answer = float(self.correct_answer)
+        self.tolerance_xml = xml.xpath('//*[@id=$id]//responseparam[@type="tolerance"]/@default',
+                                   id=xml.get('id'))[0]
+        self.tolerance = contextualize_text(self.tolerance_xml, context)
+        self.answer_id = xml.xpath('//*[@id=$id]//textline/@id',
+                                   id=xml.get('id'))[0]
+
+    def grade(self, student_answers):
+        ''' Display HTML for a numeric response '''
+        student_answer = student_answers[self.answer_id]
+        try:
+            correct = compare_with_tolerance (evaluator(dict(),dict(),student_answer), self.correct_answer, self.tolerance)
+        except:
+            raise StudentInputError('Invalid input -- please use a number only')
+
+        if correct:
+            return {self.answer_id:'correct'}
+        else:
+            return {self.answer_id:'incorrect'}
+
+    def get_answers(self):
+        return {self.answer_id:self.correct_answer}
+
+class customresponse(object):
+    def __init__(self, xml, context):
+        self.xml = xml
+        ## CRITICAL TODO: Should cover all entrytypes
+        ## NOTE: xpath will look at root of XML tree, not just 
+        ## what's in xml. @id=id keeps us in the right customresponse. 
+        self.answer_ids = xml.xpath('//*[@id=$id]//textline/@id',
+                                    id=xml.get('id'))
+        self.context = context
+        answer = xml.xpath('//*[@id=$id]//answer',
+                           id=xml.get('id'))[0]
+        answer_src = answer.get('src')
+        if answer_src != None:
+            self.code = open(settings.DATA_DIR+'src/'+answer_src).read()
+        else:
+            self.code = answer.text
+
+    def grade(self, student_answers):
+        submission = [student_answers[k] for k in sorted(self.answer_ids)]
+        self.context.update({'submission':submission})
+        exec self.code in global_context, self.context
+        return  zip(sorted(self.answer_ids), self.context['correct'])
+
+    def get_answers(self):
+        # Since this is explicitly specified in the problem, this will 
+        # be handled by capa_problem
+        return {}
+
+class StudentInputError(Exception):
+    pass
+
+class formularesponse(object):
+    def __init__(self, xml, context):
+        self.xml = xml
+        self.correct_answer = contextualize_text(xml.get('answer'), context)
+        self.samples = contextualize_text(xml.get('samples'), context)
+        self.tolerance_xml = xml.xpath('//*[@id=$id]//responseparam[@type="tolerance"]/@default',
+                                   id=xml.get('id'))[0]
+        self.tolerance = contextualize_text(self.tolerance_xml, context)
+        self.answer_id = xml.xpath('//*[@id=$id]//textline/@id',
+                                   id=xml.get('id'))[0]
+        self.context = context
+        ts = xml.get('type')
+        if ts == None:
+            typeslist = []
+        else:
+            typeslist = ts.split(',')
+        if 'ci' in typeslist: # Case insensitive
+            self.case_sensitive = False
+        elif 'cs' in typeslist: # Case sensitive
+            self.case_sensitive = True
+        else: # Default
+            self.case_sensitive = False
+
+
+    def grade(self, student_answers):
+        variables=self.samples.split('@')[0].split(',')
+        numsamples=int(self.samples.split('@')[1].split('#')[1])
+        sranges=zip(*map(lambda x:map(float, x.split(",")), 
+                         self.samples.split('@')[1].split('#')[0].split(':')))
+
+        ranges=dict(zip(variables, sranges))
+        correct = True
+        for i in range(numsamples):
+            instructor_variables = self.strip_dict(dict(self.context))
+            student_variables = dict()
+            for var in ranges:
+                value = random.uniform(*ranges[var])
+                instructor_variables[str(var)] = value
+                student_variables[str(var)] = value
+            instructor_result = evaluator(instructor_variables,dict(),self.correct_answer, cs = self.case_sensitive)
+            try: 
+                #print student_variables,dict(),student_answers[self.answer_id]
+                student_result = evaluator(student_variables,dict(),
+                                           student_answers[self.answer_id], 
+                                           cs = self.case_sensitive)
+            except UndefinedVariable as uv:
+                raise StudentInputError(uv.message+" not permitted in answer")
+            except:
+                #traceback.print_exc()
+                raise StudentInputError("Error in formula")
+            if math.isnan(student_result) or math.isinf(student_result):
+                return {self.answer_id:"incorrect"}
+            if not compare_with_tolerance(student_result, instructor_result, self.tolerance):
+                return {self.answer_id:"incorrect"}
+ 
+        return {self.answer_id:"correct"}
+
+    def strip_dict(self, d):
+        ''' Takes a dict. Returns an identical dict, with all non-word
+        keys and all non-numeric values stripped out. All values also
+        converted to float. Used so we can safely use Python contexts.
+        ''' 
+        d=dict([(k, float(d[k])) for k in d if type(k)==str and \
+                    k.isalnum() and \
+                    (type(d[k]) == float or type(d[k]) == int) ])
+        return d
+
+    def get_answers(self):
+        return {self.answer_id:self.correct_answer}
+
+class schematicresponse(object):
+    def __init__(self, xml, context):
+        self.xml = xml
+        self.answer_ids = xml.xpath('//*[@id=$id]//schematic/@id',
+                                    id=xml.get('id'))
+        self.context = context
+        answer = xml.xpath('//*[@id=$id]//answer',
+                           id=xml.get('id'))[0]
+        answer_src = answer.get('src')
+        if answer_src != None:
+            self.code = open(settings.DATA_DIR+'src/'+answer_src).read()
+        else:
+            self.code = answer.text
+
+    def grade(self, student_answers):
+        submission = [json.loads(student_answers[k]) for k in sorted(self.answer_ids)]
+        self.context.update({'submission':submission})
+        exec self.code in global_context, self.context
+        return  zip(sorted(self.answer_ids), self.context['correct'])
+
+    def get_answers(self):
+        # Since this is explicitly specified in the problem, this will 
+        # be handled by capa_problem
+        return {}

djangoapps/courseware/capa/unit.py

@@ -0,0 +1,132 @@
+import math
+import operator
+
+from numpy import eye, array
+
+from pyparsing import Word, alphas, nums, oneOf, Literal
+from pyparsing import ZeroOrMore, OneOrMore, StringStart
+from pyparsing import StringEnd, Optional, Forward
+from pyparsing import CaselessLiteral, Group, StringEnd
+from pyparsing import NoMatch, stringEnd
+
+base_units = ['meter', 'gram', 'second', 'ampere', 'kelvin', 'mole', 'cd']
+unit_vectors = dict([(base_units[i], eye(len(base_units))[:,i]) for i in range(len(base_units))])
+
+
+def unit_evaluator(unit_string, units=unit_map):
+    ''' Evaluate an expression. Variables are passed as a dictionary
+    from string to value. Unary functions are passed as a dictionary
+    from string to function '''
+    if string.strip() == "":
+        return float('nan')
+    ops = { "^" : operator.pow,
+            "*" : operator.mul,
+            "/" : operator.truediv,
+            }
+    prefixes={'%':0.01,'k':1e3,'M':1e6,'G':1e9,
+              'T':1e12,#'P':1e15,'E':1e18,'Z':1e21,'Y':1e24,
+              'c':1e-2,'m':1e-3,'u':1e-6,
+              'n':1e-9,'p':1e-12}#,'f':1e-15,'a':1e-18,'z':1e-21,'y':1e-24}
+    
+    def super_float(text):
+        ''' Like float, but with si extensions. 1k goes to 1000'''
+        if text[-1] in suffixes:
+            return float(text[:-1])*suffixes[text[-1]]
+        else:
+            return float(text)
+
+    def number_parse_action(x): # [ '7' ] ->  [ 7 ]
+        return [super_float("".join(x))]
+    def exp_parse_action(x): # [ 2 ^ 3 ^ 2 ] -> 512
+        x = [e for e in x if type(e) == float] # Ignore ^
+        x.reverse()
+        x=reduce(lambda a,b:b**a, x)
+        return x
+    def parallel(x): # Parallel resistors [ 1 2 ] => 2/3
+        if len(x) == 1:
+            return x[0]
+        if 0 in x:
+            return float('nan')
+        x = [1./e for e in x if type(e) == float] # Ignore ^
+        return 1./sum(x)
+    def sum_parse_action(x): # [ 1 + 2 - 3 ] -> 0
+        total = 0.0
+        op = ops['+']
+        for e in x:
+            if e in set('+-'):
+                op = ops[e]
+            else:
+                total=op(total, e)
+        return total
+    def prod_parse_action(x): # [ 1 * 2 / 3 ] => 0.66
+        prod = 1.0
+        op = ops['*']
+        for e in x:
+            if e in set('*/'):
+                op = ops[e]
+            else:
+                prod=op(prod, e)
+        return prod
+    def func_parse_action(x):
+        return [functions[x[0]](x[1])]
+
+    number_suffix=reduce(lambda a,b:a|b, map(Literal,suffixes.keys()), NoMatch()) # SI suffixes and percent
+    (dot,minus,plus,times,div,lpar,rpar,exp)=map(Literal,".-+*/()^")
+    
+    number_part=Word(nums)
+    inner_number = ( number_part+Optional("."+number_part) ) | ("."+number_part) # 0.33 or 7 or .34
+    number=Optional(minus | plus)+ inner_number + \
+        Optional(CaselessLiteral("E")+Optional("-")+number_part)+ \
+        Optional(number_suffix) # 0.33k or -17
+    number=number.setParseAction( number_parse_action ) # Convert to number
+    
+    # Predefine recursive variables
+    expr = Forward() 
+    factor = Forward()
+    
+    def sreduce(f, l):
+        ''' Same as reduce, but handle len 1 and len 0 lists sensibly '''
+        if len(l)==0:
+            return NoMatch()
+        if len(l)==1:
+            return l[0]
+        return reduce(f, l)
+
+    # Handle variables passed in. E.g. if we have {'R':0.5}, we make the substitution. 
+    # Special case for no variables because of how we understand PyParsing is put together
+    if len(variables)>0:
+        varnames = sreduce(lambda x,y:x|y, map(lambda x: CaselessLiteral(x), variables.keys()))
+        varnames.setParseAction(lambda x:map(lambda y:variables[y], x))
+    else:
+        varnames=NoMatch()
+    # Same thing for functions. 
+    if len(functions)>0:
+        funcnames = sreduce(lambda x,y:x|y, map(lambda x: CaselessLiteral(x), functions.keys()))
+        function = funcnames+lpar.suppress()+expr+rpar.suppress()
+        function.setParseAction(func_parse_action)
+    else:
+        function = NoMatch()
+
+    atom = number | varnames | lpar+expr+rpar | function
+    factor << (atom + ZeroOrMore(exp+atom)).setParseAction(exp_parse_action) # 7^6
+    paritem = factor + ZeroOrMore(Literal('||')+factor) # 5k || 4k
+    paritem=paritem.setParseAction(parallel)
+    term = paritem + ZeroOrMore((times|div)+paritem) # 7 * 5 / 4 - 3
+    term = term.setParseAction(prod_parse_action)
+    expr << Optional((plus|minus)) + term + ZeroOrMore((plus|minus)+term) # -5 + 4 - 3
+    expr=expr.setParseAction(sum_parse_action)
+    return (expr+stringEnd).parseString(string)[0]
+
+if __name__=='__main__':
+    variables={'R1':2.0, 'R3':4.0}
+    functions={'sin':math.sin, 'cos':math.cos}
+    print "X",evaluator(variables, functions, "10000||sin(7+5)-6k")
+    print "X",evaluator(variables, functions, "13")
+    print evaluator({'R1': 2.0, 'R3':4.0}, {}, "13")
+    # 
+    print evaluator({'a': 2.2997471478310274, 'k': 9, 'm': 8, 'x': 0.66009498411213041}, {}, "5")
+    print evaluator({},{}, "-1")
+    print evaluator({},{}, "-(7+5)")
+    print evaluator({},{}, "-0.33")
+    print evaluator({},{}, "-.33")
+    print evaluator({},{}, "5+7 QWSEKO")

djangoapps/courseware/capa/util.py

@@ -0,0 +1,6 @@
+def contextualize_text(text, context): # private
+    ''' Takes a string with variables. E.g. $a+$b. 
+    Does a substitution of those variables from the context '''
+    for key in sorted(context, lambda x,y:cmp(len(y),len(x))):
+        text=text.replace('$'+key, str(context[key]))
+    return text

djangoapps/courseware/content_parser.py

@@ -0,0 +1,282 @@
+import hashlib
+import json
+import logging
+import os
+import re
+import sys
+import urllib
+
+from datetime import timedelta
+from lxml import etree
+
+try: # This lets us do __name__ == ='__main__'
+    from django.conf import settings
+    from django.core.cache import cache
+    from student.models import UserProfile
+    from student.models import UserTestGroup
+    from mitxmako.shortcuts import render_to_response, render_to_string
+except: 
+    settings = None 
+
+''' This file will eventually form an abstraction layer between the
+course XML file and the rest of the system. 
+
+TODO: Shift everything from xml.dom.minidom to XPath (or XQuery)
+'''
+
+class ContentException(Exception):
+    pass
+
+log = logging.getLogger("mitx.courseware")
+
+
+timedelta_regex = re.compile(r'^((?P<days>\d+?) day(?:s?))?(\s)?((?P<hours>\d+?) hour(?:s?))?(\s)?((?P<minutes>\d+?) minute(?:s)?)?(\s)?((?P<seconds>\d+?) second(?:s)?)?$')
+
+def format_url_params(params):
+    return [ urllib.quote(string.replace(' ','_')) for string in params ]
+
+def parse_timedelta(time_str):
+    parts = timedelta_regex.match(time_str)
+    if not parts:
+        return
+    parts = parts.groupdict()
+    time_params = {}
+    for (name, param) in parts.iteritems():
+        if param:
+            time_params[name] = int(param)
+    return timedelta(**time_params)
+
+def fasthash(string):
+    m = hashlib.new("md4")
+    m.update(string)
+    return "id"+m.hexdigest()
+
+def xpath(xml, query_string, **args):
+    ''' Safe xpath query into an xml tree:
+        * xml is the tree.
+        * query_string is the query
+        * args are the parameters. Substitute for {params}. 
+        We should remove this with the move to lxml. 
+        We should also use lxml argument passing. '''
+    doc = etree.fromstring(xml)
+    #print type(doc)
+    def escape(x):
+        # TODO: This should escape the string. For now, we just assume it's made of valid characters. 
+        # Couldn't figure out how to escape for lxml in a few quick Googles
+        valid_chars="".join(map(chr, range(ord('a'),ord('z')+1)+range(ord('A'),ord('Z')+1)+range(ord('0'), ord('9')+1)))+"_ "
+        for e in x:
+            if e not in valid_chars:
+                raise Exception("Invalid char in xpath expression. TODO: Escape")
+        return x
+
+    args=dict( ((k, escape(args[k])) for k in args) )
+    #print args
+    results = doc.xpath(query_string.format(**args))
+    return results
+
+def xpath_remove(tree, path):
+    ''' Remove all items matching path from lxml tree.  Works in
+        place.'''
+    items = tree.xpath(path)
+    for item in items: 
+        item.getparent().remove(item)
+    return tree
+
+if __name__=='__main__':
+    print xpath('<html><problem name="Bob"></problem></html>', '/{search}/problem[@name="{name}"]', 
+                search='html', name="Bob")
+
+def item(l, default="", process=lambda x:x):
+    if len(l)==0:
+        return default
+    elif len(l)==1:
+        return process(l[0])
+    else:
+        raise Exception('Malformed XML')
+
+def id_tag(course):
+    ''' Tag all course elements with unique IDs '''
+    old_ids = {'video':'youtube',
+                   'problem':'filename',
+                   'sequential':'id',
+                   'html':'filename',
+                   'vertical':'id', 
+                   'tab':'id',
+                   'schematic':'id',
+                   'book' : 'id'}
+    import courseware.modules
+    default_ids = courseware.modules.get_default_ids()
+
+    #print default_ids, old_ids
+    #print default_ids == old_ids
+
+    # Tag elements with unique IDs
+    elements = course.xpath("|".join(['//'+c for c in default_ids]))
+    for elem in elements:
+        if elem.get('id'):
+            pass
+        elif elem.get(default_ids[elem.tag]):
+            new_id = elem.get(default_ids[elem.tag]) 
+            new_id = "".join([a for a in new_id if a.isalnum()]) # Convert to alphanumeric
+            # Without this, a conflict may occur between an hmtl or youtube id
+            new_id = default_ids[elem.tag] + new_id
+            elem.set('id', new_id)
+        else:
+            elem.set('id', fasthash(etree.tostring(elem)))
+            
+def propogate_downward_tag(element, attribute_name, parent_attribute = None):
+    ''' This call is to pass down an attribute to all children. If an element
+    has this attribute, it will be "inherited" by all of its children. If a
+    child (A) already has that attribute, A will keep the same attribute and
+    all of A's children will inherit A's attribute. This is a recursive call.'''
+    
+    if (parent_attribute == None): #This is the entry call. Select all elements with this attribute
+        all_attributed_elements = element.xpath("//*[@" + attribute_name +"]")
+        for attributed_element in all_attributed_elements:
+            attribute_value = attributed_element.get(attribute_name)
+            for child_element in attributed_element:
+                propogate_downward_tag(child_element, attribute_name, attribute_value)
+    else:
+        '''The hack below is because we would get _ContentOnlyELements from the
+        iterator that can't have attributes set. We can't find API for it. If we
+        ever have an element which subclasses BaseElement, we will not tag it'''
+        if not element.get(attribute_name) and type(element) == etree._Element:
+            element.set(attribute_name, parent_attribute)
+            
+            for child_element in element:
+                propogate_downward_tag(child_element, attribute_name, parent_attribute)
+        else:
+            #This element would have already been found by Xpath, so we return
+            #for now and trust that this element will get its turn to propogate
+            #to its children later.
+            return
+
+def user_groups(user):
+    # TODO: Rewrite in Django
+    key = 'user_group_names_{user.id}'.format(user=user)
+    cache_expiration = 60 * 60 # one hour
+    
+    # Kill caching on dev machines -- we switch groups a lot
+    if "dev" not in settings.DEFAULT_GROUPS:
+        group_names = cache.get(fasthash(key))
+    else: 
+        group_names = None
+ 
+    if group_names is None:
+        group_names = [u.name for u in UserTestGroup.objects.filter(users=user)]
+        cache.set(fasthash(key), group_names, cache_expiration)
+
+    return group_names
+
+    # return [u.name for u in UserTestGroup.objects.raw("select * from auth_user, student_usertestgroup, student_usertestgroup_users where auth_user.id = student_usertestgroup_users.user_id and student_usertestgroup_users.usertestgroup_id = student_usertestgroup.id and auth_user.id = %s", [user.id])]
+
+def course_xml_process(tree):
+    ''' Do basic pre-processing of an XML tree. Assign IDs to all
+    items without. Propagate due dates, grace periods, etc. to child
+    items. 
+    '''
+    id_tag(tree)
+    propogate_downward_tag(tree, "due")
+    propogate_downward_tag(tree, "graded")
+    propogate_downward_tag(tree, "graceperiod")
+    return tree
+
+def course_file(user):
+    ''' Given a user, return course.xml'''
+    #import logging
+    #log = logging.getLogger("tracking")
+    #log.info(  "DEBUG: cf:"+str(user) )
+
+    filename = UserProfile.objects.get(user=user).courseware # user.profile_cache.courseware 
+    groups = user_groups(user)
+    options = {'dev_content':settings.DEV_CONTENT, 
+               'groups' : groups}
+
+    
+    cache_key = filename + "_processed?dev_content:" + str(options['dev_content']) + "&groups:" + str(sorted(groups))
+    if "dev" not in settings.DEFAULT_GROUPS:
+        tree_string = cache.get(fasthash(cache_key))
+    else: 
+        tree_string = None
+
+    if not tree_string:
+        tree = course_xml_process(etree.XML(render_to_string(filename, options, namespace = 'course')))
+        tree_string = etree.tostring(tree)
+        
+        cache.set(fasthash(cache_key), tree_string, 60)
+    else:
+        tree = etree.XML(tree_string)
+
+    return tree
+
+def section_file(user, section):
+    ''' Given a user and the name of a section, return that section
+    '''
+    filename = section+".xml"
+
+    if filename not in os.listdir(settings.DATA_DIR + '/sections/'):
+        print filename+" not in "+str(os.listdir(settings.DATA_DIR + '/sections/'))
+        return None
+
+    options = {'dev_content':settings.DEV_CONTENT, 
+               'groups' : user_groups(user)}
+
+    tree = course_xml_process(etree.XML(render_to_string(filename, options, namespace = 'sections')))
+    return tree
+
+
+def module_xml(user, module, id_tag, module_id):
+    ''' Get XML for a module based on module and module_id. Assumes
+        module occurs once in courseware XML file or hidden section. '''
+    # Sanitize input
+    if not module.isalnum():
+        raise Exception("Module is not alphanumeric")
+    if not module_id.isalnum():
+        raise Exception("Module ID is not alphanumeric")
+    # Generate search
+    xpath_search='//{module}[(@{id_tag} = "{id}") or (@id = "{id}")]'.format(module=module, 
+                                                                             id_tag=id_tag,
+                                                                             id=module_id)
+    #result_set=doc.xpathEval(xpath_search)
+    doc = course_file(user)
+    section_list = (s[:-4] for s in os.listdir(settings.DATA_DIR+'/sections') if s[-4:]=='.xml')
+
+    result_set=doc.xpath(xpath_search)
+    if len(result_set)<1:
+        for section in section_list:
+            try: 
+                s = section_file(user, section)
+            except etree.XMLSyntaxError: 
+                ex= sys.exc_info()
+                raise ContentException("Malformed XML in " + section+ "("+str(ex[1].msg)+")")
+            result_set = s.xpath(xpath_search)
+            if len(result_set) != 0: 
+                break
+
+    if len(result_set)>1:
+        print "WARNING: Potentially malformed course file", module, module_id
+    if len(result_set)==0:
+        return None
+    return etree.tostring(result_set[0])
+    #return result_set[0].serialize()
+
+def toc_from_xml(dom, active_chapter, active_section):
+    name = dom.xpath('//course/@name')[0]
+
+    chapters = dom.xpath('//course[@name=$name]/chapter', name=name)
+    ch=list()
+    for c in chapters:
+        if c.get('name') == 'hidden':
+            continue
+        sections=list()
+        for s in dom.xpath('//course[@name=$name]/chapter[@name=$chname]/section', name=name, chname=c.get('name')): 
+            sections.append({'name':s.get("name") or "", 
+                             'format':s.get("subtitle") if s.get("subtitle") else s.get("format") or "", 
+                             'due':s.get("due") or "",
+                             'active':(c.get("name")==active_chapter and \
+                                           s.get("name")==active_section)})
+        ch.append({'name':c.get("name"), 
+                   'sections':sections,
+                   'active':(c.get("name")==active_chapter)})
+    return ch
+

djangoapps/courseware/grades.py

@@ -0,0 +1,228 @@
+import logging
+import urllib
+from lxml import etree
+
+import courseware.content_parser as content_parser
+from models import StudentModule
+from django.conf import settings
+import courseware.modules
+
+from student.models import UserProfile
+
+log = logging.getLogger("mitx.courseware")
+
+def get_grade(user, problem, cache):
+    ## HACK: assumes max score is fixed per problem
+    id = problem.get('id')
+    correct = 0
+    
+    # If the ID is not in the cache, add the item
+    if id not in cache: 
+        module = StudentModule(module_type = 'problem',  # TODO: Move into StudentModule.__init__?
+                               module_id = id,
+                               student = user, 
+                               state = None, 
+                               grade = 0,
+                               max_grade = None,
+                               done = 'i')
+        cache[id] = module
+
+    # Grab the # correct from cache
+    if id in cache:
+        response = cache[id]
+        if response.grade!=None:
+            correct=response.grade
+        
+    # Grab max grade from cache, or if it doesn't exist, compute and save to DB
+    if id in cache and response.max_grade != None:
+        total = response.max_grade
+    else:
+        total=courseware.modules.capa_module.Module(etree.tostring(problem), "id").max_score()
+        response.max_grade = total
+        response.save()
+
+    return (correct, total)
+
+def grade_sheet(student):
+    dom=content_parser.course_file(student)
+    course = dom.xpath('//course/@name')[0]
+    xmlChapters = dom.xpath('//course[@name=$course]/chapter', course=course)
+
+    responses=StudentModule.objects.filter(student=student)
+    response_by_id = {}
+    for response in responses:
+        response_by_id[response.module_id] = response
+    
+    
+    total_scores = {}
+    chapters=[]
+    for c in xmlChapters:
+        sections = []
+        chname=c.get('name')
+        for s in dom.xpath('//course[@name=$course]/chapter[@name=$chname]/section', 
+                           course=course, chname=chname):
+            problems=dom.xpath('//course[@name=$course]/chapter[@name=$chname]/section[@name=$section]//problem', 
+                           course=course, chname=chname, section=s.get('name'))
+
+            graded = True if s.get('graded') == "true" else False
+            scores=[]
+            if len(problems)>0:
+                for p in problems:
+                    (correct,total) = get_grade(student, p, response_by_id)
+                    # id = p.get('id')
+                    # correct = 0
+                    # if id in response_by_id:
+                    #     response = response_by_id[id]
+                    #     if response.grade!=None:
+                    #         correct=response.grade
+
+                    # total=courseware.modules.capa_module.Module(etree.tostring(p), "id").max_score() # TODO: Add state. Not useful now, but maybe someday problems will have randomized max scores? 
+                    # print correct, total
+                    if settings.GENERATE_PROFILE_SCORES:
+                        if total > 1:
+                            correct = random.randrange( max(total-2, 1) , total + 1 )
+                        else:
+                            correct = total
+                    
+                    scores.append((int(correct),total, graded ))
+
+
+                section_total = (sum([score[0] for score in scores]), 
+                                sum([score[1] for score in scores]))
+
+                graded_total = (sum([score[0] for score in scores if score[2]]), 
+                                sum([score[1] for score in scores if score[2]]))
+
+                #Add the graded total to total_scores
+                format = s.get('format') if s.get('format') else ""
+                subtitle = s.get('subtitle') if s.get('subtitle') else format
+                if format and graded_total[1] > 0:
+                    format_scores = total_scores[ format ] if format in total_scores else []
+                    format_scores.append( graded_total + (s.get("name"),) )
+                    total_scores[ format ] = format_scores
+
+                score={'section':s.get("name"),
+                       'scores':scores,
+                       'section_total' : section_total,
+                       'format' : format,
+                       'subtitle' : subtitle,
+                       'due' : s.get("due") or "",
+                       'graded' : graded,
+                       }
+                sections.append(score)
+
+        chapters.append({'course':course,
+                         'chapter' : c.get("name"),
+                         'sections' : sections,})
+                         
+    
+    def totalWithDrops(scores, drop_count):
+        #Note that this key will sort the list descending
+        sorted_scores = sorted( enumerate(scores), key=lambda x: -x[1]['percentage'] )
+        # A list of the indices of the dropped scores
+        dropped_indices = [score[0] for score in sorted_scores[-drop_count:]] 
+        aggregate_score = 0
+        for index, score in enumerate(scores):
+            if index not in dropped_indices:
+                aggregate_score += score['percentage']
+        
+        aggregate_score /= len(scores) - drop_count
+        
+        return aggregate_score, dropped_indices
+        
+    #Figure the homework scores
+    homework_scores = total_scores['Homework'] if 'Homework' in total_scores else []
+    homework_percentages = []
+    for i in range(12):
+        if i < len(homework_scores):
+            percentage = homework_scores[i][0] / float(homework_scores[i][1])
+            summary = "Homework {0} - {1} - {2:.0%} ({3:g}/{4:g})".format( i + 1, homework_scores[i][2] , percentage, homework_scores[i][0], homework_scores[i][1] )
+        else:
+            percentage = 0
+            summary = "Unreleased Homework {0} - 0% (?/?)".format(i + 1)
+        
+            if settings.GENERATE_PROFILE_SCORES:
+                points_possible = random.randrange(10, 50)
+                points_earned = random.randrange(5, points_possible)
+                percentage = points_earned / float(points_possible)
+                summary = "Random Homework - {0:.0%} ({1:g}/{2:g})".format( percentage, points_earned, points_possible )
+        
+        label = "HW {0:02d}".format(i + 1)
+        
+        homework_percentages.append( {'percentage': percentage, 'summary': summary, 'label' : label} )
+    homework_total, homework_dropped_indices = totalWithDrops(homework_percentages, 2)
+    
+    #Figure the lab scores
+    lab_scores = total_scores['Lab'] if 'Lab' in total_scores else []
+    lab_percentages = []
+    log.debug("lab_scores: {0}".format(lab_scores))
+    for i in range(12):
+        if i < len(lab_scores):
+            percentage = lab_scores[i][0] / float(lab_scores[i][1])
+            summary = "Lab {0} - {1} - {2:.0%} ({3:g}/{4:g})".format( i + 1, lab_scores[i][2] , percentage, lab_scores[i][0], lab_scores[i][1] )
+        else:
+            percentage = 0
+            summary = "Unreleased Lab {0} - 0% (?/?)".format(i + 1)
+        
+            if settings.GENERATE_PROFILE_SCORES:
+                points_possible = random.randrange(10, 50)
+                points_earned = random.randrange(5, points_possible)
+                percentage = points_earned / float(points_possible)
+                summary = "Random Lab - {0:.0%} ({1:g}/{2:g})".format( percentage, points_earned, points_possible )
+            
+        label = "Lab {0:02d}".format(i + 1)
+                
+        lab_percentages.append( {'percentage': percentage, 'summary': summary, 'label' : label} )
+    lab_total, lab_dropped_indices = totalWithDrops(lab_percentages, 2)
+    
+    
+    #TODO: Pull this data about the midterm and final from the databse. It should be exactly similar to above, but we aren't sure how exams will be done yet.
+    midterm_score = ('?', '?')
+    midterm_percentage = 0
+    
+    final_score = ('?', '?')
+    final_percentage = 0
+    
+    if settings.GENERATE_PROFILE_SCORES:
+        midterm_score = (random.randrange(50, 150), 150)
+        midterm_percentage = midterm_score[0] / float(midterm_score[1])
+        
+        final_score = (random.randrange(100, 300), 300)
+        final_percentage = final_score[0] / float(final_score[1])
+        
+    
+    grade_summary = [
+        {
+            'category': 'Homework',
+            'subscores' : homework_percentages,
+            'dropped_indices' : homework_dropped_indices,
+            'totalscore' : {'score' : homework_total, 'summary' : "Homework Average - {0:.0%}".format(homework_total)},
+            'totallabel' : 'HW Avg',
+            'weight' : 0.15,
+        },
+        {
+            'category': 'Labs',
+            'subscores' : lab_percentages,
+            'dropped_indices' : lab_dropped_indices,
+            'totalscore' : {'score' : lab_total, 'summary' : "Lab Average - {0:.0%}".format(lab_total)},
+            'totallabel' : 'Lab Avg',
+            'weight' : 0.15,
+        },
+        {
+            'category': 'Midterm',
+            'totalscore' : {'score' : midterm_percentage, 'summary' : "Midterm - {0:.0%} ({1}/{2})".format(midterm_percentage, midterm_score[0], midterm_score[1])},
+            'totallabel' : 'Midterm',
+            'weight' : 0.30,
+        },
+        {
+            'category': 'Final',
+            'totalscore' : {'score' : final_percentage, 'summary' : "Final - {0:.0%} ({1}/{2})".format(final_percentage, final_score[0], final_score[1])},
+            'totallabel' : 'Final',
+            'weight' : 0.40,
+        }
+    ]
+    
+    return {'grade_summary' : grade_summary,
+            'chapters':chapters}
+
+    

djangoapps/courseware/management/commands/check_course.py

@@ -0,0 +1,55 @@
+import os.path
+
+from lxml import etree
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.contrib.auth.models import User
+
+from mitx.courseware.content_parser import course_file
+import mitx.courseware.module_render
+import mitx.courseware.modules
+
+class Command(BaseCommand):
+    help = "Does basic validity tests on course.xml."
+    def handle(self, *args, **options):
+        check = True
+        sample_user = User.objects.all()[0]
+        print "Attempting to load courseware"
+        course = course_file(sample_user)
+        print "Confirming all problems have alphanumeric names"
+        for problem in course.xpath('//problem'):
+            filename = problem.get('filename')
+            if not filename.isalnum():
+                print "==============> Invalid (non-alphanumeric) filename", filename
+                check = False
+        print "Confirming all modules render. Nothing should print during this step. "
+        for module in course.xpath('//problem|//html|//video|//vertical|//sequential|/tab'):
+            module_class=mitx.courseware.modules.modx_modules[module.tag]
+            # TODO: Abstract this out in render_module.py
+            try: 
+                instance=module_class(etree.tostring(module), 
+                                      module.get('id'), 
+                                      ajax_url='',
+                                      state=None, 
+                                      track_function = lambda x,y,z:None, 
+                                      render_function = lambda x: {'content':'','destroy_js':'','init_js':'','type':'video'})
+            except:
+                print "==============> Error in ", etree.tostring(module)
+                check = False
+        print "Module render check finished"
+        sections_dir = settings.DATA_DIR+"sections"
+        if os.path.exists(sections_dir):
+            print "Checking all section includes are valid XML"
+            for f in os.listdir(sections_dir):
+                print f
+                etree.parse(sections_dir+'/'+f)
+        else:
+            print "Skipping check of include files -- no section includes dir ("+sections_dir+")"
+        # TODO: print "Checking course properly annotated with preprocess.py"
+        
+            
+        if check:
+            print 'Courseware passes all checks!'
+        else: 
+            print "Courseware fails some checks"

djangoapps/courseware/migrations/0001_initial.py

@@ -0,0 +1,112 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding model 'StudentModule'
+        db.create_table('courseware_studentmodule', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('module_type', self.gf('django.db.models.fields.CharField')(default='problem', max_length=32)),
+            ('module_id', self.gf('django.db.models.fields.CharField')(max_length=255)),
+            ('student', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
+            ('state', self.gf('django.db.models.fields.TextField')(null=True, blank=True)),
+            ('grade', self.gf('django.db.models.fields.FloatField')(null=True, blank=True)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+            ('modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)),
+        ))
+        db.send_create_signal('courseware', ['StudentModule'])
+
+        # Adding unique constraint on 'StudentModule', fields ['student', 'module_id', 'module_type']
+        db.create_unique('courseware_studentmodule', ['student_id', 'module_id', 'module_type'])
+
+
+    def backwards(self, orm):
+        
+        # Removing unique constraint on 'StudentModule', fields ['student', 'module_id', 'module_type']
+        db.delete_unique('courseware_studentmodule', ['student_id', 'module_id', 'module_type'])
+
+        # Deleting model 'StudentModule'
+        db.delete_table('courseware_studentmodule')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'about': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'avatar_type': ('django.db.models.fields.CharField', [], {'default': "'n'", 'max_length': '1'}),
+            'bronze': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'consecutive_days_visit_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'country': ('django_countries.fields.CountryField', [], {'max_length': '2', 'blank': 'True'}),
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'date_of_birth': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'display_tag_filter_strategy': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'email_isvalid': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'email_key': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True'}),
+            'email_tag_filter_strategy': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'gold': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'gravatar': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'ignored_tags': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'interesting_tags': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'last_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'location': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'new_response_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'questions_per_page': ('django.db.models.fields.SmallIntegerField', [], {'default': '10'}),
+            'real_name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'reputation': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}),
+            'seen_response_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'show_country': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'silver': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'status': ('django.db.models.fields.CharField', [], {'default': "'w'", 'max_length': '2'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}),
+            'website': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'courseware.studentmodule': {
+            'Meta': {'unique_together': "(('student', 'module_id', 'module_type'),)", 'object_name': 'StudentModule'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'grade': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'module_id': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'module_type': ('django.db.models.fields.CharField', [], {'default': "'problem'", 'max_length': '32'}),
+            'state': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'student': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        }
+    }
+
+    complete_apps = ['courseware']

djangoapps/courseware/migrations/0002_add_indexes.py

@@ -0,0 +1,120 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding index on 'StudentModule', fields ['created']
+        db.create_index('courseware_studentmodule', ['created'])
+
+        # Adding index on 'StudentModule', fields ['grade']
+        db.create_index('courseware_studentmodule', ['grade'])
+
+        # Adding index on 'StudentModule', fields ['modified']
+        db.create_index('courseware_studentmodule', ['modified'])
+
+        # Adding index on 'StudentModule', fields ['module_type']
+        db.create_index('courseware_studentmodule', ['module_type'])
+
+        # Adding index on 'StudentModule', fields ['module_id']
+        db.create_index('courseware_studentmodule', ['module_id'])
+
+
+    def backwards(self, orm):
+        
+        # Removing index on 'StudentModule', fields ['module_id']
+        db.delete_index('courseware_studentmodule', ['module_id'])
+
+        # Removing index on 'StudentModule', fields ['module_type']
+        db.delete_index('courseware_studentmodule', ['module_type'])
+
+        # Removing index on 'StudentModule', fields ['modified']
+        db.delete_index('courseware_studentmodule', ['modified'])
+
+        # Removing index on 'StudentModule', fields ['grade']
+        db.delete_index('courseware_studentmodule', ['grade'])
+
+        # Removing index on 'StudentModule', fields ['created']
+        db.delete_index('courseware_studentmodule', ['created'])
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'about': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'avatar_type': ('django.db.models.fields.CharField', [], {'default': "'n'", 'max_length': '1'}),
+            'bronze': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'consecutive_days_visit_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'country': ('django_countries.fields.CountryField', [], {'max_length': '2', 'blank': 'True'}),
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'date_of_birth': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'display_tag_filter_strategy': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'email_isvalid': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'email_key': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True'}),
+            'email_tag_filter_strategy': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'gold': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'gravatar': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'ignored_tags': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'interesting_tags': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'last_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'location': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'new_response_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'questions_per_page': ('django.db.models.fields.SmallIntegerField', [], {'default': '10'}),
+            'real_name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'reputation': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}),
+            'seen_response_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'show_country': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'silver': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'status': ('django.db.models.fields.CharField', [], {'default': "'w'", 'max_length': '2'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}),
+            'website': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'courseware.studentmodule': {
+            'Meta': {'unique_together': "(('student', 'module_id', 'module_type'),)", 'object_name': 'StudentModule'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'grade': ('django.db.models.fields.FloatField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'module_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'module_type': ('django.db.models.fields.CharField', [], {'default': "'problem'", 'max_length': '32', 'db_index': 'True'}),
+            'state': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'student': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        }
+    }
+
+    complete_apps = ['courseware']

djangoapps/courseware/migrations/0003_done_grade_cache.py

@@ -0,0 +1,116 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Removing unique constraint on 'StudentModule', fields ['module_id', 'module_type', 'student']
+        db.delete_unique('courseware_studentmodule', ['module_id', 'module_type', 'student_id'])
+
+        # Adding field 'StudentModule.max_grade'
+        db.add_column('courseware_studentmodule', 'max_grade', self.gf('django.db.models.fields.FloatField')(null=True, blank=True), keep_default=False)
+
+        # Adding field 'StudentModule.done'
+        db.add_column('courseware_studentmodule', 'done', self.gf('django.db.models.fields.CharField')(default='na', max_length=8, db_index=True), keep_default=False)
+
+        # Adding unique constraint on 'StudentModule', fields ['module_id', 'student']
+        db.create_unique('courseware_studentmodule', ['module_id', 'student_id'])
+
+
+    def backwards(self, orm):
+        
+        # Removing unique constraint on 'StudentModule', fields ['module_id', 'student']
+        db.delete_unique('courseware_studentmodule', ['module_id', 'student_id'])
+
+        # Deleting field 'StudentModule.max_grade'
+        db.delete_column('courseware_studentmodule', 'max_grade')
+
+        # Deleting field 'StudentModule.done'
+        db.delete_column('courseware_studentmodule', 'done')
+
+        # Adding unique constraint on 'StudentModule', fields ['module_id', 'module_type', 'student']
+        db.create_unique('courseware_studentmodule', ['module_id', 'module_type', 'student_id'])
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'about': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'avatar_type': ('django.db.models.fields.CharField', [], {'default': "'n'", 'max_length': '1'}),
+            'bronze': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'consecutive_days_visit_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'country': ('django_countries.fields.CountryField', [], {'max_length': '2', 'blank': 'True'}),
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'date_of_birth': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'display_tag_filter_strategy': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'email_isvalid': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'email_key': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True'}),
+            'email_tag_filter_strategy': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'gold': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'gravatar': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'ignored_tags': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'interesting_tags': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'last_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'location': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'new_response_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'questions_per_page': ('django.db.models.fields.SmallIntegerField', [], {'default': '10'}),
+            'real_name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'reputation': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}),
+            'seen_response_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'show_country': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'silver': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'status': ('django.db.models.fields.CharField', [], {'default': "'w'", 'max_length': '2'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}),
+            'website': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'courseware.studentmodule': {
+            'Meta': {'unique_together': "(('student', 'module_id'),)", 'object_name': 'StudentModule'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'done': ('django.db.models.fields.CharField', [], {'default': "'na'", 'max_length': '8', 'db_index': 'True'}),
+            'grade': ('django.db.models.fields.FloatField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'max_grade': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'module_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'module_type': ('django.db.models.fields.CharField', [], {'default': "'problem'", 'max_length': '32', 'db_index': 'True'}),
+            'state': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'student': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        }
+    }
+
+    complete_apps = ['courseware']

djangoapps/courseware/models.py

@@ -0,0 +1,92 @@
+"""
+WE'RE USING MIGRATIONS!
+
+If you make changes to this model, be sure to create an appropriate migration
+file and check it in at the same time as your model changes. To do that,
+
+1. Go to the mitx dir
+2. ./manage.py schemamigration courseware --auto description_of_your_change
+3. Add the migration file created in mitx/courseware/migrations/
+
+
+ASSUMPTIONS: modules have unique IDs, even across different module_types
+
+"""
+from django.db import models
+from django.db.models.signals import post_save, post_delete
+#from django.core.cache import cache
+from django.contrib.auth.models import User
+
+#from cache_toolbox import cache_model, cache_relation
+
+#CACHE_TIMEOUT = 60 * 60 * 4 # Set the cache timeout to be four hours
+
+class StudentModule(models.Model):
+    # For a homework problem, contains a JSON
+    # object consisting of state
+    MODULE_TYPES = (('problem','problem'),
+                    ('video','video'),
+                    ('html','html'),
+                    )
+    ## These three are the key for the object
+    module_type = models.CharField(max_length=32, choices=MODULE_TYPES, default='problem', db_index=True)
+    module_id = models.CharField(max_length=255, db_index=True) # Filename for homeworks, etc. 
+    student = models.ForeignKey(User, db_index=True)
+    class Meta:
+        unique_together = (('student', 'module_id'),)
+
+    ## Internal state of the object
+    state = models.TextField(null=True, blank=True)
+
+    ## Grade, and are we done? 
+    grade = models.FloatField(null=True, blank=True, db_index=True)
+    max_grade = models.FloatField(null=True, blank=True)
+    DONE_TYPES = (('na','NOT_APPLICABLE'),
+                    ('f','FINISHED'),
+                    ('i','INCOMPLETE'),
+                    )
+    done = models.CharField(max_length=8, choices=DONE_TYPES, default='na', db_index=True)
+
+    # DONE_TYPES = (('done','DONE'),          # Finished
+    #               ('incomplete','NOTDONE'), # Not finished
+    #               ('na','NA'))              # Not applicable (e.g. vertical)
+    # done = models.CharField(max_length=16, choices=DONE_TYPES)
+
+    created = models.DateTimeField(auto_now_add=True, db_index=True)
+    modified = models.DateTimeField(auto_now=True, db_index=True)
+
+    def __unicode__(self):
+        return self.module_type+'/'+self.student.username+"/"+self.module_id+'/'+str(self.state)[:20]
+
+    # @classmethod
+    # def get_with_caching(cls, student, module_id):
+    #     k = cls.key_for(student, module_id)
+    #     student_module = cache.get(k)
+    #     if student_module is None:
+    #         student_module = StudentModule.objects.filter(student=student,
+    #                                                       module_id=module_id)[0]
+    #         # It's possible it really doesn't exist...
+    #         if student_module is not None:
+    #             cache.set(k, student_module, CACHE_TIMEOUT)
+
+    #     return student_module
+
+    @classmethod
+    def key_for(cls, student, module_id):
+        return "StudentModule-student_id:{0};module_id:{1}".format(student.id, module_id)
+
+
+# def clear_cache_by_student_and_module_id(sender, instance, *args, **kwargs):
+#     k = sender.key_for(instance.student, instance.module_id)
+#     cache.delete(k)
+
+# def update_cache_by_student_and_module_id(sender, instance, *args, **kwargs):
+#     k = sender.key_for(instance.student, instance.module_id)
+#     cache.set(k, instance, CACHE_TIMEOUT)
+
+
+#post_save.connect(update_cache_by_student_and_module_id, sender=StudentModule, weak=False)
+#post_delete.connect(clear_cache_by_student_and_module_id, sender=StudentModule, weak=False)
+
+#cache_model(StudentModule)
+

djangoapps/courseware/module_render.py

@@ -0,0 +1,140 @@
+import StringIO
+import json
+import logging
+import os
+import sys
+import sys
+import urllib
+import uuid
+
+from lxml import etree
+
+from django.conf import settings
+from django.contrib.auth.models import User
+from django.core.context_processors import csrf
+from django.db import connection
+from django.http import Http404
+from django.http import HttpResponse
+from django.shortcuts import redirect
+from django.template import Context
+from django.template import Context, loader
+from mitxmako.shortcuts import render_to_response, render_to_string
+
+from models import StudentModule
+from student.models import UserProfile
+import track.views
+
+import courseware.content_parser as content_parser
+
+import courseware.modules
+
+log = logging.getLogger("mitx.courseware")
+
+def object_cache(cache, user, module_type, module_id):
+    # We don't look up on user -- all queries include user
+    # Additional lookup would require a DB hit the way Django 
+    # is broken. 
+    for o in cache: 
+        if o.module_type == module_type and \
+                o.module_id == module_id:
+            return o
+    return None
+
+def make_track_function(request):
+    def f(event_type, event):
+        return track.views.server_track(request, event_type, event, page='x_module')
+    return f
+
+def modx_dispatch(request, module=None, dispatch=None, id=None):
+    ''' Generic view for extensions. '''
+    if not request.user.is_authenticated():
+        return redirect('/')
+
+    # Grab the student information for the module from the database
+    s = StudentModule.objects.filter(student=request.user, 
+                                     module_id=id)
+    #s = StudentModule.get_with_caching(request.user, id)
+    if len(s) == 0 or s is None:
+        log.debug("Couldnt find module for user and id " + str(module) + " " + str(request.user) + " "+ str(id))
+        raise Http404
+    s = s[0]
+
+    oldgrade = s.grade
+    oldstate = s.state
+
+    dispatch=dispatch.split('?')[0]
+
+    ajax_url = '/modx/'+module+'/'+id+'/'
+
+    # Grab the XML corresponding to the request from course.xml
+    xml = content_parser.module_xml(request.user, module, 'id', id)
+
+    # Create the module
+    instance=courseware.modules.get_module_class(module)(xml, 
+                                                         id, 
+                                                         ajax_url=ajax_url, 
+                                                         state=oldstate, 
+                                                         track_function = make_track_function(request), 
+                                                         render_function = None)
+    # Let the module handle the AJAX
+    ajax_return=instance.handle_ajax(dispatch, request.POST)
+    # Save the state back to the database
+    s.state=instance.get_state()
+    if instance.get_score(): 
+        s.grade=instance.get_score()['score']
+    if s.grade != oldgrade or s.state != oldstate:
+        s.save()
+    # Return whatever the module wanted to return to the client/caller
+    return HttpResponse(ajax_return)
+
+def render_x_module(user, request, xml_module, module_object_preload):
+    ''' Generic module for extensions. This renders to HTML. '''
+    # Check if problem has an instance in DB
+    module_type=xml_module.tag
+    module_class=courseware.modules.get_module_class(module_type)
+    module_id=xml_module.get('id') #module_class.id_attribute) or "" 
+
+    # Grab state from database
+    smod = object_cache(module_object_preload, 
+                        user, 
+                        module_type, 
+                        module_id)
+
+    if not smod: # If nothing in the database...
+        state=None
+    else:
+        state = smod.state
+
+    # Create a new instance
+    ajax_url = '/modx/'+module_type+'/'+module_id+'/'
+    instance=module_class(etree.tostring(xml_module), 
+                          module_id, 
+                          ajax_url=ajax_url,
+                          state=state, 
+                          track_function = make_track_function(request), 
+                          render_function = lambda x: render_module(user, request, x, module_object_preload))
+    
+    # If instance wasn't already in the database, create it
+    if not smod:
+        smod=StudentModule(student=user, 
+                           module_type = module_type,
+                           module_id=module_id, 
+                           state=instance.get_state())
+        smod.save()
+        module_object_preload.append(smod)
+    # Grab content
+    content = instance.get_html()
+    if user.is_staff:
+        content=content+render_to_string("staff_problem_info.html", {'xml':etree.tostring(xml_module)})
+    content = {'content':content, 
+               "destroy_js":instance.get_destroy_js(),
+               'init_js':instance.get_init_js(), 
+               'type':module_type}
+
+    return content
+
+def render_module(user, request, module, module_object_preload):
+    ''' Generic dispatch for internal modules. '''
+    if module==None :
+        return {"content":""}
+    return render_x_module(user, request, module, module_object_preload)

djangoapps/courseware/modules/__init__.py

@@ -0,0 +1,69 @@
+import os
+import os.path
+
+from django.conf import settings
+
+import capa_module
+import html_module
+import schematic_module
+import seq_module
+import template_module
+import vertical_module
+import video_module
+
+from courseware import content_parser
+
+# Import all files in modules directory, excluding backups (# and . in name) 
+# and __init__
+#
+# Stick them in a list
+# modx_module_list = []
+
+# for f in os.listdir(os.path.dirname(__file__)):
+#     if f!='__init__.py' and \
+#             f[-3:] == ".py" and \
+#             "." not in f[:-3] \
+#             and '#' not in f:
+#         mod_path = 'courseware.modules.'+f[:-3]
+#         mod = __import__(mod_path, fromlist = "courseware.modules")
+#         if 'Module' in mod.__dict__:
+#             modx_module_list.append(mod)
+
+#print modx_module_list
+modx_module_list = [capa_module,  html_module,  schematic_module,  seq_module,  template_module,  vertical_module,  video_module]
+#print modx_module_list
+
+modx_modules = {}
+
+# Convert list to a dictionary for lookup by tag
+def update_modules():
+    global modx_modules
+    modx_modules = dict()
+    for module in modx_module_list:
+        for tag in module.Module.get_xml_tags():
+            modx_modules[tag] = module.Module
+
+update_modules()
+
+def get_module_class(tag):
+    ''' Given an XML tag (e.g. 'video'), return 
+    the associated module (e.g. video_module.Module). 
+    '''
+    if tag not in modx_modules:
+        update_modules()
+    return modx_modules[tag]
+
+def get_module_id(tag):
+    ''' Given an XML tag (e.g. 'video'), return 
+    the default ID for that module (e.g. 'youtube_id')
+    '''
+    return modx_modules[tag].id_attribute
+
+def get_valid_tags():
+    return modx_modules.keys()
+
+def get_default_ids():
+    tags = get_valid_tags()
+    ids = map(get_module_id, tags)
+    return dict(zip(tags, ids))
+

djangoapps/courseware/modules/capa_module.py

@@ -0,0 +1,382 @@
+import StringIO
+import datetime
+import dateutil
+import dateutil.parser
+import json
+import logging
+import math
+import numpy
+import os
+import random
+import scipy
+import struct
+import sys
+import traceback
+
+from lxml import etree
+
+## TODO: Abstract out from Django
+from django.conf import settings
+from mitxmako.shortcuts import render_to_response, render_to_string
+from django.http import Http404
+
+from x_module import XModule
+from courseware.capa.capa_problem import LoncapaProblem, StudentInputError
+import courseware.content_parser as content_parser
+
+log = logging.getLogger("mitx.courseware")
+
+class Module(XModule):
+    ''' Interface between capa_problem and x_module. Originally a hack
+    meant to be refactored out, but it seems to be serving a useful
+    prupose now. We can e.g .destroy and create the capa_problem on a
+    reset. 
+    '''
+
+    id_attribute = "filename"
+
+    @classmethod
+    def get_xml_tags(c):
+        return ["problem"]
+
+    def get_state(self):
+        state = self.lcp.get_state()
+        state['attempts'] = self.attempts
+        return json.dumps(state)
+
+    def get_score(self):
+        return self.lcp.get_score()
+
+    def max_score(self):
+        return self.lcp.get_max_score()
+
+    def get_html(self):
+        return render_to_string('problem_ajax.html', 
+                              {'id':self.item_id, 
+                               'ajax_url':self.ajax_url,
+                               })
+
+    def get_init_js(self):
+        return render_to_string('problem.js', 
+                              {'id':self.item_id, 
+                               'ajax_url':self.ajax_url,
+                               })
+
+    def get_problem_html(self, encapsulate=True):
+        html = self.lcp.get_html()
+        content={'name':self.name, 
+                 'html':html}
+        
+        check_button = True
+        reset_button = True
+        save_button = True
+
+        # If we're after deadline, or user has exhuasted attempts, 
+        # question is read-only. 
+        if self.closed():
+            check_button = False
+            reset_button = False
+            save_button = False
+            
+
+        # User submitted a problem, and hasn't reset. We don't want
+        # more submissions. 
+        if self.lcp.done and self.rerandomize == "always":
+            #print "!"
+            check_button = False
+            save_button = False
+        
+        # User hasn't submitted an answer yet -- we don't want resets
+        if not self.lcp.done:
+            reset_button = False
+
+        attempts_str = ""
+        if self.max_attempts != None: 
+            attempts_str = " ({a}/{m})".format(a=self.attempts, m=self.max_attempts)
+
+        # We don't need a "save" button if infinite number of attempts and non-randomized
+        if self.max_attempts == None and self.rerandomize != "always":
+            save_button = False
+
+        # Check if explanation is available, and if so, give a link
+        explain=""
+        if self.lcp.done and self.explain_available=='attempted':
+            explain=self.explanation
+        if self.closed() and self.explain_available=='closed':
+            explain=self.explanation
+        
+        if len(explain) == 0:
+            explain = False
+
+        html=render_to_string('problem.html', 
+                              {'problem' : content, 
+                               'id' : self.item_id, 
+                               'check_button' : check_button,
+                               'reset_button' : reset_button,
+                               'save_button' : save_button,
+                               'answer_available' : self.answer_available(),
+                               'ajax_url' : self.ajax_url,
+                               'attempts': attempts_str, 
+                               'explain': explain
+                               })
+        if encapsulate:
+            html = '<div id="main_{id}">'.format(id=self.item_id)+html+"</div>"
+            
+        return html
+
+    def __init__(self, xml, item_id, ajax_url=None, track_url=None, state=None, track_function=None, render_function = None, meta = None):
+        XModule.__init__(self, xml, item_id, ajax_url, track_url, state, track_function, render_function)
+
+        self.attempts = 0
+        self.max_attempts = None
+        
+        dom2 = etree.fromstring(xml)
+
+        self.explanation=content_parser.item(dom2.xpath('/problem/@explain'), default="closed")
+        self.explain_available=content_parser.item(dom2.xpath('/problem/@explain_available'))
+
+        display_due_date_string=content_parser.item(dom2.xpath('/problem/@due'))
+        if len(display_due_date_string)>0:
+            self.display_due_date=dateutil.parser.parse(display_due_date_string)
+            #log.debug("Parsed " + display_due_date_string + " to " + str(self.display_due_date))
+        else:
+            self.display_due_date=None
+        
+        
+        grace_period_string = content_parser.item(dom2.xpath('/problem/@graceperiod'))
+        if len(grace_period_string)>0 and self.display_due_date:
+            self.grace_period = content_parser.parse_timedelta(grace_period_string)
+            self.close_date = self.display_due_date + self.grace_period
+            #log.debug("Then parsed " + grace_period_string + " to closing date" + str(self.close_date))
+        else:
+            self.grace_period = None
+            self.close_date = self.display_due_date
+            
+        self.max_attempts=content_parser.item(dom2.xpath('/problem/@attempts'))
+        if len(self.max_attempts)>0:
+            self.max_attempts=int(self.max_attempts)
+        else:
+            self.max_attempts=None
+
+        self.show_answer=content_parser.item(dom2.xpath('/problem/@showanswer'))
+
+        if self.show_answer=="":
+            self.show_answer="closed"
+
+        self.rerandomize=content_parser.item(dom2.xpath('/problem/@rerandomize'))
+        if self.rerandomize=="" or self.rerandomize=="always" or self.rerandomize=="true":
+            self.rerandomize="always"
+        elif self.rerandomize=="false" or self.rerandomize=="per_student":
+            self.rerandomize="per_student"
+        elif self.rerandomize=="never":
+            self.rerandomize="never"
+        else:
+            raise Exception("Invalid rerandomize attribute "+self.rerandomize)
+
+        if state!=None:
+            state=json.loads(state)
+        if state!=None and 'attempts' in state:
+            self.attempts=state['attempts']
+
+        self.filename=content_parser.item(dom2.xpath('/problem/@filename'))
+        filename=settings.DATA_DIR+"/problems/"+self.filename+".xml"
+        self.name=content_parser.item(dom2.xpath('/problem/@name'))
+        if self.rerandomize == 'never':
+            seed = 1
+        else:
+            seed = None
+        self.lcp=LoncapaProblem(filename, self.item_id, state, seed = seed)
+
+    def handle_ajax(self, dispatch, get):
+        if dispatch=='problem_get':
+            response = self.get_problem(get)
+        elif False: #self.close_date > 
+            return json.dumps({"error":"Past due date"})
+        elif dispatch=='problem_check': 
+            response = self.check_problem(get)
+        elif dispatch=='problem_reset':
+            response = self.reset_problem(get)
+        elif dispatch=='problem_save':
+            response = self.save_problem(get)
+        elif dispatch=='problem_show':
+            response = self.get_answer(get)
+        else: 
+            return "Error"
+        return response
+
+    def closed(self):
+        ''' Is the student still allowed to submit answers? '''
+        if self.attempts == self.max_attempts:
+            return True
+        if self.close_date != None and datetime.datetime.utcnow() > self.close_date:
+            return True
+
+        return False
+        
+
+    def answer_available(self):
+        ''' Is the user allowed to see an answer? 
+        ''' 
+        if self.show_answer == '':
+            return False
+        if self.show_answer == "never":
+            return False
+        if self.show_answer == 'attempted' and self.attempts == 0:
+            return False
+        if self.show_answer == 'attempted' and self.attempts > 0:
+            return True
+        if self.show_answer == 'answered' and self.lcp.done:
+            return True
+        if self.show_answer == 'answered' and not self.lcp.done:
+            return False
+        if self.show_answer == 'closed' and self.closed():
+            return True
+        if self.show_answer == 'closed' and not self.closed():
+            return False
+        print "aa", self.show_answer
+        raise Http404
+
+    def get_answer(self, get):
+        if not self.answer_available():
+            raise Http404
+        else: 
+            return json.dumps(self.lcp.get_question_answers())
+
+
+    # Figure out if we should move these to capa_problem?
+    def get_problem(self, get):
+        ''' Same as get_problem_html -- if we want to reconfirm we
+            have the right thing e.g. after several AJAX calls.'''
+        return self.get_problem_html(encapsulate=False)        
+
+    def check_problem(self, get):
+        ''' Checks whether answers to a problem are correct, and
+            returns a map of correct/incorrect answers'''
+        event_info = dict()
+        event_info['state'] = self.lcp.get_state()
+        event_info['filename'] = self.filename
+
+        answers=dict()
+        # input_resistor_1 ==> resistor_1
+        for key in get:
+            answers['_'.join(key.split('_')[1:])]=get[key]
+
+#        print "XXX", answers, get
+
+        event_info['answers']=answers
+
+        # Too late. Cannot submit
+        if self.closed():
+            event_info['failure']='closed'
+            self.tracker('save_problem_check_fail', event_info)
+            print "cp"
+            raise Http404
+            
+        # Problem submitted. Student should reset before checking
+        # again.
+        if self.lcp.done and self.rerandomize == "always":
+            event_info['failure']='unreset'
+            self.tracker('save_problem_check_fail', event_info)
+            print "cpdr"
+            raise Http404
+
+        try:
+            old_state = self.lcp.get_state()
+            lcp_id = self.lcp.problem_id
+            filename = self.lcp.filename
+            correct_map = self.lcp.grade_answers(answers)
+        except StudentInputError as inst: 
+            self.lcp = LoncapaProblem(filename, id=lcp_id, state=old_state)
+            traceback.print_exc()
+#            print {'error':sys.exc_info(),
+#                   'answers':answers, 
+#                   'seed':self.lcp.seed, 
+#                   'filename':self.lcp.filename}
+            return json.dumps({'success':inst.message})
+        except: 
+            self.lcp = LoncapaProblem(filename, id=lcp_id, state=old_state)
+            traceback.print_exc()
+            return json.dumps({'success':'Unknown Error'})
+            
+
+        self.attempts = self.attempts + 1
+        self.lcp.done=True
+
+        success = 'correct'
+        for i in correct_map:
+            if correct_map[i]!='correct':
+                success = 'incorrect'
+
+        js=json.dumps({'correct_map' : correct_map,
+                       'success' : success})
+
+        event_info['correct_map']=correct_map
+        event_info['success']=success
+
+        self.tracker('save_problem_check', event_info)
+
+        return js
+
+    def save_problem(self, get):
+        event_info = dict()
+        event_info['state'] = self.lcp.get_state()
+        event_info['filename'] = self.filename
+
+        answers=dict()
+        for key in get:
+            answers['_'.join(key.split('_')[1:])]=get[key]
+        event_info['answers'] = answers
+
+        # Too late. Cannot submit
+        if self.closed():
+            event_info['failure']='closed'
+            self.tracker('save_problem_fail', event_info)
+            return "Problem is closed"
+            
+        # Problem submitted. Student should reset before saving
+        # again.
+        if self.lcp.done and self.rerandomize == "always":
+            event_info['failure']='done'
+            self.tracker('save_problem_fail', event_info)
+            return "Problem needs to be reset prior to save."
+
+        self.lcp.student_answers=answers
+
+        self.tracker('save_problem_fail', event_info)
+        return json.dumps({'success':True})
+
+    def reset_problem(self, get):
+        ''' Changes problem state to unfinished -- removes student answers, 
+            and causes problem to rerender itself. '''
+        event_info = dict()
+        event_info['old_state']=self.lcp.get_state()
+        event_info['filename']=self.filename
+
+        if self.closed():
+            event_info['failure']='closed'
+            self.tracker('reset_problem_fail', event_info)
+            return "Problem is closed"
+            
+        if not self.lcp.done:
+            event_info['failure']='not_done'
+            self.tracker('reset_problem_fail', event_info)
+            return "Refresh the page and make an attempt before resetting."
+
+        self.lcp.done=False
+        self.lcp.answers=dict()
+        self.lcp.correct_map=dict()
+        self.lcp.student_answers = dict()
+
+
+        if self.rerandomize == "always":
+            self.lcp.context=dict()
+            self.lcp.questions=dict() # Detailed info about questions in problem instance. TODO: Should be by id and not lid. 
+            self.lcp.seed=None
+
+        filename=settings.DATA_DIR+"problems/"+self.filename+".xml"
+        self.lcp=LoncapaProblem(filename, self.item_id, self.lcp.get_state())
+
+        event_info['new_state']=self.lcp.get_state()
+        self.tracker('reset_problem', event_info)
+
+        return json.dumps(self.get_problem_html(encapsulate=False))

djangoapps/courseware/modules/html_module.py

@@ -0,0 +1,38 @@
+import json
+
+## TODO: Abstract out from Django
+from django.conf import settings
+from mitxmako.shortcuts import render_to_response, render_to_string
+
+from x_module import XModule
+from lxml import etree
+
+class Module(XModule):
+    id_attribute = 'filename'
+
+    def get_state(self):
+        return json.dumps({ })
+
+    @classmethod
+    def get_xml_tags(c):
+        return ["html"]
+        
+    def get_html(self):
+        if self.filename==None:
+            xmltree=etree.fromstring(self.xml)
+            textlist=[xmltree.text]+[etree.tostring(i) for i in xmltree]+[xmltree.tail]
+            textlist=[i for i in textlist if type(i)==str]
+            return "".join(textlist)
+        try: 
+            filename=settings.DATA_DIR+"html/"+self.filename
+            return open(filename).read()
+        except: # For backwards compatibility. TODO: Remove
+            return render_to_string(self.filename, {'id': self.item_id})
+
+    def __init__(self, xml, item_id, ajax_url=None, track_url=None, state=None, track_function=None, render_function = None):
+        XModule.__init__(self, xml, item_id, ajax_url, track_url, state, track_function, render_function)
+        xmltree=etree.fromstring(xml)
+        self.filename = None
+        filename_l=xmltree.xpath("/html/@filename")
+        if len(filename_l)>0:
+            self.filename=str(filename_l[0])

djangoapps/courseware/modules/schematic_module.py

@@ -0,0 +1,24 @@
+import json
+
+## TODO: Abstract out from Django
+from django.conf import settings
+from mitxmako.shortcuts import render_to_response, render_to_string
+
+from x_module import XModule
+
+class Module(XModule):
+    id_attribute = 'id'
+
+    def get_state(self):
+        return json.dumps({ })
+
+    @classmethod
+    def get_xml_tags(c):
+        return ["schematic"]
+        
+    def get_html(self):
+        return '<input type="hidden" class="schematic" name="{item_id}" height="480" width="640">'.format(item_id=self.item_id)
+
+    def __init__(self, xml, item_id, ajax_url=None, track_url=None, state=None, render_function = None):
+        XModule.__init__(self, xml, item_id, ajax_url, track_url, state, render_function)
+

djangoapps/courseware/modules/seq_module.py

@@ -0,0 +1,121 @@
+import json
+
+from lxml import etree
+
+## TODO: Abstract out from Django
+from django.http import Http404
+from django.conf import settings
+from mitxmako.shortcuts import render_to_response, render_to_string
+
+from x_module import XModule
+
+# HACK: This shouldn't be hard-coded to two types
+# OBSOLETE: This obsoletes 'type'
+class_priority = ['video', 'problem']
+
+class Module(XModule):
+    ''' Layout module which lays out content in a temporal sequence
+    '''
+    id_attribute = 'id'
+
+    def get_state(self):
+        return json.dumps({ 'position':self.position })
+
+    @classmethod
+    def get_xml_tags(c):
+        return ["sequential", 'tab']
+        
+    def get_html(self):
+        self.render()
+        return self.content
+
+    def get_init_js(self):
+        self.render()
+        return self.init_js
+
+    def get_destroy_js(self):
+        self.render()
+        return self.destroy_js
+
+    def handle_ajax(self, dispatch, get):
+        print "GET", get
+        print "DISPATCH", dispatch
+        if dispatch=='goto_position':
+            self.position = int(get['position'])
+            return json.dumps({'success':True})
+        raise Http404()
+
+    def render(self):
+        if self.rendered:
+            return
+        def j(m): 
+            ''' jsonify contents so it can be embedded in a js array
+            We also need to split </script> tags so they don't break
+            mid-string'''
+            if 'init_js' not in m: m['init_js']=""
+            if 'type' not in m: m['init_js']=""
+            content=json.dumps(m['content']) 
+            content=content.replace('</script>', '<"+"/script>') 
+
+            return {'content':content, 
+                    "destroy_js":m['destroy_js'], 
+                    'init_js':m['init_js'], 
+                    'type': m['type']}
+
+
+        ## Returns a set of all types of all sub-children
+        child_classes = [set([i.tag for i in e.iter()]) for e in self.xmltree]
+
+        self.titles = json.dumps(["\n".join([i.get("name").strip() for i in e.iter() if i.get("name") != None]) \
+                           for e in self.xmltree])
+
+        self.contents = [j(self.render_function(e)) \
+                             for e in self.xmltree]
+
+        print self.titles
+
+        for (content, element_class) in zip(self.contents, child_classes):
+            new_class = 'other'
+            for c in class_priority:
+                if c in element_class: 
+                    new_class = c
+            content['type'] = new_class
+     
+        js=""
+
+        params={'items':self.contents,
+                'id':self.item_id,
+                'position': self.position,
+                'titles':self.titles}
+
+        # TODO/BUG: Destroy JavaScript should only be called for the active view
+        # This calls it for all the views
+        # 
+        # To fix this, we'd probably want to have some way of assigning unique
+        # IDs to sequences. 
+        destroy_js="".join([e['destroy_js'] for e in self.contents if 'destroy_js' in e])
+
+        if self.xmltree.tag == 'sequential':
+            self.init_js=js+render_to_string('seq_module.js',params)
+            self.destroy_js=destroy_js
+            self.content=render_to_string('seq_module.html',params)
+        if self.xmltree.tag == 'tab':
+            params['id'] = 'tab'
+            self.init_js=js+render_to_string('tab_module.js',params)
+            self.destroy_js=destroy_js
+            self.content=render_to_string('tab_module.html',params)
+        self.rendered = True
+        
+
+
+    def __init__(self, xml, item_id, ajax_url=None, track_url=None, state=None, track_function=None, render_function = None):
+        XModule.__init__(self, xml, item_id, ajax_url, track_url, state, track_function, render_function)
+        self.xmltree=etree.fromstring(xml)
+
+        self.position = 1
+
+        if state!=None:
+            state = json.loads(state)
+            if 'position' in state: self.position = int(state['position'])
+
+        self.rendered = False

djangoapps/courseware/modules/template_module.py

@@ -0,0 +1,29 @@
+import json
+import os
+
+## TODO: Abstract out from Django
+from django.conf import settings
+from mitxmako.shortcuts import render_to_response, render_to_string
+
+from x_module import XModule
+from lxml import etree
+
+class Module(XModule):
+    def get_state(self):
+        return json.dumps({ })
+
+    @classmethod
+    def get_xml_tags(c):
+        tags = os.listdir(settings.DATA_DIR+'/custom_tags')
+        return tags
+
+    def get_html(self):
+        return self.html
+
+    def __init__(self, xml, item_id, ajax_url=None, track_url=None, state=None, track_function=None, render_function = None):
+        XModule.__init__(self, xml, item_id, ajax_url, track_url, state, track_function, render_function)
+        xmltree = etree.fromstring(xml)
+        filename = xmltree.tag
+        params = dict(xmltree.items())
+#        print params
+        self.html = render_to_string(filename, params, namespace = 'custom_tags')

djangoapps/courseware/modules/vertical_module.py

@@ -0,0 +1,35 @@
+import json
+
+## TODO: Abstract out from Django
+from django.conf import settings
+from mitxmako.shortcuts import render_to_response, render_to_string
+
+from x_module import XModule
+from lxml import etree
+
+class Module(XModule):
+    id_attribute = 'id'
+
+    def get_state(self):
+        return json.dumps({ })
+
+    @classmethod
+    def get_xml_tags(c):
+        return ["vertical"]
+        
+    def get_html(self):
+        return render_to_string('vert_module.html',{'items':self.contents})
+
+    def get_init_js(self):
+        return self.init_js_text
+
+    def get_destroy_js(self):
+        return self.destroy_js_text
+
+    def __init__(self, xml, item_id, ajax_url=None, track_url=None, state=None, track_function=None, render_function = None):
+        XModule.__init__(self, xml, item_id, ajax_url, track_url, state, track_function, render_function)
+        xmltree=etree.fromstring(xml)
+        self.contents=[(e.get("name"),self.render_function(e)) \
+                      for e in xmltree]
+        self.init_js_text="".join([e[1]['init_js'] for e in self.contents if 'init_js' in e[1]])
+        self.destroy_js_text="".join([e[1]['destroy_js'] for e in self.contents if 'destroy_js' in e[1]])

djangoapps/courseware/modules/video_module.py

@@ -0,0 +1,70 @@
+import json
+import logging
+
+from lxml import etree
+
+## TODO: Abstract out from Django
+from django.conf import settings
+from mitxmako.shortcuts import render_to_response, render_to_string
+
+from x_module import XModule
+
+log = logging.getLogger("mitx.courseware.modules")
+
+class Module(XModule):
+    id_attribute = 'youtube'
+    video_time = 0
+
+    def handle_ajax(self, dispatch, get):
+        log.debug(u"GET {0}".format(get))
+        log.debug(u"DISPATCH {0}".format(dispatch))
+        if dispatch == 'goto_position':
+            self.position = int(float(get['position']))
+            log.debug(u"NEW POSITION {0}".format(self.position))
+            return json.dumps({'success':True})
+        raise Http404()
+
+    def get_state(self):
+        log.debug(u"STATE POSITION {0}".format(self.position))
+        return json.dumps({ 'position':self.position })
+
+    @classmethod
+    def get_xml_tags(c):
+        '''Tags in the courseware file guaranteed to correspond to the module'''
+        return ["video"]
+        
+    def video_list(self):
+        l = self.youtube.split(',')
+        l = [i.split(":") for i in l]
+        return json.dumps(dict(l))
+    
+    def get_html(self):
+        return render_to_string('video.html',{'streams':self.video_list(),
+                                              'id':self.item_id,
+                                              'position':self.position, 
+                                              'name':self.name})
+
+    def get_init_js(self):
+        '''JavaScript code to be run when problem is shown. Be aware
+        that this may happen several times on the same page
+        (e.g. student switching tabs). Common functions should be put
+        in the main course .js files for now. ''' 
+        log.debug(u"INIT POSITION {0}".format(self.position))
+        return render_to_string('video_init.js',{'streams':self.video_list(),
+                                                 'id':self.item_id,
+                                                 'position':self.position})
+
+    def get_destroy_js(self):
+        return "videoDestroy(\"{0}\");".format(self.item_id)
+
+    def __init__(self, xml, item_id, ajax_url=None, track_url=None, state=None, track_function=None, render_function = None):
+        XModule.__init__(self, xml, item_id, ajax_url, track_url, state, track_function, render_function)
+        self.youtube = etree.XML(xml).get('youtube')
+        self.name = etree.XML(xml).get('name')
+        self.position = 0
+        if state != None:
+            state = json.loads(state)
+            if 'position' in state:
+                self.position = int(float(state['position']))
+            #log.debug("POSITION IN STATE")
+        #log.debug(u"LOAD POSITION {0}".format(self.position))

djangoapps/courseware/modules/x_module.py

@@ -0,0 +1,57 @@
+import courseware.progress
+
+def dummy_track(event_type, event):
+    pass
+
+class XModule(object):
+    ''' Implements a generic learning module. 
+        Initialized on access with __init__, first time with state=None, and
+        then with state
+    '''
+    id_attribute='id' # An attribute guaranteed to be unique
+
+    @classmethod
+    def get_xml_tags(c):
+        ''' Tags in the courseware file guaranteed to correspond to the module '''
+        return []
+        
+    def get_completion(self):
+        return courseware.progress.completion()
+    
+    def get_state(self):
+        return ""
+
+    def get_score(self):
+        return None
+
+    def max_score(self):
+        return None
+
+    def get_html(self):
+        return "Unimplemented"
+
+    def get_init_js(self):
+        ''' JavaScript code to be run when problem is shown. Be aware
+        that this may happen several times on the same page
+        (e.g. student switching tabs). Common functions should be put
+        in the main course .js files for now. ''' 
+        return ""
+
+    def get_destroy_js(self):
+        return ""
+
+    def handle_ajax(self, dispatch, get):
+        ''' dispatch is last part of the URL. 
+            get is a dictionary-like object ''' 
+        return ""
+
+    def __init__(self, xml, item_id, ajax_url=None, track_url=None, state=None, track_function=None, render_function = None):
+        ''' In most cases, you must pass state or xml'''
+        self.xml = xml
+        self.item_id = item_id
+        self.ajax_url = ajax_url
+        self.track_url = track_url
+        self.state = state
+        self.tracker = track_function
+        self.render_function = render_function
+

djangoapps/courseware/progress.py

@@ -0,0 +1,38 @@
+class completion(object):
+    def __init__(self, **d):
+        self.dict = dict({'duration_total':0,
+                          'duration_watched':0,
+                          'done':True,
+                          'questions_correct':0,
+                          'questions_incorrect':0,
+                          'questions_total':0})
+        if d: 
+            self.dict.update(d)
+
+    def __getitem__(self, key):
+        return self.dict[key]
+
+    def __setitem__(self, key, value):
+        self.dict[key] = value
+
+    def __add__(self, other):
+        result = dict(self.dict)
+        for item in ['duration_total',
+                     'duration_watched',
+                     'done',
+                     'questions_correct',
+                     'questions_incorrect',
+                     'questions_total']:
+            result[item] = result[item]+other.dict[item]
+        return completion(**result)
+
+    def __contains__(self, key):
+        return key in dict
+
+    def __repr__(self):
+        return repr(self.dict)
+
+if __name__ == '__main__':
+    dict1=completion(duration_total=5)
+    dict2=completion(duration_total=7)
+    print dict1+dict2

djangoapps/courseware/tests.py

@@ -0,0 +1,53 @@
+import unittest
+
+import numpy
+
+import courseware.modules
+import courseware.capa.calc as calc
+
+class ModelsTest(unittest.TestCase):
+    def setUp(self):
+        pass
+
+    def test_get_module_class(self):
+        vc = courseware.modules.get_module_class('video')
+        vc_str = "<class 'courseware.modules.video_module.Module'>"
+        self.assertEqual(str(vc), vc_str)
+        video_id = courseware.modules.get_default_ids()['video']
+        self.assertEqual(video_id, 'youtube')
+
+    def test_calc(self):
+        variables={'R1':2.0, 'R3':4.0}
+        functions={'sin':numpy.sin, 'cos':numpy.cos}
+
+        self.assertEqual(calc.evaluator(variables, functions, "10000||sin(7+5)-6k"), 4000.0)
+        self.assertEqual(calc.evaluator({'R1': 2.0, 'R3':4.0}, {}, "13"), 13)
+        self.assertEqual(calc.evaluator(variables, functions, "13"), 13)
+        self.assertEqual(calc.evaluator({'a': 2.2997471478310274, 'k': 9, 'm': 8, 'x': 0.66009498411213041}, {}, "5"), 5)
+        self.assertEqual(calc.evaluator({},{}, "-1"), -1)
+        self.assertEqual(calc.evaluator({},{}, "-0.33"), -.33)
+        self.assertEqual(calc.evaluator({},{}, "-.33"), -.33)
+        self.assertEqual(calc.evaluator(variables, functions, "R1*R3"), 8.0)
+        self.assertTrue(abs(calc.evaluator(variables, functions, "sin(e)-0.41"))<0.01)
+        self.assertTrue(abs(calc.evaluator(variables, functions, "k*T/q-0.025"))<0.001)
+        exception_happened = False
+        try: 
+            calc.evaluator({},{}, "5+7 QWSEKO")
+        except:
+            exception_happened = True
+        self.assertTrue(exception_happened)
+
+        try: 
+            calc.evaluator({'r1':5},{}, "r1+r2")
+        except calc.UndefinedVariable:
+            pass
+        
+        self.assertEqual(calc.evaluator(variables, functions, "r1*r3"), 8.0)
+
+        exception_happened = False
+        try: 
+            calc.evaluator(variables, functions, "r1*r3", cs=True)
+        except:
+            exception_happened = True
+        self.assertTrue(exception_happened)
+

djangoapps/courseware/views.py

@@ -0,0 +1,188 @@
+import json
+import logging
+import os
+import random
+import sys
+import StringIO
+import urllib
+import uuid
+
+from django.conf import settings
+from django.core.context_processors import csrf
+from django.contrib.auth.models import User
+from django.http import HttpResponse, Http404
+from django.shortcuts import redirect
+from django.template import Context, loader
+from mitxmako.shortcuts import render_to_response, render_to_string
+#from django.views.decorators.csrf import ensure_csrf_cookie
+from django.db import connection
+from django.views.decorators.cache import cache_control
+
+from lxml import etree
+
+from module_render import render_module, modx_dispatch
+from models import StudentModule
+from student.models import UserProfile
+
+import courseware.content_parser as content_parser
+import courseware.modules.capa_module
+
+import courseware.grades as grades
+
+log = logging.getLogger("mitx.courseware")
+
+etree.set_default_parser(etree.XMLParser(dtd_validation=False, load_dtd=False,
+                                         remove_comments = True))
+
+template_imports={'urllib':urllib}
+
+@cache_control(no_cache=True, no_store=True, must_revalidate=True)
+def gradebook(request):
+    if 'course_admin' not in content_parser.user_groups(request.user):
+        raise Http404
+    student_objects = User.objects.all()[:100]
+    student_info = [{'username' :s.username,
+                     'id' : s.id,
+                     'email': s.email,
+                     'grade_info' : grades.grade_sheet(s), 
+                     'realname' : UserProfile.objects.get(user = s).name
+                     } for s in student_objects]
+
+    return render_to_response('gradebook.html',{'students':student_info})
+
+@cache_control(no_cache=True, no_store=True, must_revalidate=True)
+def profile(request, student_id = None):
+    ''' User profile. Show username, location, etc, as well as grades .
+        We need to allow the user to change some of these settings .'''
+    if not request.user.is_authenticated():
+        return redirect('/')
+
+    if student_id == None:
+        student = request.user
+    else: 
+        print content_parser.user_groups(request.user)
+        if 'course_admin' not in content_parser.user_groups(request.user):
+            raise Http404
+        student = User.objects.get( id = int(student_id))
+
+    user_info = UserProfile.objects.get(user=student) # request.user.profile_cache # 
+
+    context={'name':user_info.name,
+             'username':student.username,
+             'location':user_info.location,
+             'language':user_info.language,
+             'email':student.email,
+             'format_url_params' : content_parser.format_url_params,
+             'csrf':csrf(request)['csrf_token']
+             }
+    context.update(grades.grade_sheet(student))
+
+    return render_to_response('profile.html', context)
+
+def render_accordion(request,course,chapter,section):
+    ''' Draws navigation bar. Takes current position in accordion as
+        parameter. Returns (initialization_javascript, content)'''
+    if not course:
+        course = "6.002 Spring 2012"
+    
+    toc=content_parser.toc_from_xml(content_parser.course_file(request.user), chapter, section)
+    active_chapter=1
+    for i in range(len(toc)):
+        if toc[i]['active']:
+            active_chapter=i
+    context=dict([['active_chapter',active_chapter],
+                  ['toc',toc], 
+                  ['course_name',course],
+                  ['format_url_params',content_parser.format_url_params],
+                  ['csrf',csrf(request)['csrf_token']]] + \
+                     template_imports.items())
+    return {'init_js':render_to_string('accordion_init.js',context), 
+            'content':render_to_string('accordion.html',context)}
+
+@cache_control(no_cache=True, no_store=True, must_revalidate=True)
+def render_section(request, section):
+    ''' TODO: Consolidate with index 
+    '''
+    user = request.user
+    if not settings.COURSEWARE_ENABLED or not user.is_authenticated():
+        return redirect('/')
+
+#    try: 
+    dom = content_parser.section_file(user, section)
+    #except:
+     #   raise Http404
+
+    accordion=render_accordion(request, '', '', '')
+
+    module_ids = dom.xpath("//@id")
+    
+    module_object_preload = list(StudentModule.objects.filter(student=user, 
+                                                              module_id__in=module_ids))
+    
+    module=render_module(user, request, dom, module_object_preload)
+
+    if 'init_js' not in module:
+        module['init_js']=''
+
+    context={'init':accordion['init_js']+module['init_js'],
+             'accordion':accordion['content'],
+             'content':module['content'],
+             'csrf':csrf(request)['csrf_token']}
+
+    result = render_to_response('courseware.html', context)
+    return result
+
+
+@cache_control(no_cache=True, no_store=True, must_revalidate=True)
+def index(request, course="6.002 Spring 2012", chapter="Using the System", section="Hints"): 
+    ''' Displays courseware accordion, and any associated content. 
+    ''' 
+    user = request.user
+    if not settings.COURSEWARE_ENABLED or not user.is_authenticated():
+        return redirect('/')
+
+    # Fixes URLs -- we don't get funny encoding characters from spaces
+    # so they remain readable
+    ## TODO: Properly replace underscores
+    course=course.replace("_"," ")
+    chapter=chapter.replace("_"," ")
+    section=section.replace("_"," ")
+
+    # HACK: Force course to 6.002 for now
+    # Without this, URLs break
+    if course!="6.002 Spring 2012":
+        return redirect('/')
+
+    #import logging
+    #log = logging.getLogger("mitx")
+    #log.info(  "DEBUG: "+str(user) )
+
+    dom = content_parser.course_file(user)
+    dom_module = dom.xpath("//course[@name=$course]/chapter[@name=$chapter]//section[@name=$section]/*[1]", 
+                           course=course, chapter=chapter, section=section)
+    if len(dom_module) == 0:
+        module = None
+    else:
+        module = dom_module[0]
+
+    accordion=render_accordion(request, course, chapter, section)
+
+    module_ids = dom.xpath("//course[@name=$course]/chapter[@name=$chapter]//section[@name=$section]//@id", 
+                           course=course, chapter=chapter, section=section)
+
+    module_object_preload = list(StudentModule.objects.filter(student=user, 
+                                                              module_id__in=module_ids))
+    
+
+    module=render_module(user, request, module, module_object_preload)
+
+    if 'init_js' not in module:
+        module['init_js']=''
+
+    context={'init':accordion['init_js']+module['init_js'],
+             'accordion':accordion['content'],
+             'content':module['content'],
+             'csrf':csrf(request)['csrf_token']}
+
+    result = render_to_response('courseware.html', context)
+    return result