Merge pull request #157 from MITx/cpennington/cms-github

Enable interaction with github

cms/djangoapps/contentstore/__init__.py

@@ -0,0 +1,29 @@
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.xml import XMLModuleStore
+import logging
+
+log = logging.getLogger(__name__)
+
+
+def import_from_xml(org, course, data_dir):
+    """
+    Import the specified xml data_dir into the django defined modulestore,
+    using org and course as the location org and course.
+    """
+    module_store = XMLModuleStore(org, course, data_dir, 'xmodule.raw_module.RawDescriptor', eager=True)
+    for module in module_store.modules.itervalues():
+
+        # TODO (cpennington): This forces import to overrite the same items.
+        # This should in the future create new revisions of the items on import
+        try:
+            modulestore().create_item(module.location)
+        except:
+            log.exception('Item already exists at %s' % module.location.url())
+            pass
+        if 'data' in module.definition:
+            modulestore().update_item(module.location, module.definition['data'])
+        if 'children' in module.definition:
+            modulestore().update_children(module.location, module.definition['children'])
+        modulestore().update_metadata(module.location, dict(module.metadata))
+
+    return module_store.course

cms/djangoapps/contentstore/management/commands/import.py

@@ -3,8 +3,7 @@
 ###
 
 from django.core.management.base import BaseCommand, CommandError
-from xmodule.modulestore.django import modulestore
-from xmodule.modulestore.xml import XMLModuleStore
+from contentstore import import_from_xml
 
 unnamed_modules = 0
 
@@ -18,12 +17,4 @@ class Command(BaseCommand):
             raise CommandError("import requires 3 arguments: <org> <course> <data directory>")
 
         org, course, data_dir = args
-
-        module_store = XMLModuleStore(org, course, data_dir, 'xmodule.raw_module.RawDescriptor', eager=True)
-        for module in module_store.modules.itervalues():
-            modulestore().create_item(module.location)
-            if 'data' in module.definition:
-                modulestore().update_item(module.location, module.definition['data'])
-            if 'children' in module.definition:
-                modulestore().update_children(module.location, module.definition['children'])
-            modulestore().update_metadata(module.location, dict(module.metadata))
+        import_from_xml(org, course, data_dir)

cms/djangoapps/contentstore/views.py

@@ -1,20 +1,35 @@
-from mitxmako.shortcuts import render_to_response
-from xmodule.modulestore.django import modulestore
-from django_future.csrf import ensure_csrf_cookie
-from django.http import HttpResponse
 import json
 
+from django.http import HttpResponse
+from django_future.csrf import ensure_csrf_cookie
 from fs.osfs import OSFS
+from django.core.urlresolvers import reverse
+from xmodule.modulestore import Location
+from github_sync import repo_path_from_location, export_to_github
+
+from mitxmako.shortcuts import render_to_response
+from xmodule.modulestore.django import modulestore
+
 
 @ensure_csrf_cookie
 def index(request):
+    courses = modulestore().get_items(['i4x', None, None, 'course', None])
+    return render_to_response('index.html', {
+        'courses': [(course.metadata['display_name'],
+                    reverse('course_index', args=[
+                        course.location.org,
+                        course.location.course,
+                        course.location.name]))
+                    for course in courses]
+    })
+
+
+@ensure_csrf_cookie
+def course_index(request, org, course, name):
     # TODO (cpennington): These need to be read in from the active user
-    org = 'mit.edu'
-    course = '6002xs12'
-    name = '6.002_Spring_2012'
     course = modulestore().get_item(['i4x', org, course, 'course', name])
     weeks = course.get_children()
-    return render_to_response('index.html', {'weeks': weeks})
+    return render_to_response('course_index.html', {'weeks': weeks})
 
 
 def edit_item(request):
@@ -32,6 +47,14 @@ def save_item(request):
     item_id = request.POST['id']
     data = json.loads(request.POST['data'])
     modulestore().update_item(item_id, data)
+
+    # Export the course back to github
+    course_location = Location(item_id)._replace(category='course', name=None)
+    courses = modulestore().get_items(course_location)
+    for course in courses:
+        repo_path = repo_path_from_location(course.location)
+        export_to_github(course, repo_path, "CMS Edit")
+
     return HttpResponse(json.dumps({}))
 
 

cms/djangoapps/github_sync/__init__.py

@@ -0,0 +1,75 @@
+import logging
+import os
+
+from django.conf import settings
+from fs.osfs import OSFS
+from git import Repo, PushInfo
+
+from contentstore import import_from_xml
+from xmodule.modulestore import Location
+
+from .exceptions import GithubSyncError
+
+log = logging.getLogger(__name__)
+
+def import_from_github(repo_settings):
+    """
+    Imports data into the modulestore based on the XML stored on github
+
+    repo_settings is a dictionary with the following keys:
+        path: file system path to the local git repo
+        branch: name of the branch to track on github
+        org: name of the organization to use in the imported course
+        course: name of the coures to use in the imported course
+    """
+    repo_path = repo_settings['path']
+
+    if not os.path.isdir(repo_path):
+        Repo.clone_from(repo_settings['origin'], repo_path)
+
+    git_repo = Repo(repo_path)
+    origin = git_repo.remotes.origin
+    origin.fetch()
+
+    # Do a hard reset to the remote branch so that we have a clean import
+    git_repo.git.checkout(repo_settings['branch'])
+    git_repo.head.reset('origin/%s' % repo_settings['branch'], index=True, working_tree=True)
+
+    return git_repo.head.commit.hexsha, import_from_xml(repo_settings['org'], repo_settings['course'], repo_path)
+
+
+def repo_path_from_location(location):
+    location = Location(location)
+    for name, repo in settings.REPOS.items():
+        if repo['org'] == location.org and repo['course'] == location.course:
+            return repo['path']
+
+
+def export_to_github(course, repo_path, commit_message):
+    fs = OSFS(repo_path)
+    xml = course.export_to_xml(fs)
+
+    with fs.open('course.xml', 'w') as course_xml:
+        course_xml.write(xml)
+
+    git_repo = Repo(repo_path)
+    if git_repo.is_dirty():
+        git_repo.git.add(A=True)
+        git_repo.git.commit(m=commit_message)
+
+        origin = git_repo.remotes.origin
+        if settings.MITX_FEATURES['GITHUB_PUSH']:
+            push_infos = origin.push()
+            if len(push_infos) > 1:
+                log.error('Unexpectedly pushed multiple heads: {infos}'.format(
+                    infos="\n".join(str(info.summary) for info in push_infos)
+                ))
+
+            if push_infos[0].flags & PushInfo.ERROR:
+                log.error('Failed push: flags={p.flags}, local_ref={p.local_ref}, '
+                          'remote_ref_string={p.remote_ref_string}, '
+                          'remote_ref={p.remote_ref}, old_commit={p.old_commit}, '
+                          'summary={p.summary})'.format(p=push_infos[0]))
+                raise GithubSyncError('Failed to push: {info}'.format(
+                    info=str(push_infos[0].summary)
+                ))

cms/djangoapps/github_sync/exceptions.py

@@ -0,0 +1,2 @@
+class GithubSyncError(Exception):
+    pass

cms/djangoapps/github_sync/tests/__init__.py

@@ -0,0 +1,96 @@
+from django.test import TestCase
+from path import path
+import shutil
+from github_sync import import_from_github, export_to_github, repo_path_from_location
+from git import Repo
+from django.conf import settings
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore import Location
+from override_settings import override_settings
+from github_sync.exceptions import GithubSyncError
+
+
+class GithubSyncTestCase(TestCase):
+
+    def setUp(self):
+        self.working_dir = path(settings.TEST_ROOT)
+        self.repo_dir = self.working_dir / 'local_repo'
+        self.remote_dir = self.working_dir / 'remote_repo'
+        shutil.copytree('common/test/data/toy', self.remote_dir)
+
+        remote = Repo.init(self.remote_dir)
+        remote.git.add(A=True)
+        remote.git.commit(m='Initial commit')
+        remote.git.config("receive.denyCurrentBranch", "ignore")
+
+        modulestore().collection.drop()
+
+        self.import_revision, self.import_course = import_from_github({
+            'path': self.repo_dir,
+            'origin': self.remote_dir,
+            'branch': 'master',
+            'org': 'org',
+            'course': 'course'
+        })
+
+    def tearDown(self):
+        shutil.rmtree(self.repo_dir)
+        shutil.rmtree(self.remote_dir)
+
+    def test_initialize_repo(self):
+        """
+        Test that importing from github will create a repo if the repo doesn't already exist
+        """
+        self.assertEquals(1, len(Repo(self.repo_dir).head.reference.log()))
+
+    def test_import_contents(self):
+        """
+        Test that the import loads the correct course into the modulestore
+        """
+        self.assertEquals('Toy Course', self.import_course.metadata['display_name'])
+        self.assertIn(
+            Location('i4x://org/course/chapter/Overview'),
+            [child.location for child in self.import_course.get_children()])
+        self.assertEquals(1, len(self.import_course.get_children()))
+
+    @override_settings(MITX_FEATURES={'GITHUB_PUSH': False})
+    def test_export_no_pash(self):
+        """
+        Test that with the GITHUB_PUSH feature disabled, no content is pushed to the remote
+        """
+        export_to_github(self.import_course, self.repo_dir, 'Test no-push')
+        self.assertEquals(1, Repo(self.remote_dir).head.commit.count())
+
+    @override_settings(MITX_FEATURES={'GITHUB_PUSH': True})
+    def test_export_push(self):
+        """
+        Test that with GITHUB_PUSH enabled, content is pushed to the remote
+        """
+        self.import_course.metadata['display_name'] = 'Changed display name'
+        export_to_github(self.import_course, self.repo_dir, 'Test push')
+        self.assertEquals(2, Repo(self.remote_dir).head.commit.count())
+
+    @override_settings(MITX_FEATURES={'GITHUB_PUSH': True})
+    def test_export_conflict(self):
+        """
+        Test that if there is a conflict when pushing to the remote repo, nothing is pushed and an exception is raised
+        """
+        self.import_course.metadata['display_name'] = 'Changed display name'
+
+        remote = Repo(self.remote_dir)
+        remote.git.commit(allow_empty=True, m="Testing conflict commit")
+
+        self.assertRaises(GithubSyncError, export_to_github, self.import_course, self.repo_dir, 'Test push')
+        self.assertEquals(2, remote.head.reference.commit.count())
+        self.assertEquals("Testing conflict commit\n", remote.head.reference.commit.message)
+
+
+@override_settings(REPOS={'namea': {'path': 'patha', 'org': 'orga', 'course': 'coursea'},
+                          'nameb': {'path': 'pathb', 'org': 'orgb', 'course': 'courseb'}})
+class RepoPathLookupTestCase(TestCase):
+    def test_successful_lookup(self):
+        self.assertEquals('patha', repo_path_from_location('i4x://orga/coursea/course/foo'))
+        self.assertEquals('pathb', repo_path_from_location('i4x://orgb/courseb/course/foo'))
+
+    def test_failed_lookup(self):
+        self.assertEquals(None, repo_path_from_location('i4x://c/c/course/foo'))

cms/djangoapps/github_sync/tests/test_views.py

@@ -0,0 +1,53 @@
+import json
+from django.test.client import Client
+from django.test import TestCase
+from mock import patch, Mock
+from override_settings import override_settings
+from django.conf import settings
+
+
+@override_settings(REPOS={'repo': {'path': 'path', 'branch': 'branch'}})
+class PostReceiveTestCase(TestCase):
+    def setUp(self):
+        self.client = Client()
+
+    @patch('github_sync.views.export_to_github')
+    @patch('github_sync.views.import_from_github')
+    def test_non_branch(self, import_from_github, export_to_github):
+        self.client.post('/github_service_hook', {'payload': json.dumps({
+            'ref': 'refs/tags/foo'})
+        })
+        self.assertFalse(import_from_github.called)
+        self.assertFalse(export_to_github.called)
+
+    @patch('github_sync.views.export_to_github')
+    @patch('github_sync.views.import_from_github')
+    def test_non_watched_repo(self, import_from_github, export_to_github):
+        self.client.post('/github_service_hook', {'payload': json.dumps({
+            'ref': 'refs/heads/branch',
+            'repository': {'name': 'bad_repo'}})
+        })
+        self.assertFalse(import_from_github.called)
+        self.assertFalse(export_to_github.called)
+
+    @patch('github_sync.views.export_to_github')
+    @patch('github_sync.views.import_from_github')
+    def test_non_tracked_branch(self, import_from_github, export_to_github):
+        self.client.post('/github_service_hook', {'payload': json.dumps({
+            'ref': 'refs/heads/non_branch',
+            'repository': {'name': 'repo'}})
+        })
+        self.assertFalse(import_from_github.called)
+        self.assertFalse(export_to_github.called)
+
+    @patch('github_sync.views.export_to_github')
+    @patch('github_sync.views.import_from_github', return_value=(Mock(), Mock()))
+    def test_tracked_branch(self, import_from_github, export_to_github):
+        self.client.post('/github_service_hook', {'payload': json.dumps({
+            'ref': 'refs/heads/branch',
+            'repository': {'name': 'repo'}})
+        })
+        import_from_github.assert_called_with(settings.REPOS['repo'])
+        mock_revision, mock_course = import_from_github.return_value
+        export_to_github.assert_called_with(mock_course, 'path', "Changes from cms import of revision %s" % mock_revision)
+

cms/djangoapps/github_sync/views.py

@@ -0,0 +1,52 @@
+import logging
+import json
+
+from django.http import HttpResponse
+from django.conf import settings
+from django_future.csrf import csrf_exempt
+
+from . import import_from_github, export_to_github
+
+log = logging.getLogger()
+
+
+@csrf_exempt
+def github_post_receive(request):
+    """
+    This view recieves post-receive requests from github whenever one of
+    the watched repositiories changes.
+
+    It is responsible for updating the relevant local git repo,
+    importing the new version of the course (if anything changed),
+    and then pushing back to github any changes that happened as part of the
+    import.
+
+    The github request format is described here: https://help.github.com/articles/post-receive-hooks
+    """
+
+    payload = json.loads(request.POST['payload'])
+
+    ref = payload['ref']
+
+    if not ref.startswith('refs/heads/'):
+        log.info('Ignore changes to non-branch ref %s' % ref)
+        return HttpResponse('Ignoring non-branch')
+
+    branch_name = ref.replace('refs/heads/', '', 1)
+
+    repo_name = payload['repository']['name']
+
+    if repo_name not in settings.REPOS:
+        log.info('No repository matching %s found' % repo_name)
+        return HttpResponse('No Repo Found')
+
+    repo = settings.REPOS[repo_name]
+
+    if repo['branch'] != branch_name:
+        log.info('Ignoring changes to non-tracked branch %s in repo %s' % (branch_name, repo_name))
+        return HttpResponse('Ignoring non-tracked branch')
+
+    revision, course = import_from_github(repo)
+    export_to_github(course, repo['path'], "Changes from cms import of revision %s" % revision)
+
+    return HttpResponse('Push recieved')

cms/envs/common.py

@@ -30,6 +30,7 @@ from path import path
 
 MITX_FEATURES = {
     'USE_DJANGO_PIPELINE': True,
+    'GITHUB_PUSH': False,
 }
 
 ############################# SET PATH INFORMATION #############################

cms/envs/dev.py

@@ -29,8 +29,48 @@ DATABASES = {
     }
 }
 
+REPO_ROOT = ENV_ROOT / "content"
+
+REPOS = {
+    'edx4edx': {
+        'path': REPO_ROOT / "edx4edx",
+        'org': 'edx',
+        'course': 'edx4edx',
+        'branch': 'for_cms',
+        'origin': 'git@github.com:MITx/edx4edx.git',
+    },
+    '6002x-fall-2012': {
+        'path': REPO_ROOT / '6002x-fall-2012',
+        'org': 'mit.edu',
+        'course': '6.002x',
+        'branch': 'for_cms',
+        'origin': 'git@github.com:MITx/6002x-fall-2012.git',
+    },
+    '6.00x': {
+        'path': REPO_ROOT / '6.00x',
+        'org': 'mit.edu',
+        'course': '6.00x',
+        'branch': 'for_cms',
+        'origin': 'git@github.com:MITx/6.00x.git',
+    },
+    '7.00x': {
+        'path': REPO_ROOT / '7.00x',
+        'org': 'mit.edu',
+        'course': '7.00x',
+        'branch': 'for_cms',
+        'origin': 'git@github.com:MITx/7.00x.git',
+    },
+    '3.091x': {
+        'path': REPO_ROOT / '3.091x',
+        'org': 'mit.edu',
+        'course': '3.091x',
+        'branch': 'for_cms',
+        'origin': 'git@github.com:MITx/3.091x.git',
+    },
+}
+
 CACHES = {
-    # This is the cache used for most things. Askbot will not work without a 
+    # This is the cache used for most things. Askbot will not work without a
     # functioning cache -- it relies on caching to load its settings in places.
     # In staging/prod envs, the sessions also live here.
     'default': {

cms/envs/test.py

@@ -17,10 +17,18 @@ for app in os.listdir(PROJECT_ROOT / 'djangoapps'):
     NOSE_ARGS += ['--cover-package', app]
 TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
 
+TEST_ROOT = 'test_root'
+
 MODULESTORE = {
-    'host': 'localhost',
-    'db': 'mongo_base',
-    'collection': 'key_store',
+    'default': {
+        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
+        'OPTIONS': {
+            'default_class': 'xmodule.raw_module.RawDescriptor',
+            'host': 'localhost',
+            'db': 'test_xmodule',
+            'collection': 'modulestore',
+        }
+    }
 }
 
 DATABASES = {

cms/static/coffee/unit.coffee

@@ -4,7 +4,7 @@ class @Unit
 
         $("##{@element_id} .save-update").click (event) =>
             event.preventDefault()
-            $.post("save_item", {
+            $.post("/save_item", {
                 id: @module_id
                 data: JSON.stringify(@module.save())
             })

cms/templates/course_index.html

@@ -0,0 +1,16 @@
+<%inherit file="base.html" />
+<%block name="title">Course Manager</%block>
+
+<%block name="content">
+<section class="main-container">
+
+  <%include file="widgets/navigation.html"/>
+
+  <section class="main-content">
+    <section class="edit-pane">
+      <div id="module-html"/>
+    </section>
+  </section>
+
+</section>
+</%block>

cms/templates/index.html

@@ -1,16 +1,12 @@
 <%inherit file="base.html" />
-<%block name="title">Course Manager</%block>
+<%block name="title">Courses</%block>
 
 <%block name="content">
 <section class="main-container">
-
-  <%include file="widgets/navigation.html"/>
-
-  <section class="main-content">
-    <section class="edit-pane">
-      <div id="module-html"/>
-    </section>
-  </section>
-
+  <ol>
+    %for course, url in courses:
+    <li><a href="${url}">${course}</a></li>
+    %endfor
+  </ol>
 </section>
 </%block>

cms/urls.py

@@ -8,5 +8,7 @@ urlpatterns = patterns('',
     url(r'^$', 'contentstore.views.index', name='index'),
     url(r'^edit_item$', 'contentstore.views.edit_item', name='edit_item'),
     url(r'^save_item$', 'contentstore.views.save_item', name='save_item'),
-    url(r'^temp_force_export$', 'contentstore.views.temp_force_export')
+    url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/course/(?P<name>[^/]+)$', 'contentstore.views.course_index', name='course_index'),
+    url(r'^temp_force_export$', 'contentstore.views.temp_force_export'),
+    url(r'^github_service_hook$', 'github_sync.views.github_post_receive'),
 )

common/lib/capa/setup.py

@@ -0,0 +1,8 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="capa",
+    version="0.1",
+    packages=find_packages(exclude=["tests"]),
+    install_requires=['distribute'],
+)

common/lib/mitxmako/setup.py

@@ -0,0 +1,8 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="mitxmako",
+    version="0.1",
+    packages=find_packages(exclude=["tests"]),
+    install_requires=['distribute'],
+)

common/lib/xmodule/setup.py

@@ -8,6 +8,10 @@ setup(
     package_data={
         'xmodule': ['js/module/*']
     },
+    requires=[
+        'capa',
+        'mitxmako'
+    ],
 
     # See http://guide.python-distribute.org/creation.html#entry-points
     # for a description of entry_points

common/lib/xmodule/xmodule/html_module.py

@@ -1,4 +1,5 @@
 import logging
+from lxml import etree
 
 from xmodule.x_module import XModule
 from xmodule.raw_module import RawDescriptor
@@ -26,3 +27,8 @@ class HtmlDescriptor(RawDescriptor):
 
     js = {'coffee': [resource_string(__name__, 'js/module/html.coffee')]}
     js_module = 'HTML'
+
+    @classmethod
+    def file_to_xml(cls, file_object):
+        parser = etree.HTMLParser()
+        return etree.parse(file_object, parser).getroot()

common/lib/xmodule/xmodule/js/module/html.coffee

@@ -6,4 +6,4 @@ class @HTML
             @preview.empty().append(@edit_box.val())
         )
 
-    save: -> {text: @edit_box.val()}
+    save: -> @edit_box.val()

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -159,6 +159,18 @@ class ModuleStore(object):
             location is found
         """
         raise NotImplementedError
+    
+    def get_items(self, location, default_class=None):
+        """
+        Returns a list of XModuleDescriptor instances for the items
+        that match location. Any element of location that is None is treated
+        as a wildcard that matches any value
+
+        location: Something that can be passed to Location
+        default_class: An XModuleDescriptor subclass to use if no plugin matching the
+            location is found
+        """
+        raise NotImplementedError
 
     # TODO (cpennington): Replace with clone_item
     def create_item(self, location, editor):

common/lib/xmodule/xmodule/modulestore/mongo.py

@@ -1,4 +1,5 @@
 import pymongo
+from bson.objectid import ObjectId
 from importlib import import_module
 from xmodule.x_module import XModuleDescriptor
 from xmodule.mako_module import MakoDescriptorSystem
@@ -8,6 +9,19 @@ from . import ModuleStore, Location
 from .exceptions import ItemNotFoundError, InsufficientSpecificationError
 
 
+# TODO (cpennington): This code currently operates under the assumption that
+# there is only one revision for each item. Once we start versioning inside the CMS,
+# that assumption will have to change
+
+
+def location_to_query(loc):
+    query = {}
+    for key, val in Location(loc).dict().iteritems():
+        if val is not None:
+            query['_id.{key}'.format(key=key)] = val
+
+    return query
+
 class MongoModuleStore(ModuleStore):
     """
     A Mongodb backed ModuleStore
@@ -17,7 +31,6 @@ class MongoModuleStore(ModuleStore):
             host=host,
             port=port
         )[db][collection]
-        self.collection.ensure_index('location')
 
         # Force mongo to report errors, at the expense of performance
         self.collection.safe = True
@@ -26,6 +39,18 @@ class MongoModuleStore(ModuleStore):
         class_ = getattr(import_module(module_path), class_name)
         self.default_class = class_
 
+        # TODO (cpennington): Pass a proper resources_fs to the system
+        self.system = MakoDescriptorSystem(
+            load_item=self.get_item,
+            resources_fs=None,
+            render_template=render_to_string
+        )
+
+    def _load_item(self, item):
+        item['location'] = item['_id']
+        del item['_id']
+        return XModuleDescriptor.load_from_json(item, self.system, self.default_class)
+
     def get_item(self, location):
         """
         Returns an XModuleDescriptor instance for the item at location.
@@ -39,24 +64,26 @@ class MongoModuleStore(ModuleStore):
         location: Something that can be passed to Location
         """
 
-        query = {}
         for key, val in Location(location).dict().iteritems():
             if key != 'revision' and val is None:
                 raise InsufficientSpecificationError(location)
 
-            if val is not None:
-                query['location.{key}'.format(key=key)] = val
-
         item = self.collection.find_one(
-            query,
+            location_to_query(location),
             sort=[('revision', pymongo.ASCENDING)],
         )
         if item is None:
             raise ItemNotFoundError(location)
+        return self._load_item(item)
 
-        # TODO (cpennington): Pass a proper resources_fs to the system
-        return XModuleDescriptor.load_from_json(
-            item, MakoDescriptorSystem(load_item=self.get_item, resources_fs=None, render_template=render_to_string), self.default_class)
+    def get_items(self, location, default_class=None):
+        print location_to_query(location)
+        items = self.collection.find(
+            location_to_query(location),
+            sort=[('revision', pymongo.ASCENDING)],
+        )
+
+        return [self._load_item(item) for item in items]
 
     def create_item(self, location):
         """
@@ -65,7 +92,7 @@ class MongoModuleStore(ModuleStore):
         location: Something that can be passed to Location
         """
         self.collection.insert({
-            'location': Location(location).dict(),
+            '_id': Location(location).dict(),
         })
 
     def update_item(self, location, data):
@@ -80,8 +107,9 @@ class MongoModuleStore(ModuleStore):
         # See http://www.mongodb.org/display/DOCS/Updating for
         # atomic update syntax
         self.collection.update(
-            {'location': Location(location).dict()},
-            {'$set': {'definition.data': data}}
+            {'_id': Location(location).dict()},
+            {'$set': {'definition.data': data}},
+
         )
 
     def update_children(self, location, children):
@@ -96,7 +124,7 @@ class MongoModuleStore(ModuleStore):
         # See http://www.mongodb.org/display/DOCS/Updating for
         # atomic update syntax
         self.collection.update(
-            {'location': Location(location).dict()},
+            {'_id': Location(location).dict()},
             {'$set': {'definition.children': children}}
         )
 
@@ -112,6 +140,6 @@ class MongoModuleStore(ModuleStore):
         # See http://www.mongodb.org/display/DOCS/Updating for
         # atomic update syntax
         self.collection.update(
-            {'location': Location(location).dict()},
+            {'_id': Location(location).dict()},
             {'$set': {'metadata': metadata}}
         )

common/lib/xmodule/xmodule/xml_module.py

@@ -90,6 +90,16 @@ class XmlDescriptor(XModuleDescriptor):
             if xml_object.get(attr) is not None:
                 del xml_object.attrib[attr]
 
+    @classmethod
+    def file_to_xml(cls, file_object):
+        """
+        Used when this module wants to parse a file object to xml
+        that will be converted to the definition.
+
+        Returns an lxml Element
+        """
+        return etree.parse(file_object).getroot()
+
     @classmethod
     def from_xml(cls, xml_data, system, org=None, course=None):
         """
@@ -128,7 +138,7 @@ class XmlDescriptor(XModuleDescriptor):
                 log.debug('filepath=%s, resources_fs=%s' % (filepath,system.resources_fs))
                 with system.resources_fs.open(filepath) as file:
                     try:
-                        definition_xml = etree.parse(file).getroot()
+                        definition_xml = cls.file_to_xml(file)
                     except:
                         log.exception("Failed to parse xml in file %s" % filepath)
                         raise
@@ -149,7 +159,7 @@ class XmlDescriptor(XModuleDescriptor):
 
     @classmethod
     def _format_filepath(cls, category, name):
-        return '{category}/{name}.{ext}'.format(category=category, name=name, ext=cls.filename_extension)
+        return u'{category}/{name}.{ext}'.format(category=category, name=name, ext=cls.filename_extension)
 
     def export_to_xml(self, resource_fs):
         """

lms/envs/test.py

@@ -20,7 +20,8 @@ INSTALLED_APPS = [
 
 # Nose Test Runner
 INSTALLED_APPS += ['django_nose']
-NOSE_ARGS = ['--cover-erase', '--with-xunit', '--with-xcoverage', '--cover-html', '--cover-inclusive', '--cover-html-dir', os.environ['NOSE_COVER_HTML_DIR']]
+NOSE_ARGS = ['--cover-erase', '--with-xunit', '--with-xcoverage', '--cover-html',
+             '--cover-inclusive', '--cover-html-dir', os.environ.get('NOSE_COVER_HTML_DIR', 'cover_html')]
 for app in os.listdir(PROJECT_ROOT / 'djangoapps'):
     NOSE_ARGS += ['--cover-package', app]
 TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'

requirements.txt

@@ -1,5 +1,6 @@
 django<1.4
 pip
+numpy
 scipy
 matplotlib
 markdown
@@ -24,7 +25,12 @@ sympy
 newrelic
 glob2
 pymongo
+-e common/lib/capa
+-e common/lib/mitxmako
 -e common/lib/xmodule
 django_nose
 nosexcover
 rednose
+GitPython >= 0.3
+django-override-settings
+mock>=0.8, <0.9

test_root/.gitignore

@@ -0,0 +1,2 @@
+local_repo
+remote_repo