diff --git a/cms/djangoapps/contentstore/__init__.py b/cms/djangoapps/contentstore/__init__.py index e69de29bb2..433278c47b 100644 --- a/cms/djangoapps/contentstore/__init__.py +++ b/cms/djangoapps/contentstore/__init__.py @@ -0,0 +1,29 @@ +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.xml import XMLModuleStore +import logging + +log = logging.getLogger(__name__) + + +def import_from_xml(org, course, data_dir): + """ + Import the specified xml data_dir into the django defined modulestore, + using org and course as the location org and course. + """ + module_store = XMLModuleStore(org, course, data_dir, 'xmodule.raw_module.RawDescriptor', eager=True) + for module in module_store.modules.itervalues(): + + # TODO (cpennington): This forces import to overrite the same items. + # This should in the future create new revisions of the items on import + try: + modulestore().create_item(module.location) + except: + log.exception('Item already exists at %s' % module.location.url()) + pass + if 'data' in module.definition: + modulestore().update_item(module.location, module.definition['data']) + if 'children' in module.definition: + modulestore().update_children(module.location, module.definition['children']) + modulestore().update_metadata(module.location, dict(module.metadata)) + + return module_store.course diff --git a/cms/djangoapps/contentstore/management/commands/import.py b/cms/djangoapps/contentstore/management/commands/import.py index f97ac10d41..89323c3d9b 100644 --- a/cms/djangoapps/contentstore/management/commands/import.py +++ b/cms/djangoapps/contentstore/management/commands/import.py @@ -3,8 +3,7 @@ ### from django.core.management.base import BaseCommand, CommandError -from xmodule.modulestore.django import modulestore -from xmodule.modulestore.xml import XMLModuleStore +from contentstore import import_from_xml unnamed_modules = 0 @@ -18,12 +17,4 @@ class Command(BaseCommand): raise CommandError("import requires 3 arguments: ") org, course, data_dir = args - - module_store = XMLModuleStore(org, course, data_dir, 'xmodule.raw_module.RawDescriptor', eager=True) - for module in module_store.modules.itervalues(): - modulestore().create_item(module.location) - if 'data' in module.definition: - modulestore().update_item(module.location, module.definition['data']) - if 'children' in module.definition: - modulestore().update_children(module.location, module.definition['children']) - modulestore().update_metadata(module.location, dict(module.metadata)) + import_from_xml(org, course, data_dir) diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index 76a904a403..ca06942ae4 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -1,20 +1,35 @@ -from mitxmako.shortcuts import render_to_response -from xmodule.modulestore.django import modulestore -from django_future.csrf import ensure_csrf_cookie -from django.http import HttpResponse import json +from django.http import HttpResponse +from django_future.csrf import ensure_csrf_cookie from fs.osfs import OSFS +from django.core.urlresolvers import reverse +from xmodule.modulestore import Location +from github_sync import repo_path_from_location, export_to_github + +from mitxmako.shortcuts import render_to_response +from xmodule.modulestore.django import modulestore + @ensure_csrf_cookie def index(request): + courses = modulestore().get_items(['i4x', None, None, 'course', None]) + return render_to_response('index.html', { + 'courses': [(course.metadata['display_name'], + reverse('course_index', args=[ + course.location.org, + course.location.course, + course.location.name])) + for course in courses] + }) + + +@ensure_csrf_cookie +def course_index(request, org, course, name): # TODO (cpennington): These need to be read in from the active user - org = 'mit.edu' - course = '6002xs12' - name = '6.002_Spring_2012' course = modulestore().get_item(['i4x', org, course, 'course', name]) weeks = course.get_children() - return render_to_response('index.html', {'weeks': weeks}) + return render_to_response('course_index.html', {'weeks': weeks}) def edit_item(request): @@ -32,6 +47,14 @@ def save_item(request): item_id = request.POST['id'] data = json.loads(request.POST['data']) modulestore().update_item(item_id, data) + + # Export the course back to github + course_location = Location(item_id)._replace(category='course', name=None) + courses = modulestore().get_items(course_location) + for course in courses: + repo_path = repo_path_from_location(course.location) + export_to_github(course, repo_path, "CMS Edit") + return HttpResponse(json.dumps({})) diff --git a/cms/djangoapps/github_sync/__init__.py b/cms/djangoapps/github_sync/__init__.py new file mode 100644 index 0000000000..c901ad37e7 --- /dev/null +++ b/cms/djangoapps/github_sync/__init__.py @@ -0,0 +1,75 @@ +import logging +import os + +from django.conf import settings +from fs.osfs import OSFS +from git import Repo, PushInfo + +from contentstore import import_from_xml +from xmodule.modulestore import Location + +from .exceptions import GithubSyncError + +log = logging.getLogger(__name__) + +def import_from_github(repo_settings): + """ + Imports data into the modulestore based on the XML stored on github + + repo_settings is a dictionary with the following keys: + path: file system path to the local git repo + branch: name of the branch to track on github + org: name of the organization to use in the imported course + course: name of the coures to use in the imported course + """ + repo_path = repo_settings['path'] + + if not os.path.isdir(repo_path): + Repo.clone_from(repo_settings['origin'], repo_path) + + git_repo = Repo(repo_path) + origin = git_repo.remotes.origin + origin.fetch() + + # Do a hard reset to the remote branch so that we have a clean import + git_repo.git.checkout(repo_settings['branch']) + git_repo.head.reset('origin/%s' % repo_settings['branch'], index=True, working_tree=True) + + return git_repo.head.commit.hexsha, import_from_xml(repo_settings['org'], repo_settings['course'], repo_path) + + +def repo_path_from_location(location): + location = Location(location) + for name, repo in settings.REPOS.items(): + if repo['org'] == location.org and repo['course'] == location.course: + return repo['path'] + + +def export_to_github(course, repo_path, commit_message): + fs = OSFS(repo_path) + xml = course.export_to_xml(fs) + + with fs.open('course.xml', 'w') as course_xml: + course_xml.write(xml) + + git_repo = Repo(repo_path) + if git_repo.is_dirty(): + git_repo.git.add(A=True) + git_repo.git.commit(m=commit_message) + + origin = git_repo.remotes.origin + if settings.MITX_FEATURES['GITHUB_PUSH']: + push_infos = origin.push() + if len(push_infos) > 1: + log.error('Unexpectedly pushed multiple heads: {infos}'.format( + infos="\n".join(str(info.summary) for info in push_infos) + )) + + if push_infos[0].flags & PushInfo.ERROR: + log.error('Failed push: flags={p.flags}, local_ref={p.local_ref}, ' + 'remote_ref_string={p.remote_ref_string}, ' + 'remote_ref={p.remote_ref}, old_commit={p.old_commit}, ' + 'summary={p.summary})'.format(p=push_infos[0])) + raise GithubSyncError('Failed to push: {info}'.format( + info=str(push_infos[0].summary) + )) diff --git a/cms/djangoapps/github_sync/exceptions.py b/cms/djangoapps/github_sync/exceptions.py new file mode 100644 index 0000000000..9097ffc2a6 --- /dev/null +++ b/cms/djangoapps/github_sync/exceptions.py @@ -0,0 +1,2 @@ +class GithubSyncError(Exception): + pass diff --git a/cms/djangoapps/github_sync/tests/__init__.py b/cms/djangoapps/github_sync/tests/__init__.py new file mode 100644 index 0000000000..825fc68313 --- /dev/null +++ b/cms/djangoapps/github_sync/tests/__init__.py @@ -0,0 +1,96 @@ +from django.test import TestCase +from path import path +import shutil +from github_sync import import_from_github, export_to_github, repo_path_from_location +from git import Repo +from django.conf import settings +from xmodule.modulestore.django import modulestore +from xmodule.modulestore import Location +from override_settings import override_settings +from github_sync.exceptions import GithubSyncError + + +class GithubSyncTestCase(TestCase): + + def setUp(self): + self.working_dir = path(settings.TEST_ROOT) + self.repo_dir = self.working_dir / 'local_repo' + self.remote_dir = self.working_dir / 'remote_repo' + shutil.copytree('common/test/data/toy', self.remote_dir) + + remote = Repo.init(self.remote_dir) + remote.git.add(A=True) + remote.git.commit(m='Initial commit') + remote.git.config("receive.denyCurrentBranch", "ignore") + + modulestore().collection.drop() + + self.import_revision, self.import_course = import_from_github({ + 'path': self.repo_dir, + 'origin': self.remote_dir, + 'branch': 'master', + 'org': 'org', + 'course': 'course' + }) + + def tearDown(self): + shutil.rmtree(self.repo_dir) + shutil.rmtree(self.remote_dir) + + def test_initialize_repo(self): + """ + Test that importing from github will create a repo if the repo doesn't already exist + """ + self.assertEquals(1, len(Repo(self.repo_dir).head.reference.log())) + + def test_import_contents(self): + """ + Test that the import loads the correct course into the modulestore + """ + self.assertEquals('Toy Course', self.import_course.metadata['display_name']) + self.assertIn( + Location('i4x://org/course/chapter/Overview'), + [child.location for child in self.import_course.get_children()]) + self.assertEquals(1, len(self.import_course.get_children())) + + @override_settings(MITX_FEATURES={'GITHUB_PUSH': False}) + def test_export_no_pash(self): + """ + Test that with the GITHUB_PUSH feature disabled, no content is pushed to the remote + """ + export_to_github(self.import_course, self.repo_dir, 'Test no-push') + self.assertEquals(1, Repo(self.remote_dir).head.commit.count()) + + @override_settings(MITX_FEATURES={'GITHUB_PUSH': True}) + def test_export_push(self): + """ + Test that with GITHUB_PUSH enabled, content is pushed to the remote + """ + self.import_course.metadata['display_name'] = 'Changed display name' + export_to_github(self.import_course, self.repo_dir, 'Test push') + self.assertEquals(2, Repo(self.remote_dir).head.commit.count()) + + @override_settings(MITX_FEATURES={'GITHUB_PUSH': True}) + def test_export_conflict(self): + """ + Test that if there is a conflict when pushing to the remote repo, nothing is pushed and an exception is raised + """ + self.import_course.metadata['display_name'] = 'Changed display name' + + remote = Repo(self.remote_dir) + remote.git.commit(allow_empty=True, m="Testing conflict commit") + + self.assertRaises(GithubSyncError, export_to_github, self.import_course, self.repo_dir, 'Test push') + self.assertEquals(2, remote.head.reference.commit.count()) + self.assertEquals("Testing conflict commit\n", remote.head.reference.commit.message) + + +@override_settings(REPOS={'namea': {'path': 'patha', 'org': 'orga', 'course': 'coursea'}, + 'nameb': {'path': 'pathb', 'org': 'orgb', 'course': 'courseb'}}) +class RepoPathLookupTestCase(TestCase): + def test_successful_lookup(self): + self.assertEquals('patha', repo_path_from_location('i4x://orga/coursea/course/foo')) + self.assertEquals('pathb', repo_path_from_location('i4x://orgb/courseb/course/foo')) + + def test_failed_lookup(self): + self.assertEquals(None, repo_path_from_location('i4x://c/c/course/foo')) diff --git a/cms/djangoapps/github_sync/tests/test_views.py b/cms/djangoapps/github_sync/tests/test_views.py new file mode 100644 index 0000000000..9e8095a67b --- /dev/null +++ b/cms/djangoapps/github_sync/tests/test_views.py @@ -0,0 +1,53 @@ +import json +from django.test.client import Client +from django.test import TestCase +from mock import patch, Mock +from override_settings import override_settings +from django.conf import settings + + +@override_settings(REPOS={'repo': {'path': 'path', 'branch': 'branch'}}) +class PostReceiveTestCase(TestCase): + def setUp(self): + self.client = Client() + + @patch('github_sync.views.export_to_github') + @patch('github_sync.views.import_from_github') + def test_non_branch(self, import_from_github, export_to_github): + self.client.post('/github_service_hook', {'payload': json.dumps({ + 'ref': 'refs/tags/foo'}) + }) + self.assertFalse(import_from_github.called) + self.assertFalse(export_to_github.called) + + @patch('github_sync.views.export_to_github') + @patch('github_sync.views.import_from_github') + def test_non_watched_repo(self, import_from_github, export_to_github): + self.client.post('/github_service_hook', {'payload': json.dumps({ + 'ref': 'refs/heads/branch', + 'repository': {'name': 'bad_repo'}}) + }) + self.assertFalse(import_from_github.called) + self.assertFalse(export_to_github.called) + + @patch('github_sync.views.export_to_github') + @patch('github_sync.views.import_from_github') + def test_non_tracked_branch(self, import_from_github, export_to_github): + self.client.post('/github_service_hook', {'payload': json.dumps({ + 'ref': 'refs/heads/non_branch', + 'repository': {'name': 'repo'}}) + }) + self.assertFalse(import_from_github.called) + self.assertFalse(export_to_github.called) + + @patch('github_sync.views.export_to_github') + @patch('github_sync.views.import_from_github', return_value=(Mock(), Mock())) + def test_tracked_branch(self, import_from_github, export_to_github): + self.client.post('/github_service_hook', {'payload': json.dumps({ + 'ref': 'refs/heads/branch', + 'repository': {'name': 'repo'}}) + }) + import_from_github.assert_called_with(settings.REPOS['repo']) + mock_revision, mock_course = import_from_github.return_value + export_to_github.assert_called_with(mock_course, 'path', "Changes from cms import of revision %s" % mock_revision) + diff --git a/cms/djangoapps/github_sync/views.py b/cms/djangoapps/github_sync/views.py new file mode 100644 index 0000000000..8bf654f430 --- /dev/null +++ b/cms/djangoapps/github_sync/views.py @@ -0,0 +1,52 @@ +import logging +import json + +from django.http import HttpResponse +from django.conf import settings +from django_future.csrf import csrf_exempt + +from . import import_from_github, export_to_github + +log = logging.getLogger() + + +@csrf_exempt +def github_post_receive(request): + """ + This view recieves post-receive requests from github whenever one of + the watched repositiories changes. + + It is responsible for updating the relevant local git repo, + importing the new version of the course (if anything changed), + and then pushing back to github any changes that happened as part of the + import. + + The github request format is described here: https://help.github.com/articles/post-receive-hooks + """ + + payload = json.loads(request.POST['payload']) + + ref = payload['ref'] + + if not ref.startswith('refs/heads/'): + log.info('Ignore changes to non-branch ref %s' % ref) + return HttpResponse('Ignoring non-branch') + + branch_name = ref.replace('refs/heads/', '', 1) + + repo_name = payload['repository']['name'] + + if repo_name not in settings.REPOS: + log.info('No repository matching %s found' % repo_name) + return HttpResponse('No Repo Found') + + repo = settings.REPOS[repo_name] + + if repo['branch'] != branch_name: + log.info('Ignoring changes to non-tracked branch %s in repo %s' % (branch_name, repo_name)) + return HttpResponse('Ignoring non-tracked branch') + + revision, course = import_from_github(repo) + export_to_github(course, repo['path'], "Changes from cms import of revision %s" % revision) + + return HttpResponse('Push recieved') diff --git a/cms/envs/common.py b/cms/envs/common.py index 6edd6aa2b9..400a5138b9 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -30,6 +30,7 @@ from path import path MITX_FEATURES = { 'USE_DJANGO_PIPELINE': True, + 'GITHUB_PUSH': False, } ############################# SET PATH INFORMATION ############################# diff --git a/cms/envs/dev.py b/cms/envs/dev.py index b4bcbfa9ce..ca48481184 100644 --- a/cms/envs/dev.py +++ b/cms/envs/dev.py @@ -29,8 +29,48 @@ DATABASES = { } } +REPO_ROOT = ENV_ROOT / "content" + +REPOS = { + 'edx4edx': { + 'path': REPO_ROOT / "edx4edx", + 'org': 'edx', + 'course': 'edx4edx', + 'branch': 'for_cms', + 'origin': 'git@github.com:MITx/edx4edx.git', + }, + '6002x-fall-2012': { + 'path': REPO_ROOT / '6002x-fall-2012', + 'org': 'mit.edu', + 'course': '6.002x', + 'branch': 'for_cms', + 'origin': 'git@github.com:MITx/6002x-fall-2012.git', + }, + '6.00x': { + 'path': REPO_ROOT / '6.00x', + 'org': 'mit.edu', + 'course': '6.00x', + 'branch': 'for_cms', + 'origin': 'git@github.com:MITx/6.00x.git', + }, + '7.00x': { + 'path': REPO_ROOT / '7.00x', + 'org': 'mit.edu', + 'course': '7.00x', + 'branch': 'for_cms', + 'origin': 'git@github.com:MITx/7.00x.git', + }, + '3.091x': { + 'path': REPO_ROOT / '3.091x', + 'org': 'mit.edu', + 'course': '3.091x', + 'branch': 'for_cms', + 'origin': 'git@github.com:MITx/3.091x.git', + }, +} + CACHES = { - # This is the cache used for most things. Askbot will not work without a + # This is the cache used for most things. Askbot will not work without a # functioning cache -- it relies on caching to load its settings in places. # In staging/prod envs, the sessions also live here. 'default': { diff --git a/cms/envs/test.py b/cms/envs/test.py index 032de92953..927e2af987 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -17,10 +17,18 @@ for app in os.listdir(PROJECT_ROOT / 'djangoapps'): NOSE_ARGS += ['--cover-package', app] TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' +TEST_ROOT = 'test_root' + MODULESTORE = { - 'host': 'localhost', - 'db': 'mongo_base', - 'collection': 'key_store', + 'default': { + 'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore', + 'OPTIONS': { + 'default_class': 'xmodule.raw_module.RawDescriptor', + 'host': 'localhost', + 'db': 'test_xmodule', + 'collection': 'modulestore', + } + } } DATABASES = { diff --git a/cms/static/coffee/unit.coffee b/cms/static/coffee/unit.coffee index b81bc0df08..dda41fe42b 100644 --- a/cms/static/coffee/unit.coffee +++ b/cms/static/coffee/unit.coffee @@ -4,7 +4,7 @@ class @Unit $("##{@element_id} .save-update").click (event) => event.preventDefault() - $.post("save_item", { + $.post("/save_item", { id: @module_id data: JSON.stringify(@module.save()) }) diff --git a/cms/templates/course_index.html b/cms/templates/course_index.html new file mode 100644 index 0000000000..6bc04dc8a2 --- /dev/null +++ b/cms/templates/course_index.html @@ -0,0 +1,16 @@ +<%inherit file="base.html" /> +<%block name="title">Course Manager + +<%block name="content"> +
+ + <%include file="widgets/navigation.html"/> + +
+
+
+
+
+ +
+ diff --git a/cms/templates/index.html b/cms/templates/index.html index 6bc04dc8a2..2998cb8bd6 100644 --- a/cms/templates/index.html +++ b/cms/templates/index.html @@ -1,16 +1,12 @@ <%inherit file="base.html" /> -<%block name="title">Course Manager +<%block name="title">Courses <%block name="content">
- - <%include file="widgets/navigation.html"/> - -
-
-
-
-
- +
    + %for course, url in courses: +
  1. ${course}
  2. + %endfor +
diff --git a/cms/urls.py b/cms/urls.py index 9d827c3fe3..fe3c9216e6 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -8,5 +8,7 @@ urlpatterns = patterns('', url(r'^$', 'contentstore.views.index', name='index'), url(r'^edit_item$', 'contentstore.views.edit_item', name='edit_item'), url(r'^save_item$', 'contentstore.views.save_item', name='save_item'), - url(r'^temp_force_export$', 'contentstore.views.temp_force_export') + url(r'^(?P[^/]+)/(?P[^/]+)/course/(?P[^/]+)$', 'contentstore.views.course_index', name='course_index'), + url(r'^temp_force_export$', 'contentstore.views.temp_force_export'), + url(r'^github_service_hook$', 'github_sync.views.github_post_receive'), ) diff --git a/common/lib/capa/__init__.py b/common/lib/capa/capa/__init__.py similarity index 100% rename from common/lib/capa/__init__.py rename to common/lib/capa/capa/__init__.py diff --git a/common/lib/capa/calc.py b/common/lib/capa/capa/calc.py similarity index 100% rename from common/lib/capa/calc.py rename to common/lib/capa/capa/calc.py diff --git a/common/lib/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py similarity index 100% rename from common/lib/capa/capa_problem.py rename to common/lib/capa/capa/capa_problem.py diff --git a/common/lib/capa/checker.py b/common/lib/capa/capa/checker.py similarity index 100% rename from common/lib/capa/checker.py rename to common/lib/capa/capa/checker.py diff --git a/common/lib/capa/correctmap.py b/common/lib/capa/capa/correctmap.py similarity index 100% rename from common/lib/capa/correctmap.py rename to common/lib/capa/capa/correctmap.py diff --git a/common/lib/capa/eia.py b/common/lib/capa/capa/eia.py similarity index 100% rename from common/lib/capa/eia.py rename to common/lib/capa/capa/eia.py diff --git a/common/lib/capa/inputtypes.py b/common/lib/capa/capa/inputtypes.py similarity index 100% rename from common/lib/capa/inputtypes.py rename to common/lib/capa/capa/inputtypes.py diff --git a/common/lib/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py similarity index 100% rename from common/lib/capa/responsetypes.py rename to common/lib/capa/capa/responsetypes.py diff --git a/common/lib/capa/templates/choicegroup.html b/common/lib/capa/capa/templates/choicegroup.html similarity index 100% rename from common/lib/capa/templates/choicegroup.html rename to common/lib/capa/capa/templates/choicegroup.html diff --git a/common/lib/capa/templates/imageinput.html b/common/lib/capa/capa/templates/imageinput.html similarity index 100% rename from common/lib/capa/templates/imageinput.html rename to common/lib/capa/capa/templates/imageinput.html diff --git a/common/lib/capa/templates/jstextline.html b/common/lib/capa/capa/templates/jstextline.html similarity index 100% rename from common/lib/capa/templates/jstextline.html rename to common/lib/capa/capa/templates/jstextline.html diff --git a/common/lib/capa/templates/mathstring.html b/common/lib/capa/capa/templates/mathstring.html similarity index 100% rename from common/lib/capa/templates/mathstring.html rename to common/lib/capa/capa/templates/mathstring.html diff --git a/common/lib/capa/templates/schematicinput.html b/common/lib/capa/capa/templates/schematicinput.html similarity index 100% rename from common/lib/capa/templates/schematicinput.html rename to common/lib/capa/capa/templates/schematicinput.html diff --git a/common/lib/capa/templates/solutionspan.html b/common/lib/capa/capa/templates/solutionspan.html similarity index 100% rename from common/lib/capa/templates/solutionspan.html rename to common/lib/capa/capa/templates/solutionspan.html diff --git a/common/lib/capa/templates/textbox.html b/common/lib/capa/capa/templates/textbox.html similarity index 100% rename from common/lib/capa/templates/textbox.html rename to common/lib/capa/capa/templates/textbox.html diff --git a/common/lib/capa/templates/textinput.html b/common/lib/capa/capa/templates/textinput.html similarity index 100% rename from common/lib/capa/templates/textinput.html rename to common/lib/capa/capa/templates/textinput.html diff --git a/common/lib/capa/templates/textinput_dynamath.html b/common/lib/capa/capa/templates/textinput_dynamath.html similarity index 100% rename from common/lib/capa/templates/textinput_dynamath.html rename to common/lib/capa/capa/templates/textinput_dynamath.html diff --git a/common/lib/capa/util.py b/common/lib/capa/capa/util.py similarity index 100% rename from common/lib/capa/util.py rename to common/lib/capa/capa/util.py diff --git a/common/lib/capa/setup.py b/common/lib/capa/setup.py new file mode 100644 index 0000000000..cf66229b88 --- /dev/null +++ b/common/lib/capa/setup.py @@ -0,0 +1,8 @@ +from setuptools import setup, find_packages + +setup( + name="capa", + version="0.1", + packages=find_packages(exclude=["tests"]), + install_requires=['distribute'], +) diff --git a/common/lib/mitxmako/__init__.py b/common/lib/mitxmako/mitxmako/__init__.py similarity index 100% rename from common/lib/mitxmako/__init__.py rename to common/lib/mitxmako/mitxmako/__init__.py diff --git a/common/lib/mitxmako/middleware.py b/common/lib/mitxmako/mitxmako/middleware.py similarity index 100% rename from common/lib/mitxmako/middleware.py rename to common/lib/mitxmako/mitxmako/middleware.py diff --git a/common/lib/mitxmako/shortcuts.py b/common/lib/mitxmako/mitxmako/shortcuts.py similarity index 100% rename from common/lib/mitxmako/shortcuts.py rename to common/lib/mitxmako/mitxmako/shortcuts.py diff --git a/common/lib/mitxmako/template.py b/common/lib/mitxmako/mitxmako/template.py similarity index 100% rename from common/lib/mitxmako/template.py rename to common/lib/mitxmako/mitxmako/template.py diff --git a/common/lib/mitxmako/setup.py b/common/lib/mitxmako/setup.py new file mode 100644 index 0000000000..535d86f90e --- /dev/null +++ b/common/lib/mitxmako/setup.py @@ -0,0 +1,8 @@ +from setuptools import setup, find_packages + +setup( + name="mitxmako", + version="0.1", + packages=find_packages(exclude=["tests"]), + install_requires=['distribute'], +) diff --git a/common/lib/xmodule/setup.py b/common/lib/xmodule/setup.py index 77b0838ff2..1ce23bca90 100644 --- a/common/lib/xmodule/setup.py +++ b/common/lib/xmodule/setup.py @@ -8,6 +8,10 @@ setup( package_data={ 'xmodule': ['js/module/*'] }, + requires=[ + 'capa', + 'mitxmako' + ], # See http://guide.python-distribute.org/creation.html#entry-points # for a description of entry_points diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index 08fe4bbecc..337a833dc3 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -1,4 +1,5 @@ import logging +from lxml import etree from xmodule.x_module import XModule from xmodule.raw_module import RawDescriptor @@ -26,3 +27,8 @@ class HtmlDescriptor(RawDescriptor): js = {'coffee': [resource_string(__name__, 'js/module/html.coffee')]} js_module = 'HTML' + + @classmethod + def file_to_xml(cls, file_object): + parser = etree.HTMLParser() + return etree.parse(file_object, parser).getroot() diff --git a/common/lib/xmodule/xmodule/js/module/html.coffee b/common/lib/xmodule/xmodule/js/module/html.coffee index 5e072c27a3..699b0994bd 100644 --- a/common/lib/xmodule/xmodule/js/module/html.coffee +++ b/common/lib/xmodule/xmodule/js/module/html.coffee @@ -6,4 +6,4 @@ class @HTML @preview.empty().append(@edit_box.val()) ) - save: -> {text: @edit_box.val()} + save: -> @edit_box.val() diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py index b8ab95d85e..4d628a6841 100644 --- a/common/lib/xmodule/xmodule/modulestore/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/__init__.py @@ -159,6 +159,18 @@ class ModuleStore(object): location is found """ raise NotImplementedError + + def get_items(self, location, default_class=None): + """ + Returns a list of XModuleDescriptor instances for the items + that match location. Any element of location that is None is treated + as a wildcard that matches any value + + location: Something that can be passed to Location + default_class: An XModuleDescriptor subclass to use if no plugin matching the + location is found + """ + raise NotImplementedError # TODO (cpennington): Replace with clone_item def create_item(self, location, editor): diff --git a/common/lib/xmodule/xmodule/modulestore/mongo.py b/common/lib/xmodule/xmodule/modulestore/mongo.py index cc731c929c..ee77e59b26 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo.py @@ -1,4 +1,5 @@ import pymongo +from bson.objectid import ObjectId from importlib import import_module from xmodule.x_module import XModuleDescriptor from xmodule.mako_module import MakoDescriptorSystem @@ -8,6 +9,19 @@ from . import ModuleStore, Location from .exceptions import ItemNotFoundError, InsufficientSpecificationError +# TODO (cpennington): This code currently operates under the assumption that +# there is only one revision for each item. Once we start versioning inside the CMS, +# that assumption will have to change + + +def location_to_query(loc): + query = {} + for key, val in Location(loc).dict().iteritems(): + if val is not None: + query['_id.{key}'.format(key=key)] = val + + return query + class MongoModuleStore(ModuleStore): """ A Mongodb backed ModuleStore @@ -17,7 +31,6 @@ class MongoModuleStore(ModuleStore): host=host, port=port )[db][collection] - self.collection.ensure_index('location') # Force mongo to report errors, at the expense of performance self.collection.safe = True @@ -26,6 +39,18 @@ class MongoModuleStore(ModuleStore): class_ = getattr(import_module(module_path), class_name) self.default_class = class_ + # TODO (cpennington): Pass a proper resources_fs to the system + self.system = MakoDescriptorSystem( + load_item=self.get_item, + resources_fs=None, + render_template=render_to_string + ) + + def _load_item(self, item): + item['location'] = item['_id'] + del item['_id'] + return XModuleDescriptor.load_from_json(item, self.system, self.default_class) + def get_item(self, location): """ Returns an XModuleDescriptor instance for the item at location. @@ -39,24 +64,26 @@ class MongoModuleStore(ModuleStore): location: Something that can be passed to Location """ - query = {} for key, val in Location(location).dict().iteritems(): if key != 'revision' and val is None: raise InsufficientSpecificationError(location) - if val is not None: - query['location.{key}'.format(key=key)] = val - item = self.collection.find_one( - query, + location_to_query(location), sort=[('revision', pymongo.ASCENDING)], ) if item is None: raise ItemNotFoundError(location) + return self._load_item(item) - # TODO (cpennington): Pass a proper resources_fs to the system - return XModuleDescriptor.load_from_json( - item, MakoDescriptorSystem(load_item=self.get_item, resources_fs=None, render_template=render_to_string), self.default_class) + def get_items(self, location, default_class=None): + print location_to_query(location) + items = self.collection.find( + location_to_query(location), + sort=[('revision', pymongo.ASCENDING)], + ) + + return [self._load_item(item) for item in items] def create_item(self, location): """ @@ -65,7 +92,7 @@ class MongoModuleStore(ModuleStore): location: Something that can be passed to Location """ self.collection.insert({ - 'location': Location(location).dict(), + '_id': Location(location).dict(), }) def update_item(self, location, data): @@ -80,8 +107,9 @@ class MongoModuleStore(ModuleStore): # See http://www.mongodb.org/display/DOCS/Updating for # atomic update syntax self.collection.update( - {'location': Location(location).dict()}, - {'$set': {'definition.data': data}} + {'_id': Location(location).dict()}, + {'$set': {'definition.data': data}}, + ) def update_children(self, location, children): @@ -96,7 +124,7 @@ class MongoModuleStore(ModuleStore): # See http://www.mongodb.org/display/DOCS/Updating for # atomic update syntax self.collection.update( - {'location': Location(location).dict()}, + {'_id': Location(location).dict()}, {'$set': {'definition.children': children}} ) @@ -112,6 +140,6 @@ class MongoModuleStore(ModuleStore): # See http://www.mongodb.org/display/DOCS/Updating for # atomic update syntax self.collection.update( - {'location': Location(location).dict()}, + {'_id': Location(location).dict()}, {'$set': {'metadata': metadata}} ) diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index 0b0b1225b1..140dd8d162 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -90,6 +90,16 @@ class XmlDescriptor(XModuleDescriptor): if xml_object.get(attr) is not None: del xml_object.attrib[attr] + @classmethod + def file_to_xml(cls, file_object): + """ + Used when this module wants to parse a file object to xml + that will be converted to the definition. + + Returns an lxml Element + """ + return etree.parse(file_object).getroot() + @classmethod def from_xml(cls, xml_data, system, org=None, course=None): """ @@ -128,7 +138,7 @@ class XmlDescriptor(XModuleDescriptor): log.debug('filepath=%s, resources_fs=%s' % (filepath,system.resources_fs)) with system.resources_fs.open(filepath) as file: try: - definition_xml = etree.parse(file).getroot() + definition_xml = cls.file_to_xml(file) except: log.exception("Failed to parse xml in file %s" % filepath) raise @@ -149,7 +159,7 @@ class XmlDescriptor(XModuleDescriptor): @classmethod def _format_filepath(cls, category, name): - return '{category}/{name}.{ext}'.format(category=category, name=name, ext=cls.filename_extension) + return u'{category}/{name}.{ext}'.format(category=category, name=name, ext=cls.filename_extension) def export_to_xml(self, resource_fs): """ diff --git a/common/test/data/toy/toy_course.xml b/common/test/data/toy/course.xml similarity index 100% rename from common/test/data/toy/toy_course.xml rename to common/test/data/toy/course.xml diff --git a/lms/envs/test.py b/lms/envs/test.py index 941b855f5c..aed49d43de 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -20,7 +20,8 @@ INSTALLED_APPS = [ # Nose Test Runner INSTALLED_APPS += ['django_nose'] -NOSE_ARGS = ['--cover-erase', '--with-xunit', '--with-xcoverage', '--cover-html', '--cover-inclusive', '--cover-html-dir', os.environ['NOSE_COVER_HTML_DIR']] +NOSE_ARGS = ['--cover-erase', '--with-xunit', '--with-xcoverage', '--cover-html', + '--cover-inclusive', '--cover-html-dir', os.environ.get('NOSE_COVER_HTML_DIR', 'cover_html')] for app in os.listdir(PROJECT_ROOT / 'djangoapps'): NOSE_ARGS += ['--cover-package', app] TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' diff --git a/requirements.txt b/requirements.txt index a72f72a7da..efc8e48ddd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ django<1.4 pip +numpy scipy matplotlib markdown @@ -24,7 +25,12 @@ sympy newrelic glob2 pymongo +-e common/lib/capa +-e common/lib/mitxmako -e common/lib/xmodule django_nose nosexcover rednose +GitPython >= 0.3 +django-override-settings +mock>=0.8, <0.9 diff --git a/test_root/.gitignore b/test_root/.gitignore new file mode 100644 index 0000000000..b3e5512f73 --- /dev/null +++ b/test_root/.gitignore @@ -0,0 +1,2 @@ +local_repo +remote_repo