From 67a732a0ffceaae0c26a2db591fc0ad5e1107a12 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Tue, 3 Jul 2012 13:18:01 -0400 Subject: [PATCH 01/18] Allow the HTML module to use the lxml HTML parser when parsing html file includes --- common/lib/xmodule/xmodule/html_module.py | 6 ++++++ common/lib/xmodule/xmodule/xml_module.py | 12 +++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index 08fe4bbecc..337a833dc3 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -1,4 +1,5 @@ import logging +from lxml import etree from xmodule.x_module import XModule from xmodule.raw_module import RawDescriptor @@ -26,3 +27,8 @@ class HtmlDescriptor(RawDescriptor): js = {'coffee': [resource_string(__name__, 'js/module/html.coffee')]} js_module = 'HTML' + + @classmethod + def file_to_xml(cls, file_object): + parser = etree.HTMLParser() + return etree.parse(file_object, parser).getroot() diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index aebb024a59..5699f962cf 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -90,6 +90,16 @@ class XmlDescriptor(XModuleDescriptor): if xml_object.get(attr) is not None: del xml_object.attrib[attr] + @classmethod + def file_to_xml(cls, file_object): + """ + Used when this module wants to parse a file object to xml + that will be converted to the definition. + + Returns an lxml Element + """ + return etree.parse(file_object).getroot() + @classmethod def from_xml(cls, xml_data, system, org=None, course=None): """ @@ -127,7 +137,7 @@ class XmlDescriptor(XModuleDescriptor): filepath = cls._format_filepath(xml_object.tag, filename) with system.resources_fs.open(filepath) as file: try: - definition_xml = etree.parse(file).getroot() + definition_xml = cls.file_to_xml(file) except: log.exception("Failed to parse xml in file %s" % filepath) raise From 4e599d4ab2fb08c4c120f2849cd1fa562e46b9e1 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Tue, 3 Jul 2012 13:42:35 -0400 Subject: [PATCH 02/18] Extract import from xml into a separate function --- cms/djangoapps/contentstore/__init__.py | 19 +++++++++++++++++++ .../management/commands/import.py | 13 ++----------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/cms/djangoapps/contentstore/__init__.py b/cms/djangoapps/contentstore/__init__.py index e69de29bb2..7a38b99b5e 100644 --- a/cms/djangoapps/contentstore/__init__.py +++ b/cms/djangoapps/contentstore/__init__.py @@ -0,0 +1,19 @@ +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.xml import XMLModuleStore + + +def import_from_xml(org, course, data_dir): + """ + Import the specified xml data_dir into the django defined modulestore, + using org and course as the location org and course. + """ + module_store = XMLModuleStore(org, course, data_dir, 'xmodule.raw_module.RawDescriptor', eager=True) + for module in module_store.modules.itervalues(): + modulestore().create_item(module.location) + if 'data' in module.definition: + modulestore().update_item(module.location, module.definition['data']) + if 'children' in module.definition: + modulestore().update_children(module.location, module.definition['children']) + modulestore().update_metadata(module.location, dict(module.metadata)) + + return module_store.course diff --git a/cms/djangoapps/contentstore/management/commands/import.py b/cms/djangoapps/contentstore/management/commands/import.py index f97ac10d41..89323c3d9b 100644 --- a/cms/djangoapps/contentstore/management/commands/import.py +++ b/cms/djangoapps/contentstore/management/commands/import.py @@ -3,8 +3,7 @@ ### from django.core.management.base import BaseCommand, CommandError -from xmodule.modulestore.django import modulestore -from xmodule.modulestore.xml import XMLModuleStore +from contentstore import import_from_xml unnamed_modules = 0 @@ -18,12 +17,4 @@ class Command(BaseCommand): raise CommandError("import requires 3 arguments: ") org, course, data_dir = args - - module_store = XMLModuleStore(org, course, data_dir, 'xmodule.raw_module.RawDescriptor', eager=True) - for module in module_store.modules.itervalues(): - modulestore().create_item(module.location) - if 'data' in module.definition: - modulestore().update_item(module.location, module.definition['data']) - if 'children' in module.definition: - modulestore().update_children(module.location, module.definition['children']) - modulestore().update_metadata(module.location, dict(module.metadata)) + import_from_xml(org, course, data_dir) From 10add5666055160a3e86049eb2a363144aaf8deb Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Tue, 3 Jul 2012 13:43:56 -0400 Subject: [PATCH 03/18] Add url to point git service hooks at in order to trigger an import from github --- cms/djangoapps/contentstore/views.py | 10 +++-- cms/djangoapps/github_sync/__init__.py | 40 ++++++++++++++++++++ cms/djangoapps/github_sync/views.py | 52 ++++++++++++++++++++++++++ cms/envs/dev.py | 13 ++++++- cms/urls.py | 3 +- requirements.txt | 1 + 6 files changed, 113 insertions(+), 6 deletions(-) create mode 100644 cms/djangoapps/github_sync/__init__.py create mode 100644 cms/djangoapps/github_sync/views.py diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index 76a904a403..826b2a0028 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -1,11 +1,13 @@ -from mitxmako.shortcuts import render_to_response -from xmodule.modulestore.django import modulestore -from django_future.csrf import ensure_csrf_cookie -from django.http import HttpResponse import json +from django.http import HttpResponse +from django_future.csrf import ensure_csrf_cookie from fs.osfs import OSFS +from mitxmako.shortcuts import render_to_response +from xmodule.modulestore.django import modulestore + + @ensure_csrf_cookie def index(request): # TODO (cpennington): These need to be read in from the active user diff --git a/cms/djangoapps/github_sync/__init__.py b/cms/djangoapps/github_sync/__init__.py new file mode 100644 index 0000000000..c6bbca222f --- /dev/null +++ b/cms/djangoapps/github_sync/__init__.py @@ -0,0 +1,40 @@ +from git import Repo +from contentstore import import_from_xml +from fs.osfs import OSFS + + +def import_from_github(repo_settings): + """ + Imports data into the modulestore based on the XML stored on github + + repo_settings is a dictionary with the following keys: + path: file system path to the local git repo + branch: name of the branch to track on github + org: name of the + """ + repo_path = repo_settings['path'] + git_repo = Repo(repo_path) + origin = git_repo.remotes.origin + origin.fetch() + + # Do a hard reset to the remote branch so that we have a clean import + git_repo.heads[repo_settings['branch']].checkout() + git_repo.head.reset('origin/%s' % repo_settings['branch'], index=True, working_tree=True) + + return git_repo.head.commit.hexsha, import_from_xml(repo_settings['org'], repo_settings['course'], repo_path) + + +def export_to_github(course, repo_path, commit_message): + fs = OSFS(repo_path) + xml = course.export_to_xml(fs) + + with fs.open('course.xml', 'w') as course_xml: + course_xml.write(xml) + + git_repo = Repo(repo_path) + if git_repo.is_dirty(): + git_repo.git.add(A=True) + git_repo.git.commit(m=commit_message) + + origin = git_repo.remotes.origin + origin.push() diff --git a/cms/djangoapps/github_sync/views.py b/cms/djangoapps/github_sync/views.py new file mode 100644 index 0000000000..8bf654f430 --- /dev/null +++ b/cms/djangoapps/github_sync/views.py @@ -0,0 +1,52 @@ +import logging +import json + +from django.http import HttpResponse +from django.conf import settings +from django_future.csrf import csrf_exempt + +from . import import_from_github, export_to_github + +log = logging.getLogger() + + +@csrf_exempt +def github_post_receive(request): + """ + This view recieves post-receive requests from github whenever one of + the watched repositiories changes. + + It is responsible for updating the relevant local git repo, + importing the new version of the course (if anything changed), + and then pushing back to github any changes that happened as part of the + import. + + The github request format is described here: https://help.github.com/articles/post-receive-hooks + """ + + payload = json.loads(request.POST['payload']) + + ref = payload['ref'] + + if not ref.startswith('refs/heads/'): + log.info('Ignore changes to non-branch ref %s' % ref) + return HttpResponse('Ignoring non-branch') + + branch_name = ref.replace('refs/heads/', '', 1) + + repo_name = payload['repository']['name'] + + if repo_name not in settings.REPOS: + log.info('No repository matching %s found' % repo_name) + return HttpResponse('No Repo Found') + + repo = settings.REPOS[repo_name] + + if repo['branch'] != branch_name: + log.info('Ignoring changes to non-tracked branch %s in repo %s' % (branch_name, repo_name)) + return HttpResponse('Ignoring non-tracked branch') + + revision, course = import_from_github(repo) + export_to_github(course, repo['path'], "Changes from cms import of revision %s" % revision) + + return HttpResponse('Push recieved') diff --git a/cms/envs/dev.py b/cms/envs/dev.py index b4bcbfa9ce..8ff2a35c52 100644 --- a/cms/envs/dev.py +++ b/cms/envs/dev.py @@ -29,8 +29,19 @@ DATABASES = { } } +REPO_ROOT = ENV_ROOT / "content" + +REPOS = { + 'edx4edx': { + 'path': REPO_ROOT / "edx4edx", + 'org': 'edx', + 'course': 'edx4edx', + 'branch': 'for_cms' + } +} + CACHES = { - # This is the cache used for most things. Askbot will not work without a + # This is the cache used for most things. Askbot will not work without a # functioning cache -- it relies on caching to load its settings in places. # In staging/prod envs, the sessions also live here. 'default': { diff --git a/cms/urls.py b/cms/urls.py index 9d827c3fe3..ad5a69b824 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -8,5 +8,6 @@ urlpatterns = patterns('', url(r'^$', 'contentstore.views.index', name='index'), url(r'^edit_item$', 'contentstore.views.edit_item', name='edit_item'), url(r'^save_item$', 'contentstore.views.save_item', name='save_item'), - url(r'^temp_force_export$', 'contentstore.views.temp_force_export') + url(r'^temp_force_export$', 'contentstore.views.temp_force_export'), + url(r'^github_service_hook$', 'github_sync.views.github_post_receive'), ) diff --git a/requirements.txt b/requirements.txt index a72f72a7da..ddf218af7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,3 +28,4 @@ pymongo django_nose nosexcover rednose +GitPython >= 0.3 From a5d861d298e467b1e34508c86f24a7fa6f9d1570 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Tue, 3 Jul 2012 14:29:50 -0400 Subject: [PATCH 04/18] Make course index page read from url for org, course, name --- cms/djangoapps/contentstore/views.py | 7 ++----- cms/templates/{index.html => course_index.html} | 0 cms/urls.py | 1 + 3 files changed, 3 insertions(+), 5 deletions(-) rename cms/templates/{index.html => course_index.html} (100%) diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index 826b2a0028..2d5277fcc4 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -9,14 +9,11 @@ from xmodule.modulestore.django import modulestore @ensure_csrf_cookie -def index(request): +def course_index(request, org, course, name): # TODO (cpennington): These need to be read in from the active user - org = 'mit.edu' - course = '6002xs12' - name = '6.002_Spring_2012' course = modulestore().get_item(['i4x', org, course, 'course', name]) weeks = course.get_children() - return render_to_response('index.html', {'weeks': weeks}) + return render_to_response('course_index.html', {'weeks': weeks}) def edit_item(request): diff --git a/cms/templates/index.html b/cms/templates/course_index.html similarity index 100% rename from cms/templates/index.html rename to cms/templates/course_index.html diff --git a/cms/urls.py b/cms/urls.py index ad5a69b824..3a953261a1 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -6,6 +6,7 @@ from django.conf.urls.defaults import patterns, url urlpatterns = patterns('', url(r'^$', 'contentstore.views.index', name='index'), + url(r'^(?P[^/]+)/(?P[^/]+)/course/(?P[^/]+)$', 'contentstore.views.course_index', name='course_index'), url(r'^edit_item$', 'contentstore.views.edit_item', name='edit_item'), url(r'^save_item$', 'contentstore.views.save_item', name='save_item'), url(r'^temp_force_export$', 'contentstore.views.temp_force_export'), From 5a2b2e0126a1a739f3493f58457a9ac5300af228 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Tue, 3 Jul 2012 14:44:41 -0400 Subject: [PATCH 05/18] Add an index page that lists all courses --- cms/djangoapps/contentstore/views.py | 15 ++++++++++++ cms/templates/index.html | 12 ++++++++++ .../xmodule/xmodule/modulestore/__init__.py | 12 ++++++++++ .../lib/xmodule/xmodule/modulestore/mongo.py | 23 +++++++++++++++++++ 4 files changed, 62 insertions(+) create mode 100644 cms/templates/index.html diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index 2d5277fcc4..c4825a4ddf 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -3,11 +3,26 @@ import json from django.http import HttpResponse from django_future.csrf import ensure_csrf_cookie from fs.osfs import OSFS +from django.core.urlresolvers import reverse from mitxmako.shortcuts import render_to_response from xmodule.modulestore.django import modulestore +@ensure_csrf_cookie +def index(request): + courses = modulestore().get_items(['i4x', None, None, 'course', None]) + print courses + return render_to_response('index.html', { + 'courses': [(course.metadata['display_name'], + reverse('course_index', args=[ + course.location.org, + course.location.course, + course.location.name])) + for course in courses] + }) + + @ensure_csrf_cookie def course_index(request, org, course, name): # TODO (cpennington): These need to be read in from the active user diff --git a/cms/templates/index.html b/cms/templates/index.html new file mode 100644 index 0000000000..2998cb8bd6 --- /dev/null +++ b/cms/templates/index.html @@ -0,0 +1,12 @@ +<%inherit file="base.html" /> +<%block name="title">Courses + +<%block name="content"> +
+
    + %for course, url in courses: +
  1. ${course}
    2. + %endfor +
+
+ diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py index 00b3b13bb0..f9bc675116 100644 --- a/common/lib/xmodule/xmodule/modulestore/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/__init__.py @@ -153,6 +153,18 @@ class ModuleStore(object): location is found """ raise NotImplementedError + + def get_items(self, location, default_class=None): + """ + Returns a list of XModuleDescriptor instances for the items + that match location. Any element of location that is None is treated + as a wildcard that matches any value + + location: Something that can be passed to Location + default_class: An XModuleDescriptor subclass to use if no plugin matching the + location is found + """ + raise NotImplementedError # TODO (cpennington): Replace with clone_item def create_item(self, location, editor): diff --git a/common/lib/xmodule/xmodule/modulestore/mongo.py b/common/lib/xmodule/xmodule/modulestore/mongo.py index cc731c929c..62a289a750 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo.py @@ -58,6 +58,29 @@ class MongoModuleStore(ModuleStore): return XModuleDescriptor.load_from_json( item, MakoDescriptorSystem(load_item=self.get_item, resources_fs=None, render_template=render_to_string), self.default_class) + def get_items(self, location, default_class=None): + query = {} + for key, val in Location(location).dict().iteritems(): + if val is not None: + query['location.{key}'.format(key=key)] = val + + items = self.collection.find( + query, + sort=[('revision', pymongo.ASCENDING)], + ) + + # TODO (cpennington): Pass a proper resources_fs to the system + system = MakoDescriptorSystem( + load_item=self.get_item, + resources_fs=None, + render_template=render_to_string + ) + + return [ + XModuleDescriptor.load_from_json(item, system, self.default_class) + for item in items + ] + def create_item(self, location): """ Create an empty item at the specified location with the supplied editor From 9b134aadf4b1f8878eb80df2a88b1abeefd6302e Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Tue, 3 Jul 2012 16:04:18 -0400 Subject: [PATCH 06/18] Swallow attempts to create a new item where one already exists --- cms/djangoapps/contentstore/__init__.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cms/djangoapps/contentstore/__init__.py b/cms/djangoapps/contentstore/__init__.py index 7a38b99b5e..433278c47b 100644 --- a/cms/djangoapps/contentstore/__init__.py +++ b/cms/djangoapps/contentstore/__init__.py @@ -1,5 +1,8 @@ from xmodule.modulestore.django import modulestore from xmodule.modulestore.xml import XMLModuleStore +import logging + +log = logging.getLogger(__name__) def import_from_xml(org, course, data_dir): @@ -9,7 +12,14 @@ def import_from_xml(org, course, data_dir): """ module_store = XMLModuleStore(org, course, data_dir, 'xmodule.raw_module.RawDescriptor', eager=True) for module in module_store.modules.itervalues(): - modulestore().create_item(module.location) + + # TODO (cpennington): This forces import to overrite the same items. + # This should in the future create new revisions of the items on import + try: + modulestore().create_item(module.location) + except: + log.exception('Item already exists at %s' % module.location.url()) + pass if 'data' in module.definition: modulestore().update_item(module.location, module.definition['data']) if 'children' in module.definition: From d99d98670e4b47d7228185d97bcfd64ec1bb1416 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Tue, 3 Jul 2012 18:57:35 -0400 Subject: [PATCH 07/18] Point at root url for save_item --- cms/static/coffee/unit.coffee | 2 +- cms/urls.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cms/static/coffee/unit.coffee b/cms/static/coffee/unit.coffee index b81bc0df08..dda41fe42b 100644 --- a/cms/static/coffee/unit.coffee +++ b/cms/static/coffee/unit.coffee @@ -4,7 +4,7 @@ class @Unit $("##{@element_id} .save-update").click (event) => event.preventDefault() - $.post("save_item", { + $.post("/save_item", { id: @module_id data: JSON.stringify(@module.save()) }) diff --git a/cms/urls.py b/cms/urls.py index 3a953261a1..fe3c9216e6 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -6,9 +6,9 @@ from django.conf.urls.defaults import patterns, url urlpatterns = patterns('', url(r'^$', 'contentstore.views.index', name='index'), - url(r'^(?P[^/]+)/(?P[^/]+)/course/(?P[^/]+)$', 'contentstore.views.course_index', name='course_index'), url(r'^edit_item$', 'contentstore.views.edit_item', name='edit_item'), url(r'^save_item$', 'contentstore.views.save_item', name='save_item'), + url(r'^(?P[^/]+)/(?P[^/]+)/course/(?P[^/]+)$', 'contentstore.views.course_index', name='course_index'), url(r'^temp_force_export$', 'contentstore.views.temp_force_export'), url(r'^github_service_hook$', 'github_sync.views.github_post_receive'), ) From 9ae644e00757dc4f5a7796ad26394ddee3bbc0f8 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Tue, 3 Jul 2012 18:59:17 -0400 Subject: [PATCH 08/18] Make html save action send the correct data type --- common/lib/xmodule/xmodule/js/module/html.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/lib/xmodule/xmodule/js/module/html.coffee b/common/lib/xmodule/xmodule/js/module/html.coffee index 5e072c27a3..699b0994bd 100644 --- a/common/lib/xmodule/xmodule/js/module/html.coffee +++ b/common/lib/xmodule/xmodule/js/module/html.coffee @@ -6,4 +6,4 @@ class @HTML @preview.empty().append(@edit_box.val()) ) - save: -> {text: @edit_box.val()} + save: -> @edit_box.val() From cf2691301ede1eb2d0c3febde00a4d40a8f00f2d Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Tue, 3 Jul 2012 19:00:56 -0400 Subject: [PATCH 09/18] Use _id to store the location, rather than location --- .../lib/xmodule/xmodule/modulestore/mongo.py | 69 ++++++++++--------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/common/lib/xmodule/xmodule/modulestore/mongo.py b/common/lib/xmodule/xmodule/modulestore/mongo.py index 62a289a750..ee77e59b26 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo.py @@ -1,4 +1,5 @@ import pymongo +from bson.objectid import ObjectId from importlib import import_module from xmodule.x_module import XModuleDescriptor from xmodule.mako_module import MakoDescriptorSystem @@ -8,6 +9,19 @@ from . import ModuleStore, Location from .exceptions import ItemNotFoundError, InsufficientSpecificationError +# TODO (cpennington): This code currently operates under the assumption that +# there is only one revision for each item. Once we start versioning inside the CMS, +# that assumption will have to change + + +def location_to_query(loc): + query = {} + for key, val in Location(loc).dict().iteritems(): + if val is not None: + query['_id.{key}'.format(key=key)] = val + + return query + class MongoModuleStore(ModuleStore): """ A Mongodb backed ModuleStore @@ -17,7 +31,6 @@ class MongoModuleStore(ModuleStore): host=host, port=port )[db][collection] - self.collection.ensure_index('location') # Force mongo to report errors, at the expense of performance self.collection.safe = True @@ -26,6 +39,18 @@ class MongoModuleStore(ModuleStore): class_ = getattr(import_module(module_path), class_name) self.default_class = class_ + # TODO (cpennington): Pass a proper resources_fs to the system + self.system = MakoDescriptorSystem( + load_item=self.get_item, + resources_fs=None, + render_template=render_to_string + ) + + def _load_item(self, item): + item['location'] = item['_id'] + del item['_id'] + return XModuleDescriptor.load_from_json(item, self.system, self.default_class) + def get_item(self, location): """ Returns an XModuleDescriptor instance for the item at location. @@ -39,47 +64,26 @@ class MongoModuleStore(ModuleStore): location: Something that can be passed to Location """ - query = {} for key, val in Location(location).dict().iteritems(): if key != 'revision' and val is None: raise InsufficientSpecificationError(location) - if val is not None: - query['location.{key}'.format(key=key)] = val - item = self.collection.find_one( - query, + location_to_query(location), sort=[('revision', pymongo.ASCENDING)], ) if item is None: raise ItemNotFoundError(location) - - # TODO (cpennington): Pass a proper resources_fs to the system - return XModuleDescriptor.load_from_json( - item, MakoDescriptorSystem(load_item=self.get_item, resources_fs=None, render_template=render_to_string), self.default_class) + return self._load_item(item) def get_items(self, location, default_class=None): - query = {} - for key, val in Location(location).dict().iteritems(): - if val is not None: - query['location.{key}'.format(key=key)] = val - + print location_to_query(location) items = self.collection.find( - query, + location_to_query(location), sort=[('revision', pymongo.ASCENDING)], ) - # TODO (cpennington): Pass a proper resources_fs to the system - system = MakoDescriptorSystem( - load_item=self.get_item, - resources_fs=None, - render_template=render_to_string - ) - - return [ - XModuleDescriptor.load_from_json(item, system, self.default_class) - for item in items - ] + return [self._load_item(item) for item in items] def create_item(self, location): """ @@ -88,7 +92,7 @@ class MongoModuleStore(ModuleStore): location: Something that can be passed to Location """ self.collection.insert({ - 'location': Location(location).dict(), + '_id': Location(location).dict(), }) def update_item(self, location, data): @@ -103,8 +107,9 @@ class MongoModuleStore(ModuleStore): # See http://www.mongodb.org/display/DOCS/Updating for # atomic update syntax self.collection.update( - {'location': Location(location).dict()}, - {'$set': {'definition.data': data}} + {'_id': Location(location).dict()}, + {'$set': {'definition.data': data}}, + ) def update_children(self, location, children): @@ -119,7 +124,7 @@ class MongoModuleStore(ModuleStore): # See http://www.mongodb.org/display/DOCS/Updating for # atomic update syntax self.collection.update( - {'location': Location(location).dict()}, + {'_id': Location(location).dict()}, {'$set': {'definition.children': children}} ) @@ -135,6 +140,6 @@ class MongoModuleStore(ModuleStore): # See http://www.mongodb.org/display/DOCS/Updating for # atomic update syntax self.collection.update( - {'location': Location(location).dict()}, + {'_id': Location(location).dict()}, {'$set': {'metadata': metadata}} ) From b0ff3053b9a8f451c25a821b85b4c341fe475ce3 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Tue, 3 Jul 2012 19:06:07 -0400 Subject: [PATCH 10/18] Add development code for pushing CMS changes to github --- cms/djangoapps/contentstore/views.py | 11 ++++++++- cms/djangoapps/github_sync/__init__.py | 19 +++++++++++++-- cms/envs/common.py | 1 + cms/envs/dev.py | 33 ++++++++++++++++++++++++-- 4 files changed, 59 insertions(+), 5 deletions(-) diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index c4825a4ddf..ca06942ae4 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -4,6 +4,8 @@ from django.http import HttpResponse from django_future.csrf import ensure_csrf_cookie from fs.osfs import OSFS from django.core.urlresolvers import reverse +from xmodule.modulestore import Location +from github_sync import repo_path_from_location, export_to_github from mitxmako.shortcuts import render_to_response from xmodule.modulestore.django import modulestore @@ -12,7 +14,6 @@ from xmodule.modulestore.django import modulestore @ensure_csrf_cookie def index(request): courses = modulestore().get_items(['i4x', None, None, 'course', None]) - print courses return render_to_response('index.html', { 'courses': [(course.metadata['display_name'], reverse('course_index', args=[ @@ -46,6 +47,14 @@ def save_item(request): item_id = request.POST['id'] data = json.loads(request.POST['data']) modulestore().update_item(item_id, data) + + # Export the course back to github + course_location = Location(item_id)._replace(category='course', name=None) + courses = modulestore().get_items(course_location) + for course in courses: + repo_path = repo_path_from_location(course.location) + export_to_github(course, repo_path, "CMS Edit") + return HttpResponse(json.dumps({})) diff --git a/cms/djangoapps/github_sync/__init__.py b/cms/djangoapps/github_sync/__init__.py index c6bbca222f..178291f7db 100644 --- a/cms/djangoapps/github_sync/__init__.py +++ b/cms/djangoapps/github_sync/__init__.py @@ -1,6 +1,9 @@ from git import Repo from contentstore import import_from_xml from fs.osfs import OSFS +import os +from xmodule.modulestore import Location +from django.conf import settings def import_from_github(repo_settings): @@ -13,17 +16,28 @@ def import_from_github(repo_settings): org: name of the """ repo_path = repo_settings['path'] + + if not os.path.isdir(repo_path): + Repo.clone_from(repo_settings['origin'], repo_path) + git_repo = Repo(repo_path) origin = git_repo.remotes.origin origin.fetch() # Do a hard reset to the remote branch so that we have a clean import - git_repo.heads[repo_settings['branch']].checkout() + git_repo.git.checkout(repo_settings['branch']) git_repo.head.reset('origin/%s' % repo_settings['branch'], index=True, working_tree=True) return git_repo.head.commit.hexsha, import_from_xml(repo_settings['org'], repo_settings['course'], repo_path) +def repo_path_from_location(location): + location = Location(location) + for name, repo in settings.REPOS.items(): + if repo['org'] == location.org and repo['course'] == location.course: + return repo['path'] + + def export_to_github(course, repo_path, commit_message): fs = OSFS(repo_path) xml = course.export_to_xml(fs) @@ -37,4 +51,5 @@ def export_to_github(course, repo_path, commit_message): git_repo.git.commit(m=commit_message) origin = git_repo.remotes.origin - origin.push() + if settings.MITX_FEATURES['GITHUB_PUSH']: + origin.push() diff --git a/cms/envs/common.py b/cms/envs/common.py index 6edd6aa2b9..400a5138b9 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -30,6 +30,7 @@ from path import path MITX_FEATURES = { 'USE_DJANGO_PIPELINE': True, + 'GITHUB_PUSH': False, } ############################# SET PATH INFORMATION ############################# diff --git a/cms/envs/dev.py b/cms/envs/dev.py index 8ff2a35c52..13dd2e847b 100644 --- a/cms/envs/dev.py +++ b/cms/envs/dev.py @@ -36,8 +36,37 @@ REPOS = { 'path': REPO_ROOT / "edx4edx", 'org': 'edx', 'course': 'edx4edx', - 'branch': 'for_cms' - } + 'branch': 'for_cms', + 'origin': 'git@github.com:MITx/edx4edx.git', + }, + '6002x-fall-2012': { + 'path': REPO_ROOT / '6002x-fall-2012', + 'org': 'mit.edu', + 'course': '6.002x', + 'branch': 'master', + 'origin': 'git@github.com:MITx/6002x-fall-2012.git', + }, + '6.00x': { + 'path': REPO_ROOT / '6.00x', + 'org': 'mit.edu', + 'course': '6.00x', + 'branch': 'master', + 'origin': 'git@github.com:MITx/6.00x.git', + }, + '7.00x': { + 'path': REPO_ROOT / '7.00x', + 'org': 'mit.edu', + 'course': '7.00x', + 'branch': 'master', + 'origin': 'git@github.com:MITx/7.00x.git', + }, + '3.091x': { + 'path': REPO_ROOT / '3.091x', + 'org': 'mit.edu', + 'course': '3.091x', + 'branch': 'master', + 'origin': 'git@github.com:MITx/3.091x.git', + }, } CACHES = { From 894ae58bc3706759db885bb282200156b9bbaadb Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Tue, 3 Jul 2012 21:22:36 -0400 Subject: [PATCH 11/18] Use unicode for filepaths in data repos --- common/lib/xmodule/xmodule/xml_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index 5699f962cf..4d0ef3606a 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -158,7 +158,7 @@ class XmlDescriptor(XModuleDescriptor): @classmethod def _format_filepath(cls, type, name): - return '{type}/{name}.{ext}'.format(type=type, name=name, ext=cls.filename_extension) + return u'{type}/{name}.{ext}'.format(type=type, name=name, ext=cls.filename_extension) def export_to_xml(self, resource_fs): """ From 31288d6caf96c28ea9279992091457277bfb6fe1 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Tue, 3 Jul 2012 21:22:48 -0400 Subject: [PATCH 12/18] Only use the for_cms branch for development purposes --- cms/envs/dev.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cms/envs/dev.py b/cms/envs/dev.py index 13dd2e847b..ca48481184 100644 --- a/cms/envs/dev.py +++ b/cms/envs/dev.py @@ -43,28 +43,28 @@ REPOS = { 'path': REPO_ROOT / '6002x-fall-2012', 'org': 'mit.edu', 'course': '6.002x', - 'branch': 'master', + 'branch': 'for_cms', 'origin': 'git@github.com:MITx/6002x-fall-2012.git', }, '6.00x': { 'path': REPO_ROOT / '6.00x', 'org': 'mit.edu', 'course': '6.00x', - 'branch': 'master', + 'branch': 'for_cms', 'origin': 'git@github.com:MITx/6.00x.git', }, '7.00x': { 'path': REPO_ROOT / '7.00x', 'org': 'mit.edu', 'course': '7.00x', - 'branch': 'master', + 'branch': 'for_cms', 'origin': 'git@github.com:MITx/7.00x.git', }, '3.091x': { 'path': REPO_ROOT / '3.091x', 'org': 'mit.edu', 'course': '3.091x', - 'branch': 'master', + 'branch': 'for_cms', 'origin': 'git@github.com:MITx/3.091x.git', }, } From 83da714a50fc4285f36e53d0a441bb0be44e14db Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Thu, 5 Jul 2012 10:30:04 -0400 Subject: [PATCH 13/18] Add more docstring to import_from_github --- cms/djangoapps/github_sync/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cms/djangoapps/github_sync/__init__.py b/cms/djangoapps/github_sync/__init__.py index 178291f7db..d00949e56e 100644 --- a/cms/djangoapps/github_sync/__init__.py +++ b/cms/djangoapps/github_sync/__init__.py @@ -13,7 +13,8 @@ def import_from_github(repo_settings): repo_settings is a dictionary with the following keys: path: file system path to the local git repo branch: name of the branch to track on github - org: name of the + org: name of the organization to use in the imported course + course: name of the coures to use in the imported course """ repo_path = repo_settings['path'] From ab851c0b1ae7e528eecfce811bb6e3ebab1c712a Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Thu, 5 Jul 2012 11:54:43 -0400 Subject: [PATCH 14/18] Add tests for the core github_sync functionality --- cms/djangoapps/github_sync/__init__.py | 31 ++++-- cms/djangoapps/github_sync/exceptions.py | 2 + cms/djangoapps/github_sync/tests/__init__.py | 96 +++++++++++++++++++ cms/envs/test.py | 14 ++- .../data/toy/{toy_course.xml => course.xml} | 0 requirements.txt | 1 + test_root/.gitignore | 2 + 7 files changed, 137 insertions(+), 9 deletions(-) create mode 100644 cms/djangoapps/github_sync/exceptions.py create mode 100644 cms/djangoapps/github_sync/tests/__init__.py rename common/test/data/toy/{toy_course.xml => course.xml} (100%) create mode 100644 test_root/.gitignore diff --git a/cms/djangoapps/github_sync/__init__.py b/cms/djangoapps/github_sync/__init__.py index d00949e56e..c901ad37e7 100644 --- a/cms/djangoapps/github_sync/__init__.py +++ b/cms/djangoapps/github_sync/__init__.py @@ -1,10 +1,16 @@ -from git import Repo -from contentstore import import_from_xml -from fs.osfs import OSFS +import logging import os -from xmodule.modulestore import Location -from django.conf import settings +from django.conf import settings +from fs.osfs import OSFS +from git import Repo, PushInfo + +from contentstore import import_from_xml +from xmodule.modulestore import Location + +from .exceptions import GithubSyncError + +log = logging.getLogger(__name__) def import_from_github(repo_settings): """ @@ -53,4 +59,17 @@ def export_to_github(course, repo_path, commit_message): origin = git_repo.remotes.origin if settings.MITX_FEATURES['GITHUB_PUSH']: - origin.push() + push_infos = origin.push() + if len(push_infos) > 1: + log.error('Unexpectedly pushed multiple heads: {infos}'.format( + infos="\n".join(str(info.summary) for info in push_infos) + )) + + if push_infos[0].flags & PushInfo.ERROR: + log.error('Failed push: flags={p.flags}, local_ref={p.local_ref}, ' + 'remote_ref_string={p.remote_ref_string}, ' + 'remote_ref={p.remote_ref}, old_commit={p.old_commit}, ' + 'summary={p.summary})'.format(p=push_infos[0])) + raise GithubSyncError('Failed to push: {info}'.format( + info=str(push_infos[0].summary) + )) diff --git a/cms/djangoapps/github_sync/exceptions.py b/cms/djangoapps/github_sync/exceptions.py new file mode 100644 index 0000000000..9097ffc2a6 --- /dev/null +++ b/cms/djangoapps/github_sync/exceptions.py @@ -0,0 +1,2 @@ +class GithubSyncError(Exception): + pass diff --git a/cms/djangoapps/github_sync/tests/__init__.py b/cms/djangoapps/github_sync/tests/__init__.py new file mode 100644 index 0000000000..825fc68313 --- /dev/null +++ b/cms/djangoapps/github_sync/tests/__init__.py @@ -0,0 +1,96 @@ +from django.test import TestCase +from path import path +import shutil +from github_sync import import_from_github, export_to_github, repo_path_from_location +from git import Repo +from django.conf import settings +from xmodule.modulestore.django import modulestore +from xmodule.modulestore import Location +from override_settings import override_settings +from github_sync.exceptions import GithubSyncError + + +class GithubSyncTestCase(TestCase): + + def setUp(self): + self.working_dir = path(settings.TEST_ROOT) + self.repo_dir = self.working_dir / 'local_repo' + self.remote_dir = self.working_dir / 'remote_repo' + shutil.copytree('common/test/data/toy', self.remote_dir) + + remote = Repo.init(self.remote_dir) + remote.git.add(A=True) + remote.git.commit(m='Initial commit') + remote.git.config("receive.denyCurrentBranch", "ignore") + + modulestore().collection.drop() + + self.import_revision, self.import_course = import_from_github({ + 'path': self.repo_dir, + 'origin': self.remote_dir, + 'branch': 'master', + 'org': 'org', + 'course': 'course' + }) + + def tearDown(self): + shutil.rmtree(self.repo_dir) + shutil.rmtree(self.remote_dir) + + def test_initialize_repo(self): + """ + Test that importing from github will create a repo if the repo doesn't already exist + """ + self.assertEquals(1, len(Repo(self.repo_dir).head.reference.log())) + + def test_import_contents(self): + """ + Test that the import loads the correct course into the modulestore + """ + self.assertEquals('Toy Course', self.import_course.metadata['display_name']) + self.assertIn( + Location('i4x://org/course/chapter/Overview'), + [child.location for child in self.import_course.get_children()]) + self.assertEquals(1, len(self.import_course.get_children())) + + @override_settings(MITX_FEATURES={'GITHUB_PUSH': False}) + def test_export_no_pash(self): + """ + Test that with the GITHUB_PUSH feature disabled, no content is pushed to the remote + """ + export_to_github(self.import_course, self.repo_dir, 'Test no-push') + self.assertEquals(1, Repo(self.remote_dir).head.commit.count()) + + @override_settings(MITX_FEATURES={'GITHUB_PUSH': True}) + def test_export_push(self): + """ + Test that with GITHUB_PUSH enabled, content is pushed to the remote + """ + self.import_course.metadata['display_name'] = 'Changed display name' + export_to_github(self.import_course, self.repo_dir, 'Test push') + self.assertEquals(2, Repo(self.remote_dir).head.commit.count()) + + @override_settings(MITX_FEATURES={'GITHUB_PUSH': True}) + def test_export_conflict(self): + """ + Test that if there is a conflict when pushing to the remote repo, nothing is pushed and an exception is raised + """ + self.import_course.metadata['display_name'] = 'Changed display name' + + remote = Repo(self.remote_dir) + remote.git.commit(allow_empty=True, m="Testing conflict commit") + + self.assertRaises(GithubSyncError, export_to_github, self.import_course, self.repo_dir, 'Test push') + self.assertEquals(2, remote.head.reference.commit.count()) + self.assertEquals("Testing conflict commit\n", remote.head.reference.commit.message) + + +@override_settings(REPOS={'namea': {'path': 'patha', 'org': 'orga', 'course': 'coursea'}, + 'nameb': {'path': 'pathb', 'org': 'orgb', 'course': 'courseb'}}) +class RepoPathLookupTestCase(TestCase): + def test_successful_lookup(self): + self.assertEquals('patha', repo_path_from_location('i4x://orga/coursea/course/foo')) + self.assertEquals('pathb', repo_path_from_location('i4x://orgb/courseb/course/foo')) + + def test_failed_lookup(self): + self.assertEquals(None, repo_path_from_location('i4x://c/c/course/foo')) diff --git a/cms/envs/test.py b/cms/envs/test.py index 032de92953..927e2af987 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -17,10 +17,18 @@ for app in os.listdir(PROJECT_ROOT / 'djangoapps'): NOSE_ARGS += ['--cover-package', app] TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' +TEST_ROOT = 'test_root' + MODULESTORE = { - 'host': 'localhost', - 'db': 'mongo_base', - 'collection': 'key_store', + 'default': { + 'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore', + 'OPTIONS': { + 'default_class': 'xmodule.raw_module.RawDescriptor', + 'host': 'localhost', + 'db': 'test_xmodule', + 'collection': 'modulestore', + } + } } DATABASES = { diff --git a/common/test/data/toy/toy_course.xml b/common/test/data/toy/course.xml similarity index 100% rename from common/test/data/toy/toy_course.xml rename to common/test/data/toy/course.xml diff --git a/requirements.txt b/requirements.txt index ddf218af7c..571c530a2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,3 +29,4 @@ django_nose nosexcover rednose GitPython >= 0.3 +django-override-settings diff --git a/test_root/.gitignore b/test_root/.gitignore new file mode 100644 index 0000000000..b3e5512f73 --- /dev/null +++ b/test_root/.gitignore @@ -0,0 +1,2 @@ +local_repo +remote_repo From 386acbe1ff755d6cf11eb696c592dcf4cff55a3b Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Thu, 5 Jul 2012 12:15:49 -0400 Subject: [PATCH 15/18] Add tests of the github_sync service hook view --- .../github_sync/tests/test_views.py | 53 +++++++++++++++++++ requirements.txt | 1 + 2 files changed, 54 insertions(+) create mode 100644 cms/djangoapps/github_sync/tests/test_views.py diff --git a/cms/djangoapps/github_sync/tests/test_views.py b/cms/djangoapps/github_sync/tests/test_views.py new file mode 100644 index 0000000000..9e8095a67b --- /dev/null +++ b/cms/djangoapps/github_sync/tests/test_views.py @@ -0,0 +1,53 @@ +import json +from django.test.client import Client +from django.test import TestCase +from mock import patch, Mock +from override_settings import override_settings +from django.conf import settings + + +@override_settings(REPOS={'repo': {'path': 'path', 'branch': 'branch'}}) +class PostReceiveTestCase(TestCase): + def setUp(self): + self.client = Client() + + @patch('github_sync.views.export_to_github') + @patch('github_sync.views.import_from_github') + def test_non_branch(self, import_from_github, export_to_github): + self.client.post('/github_service_hook', {'payload': json.dumps({ + 'ref': 'refs/tags/foo'}) + }) + self.assertFalse(import_from_github.called) + self.assertFalse(export_to_github.called) + + @patch('github_sync.views.export_to_github') + @patch('github_sync.views.import_from_github') + def test_non_watched_repo(self, import_from_github, export_to_github): + self.client.post('/github_service_hook', {'payload': json.dumps({ + 'ref': 'refs/heads/branch', + 'repository': {'name': 'bad_repo'}}) + }) + self.assertFalse(import_from_github.called) + self.assertFalse(export_to_github.called) + + @patch('github_sync.views.export_to_github') + @patch('github_sync.views.import_from_github') + def test_non_tracked_branch(self, import_from_github, export_to_github): + self.client.post('/github_service_hook', {'payload': json.dumps({ + 'ref': 'refs/heads/non_branch', + 'repository': {'name': 'repo'}}) + }) + self.assertFalse(import_from_github.called) + self.assertFalse(export_to_github.called) + + @patch('github_sync.views.export_to_github') + @patch('github_sync.views.import_from_github', return_value=(Mock(), Mock())) + def test_tracked_branch(self, import_from_github, export_to_github): + self.client.post('/github_service_hook', {'payload': json.dumps({ + 'ref': 'refs/heads/branch', + 'repository': {'name': 'repo'}}) + }) + import_from_github.assert_called_with(settings.REPOS['repo']) + mock_revision, mock_course = import_from_github.return_value + export_to_github.assert_called_with(mock_course, 'path', "Changes from cms import of revision %s" % mock_revision) + diff --git a/requirements.txt b/requirements.txt index 571c530a2f..d342c46859 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,3 +30,4 @@ nosexcover rednose GitPython >= 0.3 django-override-settings +mock>=0.8, <0.9 From e9ee1566d6311de8a29c9aa0659f4f94a325c381 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Thu, 5 Jul 2012 12:48:18 -0400 Subject: [PATCH 16/18] Modularize capa and mitxmako so that xmodule can properly depend on them --- common/lib/capa/{ => capa}/__init__.py | 0 common/lib/capa/{ => capa}/calc.py | 0 common/lib/capa/{ => capa}/capa_problem.py | 0 common/lib/capa/{ => capa}/checker.py | 0 common/lib/capa/{ => capa}/correctmap.py | 0 common/lib/capa/{ => capa}/eia.py | 0 common/lib/capa/{ => capa}/inputtypes.py | 0 common/lib/capa/{ => capa}/responsetypes.py | 0 common/lib/capa/{ => capa}/templates/choicegroup.html | 0 common/lib/capa/{ => capa}/templates/imageinput.html | 0 common/lib/capa/{ => capa}/templates/jstextline.html | 0 common/lib/capa/{ => capa}/templates/mathstring.html | 0 common/lib/capa/{ => capa}/templates/schematicinput.html | 0 common/lib/capa/{ => capa}/templates/solutionspan.html | 0 common/lib/capa/{ => capa}/templates/textbox.html | 0 common/lib/capa/{ => capa}/templates/textinput.html | 0 .../lib/capa/{ => capa}/templates/textinput_dynamath.html | 0 common/lib/capa/{ => capa}/util.py | 0 common/lib/capa/setup.py | 8 ++++++++ common/lib/mitxmako/{ => mitxmako}/__init__.py | 0 common/lib/mitxmako/{ => mitxmako}/middleware.py | 0 common/lib/mitxmako/{ => mitxmako}/shortcuts.py | 0 common/lib/mitxmako/{ => mitxmako}/template.py | 0 common/lib/mitxmako/setup.py | 8 ++++++++ common/lib/xmodule/setup.py | 4 ++++ requirements.txt | 2 ++ 26 files changed, 22 insertions(+) rename common/lib/capa/{ => capa}/__init__.py (100%) rename common/lib/capa/{ => capa}/calc.py (100%) rename common/lib/capa/{ => capa}/capa_problem.py (100%) rename common/lib/capa/{ => capa}/checker.py (100%) rename common/lib/capa/{ => capa}/correctmap.py (100%) rename common/lib/capa/{ => capa}/eia.py (100%) rename common/lib/capa/{ => capa}/inputtypes.py (100%) rename common/lib/capa/{ => capa}/responsetypes.py (100%) rename common/lib/capa/{ => capa}/templates/choicegroup.html (100%) rename common/lib/capa/{ => capa}/templates/imageinput.html (100%) rename common/lib/capa/{ => capa}/templates/jstextline.html (100%) rename common/lib/capa/{ => capa}/templates/mathstring.html (100%) rename common/lib/capa/{ => capa}/templates/schematicinput.html (100%) rename common/lib/capa/{ => capa}/templates/solutionspan.html (100%) rename common/lib/capa/{ => capa}/templates/textbox.html (100%) rename common/lib/capa/{ => capa}/templates/textinput.html (100%) rename common/lib/capa/{ => capa}/templates/textinput_dynamath.html (100%) rename common/lib/capa/{ => capa}/util.py (100%) create mode 100644 common/lib/capa/setup.py rename common/lib/mitxmako/{ => mitxmako}/__init__.py (100%) rename common/lib/mitxmako/{ => mitxmako}/middleware.py (100%) rename common/lib/mitxmako/{ => mitxmako}/shortcuts.py (100%) rename common/lib/mitxmako/{ => mitxmako}/template.py (100%) create mode 100644 common/lib/mitxmako/setup.py diff --git a/common/lib/capa/__init__.py b/common/lib/capa/capa/__init__.py similarity index 100% rename from common/lib/capa/__init__.py rename to common/lib/capa/capa/__init__.py diff --git a/common/lib/capa/calc.py b/common/lib/capa/capa/calc.py similarity index 100% rename from common/lib/capa/calc.py rename to common/lib/capa/capa/calc.py diff --git a/common/lib/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py similarity index 100% rename from common/lib/capa/capa_problem.py rename to common/lib/capa/capa/capa_problem.py diff --git a/common/lib/capa/checker.py b/common/lib/capa/capa/checker.py similarity index 100% rename from common/lib/capa/checker.py rename to common/lib/capa/capa/checker.py diff --git a/common/lib/capa/correctmap.py b/common/lib/capa/capa/correctmap.py similarity index 100% rename from common/lib/capa/correctmap.py rename to common/lib/capa/capa/correctmap.py diff --git a/common/lib/capa/eia.py b/common/lib/capa/capa/eia.py similarity index 100% rename from common/lib/capa/eia.py rename to common/lib/capa/capa/eia.py diff --git a/common/lib/capa/inputtypes.py b/common/lib/capa/capa/inputtypes.py similarity index 100% rename from common/lib/capa/inputtypes.py rename to common/lib/capa/capa/inputtypes.py diff --git a/common/lib/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py similarity index 100% rename from common/lib/capa/responsetypes.py rename to common/lib/capa/capa/responsetypes.py diff --git a/common/lib/capa/templates/choicegroup.html b/common/lib/capa/capa/templates/choicegroup.html similarity index 100% rename from common/lib/capa/templates/choicegroup.html rename to common/lib/capa/capa/templates/choicegroup.html diff --git a/common/lib/capa/templates/imageinput.html b/common/lib/capa/capa/templates/imageinput.html similarity index 100% rename from common/lib/capa/templates/imageinput.html rename to common/lib/capa/capa/templates/imageinput.html diff --git a/common/lib/capa/templates/jstextline.html b/common/lib/capa/capa/templates/jstextline.html similarity index 100% rename from common/lib/capa/templates/jstextline.html rename to common/lib/capa/capa/templates/jstextline.html diff --git a/common/lib/capa/templates/mathstring.html b/common/lib/capa/capa/templates/mathstring.html similarity index 100% rename from common/lib/capa/templates/mathstring.html rename to common/lib/capa/capa/templates/mathstring.html diff --git a/common/lib/capa/templates/schematicinput.html b/common/lib/capa/capa/templates/schematicinput.html similarity index 100% rename from common/lib/capa/templates/schematicinput.html rename to common/lib/capa/capa/templates/schematicinput.html diff --git a/common/lib/capa/templates/solutionspan.html b/common/lib/capa/capa/templates/solutionspan.html similarity index 100% rename from common/lib/capa/templates/solutionspan.html rename to common/lib/capa/capa/templates/solutionspan.html diff --git a/common/lib/capa/templates/textbox.html b/common/lib/capa/capa/templates/textbox.html similarity index 100% rename from common/lib/capa/templates/textbox.html rename to common/lib/capa/capa/templates/textbox.html diff --git a/common/lib/capa/templates/textinput.html b/common/lib/capa/capa/templates/textinput.html similarity index 100% rename from common/lib/capa/templates/textinput.html rename to common/lib/capa/capa/templates/textinput.html diff --git a/common/lib/capa/templates/textinput_dynamath.html b/common/lib/capa/capa/templates/textinput_dynamath.html similarity index 100% rename from common/lib/capa/templates/textinput_dynamath.html rename to common/lib/capa/capa/templates/textinput_dynamath.html diff --git a/common/lib/capa/util.py b/common/lib/capa/capa/util.py similarity index 100% rename from common/lib/capa/util.py rename to common/lib/capa/capa/util.py diff --git a/common/lib/capa/setup.py b/common/lib/capa/setup.py new file mode 100644 index 0000000000..cf66229b88 --- /dev/null +++ b/common/lib/capa/setup.py @@ -0,0 +1,8 @@ +from setuptools import setup, find_packages + +setup( + name="capa", + version="0.1", + packages=find_packages(exclude=["tests"]), + install_requires=['distribute'], +) diff --git a/common/lib/mitxmako/__init__.py b/common/lib/mitxmako/mitxmako/__init__.py similarity index 100% rename from common/lib/mitxmako/__init__.py rename to common/lib/mitxmako/mitxmako/__init__.py diff --git a/common/lib/mitxmako/middleware.py b/common/lib/mitxmako/mitxmako/middleware.py similarity index 100% rename from common/lib/mitxmako/middleware.py rename to common/lib/mitxmako/mitxmako/middleware.py diff --git a/common/lib/mitxmako/shortcuts.py b/common/lib/mitxmako/mitxmako/shortcuts.py similarity index 100% rename from common/lib/mitxmako/shortcuts.py rename to common/lib/mitxmako/mitxmako/shortcuts.py diff --git a/common/lib/mitxmako/template.py b/common/lib/mitxmako/mitxmako/template.py similarity index 100% rename from common/lib/mitxmako/template.py rename to common/lib/mitxmako/mitxmako/template.py diff --git a/common/lib/mitxmako/setup.py b/common/lib/mitxmako/setup.py new file mode 100644 index 0000000000..535d86f90e --- /dev/null +++ b/common/lib/mitxmako/setup.py @@ -0,0 +1,8 @@ +from setuptools import setup, find_packages + +setup( + name="mitxmako", + version="0.1", + packages=find_packages(exclude=["tests"]), + install_requires=['distribute'], +) diff --git a/common/lib/xmodule/setup.py b/common/lib/xmodule/setup.py index 77b0838ff2..1ce23bca90 100644 --- a/common/lib/xmodule/setup.py +++ b/common/lib/xmodule/setup.py @@ -8,6 +8,10 @@ setup( package_data={ 'xmodule': ['js/module/*'] }, + requires=[ + 'capa', + 'mitxmako' + ], # See http://guide.python-distribute.org/creation.html#entry-points # for a description of entry_points diff --git a/requirements.txt b/requirements.txt index d342c46859..6f63769ee3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,6 +24,8 @@ sympy newrelic glob2 pymongo +-e common/lib/capa +-e common/lib/mitxmako -e common/lib/xmodule django_nose nosexcover From ce4cea0e3efa116919014f0077fa3c85855dc910 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Thu, 5 Jul 2012 13:07:29 -0400 Subject: [PATCH 17/18] Don't fail if no NOSE_COVER_HTML_DIR is set --- lms/envs/test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lms/envs/test.py b/lms/envs/test.py index 941b855f5c..aed49d43de 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -20,7 +20,8 @@ INSTALLED_APPS = [ # Nose Test Runner INSTALLED_APPS += ['django_nose'] -NOSE_ARGS = ['--cover-erase', '--with-xunit', '--with-xcoverage', '--cover-html', '--cover-inclusive', '--cover-html-dir', os.environ['NOSE_COVER_HTML_DIR']] +NOSE_ARGS = ['--cover-erase', '--with-xunit', '--with-xcoverage', '--cover-html', + '--cover-inclusive', '--cover-html-dir', os.environ.get('NOSE_COVER_HTML_DIR', 'cover_html')] for app in os.listdir(PROJECT_ROOT / 'djangoapps'): NOSE_ARGS += ['--cover-package', app] TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' From dbd82c4bc8a8c56de88adda994297d678f350487 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Thu, 5 Jul 2012 13:07:54 -0400 Subject: [PATCH 18/18] Install numpy before trying to install scipy from requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 6f63769ee3..efc8e48ddd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ django<1.4 pip +numpy scipy matplotlib markdown