From 6fee7928fc9961c08fbcbc7f7f35631a71ffbacf Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Thu, 14 Jun 2012 16:15:50 -0400 Subject: [PATCH] Successfully read course children out of mongodb --- .../management/commands/import.py | 29 +++++---- cms/djangoapps/contentstore/views.py | 5 +- cms/lib/keystore/__init__.py | 34 ++++++++++ cms/lib/keystore/exceptions.py | 4 ++ cms/lib/keystore/mongo.py | 63 ++++++++++++++++--- 5 files changed, 109 insertions(+), 26 deletions(-) diff --git a/cms/djangoapps/contentstore/management/commands/import.py b/cms/djangoapps/contentstore/management/commands/import.py index 78984f4119..690e3dbea0 100644 --- a/cms/djangoapps/contentstore/management/commands/import.py +++ b/cms/djangoapps/contentstore/management/commands/import.py @@ -6,11 +6,13 @@ #import mitxmako.middleware #from courseware import content_parser #from django.contrib.auth.models import User +import os.path +from StringIO import StringIO from mako.template import Template from mako.lookup import TemplateLookup from django.core.management.base import BaseCommand -from contentstore.models import create_item, update_item, update_children +from keystore.django import keystore from lxml import etree @@ -20,16 +22,15 @@ class Command(BaseCommand): def handle(self, *args, **options): print args data_dir = args[0] - course_file = 'course.xml' parser = etree.XMLParser(remove_comments = True) lookup = TemplateLookup(directories=[data_dir]) template = lookup.get_template("course.xml") course_string = template.render(groups=[]) - course = etree.XML(course_string, parser=parser) + course = etree.parse(StringIO(course_string), parser=parser) - elements = course.xpath("//*") + elements = list(course.iter()) tag_to_category = {# Inside HTML ==> Skip these # Custom tags @@ -39,11 +40,11 @@ class Command(BaseCommand): 'image': 'Custom', 'discuss': 'Custom', # Simple lists - 'chapter': 'Sequence', - 'course': 'Sequence', - 'sequential': 'Sequence', - 'vertical': 'Sequence', - 'section': 'Sequence', + 'chapter': 'Chapter', + 'course': 'Course', + 'sequential': 'LectureSequence', + 'vertical': 'ProblemSet', + 'section': 'Section', # True types 'video': 'VideoSegment', 'html': 'HTML', @@ -114,7 +115,7 @@ class Command(BaseCommand): results[e.attrib['url']] = {'data':{'text':text}} def handle_problem(e): - data = open(data_dir+'problems/'+e.attrib['filename']+'.xml').read() + data = open(os.path.join(data_dir, 'problems', e.attrib['filename']+'.xml')).read() results[e.attrib['url']] = {'data':{'statement':data}} element_actions = {# Inside HTML ==> Skip these @@ -149,10 +150,8 @@ class Command(BaseCommand): for k in results: print k - create_item(k, 'Piotr Mitros') + keystore.create_item(k, 'Piotr Mitros') if 'data' in results[k]: - update_item(k, results[k]['data']) + keystore.update_item(k, results[k]['data']) if 'children' in results[k]: - update_children(k, results[k]['children']) - - + keystore.update_children(k, results[k]['children']) diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index 38e9e8ad35..429fb6c26b 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -1,14 +1,11 @@ from mitxmako.shortcuts import render_to_response -from keystore import Location from keystore.django import keystore from django.contrib.auth.decorators import login_required @login_required def calendar(request, org, course): - weeks = keystore.get_children_for_item( - Location(['i4x', org, course, 'Course', 'course']) - ) + weeks = keystore.get_children_for_item(['i4x', org, course, 'Course', None]) return render_to_response('calendar.html', {'weeks': weeks}) diff --git a/cms/lib/keystore/__init__.py b/cms/lib/keystore/__init__.py index 5e6374cf4a..d0a24be797 100644 --- a/cms/lib/keystore/__init__.py +++ b/cms/lib/keystore/__init__.py @@ -60,6 +60,40 @@ class KeyStore(object): with the specified location. If no object is found at that location, raises keystore.exceptions.ItemNotFoundError + + Searches for all matches of a partially specifed location, but raises an + keystore.exceptions.InsufficientSpecificationError if more + than a single object matches the query. + + location: Something that can be passed to Location + """ + raise NotImplementedError + + def create_item(self, location, editor): + """ + Create an empty item at the specified location with the supplied editor + + location: Something that can be passed to Location + """ + raise NotImplementedError + + def update_item(self, location, data): + """ + Set the data in the item specified by the location to + data + + location: Something that can be passed to Location + data: A nested dictionary of problem data + """ + raise NotImplementedError + + def update_children(self, location, children): + """ + Set the children for the item specified by the location to + data + + location: Something that can be passed to Location + children: A list of child item identifiers """ raise NotImplementedError diff --git a/cms/lib/keystore/exceptions.py b/cms/lib/keystore/exceptions.py index b66470859f..08fd9b11d0 100644 --- a/cms/lib/keystore/exceptions.py +++ b/cms/lib/keystore/exceptions.py @@ -5,3 +5,7 @@ Exceptions thrown by KeyStore objects class ItemNotFoundError(Exception): pass + + +class InsufficientSpecificationError(Exception): + pass diff --git a/cms/lib/keystore/mongo.py b/cms/lib/keystore/mongo.py index fc190ee098..d29afb4bd2 100644 --- a/cms/lib/keystore/mongo.py +++ b/cms/lib/keystore/mongo.py @@ -1,6 +1,6 @@ import pymongo -from . import KeyStore -from .exceptions import ItemNotFoundError +from . import KeyStore, Location +from .exceptions import ItemNotFoundError, InsufficientSpecificationError class MongoKeyStore(KeyStore): @@ -12,15 +12,64 @@ class MongoKeyStore(KeyStore): host=host, port=port )[db][collection] + + # Force mongo to report errors, at the expense of performance + self.collection.safe = True def get_children_for_item(self, location): - item = self.collection.find_one( - {'location': location.dict()}, + query = dict( + ('location.{key}'.format(key=key), val) + for (key, val) + in Location(location).dict().items() + if val is not None + ) + items = self.collection.find( + query, fields={'children': True}, sort=[('revision', pymongo.ASCENDING)], + limit=1, + ) + if items.count() > 1: + raise InsufficientSpecificationError(location) + + if items.count() == 0: + raise ItemNotFoundError(location) + + return items[0]['children'] + + def create_item(self, location, editor): + """ + Create an empty item at the specified location with the supplied editor + + location: Something that can be passed to Location + """ + self.collection.insert({ + 'location': Location(location).dict(), + 'editor': editor + }) + + def update_item(self, location, data): + """ + Set the data in the item specified by the location to + data + + location: Something that can be passed to Location + data: A nested dictionary of problem data + """ + self.collection.update( + {'location': Location(location).dict()}, + {'$set': {'data': data}} ) - if item is None: - raise ItemNotFoundError() + def update_children(self, location, children): + """ + Set the children for the item specified by the location to + data - return item['children'] + location: Something that can be passed to Location + children: A list of child item identifiers + """ + self.collection.update( + {'location': Location(location).dict()}, + {'$set': {'children': children}} + )