From b9befbafcd562a037c09140aa2b48cf1f1f18e71 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Wed, 13 Jun 2012 11:42:50 -0400 Subject: [PATCH] Add contentstore from cms proto as a baseline for importing and views for the cms --- cms/djangoapps/contentstore/__init__.py | 0 .../contentstore/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../management/commands/ftpserve.py | 61 +++++++ .../management/commands/import.py | 158 ++++++++++++++++++ cms/djangoapps/contentstore/tests.py | 16 ++ cms/{ => djangoapps/contentstore}/views.py | 0 cms/envs/common.py | 6 +- cms/urls.py | 2 +- 9 files changed, 238 insertions(+), 5 deletions(-) create mode 100644 cms/djangoapps/contentstore/__init__.py create mode 100644 cms/djangoapps/contentstore/management/__init__.py create mode 100644 cms/djangoapps/contentstore/management/commands/__init__.py create mode 100644 cms/djangoapps/contentstore/management/commands/ftpserve.py create mode 100644 cms/djangoapps/contentstore/management/commands/import.py create mode 100644 cms/djangoapps/contentstore/tests.py rename cms/{ => djangoapps/contentstore}/views.py (100%) diff --git a/cms/djangoapps/contentstore/__init__.py b/cms/djangoapps/contentstore/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cms/djangoapps/contentstore/management/__init__.py b/cms/djangoapps/contentstore/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cms/djangoapps/contentstore/management/commands/__init__.py b/cms/djangoapps/contentstore/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cms/djangoapps/contentstore/management/commands/ftpserve.py b/cms/djangoapps/contentstore/management/commands/ftpserve.py new file mode 100644 index 0000000000..f0a1c19dbf --- /dev/null +++ b/cms/djangoapps/contentstore/management/commands/ftpserve.py @@ -0,0 +1,61 @@ +from django.core.management.base import BaseCommand +from django.contrib.auth.models import User +import contentstore.tasks + +from pyftpdlib import ftpserver +import os + +class DjangoAuthorizer(object): + def validate_authentication(self, username, password): + try: + u=User.objects.get(username=username) + except User.DoesNotExist: + return False + # TODO: Check security groups + return u.check_password(password) + def has_user(self, username): + print "????",username + return True + def has_perm(self, username, perm, path=None): + print "!!!!!",username, perm, path + return True + def get_home_dir(self, username): + d = "/tmp/ftp/"+username + try: + os.mkdir(d) + except OSError: + pass + return "/tmp/ftp/"+username + def get_perms(self, username): + return 'elradfmw' + def get_msg_login(self, username): + return 'Hello' + def get_msg_quit(self, username): + return 'Goodbye' + def __init__(self): + pass + def impersonate_user(self, username, password): + pass + def terminate_impersonation(self, username): + pass + +def on_upload(ftp_handler, filename): + source = ftp_handler.remote_ip + author = ftp_handler.username + print filename, author, source + # We pass on this for now: + # contentstore.tasks.on_upload + # It is a changing API, and it makes testing the FTP server slow. + +class Command(BaseCommand): + help = \ +''' Run FTP server.''' + def handle(self, *args, **options): + authorizer = DjangoAuthorizer() #ftpserver.DummyAuthorizer() + handler = ftpserver.FTPHandler + handler.on_file_received = on_upload + + handler.authorizer = authorizer + address = ("127.0.0.1", 2121) + ftpd = ftpserver.FTPServer(address, handler) + ftpd.serve_forever() diff --git a/cms/djangoapps/contentstore/management/commands/import.py b/cms/djangoapps/contentstore/management/commands/import.py new file mode 100644 index 0000000000..78984f4119 --- /dev/null +++ b/cms/djangoapps/contentstore/management/commands/import.py @@ -0,0 +1,158 @@ +### +### One-off script for importing courseware form XML format +### + + +#import mitxmako.middleware +#from courseware import content_parser +#from django.contrib.auth.models import User +from mako.template import Template +from mako.lookup import TemplateLookup + +from django.core.management.base import BaseCommand +from contentstore.models import create_item, update_item, update_children + +from lxml import etree + +class Command(BaseCommand): + help = \ +''' Run FTP server.''' + def handle(self, *args, **options): + print args + data_dir = args[0] + course_file = 'course.xml' + + parser = etree.XMLParser(remove_comments = True) + + lookup = TemplateLookup(directories=[data_dir]) + template = lookup.get_template("course.xml") + course_string = template.render(groups=[]) + course = etree.XML(course_string, parser=parser) + + elements = course.xpath("//*") + + tag_to_category = {# Inside HTML ==> Skip these + # Custom tags + 'videodev': 'Custom', + 'slides': 'Custom', + 'book': 'Custom', + 'image': 'Custom', + 'discuss': 'Custom', + # Simple lists + 'chapter': 'Sequence', + 'course': 'Sequence', + 'sequential': 'Sequence', + 'vertical': 'Sequence', + 'section': 'Sequence', + # True types + 'video': 'VideoSegment', + 'html': 'HTML', + 'problem': 'Problem', + } + + + name_index=0 + for e in elements: + name = e.attrib.get('name', None) + for f in elements: + if f != e and f.attrib.get('name', None) == name: + name = None + if not name: + name = "{tag}_{index}".format(tag = e.tag,index = name_index) + name_index = name_index + 1 + if e.tag in tag_to_category: + category = tag_to_category[e.tag] + category = category.replace('/', '-') + name = name.replace('/', '-') + e.set('url', 'i4x://mit.edu/6002xs12/{category}/{name}'.format(category = category, + name = name)) + + + def handle_skip(e): + print "Skipping ", e + + results = {} + + def handle_custom(e): + data = {'type':'i4x://mit.edu/6002xs12/tag/{tag}'.format(tag=e.tag), + 'attrib':dict(e.attrib)} + results[e.attrib['url']] = {'data':data} + + def handle_list(e): + if e.attrib.get("class", None) == "tutorials": + return + children = [{'url':le.attrib['url']} for le in e.getchildren()] + results[e.attrib['url']] = {'children':children} + + def handle_video(e): + url = e.attrib['url'] + clip_url = url.replace('VideoSegment', 'VideoClip') + # Take: 0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8 + # Make: [(0.75, 'izygArpw-Qo'), (1.0, 'p2Q6BrNhdh8'), (1.25, '1EeWXzPdhSA'), (1.5, 'rABDYkeK0x8')] + youtube_str = e.attrib['youtube'] + youtube_list = [(float(x), y) for x,y in map(lambda x:x.split(':'), youtube_str.split(','))] + clip_infos = [{ "status": "ready", + "format": "youtube", + "sane": True, + "location": "youtube", + "speed": speed, + "id": youtube_id, + "size": None} \ + for (speed, youtube_id) \ + in youtube_list] + results[clip_url] = {'data':{'clip_infos':clip_infos}} + results[url] = {'children' : [{'url':clip_url}]} + + def handle_html(e): + if 'src' in e.attrib: + text = open(data_dir+'html/'+e.attrib['src']).read() + else: + textlist=[e.text]+[etree.tostring(elem) for elem in e]+[e.tail] + textlist=[i for i in textlist if type(i)==str] + text = "".join(textlist) + + results[e.attrib['url']] = {'data':{'text':text}} + + def handle_problem(e): + data = open(data_dir+'problems/'+e.attrib['filename']+'.xml').read() + results[e.attrib['url']] = {'data':{'statement':data}} + + element_actions = {# Inside HTML ==> Skip these + 'a': handle_skip, + 'h1': handle_skip, + 'h2': handle_skip, + 'hr': handle_skip, + 'strong': handle_skip, + 'ul': handle_skip, + 'li': handle_skip, + 'p': handle_skip, + # Custom tags + 'videodev': handle_custom, + 'slides': handle_custom, + 'book': handle_custom, + 'image': handle_custom, + 'discuss': handle_custom, + # Simple lists + 'chapter': handle_list, + 'course': handle_list, + 'sequential': handle_list, + 'vertical': handle_list, + 'section': handle_list, + # True types + 'video': handle_video, + 'html': handle_html, + 'problem': handle_problem, + } + + for e in elements: + element_actions[e.tag](e) + + for k in results: + print k + create_item(k, 'Piotr Mitros') + if 'data' in results[k]: + update_item(k, results[k]['data']) + if 'children' in results[k]: + update_children(k, results[k]['children']) + + diff --git a/cms/djangoapps/contentstore/tests.py b/cms/djangoapps/contentstore/tests.py new file mode 100644 index 0000000000..501deb776c --- /dev/null +++ b/cms/djangoapps/contentstore/tests.py @@ -0,0 +1,16 @@ +""" +This file demonstrates writing tests using the unittest module. These will pass +when you run "manage.py test". + +Replace this with more appropriate tests for your application. +""" + +from django.test import TestCase + + +class SimpleTest(TestCase): + def test_basic_addition(self): + """ + Tests that 1 + 1 always equals 2. + """ + self.assertEqual(1 + 1, 2) diff --git a/cms/views.py b/cms/djangoapps/contentstore/views.py similarity index 100% rename from cms/views.py rename to cms/djangoapps/contentstore/views.py diff --git a/cms/envs/common.py b/cms/envs/common.py index 9b349a06d0..8d402b6fa9 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -144,8 +144,6 @@ INSTALLED_APPS = ( 'django.contrib.sites', 'django.contrib.messages', 'django.contrib.staticfiles', - # Uncomment the next line to enable the admin: - # 'django.contrib.admin', - # Uncomment the next line to enable admin documentation: - # 'django.contrib.admindocs', + 'contentstore', + 'instructor', ) diff --git a/cms/urls.py b/cms/urls.py index 55d7a1086e..a7266066cc 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -5,6 +5,6 @@ from django.conf.urls.defaults import patterns, url # admin.autodiscover() urlpatterns = patterns('', - url(r'^(?P[^/]+)/(?P[^/]+)/calendar/', 'cms.views.calendar', name='calendar'), + url(r'^(?P[^/]+)/(?P[^/]+)/calendar/', 'contentstore.views.calendar', name='calendar'), url(r'^accounts/login/', 'instructor.views.do_login', name='login'), )