Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #448 from MITx/feature/ichuang/add-gitreload-to-migrate

Browse Source 
add gitreload to lms-migration code
This commit is contained in:
Calen Pennington
2012-08-21 12:52:52 -07:00
parent 61192e5280 65ea54f7e2
commit dca434ad34
16 changed files with 348 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
common/djangoapps/track/views.py
View File

@@ -1,6 +1,7 @@
import json
import logging
import os
import pytz
import datetime
import dateutil.parser
@@ -84,15 +85,33 @@ def server_track(request, event_type, event, page=None):
"time": datetime.datetime.utcnow().isoformat(),
}
if event_type=="/event_logs" and request.user.is_staff: # don't log
if event_type.startswith("/event_logs") and request.user.is_staff: # don't log
return
log_event(event)
@login_required
@ensure_csrf_cookie
def view_tracking_log(request):
def view_tracking_log(request,args=''):
if not request.user.is_staff:
return redirect('/')
record_instances = TrackingLog.objects.all().order_by('-time')[0:100]
nlen = 100
username = ''
if args:
for arg in args.split('/'):
if arg.isdigit():
nlen = int(arg)
if arg.startswith('username='):
username = arg[9:]
record_instances = TrackingLog.objects.all().order_by('-time')
if username:
record_instances = record_instances.filter(username=username)
record_instances = record_instances[0:nlen]
# fix dtstamp
fmt = '%a %d-%b-%y %H:%M:%S' # "%Y-%m-%d %H:%M:%S %Z%z"
for rinst in record_instances:
rinst.dtstr = rinst.time.replace(tzinfo=pytz.utc).astimezone(pytz.timezone('US/Eastern')).strftime(fmt)
return render_to_response('tracking_log.html',{'records':record_instances})

5
common/lib/capa/capa/inputtypes.py
View File

@@ -150,6 +150,7 @@ def optioninput(element, value, status, render_template, msg=''):
'state': status,
'msg': msg,
'options': osetdict,
'inline': element.get('inline',''),
}
html = render_template("optioninput.html", context)
@@ -294,7 +295,9 @@ def textline(element, value, status, render_template, msg=""):
hidden = element.get('hidden', '') # if specified, then textline is hidden and id is stored in div of name given by hidden
escapedict = {'"': '"'}
value = saxutils.escape(value, escapedict) # otherwise, answers with quotes in them crashes the system!
context = {'id': eid, 'value': value, 'state': status, 'count': count, 'size': size, 'msg': msg, 'hidden': hidden}
context = {'id': eid, 'value': value, 'state': status, 'count': count, 'size': size, 'msg': msg, 'hidden': hidden,
'inline': element.get('inline',''),
}
html = render_template("textinput.html", context)
try:
xhtml = etree.XML(html)

12
common/lib/capa/capa/templates/textinput.html
View File

@@ -1,12 +1,14 @@
<section id="textinput_${id}" class="textinput">
<% doinline = "inline" if inline else "" %>
<section id="textinput_${id}" class="textinput ${doinline}" >
% if state == 'unsubmitted':
<div class="unanswered" id="status_${id}">
<div class="unanswered ${doinline}" id="status_${id}">
% elif state == 'correct':
<div class="correct" id="status_${id}">
<div class="correct ${doinline}" id="status_${id}">
% elif state == 'incorrect':
<div class="incorrect" id="status_${id}">
<div class="incorrect ${doinline}" id="status_${id}">
% elif state == 'incomplete':
<div class="incorrect" id="status_${id}">
<div class="incorrect ${doinline}" id="status_${id}">
% endif
% if hidden:
<div style="display:none;" name="${hidden}" inputid="input_${id}" />

4
common/lib/xmodule/xmodule/css/capa/display.scss
View File

@@ -27,6 +27,10 @@ section.problem {
}
}
.inline {
display: inline;
}
div {
p {
&.answer {

0
lms/djangoapps/lms_migration/management/__init__.py Normal file
View File

0
lms/djangoapps/lms_migration/management/commands/__init__.py Normal file
View File

58
lms/djangoapps/lms_migration/management/commands/create_groups.py Normal file
View File

@@ -0,0 +1,58 @@
#!/usr/bin/python
#
# File: create_groups.py
#
# Create all staff_* groups for classes in data directory.
import os, sys, string, re
from django.core.management.base import BaseCommand
from django.conf import settings
from django.contrib.auth.models import User, Group
from path import path
from lxml import etree
def create_groups():
'''
Create staff and instructor groups for all classes in the data_dir
'''
data_dir = settings.DATA_DIR
print "data_dir = %s" % data_dir
for course_dir in os.listdir(data_dir):
if course_dir.startswith('.'):
continue
if not os.path.isdir(path(data_dir) / course_dir):
continue
cxfn = path(data_dir) / course_dir / 'course.xml'
try:
coursexml = etree.parse(cxfn)
except Exception as err:
print "Oops, cannot read %s, skipping" % cxfn
continue
cxmlroot = coursexml.getroot()
course = cxmlroot.get('course') # TODO (vshnayder!!): read metadata from policy file(s) instead of from course.xml
if course is None:
print "oops, can't get course id for %s" % course_dir
continue
print "course=%s for course_dir=%s" % (course,course_dir)
create_group('staff_%s' % course) # staff group
create_group('instructor_%s' % course) # instructor group (can manage staff group list)
def create_group(gname):
if Group.objects.filter(name=gname):
print " group exists for %s" % gname
return
g = Group(name=gname)
g.save()
print " created group %s" % gname
class Command(BaseCommand):
help = "Create groups associated with all courses in data_dir."
def handle(self, *args, **options):
create_groups()

146
lms/djangoapps/lms_migration/management/commands/create_user.py Normal file
View File

@@ -0,0 +1,146 @@
#!/usr/bin/python
#
# File: create_user.py
#
# Create user. Prompt for groups and ExternalAuthMap
import os, sys, string, re
import datetime
from getpass import getpass
import json
from random import choice
import readline
from django.core.management.base import BaseCommand
from student.models import UserProfile, Registration
from external_auth.models import ExternalAuthMap
from django.contrib.auth.models import User, Group
class MyCompleter(object): # Custom completer
def __init__(self, options):
self.options = sorted(options)
def complete(self, text, state):
if state == 0: # on first trigger, build possible matches
if text: # cache matches (entries that start with entered text)
self.matches = [s for s in self.options
if s and s.startswith(text)]
else: # no text entered, all matches possible
self.matches = self.options[:]
# return match indexed by state
try:
return self.matches[state]
except IndexError:
return None
def GenPasswd(length=8, chars=string.letters + string.digits):
return ''.join([choice(chars) for i in range(length)])
#-----------------------------------------------------------------------------
# main command
class Command(BaseCommand):
help = "Create user, interactively; can add ExternalAuthMap for MIT user if email@MIT.EDU resolves properly."
def handle(self, *args, **options):
while True:
uname = raw_input('username: ')
if User.objects.filter(username=uname):
print "username %s already taken" % uname
else:
break
make_eamap = False
if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y':
email = '%s@MIT.EDU' % uname
if not email.endswith('@MIT.EDU'):
print "Failed - email must be @MIT.EDU"
sys.exit(-1)
mit_domain = 'ssl:MIT'
if ExternalAuthMap.objects.filter(external_id = email, external_domain = mit_domain):
print "Failed - email %s already exists as external_id" % email
sys.exit(-1)
make_eamap = True
password = GenPasswd(12)
# get name from kerberos
kname = os.popen("finger %s | grep 'name:'" % email).read().strip().split('name: ')[1].strip()
name = raw_input('Full name: [%s] ' % kname).strip()
if name=='':
name = kname
print "name = %s" % name
else:
while True:
password = getpass()
password2 = getpass()
if password == password2:
break
print "Oops, passwords do not match, please retry"
while True:
email = raw_input('email: ')
if User.objects.filter(email=email):
print "email %s already taken" % email
else:
break
name = raw_input('Full name: ')
user = User(username=uname, email=email, is_active=True)
user.set_password(password)
try:
user.save()
except IntegrityError:
print "Oops, failed to create user %s, IntegrityError" % user
raise
r = Registration()
r.register(user)
up = UserProfile(user=user)
up.name = name
up.save()
if make_eamap:
credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name,email)
eamap = ExternalAuthMap(external_id = email,
external_email = email,
external_domain = mit_domain,
external_name = name,
internal_password = password,
external_credentials = json.dumps(credentials),
)
eamap.user = user
eamap.dtsignup = datetime.datetime.now()
eamap.save()
print "User %s created successfully!" % user
if not raw_input('Add user %s to any groups? [n] ' % user).lower()=='y':
sys.exit(0)
print "Here are the groups available:"
groups = [str(g.name) for g in Group.objects.all()]
print groups
completer = MyCompleter(groups)
readline.set_completer(completer.complete)
readline.parse_and_bind('tab: complete')
while True:
gname = raw_input("Add group (tab to autocomplete, empty line to end): ")
if not gname:
break
if not gname in groups:
print "Unknown group %s" % gname
continue
g = Group.objects.get(name=gname)
user.groups.add(g)
print "Added %s to group %s" % (user,g)
print "Done!"

86
lms/djangoapps/lms_migration/migrate.py
View File

@@ -2,13 +2,21 @@
# migration tools for content team to go from stable-edx4edx to LMS+CMS
#
import json
import logging
import os
from pprint import pprint
import xmodule.modulestore.django as xmodule_django
from xmodule.modulestore.django import modulestore
from django.http import HttpResponse
from django.conf import settings
import track.views
try:
from django.views.decorators.csrf import csrf_exempt
except ImportError:
from django.contrib.csrf.middleware import csrf_exempt
log = logging.getLogger("mitx.lms_migrate")
LOCAL_DEBUG = True
@@ -18,6 +26,15 @@ def escape(s):
"""escape HTML special characters in string"""
return str(s).replace('<','&lt;').replace('>','&gt;')
def getip(request):
'''
Extract IP address of requester from header, even if behind proxy
'''
ip = request.META.get('HTTP_X_REAL_IP','') # nginx reverse proxy
if not ip:
ip = request.META.get('REMOTE_ADDR','None')
return ip
def manage_modulestores(request,reload_dir=None):
'''
Manage the static in-memory modulestores.
@@ -32,9 +49,7 @@ def manage_modulestores(request,reload_dir=None):
#----------------------------------------
# check on IP address of requester
ip = request.META.get('HTTP_X_REAL_IP','') # nginx reverse proxy
if not ip:
ip = request.META.get('REMOTE_ADDR','None')
ip = getip(request)
if LOCAL_DEBUG:
html += '<h3>IP address: %s ' % ip
@@ -48,7 +63,7 @@ def manage_modulestores(request,reload_dir=None):
html += 'Permission denied'
html += "</body></html>"
log.debug('request denied, ALLOWED_IPS=%s' % ALLOWED_IPS)
return HttpResponse(html)
return HttpResponse(html, status=403)
#----------------------------------------
# reload course if specified
@@ -108,3 +123,66 @@ def manage_modulestores(request,reload_dir=None):
html += "</body></html>"
return HttpResponse(html)
@csrf_exempt
def gitreload(request, reload_dir=None):
'''
This can be used as a github WebHook Service Hook, for reloading of the content repo used by the LMS.
If reload_dir is not None, then instruct the xml loader to reload that course directory.
'''
html = "<html><body>"
ip = getip(request)
html += '<h3>IP address: %s ' % ip
html += '<h3>User: %s ' % request.user
ALLOWED_IPS = [] # allow none by default
if hasattr(settings,'ALLOWED_GITRELOAD_IPS'): # allow override in settings
ALLOWED_IPS = ALLOWED_GITRELOAD_IPS
if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS):
if request.user and request.user.is_staff:
log.debug('request allowed because user=%s is staff' % request.user)
else:
html += 'Permission denied'
html += "</body></html>"
log.debug('request denied from %s, ALLOWED_IPS=%s' % (ip,ALLOWED_IPS))
return HttpResponse(html)
#----------------------------------------
# see if request is from github (POST with JSON)
if reload_dir is None and 'payload' in request.POST:
payload = request.POST['payload']
log.debug("payload=%s" % payload)
gitargs = json.loads(payload)
log.debug("gitargs=%s" % gitargs)
reload_dir = gitargs['repository']['name']
log.debug("github reload_dir=%s" % reload_dir)
gdir = settings.DATA_DIR / reload_dir
if not os.path.exists(gdir):
log.debug("====> ERROR in gitreload - no such directory %s" % reload_dir)
return HttpResponse('Error')
cmd = "cd %s; git reset --hard HEAD; git clean -f -d; git pull origin; chmod g+w course.xml" % gdir
log.debug(os.popen(cmd).read())
if hasattr(settings,'GITRELOAD_HOOK'): # hit this hook after reload, if set
gh = settings.GITRELOAD_HOOK
if gh:
ghurl = '%s/%s' % (gh,reload_dir)
r = requests.get(ghurl)
log.debug("GITRELOAD_HOOK to %s: %s" % (ghurl, r.text))
#----------------------------------------
# reload course if specified
if reload_dir is not None:
def_ms = modulestore()
if reload_dir not in def_ms.courses:
html += "<h2><font color='red'>Error: '%s' is not a valid course directory</font></h2>" % reload_dir
else:
html += "<h2><font color='blue'>Reloaded course directory '%s'</font></h2>" % reload_dir
def_ms.try_load_course(reload_dir)
track.views.server_track(request, 'reloaded %s' % reload_dir, {}, page='migrate')
return HttpResponse(html)

8
lms/envs/common.py
View File

@@ -260,6 +260,14 @@ USE_L10N = True
# Messages
MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage'
#################################### GITHUB #######################################
# gitreload is used in LMS-workflow to pull content from github
# gitreload requests are only allowed from these IP addresses, which are
# the advertised public IPs of the github WebHook servers.
# These are listed, eg at https://github.com/MITx/mitx/admin/hooks
ALLOWED_GITRELOAD_IPS = ['207.97.227.253', '50.57.128.197', '108.171.174.178']
#################################### AWS #######################################
# S3BotoStorage insists on a timeout for uploaded assets. We should make it
# permanent instead, but rather than trying to figure out exactly where that

2
lms/envs/dev.py
View File

@@ -73,6 +73,8 @@ MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True
MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = False # require that user be in the staff_* group to be able to enroll
MITX_FEATURES['USE_XQA_SERVER'] = 'http://xqa:server@content-qa.mitx.mit.edu/xqa'
INSTALLED_APPS += ('lms_migration',)
LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1']
################################ OpenID Auth #################################

12
lms/envs/dev_ike.py
View File

@@ -17,14 +17,19 @@ MITX_FEATURES['ENABLE_TEXTBOOK'] = False
MITX_FEATURES['ENABLE_DISCUSSION'] = False
MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True # require that user be in the staff_* group to be able to enroll
MITX_FEATURES['DISABLE_START_DATES'] = True
# MITX_FEATURES['USE_DJANGO_PIPELINE']=False # don't recompile scss
myhost = socket.gethostname()
if ('edxvm' in myhost) or ('ocw' in myhost):
MITX_FEATURES['DISABLE_LOGIN_BUTTON'] = True # auto-login with MIT certificate
MITX_FEATURES['USE_XQA_SERVER'] = 'https://qisx.mit.edu/xqa' # needs to be ssl or browser blocks it
MITX_FEATURES['USE_DJANGO_PIPELINE']=False # don't recompile scss
if ('domU' in myhost):
EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
MITX_FEATURES['REROUTE_ACTIVATION_EMAIL'] = 'ichuang@mitx.mit.edu' # nonempty string = address for all activation emails
MITX_FEATURES['USE_DJANGO_PIPELINE']=False # don't recompile scss
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTOCOL', 'https') # django 1.4 for nginx ssl proxy
@@ -33,4 +38,9 @@ SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTOCOL', 'https') # django 1.4 fo
INSTALLED_APPS = tuple([ app for app in INSTALLED_APPS if not app.startswith('debug_toolbar') ])
MIDDLEWARE_CLASSES = tuple([ mcl for mcl in MIDDLEWARE_CLASSES if not mcl.startswith('debug_toolbar') ])
TEMPLATE_LOADERS = tuple([ app for app in TEMPLATE_LOADERS if not app.startswith('askbot') ])
#TEMPLATE_LOADERS = tuple([ app for app in TEMPLATE_LOADERS if not app.startswith('askbot') ])
#TEMPLATE_LOADERS = tuple([ app for app in TEMPLATE_LOADERS if not app.startswith('mitxmako') ])
TEMPLATE_LOADERS = (
'django.template.loaders.filesystem.Loader',
'django.template.loaders.app_directories.Loader',
)

2
lms/templates/tracking_log.html
View File

@@ -3,7 +3,7 @@
<table border="1"><tr><th>datetime</th><th>username</th><th>ipaddr</th><th>source</th><th>type</th></tr>
% for rec in records:
<tr>
<td>${rec.time}</td>
<td>${rec.dtstr}</td>
<td>${rec.username}</td>
<td>${rec.ip}</td>
<td>${rec.event_source}</td>

3
lms/urls.py
View File

@@ -217,11 +217,14 @@ if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'):
urlpatterns += (
url(r'^migrate/modules$', 'lms_migration.migrate.manage_modulestores'),
url(r'^migrate/reload/(?P<reload_dir>[^/]+)$', 'lms_migration.migrate.manage_modulestores'),
url(r'^gitreload$', 'lms_migration.migrate.gitreload'),
url(r'^gitreload/(?P<reload_dir>[^/]+)$', 'lms_migration.migrate.gitreload'),
)
if settings.MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'):
urlpatterns += (
url(r'^event_logs$', 'track.views.view_tracking_log'),
url(r'^event_logs/(?P<args>.+)$', 'track.views.view_tracking_log'),
)
urlpatterns = patterns(*urlpatterns)

46
utility-scripts/create_groups.py
View File

@@ -1,46 +0,0 @@
#!/usr/bin/python
#
# File: create_groups.py
#
# Create all staff_* groups for classes in data directory.
import os, sys, string, re
sys.path.append(os.path.abspath('.'))
os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev'
try:
from lms.envs.dev import *
except Exception as err:
print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory."
sys.exit(-1)
from django.conf import settings
from django.contrib.auth.models import User, Group
from path import path
from lxml import etree
data_dir = settings.DATA_DIR
print "data_dir = %s" % data_dir
for course_dir in os.listdir(data_dir):
# print course_dir
if not os.path.isdir(path(data_dir) / course_dir):
continue
cxfn = path(data_dir) / course_dir / 'course.xml'
coursexml = etree.parse(cxfn)
cxmlroot = coursexml.getroot()
course = cxmlroot.get('course')
if course is None:
print "oops, can't get course id for %s" % course_dir
continue
print "course=%s for course_dir=%s" % (course,course_dir)
gname = 'staff_%s' % course
if Group.objects.filter(name=gname):
print "group exists for %s" % gname
continue
g = Group(name=gname)
g.save()
print "created group %s" % gname

149
utility-scripts/create_user.py
View File

@@ -1,149 +0,0 @@
#!/usr/bin/python
#
# File: create_user.py
#
# Create user. Prompt for groups and ExternalAuthMap
import os, sys, string, re
import datetime
from getpass import getpass
import json
import readline
sys.path.append(os.path.abspath('.'))
os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev'
try:
from lms.envs.dev import *
except Exception as err:
print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory."
sys.exit(-1)
from student.models import UserProfile, Registration
from external_auth.models import ExternalAuthMap
from django.contrib.auth.models import User, Group
from random import choice
class MyCompleter(object): # Custom completer
def __init__(self, options):
self.options = sorted(options)
def complete(self, text, state):
if state == 0: # on first trigger, build possible matches
if text: # cache matches (entries that start with entered text)
self.matches = [s for s in self.options
if s and s.startswith(text)]
else: # no text entered, all matches possible
self.matches = self.options[:]
# return match indexed by state
try:
return self.matches[state]
except IndexError:
return None
def GenPasswd(length=8, chars=string.letters + string.digits):
return ''.join([choice(chars) for i in range(length)])
#-----------------------------------------------------------------------------
# main
while True:
uname = raw_input('username: ')
if User.objects.filter(username=uname):
print "username %s already taken" % uname
else:
break
make_eamap = False
if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y':
email = '%s@MIT.EDU' % uname
if not email.endswith('@MIT.EDU'):
print "Failed - email must be @MIT.EDU"
sys.exit(-1)
mit_domain = 'ssl:MIT'
if ExternalAuthMap.objects.filter(external_id = email, external_domain = mit_domain):
print "Failed - email %s already exists as external_id" % email
sys.exit(-1)
make_eamap = True
password = GenPasswd(12)
# get name from kerberos
kname = os.popen("finger %s | grep 'name:'" % email).read().strip().split('name: ')[1].strip()
name = raw_input('Full name: [%s] ' % kname).strip()
if name=='':
name = kname
print "name = %s" % name
else:
while True:
password = getpass()
password2 = getpass()
if password == password2:
break
print "Oops, passwords do not match, please retry"
while True:
email = raw_input('email: ')
if User.objects.filter(email=email):
print "email %s already taken" % email
else:
break
name = raw_input('Full name: ')
user = User(username=uname, email=email, is_active=True)
user.set_password(password)
try:
user.save()
except IntegrityError:
print "Oops, failed to create user %s, IntegrityError" % user
raise
r = Registration()
r.register(user)
up = UserProfile(user=user)
up.name = name
up.save()
if make_eamap:
credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name,email)
eamap = ExternalAuthMap(external_id = email,
external_email = email,
external_domain = mit_domain,
external_name = name,
internal_password = password,
external_credentials = json.dumps(credentials),
)
eamap.user = user
eamap.dtsignup = datetime.datetime.now()
eamap.save()
print "User %s created successfully!" % user
if not raw_input('Add user %s to any groups? [n] ' % user).lower()=='y':
sys.exit(0)
print "Here are the groups available:"
groups = [str(g.name) for g in Group.objects.all()]
print groups
completer = MyCompleter(groups)
readline.set_completer(completer.complete)
readline.parse_and_bind('tab: complete')
while True:
gname = raw_input("Add group (tab to autocomplete, empty line to end): ")
if not gname:
break
if not gname in groups:
print "Unknown group %s" % gname
continue
g = Group.objects.get(name=gname)
user.groups.add(g)
print "Added %s to group %s" % (user,g)
print "Done!"