Merge pull request #448 from MITx/feature/ichuang/add-gitreload-to-migrate
add gitreload to lms-migration code
This commit is contained in:
Calen Pennington
0
lms/djangoapps/lms_migration/management/__init__.py
Normal file
0
lms/djangoapps/lms_migration/management/__init__.py
Normal file
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/python
|
||||
#
|
||||
# File: create_groups.py
|
||||
#
|
||||
# Create all staff_* groups for classes in data directory.
|
||||
|
||||
import os, sys, string, re
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.conf import settings
|
||||
from django.contrib.auth.models import User, Group
|
||||
from path import path
|
||||
from lxml import etree
|
||||
|
||||
def create_groups():
|
||||
'''
|
||||
Create staff and instructor groups for all classes in the data_dir
|
||||
'''
|
||||
|
||||
data_dir = settings.DATA_DIR
|
||||
print "data_dir = %s" % data_dir
|
||||
|
||||
for course_dir in os.listdir(data_dir):
|
||||
|
||||
if course_dir.startswith('.'):
|
||||
continue
|
||||
if not os.path.isdir(path(data_dir) / course_dir):
|
||||
continue
|
||||
|
||||
cxfn = path(data_dir) / course_dir / 'course.xml'
|
||||
try:
|
||||
coursexml = etree.parse(cxfn)
|
||||
except Exception as err:
|
||||
print "Oops, cannot read %s, skipping" % cxfn
|
||||
continue
|
||||
cxmlroot = coursexml.getroot()
|
||||
course = cxmlroot.get('course') # TODO (vshnayder!!): read metadata from policy file(s) instead of from course.xml
|
||||
if course is None:
|
||||
print "oops, can't get course id for %s" % course_dir
|
||||
continue
|
||||
print "course=%s for course_dir=%s" % (course,course_dir)
|
||||
|
||||
create_group('staff_%s' % course) # staff group
|
||||
create_group('instructor_%s' % course) # instructor group (can manage staff group list)
|
||||
|
||||
def create_group(gname):
|
||||
if Group.objects.filter(name=gname):
|
||||
print " group exists for %s" % gname
|
||||
return
|
||||
g = Group(name=gname)
|
||||
g.save()
|
||||
print " created group %s" % gname
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Create groups associated with all courses in data_dir."
|
||||
|
||||
def handle(self, *args, **options):
|
||||
create_groups()
|
||||
146
lms/djangoapps/lms_migration/management/commands/create_user.py
Normal file
146
lms/djangoapps/lms_migration/management/commands/create_user.py
Normal file
@@ -0,0 +1,146 @@
|
||||
#!/usr/bin/python
|
||||
#
|
||||
# File: create_user.py
|
||||
#
|
||||
# Create user. Prompt for groups and ExternalAuthMap
|
||||
|
||||
import os, sys, string, re
|
||||
import datetime
|
||||
from getpass import getpass
|
||||
import json
|
||||
from random import choice
|
||||
import readline
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
from student.models import UserProfile, Registration
|
||||
from external_auth.models import ExternalAuthMap
|
||||
from django.contrib.auth.models import User, Group
|
||||
|
||||
class MyCompleter(object): # Custom completer
|
||||
|
||||
def __init__(self, options):
|
||||
self.options = sorted(options)
|
||||
|
||||
def complete(self, text, state):
|
||||
if state == 0: # on first trigger, build possible matches
|
||||
if text: # cache matches (entries that start with entered text)
|
||||
self.matches = [s for s in self.options
|
||||
if s and s.startswith(text)]
|
||||
else: # no text entered, all matches possible
|
||||
self.matches = self.options[:]
|
||||
|
||||
# return match indexed by state
|
||||
try:
|
||||
return self.matches[state]
|
||||
except IndexError:
|
||||
return None
|
||||
|
||||
def GenPasswd(length=8, chars=string.letters + string.digits):
|
||||
return ''.join([choice(chars) for i in range(length)])
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# main command
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Create user, interactively; can add ExternalAuthMap for MIT user if email@MIT.EDU resolves properly."
|
||||
|
||||
def handle(self, *args, **options):
|
||||
|
||||
while True:
|
||||
uname = raw_input('username: ')
|
||||
if User.objects.filter(username=uname):
|
||||
print "username %s already taken" % uname
|
||||
else:
|
||||
break
|
||||
|
||||
make_eamap = False
|
||||
if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y':
|
||||
email = '%s@MIT.EDU' % uname
|
||||
if not email.endswith('@MIT.EDU'):
|
||||
print "Failed - email must be @MIT.EDU"
|
||||
sys.exit(-1)
|
||||
mit_domain = 'ssl:MIT'
|
||||
if ExternalAuthMap.objects.filter(external_id = email, external_domain = mit_domain):
|
||||
print "Failed - email %s already exists as external_id" % email
|
||||
sys.exit(-1)
|
||||
make_eamap = True
|
||||
password = GenPasswd(12)
|
||||
|
||||
# get name from kerberos
|
||||
kname = os.popen("finger %s | grep 'name:'" % email).read().strip().split('name: ')[1].strip()
|
||||
name = raw_input('Full name: [%s] ' % kname).strip()
|
||||
if name=='':
|
||||
name = kname
|
||||
print "name = %s" % name
|
||||
else:
|
||||
while True:
|
||||
password = getpass()
|
||||
password2 = getpass()
|
||||
if password == password2:
|
||||
break
|
||||
print "Oops, passwords do not match, please retry"
|
||||
|
||||
while True:
|
||||
email = raw_input('email: ')
|
||||
if User.objects.filter(email=email):
|
||||
print "email %s already taken" % email
|
||||
else:
|
||||
break
|
||||
|
||||
name = raw_input('Full name: ')
|
||||
|
||||
|
||||
user = User(username=uname, email=email, is_active=True)
|
||||
user.set_password(password)
|
||||
try:
|
||||
user.save()
|
||||
except IntegrityError:
|
||||
print "Oops, failed to create user %s, IntegrityError" % user
|
||||
raise
|
||||
|
||||
r = Registration()
|
||||
r.register(user)
|
||||
|
||||
up = UserProfile(user=user)
|
||||
up.name = name
|
||||
up.save()
|
||||
|
||||
if make_eamap:
|
||||
credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name,email)
|
||||
eamap = ExternalAuthMap(external_id = email,
|
||||
external_email = email,
|
||||
external_domain = mit_domain,
|
||||
external_name = name,
|
||||
internal_password = password,
|
||||
external_credentials = json.dumps(credentials),
|
||||
)
|
||||
eamap.user = user
|
||||
eamap.dtsignup = datetime.datetime.now()
|
||||
eamap.save()
|
||||
|
||||
print "User %s created successfully!" % user
|
||||
|
||||
if not raw_input('Add user %s to any groups? [n] ' % user).lower()=='y':
|
||||
sys.exit(0)
|
||||
|
||||
print "Here are the groups available:"
|
||||
|
||||
groups = [str(g.name) for g in Group.objects.all()]
|
||||
print groups
|
||||
|
||||
completer = MyCompleter(groups)
|
||||
readline.set_completer(completer.complete)
|
||||
readline.parse_and_bind('tab: complete')
|
||||
|
||||
while True:
|
||||
gname = raw_input("Add group (tab to autocomplete, empty line to end): ")
|
||||
if not gname:
|
||||
break
|
||||
if not gname in groups:
|
||||
print "Unknown group %s" % gname
|
||||
continue
|
||||
g = Group.objects.get(name=gname)
|
||||
user.groups.add(g)
|
||||
print "Added %s to group %s" % (user,g)
|
||||
|
||||
print "Done!"
|
||||
@@ -2,13 +2,21 @@
|
||||
# migration tools for content team to go from stable-edx4edx to LMS+CMS
|
||||
#
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pprint import pprint
|
||||
import xmodule.modulestore.django as xmodule_django
|
||||
from xmodule.modulestore.django import modulestore
|
||||
|
||||
from django.http import HttpResponse
|
||||
from django.conf import settings
|
||||
import track.views
|
||||
|
||||
try:
|
||||
from django.views.decorators.csrf import csrf_exempt
|
||||
except ImportError:
|
||||
from django.contrib.csrf.middleware import csrf_exempt
|
||||
|
||||
log = logging.getLogger("mitx.lms_migrate")
|
||||
LOCAL_DEBUG = True
|
||||
@@ -18,6 +26,15 @@ def escape(s):
|
||||
"""escape HTML special characters in string"""
|
||||
return str(s).replace('<','<').replace('>','>')
|
||||
|
||||
def getip(request):
|
||||
'''
|
||||
Extract IP address of requester from header, even if behind proxy
|
||||
'''
|
||||
ip = request.META.get('HTTP_X_REAL_IP','') # nginx reverse proxy
|
||||
if not ip:
|
||||
ip = request.META.get('REMOTE_ADDR','None')
|
||||
return ip
|
||||
|
||||
def manage_modulestores(request,reload_dir=None):
|
||||
'''
|
||||
Manage the static in-memory modulestores.
|
||||
@@ -32,9 +49,7 @@ def manage_modulestores(request,reload_dir=None):
|
||||
#----------------------------------------
|
||||
# check on IP address of requester
|
||||
|
||||
ip = request.META.get('HTTP_X_REAL_IP','') # nginx reverse proxy
|
||||
if not ip:
|
||||
ip = request.META.get('REMOTE_ADDR','None')
|
||||
ip = getip(request)
|
||||
|
||||
if LOCAL_DEBUG:
|
||||
html += '<h3>IP address: %s ' % ip
|
||||
@@ -48,7 +63,7 @@ def manage_modulestores(request,reload_dir=None):
|
||||
html += 'Permission denied'
|
||||
html += "</body></html>"
|
||||
log.debug('request denied, ALLOWED_IPS=%s' % ALLOWED_IPS)
|
||||
return HttpResponse(html)
|
||||
return HttpResponse(html, status=403)
|
||||
|
||||
#----------------------------------------
|
||||
# reload course if specified
|
||||
@@ -108,3 +123,66 @@ def manage_modulestores(request,reload_dir=None):
|
||||
|
||||
html += "</body></html>"
|
||||
return HttpResponse(html)
|
||||
|
||||
@csrf_exempt
|
||||
def gitreload(request, reload_dir=None):
|
||||
'''
|
||||
This can be used as a github WebHook Service Hook, for reloading of the content repo used by the LMS.
|
||||
|
||||
If reload_dir is not None, then instruct the xml loader to reload that course directory.
|
||||
'''
|
||||
html = "<html><body>"
|
||||
ip = getip(request)
|
||||
|
||||
html += '<h3>IP address: %s ' % ip
|
||||
html += '<h3>User: %s ' % request.user
|
||||
|
||||
ALLOWED_IPS = [] # allow none by default
|
||||
if hasattr(settings,'ALLOWED_GITRELOAD_IPS'): # allow override in settings
|
||||
ALLOWED_IPS = ALLOWED_GITRELOAD_IPS
|
||||
|
||||
if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS):
|
||||
if request.user and request.user.is_staff:
|
||||
log.debug('request allowed because user=%s is staff' % request.user)
|
||||
else:
|
||||
html += 'Permission denied'
|
||||
html += "</body></html>"
|
||||
log.debug('request denied from %s, ALLOWED_IPS=%s' % (ip,ALLOWED_IPS))
|
||||
return HttpResponse(html)
|
||||
|
||||
#----------------------------------------
|
||||
# see if request is from github (POST with JSON)
|
||||
|
||||
if reload_dir is None and 'payload' in request.POST:
|
||||
payload = request.POST['payload']
|
||||
log.debug("payload=%s" % payload)
|
||||
gitargs = json.loads(payload)
|
||||
log.debug("gitargs=%s" % gitargs)
|
||||
reload_dir = gitargs['repository']['name']
|
||||
log.debug("github reload_dir=%s" % reload_dir)
|
||||
gdir = settings.DATA_DIR / reload_dir
|
||||
if not os.path.exists(gdir):
|
||||
log.debug("====> ERROR in gitreload - no such directory %s" % reload_dir)
|
||||
return HttpResponse('Error')
|
||||
cmd = "cd %s; git reset --hard HEAD; git clean -f -d; git pull origin; chmod g+w course.xml" % gdir
|
||||
log.debug(os.popen(cmd).read())
|
||||
if hasattr(settings,'GITRELOAD_HOOK'): # hit this hook after reload, if set
|
||||
gh = settings.GITRELOAD_HOOK
|
||||
if gh:
|
||||
ghurl = '%s/%s' % (gh,reload_dir)
|
||||
r = requests.get(ghurl)
|
||||
log.debug("GITRELOAD_HOOK to %s: %s" % (ghurl, r.text))
|
||||
|
||||
#----------------------------------------
|
||||
# reload course if specified
|
||||
|
||||
if reload_dir is not None:
|
||||
def_ms = modulestore()
|
||||
if reload_dir not in def_ms.courses:
|
||||
html += "<h2><font color='red'>Error: '%s' is not a valid course directory</font></h2>" % reload_dir
|
||||
else:
|
||||
html += "<h2><font color='blue'>Reloaded course directory '%s'</font></h2>" % reload_dir
|
||||
def_ms.try_load_course(reload_dir)
|
||||
track.views.server_track(request, 'reloaded %s' % reload_dir, {}, page='migrate')
|
||||
|
||||
return HttpResponse(html)
|
||||
|
||||
Reference in New Issue
Block a user