Merge pull request #18985 from edx/feanil/remove_lms_migrate
Remove tools that don't seem to be used.
This commit is contained in:
Feanil Patel
@@ -1,61 +0,0 @@
|
||||
#!/usr/bin/python
|
||||
#
|
||||
# File: create_groups.py
|
||||
#
|
||||
# Create all staff_* groups for classes in data directory.
|
||||
|
||||
import os
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib.auth.models import Group
|
||||
from django.core.management.base import BaseCommand
|
||||
from lxml import etree
|
||||
from path import Path as path
|
||||
|
||||
|
||||
def create_groups():
|
||||
'''
|
||||
Create staff and instructor groups for all classes in the data_dir
|
||||
'''
|
||||
|
||||
data_dir = settings.DATA_DIR
|
||||
print "data_dir = %s" % data_dir
|
||||
|
||||
for course_dir in os.listdir(data_dir):
|
||||
|
||||
if course_dir.startswith('.'):
|
||||
continue
|
||||
if not os.path.isdir(path(data_dir) / course_dir):
|
||||
continue
|
||||
|
||||
cxfn = path(data_dir) / course_dir / 'course.xml'
|
||||
try:
|
||||
coursexml = etree.parse(cxfn)
|
||||
except Exception: # pylint: disable=broad-except
|
||||
print "Oops, cannot read %s, skipping" % cxfn
|
||||
continue
|
||||
cxmlroot = coursexml.getroot()
|
||||
course = cxmlroot.get('course') # TODO (vshnayder!!): read metadata from policy file(s) instead of from course.xml
|
||||
if course is None:
|
||||
print "oops, can't get course id for %s" % course_dir
|
||||
continue
|
||||
print "course=%s for course_dir=%s" % (course, course_dir)
|
||||
|
||||
create_group('staff_%s' % course) # staff group
|
||||
create_group('instructor_%s' % course) # instructor group (can manage staff group list)
|
||||
|
||||
|
||||
def create_group(gname):
|
||||
if Group.objects.filter(name=gname):
|
||||
print " group exists for %s" % gname
|
||||
return
|
||||
g = Group(name=gname)
|
||||
g.save()
|
||||
print " created group %s" % gname
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Create groups associated with all courses in data_dir."
|
||||
|
||||
def handle(self, *args, **options):
|
||||
create_groups()
|
||||
@@ -1,159 +0,0 @@
|
||||
#!/usr/bin/python
|
||||
#
|
||||
# File: create_user.py
|
||||
#
|
||||
# Create user. Prompt for groups and ExternalAuthMap
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import readline
|
||||
import string
|
||||
import sys
|
||||
from getpass import getpass
|
||||
from random import choice
|
||||
|
||||
from django.contrib.auth.models import Group, User
|
||||
from django.core.management.base import BaseCommand
|
||||
from pytz import UTC
|
||||
|
||||
from openedx.core.djangoapps.external_auth.models import ExternalAuthMap
|
||||
from student.models import Registration, UserProfile, email_exists_or_retired
|
||||
|
||||
|
||||
class MyCompleter(object): # Custom completer
|
||||
|
||||
def __init__(self, options):
|
||||
self.options = sorted(options)
|
||||
|
||||
def complete(self, text, state):
|
||||
if state == 0: # on first trigger, build possible matches
|
||||
if text: # cache matches (entries that start with entered text)
|
||||
self.matches = [
|
||||
option
|
||||
for option in self.options
|
||||
if option and option.startswith(text)
|
||||
]
|
||||
else: # no text entered, all matches possible
|
||||
self.matches = self.options[:]
|
||||
|
||||
# return match indexed by state
|
||||
try:
|
||||
return self.matches[state]
|
||||
except IndexError:
|
||||
return None
|
||||
|
||||
|
||||
def GenPasswd(length=8, chars=string.letters + string.digits):
|
||||
return ''.join([choice(chars) for dummy0 in range(length)])
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# main command
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Create user, interactively; can add ExternalAuthMap for MIT user if email@MIT.EDU resolves properly."
|
||||
|
||||
def handle(self, *args, **options):
|
||||
|
||||
while True:
|
||||
uname = raw_input('username: ')
|
||||
if User.objects.filter(username=uname):
|
||||
print "username %s already taken" % uname
|
||||
else:
|
||||
break
|
||||
|
||||
make_eamap = False
|
||||
if raw_input('Create MIT ExternalAuth? [n] ').lower() == 'y':
|
||||
email = '%s@MIT.EDU' % uname
|
||||
if not email.endswith('@MIT.EDU'):
|
||||
print "Failed - email must be @MIT.EDU"
|
||||
sys.exit(-1)
|
||||
mit_domain = 'ssl:MIT'
|
||||
if ExternalAuthMap.objects.filter(external_id=email, external_domain=mit_domain):
|
||||
print "Failed - email %s already exists as external_id" % email
|
||||
sys.exit(-1)
|
||||
make_eamap = True
|
||||
password = GenPasswd(12)
|
||||
|
||||
# get name from kerberos
|
||||
try:
|
||||
kname = os.popen("finger %s | grep 'name:'" % email).read().strip().split('name: ')[1].strip()
|
||||
except:
|
||||
kname = ''
|
||||
name = raw_input('Full name: [%s] ' % kname).strip()
|
||||
if name == '':
|
||||
name = kname
|
||||
print "name = %s" % name
|
||||
else:
|
||||
while True:
|
||||
password = getpass()
|
||||
password2 = getpass()
|
||||
if password == password2:
|
||||
break
|
||||
print "Oops, passwords do not match, please retry"
|
||||
|
||||
while True:
|
||||
email = raw_input('email: ')
|
||||
if email_exists_or_retired(email):
|
||||
print "email %s already taken" % email
|
||||
else:
|
||||
break
|
||||
|
||||
name = raw_input('Full name: ')
|
||||
|
||||
user = User(username=uname, email=email, is_active=True)
|
||||
user.set_password(password)
|
||||
try:
|
||||
user.save()
|
||||
except IntegrityError:
|
||||
print "Oops, failed to create user %s, IntegrityError" % user
|
||||
raise
|
||||
|
||||
r = Registration()
|
||||
r.register(user)
|
||||
|
||||
up = UserProfile(user=user)
|
||||
up.name = name
|
||||
up.save()
|
||||
|
||||
if make_eamap:
|
||||
credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name, email)
|
||||
eamap = ExternalAuthMap(
|
||||
external_id=email,
|
||||
external_email=email,
|
||||
external_domain=mit_domain,
|
||||
external_name=name,
|
||||
internal_password=password,
|
||||
external_credentials=json.dumps(credentials),
|
||||
)
|
||||
eamap.user = user
|
||||
eamap.dtsignup = datetime.datetime.now(UTC)
|
||||
eamap.save()
|
||||
|
||||
print "User %s created successfully!" % user
|
||||
|
||||
if not raw_input('Add user %s to any groups? [n] ' % user).lower() == 'y':
|
||||
sys.exit(0)
|
||||
|
||||
print "Here are the groups available:"
|
||||
|
||||
groups = [str(g.name) for g in Group.objects.all()]
|
||||
print groups
|
||||
|
||||
completer = MyCompleter(groups)
|
||||
readline.set_completer(completer.complete)
|
||||
readline.parse_and_bind('tab: complete')
|
||||
|
||||
while True:
|
||||
gname = raw_input("Add group (tab to autocomplete, empty line to end): ")
|
||||
if not gname:
|
||||
break
|
||||
if gname not in groups:
|
||||
print "Unknown group %s" % gname
|
||||
continue
|
||||
g = Group.objects.get(name=gname)
|
||||
user.groups.add(g)
|
||||
print "Added %s to group %s" % (user, g)
|
||||
|
||||
print "Done!"
|
||||
@@ -1,70 +0,0 @@
|
||||
#!/usr/bin/python
|
||||
#
|
||||
# File: manage_course_groups
|
||||
#
|
||||
# interactively list and edit membership in course staff and instructor groups
|
||||
|
||||
import re
|
||||
|
||||
from django.contrib.auth.models import Group, User
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# get all staff groups
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Manage course group membership, interactively."
|
||||
|
||||
def handle(self, *args, **options):
|
||||
|
||||
gset = Group.objects.all()
|
||||
|
||||
print "Groups:"
|
||||
for cnt, g in zip(range(len(gset)), gset):
|
||||
print "%d. %s" % (cnt, g)
|
||||
|
||||
gnum = int(raw_input('Choose group to manage (enter #): '))
|
||||
|
||||
group = gset[gnum]
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# users in group
|
||||
|
||||
uall = User.objects.all()
|
||||
if uall.count() < 50:
|
||||
print "----"
|
||||
print "List of All Users: %s" % [str(x.username) for x in uall]
|
||||
print "----"
|
||||
else:
|
||||
print "----"
|
||||
print "There are %d users, which is too many to list" % uall.count()
|
||||
print "----"
|
||||
|
||||
while True:
|
||||
|
||||
print "Users in the group:"
|
||||
|
||||
uset = group.user_set.all()
|
||||
for cnt, u in zip(range(len(uset)), uset):
|
||||
print "%d. %s" % (cnt, u)
|
||||
|
||||
action = raw_input('Choose user to delete (enter #) or enter usernames (comma delim) to add: ')
|
||||
|
||||
m = re.match('^[0-9]+$', action)
|
||||
if m:
|
||||
unum = int(action)
|
||||
u = uset[unum]
|
||||
print "Deleting user %s" % u
|
||||
u.groups.remove(group)
|
||||
|
||||
else:
|
||||
for uname in action.split(','):
|
||||
try:
|
||||
user = User.objects.get(username=action)
|
||||
except Exception as err:
|
||||
print "Error %s" % err
|
||||
continue
|
||||
print "adding %s to group %s" % (user, group)
|
||||
user.groups.add(group)
|
||||
@@ -1,232 +0,0 @@
|
||||
#
|
||||
# migration tools for content team to go from stable-edx4edx to LMS+CMS
|
||||
#
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
||||
from django.conf import settings
|
||||
from django.http import HttpResponse
|
||||
from six import text_type
|
||||
|
||||
import track.views
|
||||
import xmodule.modulestore.django as xmodule_django
|
||||
from xmodule.modulestore.django import modulestore
|
||||
|
||||
try:
|
||||
from django.views.decorators.csrf import csrf_exempt
|
||||
except ImportError:
|
||||
from django.contrib.csrf.middleware import csrf_exempt
|
||||
|
||||
log = logging.getLogger("edx.lms_migrate")
|
||||
LOCAL_DEBUG = True
|
||||
ALLOWED_IPS = settings.LMS_MIGRATION_ALLOWED_IPS
|
||||
|
||||
|
||||
def escape(s):
|
||||
"""escape HTML special characters in string"""
|
||||
return str(s).replace('<', '<').replace('>', '>')
|
||||
|
||||
|
||||
def getip(request):
|
||||
'''
|
||||
Extract IP address of requester from header, even if behind proxy
|
||||
'''
|
||||
ip = request.META.get('HTTP_X_REAL_IP', '') # nginx reverse proxy
|
||||
if not ip:
|
||||
ip = request.META.get('REMOTE_ADDR', 'None')
|
||||
return ip
|
||||
|
||||
|
||||
def get_commit_id(course):
|
||||
#return course.metadata.get('GIT_COMMIT_ID', 'No commit id')
|
||||
return getattr(course, 'GIT_COMMIT_ID', 'No commit id')
|
||||
# getattr(def_ms.courses[reload_dir], 'GIT_COMMIT_ID','No commit id')
|
||||
|
||||
|
||||
def set_commit_id(course, commit_id):
|
||||
#course.metadata['GIT_COMMIT_ID'] = commit_id
|
||||
course.GIT_COMMIT_ID = commit_id
|
||||
# def_ms.courses[reload_dir].GIT_COMMIT_ID = new_commit_id
|
||||
|
||||
|
||||
def manage_modulestores(request, reload_dir=None, commit_id=None):
|
||||
'''
|
||||
Manage the static in-memory modulestores.
|
||||
|
||||
If reload_dir is not None, then instruct the xml loader to reload that course directory.
|
||||
'''
|
||||
html = "<html><body>"
|
||||
|
||||
def_ms = modulestore()
|
||||
courses = def_ms.get_courses()
|
||||
|
||||
#----------------------------------------
|
||||
# check on IP address of requester
|
||||
|
||||
ip = getip(request)
|
||||
|
||||
if LOCAL_DEBUG:
|
||||
html += '<h3>IP address: %s <h3>' % ip
|
||||
html += '<h3>User: %s </h3>' % request.user
|
||||
html += '<h3>My pid: %s</h3>' % os.getpid()
|
||||
log.debug(u'request from ip=%s, user=%s', ip, request.user)
|
||||
|
||||
if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS):
|
||||
if request.user and request.user.is_staff:
|
||||
log.debug(u'request allowed because user=%s is staff', request.user)
|
||||
else:
|
||||
html += 'Permission denied'
|
||||
html += "</body></html>"
|
||||
log.debug('request denied, ALLOWED_IPS=%s', ALLOWED_IPS)
|
||||
return HttpResponse(html, status=403)
|
||||
|
||||
#----------------------------------------
|
||||
# reload course if specified; handle optional commit_id
|
||||
|
||||
if reload_dir is not None:
|
||||
if reload_dir not in def_ms.courses:
|
||||
html += '<h2 class="inline-error">Error: "%s" is not a valid course directory</h2>' % reload_dir
|
||||
else:
|
||||
# reloading based on commit_id is needed when running mutiple worker threads,
|
||||
# so that a given thread doesn't reload the same commit multiple times
|
||||
current_commit_id = get_commit_id(def_ms.courses[reload_dir])
|
||||
log.debug('commit_id="%s"', commit_id)
|
||||
log.debug('current_commit_id="%s"', current_commit_id)
|
||||
|
||||
if (commit_id is not None) and (commit_id == current_commit_id):
|
||||
html += "<h2>Already at commit id %s for %s</h2>" % (commit_id, reload_dir)
|
||||
track.views.server_track(request,
|
||||
'reload %s skipped already at %s (pid=%s)' % (reload_dir,
|
||||
commit_id,
|
||||
os.getpid(),
|
||||
),
|
||||
{}, page='migrate')
|
||||
else:
|
||||
html += '<h2>Reloaded course directory "%s"</h2>' % reload_dir
|
||||
def_ms.try_load_course(reload_dir)
|
||||
gdir = settings.DATA_DIR / reload_dir
|
||||
new_commit_id = os.popen('cd %s; git log -n 1 | head -1' % gdir).read().strip().split(' ')[1]
|
||||
set_commit_id(def_ms.courses[reload_dir], new_commit_id)
|
||||
html += '<p>commit_id=%s</p>' % new_commit_id
|
||||
track.views.server_track(request, 'reloaded %s now at %s (pid=%s)' % (reload_dir,
|
||||
new_commit_id,
|
||||
os.getpid()), {}, page='migrate')
|
||||
|
||||
#----------------------------------------
|
||||
|
||||
html += '<h2>Courses loaded in the modulestore</h2>'
|
||||
html += '<ol>'
|
||||
for cdir, course in def_ms.courses.items():
|
||||
html += '<li><a href="%s/migrate/reload/%s">%s</a> (%s)</li>' % (
|
||||
settings.EDX_ROOT_URL,
|
||||
escape(cdir),
|
||||
escape(cdir),
|
||||
text_type(course.location)
|
||||
)
|
||||
html += '</ol>'
|
||||
|
||||
#----------------------------------------
|
||||
|
||||
#dumpfields = ['definition', 'location', 'metadata']
|
||||
dumpfields = ['location', 'metadata']
|
||||
|
||||
for cdir, course in def_ms.courses.items():
|
||||
html += '<hr width="100%"/>'
|
||||
html += '<h2>Course: %s (%s)</h2>' % (course.display_name_with_default_escaped, cdir)
|
||||
|
||||
html += '<p>commit_id=%s</p>' % get_commit_id(course)
|
||||
|
||||
for field in dumpfields:
|
||||
data = getattr(course, field, None)
|
||||
html += '<h3>%s</h3>' % field
|
||||
if isinstance(data, dict):
|
||||
html += '<ul>'
|
||||
for k, v in data.items():
|
||||
html += '<li>%s:%s</li>' % (escape(k), escape(v))
|
||||
html += '</ul>'
|
||||
else:
|
||||
html += '<ul><li>%s</li></ul>' % escape(data)
|
||||
|
||||
#----------------------------------------
|
||||
|
||||
html += '<hr width="100%"/>'
|
||||
html += "courses: <pre>%s</pre>" % escape(courses)
|
||||
|
||||
ms = xmodule_django._MODULESTORES
|
||||
html += "modules: <pre>%s</pre>" % escape(ms)
|
||||
html += "default modulestore: <pre>%s</pre>" % escape(unicode(def_ms))
|
||||
|
||||
#----------------------------------------
|
||||
|
||||
log.debug('_MODULESTORES=%s', ms)
|
||||
log.debug('courses=%s', courses)
|
||||
log.debug('def_ms=%s', unicode(def_ms))
|
||||
|
||||
html += "</body></html>"
|
||||
return HttpResponse(html)
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
def gitreload(request, reload_dir=None):
|
||||
'''
|
||||
This can be used as a github WebHook Service Hook, for reloading of the content repo used by the LMS.
|
||||
|
||||
If reload_dir is not None, then instruct the xml loader to reload that course directory.
|
||||
'''
|
||||
html = "<html><body>"
|
||||
ip = getip(request)
|
||||
|
||||
html += '<h3>IP address: %s ' % ip
|
||||
html += '<h3>User: %s ' % request.user
|
||||
|
||||
ALLOWED_IPS = [] # allow none by default
|
||||
if hasattr(settings, 'ALLOWED_GITRELOAD_IPS'): # allow override in settings
|
||||
ALLOWED_IPS = settings.ALLOWED_GITRELOAD_IPS
|
||||
|
||||
if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS):
|
||||
if request.user and request.user.is_staff:
|
||||
log.debug(u'request allowed because user=%s is staff', request.user)
|
||||
else:
|
||||
html += 'Permission denied'
|
||||
html += "</body></html>"
|
||||
log.debug('request denied from %s, ALLOWED_IPS=%s', ip, ALLOWED_IPS)
|
||||
return HttpResponse(html)
|
||||
|
||||
#----------------------------------------
|
||||
# see if request is from github (POST with JSON)
|
||||
|
||||
if reload_dir is None and 'payload' in request.POST:
|
||||
payload = request.POST['payload']
|
||||
log.debug("payload=%s", payload)
|
||||
gitargs = json.loads(payload)
|
||||
log.debug("gitargs=%s", gitargs)
|
||||
reload_dir = gitargs['repository']['name']
|
||||
log.debug("github reload_dir=%s", reload_dir)
|
||||
gdir = settings.DATA_DIR / reload_dir
|
||||
if not os.path.exists(gdir):
|
||||
log.debug("====> ERROR in gitreload - no such directory %s", reload_dir)
|
||||
return HttpResponse('Error')
|
||||
cmd = "cd %s; git reset --hard HEAD; git clean -f -d; git pull origin; chmod g+w course.xml" % gdir
|
||||
log.debug(os.popen(cmd).read())
|
||||
if hasattr(settings, 'GITRELOAD_HOOK'): # hit this hook after reload, if set
|
||||
gh = settings.GITRELOAD_HOOK
|
||||
if gh:
|
||||
ghurl = '%s/%s' % (gh, reload_dir)
|
||||
r = requests.get(ghurl)
|
||||
log.debug("GITRELOAD_HOOK to %s: %s", ghurl, r.text)
|
||||
|
||||
#----------------------------------------
|
||||
# reload course if specified
|
||||
|
||||
if reload_dir is not None:
|
||||
def_ms = modulestore()
|
||||
if reload_dir not in def_ms.courses:
|
||||
html += '<h2 class="inline-error">Error: "%s" is not a valid course directory</font></h2>' % reload_dir
|
||||
else:
|
||||
html += "<h2>Reloaded course directory '%s'</h2>" % reload_dir
|
||||
def_ms.try_load_course(reload_dir)
|
||||
track.views.server_track(request, 'reloaded %s' % reload_dir, {}, page='migrate')
|
||||
|
||||
return HttpResponse(html)
|
||||
@@ -1077,14 +1077,6 @@ MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage'
|
||||
# Guidelines for translators
|
||||
TRANSLATORS_GUIDE = 'https://edx.readthedocs.org/projects/edx-developer-guide/en/latest/conventions/internationalization/i18n_translators_guide.html' # pylint: disable=line-too-long
|
||||
|
||||
#################################### GITHUB #######################################
|
||||
# gitreload is used in LMS-workflow to pull content from github
|
||||
# gitreload requests are only allowed from these IP addresses, which are
|
||||
# the advertised public IPs of the github WebHook servers.
|
||||
# These are listed, eg at https://github.com/edx/edx-platform/admin/hooks
|
||||
|
||||
ALLOWED_GITRELOAD_IPS = ['207.97.227.253', '50.57.128.197', '108.171.174.178']
|
||||
|
||||
#################################### AWS #######################################
|
||||
# S3BotoStorage insists on a timeout for uploaded assets. We should make it
|
||||
# permanent instead, but rather than trying to figure out exactly where that
|
||||
|
||||
13
lms/urls.py
13
lms/urls.py
@@ -26,7 +26,6 @@ from lms.djangoapps.instructor.views import coupons as instructor_coupons_views
|
||||
from lms.djangoapps.instructor.views import instructor_dashboard as instructor_dashboard_views
|
||||
from lms.djangoapps.instructor.views import registration_codes as instructor_registration_codes_views
|
||||
from lms.djangoapps.instructor_task import views as instructor_task_views
|
||||
from lms_migration import migrate as lms_migrate_views
|
||||
from notes import views as notes_views
|
||||
from notification_prefs import views as notification_prefs_views
|
||||
from openedx.core.djangoapps.auth_exchange.views import LoginWithAccessTokenView
|
||||
@@ -891,18 +890,6 @@ if settings.FEATURES.get('ENABLE_OAUTH2_PROVIDER'):
|
||||
url(r'^_o/', include('oauth2_provider.urls', namespace='oauth2_provider')),
|
||||
]
|
||||
|
||||
if settings.FEATURES.get('ENABLE_LMS_MIGRATION'):
|
||||
urlpatterns += [
|
||||
url(r'^migrate/modules$', lms_migrate_views.manage_modulestores),
|
||||
url(r'^migrate/reload/(?P<reload_dir>[^/]+)$', lms_migrate_views.manage_modulestores),
|
||||
url(
|
||||
r'^migrate/reload/(?P<reload_dir>[^/]+)/(?P<commit_id>[^/]+)$',
|
||||
lms_migrate_views.manage_modulestores
|
||||
),
|
||||
url(r'^gitreload$', lms_migrate_views.gitreload),
|
||||
url(r'^gitreload/(?P<reload_dir>[^/]+)$', lms_migrate_views.gitreload),
|
||||
]
|
||||
|
||||
if settings.FEATURES.get('ENABLE_SQL_TRACKING_LOGS'):
|
||||
urlpatterns += [
|
||||
url(r'^event_logs$', track_views.view_tracking_log),
|
||||
|
||||
Reference in New Issue
Block a user