diff --git a/lms/djangoapps/lms_migration/__init__.py b/lms/djangoapps/lms_migration/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/lms/djangoapps/lms_migration/management/__init__.py b/lms/djangoapps/lms_migration/management/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/lms/djangoapps/lms_migration/management/commands/__init__.py b/lms/djangoapps/lms_migration/management/commands/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/lms/djangoapps/lms_migration/management/commands/create_groups.py b/lms/djangoapps/lms_migration/management/commands/create_groups.py deleted file mode 100644 index 1ec0ebf0a1..0000000000 --- a/lms/djangoapps/lms_migration/management/commands/create_groups.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/python -# -# File: create_groups.py -# -# Create all staff_* groups for classes in data directory. - -import os - -from django.conf import settings -from django.contrib.auth.models import Group -from django.core.management.base import BaseCommand -from lxml import etree -from path import Path as path - - -def create_groups(): - ''' - Create staff and instructor groups for all classes in the data_dir - ''' - - data_dir = settings.DATA_DIR - print "data_dir = %s" % data_dir - - for course_dir in os.listdir(data_dir): - - if course_dir.startswith('.'): - continue - if not os.path.isdir(path(data_dir) / course_dir): - continue - - cxfn = path(data_dir) / course_dir / 'course.xml' - try: - coursexml = etree.parse(cxfn) - except Exception: # pylint: disable=broad-except - print "Oops, cannot read %s, skipping" % cxfn - continue - cxmlroot = coursexml.getroot() - course = cxmlroot.get('course') # TODO (vshnayder!!): read metadata from policy file(s) instead of from course.xml - if course is None: - print "oops, can't get course id for %s" % course_dir - continue - print "course=%s for course_dir=%s" % (course, course_dir) - - create_group('staff_%s' % course) # staff group - create_group('instructor_%s' % course) # instructor group (can manage staff group list) - - -def create_group(gname): - if Group.objects.filter(name=gname): - print " group exists for %s" % gname - return - g = Group(name=gname) - g.save() - print " created group %s" % gname - - -class Command(BaseCommand): - help = "Create groups associated with all courses in data_dir." - - def handle(self, *args, **options): - create_groups() diff --git a/lms/djangoapps/lms_migration/management/commands/create_user.py b/lms/djangoapps/lms_migration/management/commands/create_user.py deleted file mode 100644 index acab51a35c..0000000000 --- a/lms/djangoapps/lms_migration/management/commands/create_user.py +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/python -# -# File: create_user.py -# -# Create user. Prompt for groups and ExternalAuthMap - -import datetime -import json -import os -import readline -import string -import sys -from getpass import getpass -from random import choice - -from django.contrib.auth.models import Group, User -from django.core.management.base import BaseCommand -from pytz import UTC - -from openedx.core.djangoapps.external_auth.models import ExternalAuthMap -from student.models import Registration, UserProfile, email_exists_or_retired - - -class MyCompleter(object): # Custom completer - - def __init__(self, options): - self.options = sorted(options) - - def complete(self, text, state): - if state == 0: # on first trigger, build possible matches - if text: # cache matches (entries that start with entered text) - self.matches = [ - option - for option in self.options - if option and option.startswith(text) - ] - else: # no text entered, all matches possible - self.matches = self.options[:] - - # return match indexed by state - try: - return self.matches[state] - except IndexError: - return None - - -def GenPasswd(length=8, chars=string.letters + string.digits): - return ''.join([choice(chars) for dummy0 in range(length)]) - -#----------------------------------------------------------------------------- -# main command - - -class Command(BaseCommand): - help = "Create user, interactively; can add ExternalAuthMap for MIT user if email@MIT.EDU resolves properly." - - def handle(self, *args, **options): - - while True: - uname = raw_input('username: ') - if User.objects.filter(username=uname): - print "username %s already taken" % uname - else: - break - - make_eamap = False - if raw_input('Create MIT ExternalAuth? [n] ').lower() == 'y': - email = '%s@MIT.EDU' % uname - if not email.endswith('@MIT.EDU'): - print "Failed - email must be @MIT.EDU" - sys.exit(-1) - mit_domain = 'ssl:MIT' - if ExternalAuthMap.objects.filter(external_id=email, external_domain=mit_domain): - print "Failed - email %s already exists as external_id" % email - sys.exit(-1) - make_eamap = True - password = GenPasswd(12) - - # get name from kerberos - try: - kname = os.popen("finger %s | grep 'name:'" % email).read().strip().split('name: ')[1].strip() - except: - kname = '' - name = raw_input('Full name: [%s] ' % kname).strip() - if name == '': - name = kname - print "name = %s" % name - else: - while True: - password = getpass() - password2 = getpass() - if password == password2: - break - print "Oops, passwords do not match, please retry" - - while True: - email = raw_input('email: ') - if email_exists_or_retired(email): - print "email %s already taken" % email - else: - break - - name = raw_input('Full name: ') - - user = User(username=uname, email=email, is_active=True) - user.set_password(password) - try: - user.save() - except IntegrityError: - print "Oops, failed to create user %s, IntegrityError" % user - raise - - r = Registration() - r.register(user) - - up = UserProfile(user=user) - up.name = name - up.save() - - if make_eamap: - credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name, email) - eamap = ExternalAuthMap( - external_id=email, - external_email=email, - external_domain=mit_domain, - external_name=name, - internal_password=password, - external_credentials=json.dumps(credentials), - ) - eamap.user = user - eamap.dtsignup = datetime.datetime.now(UTC) - eamap.save() - - print "User %s created successfully!" % user - - if not raw_input('Add user %s to any groups? [n] ' % user).lower() == 'y': - sys.exit(0) - - print "Here are the groups available:" - - groups = [str(g.name) for g in Group.objects.all()] - print groups - - completer = MyCompleter(groups) - readline.set_completer(completer.complete) - readline.parse_and_bind('tab: complete') - - while True: - gname = raw_input("Add group (tab to autocomplete, empty line to end): ") - if not gname: - break - if gname not in groups: - print "Unknown group %s" % gname - continue - g = Group.objects.get(name=gname) - user.groups.add(g) - print "Added %s to group %s" % (user, g) - - print "Done!" diff --git a/lms/djangoapps/lms_migration/management/commands/manage_course_groups.py b/lms/djangoapps/lms_migration/management/commands/manage_course_groups.py deleted file mode 100644 index 0798790e80..0000000000 --- a/lms/djangoapps/lms_migration/management/commands/manage_course_groups.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/python -# -# File: manage_course_groups -# -# interactively list and edit membership in course staff and instructor groups - -import re - -from django.contrib.auth.models import Group, User -from django.core.management.base import BaseCommand - - -#----------------------------------------------------------------------------- -# get all staff groups - - -class Command(BaseCommand): - help = "Manage course group membership, interactively." - - def handle(self, *args, **options): - - gset = Group.objects.all() - - print "Groups:" - for cnt, g in zip(range(len(gset)), gset): - print "%d. %s" % (cnt, g) - - gnum = int(raw_input('Choose group to manage (enter #): ')) - - group = gset[gnum] - - #----------------------------------------------------------------------------- - # users in group - - uall = User.objects.all() - if uall.count() < 50: - print "----" - print "List of All Users: %s" % [str(x.username) for x in uall] - print "----" - else: - print "----" - print "There are %d users, which is too many to list" % uall.count() - print "----" - - while True: - - print "Users in the group:" - - uset = group.user_set.all() - for cnt, u in zip(range(len(uset)), uset): - print "%d. %s" % (cnt, u) - - action = raw_input('Choose user to delete (enter #) or enter usernames (comma delim) to add: ') - - m = re.match('^[0-9]+$', action) - if m: - unum = int(action) - u = uset[unum] - print "Deleting user %s" % u - u.groups.remove(group) - - else: - for uname in action.split(','): - try: - user = User.objects.get(username=action) - except Exception as err: - print "Error %s" % err - continue - print "adding %s to group %s" % (user, group) - user.groups.add(group) diff --git a/lms/djangoapps/lms_migration/migrate.py b/lms/djangoapps/lms_migration/migrate.py deleted file mode 100644 index 13190dd858..0000000000 --- a/lms/djangoapps/lms_migration/migrate.py +++ /dev/null @@ -1,232 +0,0 @@ -# -# migration tools for content team to go from stable-edx4edx to LMS+CMS -# - -import json -import logging -import os - -from django.conf import settings -from django.http import HttpResponse -from six import text_type - -import track.views -import xmodule.modulestore.django as xmodule_django -from xmodule.modulestore.django import modulestore - -try: - from django.views.decorators.csrf import csrf_exempt -except ImportError: - from django.contrib.csrf.middleware import csrf_exempt - -log = logging.getLogger("edx.lms_migrate") -LOCAL_DEBUG = True -ALLOWED_IPS = settings.LMS_MIGRATION_ALLOWED_IPS - - -def escape(s): - """escape HTML special characters in string""" - return str(s).replace('<', '<').replace('>', '>') - - -def getip(request): - ''' - Extract IP address of requester from header, even if behind proxy - ''' - ip = request.META.get('HTTP_X_REAL_IP', '') # nginx reverse proxy - if not ip: - ip = request.META.get('REMOTE_ADDR', 'None') - return ip - - -def get_commit_id(course): - #return course.metadata.get('GIT_COMMIT_ID', 'No commit id') - return getattr(course, 'GIT_COMMIT_ID', 'No commit id') - # getattr(def_ms.courses[reload_dir], 'GIT_COMMIT_ID','No commit id') - - -def set_commit_id(course, commit_id): - #course.metadata['GIT_COMMIT_ID'] = commit_id - course.GIT_COMMIT_ID = commit_id - # def_ms.courses[reload_dir].GIT_COMMIT_ID = new_commit_id - - -def manage_modulestores(request, reload_dir=None, commit_id=None): - ''' - Manage the static in-memory modulestores. - - If reload_dir is not None, then instruct the xml loader to reload that course directory. - ''' - html = "" - - def_ms = modulestore() - courses = def_ms.get_courses() - - #---------------------------------------- - # check on IP address of requester - - ip = getip(request) - - if LOCAL_DEBUG: - html += '

IP address: %s

' % ip - html += '

User: %s

' % request.user - html += '

My pid: %s

' % os.getpid() - log.debug(u'request from ip=%s, user=%s', ip, request.user) - - if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS): - if request.user and request.user.is_staff: - log.debug(u'request allowed because user=%s is staff', request.user) - else: - html += 'Permission denied' - html += "" - log.debug('request denied, ALLOWED_IPS=%s', ALLOWED_IPS) - return HttpResponse(html, status=403) - - #---------------------------------------- - # reload course if specified; handle optional commit_id - - if reload_dir is not None: - if reload_dir not in def_ms.courses: - html += '

Error: "%s" is not a valid course directory

' % reload_dir - else: - # reloading based on commit_id is needed when running mutiple worker threads, - # so that a given thread doesn't reload the same commit multiple times - current_commit_id = get_commit_id(def_ms.courses[reload_dir]) - log.debug('commit_id="%s"', commit_id) - log.debug('current_commit_id="%s"', current_commit_id) - - if (commit_id is not None) and (commit_id == current_commit_id): - html += "

Already at commit id %s for %s

" % (commit_id, reload_dir) - track.views.server_track(request, - 'reload %s skipped already at %s (pid=%s)' % (reload_dir, - commit_id, - os.getpid(), - ), - {}, page='migrate') - else: - html += '

Reloaded course directory "%s"

' % reload_dir - def_ms.try_load_course(reload_dir) - gdir = settings.DATA_DIR / reload_dir - new_commit_id = os.popen('cd %s; git log -n 1 | head -1' % gdir).read().strip().split(' ')[1] - set_commit_id(def_ms.courses[reload_dir], new_commit_id) - html += '

commit_id=%s

' % new_commit_id - track.views.server_track(request, 'reloaded %s now at %s (pid=%s)' % (reload_dir, - new_commit_id, - os.getpid()), {}, page='migrate') - - #---------------------------------------- - - html += '

Courses loaded in the modulestore

' - html += '
    ' - for cdir, course in def_ms.courses.items(): - html += '
  1. %s (%s)
    2. ' % ( - settings.EDX_ROOT_URL, - escape(cdir), - escape(cdir), - text_type(course.location) - ) - html += '
' - - #---------------------------------------- - - #dumpfields = ['definition', 'location', 'metadata'] - dumpfields = ['location', 'metadata'] - - for cdir, course in def_ms.courses.items(): - html += '
' - html += '

Course: %s (%s)

' % (course.display_name_with_default_escaped, cdir) - - html += '

commit_id=%s

' % get_commit_id(course) - - for field in dumpfields: - data = getattr(course, field, None) - html += '

%s

' % field - if isinstance(data, dict): - html += '' - else: - html += '' % escape(data) - - #---------------------------------------- - - html += '
' - html += "courses: 
%s
" % escape(courses) - - ms = xmodule_django._MODULESTORES - html += "modules: 
%s
" % escape(ms) - html += "default modulestore: 
%s
" % escape(unicode(def_ms)) - - #---------------------------------------- - - log.debug('_MODULESTORES=%s', ms) - log.debug('courses=%s', courses) - log.debug('def_ms=%s', unicode(def_ms)) - - html += "" - return HttpResponse(html) - - -@csrf_exempt -def gitreload(request, reload_dir=None): - ''' - This can be used as a github WebHook Service Hook, for reloading of the content repo used by the LMS. - - If reload_dir is not None, then instruct the xml loader to reload that course directory. - ''' - html = "" - ip = getip(request) - - html += '

IP address: %s ' % ip - html += '

User: %s ' % request.user - - ALLOWED_IPS = [] # allow none by default - if hasattr(settings, 'ALLOWED_GITRELOAD_IPS'): # allow override in settings - ALLOWED_IPS = settings.ALLOWED_GITRELOAD_IPS - - if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS): - if request.user and request.user.is_staff: - log.debug(u'request allowed because user=%s is staff', request.user) - else: - html += 'Permission denied' - html += "" - log.debug('request denied from %s, ALLOWED_IPS=%s', ip, ALLOWED_IPS) - return HttpResponse(html) - - #---------------------------------------- - # see if request is from github (POST with JSON) - - if reload_dir is None and 'payload' in request.POST: - payload = request.POST['payload'] - log.debug("payload=%s", payload) - gitargs = json.loads(payload) - log.debug("gitargs=%s", gitargs) - reload_dir = gitargs['repository']['name'] - log.debug("github reload_dir=%s", reload_dir) - gdir = settings.DATA_DIR / reload_dir - if not os.path.exists(gdir): - log.debug("====> ERROR in gitreload - no such directory %s", reload_dir) - return HttpResponse('Error') - cmd = "cd %s; git reset --hard HEAD; git clean -f -d; git pull origin; chmod g+w course.xml" % gdir - log.debug(os.popen(cmd).read()) - if hasattr(settings, 'GITRELOAD_HOOK'): # hit this hook after reload, if set - gh = settings.GITRELOAD_HOOK - if gh: - ghurl = '%s/%s' % (gh, reload_dir) - r = requests.get(ghurl) - log.debug("GITRELOAD_HOOK to %s: %s", ghurl, r.text) - - #---------------------------------------- - # reload course if specified - - if reload_dir is not None: - def_ms = modulestore() - if reload_dir not in def_ms.courses: - html += '

Error: "%s" is not a valid course directory

' % reload_dir - else: - html += "

Reloaded course directory '%s'

" % reload_dir - def_ms.try_load_course(reload_dir) - track.views.server_track(request, 'reloaded %s' % reload_dir, {}, page='migrate') - - return HttpResponse(html) diff --git a/lms/envs/common.py b/lms/envs/common.py index 33f026c16e..6b82a8be1b 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1077,14 +1077,6 @@ MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage' # Guidelines for translators TRANSLATORS_GUIDE = 'https://edx.readthedocs.org/projects/edx-developer-guide/en/latest/conventions/internationalization/i18n_translators_guide.html' # pylint: disable=line-too-long -#################################### GITHUB ####################################### -# gitreload is used in LMS-workflow to pull content from github -# gitreload requests are only allowed from these IP addresses, which are -# the advertised public IPs of the github WebHook servers. -# These are listed, eg at https://github.com/edx/edx-platform/admin/hooks - -ALLOWED_GITRELOAD_IPS = ['207.97.227.253', '50.57.128.197', '108.171.174.178'] - #################################### AWS ####################################### # S3BotoStorage insists on a timeout for uploaded assets. We should make it # permanent instead, but rather than trying to figure out exactly where that diff --git a/lms/urls.py b/lms/urls.py index 928eb398aa..4653abe183 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -26,7 +26,6 @@ from lms.djangoapps.instructor.views import coupons as instructor_coupons_views from lms.djangoapps.instructor.views import instructor_dashboard as instructor_dashboard_views from lms.djangoapps.instructor.views import registration_codes as instructor_registration_codes_views from lms.djangoapps.instructor_task import views as instructor_task_views -from lms_migration import migrate as lms_migrate_views from notes import views as notes_views from notification_prefs import views as notification_prefs_views from openedx.core.djangoapps.auth_exchange.views import LoginWithAccessTokenView @@ -891,18 +890,6 @@ if settings.FEATURES.get('ENABLE_OAUTH2_PROVIDER'): url(r'^_o/', include('oauth2_provider.urls', namespace='oauth2_provider')), ] -if settings.FEATURES.get('ENABLE_LMS_MIGRATION'): - urlpatterns += [ - url(r'^migrate/modules$', lms_migrate_views.manage_modulestores), - url(r'^migrate/reload/(?P[^/]+)$', lms_migrate_views.manage_modulestores), - url( - r'^migrate/reload/(?P[^/]+)/(?P[^/]+)$', - lms_migrate_views.manage_modulestores - ), - url(r'^gitreload$', lms_migrate_views.gitreload), - url(r'^gitreload/(?P[^/]+)$', lms_migrate_views.gitreload), - ] - if settings.FEATURES.get('ENABLE_SQL_TRACKING_LOGS'): urlpatterns += [ url(r'^event_logs$', track_views.view_tracking_log),