diff --git a/lms/djangoapps/lms_migration/migrate.py b/lms/djangoapps/lms_migration/migrate.py deleted file mode 100644 index 13190dd858..0000000000 --- a/lms/djangoapps/lms_migration/migrate.py +++ /dev/null @@ -1,232 +0,0 @@ -# -# migration tools for content team to go from stable-edx4edx to LMS+CMS -# - -import json -import logging -import os - -from django.conf import settings -from django.http import HttpResponse -from six import text_type - -import track.views -import xmodule.modulestore.django as xmodule_django -from xmodule.modulestore.django import modulestore - -try: - from django.views.decorators.csrf import csrf_exempt -except ImportError: - from django.contrib.csrf.middleware import csrf_exempt - -log = logging.getLogger("edx.lms_migrate") -LOCAL_DEBUG = True -ALLOWED_IPS = settings.LMS_MIGRATION_ALLOWED_IPS - - -def escape(s): - """escape HTML special characters in string""" - return str(s).replace('<', '<').replace('>', '>') - - -def getip(request): - ''' - Extract IP address of requester from header, even if behind proxy - ''' - ip = request.META.get('HTTP_X_REAL_IP', '') # nginx reverse proxy - if not ip: - ip = request.META.get('REMOTE_ADDR', 'None') - return ip - - -def get_commit_id(course): - #return course.metadata.get('GIT_COMMIT_ID', 'No commit id') - return getattr(course, 'GIT_COMMIT_ID', 'No commit id') - # getattr(def_ms.courses[reload_dir], 'GIT_COMMIT_ID','No commit id') - - -def set_commit_id(course, commit_id): - #course.metadata['GIT_COMMIT_ID'] = commit_id - course.GIT_COMMIT_ID = commit_id - # def_ms.courses[reload_dir].GIT_COMMIT_ID = new_commit_id - - -def manage_modulestores(request, reload_dir=None, commit_id=None): - ''' - Manage the static in-memory modulestores. - - If reload_dir is not None, then instruct the xml loader to reload that course directory. - ''' - html = "" - - def_ms = modulestore() - courses = def_ms.get_courses() - - #---------------------------------------- - # check on IP address of requester - - ip = getip(request) - - if LOCAL_DEBUG: - html += '

IP address: %s

' % ip - html += '

User: %s

' % request.user - html += '

My pid: %s

' % os.getpid() - log.debug(u'request from ip=%s, user=%s', ip, request.user) - - if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS): - if request.user and request.user.is_staff: - log.debug(u'request allowed because user=%s is staff', request.user) - else: - html += 'Permission denied' - html += "" - log.debug('request denied, ALLOWED_IPS=%s', ALLOWED_IPS) - return HttpResponse(html, status=403) - - #---------------------------------------- - # reload course if specified; handle optional commit_id - - if reload_dir is not None: - if reload_dir not in def_ms.courses: - html += '

Error: "%s" is not a valid course directory

' % reload_dir - else: - # reloading based on commit_id is needed when running mutiple worker threads, - # so that a given thread doesn't reload the same commit multiple times - current_commit_id = get_commit_id(def_ms.courses[reload_dir]) - log.debug('commit_id="%s"', commit_id) - log.debug('current_commit_id="%s"', current_commit_id) - - if (commit_id is not None) and (commit_id == current_commit_id): - html += "

Already at commit id %s for %s

" % (commit_id, reload_dir) - track.views.server_track(request, - 'reload %s skipped already at %s (pid=%s)' % (reload_dir, - commit_id, - os.getpid(), - ), - {}, page='migrate') - else: - html += '

Reloaded course directory "%s"

' % reload_dir - def_ms.try_load_course(reload_dir) - gdir = settings.DATA_DIR / reload_dir - new_commit_id = os.popen('cd %s; git log -n 1 | head -1' % gdir).read().strip().split(' ')[1] - set_commit_id(def_ms.courses[reload_dir], new_commit_id) - html += '

commit_id=%s

' % new_commit_id - track.views.server_track(request, 'reloaded %s now at %s (pid=%s)' % (reload_dir, - new_commit_id, - os.getpid()), {}, page='migrate') - - #---------------------------------------- - - html += '

Courses loaded in the modulestore

' - html += '
    ' - for cdir, course in def_ms.courses.items(): - html += '
  1. %s (%s)
    2. ' % ( - settings.EDX_ROOT_URL, - escape(cdir), - escape(cdir), - text_type(course.location) - ) - html += '
' - - #---------------------------------------- - - #dumpfields = ['definition', 'location', 'metadata'] - dumpfields = ['location', 'metadata'] - - for cdir, course in def_ms.courses.items(): - html += '
' - html += '

Course: %s (%s)

' % (course.display_name_with_default_escaped, cdir) - - html += '

commit_id=%s

' % get_commit_id(course) - - for field in dumpfields: - data = getattr(course, field, None) - html += '

%s

' % field - if isinstance(data, dict): - html += '' - else: - html += '' % escape(data) - - #---------------------------------------- - - html += '
' - html += "courses: 
%s
" % escape(courses) - - ms = xmodule_django._MODULESTORES - html += "modules: 
%s
" % escape(ms) - html += "default modulestore: 
%s
" % escape(unicode(def_ms)) - - #---------------------------------------- - - log.debug('_MODULESTORES=%s', ms) - log.debug('courses=%s', courses) - log.debug('def_ms=%s', unicode(def_ms)) - - html += "" - return HttpResponse(html) - - -@csrf_exempt -def gitreload(request, reload_dir=None): - ''' - This can be used as a github WebHook Service Hook, for reloading of the content repo used by the LMS. - - If reload_dir is not None, then instruct the xml loader to reload that course directory. - ''' - html = "" - ip = getip(request) - - html += '

IP address: %s ' % ip - html += '

User: %s ' % request.user - - ALLOWED_IPS = [] # allow none by default - if hasattr(settings, 'ALLOWED_GITRELOAD_IPS'): # allow override in settings - ALLOWED_IPS = settings.ALLOWED_GITRELOAD_IPS - - if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS): - if request.user and request.user.is_staff: - log.debug(u'request allowed because user=%s is staff', request.user) - else: - html += 'Permission denied' - html += "" - log.debug('request denied from %s, ALLOWED_IPS=%s', ip, ALLOWED_IPS) - return HttpResponse(html) - - #---------------------------------------- - # see if request is from github (POST with JSON) - - if reload_dir is None and 'payload' in request.POST: - payload = request.POST['payload'] - log.debug("payload=%s", payload) - gitargs = json.loads(payload) - log.debug("gitargs=%s", gitargs) - reload_dir = gitargs['repository']['name'] - log.debug("github reload_dir=%s", reload_dir) - gdir = settings.DATA_DIR / reload_dir - if not os.path.exists(gdir): - log.debug("====> ERROR in gitreload - no such directory %s", reload_dir) - return HttpResponse('Error') - cmd = "cd %s; git reset --hard HEAD; git clean -f -d; git pull origin; chmod g+w course.xml" % gdir - log.debug(os.popen(cmd).read()) - if hasattr(settings, 'GITRELOAD_HOOK'): # hit this hook after reload, if set - gh = settings.GITRELOAD_HOOK - if gh: - ghurl = '%s/%s' % (gh, reload_dir) - r = requests.get(ghurl) - log.debug("GITRELOAD_HOOK to %s: %s", ghurl, r.text) - - #---------------------------------------- - # reload course if specified - - if reload_dir is not None: - def_ms = modulestore() - if reload_dir not in def_ms.courses: - html += '

Error: "%s" is not a valid course directory

' % reload_dir - else: - html += "

Reloaded course directory '%s'

" % reload_dir - def_ms.try_load_course(reload_dir) - track.views.server_track(request, 'reloaded %s' % reload_dir, {}, page='migrate') - - return HttpResponse(html) diff --git a/lms/envs/common.py b/lms/envs/common.py index 3a0c8f38de..c9f4fa253b 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1077,14 +1077,6 @@ MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage' # Guidelines for translators TRANSLATORS_GUIDE = 'https://edx.readthedocs.org/projects/edx-developer-guide/en/latest/conventions/internationalization/i18n_translators_guide.html' # pylint: disable=line-too-long -#################################### GITHUB ####################################### -# gitreload is used in LMS-workflow to pull content from github -# gitreload requests are only allowed from these IP addresses, which are -# the advertised public IPs of the github WebHook servers. -# These are listed, eg at https://github.com/edx/edx-platform/admin/hooks - -ALLOWED_GITRELOAD_IPS = ['207.97.227.253', '50.57.128.197', '108.171.174.178'] - #################################### AWS ####################################### # S3BotoStorage insists on a timeout for uploaded assets. We should make it # permanent instead, but rather than trying to figure out exactly where that diff --git a/lms/urls.py b/lms/urls.py index 928eb398aa..4653abe183 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -26,7 +26,6 @@ from lms.djangoapps.instructor.views import coupons as instructor_coupons_views from lms.djangoapps.instructor.views import instructor_dashboard as instructor_dashboard_views from lms.djangoapps.instructor.views import registration_codes as instructor_registration_codes_views from lms.djangoapps.instructor_task import views as instructor_task_views -from lms_migration import migrate as lms_migrate_views from notes import views as notes_views from notification_prefs import views as notification_prefs_views from openedx.core.djangoapps.auth_exchange.views import LoginWithAccessTokenView @@ -891,18 +890,6 @@ if settings.FEATURES.get('ENABLE_OAUTH2_PROVIDER'): url(r'^_o/', include('oauth2_provider.urls', namespace='oauth2_provider')), ] -if settings.FEATURES.get('ENABLE_LMS_MIGRATION'): - urlpatterns += [ - url(r'^migrate/modules$', lms_migrate_views.manage_modulestores), - url(r'^migrate/reload/(?P[^/]+)$', lms_migrate_views.manage_modulestores), - url( - r'^migrate/reload/(?P[^/]+)/(?P[^/]+)$', - lms_migrate_views.manage_modulestores - ), - url(r'^gitreload$', lms_migrate_views.gitreload), - url(r'^gitreload/(?P[^/]+)$', lms_migrate_views.gitreload), - ] - if settings.FEATURES.get('ENABLE_SQL_TRACKING_LOGS'): urlpatterns += [ url(r'^event_logs$', track_views.view_tracking_log),