diff --git a/cms/djangoapps/contentstore/management/commands/import.py b/cms/djangoapps/contentstore/management/commands/import.py index e0d58b32f0..2e16bf1258 100644 --- a/cms/djangoapps/contentstore/management/commands/import.py +++ b/cms/djangoapps/contentstore/management/commands/import.py @@ -31,9 +31,16 @@ class Command(BaseCommand): course_dirs = args[1:] else: course_dirs = None - print("Importing. Data_dir={data}, course_dirs={courses}".format( + self.stdout.write("Importing. Data_dir={data}, course_dirs={courses}\n".format( data=data_dir, courses=course_dirs, dis=do_import_static)) - import_from_xml(modulestore('direct'), data_dir, course_dirs, load_error_modules=False, + try: + mstore = modulestore('direct') + except KeyError: + self.stdout.write('Unable to load direct modulestore, trying ' + 'default\n') + mstore = modulestore('default') + + import_from_xml(mstore, data_dir, course_dirs, load_error_modules=False, static_content_store=contentstore(), verbose=True, do_import_static=do_import_static) diff --git a/lms/djangoapps/courseware/management/commands/import.py b/lms/djangoapps/courseware/management/commands/import.py new file mode 120000 index 0000000000..36b7e3c6fc --- /dev/null +++ b/lms/djangoapps/courseware/management/commands/import.py @@ -0,0 +1 @@ +../../../../../cms/djangoapps/contentstore/management/commands/import.py \ No newline at end of file diff --git a/lms/djangoapps/dashboard/management/__init__.py b/lms/djangoapps/dashboard/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lms/djangoapps/dashboard/management/commands/__init__.py b/lms/djangoapps/dashboard/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lms/djangoapps/dashboard/management/commands/git_add_course.py b/lms/djangoapps/dashboard/management/commands/git_add_course.py new file mode 100644 index 0000000000..1d095b11a6 --- /dev/null +++ b/lms/djangoapps/dashboard/management/commands/git_add_course.py @@ -0,0 +1,242 @@ +""" +Script for importing courseware from git/xml into a mongo modulestore +""" + +import os +import re +import datetime +import StringIO +import subprocess +import logging + +from django.conf import settings +from django.core import management +from django.core.management.base import BaseCommand, CommandError +from django.utils.translation import ugettext as _ +import mongoengine + +from dashboard.models import CourseImportLog +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.xml import XMLModuleStore + +log = logging.getLogger(__name__) + +GIT_REPO_DIR = getattr(settings, 'GIT_REPO_DIR', '/opt/edx/course_repos') +GIT_IMPORT_STATIC = getattr(settings, 'GIT_IMPORT_STATIC', True) + +GIT_IMPORT_NO_DIR = -1 +GIT_IMPORT_URL_BAD = -2 +GIT_IMPORT_CANNOT_PULL = -3 +GIT_IMPORT_XML_IMPORT_FAILED = -4 +GIT_IMPORT_UNSUPPORTED_STORE = -5 +GIT_IMPORT_MONGODB_FAIL = -6 +GIT_IMPORT_BAD_REPO = -7 + + +def add_repo(repo, rdir_in): + """This will add a git repo into the mongo modulestore""" + # pylint: disable=R0915 + + # Set defaults even if it isn't defined in settings + mongo_db = { + 'host': 'localhost', + 'user': '', + 'password': '', + 'db': 'xlog', + } + + # Allow overrides + if hasattr(settings, 'MONGODB_LOG'): + for config_item in ['host', 'user', 'password', 'db', ]: + mongo_db[config_item] = settings.MONGODB_LOG.get( + config_item, mongo_db[config_item]) + + if not os.path.isdir(GIT_REPO_DIR): + log.critical(_("Path {0} doesn't exist, please create it, " + "or configure a different path with " + "GIT_REPO_DIR").format(GIT_REPO_DIR)) + return GIT_IMPORT_NO_DIR + + # pull from git + if not repo.endswith('.git') or not ( + repo.startswith('http:') or + repo.startswith('https:') or + repo.startswith('git:') or + repo.startswith('file:')): + + log.error(_('Oops, not a git ssh url?')) + log.error(_('Expecting something like ' + 'git@github.com:mitocw/edx4edx_lite.git')) + return GIT_IMPORT_URL_BAD + + if rdir_in: + rdir = rdir_in + rdir = os.path.basename(rdir) + else: + rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0] + + log.debug('rdir = {0}'.format(rdir)) + + rdirp = '{0}/{1}'.format(GIT_REPO_DIR, rdir) + if os.path.exists(rdirp): + log.info(_('directory already exists, doing a git pull instead ' + 'of git clone')) + cmd = ['git', 'pull', ] + cwd = '{0}/{1}'.format(GIT_REPO_DIR, rdir) + else: + cmd = ['git', 'clone', repo, ] + cwd = GIT_REPO_DIR + + log.debug(cmd) + cwd = os.path.abspath(cwd) + try: + ret_git = subprocess.check_output(cmd, cwd=cwd) + except subprocess.CalledProcessError: + log.exception(_('git clone or pull failed!')) + return GIT_IMPORT_CANNOT_PULL + log.debug(ret_git) + + # get commit id + cmd = ['git', 'log', '-1', '--format=%H', ] + try: + commit_id = subprocess.check_output(cmd, cwd=rdirp) + except subprocess.CalledProcessError: + log.exception(_('Unable to get git log')) + return GIT_IMPORT_BAD_REPO + + ret_git += _('\nCommit ID: {0}').format(commit_id) + + # get branch + cmd = ['git', 'rev-parse', '--abbrev-ref', 'HEAD', ] + try: + branch = subprocess.check_output(cmd, cwd=rdirp) + except subprocess.CalledProcessError: + log.exception(_('Unable to get branch info')) + return GIT_IMPORT_BAD_REPO + + ret_git += ' \nBranch: {0}'.format(branch) + + # Get XML logging logger and capture debug to parse results + output = StringIO.StringIO() + import_log_handler = logging.StreamHandler(output) + import_log_handler.setLevel(logging.DEBUG) + + logger_names = ['xmodule.modulestore.xml_importer', 'git_add_course', + 'xmodule.modulestore.xml', 'xmodule.seq_module', ] + loggers = [] + + for logger_name in logger_names: + logger = logging.getLogger(logger_name) + logger.old_level = logger.level + logger.setLevel(logging.DEBUG) + logger.addHandler(import_log_handler) + loggers.append(logger) + + try: + management.call_command('import', GIT_REPO_DIR, rdir, + nostatic=not GIT_IMPORT_STATIC) + except CommandError: + log.exception(_('Unable to run import command.')) + return GIT_IMPORT_XML_IMPORT_FAILED + except NotImplementedError: + log.exception(_('The underlying module store does not support import.')) + return GIT_IMPORT_UNSUPPORTED_STORE + + ret_import = output.getvalue() + + # Remove handler hijacks + for logger in loggers: + logger.setLevel(logger.old_level) + logger.removeHandler(import_log_handler) + + course_id = 'unknown' + location = 'unknown' + + # extract course ID from output of import-command-run and make symlink + # this is needed in order for custom course scripts to work + match = re.search('(?ms)===> IMPORTING course to location ([^ \n]+)', + ret_import) + if match: + location = match.group(1).strip() + log.debug('location = {0}'.format(location)) + course_id = location.replace('i4x://', '').replace( + '/course/', '/').split('\n')[0].strip() + + cdir = '{0}/{1}'.format(GIT_REPO_DIR, course_id.split('/')[1]) + log.debug(_('Studio course dir = {0}').format(cdir)) + + if os.path.exists(cdir) and not os.path.islink(cdir): + log.debug(_(' -> exists, but is not symlink')) + log.debug(subprocess.check_output(['ls', '-l', ], + cwd=os.path.abspath(cdir))) + try: + os.rmdir(os.path.abspath(cdir)) + except OSError: + log.exception(_('Failed to remove course directory')) + + if not os.path.exists(cdir): + log.debug(_(' -> creating symlink between {0} and {1}').format(rdirp, cdir)) + try: + os.symlink(os.path.abspath(rdirp), os.path.abspath(cdir)) + except OSError: + log.exception(_('Unable to create course symlink')) + log.debug(subprocess.check_output(['ls', '-l', ], + cwd=os.path.abspath(cdir))) + + # store import-command-run output in mongo + mongouri = 'mongodb://{user}:{password}@{host}/{db}'.format(**mongo_db) + + try: + if mongo_db['user'] and mongo_db['password']: + mdb = mongoengine.connect(mongo_db['db'], host=mongouri) + else: + mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host']) + except mongoengine.connection.ConnectionError: + log.exception(_('Unable to connect to mongodb to save log, please ' + 'check MONGODB_LOG settings')) + return GIT_IMPORT_MONGODB_FAIL + cil = CourseImportLog( + course_id=course_id, + location=location, + repo_dir=rdir, + created=datetime.datetime.now(), + import_log=ret_import, + git_log=ret_git, + ) + cil.save() + + log.debug(_('saved CourseImportLog for {0}').format(cil.course_id)) + mdb.disconnect() + return 0 + + +class Command(BaseCommand): + """ + Pull a git repo and import into the mongo based content database. + """ + + help = _('Import the specified git repository into the ' + 'modulestore and directory') + + def handle(self, *args, **options): + """Check inputs and run the command""" + + if isinstance(modulestore, XMLModuleStore): + raise CommandError(_('This script requires a mongo module store')) + + if len(args) < 1: + raise CommandError(_('This script requires at least one argument, ' + 'the git URL')) + + if len(args) > 2: + raise CommandError(_('This script requires no more than two ' + 'arguments')) + + rdir_arg = None + + if len(args) > 1: + rdir_arg = args[1] + + if add_repo(args[0], rdir_arg) != 0: + raise CommandError(_('Repo was not added, check log output ' + 'for details')) diff --git a/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py b/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py new file mode 100644 index 0000000000..0b3c685fbd --- /dev/null +++ b/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py @@ -0,0 +1,104 @@ +""" +Provide tests for git_add_course management command. +""" + +import unittest +import os +import shutil +import subprocess + +from django.conf import settings +from django.core.management import call_command +from django.core.management.base import CommandError +from django.test.utils import override_settings + +from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +import dashboard.management.commands.git_add_course as git_add_course + +TEST_MONGODB_LOG = { + 'host': 'localhost', + 'user': '', + 'password': '', + 'db': 'test_xlog', +} + +FEATURES_WITH_SSL_AUTH = settings.FEATURES.copy() +FEATURES_WITH_SSL_AUTH['AUTH_USE_MIT_CERTIFICATES'] = True + + +@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) +@override_settings(MONGODB_LOG=TEST_MONGODB_LOG) +@unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'), + "ENABLE_SYSADMIN_DASHBOARD not set") +class TestGitAddCourse(ModuleStoreTestCase): + """ + Tests the git_add_course management command for proper functions. + """ + + TEST_REPO = 'https://github.com/mitocw/edx4edx_lite.git' + + def assertCommandFailureRegexp(self, regex, *args): + """ + Convenience function for testing command failures + """ + with self.assertRaises(SystemExit): + self.assertRaisesRegexp(CommandError, regex, + call_command('git_add_course', *args)) + + def test_command_args(self): + """ + Validate argument checking + """ + self.assertCommandFailureRegexp( + 'This script requires at least one argument, the git URL') + self.assertCommandFailureRegexp( + 'This script requires no more than two arguments', + 'blah', 'blah', 'blah') + self.assertCommandFailureRegexp( + 'Repo was not added, check log output for details', + 'blah') + # Test successful import from command + try: + os.mkdir(getattr(settings, 'GIT_REPO_DIR')) + except OSError: + pass + + # Make a course dir that will be replaced with a symlink + # while we are at it. + if not os.path.isdir(getattr(settings, 'GIT_REPO_DIR') / 'edx4edx'): + os.mkdir(getattr(settings, 'GIT_REPO_DIR') / 'edx4edx') + + call_command('git_add_course', self.TEST_REPO, + getattr(settings, 'GIT_REPO_DIR') / 'edx4edx_lite') + if os.path.isdir(getattr(settings, 'GIT_REPO_DIR')): + shutil.rmtree(getattr(settings, 'GIT_REPO_DIR')) + + def test_add_repo(self): + """ + Various exit path tests for test_add_repo + """ + self.assertEqual(git_add_course.GIT_IMPORT_NO_DIR, + git_add_course.add_repo(self.TEST_REPO, None)) + try: + os.mkdir(getattr(settings, 'GIT_REPO_DIR')) + except OSError: + pass + self.assertEqual(git_add_course.GIT_IMPORT_URL_BAD, + git_add_course.add_repo('foo', None)) + + self.assertEqual( + git_add_course.GIT_IMPORT_CANNOT_PULL, + git_add_course.add_repo('file:///foobar.git', None) + ) + + # Test git repo that exists, but is "broken" + bare_repo = os.path.abspath('{0}/{1}'.format(settings.TEST_ROOT, 'bare.git')) + os.mkdir(os.path.abspath(bare_repo)) + subprocess.call(['git', '--bare', 'init', ], cwd=bare_repo) + + self.assertEqual( + git_add_course.GIT_IMPORT_BAD_REPO, + git_add_course.add_repo('file://{0}'.format(bare_repo), None) + ) + shutil.rmtree(bare_repo) diff --git a/lms/djangoapps/dashboard/models.py b/lms/djangoapps/dashboard/models.py index 6b20219993..096288f6d4 100644 --- a/lms/djangoapps/dashboard/models.py +++ b/lms/djangoapps/dashboard/models.py @@ -1 +1,17 @@ -# Create your models here. +"""Models for dashboard application""" + +import mongoengine + + +class CourseImportLog(mongoengine.Document): + """Mongoengine model for git log""" + # pylint: disable=R0924 + + course_id = mongoengine.StringField(max_length=128) + location = mongoengine.StringField(max_length=168) + import_log = mongoengine.StringField(max_length=20 * 65535) + git_log = mongoengine.StringField(max_length=65535) + repo_dir = mongoengine.StringField(max_length=128) + created = mongoengine.DateTimeField() + meta = {'indexes': ['course_id', 'created'], + 'allow_inheritance': False} diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py new file mode 100644 index 0000000000..9627e37f0b --- /dev/null +++ b/lms/djangoapps/dashboard/sysadmin.py @@ -0,0 +1,692 @@ +""" +This module creates a sysadmin dashboard for managing and viewing +courses. +""" +import csv +import json +import logging +import os +import subprocess +import time +import StringIO +from datetime import datetime + +from django.conf import settings +from django.contrib.auth import authenticate +from django.contrib.auth.decorators import login_required +from django.contrib.auth.models import User +from django.core.exceptions import PermissionDenied +from django.db import IntegrityError +from django.http import HttpResponse, Http404 +from django.utils.decorators import method_decorator +from django.utils.html import escape +from django.utils.translation import ugettext as _ +from django.views.decorators.cache import cache_control +from django.views.generic.base import TemplateView +from django.views.decorators.http import condition +from django_future.csrf import ensure_csrf_cookie +from edxmako.shortcuts import render_to_response +import mongoengine + +from courseware.courses import get_course_by_id +from courseware.roles import CourseStaffRole, CourseInstructorRole +import dashboard.management.commands.git_add_course as git_add_course +from dashboard.models import CourseImportLog +from external_auth.models import ExternalAuthMap +from external_auth.views import generate_password +from student.models import CourseEnrollment, UserProfile, Registration +import track.views +from xmodule.contentstore.django import contentstore +from xmodule.modulestore import MONGO_MODULESTORE_TYPE +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.store_utilities import delete_course +from xmodule.modulestore.xml import XMLModuleStore + + +log = logging.getLogger(__name__) + + +class SysadminDashboardView(TemplateView): + """Base class for sysadmin dashboard views with common methods""" + + template_name = 'sysadmin_dashboard.html' + + def __init__(self, **kwargs): + """ + Initialize base sysadmin dashboard class with modulestore, + modulestore_type and return msg + """ + + self.def_ms = modulestore() + self.is_using_mongo = True + if isinstance(self.def_ms, XMLModuleStore): + self.is_using_mongo = False + self.msg = u'' + self.datatable = [] + super(SysadminDashboardView, self).__init__(**kwargs) + + @method_decorator(ensure_csrf_cookie) + @method_decorator(login_required) + @method_decorator(cache_control(no_cache=True, no_store=True, + must_revalidate=True)) + @method_decorator(condition(etag_func=None)) + def dispatch(self, *args, **kwargs): + return super(SysadminDashboardView, self).dispatch(*args, **kwargs) + + def get_courses(self): + """ Get an iterable list of courses.""" + + courses = self.def_ms.get_courses() + courses = dict([c.id, c] for c in courses) # no course directory + + return courses + + def return_csv(self, filename, header, data): + """ + Convenient function for handling the http response of a csv. + data should be iterable and is used to stream object over http + """ + + csv_file = StringIO.StringIO() + writer = csv.writer(csv_file, dialect='excel', quotechar='"', + quoting=csv.QUOTE_ALL) + + writer.writerow(header) + + # Setup streaming of the data + def read_and_flush(): + """Read and clear buffer for optimization""" + csv_file.seek(0) + csv_data = csv_file.read() + csv_file.seek(0) + csv_file.truncate() + return csv_data + + def csv_data(): + """Generator for handling potentially large CSVs""" + for row in data: + writer.writerow(row) + csv_data = read_and_flush() + yield csv_data + response = HttpResponse(csv_data(), mimetype='text/csv') + response['Content-Disposition'] = 'attachment; filename={0}'.format( + filename) + return response + + +class Users(SysadminDashboardView): + """ + The status view provides Web based user management, a listing of + courses loaded, and user statistics + """ + + def fix_external_auth_map_passwords(self): + """ + This corrects any passwords that have drifted from eamap to + internal django auth. Needs to be removed when fixed in external_auth + """ + + msg = '' + for eamap in ExternalAuthMap.objects.all(): + euser = eamap.user + epass = eamap.internal_password + if euser is None: + continue + try: + testuser = authenticate(username=euser.username, password=epass) + except (TypeError, PermissionDenied, AttributeError), err: + msg += _('Failed in authenticating {0}, error {1}\n' + ).format(euser, err) + continue + if testuser is None: + msg += _('Failed in authenticating {0}\n').format(euser) + msg += _('fixed password') + euser.set_password(epass) + euser.save() + continue + if not msg: + msg = _('All ok!') + return msg + + def create_user(self, uname, name, password=None): + """ Creates a user (both SSL and regular)""" + + if not uname: + return _('Must provide username') + if not name: + return _('Must provide full name') + + email_domain = getattr(settings, 'SSL_AUTH_EMAIL_DOMAIN', 'MIT.EDU') + + msg = u'' + if settings.FEATURES['AUTH_USE_MIT_CERTIFICATES']: + if not '@' in uname: + email = '{0}@{1}'.format(uname, email_domain) + else: + email = uname + if not email.endswith('@{0}'.format(email_domain)): + msg += u'{0} @{1}'.format(_('email must end in'), email_domain) + return msg + mit_domain = 'ssl:MIT' + if ExternalAuthMap.objects.filter(external_id=email, + external_domain=mit_domain): + msg += _('Failed - email {0} already exists as ' + 'external_id').format(email) + return msg + new_password = generate_password() + else: + if not password: + return _('Password must be supplied if not using certificates') + + email = uname + + if not '@' in email: + msg += _('email address required (not username)') + return msg + new_password = password + + user = User(username=uname, email=email, is_active=True) + user.set_password(new_password) + try: + user.save() + except IntegrityError: + msg += _('Oops, failed to create user {0}, ' + 'IntegrityError').format(user) + return msg + + reg = Registration() + reg.register(user) + + profile = UserProfile(user=user) + profile.name = name + profile.save() + + if settings.FEATURES['AUTH_USE_MIT_CERTIFICATES']: + credential_string = getattr(settings, 'SSL_AUTH_DN_FORMAT_STRING', + '/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}') + credentials = credential_string.format(name, email) + eamap = ExternalAuthMap( + external_id=email, + external_email=email, + external_domain=mit_domain, + external_name=name, + internal_password=new_password, + external_credentials=json.dumps(credentials), + ) + eamap.user = user + eamap.dtsignup = datetime.now() + eamap.save() + + msg += _('User {0} created successfully!').format(user) + return msg + + def delete_user(self, uname): + """Deletes a user from django auth""" + + if not uname: + return _('Must provide username') + if '@' in uname: + try: + user = User.objects.get(email=uname) + except User.DoesNotExist, err: + msg = _('Cannot find user with email address {0}').format(uname) + return msg + else: + try: + user = User.objects.get(username=uname) + except User.DoesNotExist, err: + msg = _('Cannot find user with username {0} - {1}' + ).format(uname, str(err)) + return msg + user.delete() + return _('Deleted user {0}').format(uname) + + def make_common_context(self): + """Returns the datatable used for this view""" + + self.datatable = {} + courses = self.get_courses() + + self.datatable = dict(header=[_('Statistic'), _('Value')], + title=_('Site statistics')) + self.datatable['data'] = [[_('Total number of users'), + User.objects.all().count()]] + + self.msg += u'

{0}

'.format( + _('Courses loaded in the modulestore')) + self.msg += u'
    ' + for (cdir, course) in courses.items(): + self.msg += u'
  1. {0} ({1})
  2. '.format( + escape(cdir), course.location.url()) + self.msg += u'
' + + def get(self, request): + + if not request.user.is_staff: + raise Http404 + self.make_common_context() + + context = { + 'datatable': self.datatable, + 'msg': self.msg, + 'djangopid': os.getpid(), + 'modeflag': {'users': 'active-section'}, + 'mitx_version': getattr(settings, 'VERSION_STRING', ''), + } + return render_to_response(self.template_name, context) + + def post(self, request): + """Handle various actions available on page""" + + if not request.user.is_staff: + raise Http404 + + self.make_common_context() + + action = request.POST.get('action', '') + track.views.server_track(request, action, {}, page='user_sysdashboard') + + if action == 'download_users': + header = [_('username'), _('email'), ] + data = ([u.username, u.email] for u in + (User.objects.all().iterator())) + return self.return_csv('users_{0}.csv'.format( + request.META['SERVER_NAME']), header, data) + elif action == 'repair_eamap': + self.msg = u'

{0}

{1}
{2}'.format( + _('Repair Results'), + self.fix_external_auth_map_passwords(), + self.msg) + self.datatable = {} + elif action == 'create_user': + uname = request.POST.get('student_uname', '').strip() + name = request.POST.get('student_fullname', '').strip() + password = request.POST.get('student_password', '').strip() + self.msg = u'

{0}

{1}


{2}'.format( + _('Create User Results'), + self.create_user(uname, name, password), self.msg) + elif action == 'del_user': + uname = request.POST.get('student_uname', '').strip() + self.msg = u'

{0}

{1}


{2}'.format( + _('Delete User Results'), self.delete_user(uname), self.msg) + + context = { + 'datatable': self.datatable, + 'msg': self.msg, + 'djangopid': os.getpid(), + 'modeflag': {'users': 'active-section'}, + 'mitx_version': getattr(settings, 'VERSION_STRING', ''), + } + return render_to_response(self.template_name, context) + + +class Courses(SysadminDashboardView): + """ + This manages adding/updating courses from git, deleting courses, and + provides course listing information. + """ + + def git_info_for_course(self, cdir): + """This pulls out some git info like the last commit""" + + cmd = '' + gdir = settings.DATA_DIR / cdir + info = ['', '', ''] + if not os.path.exists(gdir): + return info + + cmd = ['git', 'log', '-1', + '--format=format:{ "commit": "%H", "author": "%an %ae", "date": "%ad"}', ] + try: + output_json = json.loads(subprocess.check_output(cmd, cwd=gdir)) + info = [output_json['commit'], + output_json['date'], + output_json['author'], ] + except (ValueError, subprocess.CalledProcessError): + pass + + return info + + def get_course_from_git(self, gitloc, datatable): + """This downloads and runs the checks for importing a course in git""" + + if not (gitloc.endswith('.git') or gitloc.startswith('http:') or + gitloc.startswith('https:') or gitloc.startswith('git:')): + return _("The git repo location should end with '.git', " + "and be a valid url") + + if self.is_using_mongo: + return self.import_mongo_course(gitloc) + + return self.import_xml_course(gitloc, datatable) + + def import_mongo_course(self, gitloc): + """ + Imports course using management command and captures logging output + at debug level for display in template + """ + + msg = u'' + + logging.debug(_('Adding course using git repo {0}').format(gitloc)) + + # Grab logging output for debugging imports + output = StringIO.StringIO() + import_log_handler = logging.StreamHandler(output) + import_log_handler.setLevel(logging.DEBUG) + + logger_names = ['xmodule.modulestore.xml_importer', + 'dashboard.management.commands.git_add_course', + 'xmodule.modulestore.xml', 'xmodule.seq_module', ] + loggers = [] + + for logger_name in logger_names: + logger = logging.getLogger(logger_name) + logger.old_level = logger.level + logger.setLevel(logging.DEBUG) + logger.addHandler(import_log_handler) + loggers.append(logger) + + git_add_course.add_repo(gitloc, None) + ret = output.getvalue() + + # Remove handler hijacks + for logger in loggers: + logger.setLevel(logger.old_level) + logger.removeHandler(import_log_handler) + + msg = u"

{0} {1}

".format( + _('Added course from'), gitloc) + msg += _("
{0}
").format(escape(ret)) + return msg + + def import_xml_course(self, gitloc, datatable): + """Imports a git course into the XMLModuleStore""" + + msg = u'' + if not getattr(settings, 'GIT_IMPORT_WITH_XMLMODULESTORE', False): + return _('Refusing to import. GIT_IMPORT_WITH_XMLMODULESTORE is ' + 'not turned on, and it is generally not safe to import ' + 'into an XMLModuleStore with multithreaded. We ' + 'recommend you enable the MongoDB based module store ' + 'instead, unless this is a development environment.') + cdir = (gitloc.rsplit('/', 1)[1])[:-4] + gdir = settings.DATA_DIR / cdir + if os.path.exists(gdir): + msg += _("The course {0} already exists in the data directory! " + "(reloading anyway)").format(cdir) + cmd = ['git', 'pull', ] + cwd = gdir + else: + cmd = ['git', 'clone', gitloc, ] + cwd = settings.DATA_DIR + cwd = os.path.abspath(cwd) + try: + cmd_output = escape(subprocess.check_output(cmd, cwd=cwd)) + except subprocess.CalledProcessError: + return _('Unable to clone or pull repository. Please check your url.') + + msg += u'
{0}
'.format(cmd_output) + if not os.path.exists(gdir): + msg += _('Failed to clone repository to {0}').format(gdir) + return msg + self.def_ms.try_load_course(os.path.abspath(gdir)) + errlog = self.def_ms.errored_courses.get(cdir, '') + if errlog: + msg += u'
{0}
'.format(escape(errlog)) + else: + course = self.def_ms.courses[os.path.abspath(gdir)] + msg += _('Loaded course {0} {1}
Errors:').format( + cdir, course.display_name) + errors = self.def_ms.get_item_errors(course.location) + if not errors: + msg += u'None' + else: + msg += u'' + datatable['data'].append([course.display_name, cdir] + + self.git_info_for_course(cdir)) + return msg + + def make_datatable(self): + """Creates course information datatable""" + + data = [] + courses = self.get_courses() + + for (cdir, course) in courses.items(): + gdir = cdir + if '/' in cdir: + gdir = cdir.rsplit('/', 1)[1] + data.append([course.display_name, cdir] + + self.git_info_for_course(gdir)) + + return dict(header=[_('Course Name'), _('Directory/ID'), + _('Git Commit'), _('Last Change'), + _('Last Editor')], + title=_('Information about all courses'), + data=data) + + def get(self, request): + """Displays forms and course information""" + + if not request.user.is_staff: + raise Http404 + + context = { + 'datatable': self.make_datatable(), + 'msg': self.msg, + 'djangopid': os.getpid(), + 'modeflag': {'courses': 'active-section'}, + 'mitx_version': getattr(settings, 'VERSION_STRING', ''), + } + return render_to_response(self.template_name, context) + + def post(self, request): + """Handle all actions from courses view""" + + if not request.user.is_staff: + raise Http404 + + action = request.POST.get('action', '') + track.views.server_track(request, action, {}, + page='courses_sysdashboard') + + courses = self.get_courses() + if action == 'add_course': + gitloc = request.POST.get('repo_location', '').strip().replace(' ', '').replace(';', '') + datatable = self.make_datatable() + self.msg += self.get_course_from_git(gitloc, datatable) + + elif action == 'del_course': + course_id = request.POST.get('course_id', '').strip() + course_found = False + if course_id in courses: + course_found = True + course = courses[course_id] + else: + try: + course = get_course_by_id(course_id) + course_found = True + except Exception, err: # pylint: disable=broad-except + self.msg += _('Error - cannot get course with ID ' + '{0}
{1}
').format( + course_id, escape(str(err)) + ) + + is_mongo_course = (modulestore().get_modulestore_type(course_id) == MONGO_MODULESTORE_TYPE) + if course_found and not is_mongo_course: + cdir = course.data_dir + self.def_ms.courses.pop(cdir) + + # now move the directory (don't actually delete it) + new_dir = "{course_dir}_deleted_{timestamp}".format( + course_dir=cdir, + timestamp=int(time.time()) + ) + os.rename(settings.DATA_DIR / cdir, settings.DATA_DIR / new_dir) + + self.msg += (u"Deleted " + u"{0} = {1} ({2})".format( + cdir, course.id, course.display_name)) + + elif course_found and is_mongo_course: + # delete course that is stored with mongodb backend + loc = course.location + content_store = contentstore() + commit = True + delete_course(self.def_ms, content_store, loc, commit) + # don't delete user permission groups, though + self.msg += \ + u"{0} {1} = {2} ({3})".format( + _('Deleted'), loc, course.id, course.display_name) + datatable = self.make_datatable() + + context = { + 'datatable': datatable, + 'msg': self.msg, + 'djangopid': os.getpid(), + 'modeflag': {'courses': 'active-section'}, + 'mitx_version': getattr(settings, 'VERSION_STRING', ''), + } + return render_to_response(self.template_name, context) + + +class Staffing(SysadminDashboardView): + """ + The status view provides a view of staffing and enrollment in + courses that include an option to download the data as a csv. + """ + + def get(self, request): + """Displays course Enrollment and staffing course statistics""" + + if not request.user.is_staff: + raise Http404 + data = [] + + courses = self.get_courses() + + for (cdir, course) in courses.items(): # pylint: disable=unused-variable + datum = [course.display_name, course.id] + datum += [CourseEnrollment.objects.filter( + course_id=course.id).count()] + datum += [CourseStaffRole(course.location).users_with_role().count()] + datum += [','.join([x.username for x in CourseInstructorRole( + course.location).users_with_role()])] + data.append(datum) + + datatable = dict(header=[_('Course Name'), _('course_id'), + _('# enrolled'), _('# staff'), + _('instructors')], + title=_('Enrollment information for all courses'), + data=data) + context = { + 'datatable': datatable, + 'msg': self.msg, + 'djangopid': os.getpid(), + 'modeflag': {'staffing': 'active-section'}, + 'mitx_version': getattr(settings, 'VERSION_STRING', ''), + } + return render_to_response(self.template_name, context) + + def post(self, request): + """Handle all actions from staffing and enrollment view""" + + action = request.POST.get('action', '') + track.views.server_track(request, action, {}, + page='staffing_sysdashboard') + + if action == 'get_staff_csv': + data = [] + roles = [CourseInstructorRole, CourseStaffRole, ] + + courses = self.get_courses() + + for (cdir, course) in courses.items(): # pylint: disable=unused-variable + for role in roles: + for user in role(course.location).users_with_role(): + datum = [course.id, role, user.username, user.email, + user.profile.name] + data.append(datum) + header = [_('course_id'), + _('role'), _('username'), + _('email'), _('full_name'), ] + return self.return_csv('staff_{0}.csv'.format( + request.META['SERVER_NAME']), header, data) + + return self.get(request) + + +class GitLogs(TemplateView): + """ + This provides a view into the import of courses from git repositories. + It is convenient for allowing course teams to see what may be wrong with + their xml + """ + + template_name = 'sysadmin_dashboard_gitlogs.html' + + @method_decorator(login_required) + def get(self, request, *args, **kwargs): + """Shows logs of imports that happened as a result of a git import""" + + course_id = kwargs.get('course_id') + + # Set mongodb defaults even if it isn't defined in settings + mongo_db = { + 'host': 'localhost', + 'user': '', + 'password': '', + 'db': 'xlog', + } + + # Allow overrides + if hasattr(settings, 'MONGODB_LOG'): + for config_item in ['host', 'user', 'password', 'db', ]: + mongo_db[config_item] = settings.MONGODB_LOG.get( + config_item, mongo_db[config_item]) + + mongouri = 'mongodb://{user}:{password}@{host}/{db}'.format(**mongo_db) + + error_msg = '' + + try: + if mongo_db['user'] and mongo_db['password']: + mdb = mongoengine.connect(mongo_db['db'], host=mongouri) + else: + mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host']) + except mongoengine.connection.ConnectionError: + logging.exception(_('Unable to connect to mongodb to save log, ' + 'please check MONGODB_LOG settings.')) + + if course_id is None: + # Require staff if not going to specific course + if not request.user.is_staff: + raise Http404 + cilset = CourseImportLog.objects.all().order_by('-created') + else: + try: + course = get_course_by_id(course_id) + except Exception: # pylint: disable=broad-except + cilset = None + error_msg = _('Cannot find course {0}').format(course_id) + + # Allow only course team, instructors, and staff + if not (request.user.is_staff or + CourseInstructorRole(course.location).has_user(request.user) or + CourseStaffRole(course.location).has_user(request.user)): + raise Http404 + log.debug('course_id={0}'.format(course_id)) + cilset = CourseImportLog.objects.filter( + course_id=course_id).order_by('-created') + log.debug('cilset length={0}'.format(len(cilset))) + mdb.disconnect() + context = {'cilset': cilset, + 'course_id': course_id, + 'error_msg': error_msg} + + return render_to_response(self.template_name, context) diff --git a/lms/djangoapps/dashboard/sysadmin_urls.py b/lms/djangoapps/dashboard/sysadmin_urls.py new file mode 100644 index 0000000000..e543537415 --- /dev/null +++ b/lms/djangoapps/dashboard/sysadmin_urls.py @@ -0,0 +1,18 @@ +""" +Urls for sysadmin dashboard feature +""" +# pylint: disable=E1120 + +from django.conf.urls import patterns, url + +from dashboard import sysadmin + +urlpatterns = patterns( + '', + url(r'^$', sysadmin.Users.as_view(), name="sysadmin"), + url(r'^courses/?$', sysadmin.Courses.as_view(), name="sysadmin_courses"), + url(r'^staffing/?$', sysadmin.Staffing.as_view(), name="sysadmin_staffing"), + url(r'^gitlogs/?$', sysadmin.GitLogs.as_view(), name="gitlogs"), + url(r'^gitlogs/(?P.+)$', sysadmin.GitLogs.as_view(), + name="gitlogs_detail"), +) diff --git a/lms/djangoapps/dashboard/tests/test_sysadmin.py b/lms/djangoapps/dashboard/tests/test_sysadmin.py new file mode 100644 index 0000000000..37080723d0 --- /dev/null +++ b/lms/djangoapps/dashboard/tests/test_sysadmin.py @@ -0,0 +1,506 @@ +""" +Provide tests for sysadmin dashboard feature in sysadmin.py +""" + +import unittest +import os +import shutil + +from django.conf import settings +from django.contrib.auth.hashers import check_password +from django.contrib.auth.models import User +from django.core.urlresolvers import reverse +from django.test.client import Client +from django.test.utils import override_settings +from django.utils.html import escape +from django.utils.translation import ugettext as _ +import mongoengine + +from courseware.roles import CourseStaffRole, GlobalStaff +from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE +from dashboard.models import CourseImportLog +from dashboard.sysadmin import Users +from external_auth.models import ExternalAuthMap +from student.tests.factories import UserFactory +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.xml import XMLModuleStore + + +TEST_MONGODB_LOG = { + 'host': 'localhost', + 'user': '', + 'password': '', + 'db': 'test_xlog', +} + +FEATURES_WITH_SSL_AUTH = settings.FEATURES.copy() +FEATURES_WITH_SSL_AUTH['AUTH_USE_MIT_CERTIFICATES'] = True + + +class SysadminBaseTestCase(ModuleStoreTestCase): + """ + Base class with common methods used in XML and Mongo tests + """ + + @classmethod + def tearDownClass(cls): + """Delete all repos imported during tests.""" + super(SysadminBaseTestCase, cls).tearDownClass() + try: + shutil.rmtree(getattr(settings, 'GIT_REPO_DIR')) + except OSError: + pass + + def setUp(self): + """Setup test case by adding primary user.""" + super(SysadminBaseTestCase, self).setUp() + self.user = UserFactory.create(username='test_user', + email='test_user+sysadmin@edx.org', + password='foo') + self.client = Client() + + def _setstaff_login(self): + """Makes the test user staff and logs them in""" + GlobalStaff().add_users(self.user) + self.client.login(username=self.user.username, password='foo') + + def _add_edx4edx(self): + """Adds the edx4edx sample course""" + return self.client.post(reverse('sysadmin_courses'), { + 'repo_location': 'https://github.com/mitocw/edx4edx_lite.git', + 'action': 'add_course', }) + + def _rm_edx4edx(self): + """Deletes the sample course from the XML store""" + def_ms = modulestore() + try: + # using XML store + course = def_ms.courses.get('{0}/edx4edx_lite'.format( + os.path.abspath(settings.DATA_DIR)), None) + except AttributeError: + # Using mongo store + course = def_ms.get_course('MITx/edx4edx/edx4edx') + + # Delete git loaded course + return self.client.post(reverse('sysadmin_courses'), + {'course_id': course.id, + 'action': 'del_course', }) + + +@unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'), + "ENABLE_SYSADMIN_DASHBOARD not set") +@override_settings(GIT_IMPORT_WITH_XMLMODULESTORE=True) +class TestSysadmin(SysadminBaseTestCase): + """ + Test sysadmin dashboard features using XMLModuleStore + """ + + def test_staff_access(self): + """Test access controls.""" + + test_views = ['sysadmin', 'sysadmin_courses', 'sysadmin_staffing', ] + for view in test_views: + response = self.client.get(reverse(view)) + self.assertEqual(response.status_code, 302) + + logged_in = self.client.login(username=self.user.username, + password='foo') + self.assertTrue(logged_in) + + for view in test_views: + response = self.client.get(reverse(view)) + self.assertEqual(response.status_code, 404) + + response = self.client.get(reverse('gitlogs')) + self.assertEqual(response.status_code, 404) + + self.user.is_staff = True + self.user.save() + + self.client.logout() + self.client.login(username=self.user.username, password='foo') + + for view in test_views: + response = self.client.get(reverse(view)) + self.assertTrue(response.status_code, 200) + + response = self.client.get(reverse('gitlogs')) + self.assertTrue(response.status_code, 200) + + def test_user_mod(self): + """Create and delete a user""" + + self._setstaff_login() + + self.client.login(username=self.user.username, password='foo') + + # Create user tests + + # No uname + response = self.client.post(reverse('sysadmin'), + {'action': 'create_user', + 'student_fullname': 'blah', + 'student_password': 'foozor', }) + self.assertIn(_('Must provide username'), response.content) + # no full name + response = self.client.post(reverse('sysadmin'), + {'action': 'create_user', + 'student_uname': 'test_cuser+sysadmin@edx.org', + 'student_password': 'foozor', }) + self.assertIn(_('Must provide full name'), response.content) + + # Test create valid user + self.client.post(reverse('sysadmin'), + {'action': 'create_user', + 'student_uname': 'test_cuser+sysadmin@edx.org', + 'student_fullname': 'test cuser', + 'student_password': 'foozor', }) + + self.assertIsNotNone( + User.objects.get(username='test_cuser+sysadmin@edx.org', + email='test_cuser+sysadmin@edx.org')) + + # login as new user to confirm + self.assertTrue(self.client.login( + username='test_cuser+sysadmin@edx.org', password='foozor')) + + self.client.logout() + self.client.login(username=self.user.username, password='foo') + + # Delete user tests + + # Try no username + response = self.client.post(reverse('sysadmin'), + {'action': 'del_user', }) + self.assertIn(_('Must provide username'), response.content) + + # Try bad usernames + response = self.client.post(reverse('sysadmin'), + {'action': 'del_user', + 'student_uname': 'flabbergast@example.com', + 'student_fullname': 'enigma jones', }) + self.assertIn(_('Cannot find user with email address'), response.content) + + response = self.client.post(reverse('sysadmin'), + {'action': 'del_user', + 'student_uname': 'flabbergast', + 'student_fullname': 'enigma jones', }) + self.assertIn(_('Cannot find user with username'), response.content) + + self.client.post(reverse('sysadmin'), + {'action': 'del_user', + 'student_uname': 'test_cuser+sysadmin@edx.org', + 'student_fullname': 'test cuser', }) + + self.assertEqual(0, len(User.objects.filter( + username='test_cuser+sysadmin@edx.org', + email='test_cuser+sysadmin@edx.org'))) + + self.assertEqual(1, len(User.objects.all())) + + def test_user_csv(self): + """Download and validate user CSV""" + + num_test_users = 100 + self._setstaff_login() + + # Stuff full of users to test streaming + for user_num in xrange(num_test_users): + Users().create_user('testingman_with_long_name{}'.format(user_num), + 'test test') + + response = self.client.post(reverse('sysadmin'), + {'action': 'download_users', }) + + self.assertIn('attachment', response['Content-Disposition']) + self.assertEqual('text/csv', response['Content-Type']) + self.assertIn('test_user', response.content) + self.assertTrue(num_test_users + 2, len(response.content.splitlines())) + + # Clean up + User.objects.filter( + username__startswith='testingman_with_long_name').delete() + + @override_settings(FEATURES=FEATURES_WITH_SSL_AUTH) + def test_authmap_repair(self): + """Run authmap check and repair""" + + self._setstaff_login() + + Users().create_user('test0', 'test test') + # Will raise exception, so no assert needed + eamap = ExternalAuthMap.objects.get(external_name='test test') + mitu = User.objects.get(username='test0') + + self.assertTrue(check_password(eamap.internal_password, mitu.password)) + mitu.set_password('not autogenerated') + mitu.save() + self.assertFalse(check_password(eamap.internal_password, mitu.password)) + + # Create really non user AuthMap + ExternalAuthMap(external_id='ll', + external_domain='ll', + external_credentials='{}', + external_email='a@b.c', + external_name='c', + internal_password='').save() + + response = self.client.post(reverse('sysadmin'), + {'action': 'repair_eamap', }) + + self.assertIn('{0} test0'.format(_('Failed in authenticating')), + response.content) + self.assertIn(_('fixed password'), response.content) + + self.assertTrue(self.client.login(username='test0', + password=eamap.internal_password)) + + # Check for all OK + self._setstaff_login() + response = self.client.post(reverse('sysadmin'), + {'action': 'repair_eamap', }) + self.assertIn(_('All ok!'), response.content) + + def test_xml_course_add_delete(self): + """add and delete course from xml module store""" + + self._setstaff_login() + + # Try bad git repo + response = self.client.post(reverse('sysadmin_courses'), { + 'repo_location': 'github.com/mitocw/edx4edx_lite', + 'action': 'add_course', }) + self.assertIn(_("The git repo location should end with '.git', " + "and be a valid url"), response.content.decode('utf-8')) + + response = self.client.post(reverse('sysadmin_courses'), { + 'repo_location': 'http://example.com/not_real.git', + 'action': 'add_course', }) + self.assertIn(_('Unable to clone or pull repository'), + response.content.decode('utf-8')) + # Create git loaded course + response = self._add_edx4edx() + + def_ms = modulestore() + self.assertIn('xml', str(def_ms.__class__)) + course = def_ms.courses.get('{0}/edx4edx_lite'.format( + os.path.abspath(settings.DATA_DIR)), None) + self.assertIsNotNone(course) + + # Delete a course + response = self._rm_edx4edx() + course = def_ms.courses.get('{0}/edx4edx_lite'.format( + os.path.abspath(settings.DATA_DIR)), None) + self.assertIsNone(course) + + # Try and delete a non-existent course + response = self.client.post(reverse('sysadmin_courses'), + {'course_id': 'foobar/foo/blah', + 'action': 'del_course', }) + self.assertIn(_('Error - cannot get course with ID'), + response.content.decode('utf-8')) + + @override_settings(GIT_IMPORT_WITH_XMLMODULESTORE=False) + def test_xml_safety_flag(self): + """Make sure the settings flag to disable xml imports is working""" + + self._setstaff_login() + response = self._add_edx4edx() + self.assertIn('GIT_IMPORT_WITH_XMLMODULESTORE', response.content) + + def_ms = modulestore() + course = def_ms.courses.get('{0}/edx4edx_lite'.format( + os.path.abspath(settings.DATA_DIR)), None) + self.assertIsNone(course) + + def test_git_pull(self): + """Make sure we can pull""" + + self._setstaff_login() + + response = self._add_edx4edx() + response = self._add_edx4edx() + self.assertIn(_("The course {0} already exists in the data directory! " + "(reloading anyway)").format('edx4edx_lite'), + response.content.decode('utf-8')) + self._rm_edx4edx() + + def test_staff_csv(self): + """Download and validate staff CSV""" + + self._setstaff_login() + self._add_edx4edx() + + def_ms = modulestore() + course = def_ms.get_course('MITx/edx4edx/edx4edx') + CourseStaffRole(course.location).add_users(self.user) + + response = self.client.post(reverse('sysadmin_staffing'), + {'action': 'get_staff_csv', }) + self.assertIn('attachment', response['Content-Disposition']) + self.assertEqual('text/csv', response['Content-Type']) + columns = [_('course_id'), _('role'), _('username'), + _('email'), _('full_name'), ] + self.assertIn(','.join('"' + c + '"' for c in columns), + response.content) + + self._rm_edx4edx() + + def test_enrollment_page(self): + """ + Adds a course and makes sure that it shows up on the staffing and + enrollment page + """ + + self._setstaff_login() + self._add_edx4edx() + response = self.client.get(reverse('sysadmin_staffing')) + self.assertIn('edx4edx', response.content) + self._rm_edx4edx() + + +@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) +@override_settings(MONGODB_LOG=TEST_MONGODB_LOG) +@unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'), + "ENABLE_SYSADMIN_DASHBOARD not set") +class TestSysAdminMongoCourseImport(SysadminBaseTestCase): + """ + Check that importing into the mongo module store works + """ + + @classmethod + def tearDownClass(cls): + """Delete mongo log entries after test.""" + super(TestSysAdminMongoCourseImport, cls).tearDownClass() + try: + mongoengine.connect(TEST_MONGODB_LOG['db']) + CourseImportLog.objects.all().delete() + except mongoengine.connection.ConnectionError: + pass + + def _setstaff_login(self): + """ + Makes the test user staff and logs them in + """ + + self.user.is_staff = True + self.user.save() + + self.client.login(username=self.user.username, password='foo') + + def test_missing_repo_dir(self): + """ + Ensure that we handle a missing repo dir + """ + + self._setstaff_login() + + if os.path.isdir(getattr(settings, 'GIT_REPO_DIR')): + shutil.rmtree(getattr(settings, 'GIT_REPO_DIR')) + + # Create git loaded course + response = self._add_edx4edx() + self.assertIn(escape(_("Path {0} doesn't exist, please create it, or " + "configure a different path with " + "GIT_REPO_DIR").format(settings.GIT_REPO_DIR)), + response.content.decode('UTF-8')) + + def test_mongo_course_add_delete(self): + """ + This is the same as TestSysadmin.test_xml_course_add_delete, + but it uses a mongo store + """ + + self._setstaff_login() + if not os.path.isdir(getattr(settings, 'GIT_REPO_DIR')): + os.mkdir(getattr(settings, 'GIT_REPO_DIR')) + + def_ms = modulestore() + self.assertFalse(isinstance(def_ms, XMLModuleStore)) + + self._add_edx4edx() + course = def_ms.get_course('MITx/edx4edx/edx4edx') + self.assertIsNotNone(course) + + self._rm_edx4edx() + course = def_ms.get_course('MITx/edx4edx/edx4edx') + self.assertIsNone(course) + + def test_gitlogs(self): + """ + Create a log entry and make sure it exists + """ + + self._setstaff_login() + try: + os.mkdir(getattr(settings, 'GIT_REPO_DIR')) + except OSError: + pass + + self._add_edx4edx() + response = self.client.get(reverse('gitlogs')) + + # Check that our earlier import has a log with a link to details + self.assertIn('/gitlogs/MITx/edx4edx/edx4edx', response.content) + + response = self.client.get( + reverse('gitlogs_detail', kwargs={ + 'course_id': 'MITx/edx4edx/edx4edx'})) + + self.assertIn('======> IMPORTING course to location', + response.content) + + self._rm_edx4edx() + + def test_gitlog_bad_course(self): + """ + Make sure we gracefully handle courses that don't exist. + """ + self._setstaff_login() + response = self.client.get( + reverse('gitlogs_detail', kwargs={ + 'course_id': 'Not/Real/Testing'})) + self.assertIn(_('Cannot find course'), response.content) + + def test_gitlog_courseteam_access(self): + """ + Ensure course team users are allowed to access only their own course. + """ + + try: + os.mkdir(getattr(settings, 'GIT_REPO_DIR')) + except OSError: + pass + + self._setstaff_login() + self._add_edx4edx() + self.user.is_staff = False + self.user.save() + logged_in = self.client.login(username=self.user.username, + password='foo') + response = self.client.get(reverse('gitlogs')) + # Make sure our non privileged user doesn't have access to all logs + self.assertEqual(response.status_code, 404) + # Or specific logs + response = self.client.get(reverse('gitlogs_detail', kwargs={ + 'course_id': 'MITx/edx4edx/edx4edx'})) + self.assertEqual(response.status_code, 404) + + # Add user as staff in course team + def_ms = modulestore() + course = def_ms.get_course('MITx/edx4edx/edx4edx') + CourseStaffRole(course.location).add_users(self.user) + + self.assertTrue(CourseStaffRole(course.location).has_user(self.user)) + logged_in = self.client.login(username=self.user.username, + password='foo') + self.assertTrue(logged_in) + + response = self.client.get( + reverse('gitlogs_detail', kwargs={ + 'course_id': 'MITx/edx4edx/edx4edx'})) + self.assertIn('======> IMPORTING course to location', + response.content) + + self._rm_edx4edx() diff --git a/lms/envs/aws.py b/lms/envs/aws.py index 0e3e87b299..8f39618fcb 100644 --- a/lms/envs/aws.py +++ b/lms/envs/aws.py @@ -237,6 +237,10 @@ ZENDESK_URL = ENV_TOKENS.get("ZENDESK_URL") FEEDBACK_SUBMISSION_EMAIL = ENV_TOKENS.get("FEEDBACK_SUBMISSION_EMAIL") MKTG_URLS = ENV_TOKENS.get('MKTG_URLS', MKTG_URLS) +# git repo loading environment +GIT_REPO_DIR = ENV_TOKENS.get('GIT_REPO_DIR', '/edx/var/edxapp/course_repos') +GIT_IMPORT_STATIC = ENV_TOKENS.get('GIT_IMPORT_STATIC', True) + for name, value in ENV_TOKENS.get("CODE_JAIL", {}).items(): oldvalue = CODE_JAIL.get(name) if isinstance(oldvalue, dict): @@ -251,6 +255,11 @@ COURSES_WITH_UNSAFE_CODE = ENV_TOKENS.get("COURSES_WITH_UNSAFE_CODE", []) if "TRACKING_IGNORE_URL_PATTERNS" in ENV_TOKENS: TRACKING_IGNORE_URL_PATTERNS = ENV_TOKENS.get("TRACKING_IGNORE_URL_PATTERNS") +# SSL external authentication settings +SSL_AUTH_EMAIL_DOMAIN = ENV_TOKENS.get("SSL_AUTH_EMAIL_DOMAIN", "MIT.EDU") +SSL_AUTH_DN_FORMAT_STRING = ENV_TOKENS.get("SSL_AUTH_DN_FORMAT_STRING", + "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}") + ############################## SECURE AUTH ITEMS ############### # Secret things: passwords, access keys, etc. @@ -286,6 +295,7 @@ XQUEUE_INTERFACE = AUTH_TOKENS['XQUEUE_INTERFACE'] MODULESTORE = AUTH_TOKENS.get('MODULESTORE', MODULESTORE) CONTENTSTORE = AUTH_TOKENS.get('CONTENTSTORE', CONTENTSTORE) DOC_STORE_CONFIG = AUTH_TOKENS.get('DOC_STORE_CONFIG',DOC_STORE_CONFIG) +MONGODB_LOG = AUTH_TOKENS.get('MONGODB_LOG', {}) OPEN_ENDED_GRADING_INTERFACE = AUTH_TOKENS.get('OPEN_ENDED_GRADING_INTERFACE', OPEN_ENDED_GRADING_INTERFACE) diff --git a/lms/envs/common.py b/lms/envs/common.py index 893e537f77..378c390e69 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -89,6 +89,8 @@ FEATURES = { 'ENABLE_MASQUERADE': True, # allow course staff to change to student view of courseware + 'ENABLE_SYSADMIN_DASHBOARD': False, # sysadmin dashboard, to see what courses are loaded, to delete & load courses + 'DISABLE_LOGIN_BUTTON': False, # used in systems where login is automatic, eg MIT SSL # extrernal access methods @@ -944,6 +946,7 @@ INSTALLED_APPS = ( 'eventtracking.django', 'util', 'certificates', + 'dashboard', 'instructor', 'instructor_task', 'open_ended_grading', diff --git a/lms/envs/test.py b/lms/envs/test.py index 31636f6aad..6fe96d760b 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -207,6 +207,10 @@ CC_PROCESSOR['CyberSource']['MERCHANT_ID'] = "edx" CC_PROCESSOR['CyberSource']['SERIAL_NUMBER'] = "0123456789012345678901" CC_PROCESSOR['CyberSource']['PURCHASE_ENDPOINT'] = "/shoppingcart/payment_fake" +########################### SYSADMIN DASHBOARD ################################ +FEATURES['ENABLE_SYSADMIN_DASHBOARD'] = True +GIT_REPO_DIR = TEST_ROOT / "course_repos" + ################################# CELERY ###################################### CELERY_ALWAYS_EAGER = True diff --git a/lms/templates/sysadmin_dashboard.html b/lms/templates/sysadmin_dashboard.html new file mode 100644 index 0000000000..21dcf4c869 --- /dev/null +++ b/lms/templates/sysadmin_dashboard.html @@ -0,0 +1,206 @@ +<%inherit file="/main.html" /> +<%! from django.core.urlresolvers import reverse %> +<%! from django.utils.translation import ugettext as _ %> +<%namespace name='static' file='/static_content.html'/> + +<%block name="headextra"> + <%static:css group='style-course'/> + + + + + + +
+
+ +
+

${_('Sysadmin Dashboard')}

+
+

+ ${_('Users')} + ${_('Courses')} + ${_('Staffing and Enrollment')} + ${_('Git Logs')} +

+
+%if modeflag.get('users'): +

${_('User Management')}

+ +
+ + +
    +
  • + + +
  • + +
  • + + +
  • +
  • + + +
  • +
+ +
+

+ + +

+
+ +
+

+ +

+ +

+ +

+ +
+
+ %endif + +%if modeflag.get('staffing'): + +

${_("Go to each individual course's Instructor dashboard to manage course enrollment.")}

+
+ +

${_('Manage course staff and instructors')}


+
+ + +
+ +%endif + +%if modeflag.get('courses'): +

${_('Administer Courses')}


+ +
+ +
    +
  • + + +
  • +
+
+ +
+
+
    +
  • + + +
  • +
+
+ +
+
+
+%endif + +%if msg: +

${msg}

+%endif + +%if datatable: + +
+
+

+


+

${datatable['title']}

+ + + %for hname in datatable['header']: + + %endfor + + %for row in datatable['data']: + + %for value in row: + + %endfor + + %endfor +
${hname}
${value}
+

+%endif + +%if plots: + + %for plot in plots: +
+

${plot['title']}

+
+

${plot['info']}

+
+
+ +
+
+ %endfor + +%endif + +
+
${_('Django PID')}: ${djangopid} + | ${_('Platform Version')}: ${mitx_version}
+
+
diff --git a/lms/templates/sysadmin_dashboard_gitlogs.html b/lms/templates/sysadmin_dashboard_gitlogs.html new file mode 100644 index 0000000000..d7709c675d --- /dev/null +++ b/lms/templates/sysadmin_dashboard_gitlogs.html @@ -0,0 +1,125 @@ +<%inherit file="/main.html" /> +<%! from django.core.urlresolvers import reverse %> +<%! from django.utils.translation import ugettext as _ %> +<%namespace name='static' file='/static_content.html'/> + +<%block name="headextra"> + <%static:css group='style-course'/> + + + + + +
+
+ +
+

${_('Sysadmin Dashboard')}

+
+

+ ${_('Users')} + ${_('Courses')} + ${_('Staffing and Enrollment')} + ${_('Git Logs')} +

+
+ +
+ + + +

${_('Git Logs')}

+ +%if course_id is None: + + + + + + + + + + %for cil in cilset[:10]: + + + + + + %endfor + +
${_('Date')}${_('Course ID')}${_('Git Action')}
${cil.created}${cil.course_id}${cil.git_log}
+ +%else: + +

${_('Recent git load activity for')} ${course_id}

+ %if error_msg: +

${_('Error')}:

+

${error_msg}

+ %endif + + + + + + + + + + + % for cil in cilset[:2]: + + + + + + + + + + % endfor + +
${_('Date')}${_('Course ID')}${_('git action')}
${cil.created}${cil.course_id}${cil.git_log}
+
${cil.import_log | h}
+
+ +% endif + +
+
+
diff --git a/lms/urls.py b/lms/urls.py index 18351f37a6..ca154c21c5 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -80,6 +80,12 @@ urlpatterns += ( url(r'^jsi18n/$', 'django.views.i18n.javascript_catalog', js_info_dict), ) +# sysadmin dashboard, to see what courses are loaded, to delete & load courses +if settings.FEATURES["ENABLE_SYSADMIN_DASHBOARD"]: + urlpatterns += ( + url(r'^sysadmin/', include('dashboard.sysadmin_urls')), + ) + #Semi-static views (these need to be rendered and have the login bar, but don't change) urlpatterns += ( url(r'^404$', 'static_template_view.views.render', diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 6e50056d1c..e8dcb9191b 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -42,6 +42,7 @@ lazy==1.1 lxml==3.0.1 mako==0.7.3 Markdown==2.2.1 +mongoengine==0.7.10 networkx==1.7 nltk==2.0.4 oauthlib==0.5.1