Several code enhancements to sysadmin dashboard
Improved testing (cleaning up afterwards, catching stdout) Refactored import function to it's own file Removed monkey patch to log capturing, but still needs work Translation fixes Removal of several gettext phrases
This commit is contained in:
Carson Gee
207
lms/djangoapps/dashboard/git_import.py
Normal file
207
lms/djangoapps/dashboard/git_import.py
Normal file
@@ -0,0 +1,207 @@
|
||||
"""
|
||||
Provides a function for importing a git repository into the lms
|
||||
instance when using a mongo modulestore
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import StringIO
|
||||
import subprocess
|
||||
import logging
|
||||
|
||||
from django.conf import settings
|
||||
from django.core import management
|
||||
from django.core.management.base import CommandError
|
||||
from django.utils import timezone
|
||||
from django.utils.translation import ugettext as _
|
||||
import mongoengine
|
||||
|
||||
from dashboard.models import CourseImportLog
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
GIT_REPO_DIR = getattr(settings, 'GIT_REPO_DIR', '/opt/edx/course_repos')
|
||||
GIT_IMPORT_STATIC = getattr(settings, 'GIT_IMPORT_STATIC', True)
|
||||
|
||||
|
||||
class GitImportError(Exception):
|
||||
"""
|
||||
Exception class for handling the typical errors in a git import.
|
||||
"""
|
||||
|
||||
NO_DIR = _("Path {0} doesn't exist, please create it, "
|
||||
"or configure a different path with "
|
||||
"GIT_REPO_DIR").format(GIT_REPO_DIR)
|
||||
URL_BAD = _('Non usable git url provided. Expecting something like:'
|
||||
' git@github.com:mitocw/edx4edx_lite.git')
|
||||
BAD_REPO = _('Unable to get git log')
|
||||
CANNOT_PULL = _('git clone or pull failed!')
|
||||
XML_IMPORT_FAILED = _('Unable to run import command.')
|
||||
UNSUPPORTED_STORE = _('The underlying module store does not support import.')
|
||||
|
||||
|
||||
def cmd_log(cmd, cwd):
|
||||
"""
|
||||
Helper function to redirect stderr to stdout and log the command
|
||||
used along with the output. Will raise subprocess.CalledProcessError if
|
||||
command doesn't return 0, and returns the command's output.
|
||||
"""
|
||||
output = subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT)
|
||||
log.debug('Command was: {0!r}. '
|
||||
'Working directory was: {1!r}'.format(' '.join(cmd), cwd))
|
||||
log.debug('Command output was: {0!r}'.format(output))
|
||||
return output
|
||||
|
||||
|
||||
def add_repo(repo, rdir_in):
|
||||
"""This will add a git repo into the mongo modulestore"""
|
||||
# pylint: disable=R0915
|
||||
|
||||
# Set defaults even if it isn't defined in settings
|
||||
mongo_db = {
|
||||
'host': 'localhost',
|
||||
'user': '',
|
||||
'password': '',
|
||||
'db': 'xlog',
|
||||
}
|
||||
|
||||
# Allow overrides
|
||||
if hasattr(settings, 'MONGODB_LOG'):
|
||||
for config_item in ['host', 'user', 'password', 'db', ]:
|
||||
mongo_db[config_item] = settings.MONGODB_LOG.get(
|
||||
config_item, mongo_db[config_item])
|
||||
|
||||
if not os.path.isdir(GIT_REPO_DIR):
|
||||
raise GitImportError(GitImportError.NO_DIR)
|
||||
# pull from git
|
||||
if not (repo.endswith('.git') or
|
||||
repo.startswith(('http:', 'https:', 'git:', 'file:'))):
|
||||
raise GitImportError(GitImportError.URL_BAD)
|
||||
|
||||
if rdir_in:
|
||||
rdir = os.path.basename(rdir_in)
|
||||
else:
|
||||
rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0]
|
||||
log.debug('rdir = {0}'.format(rdir))
|
||||
|
||||
rdirp = '{0}/{1}'.format(GIT_REPO_DIR, rdir)
|
||||
if os.path.exists(rdirp):
|
||||
log.info('directory already exists, doing a git pull instead '
|
||||
'of git clone')
|
||||
cmd = ['git', 'pull', ]
|
||||
cwd = rdirp
|
||||
else:
|
||||
cmd = ['git', 'clone', repo, ]
|
||||
cwd = GIT_REPO_DIR
|
||||
|
||||
cwd = os.path.abspath(cwd)
|
||||
try:
|
||||
ret_git = cmd_log(cmd, cwd=cwd)
|
||||
except subprocess.CalledProcessError:
|
||||
raise GitImportError(GitImportError.CANNOT_PULL)
|
||||
|
||||
# get commit id
|
||||
cmd = ['git', 'log', '-1', '--format=%H', ]
|
||||
try:
|
||||
commit_id = cmd_log(cmd, cwd=rdirp)
|
||||
except subprocess.CalledProcessError:
|
||||
raise GitImportError(GitImportError.BAD_REPO)
|
||||
|
||||
ret_git += '\nCommit ID: {0}'.format(commit_id)
|
||||
|
||||
# get branch
|
||||
cmd = ['git', 'rev-parse', '--abbrev-ref', 'HEAD', ]
|
||||
try:
|
||||
branch = cmd_log(cmd, cwd=rdirp)
|
||||
except subprocess.CalledProcessError:
|
||||
raise GitImportError(GitImportError.BAD_REPO)
|
||||
|
||||
ret_git += '{0}Branch: {1}'.format(' \n', branch)
|
||||
|
||||
# Get XML logging logger and capture debug to parse results
|
||||
output = StringIO.StringIO()
|
||||
import_log_handler = logging.StreamHandler(output)
|
||||
import_log_handler.setLevel(logging.DEBUG)
|
||||
|
||||
logger_names = ['xmodule.modulestore.xml_importer', 'git_add_course',
|
||||
'xmodule.modulestore.xml', 'xmodule.seq_module', ]
|
||||
loggers = []
|
||||
|
||||
for logger_name in logger_names:
|
||||
logger = logging.getLogger(logger_name)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.addHandler(import_log_handler)
|
||||
loggers.append(logger)
|
||||
|
||||
try:
|
||||
management.call_command('import', GIT_REPO_DIR, rdir,
|
||||
nostatic=not GIT_IMPORT_STATIC)
|
||||
except CommandError:
|
||||
raise GitImportError(GitImportError.XML_IMPORT_FAILED)
|
||||
except NotImplementedError:
|
||||
raise GitImportError(GitImportError.UNSUPPORTED_STORE)
|
||||
|
||||
ret_import = output.getvalue()
|
||||
|
||||
# Remove handler hijacks
|
||||
for logger in loggers:
|
||||
logger.setLevel(logging.NOTSET)
|
||||
logger.removeHandler(import_log_handler)
|
||||
|
||||
course_id = 'unknown'
|
||||
location = 'unknown'
|
||||
|
||||
# extract course ID from output of import-command-run and make symlink
|
||||
# this is needed in order for custom course scripts to work
|
||||
match = re.search('(?ms)===> IMPORTING course to location ([^ \n]+)',
|
||||
ret_import)
|
||||
if match:
|
||||
location = match.group(1).strip()
|
||||
log.debug('location = {0}'.format(location))
|
||||
course_id = location.replace('i4x://', '').replace(
|
||||
'/course/', '/').split('\n')[0].strip()
|
||||
|
||||
cdir = '{0}/{1}'.format(GIT_REPO_DIR, course_id.split('/')[1])
|
||||
log.debug('Studio course dir = {0}'.format(cdir))
|
||||
|
||||
if os.path.exists(cdir) and not os.path.islink(cdir):
|
||||
log.debug(' -> exists, but is not symlink')
|
||||
log.debug(subprocess.check_output(['ls', '-l', ],
|
||||
cwd=os.path.abspath(cdir)))
|
||||
try:
|
||||
os.rmdir(os.path.abspath(cdir))
|
||||
except OSError:
|
||||
log.exception('Failed to remove course directory')
|
||||
|
||||
if not os.path.exists(cdir):
|
||||
log.debug(' -> creating symlink between {0} and {1}'.format(rdirp, cdir))
|
||||
try:
|
||||
os.symlink(os.path.abspath(rdirp), os.path.abspath(cdir))
|
||||
except OSError:
|
||||
log.exception('Unable to create course symlink')
|
||||
log.debug(subprocess.check_output(['ls', '-l', ],
|
||||
cwd=os.path.abspath(cdir)))
|
||||
|
||||
# store import-command-run output in mongo
|
||||
mongouri = 'mongodb://{user}:{password}@{host}/{db}'.format(**mongo_db)
|
||||
|
||||
try:
|
||||
if mongo_db['user'] and mongo_db['password']:
|
||||
mdb = mongoengine.connect(mongo_db['db'], host=mongouri)
|
||||
else:
|
||||
mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'])
|
||||
except mongoengine.connection.ConnectionError:
|
||||
log.exception('Unable to connect to mongodb to save log, please '
|
||||
'check MONGODB_LOG settings')
|
||||
cil = CourseImportLog(
|
||||
course_id=course_id,
|
||||
location=location,
|
||||
repo_dir=rdir,
|
||||
created=timezone.now(),
|
||||
import_log=ret_import,
|
||||
git_log=ret_git,
|
||||
)
|
||||
cil.save()
|
||||
|
||||
log.debug('saved CourseImportLog for {0}'.format(cil.course_id))
|
||||
mdb.disconnect()
|
||||
@@ -4,212 +4,22 @@ Script for importing courseware from git/xml into a mongo modulestore
|
||||
|
||||
import os
|
||||
import re
|
||||
import datetime
|
||||
import StringIO
|
||||
import subprocess
|
||||
import logging
|
||||
|
||||
from django.conf import settings
|
||||
from django.core import management
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.utils.translation import ugettext as _
|
||||
import mongoengine
|
||||
|
||||
import dashboard.git_import
|
||||
from dashboard.git_import import GitImportError
|
||||
from dashboard.models import CourseImportLog
|
||||
from xmodule.modulestore.django import modulestore
|
||||
from xmodule.modulestore.xml import XMLModuleStore
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
GIT_REPO_DIR = getattr(settings, 'GIT_REPO_DIR', '/opt/edx/course_repos')
|
||||
GIT_IMPORT_STATIC = getattr(settings, 'GIT_IMPORT_STATIC', True)
|
||||
|
||||
GIT_IMPORT_NO_DIR = -1
|
||||
GIT_IMPORT_URL_BAD = -2
|
||||
GIT_IMPORT_CANNOT_PULL = -3
|
||||
GIT_IMPORT_XML_IMPORT_FAILED = -4
|
||||
GIT_IMPORT_UNSUPPORTED_STORE = -5
|
||||
GIT_IMPORT_MONGODB_FAIL = -6
|
||||
GIT_IMPORT_BAD_REPO = -7
|
||||
|
||||
|
||||
def add_repo(repo, rdir_in):
|
||||
"""This will add a git repo into the mongo modulestore"""
|
||||
# pylint: disable=R0915
|
||||
|
||||
# Set defaults even if it isn't defined in settings
|
||||
mongo_db = {
|
||||
'host': 'localhost',
|
||||
'user': '',
|
||||
'password': '',
|
||||
'db': 'xlog',
|
||||
}
|
||||
|
||||
# Allow overrides
|
||||
if hasattr(settings, 'MONGODB_LOG'):
|
||||
for config_item in ['host', 'user', 'password', 'db', ]:
|
||||
mongo_db[config_item] = settings.MONGODB_LOG.get(
|
||||
config_item, mongo_db[config_item])
|
||||
|
||||
if not os.path.isdir(GIT_REPO_DIR):
|
||||
log.critical(_("Path {0} doesn't exist, please create it, "
|
||||
"or configure a different path with "
|
||||
"GIT_REPO_DIR").format(GIT_REPO_DIR))
|
||||
return GIT_IMPORT_NO_DIR
|
||||
|
||||
# pull from git
|
||||
if not repo.endswith('.git') or not (
|
||||
repo.startswith('http:') or
|
||||
repo.startswith('https:') or
|
||||
repo.startswith('git:') or
|
||||
repo.startswith('file:')):
|
||||
|
||||
log.error(_('Oops, not a git ssh url?'))
|
||||
log.error(_('Expecting something like '
|
||||
'git@github.com:mitocw/edx4edx_lite.git'))
|
||||
return GIT_IMPORT_URL_BAD
|
||||
|
||||
if rdir_in:
|
||||
rdir = rdir_in
|
||||
rdir = os.path.basename(rdir)
|
||||
else:
|
||||
rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0]
|
||||
|
||||
log.debug('rdir = {0}'.format(rdir))
|
||||
|
||||
rdirp = '{0}/{1}'.format(GIT_REPO_DIR, rdir)
|
||||
if os.path.exists(rdirp):
|
||||
log.info(_('directory already exists, doing a git pull instead '
|
||||
'of git clone'))
|
||||
cmd = ['git', 'pull', ]
|
||||
cwd = '{0}/{1}'.format(GIT_REPO_DIR, rdir)
|
||||
else:
|
||||
cmd = ['git', 'clone', repo, ]
|
||||
cwd = GIT_REPO_DIR
|
||||
|
||||
log.debug(cmd)
|
||||
cwd = os.path.abspath(cwd)
|
||||
try:
|
||||
ret_git = subprocess.check_output(cmd, cwd=cwd)
|
||||
except subprocess.CalledProcessError:
|
||||
log.exception(_('git clone or pull failed!'))
|
||||
return GIT_IMPORT_CANNOT_PULL
|
||||
log.debug(ret_git)
|
||||
|
||||
# get commit id
|
||||
cmd = ['git', 'log', '-1', '--format=%H', ]
|
||||
try:
|
||||
commit_id = subprocess.check_output(cmd, cwd=rdirp)
|
||||
except subprocess.CalledProcessError:
|
||||
log.exception(_('Unable to get git log'))
|
||||
return GIT_IMPORT_BAD_REPO
|
||||
|
||||
ret_git += _('\nCommit ID: {0}').format(commit_id)
|
||||
|
||||
# get branch
|
||||
cmd = ['git', 'rev-parse', '--abbrev-ref', 'HEAD', ]
|
||||
try:
|
||||
branch = subprocess.check_output(cmd, cwd=rdirp)
|
||||
except subprocess.CalledProcessError:
|
||||
log.exception(_('Unable to get branch info'))
|
||||
return GIT_IMPORT_BAD_REPO
|
||||
|
||||
ret_git += ' \nBranch: {0}'.format(branch)
|
||||
|
||||
# Get XML logging logger and capture debug to parse results
|
||||
output = StringIO.StringIO()
|
||||
import_log_handler = logging.StreamHandler(output)
|
||||
import_log_handler.setLevel(logging.DEBUG)
|
||||
|
||||
logger_names = ['xmodule.modulestore.xml_importer', 'git_add_course',
|
||||
'xmodule.modulestore.xml', 'xmodule.seq_module', ]
|
||||
loggers = []
|
||||
|
||||
for logger_name in logger_names:
|
||||
logger = logging.getLogger(logger_name)
|
||||
logger.old_level = logger.level
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.addHandler(import_log_handler)
|
||||
loggers.append(logger)
|
||||
|
||||
try:
|
||||
management.call_command('import', GIT_REPO_DIR, rdir,
|
||||
nostatic=not GIT_IMPORT_STATIC)
|
||||
except CommandError:
|
||||
log.exception(_('Unable to run import command.'))
|
||||
return GIT_IMPORT_XML_IMPORT_FAILED
|
||||
except NotImplementedError:
|
||||
log.exception(_('The underlying module store does not support import.'))
|
||||
return GIT_IMPORT_UNSUPPORTED_STORE
|
||||
|
||||
ret_import = output.getvalue()
|
||||
|
||||
# Remove handler hijacks
|
||||
for logger in loggers:
|
||||
logger.setLevel(logger.old_level)
|
||||
logger.removeHandler(import_log_handler)
|
||||
|
||||
course_id = 'unknown'
|
||||
location = 'unknown'
|
||||
|
||||
# extract course ID from output of import-command-run and make symlink
|
||||
# this is needed in order for custom course scripts to work
|
||||
match = re.search('(?ms)===> IMPORTING course to location ([^ \n]+)',
|
||||
ret_import)
|
||||
if match:
|
||||
location = match.group(1).strip()
|
||||
log.debug('location = {0}'.format(location))
|
||||
course_id = location.replace('i4x://', '').replace(
|
||||
'/course/', '/').split('\n')[0].strip()
|
||||
|
||||
cdir = '{0}/{1}'.format(GIT_REPO_DIR, course_id.split('/')[1])
|
||||
log.debug(_('Studio course dir = {0}').format(cdir))
|
||||
|
||||
if os.path.exists(cdir) and not os.path.islink(cdir):
|
||||
log.debug(_(' -> exists, but is not symlink'))
|
||||
log.debug(subprocess.check_output(['ls', '-l', ],
|
||||
cwd=os.path.abspath(cdir)))
|
||||
try:
|
||||
os.rmdir(os.path.abspath(cdir))
|
||||
except OSError:
|
||||
log.exception(_('Failed to remove course directory'))
|
||||
|
||||
if not os.path.exists(cdir):
|
||||
log.debug(_(' -> creating symlink between {0} and {1}').format(rdirp, cdir))
|
||||
try:
|
||||
os.symlink(os.path.abspath(rdirp), os.path.abspath(cdir))
|
||||
except OSError:
|
||||
log.exception(_('Unable to create course symlink'))
|
||||
log.debug(subprocess.check_output(['ls', '-l', ],
|
||||
cwd=os.path.abspath(cdir)))
|
||||
|
||||
# store import-command-run output in mongo
|
||||
mongouri = 'mongodb://{user}:{password}@{host}/{db}'.format(**mongo_db)
|
||||
|
||||
try:
|
||||
if mongo_db['user'] and mongo_db['password']:
|
||||
mdb = mongoengine.connect(mongo_db['db'], host=mongouri)
|
||||
else:
|
||||
mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'])
|
||||
except mongoengine.connection.ConnectionError:
|
||||
log.exception(_('Unable to connect to mongodb to save log, please '
|
||||
'check MONGODB_LOG settings'))
|
||||
return GIT_IMPORT_MONGODB_FAIL
|
||||
cil = CourseImportLog(
|
||||
course_id=course_id,
|
||||
location=location,
|
||||
repo_dir=rdir,
|
||||
created=datetime.datetime.now(),
|
||||
import_log=ret_import,
|
||||
git_log=ret_git,
|
||||
)
|
||||
cil.save()
|
||||
|
||||
log.debug(_('saved CourseImportLog for {0}').format(cil.course_id))
|
||||
mdb.disconnect()
|
||||
return 0
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
"""
|
||||
Pull a git repo and import into the mongo based content database.
|
||||
@@ -222,21 +32,22 @@ class Command(BaseCommand):
|
||||
"""Check inputs and run the command"""
|
||||
|
||||
if isinstance(modulestore, XMLModuleStore):
|
||||
raise CommandError(_('This script requires a mongo module store'))
|
||||
raise CommandError('This script requires a mongo module store')
|
||||
|
||||
if len(args) < 1:
|
||||
raise CommandError(_('This script requires at least one argument, '
|
||||
'the git URL'))
|
||||
raise CommandError('This script requires at least one argument, '
|
||||
'the git URL')
|
||||
|
||||
if len(args) > 2:
|
||||
raise CommandError(_('This script requires no more than two '
|
||||
'arguments'))
|
||||
raise CommandError('This script requires no more than two '
|
||||
'arguments')
|
||||
|
||||
rdir_arg = None
|
||||
|
||||
if len(args) > 1:
|
||||
rdir_arg = args[1]
|
||||
|
||||
if add_repo(args[0], rdir_arg) != 0:
|
||||
raise CommandError(_('Repo was not added, check log output '
|
||||
'for details'))
|
||||
try:
|
||||
dashboard.git_import.add_repo(args[0], rdir_arg)
|
||||
except GitImportError as ex:
|
||||
raise CommandError(str(ex))
|
||||
|
||||
@@ -5,6 +5,7 @@ Provide tests for git_add_course management command.
|
||||
import unittest
|
||||
import os
|
||||
import shutil
|
||||
import StringIO
|
||||
import subprocess
|
||||
|
||||
from django.conf import settings
|
||||
@@ -14,7 +15,8 @@ from django.test.utils import override_settings
|
||||
|
||||
from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
|
||||
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
|
||||
import dashboard.management.commands.git_add_course as git_add_course
|
||||
import dashboard.git_import as git_import
|
||||
from dashboard.git_import import GitImportError
|
||||
|
||||
TEST_MONGODB_LOG = {
|
||||
'host': 'localhost',
|
||||
@@ -43,8 +45,9 @@ class TestGitAddCourse(ModuleStoreTestCase):
|
||||
Convenience function for testing command failures
|
||||
"""
|
||||
with self.assertRaises(SystemExit):
|
||||
self.assertRaisesRegexp(CommandError, regex,
|
||||
call_command('git_add_course', *args))
|
||||
with self.assertRaisesRegexp(CommandError, regex):
|
||||
call_command('git_add_course', *args,
|
||||
stderr=StringIO.StringIO())
|
||||
|
||||
def test_command_args(self):
|
||||
"""
|
||||
@@ -78,27 +81,24 @@ class TestGitAddCourse(ModuleStoreTestCase):
|
||||
"""
|
||||
Various exit path tests for test_add_repo
|
||||
"""
|
||||
self.assertEqual(git_add_course.GIT_IMPORT_NO_DIR,
|
||||
git_add_course.add_repo(self.TEST_REPO, None))
|
||||
try:
|
||||
os.mkdir(getattr(settings, 'GIT_REPO_DIR'))
|
||||
except OSError:
|
||||
pass
|
||||
self.assertEqual(git_add_course.GIT_IMPORT_URL_BAD,
|
||||
git_add_course.add_repo('foo', None))
|
||||
with self.assertRaisesRegexp(GitImportError, GitImportError.NO_DIR):
|
||||
git_import.add_repo(self.TEST_REPO, None)
|
||||
|
||||
self.assertEqual(
|
||||
git_add_course.GIT_IMPORT_CANNOT_PULL,
|
||||
git_add_course.add_repo('file:///foobar.git', None)
|
||||
)
|
||||
os.mkdir(getattr(settings, 'GIT_REPO_DIR'))
|
||||
self.addCleanup(shutil.rmtree, getattr(settings, 'GIT_REPO_DIR'))
|
||||
|
||||
with self.assertRaisesRegexp(GitImportError, GitImportError.URL_BAD):
|
||||
git_import.add_repo('foo', None)
|
||||
|
||||
with self.assertRaisesRegexp(GitImportError, GitImportError.CANNOT_PULL):
|
||||
git_import.add_repo('file:///foobar.git', None)
|
||||
|
||||
# Test git repo that exists, but is "broken"
|
||||
bare_repo = os.path.abspath('{0}/{1}'.format(settings.TEST_ROOT, 'bare.git'))
|
||||
os.mkdir(os.path.abspath(bare_repo))
|
||||
subprocess.call(['git', '--bare', 'init', ], cwd=bare_repo)
|
||||
os.mkdir(bare_repo)
|
||||
self.addCleanup(shutil.rmtree, bare_repo)
|
||||
subprocess.check_output(['git', '--bare', 'init', ], stderr=subprocess.STDOUT,
|
||||
cwd=bare_repo)
|
||||
|
||||
self.assertEqual(
|
||||
git_add_course.GIT_IMPORT_BAD_REPO,
|
||||
git_add_course.add_repo('file://{0}'.format(bare_repo), None)
|
||||
)
|
||||
shutil.rmtree(bare_repo)
|
||||
with self.assertRaisesRegexp(GitImportError, GitImportError.BAD_REPO):
|
||||
git_import.add_repo('file://{0}'.format(bare_repo), None)
|
||||
|
||||
@@ -9,7 +9,6 @@ import os
|
||||
import subprocess
|
||||
import time
|
||||
import StringIO
|
||||
from datetime import datetime
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib.auth import authenticate
|
||||
@@ -20,6 +19,7 @@ from django.db import IntegrityError
|
||||
from django.http import HttpResponse, Http404
|
||||
from django.utils.decorators import method_decorator
|
||||
from django.utils.html import escape
|
||||
from django.utils import timezone
|
||||
from django.utils.translation import ugettext as _
|
||||
from django.views.decorators.cache import cache_control
|
||||
from django.views.generic.base import TemplateView
|
||||
@@ -30,7 +30,8 @@ import mongoengine
|
||||
|
||||
from courseware.courses import get_course_by_id
|
||||
from courseware.roles import CourseStaffRole, CourseInstructorRole
|
||||
import dashboard.management.commands.git_add_course as git_add_course
|
||||
import dashboard.git_import as git_import
|
||||
from dashboard.git_import import GitImportError
|
||||
from dashboard.models import CourseImportLog
|
||||
from external_auth.models import ExternalAuthMap
|
||||
from external_auth.views import generate_password
|
||||
@@ -46,6 +47,7 @@ from xmodule.modulestore.xml import XMLModuleStore
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
||||
class SysadminDashboardView(TemplateView):
|
||||
"""Base class for sysadmin dashboard views with common methods"""
|
||||
|
||||
@@ -214,7 +216,7 @@ class Users(SysadminDashboardView):
|
||||
external_credentials=json.dumps(credentials),
|
||||
)
|
||||
eamap.user = user
|
||||
eamap.dtsignup = datetime.now()
|
||||
eamap.dtsignup = timezone.now()
|
||||
eamap.save()
|
||||
|
||||
msg += _('User {0} created successfully!').format(user)
|
||||
@@ -368,7 +370,7 @@ class Courses(SysadminDashboardView):
|
||||
|
||||
msg = u''
|
||||
|
||||
logging.debug(_('Adding course using git repo {0}').format(gitloc))
|
||||
logging.debug('Adding course using git repo {0}'.format(gitloc))
|
||||
|
||||
# Grab logging output for debugging imports
|
||||
output = StringIO.StringIO()
|
||||
@@ -376,28 +378,38 @@ class Courses(SysadminDashboardView):
|
||||
import_log_handler.setLevel(logging.DEBUG)
|
||||
|
||||
logger_names = ['xmodule.modulestore.xml_importer',
|
||||
'dashboard.management.commands.git_add_course',
|
||||
'xmodule.modulestore.xml', 'xmodule.seq_module', ]
|
||||
'dashboard.git_import',
|
||||
'xmodule.modulestore.xml',
|
||||
'xmodule.seq_module', ]
|
||||
loggers = []
|
||||
|
||||
for logger_name in logger_names:
|
||||
logger = logging.getLogger(logger_name)
|
||||
logger.old_level = logger.level
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.addHandler(import_log_handler)
|
||||
loggers.append(logger)
|
||||
|
||||
git_add_course.add_repo(gitloc, None)
|
||||
error_msg = ''
|
||||
try:
|
||||
git_import.add_repo(gitloc, None)
|
||||
except GitImportError as ex:
|
||||
error_msg = str(ex)
|
||||
ret = output.getvalue()
|
||||
|
||||
# Remove handler hijacks
|
||||
for logger in loggers:
|
||||
logger.setLevel(logger.old_level)
|
||||
logger.setLevel(logging.NOTSET)
|
||||
logger.removeHandler(import_log_handler)
|
||||
|
||||
msg = u"<h4 style='color:blue'>{0} {1}</h4>".format(
|
||||
_('Added course from'), gitloc)
|
||||
msg += _("<pre>{0}</pre>").format(escape(ret))
|
||||
if error_msg:
|
||||
msg_header = error_msg
|
||||
color = 'red'
|
||||
else:
|
||||
msg_header = _('Added Course')
|
||||
color = 'blue'
|
||||
|
||||
msg = u"<h4 style='color:{0}'>{1}</h4>".format(color, msg_header)
|
||||
msg += "<pre>{0}</pre>".format(escape(ret))
|
||||
return msg
|
||||
|
||||
def import_xml_course(self, gitloc, datatable):
|
||||
@@ -422,7 +434,9 @@ class Courses(SysadminDashboardView):
|
||||
cwd = settings.DATA_DIR
|
||||
cwd = os.path.abspath(cwd)
|
||||
try:
|
||||
cmd_output = escape(subprocess.check_output(cmd, cwd=cwd))
|
||||
cmd_output = escape(
|
||||
subprocess.check_output(cmd, stderr=subprocess.STDOUT, cwd=cwd)
|
||||
)
|
||||
except subprocess.CalledProcessError:
|
||||
return _('Unable to clone or pull repository. Please check your url.')
|
||||
|
||||
@@ -660,8 +674,8 @@ class GitLogs(TemplateView):
|
||||
else:
|
||||
mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'])
|
||||
except mongoengine.connection.ConnectionError:
|
||||
logging.exception(_('Unable to connect to mongodb to save log, '
|
||||
'please check MONGODB_LOG settings.'))
|
||||
logging.exception('Unable to connect to mongodb to save log, '
|
||||
'please check MONGODB_LOG settings.')
|
||||
|
||||
if course_id is None:
|
||||
# Require staff if not going to specific course
|
||||
|
||||
@@ -2,9 +2,10 @@
|
||||
Provide tests for sysadmin dashboard feature in sysadmin.py
|
||||
"""
|
||||
|
||||
import unittest
|
||||
import glob
|
||||
import os
|
||||
import shutil
|
||||
import unittest
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib.auth.hashers import check_password
|
||||
@@ -20,6 +21,7 @@ from courseware.roles import CourseStaffRole, GlobalStaff
|
||||
from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
|
||||
from dashboard.models import CourseImportLog
|
||||
from dashboard.sysadmin import Users
|
||||
from dashboard.git_import import GitImportError
|
||||
from external_auth.models import ExternalAuthMap
|
||||
from student.tests.factories import UserFactory
|
||||
from xmodule.modulestore.django import modulestore
|
||||
@@ -43,15 +45,6 @@ class SysadminBaseTestCase(ModuleStoreTestCase):
|
||||
Base class with common methods used in XML and Mongo tests
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
"""Delete all repos imported during tests."""
|
||||
super(SysadminBaseTestCase, cls).tearDownClass()
|
||||
try:
|
||||
shutil.rmtree(getattr(settings, 'GIT_REPO_DIR'))
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def setUp(self):
|
||||
"""Setup test case by adding primary user."""
|
||||
super(SysadminBaseTestCase, self).setUp()
|
||||
@@ -74,18 +67,37 @@ class SysadminBaseTestCase(ModuleStoreTestCase):
|
||||
def _rm_edx4edx(self):
|
||||
"""Deletes the sample course from the XML store"""
|
||||
def_ms = modulestore()
|
||||
course_path = '{0}/edx4edx_lite'.format(
|
||||
os.path.abspath(settings.DATA_DIR))
|
||||
try:
|
||||
# using XML store
|
||||
course = def_ms.courses.get('{0}/edx4edx_lite'.format(
|
||||
os.path.abspath(settings.DATA_DIR)), None)
|
||||
course = def_ms.courses.get(course_path, None)
|
||||
except AttributeError:
|
||||
# Using mongo store
|
||||
course = def_ms.get_course('MITx/edx4edx/edx4edx')
|
||||
|
||||
# Delete git loaded course
|
||||
return self.client.post(reverse('sysadmin_courses'),
|
||||
response = self.client.post(reverse('sysadmin_courses'),
|
||||
{'course_id': course.id,
|
||||
'action': 'del_course', })
|
||||
self.addCleanup(self._rm_glob, '{0}_deleted_*'.format(course_path))
|
||||
|
||||
return response
|
||||
|
||||
def _rm_glob(self, path):
|
||||
"""
|
||||
Create a shell expansion of passed in parameter and iteratively
|
||||
remove them. Must only expand to directories.
|
||||
"""
|
||||
for path in glob.glob(path):
|
||||
shutil.rmtree(path)
|
||||
|
||||
def _mkdir(self, path):
|
||||
"""
|
||||
Create directory and add the cleanup for it.
|
||||
"""
|
||||
os.mkdir(path)
|
||||
self.addCleanup(shutil.rmtree, path)
|
||||
|
||||
|
||||
@unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'),
|
||||
@@ -401,9 +413,7 @@ class TestSysAdminMongoCourseImport(SysadminBaseTestCase):
|
||||
|
||||
# Create git loaded course
|
||||
response = self._add_edx4edx()
|
||||
self.assertIn(escape(_("Path {0} doesn't exist, please create it, or "
|
||||
"configure a different path with "
|
||||
"GIT_REPO_DIR").format(settings.GIT_REPO_DIR)),
|
||||
self.assertIn(GitImportError.NO_DIR,
|
||||
response.content.decode('UTF-8'))
|
||||
|
||||
def test_mongo_course_add_delete(self):
|
||||
@@ -413,8 +423,7 @@ class TestSysAdminMongoCourseImport(SysadminBaseTestCase):
|
||||
"""
|
||||
|
||||
self._setstaff_login()
|
||||
if not os.path.isdir(getattr(settings, 'GIT_REPO_DIR')):
|
||||
os.mkdir(getattr(settings, 'GIT_REPO_DIR'))
|
||||
self._mkdir(getattr(settings, 'GIT_REPO_DIR'))
|
||||
|
||||
def_ms = modulestore()
|
||||
self.assertFalse(isinstance(def_ms, XMLModuleStore))
|
||||
@@ -433,10 +442,7 @@ class TestSysAdminMongoCourseImport(SysadminBaseTestCase):
|
||||
"""
|
||||
|
||||
self._setstaff_login()
|
||||
try:
|
||||
os.mkdir(getattr(settings, 'GIT_REPO_DIR'))
|
||||
except OSError:
|
||||
pass
|
||||
self._mkdir(getattr(settings, 'GIT_REPO_DIR'))
|
||||
|
||||
self._add_edx4edx()
|
||||
response = self.client.get(reverse('gitlogs'))
|
||||
@@ -468,10 +474,7 @@ class TestSysAdminMongoCourseImport(SysadminBaseTestCase):
|
||||
Ensure course team users are allowed to access only their own course.
|
||||
"""
|
||||
|
||||
try:
|
||||
os.mkdir(getattr(settings, 'GIT_REPO_DIR'))
|
||||
except OSError:
|
||||
pass
|
||||
self._mkdir(getattr(settings, 'GIT_REPO_DIR'))
|
||||
|
||||
self._setstaff_login()
|
||||
self._add_edx4edx()
|
||||
|
||||
Reference in New Issue
Block a user