feat!: remove sysadmin dashboard feature w.r.t 0002-deprecate-sysadmin-dashboard-adr.rst

The sysadmin dashboard feature is converted into a plugable app named edx-sysadmin (https://github.com/mitodl/edx-sysadmin) according to the decisison made at https://github.com/edx/edx-platform/blob/master/lms/djangoapps/dashboard/decisions/0002-deprecate-sysadmin-dashboard-adr.rst. Instances using sysadmin dashboard should use the new plugin from now onwards.

BREAKING CHANGE: sysadmin dashboard is removed
This commit is contained in:
HamzaIbnFarooq
2021-04-02 13:29:16 +05:00
committed by David Ormsbee
parent 6a79d47589
commit 582c02afc4
39 changed files with 2 additions and 2089 deletions

View File

@@ -258,7 +258,6 @@ FEATURES:
ENABLE_PUBLISHER: false
ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES: true
ENABLE_SPECIAL_EXAMS: false
ENABLE_SYSADMIN_DASHBOARD: false
ENABLE_THIRD_PARTY_AUTH: true
ENABLE_VIDEO_UPLOAD_PIPELINE: false
PREVIEW_LMS_BASE: preview.localhost:8000

View File

@@ -351,7 +351,6 @@ FEATURES = {
'ENABLE_COUNTRY_ACCESS': False,
'ENABLE_CREDIT_API': False,
'ENABLE_OAUTH2_PROVIDER': False,
'ENABLE_SYSADMIN_DASHBOARD': False,
'ENABLE_MOBILE_REST_API': False,
'CUSTOM_COURSES_EDX': False,
'ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES': True,

View File

@@ -1,8 +0,0 @@
"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh."""
# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long
from import_shims.warn import warn_deprecated_import
warn_deprecated_import('dashboard', 'lms.djangoapps.dashboard')
from lms.djangoapps.dashboard import *

View File

@@ -1,8 +0,0 @@
"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh."""
# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long
from import_shims.warn import warn_deprecated_import
warn_deprecated_import('dashboard.git_import', 'lms.djangoapps.dashboard.git_import')
from lms.djangoapps.dashboard.git_import import *

View File

@@ -1,8 +0,0 @@
"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh."""
# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long
from import_shims.warn import warn_deprecated_import
warn_deprecated_import('dashboard.management', 'lms.djangoapps.dashboard.management')
from lms.djangoapps.dashboard.management import *

View File

@@ -1,8 +0,0 @@
"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh."""
# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long
from import_shims.warn import warn_deprecated_import
warn_deprecated_import('dashboard.management.commands', 'lms.djangoapps.dashboard.management.commands')
from lms.djangoapps.dashboard.management.commands import *

View File

@@ -1,8 +0,0 @@
"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh."""
# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long
from import_shims.warn import warn_deprecated_import
warn_deprecated_import('dashboard.management.commands.git_add_course', 'lms.djangoapps.dashboard.management.commands.git_add_course')
from lms.djangoapps.dashboard.management.commands.git_add_course import *

View File

@@ -1,8 +0,0 @@
"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh."""
# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long
from import_shims.warn import warn_deprecated_import
warn_deprecated_import('dashboard.management.commands.tests', 'lms.djangoapps.dashboard.management.commands.tests')
from lms.djangoapps.dashboard.management.commands.tests import *

View File

@@ -1,8 +0,0 @@
"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh."""
# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long
from import_shims.warn import warn_deprecated_import
warn_deprecated_import('dashboard.management.commands.tests.test_git_add_course', 'lms.djangoapps.dashboard.management.commands.tests.test_git_add_course')
from lms.djangoapps.dashboard.management.commands.tests.test_git_add_course import *

View File

@@ -1,8 +0,0 @@
"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh."""
# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long
from import_shims.warn import warn_deprecated_import
warn_deprecated_import('dashboard.models', 'lms.djangoapps.dashboard.models')
from lms.djangoapps.dashboard.models import *

View File

@@ -1,8 +0,0 @@
"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh."""
# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long
from import_shims.warn import warn_deprecated_import
warn_deprecated_import('dashboard.sysadmin', 'lms.djangoapps.dashboard.sysadmin')
from lms.djangoapps.dashboard.sysadmin import *

View File

@@ -1,8 +0,0 @@
"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh."""
# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long
from import_shims.warn import warn_deprecated_import
warn_deprecated_import('dashboard.sysadmin_urls', 'lms.djangoapps.dashboard.sysadmin_urls')
from lms.djangoapps.dashboard.sysadmin_urls import *

View File

@@ -1,8 +0,0 @@
"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh."""
# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long
from import_shims.warn import warn_deprecated_import
warn_deprecated_import('dashboard.tests', 'lms.djangoapps.dashboard.tests')
from lms.djangoapps.dashboard.tests import *

View File

@@ -1,8 +0,0 @@
"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh."""
# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long
from import_shims.warn import warn_deprecated_import
warn_deprecated_import('dashboard.tests.test_sysadmin', 'lms.djangoapps.dashboard.tests.test_sysadmin')
from lms.djangoapps.dashboard.tests.test_sysadmin import *

View File

@@ -290,7 +290,6 @@ FEATURES:
ENABLE_PUBLISHER: false
ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES: true
ENABLE_SPECIAL_EXAMS: false
ENABLE_SYSADMIN_DASHBOARD: false
ENABLE_THIRD_PARTY_AUTH: true
ENABLE_VIDEO_UPLOAD_PIPELINE: false
PREVIEW_LMS_BASE: preview.localhost:18000
@@ -307,7 +306,6 @@ FINANCIAL_REPORTS:
STORAGE_TYPE: localfs
FOOTER_ORGANIZATION_IMAGE: images/logo.png
GITHUB_REPO_ROOT: /edx/var/edxapp/data
GIT_REPO_DIR: /edx/var/edxapp/course_repos
GOOGLE_ANALYTICS_ACCOUNT: null
GOOGLE_ANALYTICS_LINKEDIN: ''
GOOGLE_ANALYTICS_TRACKING_ID: ''

View File

@@ -1,352 +0,0 @@
"""
Provides a function for importing a git repository into the lms
instance when using a mongo modulestore
"""
import logging
import os
import re
import subprocess
import mongoengine
from django.conf import settings
from django.core import management
from django.core.management.base import CommandError
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from opaque_keys.edx.locator import CourseLocator
from six import StringIO
from xmodule.util.sandboxing import DEFAULT_PYTHON_LIB_FILENAME
from lms.djangoapps.dashboard.models import CourseImportLog
log = logging.getLogger(__name__)
DEFAULT_GIT_REPO_DIR = '/edx/var/app/edxapp/course_repos'
class GitImportError(Exception):
"""
Exception class for handling the typical errors in a git import.
"""
MESSAGE = None
def __init__(self, message=None):
if message is None:
message = self.MESSAGE
super().__init__(message)
class GitImportErrorNoDir(GitImportError):
"""
GitImportError when no directory exists at the specified path.
"""
def __init__(self, repo_dir):
super().__init__(
_(
"Path {0} doesn't exist, please create it, "
"or configure a different path with "
"GIT_REPO_DIR"
).format(repo_dir)
)
class GitImportErrorUrlBad(GitImportError):
"""
GitImportError when the git url provided wasn't usable.
"""
MESSAGE = _(
'Non usable git url provided. Expecting something like:'
' git@github.com:edx/edx4edx_lite.git'
)
class GitImportErrorBadRepo(GitImportError):
"""
GitImportError when the cloned repository was malformed.
"""
MESSAGE = _('Unable to get git log')
class GitImportErrorCannotPull(GitImportError):
"""
GitImportError when the clone of the repository failed.
"""
MESSAGE = _('git clone or pull failed!')
class GitImportErrorXmlImportFailed(GitImportError):
"""
GitImportError when the course import command failed.
"""
MESSAGE = _('Unable to run import command.')
class GitImportErrorUnsupportedStore(GitImportError):
"""
GitImportError when the modulestore doesn't support imports.
"""
MESSAGE = _('The underlying module store does not support import.')
class GitImportErrorRemoteBranchMissing(GitImportError):
"""
GitImportError when the remote branch doesn't exist.
"""
# Translators: This is an error message when they ask for a
# particular version of a git repository and that version isn't
# available from the remote source they specified
MESSAGE = _('The specified remote branch is not available.')
class GitImportErrorCannotBranch(GitImportError):
"""
GitImportError when the local branch doesn't exist.
"""
# Translators: Error message shown when they have asked for a git
# repository branch, a specific version within a repository, that
# doesn't exist, or there is a problem changing to it.
MESSAGE = _('Unable to switch to specified branch. Please check your branch name.')
def cmd_log(cmd, cwd):
"""
Helper function to redirect stderr to stdout and log the command
used along with the output. Will raise subprocess.CalledProcessError if
command doesn't return 0, and returns the command's output.
"""
output = subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT).decode('utf-8')
log.debug('Command was: %r. Working directory was: %r', ' '.join(cmd), cwd)
log.debug('Command output was: %r', output)
return output
def switch_branch(branch, rdir):
"""
This will determine how to change the branch of the repo, and then
use the appropriate git commands to do so.
Raises an appropriate GitImportError exception if there is any issues with changing
branches.
"""
# Get the latest remote
try:
cmd_log(['git', 'fetch', ], rdir)
except subprocess.CalledProcessError as ex:
log.exception('Unable to fetch remote: %r', ex.output)
raise GitImportErrorCannotBranch() # lint-amnesty, pylint: disable=raise-missing-from
# Check if the branch is available from the remote.
cmd = ['git', 'ls-remote', 'origin', '-h', f'refs/heads/{branch}', ]
try:
output = cmd_log(cmd, rdir)
except subprocess.CalledProcessError as ex:
log.exception('Getting a list of remote branches failed: %r', ex.output)
raise GitImportErrorCannotBranch() # lint-amnesty, pylint: disable=raise-missing-from
if branch not in output:
raise GitImportErrorRemoteBranchMissing()
# Check it the remote branch has already been made locally
cmd = ['git', 'branch', '-a', ]
try:
output = cmd_log(cmd, rdir)
except subprocess.CalledProcessError as ex:
log.exception('Getting a list of local branches failed: %r', ex.output)
raise GitImportErrorCannotBranch() # lint-amnesty, pylint: disable=raise-missing-from
branches = []
for line in output.split('\n'):
branches.append(line.replace('*', '').strip())
if branch not in branches:
# Checkout with -b since it is remote only
cmd = ['git', 'checkout', '--force', '--track',
'-b', branch, f'origin/{branch}', ]
try:
cmd_log(cmd, rdir)
except subprocess.CalledProcessError as ex:
log.exception('Unable to checkout remote branch: %r', ex.output)
raise GitImportErrorCannotBranch() # lint-amnesty, pylint: disable=raise-missing-from
# Go ahead and reset hard to the newest version of the branch now that we know
# it is local.
try:
cmd_log(['git', 'reset', '--hard', f'origin/{branch}', ], rdir)
except subprocess.CalledProcessError as ex:
log.exception('Unable to reset to branch: %r', ex.output)
raise GitImportErrorCannotBranch() # lint-amnesty, pylint: disable=raise-missing-from
def add_repo(repo, rdir_in, branch=None):
"""
This will add a git repo into the mongo modulestore.
If branch is left as None, it will fetch the most recent
version of the current branch.
"""
# pylint: disable=too-many-statements
git_repo_dir = getattr(settings, 'GIT_REPO_DIR', DEFAULT_GIT_REPO_DIR)
git_import_static = getattr(settings, 'GIT_IMPORT_STATIC', True)
git_import_python_lib = getattr(settings, 'GIT_IMPORT_PYTHON_LIB', True)
python_lib_filename = getattr(settings, 'PYTHON_LIB_FILENAME', DEFAULT_PYTHON_LIB_FILENAME)
# Set defaults even if it isn't defined in settings
mongo_db = {
'host': 'localhost',
'port': 27017,
'user': '',
'password': '',
'db': 'xlog',
}
# Allow overrides
if hasattr(settings, 'MONGODB_LOG'):
for config_item in ['host', 'user', 'password', 'db', 'port']:
mongo_db[config_item] = settings.MONGODB_LOG.get(
config_item, mongo_db[config_item])
if not os.path.isdir(git_repo_dir):
raise GitImportErrorNoDir(git_repo_dir)
# pull from git
if not (repo.endswith('.git') or
repo.startswith(('http:', 'https:', 'git:', 'file:'))):
raise GitImportErrorUrlBad()
if rdir_in:
rdir = os.path.basename(rdir_in)
else:
rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0]
log.debug('rdir = %s', rdir)
rdirp = f'{git_repo_dir}/{rdir}'
if os.path.exists(rdirp):
log.info('directory already exists, doing a git pull instead '
'of git clone')
cmd = ['git', 'pull', ]
cwd = rdirp
else:
cmd = ['git', 'clone', repo, ]
cwd = git_repo_dir
cwd = os.path.abspath(cwd)
try:
ret_git = cmd_log(cmd, cwd=cwd)
except subprocess.CalledProcessError as ex:
log.exception('Error running git pull: %r', ex.output)
raise GitImportErrorCannotPull() # lint-amnesty, pylint: disable=raise-missing-from
if branch:
switch_branch(branch, rdirp)
# get commit id
cmd = ['git', 'log', '-1', '--format=%H', ]
try:
commit_id = cmd_log(cmd, cwd=rdirp)
except subprocess.CalledProcessError as ex:
log.exception('Unable to get git log: %r', ex.output)
raise GitImportErrorBadRepo() # lint-amnesty, pylint: disable=raise-missing-from
ret_git += f'\nCommit ID: {commit_id}'
# get branch
cmd = ['git', 'symbolic-ref', '--short', 'HEAD', ]
try:
branch = cmd_log(cmd, cwd=rdirp)
except subprocess.CalledProcessError as ex:
# I can't discover a way to excercise this, but git is complex
# so still logging and raising here in case.
log.exception('Unable to determine branch: %r', ex.output)
raise GitImportErrorBadRepo() # lint-amnesty, pylint: disable=raise-missing-from
ret_git += '{}Branch: {}'.format(' \n', branch)
# Get XML logging logger and capture debug to parse results
output = StringIO()
import_log_handler = logging.StreamHandler(output)
import_log_handler.setLevel(logging.DEBUG)
logger_names = ['xmodule.modulestore.xml_importer', 'git_add_course',
'xmodule.modulestore.xml', 'xmodule.seq_module', ]
loggers = []
for logger_name in logger_names:
logger = logging.getLogger(logger_name)
logger.setLevel(logging.DEBUG)
logger.addHandler(import_log_handler)
loggers.append(logger)
try:
management.call_command(
'import', git_repo_dir, rdir,
nostatic=not git_import_static, nopythonlib=not git_import_python_lib,
python_lib_filename=python_lib_filename
)
except CommandError:
raise GitImportErrorXmlImportFailed() # lint-amnesty, pylint: disable=raise-missing-from
except NotImplementedError:
raise GitImportErrorUnsupportedStore() # lint-amnesty, pylint: disable=raise-missing-from
ret_import = output.getvalue()
# Remove handler hijacks
for logger in loggers:
logger.setLevel(logging.NOTSET)
logger.removeHandler(import_log_handler)
course_key = None
location = 'unknown'
# extract course ID from output of import-command-run and make symlink
# this is needed in order for custom course scripts to work
match = re.search(r'(?ms)===> IMPORTING courselike (\S+)', ret_import)
if match:
course_id = match.group(1).split('/')
# we need to transform course key extracted from logs into CourseLocator instance, because
# we are using split module store and course keys store as instance of CourseLocator.
# please see common.lib.xmodule.xmodule.modulestore.split_mongo.split.SplitMongoModuleStore#make_course_key
# We want set course id in CourseImportLog as CourseLocator. So that in split module
# environment course id remain consistent as CourseLocator instance.
course_key = CourseLocator(*course_id)
cdir = f'{git_repo_dir}/{course_key.course}'
log.debug('Studio course dir = %s', cdir)
if os.path.exists(cdir) and not os.path.islink(cdir):
log.debug(' -> exists, but is not symlink')
log.debug(subprocess.check_output(['ls', '-l', ],
cwd=os.path.abspath(cdir)))
try:
os.rmdir(os.path.abspath(cdir))
except OSError:
log.exception('Failed to remove course directory')
if not os.path.exists(cdir):
log.debug(' -> creating symlink between %s and %s', rdirp, cdir)
try:
os.symlink(os.path.abspath(rdirp), os.path.abspath(cdir))
except OSError:
log.exception('Unable to create course symlink')
log.debug(subprocess.check_output(['ls', '-l', ],
cwd=os.path.abspath(cdir)))
# store import-command-run output in mongo
mongouri = 'mongodb://{user}:{password}@{host}:{port}/{db}'.format(**mongo_db)
try:
if mongo_db['user'] and mongo_db['password']:
mdb = mongoengine.connect(mongo_db['db'], host=mongouri)
else:
mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'], port=mongo_db['port'])
except mongoengine.connection.ConnectionFailure:
log.exception('Unable to connect to mongodb to save log, please '
'check MONGODB_LOG settings')
cil = CourseImportLog(
course_id=course_key,
location=location,
repo_dir=rdir,
created=timezone.now(),
import_log=ret_import,
git_log=ret_git,
)
cil.save()
log.debug('saved CourseImportLog for %s', cil.course_id)
mdb.close()

View File

@@ -1,54 +0,0 @@
"""
Script for importing courseware from git/xml into a mongo modulestore
"""
import logging
from django.core.management.base import BaseCommand, CommandError
from django.utils.translation import ugettext as _
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.xml import XMLModuleStore
from lms.djangoapps.dashboard import git_import
log = logging.getLogger(__name__)
class Command(BaseCommand):
"""
Pull a git repo and import into the mongo based content database.
"""
# Translators: A git repository is a place to store a grouping of
# versioned files. A branch is a sub grouping of a repository that
# has a specific version of the repository. A modulestore is the database used
# to store the courses for use on the Web site.
help = ('Usage: '
'git_add_course repository_url [directory to check out into] [repository_branch] '
'\n{}'.format(_('Import the specified git repository and optional branch into the '
'modulestore and optionally specified directory.')))
def add_arguments(self, parser):
# Positional arguments
parser.add_argument('repository_url')
parser.add_argument('--directory_path', action='store')
parser.add_argument('--repository_branch', action='store')
def handle(self, *args, **options):
"""Check inputs and run the command"""
if isinstance(modulestore, XMLModuleStore):
raise CommandError('This script requires a mongo module store')
rdir_arg = None
branch = None
if options['directory_path']:
rdir_arg = options['directory_path']
if options['repository_branch']:
branch = options['repository_branch']
try:
git_import.add_repo(options['repository_url'], rdir_arg, branch)
except git_import.GitImportError as ex:
raise CommandError(str(ex)) # lint-amnesty, pylint: disable=raise-missing-from

View File

@@ -1,226 +0,0 @@
"""
Provide tests for git_add_course management command.
"""
import logging
import os
import shutil
import subprocess
import unittest
from uuid import uuid4
import pytest
from django.conf import settings
from django.core.management import call_command
from django.core.management.base import CommandError
from django.test.utils import override_settings
from opaque_keys.edx.keys import CourseKey
from six import StringIO
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
from xmodule.modulestore.tests.mongo_connection import MONGO_HOST, MONGO_PORT_NUM
import lms.djangoapps.dashboard.git_import as git_import
from lms.djangoapps.dashboard.git_import import (
GitImportError,
GitImportErrorBadRepo,
GitImportErrorCannotPull,
GitImportErrorNoDir,
GitImportErrorRemoteBranchMissing,
GitImportErrorUrlBad
)
TEST_MONGODB_LOG = {
'host': MONGO_HOST,
'port': MONGO_PORT_NUM,
'user': '',
'password': '',
'db': 'test_xlog',
}
@override_settings(
MONGODB_LOG=TEST_MONGODB_LOG,
GIT_REPO_DIR=settings.TEST_ROOT / f"course_repos_{uuid4().hex}"
)
@unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'),
"ENABLE_SYSADMIN_DASHBOARD not set")
class TestGitAddCourse(SharedModuleStoreTestCase):
"""
Tests the git_add_course management command for proper functions.
"""
TEST_REPO = 'https://github.com/edx/edx4edx_lite.git'
TEST_COURSE = 'MITx/edx4edx/edx4edx'
TEST_BRANCH = 'testing_do_not_delete'
TEST_BRANCH_COURSE = CourseKey.from_string('MITx/edx4edx_branch/edx4edx')
ENABLED_CACHES = ['default', 'mongo_metadata_inheritance', 'loc_cache']
def setUp(self):
super().setUp()
self.git_repo_dir = settings.GIT_REPO_DIR
def assertCommandFailureRegexp(self, regex, *args):
"""
Convenience function for testing command failures
"""
with self.assertRaisesRegex(CommandError, regex):
call_command('git_add_course', *args, stderr=StringIO())
def test_command_args(self):
"""
Validate argument checking
"""
# No argument given.
self.assertCommandFailureRegexp('Error: the following arguments are required: repository_url')
# Extra/Un-named arguments given.
self.assertCommandFailureRegexp(
'Error: unrecognized arguments: blah blah blah',
'blah', 'blah', 'blah', 'blah')
# Not a valid path.
self.assertCommandFailureRegexp(
f'Path {self.git_repo_dir} doesn\'t exist, please create it,',
'blah')
# Test successful import from command
if not os.path.isdir(self.git_repo_dir):
os.mkdir(self.git_repo_dir)
self.addCleanup(shutil.rmtree, self.git_repo_dir)
# Make a course dir that will be replaced with a symlink
# while we are at it.
if not os.path.isdir(self.git_repo_dir / 'edx4edx'):
os.mkdir(self.git_repo_dir / 'edx4edx')
call_command('git_add_course', self.TEST_REPO,
directory_path=self.git_repo_dir / 'edx4edx_lite')
# Test with all three args (branch)
call_command('git_add_course', self.TEST_REPO,
directory_path=self.git_repo_dir / 'edx4edx_lite',
repository_branch=self.TEST_BRANCH)
def test_add_repo(self):
"""
Various exit path tests for test_add_repo
"""
with pytest.raises(GitImportErrorNoDir):
git_import.add_repo(self.TEST_REPO, None, None)
os.mkdir(self.git_repo_dir)
self.addCleanup(shutil.rmtree, self.git_repo_dir)
with pytest.raises(GitImportErrorUrlBad):
git_import.add_repo('foo', None, None)
with pytest.raises(GitImportErrorCannotPull):
git_import.add_repo('file:///foobar.git', None, None)
# Test git repo that exists, but is "broken"
bare_repo = os.path.abspath('{}/{}'.format(settings.TEST_ROOT, 'bare.git'))
os.mkdir(bare_repo)
self.addCleanup(shutil.rmtree, bare_repo)
subprocess.check_output(['git', '--bare', 'init', ], stderr=subprocess.STDOUT,
cwd=bare_repo)
with pytest.raises(GitImportErrorBadRepo):
git_import.add_repo(f'file://{bare_repo}', None, None)
def test_detached_repo(self):
"""
Test repo that is in detached head state.
"""
repo_dir = self.git_repo_dir
# Test successful import from command
try:
os.mkdir(repo_dir)
except OSError:
pass
self.addCleanup(shutil.rmtree, repo_dir)
git_import.add_repo(self.TEST_REPO, repo_dir / 'edx4edx_lite', None)
subprocess.check_output(['git', 'checkout', 'HEAD~2', ],
stderr=subprocess.STDOUT,
cwd=repo_dir / 'edx4edx_lite')
with pytest.raises(GitImportErrorCannotPull):
git_import.add_repo(self.TEST_REPO, repo_dir / 'edx4edx_lite', None)
def test_branching(self):
"""
Exercise branching code of import
"""
repo_dir = self.git_repo_dir
# Test successful import from command
if not os.path.isdir(repo_dir):
os.mkdir(repo_dir)
self.addCleanup(shutil.rmtree, repo_dir)
# Checkout non existent branch
with pytest.raises(GitImportErrorRemoteBranchMissing):
git_import.add_repo(self.TEST_REPO, repo_dir / 'edx4edx_lite', 'asdfasdfasdf')
# Checkout new branch
git_import.add_repo(self.TEST_REPO,
repo_dir / 'edx4edx_lite',
self.TEST_BRANCH)
def_ms = modulestore()
# Validate that it is different than master
assert def_ms.get_course(self.TEST_BRANCH_COURSE) is not None
# Attempt to check out the same branch again to validate branch choosing
# works
git_import.add_repo(self.TEST_REPO,
repo_dir / 'edx4edx_lite',
self.TEST_BRANCH)
# Delete to test branching back to master
def_ms.delete_course(self.TEST_BRANCH_COURSE, ModuleStoreEnum.UserID.test)
assert def_ms.get_course(self.TEST_BRANCH_COURSE) is None
git_import.add_repo(self.TEST_REPO,
repo_dir / 'edx4edx_lite',
'master')
assert def_ms.get_course(self.TEST_BRANCH_COURSE) is None
assert def_ms.get_course(CourseKey.from_string(self.TEST_COURSE)) is not None
def test_branch_exceptions(self):
"""
This wil create conditions to exercise bad paths in the switch_branch function.
"""
# create bare repo that we can mess with and attempt an import
bare_repo = os.path.abspath('{}/{}'.format(settings.TEST_ROOT, 'bare.git'))
os.mkdir(bare_repo)
self.addCleanup(shutil.rmtree, bare_repo)
subprocess.check_output(['git', '--bare', 'init', ], stderr=subprocess.STDOUT,
cwd=bare_repo)
# Build repo dir
repo_dir = self.git_repo_dir
if not os.path.isdir(repo_dir):
os.mkdir(repo_dir)
self.addCleanup(shutil.rmtree, repo_dir)
rdir = '{0}/bare'.format(repo_dir)
with pytest.raises(GitImportErrorBadRepo):
git_import.add_repo(f'file://{bare_repo}', None, None)
# Get logger for checking strings in logs
output = StringIO()
test_log_handler = logging.StreamHandler(output)
test_log_handler.setLevel(logging.DEBUG)
glog = git_import.log
glog.addHandler(test_log_handler)
# Move remote so fetch fails
shutil.move(bare_repo, f'{settings.TEST_ROOT}/not_bare.git')
try:
git_import.switch_branch('master', rdir)
except GitImportError:
assert 'Unable to fetch remote' in output.getvalue()
shutil.move(f'{settings.TEST_ROOT}/not_bare.git', bare_repo)
output.truncate(0)
# Replace origin with a different remote
subprocess.check_output(
['git', 'remote', 'rename', 'origin', 'blah', ],
stderr=subprocess.STDOUT, cwd=rdir
)
with pytest.raises(GitImportError):
git_import.switch_branch('master', rdir)
assert 'Getting a list of remote branches failed' in output.getvalue()

View File

@@ -1,21 +0,0 @@
"""Models for dashboard application"""
import mongoengine
from xmodule.modulestore.mongoengine_fields import CourseKeyField
class CourseImportLog(mongoengine.Document):
"""Mongoengine model for git log"""
course_id = CourseKeyField(max_length=128)
# NOTE: this location is not a Location object but a pathname
location = mongoengine.StringField(max_length=168)
import_log = mongoengine.StringField(max_length=20 * 65535)
git_log = mongoengine.StringField(max_length=65535)
repo_dir = mongoengine.StringField(max_length=128)
commit = mongoengine.StringField(max_length=40, null=True)
author = mongoengine.StringField(max_length=500, null=True)
date = mongoengine.DateTimeField()
created = mongoengine.DateTimeField()
meta = {'indexes': ['course_id', 'created'],
'allow_inheritance': False}

View File

@@ -1,500 +0,0 @@
"""
This module creates a sysadmin dashboard for managing and viewing
courses.
"""
import json
import logging
import os
import subprocess
import warnings
from io import StringIO
import mongoengine
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User # lint-amnesty, pylint: disable=imported-auth-user
from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator
from django.db import IntegrityError
from django.http import Http404
from django.utils.decorators import method_decorator
from django.utils.html import escape
from django.utils.translation import ugettext as _
from django.views.decorators.cache import cache_control
from django.views.decorators.csrf import ensure_csrf_cookie
from django.views.decorators.http import condition
from django.views.generic.base import TemplateView
from opaque_keys.edx.keys import CourseKey
from path import Path as path
from xmodule.modulestore.django import modulestore
import lms.djangoapps.dashboard.git_import as git_import
from common.djangoapps.edxmako.shortcuts import render_to_response
from common.djangoapps.student.models import CourseEnrollment, Registration, UserProfile
from common.djangoapps.student.roles import CourseInstructorRole, CourseStaffRole
from common.djangoapps.track import views as track_views
from lms.djangoapps.courseware.courses import get_course_by_id
from lms.djangoapps.dashboard.git_import import GitImportError
from lms.djangoapps.dashboard.models import CourseImportLog
from openedx.core.djangolib.markup import HTML
log = logging.getLogger(__name__)
class SysadminDashboardView(TemplateView):
"""Base class for sysadmin dashboard views with common methods"""
template_name = 'sysadmin_dashboard.html'
def __init__(self, **kwargs):
"""
Initialize base sysadmin dashboard class with modulestore,
modulestore_type and return msg
"""
# Deprecation log for Sysadmin Dashboard
warnings.warn("Sysadmin Dashboard is deprecated. See DEPR-118.", DeprecationWarning)
self.def_ms = modulestore()
self.msg = ''
self.datatable = []
super().__init__(**kwargs)
@method_decorator(ensure_csrf_cookie)
@method_decorator(login_required)
@method_decorator(cache_control(no_cache=True, no_store=True,
must_revalidate=True))
@method_decorator(condition(etag_func=None))
def dispatch(self, *args, **kwargs):
return super().dispatch(*args, **kwargs)
def get_courses(self):
""" Get an iterable list of courses."""
return self.def_ms.get_courses()
class Users(SysadminDashboardView):
"""
The status view provides Web based user management, a listing of
courses loaded, and user statistics
"""
def create_user(self, uname, name, password=None):
""" Creates a user """
if not uname:
return _('Must provide username')
if not name:
return _('Must provide full name')
msg = ''
if not password:
return _('Password must be supplied')
email = uname
if '@' not in email:
msg += _('email address required (not username)')
return msg
new_password = password
user = User(username=uname, email=email, is_active=True)
user.set_password(new_password)
try:
user.save()
except IntegrityError:
msg += _('Oops, failed to create user {user}, {error}').format(
user=user,
error="IntegrityError"
)
return msg
reg = Registration()
reg.register(user)
profile = UserProfile(user=user)
profile.name = name
profile.save()
msg += _('User {user} created successfully!').format(user=user)
return msg
def delete_user(self, uname):
"""Deletes a user from django auth"""
if not uname:
return _('Must provide username')
if '@' in uname:
try:
user = User.objects.get(email=uname)
except User.DoesNotExist as err:
msg = _('Cannot find user with email address {email_addr}').format(email_addr=uname)
return msg
else:
try:
user = User.objects.get(username=uname)
except User.DoesNotExist as err:
msg = _('Cannot find user with username {username} - {error}').format(
username=uname,
error=str(err)
)
return msg
user.delete()
return _('Deleted user {username}').format(username=uname)
def make_datatable(self):
"""
Build the datatable for this view
"""
datatable = {
'header': [
_('Statistic'),
_('Value'),
],
'title': _('Site statistics'),
'data': [
[
_('Total number of users'),
User.objects.all().count(),
],
],
}
return datatable
def get(self, request): # lint-amnesty, pylint: disable=arguments-differ
if not request.user.is_staff:
raise Http404
context = {
'datatable': self.make_datatable(),
'msg': self.msg,
'djangopid': os.getpid(),
'modeflag': {'users': 'active-section'},
}
return render_to_response(self.template_name, context)
def post(self, request):
"""Handle various actions available on page"""
if not request.user.is_staff:
raise Http404
action = request.POST.get('action', '')
track_views.server_track(request, action, {}, page='user_sysdashboard')
if action == 'create_user':
uname = request.POST.get('student_uname', '').strip()
name = request.POST.get('student_fullname', '').strip()
password = request.POST.get('student_password', '').strip()
self.msg = HTML('<h4>{0}</h4><p>{1}</p><hr />{2}').format(
_('Create User Results'),
self.create_user(uname, name, password), self.msg)
elif action == 'del_user':
uname = request.POST.get('student_uname', '').strip()
self.msg = HTML('<h4>{0}</h4><p>{1}</p><hr />{2}').format(
_('Delete User Results'), self.delete_user(uname), self.msg)
context = {
'datatable': self.make_datatable(),
'msg': self.msg,
'djangopid': os.getpid(),
'modeflag': {'users': 'active-section'},
}
return render_to_response(self.template_name, context)
class Courses(SysadminDashboardView):
"""
This manages adding/updating courses from git, deleting courses, and
provides course listing information.
"""
def git_info_for_course(self, cdir):
"""This pulls out some git info like the last commit"""
cmd = ''
gdir = settings.DATA_DIR / cdir
info = ['', '', '']
# Try the data dir, then try to find it in the git import dir
if not gdir.exists():
git_repo_dir = getattr(settings, 'GIT_REPO_DIR', git_import.DEFAULT_GIT_REPO_DIR)
gdir = path(git_repo_dir) / cdir
if not gdir.exists():
return info
cmd = ['git', 'log', '-1',
'--format=format:{ "commit": "%H", "author": "%an %ae", "date": "%ad"}', ]
try:
output_json = json.loads(subprocess.check_output(cmd, cwd=gdir).decode('utf-8'))
info = [output_json['commit'],
output_json['date'],
output_json['author'], ]
except OSError as error:
log.warning("Error fetching git data: %s - %s", str(cdir), str(error))
except (ValueError, subprocess.CalledProcessError):
pass
return info
def get_course_from_git(self, gitloc, branch):
"""This downloads and runs the checks for importing a course in git"""
if not (gitloc.endswith('.git') or gitloc.startswith('http:') or
gitloc.startswith('https:') or gitloc.startswith('git:')):
return _("The git repo location should end with '.git', "
"and be a valid url")
return self.import_mongo_course(gitloc, branch)
def import_mongo_course(self, gitloc, branch):
"""
Imports course using management command and captures logging output
at debug level for display in template
"""
msg = ''
log.debug('Adding course using git repo %s', gitloc)
# Grab logging output for debugging imports
output = StringIO()
import_log_handler = logging.StreamHandler(output)
import_log_handler.setLevel(logging.DEBUG)
logger_names = ['xmodule.modulestore.xml_importer',
'lms.djangoapps.dashboard.git_import',
'xmodule.modulestore.xml',
'xmodule.seq_module', ]
loggers = []
for logger_name in logger_names:
logger = logging.getLogger(logger_name)
logger.setLevel(logging.DEBUG)
logger.addHandler(import_log_handler)
loggers.append(logger)
error_msg = ''
try:
git_import.add_repo(gitloc, None, branch)
except GitImportError as ex:
error_msg = str(ex)
ret = output.getvalue()
# Remove handler hijacks
for logger in loggers:
logger.setLevel(logging.NOTSET)
logger.removeHandler(import_log_handler)
if error_msg:
msg_header = error_msg
color = 'red'
else:
msg_header = _('Added Course')
color = 'blue'
msg = HTML("<h4 style='color:{0}'>{1}</h4>").format(color, msg_header)
msg += HTML("<pre>{0}</pre>").format(escape(ret))
return msg
def make_datatable(self, courses=None):
"""Creates course information datatable"""
data = []
courses = courses or self.get_courses()
for course in courses:
gdir = course.id.course
data.append([course.display_name, str(course.id)]
+ self.git_info_for_course(gdir))
return dict(header=[_('Course Name'),
_('Directory/ID'),
# Translators: "Git Commit" is a computer command; see http://gitref.org/basic/#commit
_('Git Commit'),
_('Last Change'),
_('Last Editor')],
title=_('Information about all courses'),
data=data)
def get(self, request): # lint-amnesty, pylint: disable=arguments-differ
"""Displays forms and course information"""
if not request.user.is_staff:
raise Http404
context = {
'datatable': self.make_datatable(),
'msg': self.msg,
'djangopid': os.getpid(),
'modeflag': {'courses': 'active-section'},
}
return render_to_response(self.template_name, context)
def post(self, request):
"""Handle all actions from courses view"""
if not request.user.is_staff:
raise Http404
action = request.POST.get('action', '')
track_views.server_track(request, action, {},
page='courses_sysdashboard')
courses = {course.id: course for course in self.get_courses()}
if action == 'add_course':
gitloc = request.POST.get('repo_location', '').strip().replace(' ', '').replace(';', '')
branch = request.POST.get('repo_branch', '').strip().replace(' ', '').replace(';', '')
self.msg += self.get_course_from_git(gitloc, branch)
elif action == 'del_course':
course_id = request.POST.get('course_id', '').strip()
course_key = CourseKey.from_string(course_id)
course_found = False
if course_key in courses:
course_found = True
course = courses[course_key]
else:
try:
course = get_course_by_id(course_key)
course_found = True
except Exception as err: # pylint: disable=broad-except
self.msg += _( # lint-amnesty, pylint: disable=translation-of-non-string
HTML('Error - cannot get course with ID {0}<br/><pre>{1}</pre>')
).format(
course_key,
escape(str(err))
)
if course_found:
# delete course that is stored with mongodb backend
self.def_ms.delete_course(course.id, request.user.id)
# don't delete user permission groups, though
self.msg += \
HTML("<font color='red'>{0} {1} = {2} ({3})</font>").format(
_('Deleted'), str(course.location), str(course.id), course.display_name)
context = {
'datatable': self.make_datatable(list(courses.values())),
'msg': self.msg,
'djangopid': os.getpid(),
'modeflag': {'courses': 'active-section'},
}
return render_to_response(self.template_name, context)
class Staffing(SysadminDashboardView):
"""
The status view provides a view of staffing and enrollment in
courses.
"""
def get(self, request): # lint-amnesty, pylint: disable=arguments-differ
"""Displays course Enrollment and staffing course statistics"""
if not request.user.is_staff:
raise Http404
data = []
for course in self.get_courses():
datum = [course.display_name, course.id]
datum += [CourseEnrollment.objects.filter(
course_id=course.id).count()]
datum += [CourseStaffRole(course.id).users_with_role().count()]
datum += [','.join([x.username for x in CourseInstructorRole(
course.id).users_with_role()])]
data.append(datum)
datatable = dict(header=[_('Course Name'), _('course_id'),
_('# enrolled'), _('# staff'),
_('instructors')],
title=_('Enrollment information for all courses'),
data=data)
context = {
'datatable': datatable,
'msg': self.msg,
'djangopid': os.getpid(),
'modeflag': {'staffing': 'active-section'},
}
return render_to_response(self.template_name, context)
class GitLogs(TemplateView):
"""
This provides a view into the import of courses from git repositories.
It is convenient for allowing course teams to see what may be wrong with
their xml
"""
template_name = 'sysadmin_dashboard_gitlogs.html'
@method_decorator(login_required)
def get(self, request, *args, **kwargs):
"""Shows logs of imports that happened as a result of a git import"""
course_id = kwargs.get('course_id')
if course_id:
course_id = CourseKey.from_string(course_id)
page_size = 10
# Set mongodb defaults even if it isn't defined in settings
mongo_db = {
'host': 'localhost',
'user': '',
'password': '',
'db': 'xlog',
}
# Allow overrides
if hasattr(settings, 'MONGODB_LOG'):
for config_item in ['host', 'user', 'password', 'db', ]:
mongo_db[config_item] = settings.MONGODB_LOG.get(
config_item, mongo_db[config_item])
mongouri = 'mongodb://{user}:{password}@{host}/{db}'.format(**mongo_db)
error_msg = ''
try:
if mongo_db['user'] and mongo_db['password']:
mdb = mongoengine.connect(mongo_db['db'], host=mongouri)
else:
mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'])
except mongoengine.connection.ConnectionError: # lint-amnesty, pylint: disable=no-member
log.exception('Unable to connect to mongodb to save log, '
'please check MONGODB_LOG settings.')
if course_id is None:
# Require staff if not going to specific course
if not request.user.is_staff:
raise Http404
cilset = CourseImportLog.objects.order_by('-created')
else:
# Allow only course team, instructors, and staff
if not (request.user.is_staff or
CourseInstructorRole(course_id).has_user(request.user) or
CourseStaffRole(course_id).has_user(request.user)):
raise Http404
log.debug('course_id=%s', course_id)
cilset = CourseImportLog.objects.filter(
course_id=course_id
).order_by('-created')
log.debug('cilset length=%s', len(cilset))
# Paginate the query set
paginator = Paginator(cilset, page_size)
try:
logs = paginator.page(request.GET.get('page'))
except PageNotAnInteger:
logs = paginator.page(1)
except EmptyPage:
# If the page is too high or low
given_page = int(request.GET.get('page'))
page = min(max(1, given_page), paginator.num_pages)
logs = paginator.page(page)
mdb.close()
context = {
'logs': logs,
'course_id': str(course_id) if course_id else None,
'error_msg': error_msg,
'page_size': page_size
}
return render_to_response(self.template_name, context)

View File

@@ -1,17 +0,0 @@
"""
Urls for sysadmin dashboard feature
"""
from django.conf.urls import url
from . import sysadmin
urlpatterns = [
url(r'^$', sysadmin.Users.as_view(), name="sysadmin"),
url(r'^courses/?$', sysadmin.Courses.as_view(), name="sysadmin_courses"),
url(r'^staffing/?$', sysadmin.Staffing.as_view(), name="sysadmin_staffing"),
url(r'^gitlogs/?$', sysadmin.GitLogs.as_view(), name="gitlogs"),
url(r'^gitlogs/(?P<course_id>.+)$', sysadmin.GitLogs.as_view(),
name="gitlogs_detail"),
]

View File

@@ -1,355 +0,0 @@
"""
Provide tests for sysadmin dashboard feature in sysadmin.py
"""
import glob
import os
import re
import shutil
import unittest
from datetime import datetime
from uuid import uuid4
import mongoengine
from django.conf import settings
from django.test.client import Client
from django.test.utils import override_settings
from django.urls import reverse
from opaque_keys.edx.locator import CourseLocator
from pytz import UTC
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.django_utils import TEST_DATA_SPLIT_MODULESTORE, SharedModuleStoreTestCase
from xmodule.modulestore.tests.mongo_connection import MONGO_HOST, MONGO_PORT_NUM
from common.djangoapps.student.roles import CourseStaffRole, GlobalStaff
from common.djangoapps.student.tests.factories import UserFactory
from common.djangoapps.util.date_utils import DEFAULT_DATE_TIME_FORMAT, get_time_display
from lms.djangoapps.dashboard.git_import import GitImportErrorNoDir
from lms.djangoapps.dashboard.models import CourseImportLog
from openedx.core.djangolib.markup import Text
TEST_MONGODB_LOG = {
'host': MONGO_HOST,
'port': MONGO_PORT_NUM,
'user': '',
'password': '',
'db': 'test_xlog',
}
class SysadminBaseTestCase(SharedModuleStoreTestCase):
"""
Base class with common methods used in XML and Mongo tests
"""
TEST_REPO = 'https://github.com/edx/edx4edx_lite.git'
TEST_BRANCH = 'testing_do_not_delete'
TEST_BRANCH_COURSE = CourseLocator.from_string('course-v1:MITx+edx4edx_branch+edx4edx')
MODULESTORE = TEST_DATA_SPLIT_MODULESTORE
def setUp(self):
"""Setup test case by adding primary user."""
super().setUp()
self.user = UserFactory.create(username='test_user',
email='test_user+sysadmin@edx.org',
password='foo')
self.client = Client()
def _setstaff_login(self):
"""Makes the test user staff and logs them in"""
GlobalStaff().add_users(self.user)
self.client.login(username=self.user.username, password='foo')
def _add_edx4edx(self, branch=None):
"""Adds the edx4edx sample course"""
post_dict = {'repo_location': self.TEST_REPO, 'action': 'add_course', }
if branch:
post_dict['repo_branch'] = branch
return self.client.post(reverse('sysadmin_courses'), post_dict)
def _rm_edx4edx(self):
"""Deletes the sample course from the XML store"""
def_ms = modulestore()
course_path = '{}/edx4edx_lite'.format(
os.path.abspath(settings.DATA_DIR))
try:
# using XML store
course = def_ms.courses.get(course_path, None)
except AttributeError:
# Using mongo store
course = def_ms.get_course(CourseLocator('MITx', 'edx4edx', 'edx4edx'))
# Delete git loaded course
response = self.client.post(
reverse('sysadmin_courses'),
{
'course_id': str(course.id),
'action': 'del_course',
}
)
self.addCleanup(self._rm_glob, f'{course_path}_deleted_*')
return response
def _rm_glob(self, path):
"""
Create a shell expansion of passed in parameter and iteratively
remove them. Must only expand to directories.
"""
for path in glob.glob(path): # lint-amnesty, pylint: disable=redefined-argument-from-local
shutil.rmtree(path)
def _mkdir(self, path):
"""
Create directory and add the cleanup for it.
"""
os.mkdir(path)
self.addCleanup(shutil.rmtree, path)
@override_settings(
MONGODB_LOG=TEST_MONGODB_LOG,
GIT_REPO_DIR=settings.TEST_ROOT / f"course_repos_{uuid4().hex}"
)
@unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'),
"ENABLE_SYSADMIN_DASHBOARD not set")
class TestSysAdminMongoCourseImport(SysadminBaseTestCase):
"""
Check that importing into the mongo module store works
"""
@classmethod
def tearDownClass(cls):
"""Delete mongo log entries after test."""
super().tearDownClass()
try:
mongoengine.connect(TEST_MONGODB_LOG['db'])
CourseImportLog.objects.all().delete()
except mongoengine.connection.ConnectionFailure:
pass
def _setstaff_login(self):
"""
Makes the test user staff and logs them in
"""
self.user.is_staff = True
self.user.save()
self.client.login(username=self.user.username, password='foo')
def test_missing_repo_dir(self):
"""
Ensure that we handle a missing repo dir
"""
self._setstaff_login()
if os.path.isdir(settings.GIT_REPO_DIR):
shutil.rmtree(settings.GIT_REPO_DIR)
# Create git loaded course
response = self._add_edx4edx()
self.assertContains(response, Text(str(GitImportErrorNoDir(settings.GIT_REPO_DIR))))
def test_mongo_course_add_delete(self):
"""
This is the same as TestSysadmin.test_xml_course_add_delete,
but it uses a mongo store
"""
self._setstaff_login()
self._mkdir(settings.GIT_REPO_DIR)
def_ms = modulestore()
assert 'xml' != def_ms.get_modulestore_type(None)
self._add_edx4edx()
course = def_ms.get_course(CourseLocator('MITx', 'edx4edx', 'edx4edx'))
assert course is not None
self._rm_edx4edx()
course = def_ms.get_course(CourseLocator('MITx', 'edx4edx', 'edx4edx'))
assert course is None
def test_course_info(self):
"""
Check to make sure we are getting git info for courses
"""
# Regex of first 3 columns of course information table row for
# test course loaded from git. Would not have sha1 if
# git_info_for_course failed.
table_re = re.compile("""
<tr>\\s+
<td>edX\\sAuthor\\sCourse</td>\\s+ # expected test git course name
<td>course-v1:MITx\\+edx4edx\\+edx4edx</td>\\s+ # expected test git course_id
<td>[a-fA-F\\d]{40}</td> # git sha1 hash
""", re.VERBOSE)
self._setstaff_login()
self._mkdir(settings.GIT_REPO_DIR)
# Make sure we don't have any git hashes on the page
response = self.client.get(reverse('sysadmin_courses'))
self.assertNotRegex(response.content.decode('utf-8'), table_re)
# Now add the course and make sure it does match
response = self._add_edx4edx()
self.assertRegex(response.content.decode('utf-8'), table_re)
def test_gitlogs(self):
"""
Create a log entry and make sure it exists
"""
self._setstaff_login()
self._mkdir(settings.GIT_REPO_DIR)
self._add_edx4edx()
response = self.client.get(reverse('gitlogs'))
# Check that our earlier import has a log with a link to details
self.assertContains(response, '/gitlogs/course-v1:MITx+edx4edx+edx4edx')
response = self.client.get(
reverse('gitlogs_detail', kwargs={
'course_id': 'course-v1:MITx+edx4edx+edx4edx'}))
self.assertContains(response, '======&gt; IMPORTING course')
self._rm_edx4edx()
def test_gitlog_date(self):
"""
Make sure the date is timezone-aware and being converted/formatted
properly.
"""
tz_names = [
'America/New_York', # UTC - 5
'Asia/Pyongyang', # UTC + 9
'Europe/London', # UTC
'Canada/Yukon', # UTC - 8
'Europe/Moscow', # UTC + 4
]
tz_format = DEFAULT_DATE_TIME_FORMAT
self._setstaff_login()
self._mkdir(settings.GIT_REPO_DIR)
self._add_edx4edx()
date = CourseImportLog.objects.first().created.replace(tzinfo=UTC)
for timezone in tz_names:
with (override_settings(TIME_ZONE=timezone)): # lint-amnesty, pylint: disable=superfluous-parens
date_text = get_time_display(date, tz_format, settings.TIME_ZONE)
response = self.client.get(reverse('gitlogs'))
self.assertContains(response, date_text)
self._rm_edx4edx()
def test_gitlog_bad_course(self):
"""
Make sure we gracefully handle courses that don't exist.
"""
self._setstaff_login()
response = self.client.get(
reverse('gitlogs_detail', kwargs={
'course_id': 'Not/Real/Testing'}))
self.assertContains(
response,
'No git import logs have been recorded for this course.',
)
def test_gitlog_no_logs(self):
"""
Make sure the template behaves well when rendered despite there not being any logs.
(This is for courses imported using methods other than the git_add_course command)
"""
self._setstaff_login()
self._mkdir(settings.GIT_REPO_DIR)
self._add_edx4edx()
# Simulate a lack of git import logs
import_logs = CourseImportLog.objects.all()
import_logs.delete()
response = self.client.get(
reverse('gitlogs_detail', kwargs={
'course_id': 'course-v1:MITx+edx4edx+edx4edx'
})
)
self.assertContains(response, 'No git import logs have been recorded for this course.')
self._rm_edx4edx()
def test_gitlog_pagination_out_of_range_invalid(self):
"""
Make sure the pagination behaves properly when the requested page is out
of range.
"""
self._setstaff_login()
mongoengine.connect(TEST_MONGODB_LOG['db'])
for _ in range(15):
CourseImportLog(
course_id=CourseLocator.from_string("test/test/test"),
location="location",
import_log="import_log",
git_log="git_log",
repo_dir="repo_dir",
created=datetime.now()
).save()
for page, expected in [(-1, 1), (1, 1), (2, 2), (30, 2), ('abc', 1)]:
response = self.client.get(
'{}?page={}'.format(
reverse('gitlogs'),
page
)
)
self.assertContains(response, f'Page {expected} of 2')
CourseImportLog.objects.delete()
def test_gitlog_courseteam_access(self):
"""
Ensure course team users are allowed to access only their own course.
"""
self._mkdir(settings.GIT_REPO_DIR)
self._setstaff_login()
self._add_edx4edx()
self.user.is_staff = False
self.user.save()
logged_in = self.client.login(username=self.user.username,
password='foo')
response = self.client.get(reverse('gitlogs'))
# Make sure our non privileged user doesn't have access to all logs
assert response.status_code == 404
# Or specific logs
response = self.client.get(reverse('gitlogs_detail', kwargs={
'course_id': 'course-v1:MITx+edx4edx+edx4edx'
}))
assert response.status_code == 404
# Add user as staff in course team
def_ms = modulestore()
course = def_ms.get_course(CourseLocator('MITx', 'edx4edx', 'edx4edx'))
CourseStaffRole(course.id).add_users(self.user)
assert CourseStaffRole(course.id).has_user(self.user)
logged_in = self.client.login(username=self.user.username,
password='foo')
assert logged_in
response = self.client.get(
reverse('gitlogs_detail', kwargs={
'course_id': 'course-v1:MITx+edx4edx+edx4edx'
}))
self.assertContains(response, '======&gt; IMPORTING course')
self._rm_edx4edx()

View File

@@ -427,12 +427,6 @@ def _section_course_info(course, access):
).format(dashboard_link=dashboard_link)
section_data['enrollment_message'] = message
if settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'):
section_data['detailed_gitlogs_url'] = reverse(
'gitlogs_detail',
kwargs={'course_id': str(course_key)}
)
try:
sorted_cutoffs = sorted(list(course.grade_cutoffs.items()), key=lambda i: i[1], reverse=True)
advance = lambda memo, letter_score_tuple: "{}: {}, ".format(letter_score_tuple[0], letter_score_tuple[1]) \

View File

@@ -195,17 +195,6 @@ FEATURES = {
# .. toggle_creation_date: 2013-04-13
'ENABLE_MASQUERADE': True,
# .. toggle_name: FEATURES['ENABLE_SYSADMIN_DASHBOARD']
# .. toggle_implementation: DjangoSetting
# .. toggle_default: False
# .. toggle_description: enables dashboard at /syadmin/ for django staff, for seeing overview of system status, for
# deleting and loading courses, for seeing log of git imports of courseware. Note that some views are noopen_edx
# .. toggle_use_cases: temporary, open_edx
# .. toggle_creation_date: 2013-12-12
# .. toggle_target_removal_date: None
# .. toggle_warnings: This feature is not supported anymore and should have a target removal date.
'ENABLE_SYSADMIN_DASHBOARD': False, # sysadmin dashboard, to see what courses are loaded, to delete & load courses
# .. toggle_name: FEATURES['DISABLE_LOGIN_BUTTON']
# .. toggle_implementation: DjangoSetting
# .. toggle_default: False
@@ -1268,8 +1257,6 @@ DATA_DIR = '/edx/var/edxapp/data'
# The banner is only rendered when the switch is activated.
MAINTENANCE_BANNER_TEXT = 'Sample banner message'
GIT_REPO_DIR = '/edx/var/edxapp/course_repos'
DJFS = {
'type': 'osfs',
'directory_root': '/edx/var/edxapp/django-pyfs/static/django-pyfs',
@@ -2845,7 +2832,6 @@ INSTALLED_APPS = [
'eventtracking.django.apps.EventTrackingConfig',
'common.djangoapps.util',
'lms.djangoapps.certificates.apps.CertificatesConfig',
'lms.djangoapps.dashboard',
'lms.djangoapps.instructor_task',
'openedx.core.djangoapps.course_groups',
'lms.djangoapps.bulk_email',

View File

@@ -334,12 +334,10 @@ COMMENTS_SERVICE_URL = ENV_TOKENS.get("COMMENTS_SERVICE_URL", '')
COMMENTS_SERVICE_KEY = ENV_TOKENS.get("COMMENTS_SERVICE_KEY", '')
CERT_QUEUE = ENV_TOKENS.get("CERT_QUEUE", 'test-pull')
# git repo loading environment
GIT_REPO_DIR = ENV_TOKENS.get('GIT_REPO_DIR', '/edx/var/edxapp/course_repos')
GIT_IMPORT_STATIC = ENV_TOKENS.get('GIT_IMPORT_STATIC', True)
GIT_IMPORT_PYTHON_LIB = ENV_TOKENS.get('GIT_IMPORT_PYTHON_LIB', True)
# Python lib settings
PYTHON_LIB_FILENAME = ENV_TOKENS.get('PYTHON_LIB_FILENAME', 'python_lib.zip')
# Code jail settings
for name, value in ENV_TOKENS.get("CODE_JAIL", {}).items():
oldvalue = CODE_JAIL.get(name)
if isinstance(oldvalue, dict):

View File

@@ -270,10 +270,6 @@ OAUTH_ENFORCE_SECURE = False
FEATURES['ENABLE_MOBILE_REST_API'] = True
FEATURES['ENABLE_VIDEO_ABSTRACTION_LAYER_API'] = True
########################### SYSADMIN DASHBOARD ################################
FEATURES['ENABLE_SYSADMIN_DASHBOARD'] = True
GIT_REPO_DIR = TEST_ROOT / "course_repos"
################################# CELERY ######################################
CELERY_ALWAYS_EAGER = True

View File

@@ -12,7 +12,6 @@ from openedx.core.djangoapps.site_configuration import helpers as configuration_
<%
show_explore_courses = settings.FEATURES.get('COURSES_ARE_BROWSABLE')
show_sysadmin_dashboard = settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD','') and user.is_staff
self.real_user = getattr(user, 'real_user', user)
enable_help_link = settings.FEATURES.get('ENABLE_HELP_LINK')
@@ -53,12 +52,6 @@ from openedx.core.djangoapps.site_configuration import helpers as configuration_
</a>
</div>
% endif
% if show_sysadmin_dashboard:
<div class="mobile-nav-item hidden-mobile nav-item nav-tab">
## Translators: This is short for "System administration".
<a class="tab-nav-link" href="${reverse('sysadmin')}">${_("Sysadmin")}</a>
</div>
% endif
</div>
<div class="secondary">
% if enable_help_link:

View File

@@ -114,16 +114,6 @@ from openedx.core.djangolib.markup import HTML, Text
</li>
</ul>
%if settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD', '') and user.is_staff:
<p>
## Translators: git is a version-control system; see http://git-scm.com/about
${Text(_("View detailed Git import logs for this course {link_start}by clicking here{link_end}.")).format(
link_start=HTML('<a href="{}">').format(section_data['detailed_gitlogs_url']),
link_end=HTML('</a>')
)}
</p>
%endif
</div>
%if settings.FEATURES.get('ENABLE_INSTRUCTOR_BACKGROUND_TASKS'):

View File

@@ -47,13 +47,6 @@ from django.utils.translation import ugettext as _
</li>
% endif
% endif
% if settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD','') and user.is_staff:
<li class="nav-item mt-2 nav-item-open-collapsed">
## Translators: This is short for "System administration".
<a class="nav-link" href="${reverse('sysadmin')}">${_("Sysadmin")}</a>
</li>
% endif
</ul>
<ul class="navbar-nav navbar-right">

View File

@@ -30,12 +30,6 @@ from django.utils.translation import ugettext as _
</li>
% endif
% endif
% if settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD','') and user.is_staff:
<li class="item">
## Translators: This is short for "System administration".
<a class="btn" href="${reverse('sysadmin')}">${_("Sysadmin")}</a>
</li>
% endif
</%block>
</ol>

View File

@@ -1,192 +0,0 @@
<%page expression_filter="h"/>
<%inherit file="/main.html" />
<%namespace name='static' file='/static_content.html'/>
<%!
from django.urls import reverse
from django.utils.translation import ugettext as _
%>
<%block name="headextra">
<%static:css group='style-course'/>
</%block>
<style type="text/css">
.warning-alert h5{
color: red;
text-align: left;
text-decoration: underline;
margin-bottom: 10px;
}
a.active-section {
color: #551A8B;
}
.sysadmin-dashboard-content h2 a {
margin-right: 1.2em;
}
table.stat_table {
font-family: verdana,arial,sans-serif;
font-size:11px;
color:#333333;
border-width: 1px;
border-color: #666666;
border-collapse: collapse;
}
table.stat_table th {
border-width: 1px;
padding: 8px;
border-style: solid;
border-color: #666666;
background-color: #dedede;
}
table.stat_table td {
border-width: 1px;
padding: 8px;
border-style: solid;
border-color: #666666;
background-color: #ffffff;
}
a.selectedmode { background-color: yellow; }
textarea {
height: 200px;
}
</style>
<section class="container">
<div class="sysadmin-dashboard-wrapper">
<section class="sysadmin-dashboard-content" style="margin-left:10pt;margin-top:10pt;margin-right:10pt;margin-bottom:20pt">
<h1>${_('Sysadmin Dashboard')}</h1>
<hr />
<h2 class="instructor-nav">
<a href="${reverse('sysadmin')}" class="${modeflag.get('users')}">${_('Users')}</a>
<a href="${reverse('sysadmin_courses')}" class="${modeflag.get('courses')}">${_('Courses')}</a>
<a href="${reverse('sysadmin_staffing')}" class="${modeflag.get('staffing')}">${_('Staffing and Enrollment')}</a>
## Translators: refers to http://git-scm.com/docs/git-log
<a href="${reverse('gitlogs')}">${_('Git Logs')}</a>
</h2>
<hr />
<div class="warning-alert">
<h5>
${_('Note: Sysadmin panel will be deprecated in the next release.')}
</h5>
</div>
%if modeflag.get('users'):
<h3>${_('User Management')}</h3>
<form name="action" method="POST">
<input type="hidden" name="csrfmiddlewaretoken" value="${ csrf_token }" />
<ul class="list-input">
<li class="field text" style="padding-bottom: 1.2em">
<label for="student_uname">${_('Email or username')}</label>
<input type="text" name="student_uname" size=40 />
</li>
<li class="field text">
<label for="student_fullname">${_('Full Name')}</label>
<input type="text" name="student_fullname" size=40 />
</li>
<li class="field text">
<label for="student_password">${_('Password')}</label>
<input type="password" name="student_password" size=40 />
</li>
</ul>
<div class="form-actions">
<p>
<button type="submit" name="action" value="del_user">${_('Delete user')}</button>
<button type="submit" name="action" value="create_user">${_('Create user')}</button>
</p>
</div>
<hr />
<hr width="40%" style="align:left">
</form>
%endif
%if modeflag.get('staffing'):
<p>${_("Go to each individual course's Instructor dashboard to manage course enrollment.")}</p>
<hr />
%endif
%if modeflag.get('courses'):
<h3>${_('Administer Courses')}</h3><br/>
<form name="action" method="POST">
<input type="hidden" name="csrfmiddlewaretoken" value="${ csrf_token }" />
<ul class="list-input">
<li class="field text">
<label for="repo_location">
## Translators: Repo is short for git repository or source of
## courseware; see http://git-scm.com/about
${_('Repo Location')}:
</label>
<input type="text" name="repo_location" style="width:60%" />
</li>
<li class="field text">
<label for="repo_location">
## Translators: Repo is short for git repository (http://git-scm.com/about) or source of
## courseware and branch is a specific version within that repository
${_('Repo Branch (optional)')}:
</label>
<input type="text" name="repo_branch" style="width:60%" />
</li>
</ul>
<div class="form-actions">
## Translators: GitHub is a popular website for hosting code
<button type="submit" name="action" value="add_course">${_('Load new course from GitHub')}</button>
</div>
<hr />
<ul class="list-input">
<li class="field text">
<label for="course_id">
## Translators: 'dir' is short for 'directory'
${_('Course ID or dir')}:
</label>
<input type="text" name="course_id" style="width:60%" />
</li>
</ul>
<div class="form-actions">
<button type="submit" name="action" value="del_course">${_('Delete course from site')}</button>
</div>
</form>
<hr style="width:40%" />
%endif
%if msg:
<p>${msg}</p>
%endif
%if datatable:
<br/>
<br/>
<p>
<hr width="100%">
<h2>${datatable['title']}</h2>
<table class="stat_table">
<tr>
%for hname in datatable['header']:
<th>${hname}</th>
%endfor
</tr>
%for row in datatable['data']:
<tr>
%for value in row:
<td>${value}</td>
%endfor
</tr>
%endfor
</table>
</p>
%endif
</section>
<div style="text-align:right; float: right"><span id="djangopid">'Django PID': ${djangopid}</span>
</div>
</div>
</section>

View File

@@ -1,207 +0,0 @@
<%page expression_filter="h"/>
<%inherit file="/main.html" />
<%!
import six
from django.urls import reverse
from django.utils.translation import ugettext as _
from django.utils.timezone import utc as UTC
from common.djangoapps.util.date_utils import get_time_display, DEFAULT_DATE_TIME_FORMAT
from django.conf import settings
%>
<%namespace name='static' file='/static_content.html'/>
<%block name="headextra">
<%static:css group='style-course'/>
<script type="text/javascript" src="${static.url('js/vendor/flot/jquery.flot.js')}"></script>
<script type="text/javascript" src="${static.url('js/vendor/flot/jquery.flot.axislabels.js')}"></script>
<script>
$(function() {
$(".toggle-import-log").click(function(e) {
var self = $(this);
var id = self.data("import-log");
$("#import-log-" + id).toggle({
duration: 200
});
if (self.html() === "[ + ]") {
self.html(edx.HtmlUtils.HTML("[ &#8722; ]").toString());
} else {
self.text("[ + ]");
}
e.preventDefault();
});
});
</script>
</%block>
<%def name="pagination()">
<div class="pagination">
%if logs.has_previous():
<span class="previous-page">
<a href="?page=${logs.previous_page_number()}">
${_("previous")}
</a>
</span>
%endif
${_("Page {current_page} of {total_pages}".format(
current_page=logs.number,
total_pages=logs.paginator.num_pages
))}
%if logs.has_next():
<span class="next-page">
<a href="?page=${logs.next_page_number()}">
${_("next")}
</a>
</span>
%endif
</div>
</%def>
<style type="text/css">
a.active-section {
color: #551A8B;
}
.sysadmin-dashboard-content h2 a {
margin-right: 1.2em;
}
table.stat_table {
font-family: verdana,arial,sans-serif;
font-size:11px;
color:#333333;
border-width: 1px;
border-color: #666666;
border-collapse: collapse;
}
table.stat_table th {
border-width: 1px;
padding: 8px;
border-style: solid;
border-color: #666666;
background-color: #dedede;
}
table.stat_table td {
border-width: 1px;
padding: 8px;
border-style: solid;
border-color: #666666;
background-color: #ffffff;
}
.import-log {
display: none;
}
.pagination, .page-status {
text-align: center;
padding: 12px 0 12px 0;
}
.pagination .previous-page {
padding-right: 10px;
}
.pagination .next-page {
padding-left: 10px;
}
a.selectedmode { background-color: yellow; }
textarea {
height: 200px;
}
</style>
<section class="container">
<div class="sysadmin-dashboard-wrapper">
<section class="sysadmin-dashboard-content" style="margin-left:10pt;margin-top:10pt;margin-right:10pt;margin-bottom:20pt">
<h1>${_('Sysadmin Dashboard')}</h1>
<hr />
<h2 class="instructor-nav">
<a href="${reverse('sysadmin')}">${_('Users')}</a>
<a href="${reverse('sysadmin_courses')}">${_('Courses')}</a>
<a href="${reverse('sysadmin_staffing')}">${_('Staffing and Enrollment')}</a>
## Translators: refers to http://git-scm.com/docs/git-log
<a href="${reverse('gitlogs')}" class="active-section">${_('Git Logs')}</a>
</h2>
<hr />
<form name="dashform" method="POST" action="${reverse('sysadmin')}">
<input type="hidden" name="csrfmiddlewaretoken" value="${ csrf_token }">
<input type="hidden" name="dash_mode" value="">
</form>
## Translators: refers to http://git-scm.com/docs/git-log
<h3>${_('Git Logs')}</h3>
%if course_id is not None:
## Translators: Git is a version-control system; see http://git-scm.com/about
<h2>${_('Recent git load activity for {course_id}').format(course_id=course_id)}</h2>
%if error_msg:
<h3>${_('Error')}:</h3>
<p>${error_msg}</p>
%endif
%endif
%if len(logs):
${pagination()}
<table class="stat_table" width="100%">
<thead>
<tr>
<th width="15%">${_('Date')}</th>
<th width="15%">${_('Course ID')}</th>
## Translators: Git is a version-control system; see http://git-scm.com/about
<th>${_('Git Action')}</th>
</tr>
</thead>
<tbody>
%for index, cil in enumerate(logs):
<%
# Appropriate datetime string for current locale and timezone
date = get_time_display(cil.created.replace(tzinfo=UTC),
DEFAULT_DATE_TIME_FORMAT, coerce_tz=settings.TIME_ZONE)
%>
<tr>
<td>${date}</td>
<td>
<a href="${reverse('gitlogs_detail', kwargs={'course_id': six.text_type(cil.course_id)})}">
${cil.course_id}
</a>
</td>
<td>
%if course_id is not None:
<a class="toggle-import-log" data-import-log="${index}" href="#">[ + ]</a>
%endif
${cil.git_log}
</td>
</tr>
## Show the full log of the latest import if viewing logs for a specific course
%if course_id is not None:
<tr class="import-log" id="import-log-${index}">
<td colspan="3">
<pre>
${cil.import_log}
</pre>
</td>
</tr>
%endif
%endfor
</tbody>
</table>
${pagination()}
%else:
<div class="page-status">
%if not course_id:
## If viewing all logs there are no logs available, let the user know.
## Translators: git is a version-control system; see http://git-scm.com/about
${_('No git import logs have been recorded.')}
%else:
## If viewing a single course and there are no logs available, let the user know.
## Translators: git is a version-control system; see http://git-scm.com/about
${_('No git import logs have been recorded for this course.')}
%endif
</div>
%endif
</section>
</div>
</section>

View File

@@ -222,12 +222,6 @@ urlpatterns += [
url(r'^openassessment/fileupload/', include('openassessment.fileupload.urls')),
]
# sysadmin dashboard, to see what courses are loaded, to delete & load courses
if settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'):
urlpatterns += [
url(r'^sysadmin/', include('lms.djangoapps.dashboard.sysadmin_urls')),
]
urlpatterns += [
url(r'^support/', include('lms.djangoapps.support.urls')),
]

View File

@@ -12,7 +12,6 @@ from openedx.core.djangoapps.site_configuration import helpers as configuration_
<%
show_explore_courses = settings.FEATURES.get('COURSES_ARE_BROWSABLE')
show_sysadmin_dashboard = settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD','') and user.is_staff
self.real_user = getattr(user, 'real_user', user)
enable_help_link = settings.FEATURES.get('ENABLE_HELP_LINK')
%>