diff --git a/cms/devstack.yml b/cms/devstack.yml index dd70f3e180..109e9c8a39 100644 --- a/cms/devstack.yml +++ b/cms/devstack.yml @@ -258,7 +258,6 @@ FEATURES: ENABLE_PUBLISHER: false ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES: true ENABLE_SPECIAL_EXAMS: false - ENABLE_SYSADMIN_DASHBOARD: false ENABLE_THIRD_PARTY_AUTH: true ENABLE_VIDEO_UPLOAD_PIPELINE: false PREVIEW_LMS_BASE: preview.localhost:8000 diff --git a/cms/envs/common.py b/cms/envs/common.py index 7dd7939d50..e3fcae6567 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -351,7 +351,6 @@ FEATURES = { 'ENABLE_COUNTRY_ACCESS': False, 'ENABLE_CREDIT_API': False, 'ENABLE_OAUTH2_PROVIDER': False, - 'ENABLE_SYSADMIN_DASHBOARD': False, 'ENABLE_MOBILE_REST_API': False, 'CUSTOM_COURSES_EDX': False, 'ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES': True, diff --git a/import_shims/lms/dashboard/__init__.py b/import_shims/lms/dashboard/__init__.py deleted file mode 100644 index f98ec5de42..0000000000 --- a/import_shims/lms/dashboard/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard', 'lms.djangoapps.dashboard') - -from lms.djangoapps.dashboard import * diff --git a/import_shims/lms/dashboard/git_import.py b/import_shims/lms/dashboard/git_import.py deleted file mode 100644 index ddc930d99f..0000000000 --- a/import_shims/lms/dashboard/git_import.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.git_import', 'lms.djangoapps.dashboard.git_import') - -from lms.djangoapps.dashboard.git_import import * diff --git a/import_shims/lms/dashboard/management/__init__.py b/import_shims/lms/dashboard/management/__init__.py deleted file mode 100644 index 343abc8990..0000000000 --- a/import_shims/lms/dashboard/management/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.management', 'lms.djangoapps.dashboard.management') - -from lms.djangoapps.dashboard.management import * diff --git a/import_shims/lms/dashboard/management/commands/__init__.py b/import_shims/lms/dashboard/management/commands/__init__.py deleted file mode 100644 index 81a76be9a8..0000000000 --- a/import_shims/lms/dashboard/management/commands/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.management.commands', 'lms.djangoapps.dashboard.management.commands') - -from lms.djangoapps.dashboard.management.commands import * diff --git a/import_shims/lms/dashboard/management/commands/git_add_course.py b/import_shims/lms/dashboard/management/commands/git_add_course.py deleted file mode 100644 index 2180a53191..0000000000 --- a/import_shims/lms/dashboard/management/commands/git_add_course.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.management.commands.git_add_course', 'lms.djangoapps.dashboard.management.commands.git_add_course') - -from lms.djangoapps.dashboard.management.commands.git_add_course import * diff --git a/import_shims/lms/dashboard/management/commands/tests/__init__.py b/import_shims/lms/dashboard/management/commands/tests/__init__.py deleted file mode 100644 index c487cd88c0..0000000000 --- a/import_shims/lms/dashboard/management/commands/tests/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.management.commands.tests', 'lms.djangoapps.dashboard.management.commands.tests') - -from lms.djangoapps.dashboard.management.commands.tests import * diff --git a/import_shims/lms/dashboard/management/commands/tests/test_git_add_course.py b/import_shims/lms/dashboard/management/commands/tests/test_git_add_course.py deleted file mode 100644 index e1c9c8276f..0000000000 --- a/import_shims/lms/dashboard/management/commands/tests/test_git_add_course.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.management.commands.tests.test_git_add_course', 'lms.djangoapps.dashboard.management.commands.tests.test_git_add_course') - -from lms.djangoapps.dashboard.management.commands.tests.test_git_add_course import * diff --git a/import_shims/lms/dashboard/models.py b/import_shims/lms/dashboard/models.py deleted file mode 100644 index efd61cdb64..0000000000 --- a/import_shims/lms/dashboard/models.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.models', 'lms.djangoapps.dashboard.models') - -from lms.djangoapps.dashboard.models import * diff --git a/import_shims/lms/dashboard/sysadmin.py b/import_shims/lms/dashboard/sysadmin.py deleted file mode 100644 index be756214a7..0000000000 --- a/import_shims/lms/dashboard/sysadmin.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.sysadmin', 'lms.djangoapps.dashboard.sysadmin') - -from lms.djangoapps.dashboard.sysadmin import * diff --git a/import_shims/lms/dashboard/sysadmin_urls.py b/import_shims/lms/dashboard/sysadmin_urls.py deleted file mode 100644 index 87d00a1c20..0000000000 --- a/import_shims/lms/dashboard/sysadmin_urls.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.sysadmin_urls', 'lms.djangoapps.dashboard.sysadmin_urls') - -from lms.djangoapps.dashboard.sysadmin_urls import * diff --git a/import_shims/lms/dashboard/tests/__init__.py b/import_shims/lms/dashboard/tests/__init__.py deleted file mode 100644 index 01a4a959ff..0000000000 --- a/import_shims/lms/dashboard/tests/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.tests', 'lms.djangoapps.dashboard.tests') - -from lms.djangoapps.dashboard.tests import * diff --git a/import_shims/lms/dashboard/tests/test_sysadmin.py b/import_shims/lms/dashboard/tests/test_sysadmin.py deleted file mode 100644 index 6c61277907..0000000000 --- a/import_shims/lms/dashboard/tests/test_sysadmin.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.tests.test_sysadmin', 'lms.djangoapps.dashboard.tests.test_sysadmin') - -from lms.djangoapps.dashboard.tests.test_sysadmin import * diff --git a/lms/devstack.yml b/lms/devstack.yml index 41a72e2938..0e71b134cf 100644 --- a/lms/devstack.yml +++ b/lms/devstack.yml @@ -290,7 +290,6 @@ FEATURES: ENABLE_PUBLISHER: false ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES: true ENABLE_SPECIAL_EXAMS: false - ENABLE_SYSADMIN_DASHBOARD: false ENABLE_THIRD_PARTY_AUTH: true ENABLE_VIDEO_UPLOAD_PIPELINE: false PREVIEW_LMS_BASE: preview.localhost:18000 @@ -307,7 +306,6 @@ FINANCIAL_REPORTS: STORAGE_TYPE: localfs FOOTER_ORGANIZATION_IMAGE: images/logo.png GITHUB_REPO_ROOT: /edx/var/edxapp/data -GIT_REPO_DIR: /edx/var/edxapp/course_repos GOOGLE_ANALYTICS_ACCOUNT: null GOOGLE_ANALYTICS_LINKEDIN: '' GOOGLE_ANALYTICS_TRACKING_ID: '' diff --git a/lms/djangoapps/dashboard/__init__.py b/lms/djangoapps/dashboard/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/lms/djangoapps/dashboard/git_import.py b/lms/djangoapps/dashboard/git_import.py deleted file mode 100644 index 4a7d46ce97..0000000000 --- a/lms/djangoapps/dashboard/git_import.py +++ /dev/null @@ -1,352 +0,0 @@ -""" -Provides a function for importing a git repository into the lms -instance when using a mongo modulestore -""" - - -import logging -import os -import re -import subprocess - -import mongoengine -from django.conf import settings -from django.core import management -from django.core.management.base import CommandError -from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ -from opaque_keys.edx.locator import CourseLocator -from six import StringIO -from xmodule.util.sandboxing import DEFAULT_PYTHON_LIB_FILENAME - -from lms.djangoapps.dashboard.models import CourseImportLog - -log = logging.getLogger(__name__) - -DEFAULT_GIT_REPO_DIR = '/edx/var/app/edxapp/course_repos' - - -class GitImportError(Exception): - """ - Exception class for handling the typical errors in a git import. - """ - MESSAGE = None - - def __init__(self, message=None): - if message is None: - message = self.MESSAGE - super().__init__(message) - - -class GitImportErrorNoDir(GitImportError): - """ - GitImportError when no directory exists at the specified path. - """ - def __init__(self, repo_dir): - super().__init__( - _( - "Path {0} doesn't exist, please create it, " - "or configure a different path with " - "GIT_REPO_DIR" - ).format(repo_dir) - ) - - -class GitImportErrorUrlBad(GitImportError): - """ - GitImportError when the git url provided wasn't usable. - """ - MESSAGE = _( - 'Non usable git url provided. Expecting something like:' - ' git@github.com:edx/edx4edx_lite.git' - ) - - -class GitImportErrorBadRepo(GitImportError): - """ - GitImportError when the cloned repository was malformed. - """ - MESSAGE = _('Unable to get git log') - - -class GitImportErrorCannotPull(GitImportError): - """ - GitImportError when the clone of the repository failed. - """ - MESSAGE = _('git clone or pull failed!') - - -class GitImportErrorXmlImportFailed(GitImportError): - """ - GitImportError when the course import command failed. - """ - MESSAGE = _('Unable to run import command.') - - -class GitImportErrorUnsupportedStore(GitImportError): - """ - GitImportError when the modulestore doesn't support imports. - """ - MESSAGE = _('The underlying module store does not support import.') - - -class GitImportErrorRemoteBranchMissing(GitImportError): - """ - GitImportError when the remote branch doesn't exist. - """ - # Translators: This is an error message when they ask for a - # particular version of a git repository and that version isn't - # available from the remote source they specified - MESSAGE = _('The specified remote branch is not available.') - - -class GitImportErrorCannotBranch(GitImportError): - """ - GitImportError when the local branch doesn't exist. - """ - # Translators: Error message shown when they have asked for a git - # repository branch, a specific version within a repository, that - # doesn't exist, or there is a problem changing to it. - MESSAGE = _('Unable to switch to specified branch. Please check your branch name.') - - -def cmd_log(cmd, cwd): - """ - Helper function to redirect stderr to stdout and log the command - used along with the output. Will raise subprocess.CalledProcessError if - command doesn't return 0, and returns the command's output. - """ - output = subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT).decode('utf-8') - - log.debug('Command was: %r. Working directory was: %r', ' '.join(cmd), cwd) - log.debug('Command output was: %r', output) - return output - - -def switch_branch(branch, rdir): - """ - This will determine how to change the branch of the repo, and then - use the appropriate git commands to do so. - - Raises an appropriate GitImportError exception if there is any issues with changing - branches. - """ - # Get the latest remote - try: - cmd_log(['git', 'fetch', ], rdir) - except subprocess.CalledProcessError as ex: - log.exception('Unable to fetch remote: %r', ex.output) - raise GitImportErrorCannotBranch() # lint-amnesty, pylint: disable=raise-missing-from - - # Check if the branch is available from the remote. - cmd = ['git', 'ls-remote', 'origin', '-h', f'refs/heads/{branch}', ] - try: - output = cmd_log(cmd, rdir) - except subprocess.CalledProcessError as ex: - log.exception('Getting a list of remote branches failed: %r', ex.output) - raise GitImportErrorCannotBranch() # lint-amnesty, pylint: disable=raise-missing-from - if branch not in output: - raise GitImportErrorRemoteBranchMissing() - # Check it the remote branch has already been made locally - cmd = ['git', 'branch', '-a', ] - try: - output = cmd_log(cmd, rdir) - except subprocess.CalledProcessError as ex: - log.exception('Getting a list of local branches failed: %r', ex.output) - raise GitImportErrorCannotBranch() # lint-amnesty, pylint: disable=raise-missing-from - branches = [] - for line in output.split('\n'): - branches.append(line.replace('*', '').strip()) - - if branch not in branches: - # Checkout with -b since it is remote only - cmd = ['git', 'checkout', '--force', '--track', - '-b', branch, f'origin/{branch}', ] - try: - cmd_log(cmd, rdir) - except subprocess.CalledProcessError as ex: - log.exception('Unable to checkout remote branch: %r', ex.output) - raise GitImportErrorCannotBranch() # lint-amnesty, pylint: disable=raise-missing-from - # Go ahead and reset hard to the newest version of the branch now that we know - # it is local. - try: - cmd_log(['git', 'reset', '--hard', f'origin/{branch}', ], rdir) - except subprocess.CalledProcessError as ex: - log.exception('Unable to reset to branch: %r', ex.output) - raise GitImportErrorCannotBranch() # lint-amnesty, pylint: disable=raise-missing-from - - -def add_repo(repo, rdir_in, branch=None): - """ - This will add a git repo into the mongo modulestore. - If branch is left as None, it will fetch the most recent - version of the current branch. - """ - # pylint: disable=too-many-statements - - git_repo_dir = getattr(settings, 'GIT_REPO_DIR', DEFAULT_GIT_REPO_DIR) - git_import_static = getattr(settings, 'GIT_IMPORT_STATIC', True) - git_import_python_lib = getattr(settings, 'GIT_IMPORT_PYTHON_LIB', True) - python_lib_filename = getattr(settings, 'PYTHON_LIB_FILENAME', DEFAULT_PYTHON_LIB_FILENAME) - - # Set defaults even if it isn't defined in settings - mongo_db = { - 'host': 'localhost', - 'port': 27017, - 'user': '', - 'password': '', - 'db': 'xlog', - } - - # Allow overrides - if hasattr(settings, 'MONGODB_LOG'): - for config_item in ['host', 'user', 'password', 'db', 'port']: - mongo_db[config_item] = settings.MONGODB_LOG.get( - config_item, mongo_db[config_item]) - - if not os.path.isdir(git_repo_dir): - raise GitImportErrorNoDir(git_repo_dir) - # pull from git - if not (repo.endswith('.git') or - repo.startswith(('http:', 'https:', 'git:', 'file:'))): - raise GitImportErrorUrlBad() - - if rdir_in: - rdir = os.path.basename(rdir_in) - else: - rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0] - log.debug('rdir = %s', rdir) - - rdirp = f'{git_repo_dir}/{rdir}' - if os.path.exists(rdirp): - log.info('directory already exists, doing a git pull instead ' - 'of git clone') - cmd = ['git', 'pull', ] - cwd = rdirp - else: - cmd = ['git', 'clone', repo, ] - cwd = git_repo_dir - - cwd = os.path.abspath(cwd) - try: - ret_git = cmd_log(cmd, cwd=cwd) - except subprocess.CalledProcessError as ex: - log.exception('Error running git pull: %r', ex.output) - raise GitImportErrorCannotPull() # lint-amnesty, pylint: disable=raise-missing-from - - if branch: - switch_branch(branch, rdirp) - - # get commit id - cmd = ['git', 'log', '-1', '--format=%H', ] - try: - commit_id = cmd_log(cmd, cwd=rdirp) - except subprocess.CalledProcessError as ex: - log.exception('Unable to get git log: %r', ex.output) - raise GitImportErrorBadRepo() # lint-amnesty, pylint: disable=raise-missing-from - - ret_git += f'\nCommit ID: {commit_id}' - - # get branch - cmd = ['git', 'symbolic-ref', '--short', 'HEAD', ] - try: - branch = cmd_log(cmd, cwd=rdirp) - except subprocess.CalledProcessError as ex: - # I can't discover a way to excercise this, but git is complex - # so still logging and raising here in case. - log.exception('Unable to determine branch: %r', ex.output) - raise GitImportErrorBadRepo() # lint-amnesty, pylint: disable=raise-missing-from - - ret_git += '{}Branch: {}'.format(' \n', branch) - - # Get XML logging logger and capture debug to parse results - output = StringIO() - import_log_handler = logging.StreamHandler(output) - import_log_handler.setLevel(logging.DEBUG) - - logger_names = ['xmodule.modulestore.xml_importer', 'git_add_course', - 'xmodule.modulestore.xml', 'xmodule.seq_module', ] - loggers = [] - - for logger_name in logger_names: - logger = logging.getLogger(logger_name) - logger.setLevel(logging.DEBUG) - logger.addHandler(import_log_handler) - loggers.append(logger) - - try: - management.call_command( - 'import', git_repo_dir, rdir, - nostatic=not git_import_static, nopythonlib=not git_import_python_lib, - python_lib_filename=python_lib_filename - ) - except CommandError: - raise GitImportErrorXmlImportFailed() # lint-amnesty, pylint: disable=raise-missing-from - except NotImplementedError: - raise GitImportErrorUnsupportedStore() # lint-amnesty, pylint: disable=raise-missing-from - - ret_import = output.getvalue() - - # Remove handler hijacks - for logger in loggers: - logger.setLevel(logging.NOTSET) - logger.removeHandler(import_log_handler) - - course_key = None - location = 'unknown' - - # extract course ID from output of import-command-run and make symlink - # this is needed in order for custom course scripts to work - match = re.search(r'(?ms)===> IMPORTING courselike (\S+)', ret_import) - if match: - course_id = match.group(1).split('/') - # we need to transform course key extracted from logs into CourseLocator instance, because - # we are using split module store and course keys store as instance of CourseLocator. - # please see common.lib.xmodule.xmodule.modulestore.split_mongo.split.SplitMongoModuleStore#make_course_key - # We want set course id in CourseImportLog as CourseLocator. So that in split module - # environment course id remain consistent as CourseLocator instance. - course_key = CourseLocator(*course_id) - cdir = f'{git_repo_dir}/{course_key.course}' - log.debug('Studio course dir = %s', cdir) - - if os.path.exists(cdir) and not os.path.islink(cdir): - log.debug(' -> exists, but is not symlink') - log.debug(subprocess.check_output(['ls', '-l', ], - cwd=os.path.abspath(cdir))) - try: - os.rmdir(os.path.abspath(cdir)) - except OSError: - log.exception('Failed to remove course directory') - - if not os.path.exists(cdir): - log.debug(' -> creating symlink between %s and %s', rdirp, cdir) - try: - os.symlink(os.path.abspath(rdirp), os.path.abspath(cdir)) - except OSError: - log.exception('Unable to create course symlink') - log.debug(subprocess.check_output(['ls', '-l', ], - cwd=os.path.abspath(cdir))) - - # store import-command-run output in mongo - mongouri = 'mongodb://{user}:{password}@{host}:{port}/{db}'.format(**mongo_db) - - try: - if mongo_db['user'] and mongo_db['password']: - mdb = mongoengine.connect(mongo_db['db'], host=mongouri) - else: - mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'], port=mongo_db['port']) - except mongoengine.connection.ConnectionFailure: - log.exception('Unable to connect to mongodb to save log, please ' - 'check MONGODB_LOG settings') - cil = CourseImportLog( - course_id=course_key, - location=location, - repo_dir=rdir, - created=timezone.now(), - import_log=ret_import, - git_log=ret_git, - ) - cil.save() - - log.debug('saved CourseImportLog for %s', cil.course_id) - mdb.close() diff --git a/lms/djangoapps/dashboard/management/__init__.py b/lms/djangoapps/dashboard/management/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/lms/djangoapps/dashboard/management/commands/__init__.py b/lms/djangoapps/dashboard/management/commands/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/lms/djangoapps/dashboard/management/commands/git_add_course.py b/lms/djangoapps/dashboard/management/commands/git_add_course.py deleted file mode 100644 index 331079a00b..0000000000 --- a/lms/djangoapps/dashboard/management/commands/git_add_course.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -Script for importing courseware from git/xml into a mongo modulestore -""" - - -import logging - -from django.core.management.base import BaseCommand, CommandError -from django.utils.translation import ugettext as _ -from xmodule.modulestore.django import modulestore -from xmodule.modulestore.xml import XMLModuleStore - -from lms.djangoapps.dashboard import git_import - -log = logging.getLogger(__name__) - - -class Command(BaseCommand): - """ - Pull a git repo and import into the mongo based content database. - """ - - # Translators: A git repository is a place to store a grouping of - # versioned files. A branch is a sub grouping of a repository that - # has a specific version of the repository. A modulestore is the database used - # to store the courses for use on the Web site. - help = ('Usage: ' - 'git_add_course repository_url [directory to check out into] [repository_branch] ' - '\n{}'.format(_('Import the specified git repository and optional branch into the ' - 'modulestore and optionally specified directory.'))) - - def add_arguments(self, parser): - # Positional arguments - parser.add_argument('repository_url') - parser.add_argument('--directory_path', action='store') - parser.add_argument('--repository_branch', action='store') - - def handle(self, *args, **options): - """Check inputs and run the command""" - - if isinstance(modulestore, XMLModuleStore): - raise CommandError('This script requires a mongo module store') - - rdir_arg = None - branch = None - if options['directory_path']: - rdir_arg = options['directory_path'] - if options['repository_branch']: - branch = options['repository_branch'] - - try: - git_import.add_repo(options['repository_url'], rdir_arg, branch) - except git_import.GitImportError as ex: - raise CommandError(str(ex)) # lint-amnesty, pylint: disable=raise-missing-from diff --git a/lms/djangoapps/dashboard/management/commands/tests/__init__.py b/lms/djangoapps/dashboard/management/commands/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py b/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py deleted file mode 100644 index 4b1080b0ba..0000000000 --- a/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py +++ /dev/null @@ -1,226 +0,0 @@ -""" -Provide tests for git_add_course management command. -""" -import logging -import os -import shutil -import subprocess -import unittest -from uuid import uuid4 - -import pytest -from django.conf import settings -from django.core.management import call_command -from django.core.management.base import CommandError -from django.test.utils import override_settings -from opaque_keys.edx.keys import CourseKey -from six import StringIO -from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.django import modulestore -from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase -from xmodule.modulestore.tests.mongo_connection import MONGO_HOST, MONGO_PORT_NUM - -import lms.djangoapps.dashboard.git_import as git_import -from lms.djangoapps.dashboard.git_import import ( - GitImportError, - GitImportErrorBadRepo, - GitImportErrorCannotPull, - GitImportErrorNoDir, - GitImportErrorRemoteBranchMissing, - GitImportErrorUrlBad -) -TEST_MONGODB_LOG = { - 'host': MONGO_HOST, - 'port': MONGO_PORT_NUM, - 'user': '', - 'password': '', - 'db': 'test_xlog', -} - - -@override_settings( - MONGODB_LOG=TEST_MONGODB_LOG, - GIT_REPO_DIR=settings.TEST_ROOT / f"course_repos_{uuid4().hex}" -) -@unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'), - "ENABLE_SYSADMIN_DASHBOARD not set") -class TestGitAddCourse(SharedModuleStoreTestCase): - """ - Tests the git_add_course management command for proper functions. - """ - TEST_REPO = 'https://github.com/edx/edx4edx_lite.git' - TEST_COURSE = 'MITx/edx4edx/edx4edx' - TEST_BRANCH = 'testing_do_not_delete' - TEST_BRANCH_COURSE = CourseKey.from_string('MITx/edx4edx_branch/edx4edx') - - ENABLED_CACHES = ['default', 'mongo_metadata_inheritance', 'loc_cache'] - - def setUp(self): - super().setUp() - self.git_repo_dir = settings.GIT_REPO_DIR - - def assertCommandFailureRegexp(self, regex, *args): - """ - Convenience function for testing command failures - """ - with self.assertRaisesRegex(CommandError, regex): - call_command('git_add_course', *args, stderr=StringIO()) - - def test_command_args(self): - """ - Validate argument checking - """ - # No argument given. - self.assertCommandFailureRegexp('Error: the following arguments are required: repository_url') - # Extra/Un-named arguments given. - self.assertCommandFailureRegexp( - 'Error: unrecognized arguments: blah blah blah', - 'blah', 'blah', 'blah', 'blah') - # Not a valid path. - self.assertCommandFailureRegexp( - f'Path {self.git_repo_dir} doesn\'t exist, please create it,', - 'blah') - # Test successful import from command - if not os.path.isdir(self.git_repo_dir): - os.mkdir(self.git_repo_dir) - self.addCleanup(shutil.rmtree, self.git_repo_dir) - - # Make a course dir that will be replaced with a symlink - # while we are at it. - if not os.path.isdir(self.git_repo_dir / 'edx4edx'): - os.mkdir(self.git_repo_dir / 'edx4edx') - - call_command('git_add_course', self.TEST_REPO, - directory_path=self.git_repo_dir / 'edx4edx_lite') - - # Test with all three args (branch) - call_command('git_add_course', self.TEST_REPO, - directory_path=self.git_repo_dir / 'edx4edx_lite', - repository_branch=self.TEST_BRANCH) - - def test_add_repo(self): - """ - Various exit path tests for test_add_repo - """ - with pytest.raises(GitImportErrorNoDir): - git_import.add_repo(self.TEST_REPO, None, None) - - os.mkdir(self.git_repo_dir) - self.addCleanup(shutil.rmtree, self.git_repo_dir) - - with pytest.raises(GitImportErrorUrlBad): - git_import.add_repo('foo', None, None) - - with pytest.raises(GitImportErrorCannotPull): - git_import.add_repo('file:///foobar.git', None, None) - - # Test git repo that exists, but is "broken" - bare_repo = os.path.abspath('{}/{}'.format(settings.TEST_ROOT, 'bare.git')) - os.mkdir(bare_repo) - self.addCleanup(shutil.rmtree, bare_repo) - subprocess.check_output(['git', '--bare', 'init', ], stderr=subprocess.STDOUT, - cwd=bare_repo) - - with pytest.raises(GitImportErrorBadRepo): - git_import.add_repo(f'file://{bare_repo}', None, None) - - def test_detached_repo(self): - """ - Test repo that is in detached head state. - """ - repo_dir = self.git_repo_dir - # Test successful import from command - try: - os.mkdir(repo_dir) - except OSError: - pass - self.addCleanup(shutil.rmtree, repo_dir) - git_import.add_repo(self.TEST_REPO, repo_dir / 'edx4edx_lite', None) - subprocess.check_output(['git', 'checkout', 'HEAD~2', ], - stderr=subprocess.STDOUT, - cwd=repo_dir / 'edx4edx_lite') - with pytest.raises(GitImportErrorCannotPull): - git_import.add_repo(self.TEST_REPO, repo_dir / 'edx4edx_lite', None) - - def test_branching(self): - """ - Exercise branching code of import - """ - repo_dir = self.git_repo_dir - # Test successful import from command - if not os.path.isdir(repo_dir): - os.mkdir(repo_dir) - self.addCleanup(shutil.rmtree, repo_dir) - - # Checkout non existent branch - with pytest.raises(GitImportErrorRemoteBranchMissing): - git_import.add_repo(self.TEST_REPO, repo_dir / 'edx4edx_lite', 'asdfasdfasdf') - - # Checkout new branch - git_import.add_repo(self.TEST_REPO, - repo_dir / 'edx4edx_lite', - self.TEST_BRANCH) - def_ms = modulestore() - # Validate that it is different than master - assert def_ms.get_course(self.TEST_BRANCH_COURSE) is not None - - # Attempt to check out the same branch again to validate branch choosing - # works - git_import.add_repo(self.TEST_REPO, - repo_dir / 'edx4edx_lite', - self.TEST_BRANCH) - - # Delete to test branching back to master - def_ms.delete_course(self.TEST_BRANCH_COURSE, ModuleStoreEnum.UserID.test) - assert def_ms.get_course(self.TEST_BRANCH_COURSE) is None - git_import.add_repo(self.TEST_REPO, - repo_dir / 'edx4edx_lite', - 'master') - assert def_ms.get_course(self.TEST_BRANCH_COURSE) is None - assert def_ms.get_course(CourseKey.from_string(self.TEST_COURSE)) is not None - - def test_branch_exceptions(self): - """ - This wil create conditions to exercise bad paths in the switch_branch function. - """ - # create bare repo that we can mess with and attempt an import - bare_repo = os.path.abspath('{}/{}'.format(settings.TEST_ROOT, 'bare.git')) - os.mkdir(bare_repo) - self.addCleanup(shutil.rmtree, bare_repo) - subprocess.check_output(['git', '--bare', 'init', ], stderr=subprocess.STDOUT, - cwd=bare_repo) - - # Build repo dir - repo_dir = self.git_repo_dir - if not os.path.isdir(repo_dir): - os.mkdir(repo_dir) - self.addCleanup(shutil.rmtree, repo_dir) - - rdir = '{0}/bare'.format(repo_dir) - with pytest.raises(GitImportErrorBadRepo): - git_import.add_repo(f'file://{bare_repo}', None, None) - - # Get logger for checking strings in logs - output = StringIO() - test_log_handler = logging.StreamHandler(output) - test_log_handler.setLevel(logging.DEBUG) - glog = git_import.log - glog.addHandler(test_log_handler) - - # Move remote so fetch fails - shutil.move(bare_repo, f'{settings.TEST_ROOT}/not_bare.git') - try: - git_import.switch_branch('master', rdir) - except GitImportError: - assert 'Unable to fetch remote' in output.getvalue() - shutil.move(f'{settings.TEST_ROOT}/not_bare.git', bare_repo) - output.truncate(0) - - # Replace origin with a different remote - subprocess.check_output( - ['git', 'remote', 'rename', 'origin', 'blah', ], - stderr=subprocess.STDOUT, cwd=rdir - ) - with pytest.raises(GitImportError): - git_import.switch_branch('master', rdir) - assert 'Getting a list of remote branches failed' in output.getvalue() diff --git a/lms/djangoapps/dashboard/models.py b/lms/djangoapps/dashboard/models.py deleted file mode 100644 index 9196f5ecea..0000000000 --- a/lms/djangoapps/dashboard/models.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Models for dashboard application""" - - -import mongoengine -from xmodule.modulestore.mongoengine_fields import CourseKeyField - - -class CourseImportLog(mongoengine.Document): - """Mongoengine model for git log""" - course_id = CourseKeyField(max_length=128) - # NOTE: this location is not a Location object but a pathname - location = mongoengine.StringField(max_length=168) - import_log = mongoengine.StringField(max_length=20 * 65535) - git_log = mongoengine.StringField(max_length=65535) - repo_dir = mongoengine.StringField(max_length=128) - commit = mongoengine.StringField(max_length=40, null=True) - author = mongoengine.StringField(max_length=500, null=True) - date = mongoengine.DateTimeField() - created = mongoengine.DateTimeField() - meta = {'indexes': ['course_id', 'created'], - 'allow_inheritance': False} diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py deleted file mode 100644 index 656d45b603..0000000000 --- a/lms/djangoapps/dashboard/sysadmin.py +++ /dev/null @@ -1,500 +0,0 @@ -""" -This module creates a sysadmin dashboard for managing and viewing -courses. -""" -import json -import logging -import os -import subprocess -import warnings -from io import StringIO - -import mongoengine -from django.conf import settings -from django.contrib.auth.decorators import login_required -from django.contrib.auth.models import User # lint-amnesty, pylint: disable=imported-auth-user -from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator -from django.db import IntegrityError -from django.http import Http404 -from django.utils.decorators import method_decorator -from django.utils.html import escape -from django.utils.translation import ugettext as _ -from django.views.decorators.cache import cache_control -from django.views.decorators.csrf import ensure_csrf_cookie -from django.views.decorators.http import condition -from django.views.generic.base import TemplateView -from opaque_keys.edx.keys import CourseKey -from path import Path as path -from xmodule.modulestore.django import modulestore - -import lms.djangoapps.dashboard.git_import as git_import -from common.djangoapps.edxmako.shortcuts import render_to_response -from common.djangoapps.student.models import CourseEnrollment, Registration, UserProfile -from common.djangoapps.student.roles import CourseInstructorRole, CourseStaffRole -from common.djangoapps.track import views as track_views -from lms.djangoapps.courseware.courses import get_course_by_id -from lms.djangoapps.dashboard.git_import import GitImportError -from lms.djangoapps.dashboard.models import CourseImportLog -from openedx.core.djangolib.markup import HTML - -log = logging.getLogger(__name__) - - -class SysadminDashboardView(TemplateView): - """Base class for sysadmin dashboard views with common methods""" - - template_name = 'sysadmin_dashboard.html' - - def __init__(self, **kwargs): - """ - Initialize base sysadmin dashboard class with modulestore, - modulestore_type and return msg - """ - # Deprecation log for Sysadmin Dashboard - warnings.warn("Sysadmin Dashboard is deprecated. See DEPR-118.", DeprecationWarning) - - self.def_ms = modulestore() - self.msg = '' - self.datatable = [] - super().__init__(**kwargs) - - @method_decorator(ensure_csrf_cookie) - @method_decorator(login_required) - @method_decorator(cache_control(no_cache=True, no_store=True, - must_revalidate=True)) - @method_decorator(condition(etag_func=None)) - def dispatch(self, *args, **kwargs): - return super().dispatch(*args, **kwargs) - - def get_courses(self): - """ Get an iterable list of courses.""" - - return self.def_ms.get_courses() - - -class Users(SysadminDashboardView): - """ - The status view provides Web based user management, a listing of - courses loaded, and user statistics - """ - - def create_user(self, uname, name, password=None): - """ Creates a user """ - - if not uname: - return _('Must provide username') - if not name: - return _('Must provide full name') - - msg = '' - if not password: - return _('Password must be supplied') - - email = uname - - if '@' not in email: - msg += _('email address required (not username)') - return msg - new_password = password - - user = User(username=uname, email=email, is_active=True) - user.set_password(new_password) - try: - user.save() - except IntegrityError: - msg += _('Oops, failed to create user {user}, {error}').format( - user=user, - error="IntegrityError" - ) - return msg - - reg = Registration() - reg.register(user) - - profile = UserProfile(user=user) - profile.name = name - profile.save() - - msg += _('User {user} created successfully!').format(user=user) - return msg - - def delete_user(self, uname): - """Deletes a user from django auth""" - - if not uname: - return _('Must provide username') - if '@' in uname: - try: - user = User.objects.get(email=uname) - except User.DoesNotExist as err: - msg = _('Cannot find user with email address {email_addr}').format(email_addr=uname) - return msg - else: - try: - user = User.objects.get(username=uname) - except User.DoesNotExist as err: - msg = _('Cannot find user with username {username} - {error}').format( - username=uname, - error=str(err) - ) - return msg - user.delete() - return _('Deleted user {username}').format(username=uname) - - def make_datatable(self): - """ - Build the datatable for this view - """ - datatable = { - 'header': [ - _('Statistic'), - _('Value'), - ], - 'title': _('Site statistics'), - 'data': [ - [ - _('Total number of users'), - User.objects.all().count(), - ], - ], - } - return datatable - - def get(self, request): # lint-amnesty, pylint: disable=arguments-differ - if not request.user.is_staff: - raise Http404 - context = { - 'datatable': self.make_datatable(), - 'msg': self.msg, - 'djangopid': os.getpid(), - 'modeflag': {'users': 'active-section'}, - } - return render_to_response(self.template_name, context) - - def post(self, request): - """Handle various actions available on page""" - - if not request.user.is_staff: - raise Http404 - action = request.POST.get('action', '') - track_views.server_track(request, action, {}, page='user_sysdashboard') - - if action == 'create_user': - uname = request.POST.get('student_uname', '').strip() - name = request.POST.get('student_fullname', '').strip() - password = request.POST.get('student_password', '').strip() - self.msg = HTML('
{1}
{1}
{0}").format(escape(ret))
- return msg
-
- def make_datatable(self, courses=None):
- """Creates course information datatable"""
-
- data = []
- courses = courses or self.get_courses()
- for course in courses:
- gdir = course.id.course
- data.append([course.display_name, str(course.id)]
- + self.git_info_for_course(gdir))
-
- return dict(header=[_('Course Name'),
- _('Directory/ID'),
- # Translators: "Git Commit" is a computer command; see http://gitref.org/basic/#commit
- _('Git Commit'),
- _('Last Change'),
- _('Last Editor')],
- title=_('Information about all courses'),
- data=data)
-
- def get(self, request): # lint-amnesty, pylint: disable=arguments-differ
- """Displays forms and course information"""
-
- if not request.user.is_staff:
- raise Http404
-
- context = {
- 'datatable': self.make_datatable(),
- 'msg': self.msg,
- 'djangopid': os.getpid(),
- 'modeflag': {'courses': 'active-section'},
- }
- return render_to_response(self.template_name, context)
-
- def post(self, request):
- """Handle all actions from courses view"""
-
- if not request.user.is_staff:
- raise Http404
-
- action = request.POST.get('action', '')
- track_views.server_track(request, action, {},
- page='courses_sysdashboard')
-
- courses = {course.id: course for course in self.get_courses()}
- if action == 'add_course':
- gitloc = request.POST.get('repo_location', '').strip().replace(' ', '').replace(';', '')
- branch = request.POST.get('repo_branch', '').strip().replace(' ', '').replace(';', '')
- self.msg += self.get_course_from_git(gitloc, branch)
-
- elif action == 'del_course':
- course_id = request.POST.get('course_id', '').strip()
- course_key = CourseKey.from_string(course_id)
- course_found = False
- if course_key in courses:
- course_found = True
- course = courses[course_key]
- else:
- try:
- course = get_course_by_id(course_key)
- course_found = True
- except Exception as err: # pylint: disable=broad-except
- self.msg += _( # lint-amnesty, pylint: disable=translation-of-non-string
- HTML('Error - cannot get course with ID {0}{1}')
- ).format(
- course_key,
- escape(str(err))
- )
-
- if course_found:
- # delete course that is stored with mongodb backend
- self.def_ms.delete_course(course.id, request.user.id)
- # don't delete user permission groups, though
- self.msg += \
- HTML("{0} {1} = {2} ({3})").format(
- _('Deleted'), str(course.location), str(course.id), course.display_name)
-
- context = {
- 'datatable': self.make_datatable(list(courses.values())),
- 'msg': self.msg,
- 'djangopid': os.getpid(),
- 'modeflag': {'courses': 'active-section'},
- }
- return render_to_response(self.template_name, context)
-
-
-class Staffing(SysadminDashboardView):
- """
- The status view provides a view of staffing and enrollment in
- courses.
- """
-
- def get(self, request): # lint-amnesty, pylint: disable=arguments-differ
- """Displays course Enrollment and staffing course statistics"""
-
- if not request.user.is_staff:
- raise Http404
- data = []
-
- for course in self.get_courses():
- datum = [course.display_name, course.id]
- datum += [CourseEnrollment.objects.filter(
- course_id=course.id).count()]
- datum += [CourseStaffRole(course.id).users_with_role().count()]
- datum += [','.join([x.username for x in CourseInstructorRole(
- course.id).users_with_role()])]
- data.append(datum)
-
- datatable = dict(header=[_('Course Name'), _('course_id'),
- _('# enrolled'), _('# staff'),
- _('instructors')],
- title=_('Enrollment information for all courses'),
- data=data)
- context = {
- 'datatable': datatable,
- 'msg': self.msg,
- 'djangopid': os.getpid(),
- 'modeflag': {'staffing': 'active-section'},
- }
- return render_to_response(self.template_name, context)
-
-
-class GitLogs(TemplateView):
- """
- This provides a view into the import of courses from git repositories.
- It is convenient for allowing course teams to see what may be wrong with
- their xml
- """
-
- template_name = 'sysadmin_dashboard_gitlogs.html'
-
- @method_decorator(login_required)
- def get(self, request, *args, **kwargs):
- """Shows logs of imports that happened as a result of a git import"""
-
- course_id = kwargs.get('course_id')
- if course_id:
- course_id = CourseKey.from_string(course_id)
-
- page_size = 10
-
- # Set mongodb defaults even if it isn't defined in settings
- mongo_db = {
- 'host': 'localhost',
- 'user': '',
- 'password': '',
- 'db': 'xlog',
- }
-
- # Allow overrides
- if hasattr(settings, 'MONGODB_LOG'):
- for config_item in ['host', 'user', 'password', 'db', ]:
- mongo_db[config_item] = settings.MONGODB_LOG.get(
- config_item, mongo_db[config_item])
-
- mongouri = 'mongodb://{user}:{password}@{host}/{db}'.format(**mongo_db)
-
- error_msg = ''
-
- try:
- if mongo_db['user'] and mongo_db['password']:
- mdb = mongoengine.connect(mongo_db['db'], host=mongouri)
- else:
- mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'])
- except mongoengine.connection.ConnectionError: # lint-amnesty, pylint: disable=no-member
- log.exception('Unable to connect to mongodb to save log, '
- 'please check MONGODB_LOG settings.')
-
- if course_id is None:
- # Require staff if not going to specific course
- if not request.user.is_staff:
- raise Http404
- cilset = CourseImportLog.objects.order_by('-created')
- else:
- # Allow only course team, instructors, and staff
- if not (request.user.is_staff or
- CourseInstructorRole(course_id).has_user(request.user) or
- CourseStaffRole(course_id).has_user(request.user)):
- raise Http404
- log.debug('course_id=%s', course_id)
- cilset = CourseImportLog.objects.filter(
- course_id=course_id
- ).order_by('-created')
- log.debug('cilset length=%s', len(cilset))
-
- # Paginate the query set
- paginator = Paginator(cilset, page_size)
- try:
- logs = paginator.page(request.GET.get('page'))
- except PageNotAnInteger:
- logs = paginator.page(1)
- except EmptyPage:
- # If the page is too high or low
- given_page = int(request.GET.get('page'))
- page = min(max(1, given_page), paginator.num_pages)
- logs = paginator.page(page)
-
- mdb.close()
- context = {
- 'logs': logs,
- 'course_id': str(course_id) if course_id else None,
- 'error_msg': error_msg,
- 'page_size': page_size
- }
-
- return render_to_response(self.template_name, context)
diff --git a/lms/djangoapps/dashboard/sysadmin_urls.py b/lms/djangoapps/dashboard/sysadmin_urls.py
deleted file mode 100644
index f5d6a84243..0000000000
--- a/lms/djangoapps/dashboard/sysadmin_urls.py
+++ /dev/null
@@ -1,17 +0,0 @@
-"""
-Urls for sysadmin dashboard feature
-"""
-
-
-from django.conf.urls import url
-
-from . import sysadmin
-
-urlpatterns = [
- url(r'^$', sysadmin.Users.as_view(), name="sysadmin"),
- url(r'^courses/?$', sysadmin.Courses.as_view(), name="sysadmin_courses"),
- url(r'^staffing/?$', sysadmin.Staffing.as_view(), name="sysadmin_staffing"),
- url(r'^gitlogs/?$', sysadmin.GitLogs.as_view(), name="gitlogs"),
- url(r'^gitlogs/(?P- ## Translators: git is a version-control system; see http://git-scm.com/about - ${Text(_("View detailed Git import logs for this course {link_start}by clicking here{link_end}.")).format( - link_start=HTML('').format(section_data['detailed_gitlogs_url']), - link_end=HTML('') - )} -
- %endif -