From 582c02afc4ceec366ad5a125be20c2ca32fed850 Mon Sep 17 00:00:00 2001 From: HamzaIbnFarooq Date: Fri, 2 Apr 2021 13:29:16 +0500 Subject: [PATCH] feat!: remove sysadmin dashboard feature w.r.t 0002-deprecate-sysadmin-dashboard-adr.rst The sysadmin dashboard feature is converted into a plugable app named edx-sysadmin (https://github.com/mitodl/edx-sysadmin) according to the decisison made at https://github.com/edx/edx-platform/blob/master/lms/djangoapps/dashboard/decisions/0002-deprecate-sysadmin-dashboard-adr.rst. Instances using sysadmin dashboard should use the new plugin from now onwards. BREAKING CHANGE: sysadmin dashboard is removed --- cms/devstack.yml | 1 - cms/envs/common.py | 1 - import_shims/lms/dashboard/__init__.py | 8 - import_shims/lms/dashboard/git_import.py | 8 - .../lms/dashboard/management/__init__.py | 8 - .../dashboard/management/commands/__init__.py | 8 - .../management/commands/git_add_course.py | 8 - .../management/commands/tests/__init__.py | 8 - .../commands/tests/test_git_add_course.py | 8 - import_shims/lms/dashboard/models.py | 8 - import_shims/lms/dashboard/sysadmin.py | 8 - import_shims/lms/dashboard/sysadmin_urls.py | 8 - import_shims/lms/dashboard/tests/__init__.py | 8 - .../lms/dashboard/tests/test_sysadmin.py | 8 - lms/devstack.yml | 2 - lms/djangoapps/dashboard/__init__.py | 0 lms/djangoapps/dashboard/git_import.py | 352 ------------ .../dashboard/management/__init__.py | 0 .../dashboard/management/commands/__init__.py | 0 .../management/commands/git_add_course.py | 54 -- .../management/commands/tests/__init__.py | 0 .../commands/tests/test_git_add_course.py | 226 -------- lms/djangoapps/dashboard/models.py | 21 - lms/djangoapps/dashboard/sysadmin.py | 500 ------------------ lms/djangoapps/dashboard/sysadmin_urls.py | 17 - lms/djangoapps/dashboard/tests/__init__.py | 0 .../dashboard/tests/test_sysadmin.py | 355 ------------- .../instructor/views/instructor_dashboard.py | 6 - lms/envs/common.py | 14 - lms/envs/production.py | 6 +- lms/envs/test.py | 4 - .../header/navbar-authenticated.html | 7 - .../instructor_dashboard_2/course_info.html | 10 - .../bootstrap/navbar-authenticated.html | 7 - .../navigation/navbar-authenticated.html | 6 - lms/templates/sysadmin_dashboard.html | 192 ------- lms/templates/sysadmin_dashboard_gitlogs.html | 207 -------- lms/urls.py | 6 - .../header/navbar-authenticated.html | 1 - 39 files changed, 2 insertions(+), 2089 deletions(-) delete mode 100644 import_shims/lms/dashboard/__init__.py delete mode 100644 import_shims/lms/dashboard/git_import.py delete mode 100644 import_shims/lms/dashboard/management/__init__.py delete mode 100644 import_shims/lms/dashboard/management/commands/__init__.py delete mode 100644 import_shims/lms/dashboard/management/commands/git_add_course.py delete mode 100644 import_shims/lms/dashboard/management/commands/tests/__init__.py delete mode 100644 import_shims/lms/dashboard/management/commands/tests/test_git_add_course.py delete mode 100644 import_shims/lms/dashboard/models.py delete mode 100644 import_shims/lms/dashboard/sysadmin.py delete mode 100644 import_shims/lms/dashboard/sysadmin_urls.py delete mode 100644 import_shims/lms/dashboard/tests/__init__.py delete mode 100644 import_shims/lms/dashboard/tests/test_sysadmin.py delete mode 100644 lms/djangoapps/dashboard/__init__.py delete mode 100644 lms/djangoapps/dashboard/git_import.py delete mode 100644 lms/djangoapps/dashboard/management/__init__.py delete mode 100644 lms/djangoapps/dashboard/management/commands/__init__.py delete mode 100644 lms/djangoapps/dashboard/management/commands/git_add_course.py delete mode 100644 lms/djangoapps/dashboard/management/commands/tests/__init__.py delete mode 100644 lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py delete mode 100644 lms/djangoapps/dashboard/models.py delete mode 100644 lms/djangoapps/dashboard/sysadmin.py delete mode 100644 lms/djangoapps/dashboard/sysadmin_urls.py delete mode 100644 lms/djangoapps/dashboard/tests/__init__.py delete mode 100644 lms/djangoapps/dashboard/tests/test_sysadmin.py delete mode 100644 lms/templates/sysadmin_dashboard.html delete mode 100644 lms/templates/sysadmin_dashboard_gitlogs.html diff --git a/cms/devstack.yml b/cms/devstack.yml index dd70f3e180..109e9c8a39 100644 --- a/cms/devstack.yml +++ b/cms/devstack.yml @@ -258,7 +258,6 @@ FEATURES: ENABLE_PUBLISHER: false ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES: true ENABLE_SPECIAL_EXAMS: false - ENABLE_SYSADMIN_DASHBOARD: false ENABLE_THIRD_PARTY_AUTH: true ENABLE_VIDEO_UPLOAD_PIPELINE: false PREVIEW_LMS_BASE: preview.localhost:8000 diff --git a/cms/envs/common.py b/cms/envs/common.py index 7dd7939d50..e3fcae6567 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -351,7 +351,6 @@ FEATURES = { 'ENABLE_COUNTRY_ACCESS': False, 'ENABLE_CREDIT_API': False, 'ENABLE_OAUTH2_PROVIDER': False, - 'ENABLE_SYSADMIN_DASHBOARD': False, 'ENABLE_MOBILE_REST_API': False, 'CUSTOM_COURSES_EDX': False, 'ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES': True, diff --git a/import_shims/lms/dashboard/__init__.py b/import_shims/lms/dashboard/__init__.py deleted file mode 100644 index f98ec5de42..0000000000 --- a/import_shims/lms/dashboard/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard', 'lms.djangoapps.dashboard') - -from lms.djangoapps.dashboard import * diff --git a/import_shims/lms/dashboard/git_import.py b/import_shims/lms/dashboard/git_import.py deleted file mode 100644 index ddc930d99f..0000000000 --- a/import_shims/lms/dashboard/git_import.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.git_import', 'lms.djangoapps.dashboard.git_import') - -from lms.djangoapps.dashboard.git_import import * diff --git a/import_shims/lms/dashboard/management/__init__.py b/import_shims/lms/dashboard/management/__init__.py deleted file mode 100644 index 343abc8990..0000000000 --- a/import_shims/lms/dashboard/management/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.management', 'lms.djangoapps.dashboard.management') - -from lms.djangoapps.dashboard.management import * diff --git a/import_shims/lms/dashboard/management/commands/__init__.py b/import_shims/lms/dashboard/management/commands/__init__.py deleted file mode 100644 index 81a76be9a8..0000000000 --- a/import_shims/lms/dashboard/management/commands/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.management.commands', 'lms.djangoapps.dashboard.management.commands') - -from lms.djangoapps.dashboard.management.commands import * diff --git a/import_shims/lms/dashboard/management/commands/git_add_course.py b/import_shims/lms/dashboard/management/commands/git_add_course.py deleted file mode 100644 index 2180a53191..0000000000 --- a/import_shims/lms/dashboard/management/commands/git_add_course.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.management.commands.git_add_course', 'lms.djangoapps.dashboard.management.commands.git_add_course') - -from lms.djangoapps.dashboard.management.commands.git_add_course import * diff --git a/import_shims/lms/dashboard/management/commands/tests/__init__.py b/import_shims/lms/dashboard/management/commands/tests/__init__.py deleted file mode 100644 index c487cd88c0..0000000000 --- a/import_shims/lms/dashboard/management/commands/tests/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.management.commands.tests', 'lms.djangoapps.dashboard.management.commands.tests') - -from lms.djangoapps.dashboard.management.commands.tests import * diff --git a/import_shims/lms/dashboard/management/commands/tests/test_git_add_course.py b/import_shims/lms/dashboard/management/commands/tests/test_git_add_course.py deleted file mode 100644 index e1c9c8276f..0000000000 --- a/import_shims/lms/dashboard/management/commands/tests/test_git_add_course.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.management.commands.tests.test_git_add_course', 'lms.djangoapps.dashboard.management.commands.tests.test_git_add_course') - -from lms.djangoapps.dashboard.management.commands.tests.test_git_add_course import * diff --git a/import_shims/lms/dashboard/models.py b/import_shims/lms/dashboard/models.py deleted file mode 100644 index efd61cdb64..0000000000 --- a/import_shims/lms/dashboard/models.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.models', 'lms.djangoapps.dashboard.models') - -from lms.djangoapps.dashboard.models import * diff --git a/import_shims/lms/dashboard/sysadmin.py b/import_shims/lms/dashboard/sysadmin.py deleted file mode 100644 index be756214a7..0000000000 --- a/import_shims/lms/dashboard/sysadmin.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.sysadmin', 'lms.djangoapps.dashboard.sysadmin') - -from lms.djangoapps.dashboard.sysadmin import * diff --git a/import_shims/lms/dashboard/sysadmin_urls.py b/import_shims/lms/dashboard/sysadmin_urls.py deleted file mode 100644 index 87d00a1c20..0000000000 --- a/import_shims/lms/dashboard/sysadmin_urls.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.sysadmin_urls', 'lms.djangoapps.dashboard.sysadmin_urls') - -from lms.djangoapps.dashboard.sysadmin_urls import * diff --git a/import_shims/lms/dashboard/tests/__init__.py b/import_shims/lms/dashboard/tests/__init__.py deleted file mode 100644 index 01a4a959ff..0000000000 --- a/import_shims/lms/dashboard/tests/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.tests', 'lms.djangoapps.dashboard.tests') - -from lms.djangoapps.dashboard.tests import * diff --git a/import_shims/lms/dashboard/tests/test_sysadmin.py b/import_shims/lms/dashboard/tests/test_sysadmin.py deleted file mode 100644 index 6c61277907..0000000000 --- a/import_shims/lms/dashboard/tests/test_sysadmin.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh.""" -# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long - -from import_shims.warn import warn_deprecated_import - -warn_deprecated_import('dashboard.tests.test_sysadmin', 'lms.djangoapps.dashboard.tests.test_sysadmin') - -from lms.djangoapps.dashboard.tests.test_sysadmin import * diff --git a/lms/devstack.yml b/lms/devstack.yml index 41a72e2938..0e71b134cf 100644 --- a/lms/devstack.yml +++ b/lms/devstack.yml @@ -290,7 +290,6 @@ FEATURES: ENABLE_PUBLISHER: false ENABLE_READING_FROM_MULTIPLE_HISTORY_TABLES: true ENABLE_SPECIAL_EXAMS: false - ENABLE_SYSADMIN_DASHBOARD: false ENABLE_THIRD_PARTY_AUTH: true ENABLE_VIDEO_UPLOAD_PIPELINE: false PREVIEW_LMS_BASE: preview.localhost:18000 @@ -307,7 +306,6 @@ FINANCIAL_REPORTS: STORAGE_TYPE: localfs FOOTER_ORGANIZATION_IMAGE: images/logo.png GITHUB_REPO_ROOT: /edx/var/edxapp/data -GIT_REPO_DIR: /edx/var/edxapp/course_repos GOOGLE_ANALYTICS_ACCOUNT: null GOOGLE_ANALYTICS_LINKEDIN: '' GOOGLE_ANALYTICS_TRACKING_ID: '' diff --git a/lms/djangoapps/dashboard/__init__.py b/lms/djangoapps/dashboard/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/lms/djangoapps/dashboard/git_import.py b/lms/djangoapps/dashboard/git_import.py deleted file mode 100644 index 4a7d46ce97..0000000000 --- a/lms/djangoapps/dashboard/git_import.py +++ /dev/null @@ -1,352 +0,0 @@ -""" -Provides a function for importing a git repository into the lms -instance when using a mongo modulestore -""" - - -import logging -import os -import re -import subprocess - -import mongoengine -from django.conf import settings -from django.core import management -from django.core.management.base import CommandError -from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ -from opaque_keys.edx.locator import CourseLocator -from six import StringIO -from xmodule.util.sandboxing import DEFAULT_PYTHON_LIB_FILENAME - -from lms.djangoapps.dashboard.models import CourseImportLog - -log = logging.getLogger(__name__) - -DEFAULT_GIT_REPO_DIR = '/edx/var/app/edxapp/course_repos' - - -class GitImportError(Exception): - """ - Exception class for handling the typical errors in a git import. - """ - MESSAGE = None - - def __init__(self, message=None): - if message is None: - message = self.MESSAGE - super().__init__(message) - - -class GitImportErrorNoDir(GitImportError): - """ - GitImportError when no directory exists at the specified path. - """ - def __init__(self, repo_dir): - super().__init__( - _( - "Path {0} doesn't exist, please create it, " - "or configure a different path with " - "GIT_REPO_DIR" - ).format(repo_dir) - ) - - -class GitImportErrorUrlBad(GitImportError): - """ - GitImportError when the git url provided wasn't usable. - """ - MESSAGE = _( - 'Non usable git url provided. Expecting something like:' - ' git@github.com:edx/edx4edx_lite.git' - ) - - -class GitImportErrorBadRepo(GitImportError): - """ - GitImportError when the cloned repository was malformed. - """ - MESSAGE = _('Unable to get git log') - - -class GitImportErrorCannotPull(GitImportError): - """ - GitImportError when the clone of the repository failed. - """ - MESSAGE = _('git clone or pull failed!') - - -class GitImportErrorXmlImportFailed(GitImportError): - """ - GitImportError when the course import command failed. - """ - MESSAGE = _('Unable to run import command.') - - -class GitImportErrorUnsupportedStore(GitImportError): - """ - GitImportError when the modulestore doesn't support imports. - """ - MESSAGE = _('The underlying module store does not support import.') - - -class GitImportErrorRemoteBranchMissing(GitImportError): - """ - GitImportError when the remote branch doesn't exist. - """ - # Translators: This is an error message when they ask for a - # particular version of a git repository and that version isn't - # available from the remote source they specified - MESSAGE = _('The specified remote branch is not available.') - - -class GitImportErrorCannotBranch(GitImportError): - """ - GitImportError when the local branch doesn't exist. - """ - # Translators: Error message shown when they have asked for a git - # repository branch, a specific version within a repository, that - # doesn't exist, or there is a problem changing to it. - MESSAGE = _('Unable to switch to specified branch. Please check your branch name.') - - -def cmd_log(cmd, cwd): - """ - Helper function to redirect stderr to stdout and log the command - used along with the output. Will raise subprocess.CalledProcessError if - command doesn't return 0, and returns the command's output. - """ - output = subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT).decode('utf-8') - - log.debug('Command was: %r. Working directory was: %r', ' '.join(cmd), cwd) - log.debug('Command output was: %r', output) - return output - - -def switch_branch(branch, rdir): - """ - This will determine how to change the branch of the repo, and then - use the appropriate git commands to do so. - - Raises an appropriate GitImportError exception if there is any issues with changing - branches. - """ - # Get the latest remote - try: - cmd_log(['git', 'fetch', ], rdir) - except subprocess.CalledProcessError as ex: - log.exception('Unable to fetch remote: %r', ex.output) - raise GitImportErrorCannotBranch() # lint-amnesty, pylint: disable=raise-missing-from - - # Check if the branch is available from the remote. - cmd = ['git', 'ls-remote', 'origin', '-h', f'refs/heads/{branch}', ] - try: - output = cmd_log(cmd, rdir) - except subprocess.CalledProcessError as ex: - log.exception('Getting a list of remote branches failed: %r', ex.output) - raise GitImportErrorCannotBranch() # lint-amnesty, pylint: disable=raise-missing-from - if branch not in output: - raise GitImportErrorRemoteBranchMissing() - # Check it the remote branch has already been made locally - cmd = ['git', 'branch', '-a', ] - try: - output = cmd_log(cmd, rdir) - except subprocess.CalledProcessError as ex: - log.exception('Getting a list of local branches failed: %r', ex.output) - raise GitImportErrorCannotBranch() # lint-amnesty, pylint: disable=raise-missing-from - branches = [] - for line in output.split('\n'): - branches.append(line.replace('*', '').strip()) - - if branch not in branches: - # Checkout with -b since it is remote only - cmd = ['git', 'checkout', '--force', '--track', - '-b', branch, f'origin/{branch}', ] - try: - cmd_log(cmd, rdir) - except subprocess.CalledProcessError as ex: - log.exception('Unable to checkout remote branch: %r', ex.output) - raise GitImportErrorCannotBranch() # lint-amnesty, pylint: disable=raise-missing-from - # Go ahead and reset hard to the newest version of the branch now that we know - # it is local. - try: - cmd_log(['git', 'reset', '--hard', f'origin/{branch}', ], rdir) - except subprocess.CalledProcessError as ex: - log.exception('Unable to reset to branch: %r', ex.output) - raise GitImportErrorCannotBranch() # lint-amnesty, pylint: disable=raise-missing-from - - -def add_repo(repo, rdir_in, branch=None): - """ - This will add a git repo into the mongo modulestore. - If branch is left as None, it will fetch the most recent - version of the current branch. - """ - # pylint: disable=too-many-statements - - git_repo_dir = getattr(settings, 'GIT_REPO_DIR', DEFAULT_GIT_REPO_DIR) - git_import_static = getattr(settings, 'GIT_IMPORT_STATIC', True) - git_import_python_lib = getattr(settings, 'GIT_IMPORT_PYTHON_LIB', True) - python_lib_filename = getattr(settings, 'PYTHON_LIB_FILENAME', DEFAULT_PYTHON_LIB_FILENAME) - - # Set defaults even if it isn't defined in settings - mongo_db = { - 'host': 'localhost', - 'port': 27017, - 'user': '', - 'password': '', - 'db': 'xlog', - } - - # Allow overrides - if hasattr(settings, 'MONGODB_LOG'): - for config_item in ['host', 'user', 'password', 'db', 'port']: - mongo_db[config_item] = settings.MONGODB_LOG.get( - config_item, mongo_db[config_item]) - - if not os.path.isdir(git_repo_dir): - raise GitImportErrorNoDir(git_repo_dir) - # pull from git - if not (repo.endswith('.git') or - repo.startswith(('http:', 'https:', 'git:', 'file:'))): - raise GitImportErrorUrlBad() - - if rdir_in: - rdir = os.path.basename(rdir_in) - else: - rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0] - log.debug('rdir = %s', rdir) - - rdirp = f'{git_repo_dir}/{rdir}' - if os.path.exists(rdirp): - log.info('directory already exists, doing a git pull instead ' - 'of git clone') - cmd = ['git', 'pull', ] - cwd = rdirp - else: - cmd = ['git', 'clone', repo, ] - cwd = git_repo_dir - - cwd = os.path.abspath(cwd) - try: - ret_git = cmd_log(cmd, cwd=cwd) - except subprocess.CalledProcessError as ex: - log.exception('Error running git pull: %r', ex.output) - raise GitImportErrorCannotPull() # lint-amnesty, pylint: disable=raise-missing-from - - if branch: - switch_branch(branch, rdirp) - - # get commit id - cmd = ['git', 'log', '-1', '--format=%H', ] - try: - commit_id = cmd_log(cmd, cwd=rdirp) - except subprocess.CalledProcessError as ex: - log.exception('Unable to get git log: %r', ex.output) - raise GitImportErrorBadRepo() # lint-amnesty, pylint: disable=raise-missing-from - - ret_git += f'\nCommit ID: {commit_id}' - - # get branch - cmd = ['git', 'symbolic-ref', '--short', 'HEAD', ] - try: - branch = cmd_log(cmd, cwd=rdirp) - except subprocess.CalledProcessError as ex: - # I can't discover a way to excercise this, but git is complex - # so still logging and raising here in case. - log.exception('Unable to determine branch: %r', ex.output) - raise GitImportErrorBadRepo() # lint-amnesty, pylint: disable=raise-missing-from - - ret_git += '{}Branch: {}'.format(' \n', branch) - - # Get XML logging logger and capture debug to parse results - output = StringIO() - import_log_handler = logging.StreamHandler(output) - import_log_handler.setLevel(logging.DEBUG) - - logger_names = ['xmodule.modulestore.xml_importer', 'git_add_course', - 'xmodule.modulestore.xml', 'xmodule.seq_module', ] - loggers = [] - - for logger_name in logger_names: - logger = logging.getLogger(logger_name) - logger.setLevel(logging.DEBUG) - logger.addHandler(import_log_handler) - loggers.append(logger) - - try: - management.call_command( - 'import', git_repo_dir, rdir, - nostatic=not git_import_static, nopythonlib=not git_import_python_lib, - python_lib_filename=python_lib_filename - ) - except CommandError: - raise GitImportErrorXmlImportFailed() # lint-amnesty, pylint: disable=raise-missing-from - except NotImplementedError: - raise GitImportErrorUnsupportedStore() # lint-amnesty, pylint: disable=raise-missing-from - - ret_import = output.getvalue() - - # Remove handler hijacks - for logger in loggers: - logger.setLevel(logging.NOTSET) - logger.removeHandler(import_log_handler) - - course_key = None - location = 'unknown' - - # extract course ID from output of import-command-run and make symlink - # this is needed in order for custom course scripts to work - match = re.search(r'(?ms)===> IMPORTING courselike (\S+)', ret_import) - if match: - course_id = match.group(1).split('/') - # we need to transform course key extracted from logs into CourseLocator instance, because - # we are using split module store and course keys store as instance of CourseLocator. - # please see common.lib.xmodule.xmodule.modulestore.split_mongo.split.SplitMongoModuleStore#make_course_key - # We want set course id in CourseImportLog as CourseLocator. So that in split module - # environment course id remain consistent as CourseLocator instance. - course_key = CourseLocator(*course_id) - cdir = f'{git_repo_dir}/{course_key.course}' - log.debug('Studio course dir = %s', cdir) - - if os.path.exists(cdir) and not os.path.islink(cdir): - log.debug(' -> exists, but is not symlink') - log.debug(subprocess.check_output(['ls', '-l', ], - cwd=os.path.abspath(cdir))) - try: - os.rmdir(os.path.abspath(cdir)) - except OSError: - log.exception('Failed to remove course directory') - - if not os.path.exists(cdir): - log.debug(' -> creating symlink between %s and %s', rdirp, cdir) - try: - os.symlink(os.path.abspath(rdirp), os.path.abspath(cdir)) - except OSError: - log.exception('Unable to create course symlink') - log.debug(subprocess.check_output(['ls', '-l', ], - cwd=os.path.abspath(cdir))) - - # store import-command-run output in mongo - mongouri = 'mongodb://{user}:{password}@{host}:{port}/{db}'.format(**mongo_db) - - try: - if mongo_db['user'] and mongo_db['password']: - mdb = mongoengine.connect(mongo_db['db'], host=mongouri) - else: - mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'], port=mongo_db['port']) - except mongoengine.connection.ConnectionFailure: - log.exception('Unable to connect to mongodb to save log, please ' - 'check MONGODB_LOG settings') - cil = CourseImportLog( - course_id=course_key, - location=location, - repo_dir=rdir, - created=timezone.now(), - import_log=ret_import, - git_log=ret_git, - ) - cil.save() - - log.debug('saved CourseImportLog for %s', cil.course_id) - mdb.close() diff --git a/lms/djangoapps/dashboard/management/__init__.py b/lms/djangoapps/dashboard/management/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/lms/djangoapps/dashboard/management/commands/__init__.py b/lms/djangoapps/dashboard/management/commands/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/lms/djangoapps/dashboard/management/commands/git_add_course.py b/lms/djangoapps/dashboard/management/commands/git_add_course.py deleted file mode 100644 index 331079a00b..0000000000 --- a/lms/djangoapps/dashboard/management/commands/git_add_course.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -Script for importing courseware from git/xml into a mongo modulestore -""" - - -import logging - -from django.core.management.base import BaseCommand, CommandError -from django.utils.translation import ugettext as _ -from xmodule.modulestore.django import modulestore -from xmodule.modulestore.xml import XMLModuleStore - -from lms.djangoapps.dashboard import git_import - -log = logging.getLogger(__name__) - - -class Command(BaseCommand): - """ - Pull a git repo and import into the mongo based content database. - """ - - # Translators: A git repository is a place to store a grouping of - # versioned files. A branch is a sub grouping of a repository that - # has a specific version of the repository. A modulestore is the database used - # to store the courses for use on the Web site. - help = ('Usage: ' - 'git_add_course repository_url [directory to check out into] [repository_branch] ' - '\n{}'.format(_('Import the specified git repository and optional branch into the ' - 'modulestore and optionally specified directory.'))) - - def add_arguments(self, parser): - # Positional arguments - parser.add_argument('repository_url') - parser.add_argument('--directory_path', action='store') - parser.add_argument('--repository_branch', action='store') - - def handle(self, *args, **options): - """Check inputs and run the command""" - - if isinstance(modulestore, XMLModuleStore): - raise CommandError('This script requires a mongo module store') - - rdir_arg = None - branch = None - if options['directory_path']: - rdir_arg = options['directory_path'] - if options['repository_branch']: - branch = options['repository_branch'] - - try: - git_import.add_repo(options['repository_url'], rdir_arg, branch) - except git_import.GitImportError as ex: - raise CommandError(str(ex)) # lint-amnesty, pylint: disable=raise-missing-from diff --git a/lms/djangoapps/dashboard/management/commands/tests/__init__.py b/lms/djangoapps/dashboard/management/commands/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py b/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py deleted file mode 100644 index 4b1080b0ba..0000000000 --- a/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py +++ /dev/null @@ -1,226 +0,0 @@ -""" -Provide tests for git_add_course management command. -""" -import logging -import os -import shutil -import subprocess -import unittest -from uuid import uuid4 - -import pytest -from django.conf import settings -from django.core.management import call_command -from django.core.management.base import CommandError -from django.test.utils import override_settings -from opaque_keys.edx.keys import CourseKey -from six import StringIO -from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.django import modulestore -from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase -from xmodule.modulestore.tests.mongo_connection import MONGO_HOST, MONGO_PORT_NUM - -import lms.djangoapps.dashboard.git_import as git_import -from lms.djangoapps.dashboard.git_import import ( - GitImportError, - GitImportErrorBadRepo, - GitImportErrorCannotPull, - GitImportErrorNoDir, - GitImportErrorRemoteBranchMissing, - GitImportErrorUrlBad -) -TEST_MONGODB_LOG = { - 'host': MONGO_HOST, - 'port': MONGO_PORT_NUM, - 'user': '', - 'password': '', - 'db': 'test_xlog', -} - - -@override_settings( - MONGODB_LOG=TEST_MONGODB_LOG, - GIT_REPO_DIR=settings.TEST_ROOT / f"course_repos_{uuid4().hex}" -) -@unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'), - "ENABLE_SYSADMIN_DASHBOARD not set") -class TestGitAddCourse(SharedModuleStoreTestCase): - """ - Tests the git_add_course management command for proper functions. - """ - TEST_REPO = 'https://github.com/edx/edx4edx_lite.git' - TEST_COURSE = 'MITx/edx4edx/edx4edx' - TEST_BRANCH = 'testing_do_not_delete' - TEST_BRANCH_COURSE = CourseKey.from_string('MITx/edx4edx_branch/edx4edx') - - ENABLED_CACHES = ['default', 'mongo_metadata_inheritance', 'loc_cache'] - - def setUp(self): - super().setUp() - self.git_repo_dir = settings.GIT_REPO_DIR - - def assertCommandFailureRegexp(self, regex, *args): - """ - Convenience function for testing command failures - """ - with self.assertRaisesRegex(CommandError, regex): - call_command('git_add_course', *args, stderr=StringIO()) - - def test_command_args(self): - """ - Validate argument checking - """ - # No argument given. - self.assertCommandFailureRegexp('Error: the following arguments are required: repository_url') - # Extra/Un-named arguments given. - self.assertCommandFailureRegexp( - 'Error: unrecognized arguments: blah blah blah', - 'blah', 'blah', 'blah', 'blah') - # Not a valid path. - self.assertCommandFailureRegexp( - f'Path {self.git_repo_dir} doesn\'t exist, please create it,', - 'blah') - # Test successful import from command - if not os.path.isdir(self.git_repo_dir): - os.mkdir(self.git_repo_dir) - self.addCleanup(shutil.rmtree, self.git_repo_dir) - - # Make a course dir that will be replaced with a symlink - # while we are at it. - if not os.path.isdir(self.git_repo_dir / 'edx4edx'): - os.mkdir(self.git_repo_dir / 'edx4edx') - - call_command('git_add_course', self.TEST_REPO, - directory_path=self.git_repo_dir / 'edx4edx_lite') - - # Test with all three args (branch) - call_command('git_add_course', self.TEST_REPO, - directory_path=self.git_repo_dir / 'edx4edx_lite', - repository_branch=self.TEST_BRANCH) - - def test_add_repo(self): - """ - Various exit path tests for test_add_repo - """ - with pytest.raises(GitImportErrorNoDir): - git_import.add_repo(self.TEST_REPO, None, None) - - os.mkdir(self.git_repo_dir) - self.addCleanup(shutil.rmtree, self.git_repo_dir) - - with pytest.raises(GitImportErrorUrlBad): - git_import.add_repo('foo', None, None) - - with pytest.raises(GitImportErrorCannotPull): - git_import.add_repo('file:///foobar.git', None, None) - - # Test git repo that exists, but is "broken" - bare_repo = os.path.abspath('{}/{}'.format(settings.TEST_ROOT, 'bare.git')) - os.mkdir(bare_repo) - self.addCleanup(shutil.rmtree, bare_repo) - subprocess.check_output(['git', '--bare', 'init', ], stderr=subprocess.STDOUT, - cwd=bare_repo) - - with pytest.raises(GitImportErrorBadRepo): - git_import.add_repo(f'file://{bare_repo}', None, None) - - def test_detached_repo(self): - """ - Test repo that is in detached head state. - """ - repo_dir = self.git_repo_dir - # Test successful import from command - try: - os.mkdir(repo_dir) - except OSError: - pass - self.addCleanup(shutil.rmtree, repo_dir) - git_import.add_repo(self.TEST_REPO, repo_dir / 'edx4edx_lite', None) - subprocess.check_output(['git', 'checkout', 'HEAD~2', ], - stderr=subprocess.STDOUT, - cwd=repo_dir / 'edx4edx_lite') - with pytest.raises(GitImportErrorCannotPull): - git_import.add_repo(self.TEST_REPO, repo_dir / 'edx4edx_lite', None) - - def test_branching(self): - """ - Exercise branching code of import - """ - repo_dir = self.git_repo_dir - # Test successful import from command - if not os.path.isdir(repo_dir): - os.mkdir(repo_dir) - self.addCleanup(shutil.rmtree, repo_dir) - - # Checkout non existent branch - with pytest.raises(GitImportErrorRemoteBranchMissing): - git_import.add_repo(self.TEST_REPO, repo_dir / 'edx4edx_lite', 'asdfasdfasdf') - - # Checkout new branch - git_import.add_repo(self.TEST_REPO, - repo_dir / 'edx4edx_lite', - self.TEST_BRANCH) - def_ms = modulestore() - # Validate that it is different than master - assert def_ms.get_course(self.TEST_BRANCH_COURSE) is not None - - # Attempt to check out the same branch again to validate branch choosing - # works - git_import.add_repo(self.TEST_REPO, - repo_dir / 'edx4edx_lite', - self.TEST_BRANCH) - - # Delete to test branching back to master - def_ms.delete_course(self.TEST_BRANCH_COURSE, ModuleStoreEnum.UserID.test) - assert def_ms.get_course(self.TEST_BRANCH_COURSE) is None - git_import.add_repo(self.TEST_REPO, - repo_dir / 'edx4edx_lite', - 'master') - assert def_ms.get_course(self.TEST_BRANCH_COURSE) is None - assert def_ms.get_course(CourseKey.from_string(self.TEST_COURSE)) is not None - - def test_branch_exceptions(self): - """ - This wil create conditions to exercise bad paths in the switch_branch function. - """ - # create bare repo that we can mess with and attempt an import - bare_repo = os.path.abspath('{}/{}'.format(settings.TEST_ROOT, 'bare.git')) - os.mkdir(bare_repo) - self.addCleanup(shutil.rmtree, bare_repo) - subprocess.check_output(['git', '--bare', 'init', ], stderr=subprocess.STDOUT, - cwd=bare_repo) - - # Build repo dir - repo_dir = self.git_repo_dir - if not os.path.isdir(repo_dir): - os.mkdir(repo_dir) - self.addCleanup(shutil.rmtree, repo_dir) - - rdir = '{0}/bare'.format(repo_dir) - with pytest.raises(GitImportErrorBadRepo): - git_import.add_repo(f'file://{bare_repo}', None, None) - - # Get logger for checking strings in logs - output = StringIO() - test_log_handler = logging.StreamHandler(output) - test_log_handler.setLevel(logging.DEBUG) - glog = git_import.log - glog.addHandler(test_log_handler) - - # Move remote so fetch fails - shutil.move(bare_repo, f'{settings.TEST_ROOT}/not_bare.git') - try: - git_import.switch_branch('master', rdir) - except GitImportError: - assert 'Unable to fetch remote' in output.getvalue() - shutil.move(f'{settings.TEST_ROOT}/not_bare.git', bare_repo) - output.truncate(0) - - # Replace origin with a different remote - subprocess.check_output( - ['git', 'remote', 'rename', 'origin', 'blah', ], - stderr=subprocess.STDOUT, cwd=rdir - ) - with pytest.raises(GitImportError): - git_import.switch_branch('master', rdir) - assert 'Getting a list of remote branches failed' in output.getvalue() diff --git a/lms/djangoapps/dashboard/models.py b/lms/djangoapps/dashboard/models.py deleted file mode 100644 index 9196f5ecea..0000000000 --- a/lms/djangoapps/dashboard/models.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Models for dashboard application""" - - -import mongoengine -from xmodule.modulestore.mongoengine_fields import CourseKeyField - - -class CourseImportLog(mongoengine.Document): - """Mongoengine model for git log""" - course_id = CourseKeyField(max_length=128) - # NOTE: this location is not a Location object but a pathname - location = mongoengine.StringField(max_length=168) - import_log = mongoengine.StringField(max_length=20 * 65535) - git_log = mongoengine.StringField(max_length=65535) - repo_dir = mongoengine.StringField(max_length=128) - commit = mongoengine.StringField(max_length=40, null=True) - author = mongoengine.StringField(max_length=500, null=True) - date = mongoengine.DateTimeField() - created = mongoengine.DateTimeField() - meta = {'indexes': ['course_id', 'created'], - 'allow_inheritance': False} diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py deleted file mode 100644 index 656d45b603..0000000000 --- a/lms/djangoapps/dashboard/sysadmin.py +++ /dev/null @@ -1,500 +0,0 @@ -""" -This module creates a sysadmin dashboard for managing and viewing -courses. -""" -import json -import logging -import os -import subprocess -import warnings -from io import StringIO - -import mongoengine -from django.conf import settings -from django.contrib.auth.decorators import login_required -from django.contrib.auth.models import User # lint-amnesty, pylint: disable=imported-auth-user -from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator -from django.db import IntegrityError -from django.http import Http404 -from django.utils.decorators import method_decorator -from django.utils.html import escape -from django.utils.translation import ugettext as _ -from django.views.decorators.cache import cache_control -from django.views.decorators.csrf import ensure_csrf_cookie -from django.views.decorators.http import condition -from django.views.generic.base import TemplateView -from opaque_keys.edx.keys import CourseKey -from path import Path as path -from xmodule.modulestore.django import modulestore - -import lms.djangoapps.dashboard.git_import as git_import -from common.djangoapps.edxmako.shortcuts import render_to_response -from common.djangoapps.student.models import CourseEnrollment, Registration, UserProfile -from common.djangoapps.student.roles import CourseInstructorRole, CourseStaffRole -from common.djangoapps.track import views as track_views -from lms.djangoapps.courseware.courses import get_course_by_id -from lms.djangoapps.dashboard.git_import import GitImportError -from lms.djangoapps.dashboard.models import CourseImportLog -from openedx.core.djangolib.markup import HTML - -log = logging.getLogger(__name__) - - -class SysadminDashboardView(TemplateView): - """Base class for sysadmin dashboard views with common methods""" - - template_name = 'sysadmin_dashboard.html' - - def __init__(self, **kwargs): - """ - Initialize base sysadmin dashboard class with modulestore, - modulestore_type and return msg - """ - # Deprecation log for Sysadmin Dashboard - warnings.warn("Sysadmin Dashboard is deprecated. See DEPR-118.", DeprecationWarning) - - self.def_ms = modulestore() - self.msg = '' - self.datatable = [] - super().__init__(**kwargs) - - @method_decorator(ensure_csrf_cookie) - @method_decorator(login_required) - @method_decorator(cache_control(no_cache=True, no_store=True, - must_revalidate=True)) - @method_decorator(condition(etag_func=None)) - def dispatch(self, *args, **kwargs): - return super().dispatch(*args, **kwargs) - - def get_courses(self): - """ Get an iterable list of courses.""" - - return self.def_ms.get_courses() - - -class Users(SysadminDashboardView): - """ - The status view provides Web based user management, a listing of - courses loaded, and user statistics - """ - - def create_user(self, uname, name, password=None): - """ Creates a user """ - - if not uname: - return _('Must provide username') - if not name: - return _('Must provide full name') - - msg = '' - if not password: - return _('Password must be supplied') - - email = uname - - if '@' not in email: - msg += _('email address required (not username)') - return msg - new_password = password - - user = User(username=uname, email=email, is_active=True) - user.set_password(new_password) - try: - user.save() - except IntegrityError: - msg += _('Oops, failed to create user {user}, {error}').format( - user=user, - error="IntegrityError" - ) - return msg - - reg = Registration() - reg.register(user) - - profile = UserProfile(user=user) - profile.name = name - profile.save() - - msg += _('User {user} created successfully!').format(user=user) - return msg - - def delete_user(self, uname): - """Deletes a user from django auth""" - - if not uname: - return _('Must provide username') - if '@' in uname: - try: - user = User.objects.get(email=uname) - except User.DoesNotExist as err: - msg = _('Cannot find user with email address {email_addr}').format(email_addr=uname) - return msg - else: - try: - user = User.objects.get(username=uname) - except User.DoesNotExist as err: - msg = _('Cannot find user with username {username} - {error}').format( - username=uname, - error=str(err) - ) - return msg - user.delete() - return _('Deleted user {username}').format(username=uname) - - def make_datatable(self): - """ - Build the datatable for this view - """ - datatable = { - 'header': [ - _('Statistic'), - _('Value'), - ], - 'title': _('Site statistics'), - 'data': [ - [ - _('Total number of users'), - User.objects.all().count(), - ], - ], - } - return datatable - - def get(self, request): # lint-amnesty, pylint: disable=arguments-differ - if not request.user.is_staff: - raise Http404 - context = { - 'datatable': self.make_datatable(), - 'msg': self.msg, - 'djangopid': os.getpid(), - 'modeflag': {'users': 'active-section'}, - } - return render_to_response(self.template_name, context) - - def post(self, request): - """Handle various actions available on page""" - - if not request.user.is_staff: - raise Http404 - action = request.POST.get('action', '') - track_views.server_track(request, action, {}, page='user_sysdashboard') - - if action == 'create_user': - uname = request.POST.get('student_uname', '').strip() - name = request.POST.get('student_fullname', '').strip() - password = request.POST.get('student_password', '').strip() - self.msg = HTML('

{0}

{1}

{2}').format( - _('Create User Results'), - self.create_user(uname, name, password), self.msg) - elif action == 'del_user': - uname = request.POST.get('student_uname', '').strip() - self.msg = HTML('

{0}

{1}

{2}').format( - _('Delete User Results'), self.delete_user(uname), self.msg) - context = { - 'datatable': self.make_datatable(), - 'msg': self.msg, - 'djangopid': os.getpid(), - 'modeflag': {'users': 'active-section'}, - } - return render_to_response(self.template_name, context) - - -class Courses(SysadminDashboardView): - """ - This manages adding/updating courses from git, deleting courses, and - provides course listing information. - """ - - def git_info_for_course(self, cdir): - """This pulls out some git info like the last commit""" - - cmd = '' - gdir = settings.DATA_DIR / cdir - info = ['', '', ''] - - # Try the data dir, then try to find it in the git import dir - if not gdir.exists(): - git_repo_dir = getattr(settings, 'GIT_REPO_DIR', git_import.DEFAULT_GIT_REPO_DIR) - gdir = path(git_repo_dir) / cdir - if not gdir.exists(): - return info - - cmd = ['git', 'log', '-1', - '--format=format:{ "commit": "%H", "author": "%an %ae", "date": "%ad"}', ] - try: - output_json = json.loads(subprocess.check_output(cmd, cwd=gdir).decode('utf-8')) - info = [output_json['commit'], - output_json['date'], - output_json['author'], ] - except OSError as error: - log.warning("Error fetching git data: %s - %s", str(cdir), str(error)) - except (ValueError, subprocess.CalledProcessError): - pass - - return info - - def get_course_from_git(self, gitloc, branch): - """This downloads and runs the checks for importing a course in git""" - - if not (gitloc.endswith('.git') or gitloc.startswith('http:') or - gitloc.startswith('https:') or gitloc.startswith('git:')): - return _("The git repo location should end with '.git', " - "and be a valid url") - - return self.import_mongo_course(gitloc, branch) - - def import_mongo_course(self, gitloc, branch): - """ - Imports course using management command and captures logging output - at debug level for display in template - """ - - msg = '' - - log.debug('Adding course using git repo %s', gitloc) - - # Grab logging output for debugging imports - output = StringIO() - import_log_handler = logging.StreamHandler(output) - import_log_handler.setLevel(logging.DEBUG) - - logger_names = ['xmodule.modulestore.xml_importer', - 'lms.djangoapps.dashboard.git_import', - 'xmodule.modulestore.xml', - 'xmodule.seq_module', ] - loggers = [] - - for logger_name in logger_names: - logger = logging.getLogger(logger_name) - logger.setLevel(logging.DEBUG) - logger.addHandler(import_log_handler) - loggers.append(logger) - - error_msg = '' - try: - git_import.add_repo(gitloc, None, branch) - except GitImportError as ex: - error_msg = str(ex) - ret = output.getvalue() - - # Remove handler hijacks - for logger in loggers: - logger.setLevel(logging.NOTSET) - logger.removeHandler(import_log_handler) - - if error_msg: - msg_header = error_msg - color = 'red' - else: - msg_header = _('Added Course') - color = 'blue' - - msg = HTML("

{1}

").format(color, msg_header) - msg += HTML("
{0}
").format(escape(ret)) - return msg - - def make_datatable(self, courses=None): - """Creates course information datatable""" - - data = [] - courses = courses or self.get_courses() - for course in courses: - gdir = course.id.course - data.append([course.display_name, str(course.id)] - + self.git_info_for_course(gdir)) - - return dict(header=[_('Course Name'), - _('Directory/ID'), - # Translators: "Git Commit" is a computer command; see http://gitref.org/basic/#commit - _('Git Commit'), - _('Last Change'), - _('Last Editor')], - title=_('Information about all courses'), - data=data) - - def get(self, request): # lint-amnesty, pylint: disable=arguments-differ - """Displays forms and course information""" - - if not request.user.is_staff: - raise Http404 - - context = { - 'datatable': self.make_datatable(), - 'msg': self.msg, - 'djangopid': os.getpid(), - 'modeflag': {'courses': 'active-section'}, - } - return render_to_response(self.template_name, context) - - def post(self, request): - """Handle all actions from courses view""" - - if not request.user.is_staff: - raise Http404 - - action = request.POST.get('action', '') - track_views.server_track(request, action, {}, - page='courses_sysdashboard') - - courses = {course.id: course for course in self.get_courses()} - if action == 'add_course': - gitloc = request.POST.get('repo_location', '').strip().replace(' ', '').replace(';', '') - branch = request.POST.get('repo_branch', '').strip().replace(' ', '').replace(';', '') - self.msg += self.get_course_from_git(gitloc, branch) - - elif action == 'del_course': - course_id = request.POST.get('course_id', '').strip() - course_key = CourseKey.from_string(course_id) - course_found = False - if course_key in courses: - course_found = True - course = courses[course_key] - else: - try: - course = get_course_by_id(course_key) - course_found = True - except Exception as err: # pylint: disable=broad-except - self.msg += _( # lint-amnesty, pylint: disable=translation-of-non-string - HTML('Error - cannot get course with ID {0}
{1}
') - ).format( - course_key, - escape(str(err)) - ) - - if course_found: - # delete course that is stored with mongodb backend - self.def_ms.delete_course(course.id, request.user.id) - # don't delete user permission groups, though - self.msg += \ - HTML("{0} {1} = {2} ({3})").format( - _('Deleted'), str(course.location), str(course.id), course.display_name) - - context = { - 'datatable': self.make_datatable(list(courses.values())), - 'msg': self.msg, - 'djangopid': os.getpid(), - 'modeflag': {'courses': 'active-section'}, - } - return render_to_response(self.template_name, context) - - -class Staffing(SysadminDashboardView): - """ - The status view provides a view of staffing and enrollment in - courses. - """ - - def get(self, request): # lint-amnesty, pylint: disable=arguments-differ - """Displays course Enrollment and staffing course statistics""" - - if not request.user.is_staff: - raise Http404 - data = [] - - for course in self.get_courses(): - datum = [course.display_name, course.id] - datum += [CourseEnrollment.objects.filter( - course_id=course.id).count()] - datum += [CourseStaffRole(course.id).users_with_role().count()] - datum += [','.join([x.username for x in CourseInstructorRole( - course.id).users_with_role()])] - data.append(datum) - - datatable = dict(header=[_('Course Name'), _('course_id'), - _('# enrolled'), _('# staff'), - _('instructors')], - title=_('Enrollment information for all courses'), - data=data) - context = { - 'datatable': datatable, - 'msg': self.msg, - 'djangopid': os.getpid(), - 'modeflag': {'staffing': 'active-section'}, - } - return render_to_response(self.template_name, context) - - -class GitLogs(TemplateView): - """ - This provides a view into the import of courses from git repositories. - It is convenient for allowing course teams to see what may be wrong with - their xml - """ - - template_name = 'sysadmin_dashboard_gitlogs.html' - - @method_decorator(login_required) - def get(self, request, *args, **kwargs): - """Shows logs of imports that happened as a result of a git import""" - - course_id = kwargs.get('course_id') - if course_id: - course_id = CourseKey.from_string(course_id) - - page_size = 10 - - # Set mongodb defaults even if it isn't defined in settings - mongo_db = { - 'host': 'localhost', - 'user': '', - 'password': '', - 'db': 'xlog', - } - - # Allow overrides - if hasattr(settings, 'MONGODB_LOG'): - for config_item in ['host', 'user', 'password', 'db', ]: - mongo_db[config_item] = settings.MONGODB_LOG.get( - config_item, mongo_db[config_item]) - - mongouri = 'mongodb://{user}:{password}@{host}/{db}'.format(**mongo_db) - - error_msg = '' - - try: - if mongo_db['user'] and mongo_db['password']: - mdb = mongoengine.connect(mongo_db['db'], host=mongouri) - else: - mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host']) - except mongoengine.connection.ConnectionError: # lint-amnesty, pylint: disable=no-member - log.exception('Unable to connect to mongodb to save log, ' - 'please check MONGODB_LOG settings.') - - if course_id is None: - # Require staff if not going to specific course - if not request.user.is_staff: - raise Http404 - cilset = CourseImportLog.objects.order_by('-created') - else: - # Allow only course team, instructors, and staff - if not (request.user.is_staff or - CourseInstructorRole(course_id).has_user(request.user) or - CourseStaffRole(course_id).has_user(request.user)): - raise Http404 - log.debug('course_id=%s', course_id) - cilset = CourseImportLog.objects.filter( - course_id=course_id - ).order_by('-created') - log.debug('cilset length=%s', len(cilset)) - - # Paginate the query set - paginator = Paginator(cilset, page_size) - try: - logs = paginator.page(request.GET.get('page')) - except PageNotAnInteger: - logs = paginator.page(1) - except EmptyPage: - # If the page is too high or low - given_page = int(request.GET.get('page')) - page = min(max(1, given_page), paginator.num_pages) - logs = paginator.page(page) - - mdb.close() - context = { - 'logs': logs, - 'course_id': str(course_id) if course_id else None, - 'error_msg': error_msg, - 'page_size': page_size - } - - return render_to_response(self.template_name, context) diff --git a/lms/djangoapps/dashboard/sysadmin_urls.py b/lms/djangoapps/dashboard/sysadmin_urls.py deleted file mode 100644 index f5d6a84243..0000000000 --- a/lms/djangoapps/dashboard/sysadmin_urls.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -Urls for sysadmin dashboard feature -""" - - -from django.conf.urls import url - -from . import sysadmin - -urlpatterns = [ - url(r'^$', sysadmin.Users.as_view(), name="sysadmin"), - url(r'^courses/?$', sysadmin.Courses.as_view(), name="sysadmin_courses"), - url(r'^staffing/?$', sysadmin.Staffing.as_view(), name="sysadmin_staffing"), - url(r'^gitlogs/?$', sysadmin.GitLogs.as_view(), name="gitlogs"), - url(r'^gitlogs/(?P.+)$', sysadmin.GitLogs.as_view(), - name="gitlogs_detail"), -] diff --git a/lms/djangoapps/dashboard/tests/__init__.py b/lms/djangoapps/dashboard/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/lms/djangoapps/dashboard/tests/test_sysadmin.py b/lms/djangoapps/dashboard/tests/test_sysadmin.py deleted file mode 100644 index a179debfa9..0000000000 --- a/lms/djangoapps/dashboard/tests/test_sysadmin.py +++ /dev/null @@ -1,355 +0,0 @@ -""" -Provide tests for sysadmin dashboard feature in sysadmin.py -""" -import glob -import os -import re -import shutil -import unittest -from datetime import datetime -from uuid import uuid4 - -import mongoengine -from django.conf import settings -from django.test.client import Client -from django.test.utils import override_settings -from django.urls import reverse -from opaque_keys.edx.locator import CourseLocator -from pytz import UTC -from xmodule.modulestore.django import modulestore -from xmodule.modulestore.tests.django_utils import TEST_DATA_SPLIT_MODULESTORE, SharedModuleStoreTestCase -from xmodule.modulestore.tests.mongo_connection import MONGO_HOST, MONGO_PORT_NUM - -from common.djangoapps.student.roles import CourseStaffRole, GlobalStaff -from common.djangoapps.student.tests.factories import UserFactory -from common.djangoapps.util.date_utils import DEFAULT_DATE_TIME_FORMAT, get_time_display -from lms.djangoapps.dashboard.git_import import GitImportErrorNoDir -from lms.djangoapps.dashboard.models import CourseImportLog -from openedx.core.djangolib.markup import Text - -TEST_MONGODB_LOG = { - 'host': MONGO_HOST, - 'port': MONGO_PORT_NUM, - 'user': '', - 'password': '', - 'db': 'test_xlog', -} - - -class SysadminBaseTestCase(SharedModuleStoreTestCase): - """ - Base class with common methods used in XML and Mongo tests - """ - - TEST_REPO = 'https://github.com/edx/edx4edx_lite.git' - TEST_BRANCH = 'testing_do_not_delete' - TEST_BRANCH_COURSE = CourseLocator.from_string('course-v1:MITx+edx4edx_branch+edx4edx') - MODULESTORE = TEST_DATA_SPLIT_MODULESTORE - - def setUp(self): - """Setup test case by adding primary user.""" - super().setUp() - self.user = UserFactory.create(username='test_user', - email='test_user+sysadmin@edx.org', - password='foo') - self.client = Client() - - def _setstaff_login(self): - """Makes the test user staff and logs them in""" - GlobalStaff().add_users(self.user) - self.client.login(username=self.user.username, password='foo') - - def _add_edx4edx(self, branch=None): - """Adds the edx4edx sample course""" - post_dict = {'repo_location': self.TEST_REPO, 'action': 'add_course', } - if branch: - post_dict['repo_branch'] = branch - return self.client.post(reverse('sysadmin_courses'), post_dict) - - def _rm_edx4edx(self): - """Deletes the sample course from the XML store""" - def_ms = modulestore() - course_path = '{}/edx4edx_lite'.format( - os.path.abspath(settings.DATA_DIR)) - try: - # using XML store - course = def_ms.courses.get(course_path, None) - except AttributeError: - # Using mongo store - course = def_ms.get_course(CourseLocator('MITx', 'edx4edx', 'edx4edx')) - - # Delete git loaded course - response = self.client.post( - reverse('sysadmin_courses'), - { - 'course_id': str(course.id), - 'action': 'del_course', - } - ) - self.addCleanup(self._rm_glob, f'{course_path}_deleted_*') - - return response - - def _rm_glob(self, path): - """ - Create a shell expansion of passed in parameter and iteratively - remove them. Must only expand to directories. - """ - for path in glob.glob(path): # lint-amnesty, pylint: disable=redefined-argument-from-local - shutil.rmtree(path) - - def _mkdir(self, path): - """ - Create directory and add the cleanup for it. - """ - os.mkdir(path) - self.addCleanup(shutil.rmtree, path) - - -@override_settings( - MONGODB_LOG=TEST_MONGODB_LOG, - GIT_REPO_DIR=settings.TEST_ROOT / f"course_repos_{uuid4().hex}" -) -@unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'), - "ENABLE_SYSADMIN_DASHBOARD not set") -class TestSysAdminMongoCourseImport(SysadminBaseTestCase): - """ - Check that importing into the mongo module store works - """ - - @classmethod - def tearDownClass(cls): - """Delete mongo log entries after test.""" - super().tearDownClass() - try: - mongoengine.connect(TEST_MONGODB_LOG['db']) - CourseImportLog.objects.all().delete() - except mongoengine.connection.ConnectionFailure: - pass - - def _setstaff_login(self): - """ - Makes the test user staff and logs them in - """ - - self.user.is_staff = True - self.user.save() - - self.client.login(username=self.user.username, password='foo') - - def test_missing_repo_dir(self): - """ - Ensure that we handle a missing repo dir - """ - - self._setstaff_login() - - if os.path.isdir(settings.GIT_REPO_DIR): - shutil.rmtree(settings.GIT_REPO_DIR) - - # Create git loaded course - response = self._add_edx4edx() - self.assertContains(response, Text(str(GitImportErrorNoDir(settings.GIT_REPO_DIR)))) - - def test_mongo_course_add_delete(self): - """ - This is the same as TestSysadmin.test_xml_course_add_delete, - but it uses a mongo store - """ - - self._setstaff_login() - self._mkdir(settings.GIT_REPO_DIR) - - def_ms = modulestore() - assert 'xml' != def_ms.get_modulestore_type(None) - - self._add_edx4edx() - course = def_ms.get_course(CourseLocator('MITx', 'edx4edx', 'edx4edx')) - assert course is not None - - self._rm_edx4edx() - course = def_ms.get_course(CourseLocator('MITx', 'edx4edx', 'edx4edx')) - assert course is None - - def test_course_info(self): - """ - Check to make sure we are getting git info for courses - """ - # Regex of first 3 columns of course information table row for - # test course loaded from git. Would not have sha1 if - # git_info_for_course failed. - table_re = re.compile(""" - \\s+ - edX\\sAuthor\\sCourse\\s+ # expected test git course name - course-v1:MITx\\+edx4edx\\+edx4edx\\s+ # expected test git course_id - [a-fA-F\\d]{40} # git sha1 hash - """, re.VERBOSE) - self._setstaff_login() - self._mkdir(settings.GIT_REPO_DIR) - - # Make sure we don't have any git hashes on the page - response = self.client.get(reverse('sysadmin_courses')) - self.assertNotRegex(response.content.decode('utf-8'), table_re) - - # Now add the course and make sure it does match - response = self._add_edx4edx() - self.assertRegex(response.content.decode('utf-8'), table_re) - - def test_gitlogs(self): - """ - Create a log entry and make sure it exists - """ - - self._setstaff_login() - self._mkdir(settings.GIT_REPO_DIR) - - self._add_edx4edx() - response = self.client.get(reverse('gitlogs')) - - # Check that our earlier import has a log with a link to details - self.assertContains(response, '/gitlogs/course-v1:MITx+edx4edx+edx4edx') - - response = self.client.get( - reverse('gitlogs_detail', kwargs={ - 'course_id': 'course-v1:MITx+edx4edx+edx4edx'})) - - self.assertContains(response, '======> IMPORTING course') - - self._rm_edx4edx() - - def test_gitlog_date(self): - """ - Make sure the date is timezone-aware and being converted/formatted - properly. - """ - - tz_names = [ - 'America/New_York', # UTC - 5 - 'Asia/Pyongyang', # UTC + 9 - 'Europe/London', # UTC - 'Canada/Yukon', # UTC - 8 - 'Europe/Moscow', # UTC + 4 - ] - tz_format = DEFAULT_DATE_TIME_FORMAT - - self._setstaff_login() - self._mkdir(settings.GIT_REPO_DIR) - - self._add_edx4edx() - date = CourseImportLog.objects.first().created.replace(tzinfo=UTC) - - for timezone in tz_names: - with (override_settings(TIME_ZONE=timezone)): # lint-amnesty, pylint: disable=superfluous-parens - date_text = get_time_display(date, tz_format, settings.TIME_ZONE) - response = self.client.get(reverse('gitlogs')) - self.assertContains(response, date_text) - - self._rm_edx4edx() - - def test_gitlog_bad_course(self): - """ - Make sure we gracefully handle courses that don't exist. - """ - self._setstaff_login() - response = self.client.get( - reverse('gitlogs_detail', kwargs={ - 'course_id': 'Not/Real/Testing'})) - self.assertContains( - response, - 'No git import logs have been recorded for this course.', - ) - - def test_gitlog_no_logs(self): - """ - Make sure the template behaves well when rendered despite there not being any logs. - (This is for courses imported using methods other than the git_add_course command) - """ - - self._setstaff_login() - self._mkdir(settings.GIT_REPO_DIR) - - self._add_edx4edx() - - # Simulate a lack of git import logs - import_logs = CourseImportLog.objects.all() - import_logs.delete() - - response = self.client.get( - reverse('gitlogs_detail', kwargs={ - 'course_id': 'course-v1:MITx+edx4edx+edx4edx' - }) - ) - self.assertContains(response, 'No git import logs have been recorded for this course.') - - self._rm_edx4edx() - - def test_gitlog_pagination_out_of_range_invalid(self): - """ - Make sure the pagination behaves properly when the requested page is out - of range. - """ - - self._setstaff_login() - - mongoengine.connect(TEST_MONGODB_LOG['db']) - - for _ in range(15): - CourseImportLog( - course_id=CourseLocator.from_string("test/test/test"), - location="location", - import_log="import_log", - git_log="git_log", - repo_dir="repo_dir", - created=datetime.now() - ).save() - - for page, expected in [(-1, 1), (1, 1), (2, 2), (30, 2), ('abc', 1)]: - response = self.client.get( - '{}?page={}'.format( - reverse('gitlogs'), - page - ) - ) - self.assertContains(response, f'Page {expected} of 2') - - CourseImportLog.objects.delete() - - def test_gitlog_courseteam_access(self): - """ - Ensure course team users are allowed to access only their own course. - """ - - self._mkdir(settings.GIT_REPO_DIR) - - self._setstaff_login() - self._add_edx4edx() - self.user.is_staff = False - self.user.save() - logged_in = self.client.login(username=self.user.username, - password='foo') - response = self.client.get(reverse('gitlogs')) - # Make sure our non privileged user doesn't have access to all logs - assert response.status_code == 404 - # Or specific logs - response = self.client.get(reverse('gitlogs_detail', kwargs={ - 'course_id': 'course-v1:MITx+edx4edx+edx4edx' - })) - assert response.status_code == 404 - - # Add user as staff in course team - def_ms = modulestore() - course = def_ms.get_course(CourseLocator('MITx', 'edx4edx', 'edx4edx')) - CourseStaffRole(course.id).add_users(self.user) - - assert CourseStaffRole(course.id).has_user(self.user) - logged_in = self.client.login(username=self.user.username, - password='foo') - assert logged_in - - response = self.client.get( - reverse('gitlogs_detail', kwargs={ - 'course_id': 'course-v1:MITx+edx4edx+edx4edx' - })) - self.assertContains(response, '======> IMPORTING course') - - self._rm_edx4edx() diff --git a/lms/djangoapps/instructor/views/instructor_dashboard.py b/lms/djangoapps/instructor/views/instructor_dashboard.py index 6e95191635..1832dbcc2b 100644 --- a/lms/djangoapps/instructor/views/instructor_dashboard.py +++ b/lms/djangoapps/instructor/views/instructor_dashboard.py @@ -427,12 +427,6 @@ def _section_course_info(course, access): ).format(dashboard_link=dashboard_link) section_data['enrollment_message'] = message - if settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'): - section_data['detailed_gitlogs_url'] = reverse( - 'gitlogs_detail', - kwargs={'course_id': str(course_key)} - ) - try: sorted_cutoffs = sorted(list(course.grade_cutoffs.items()), key=lambda i: i[1], reverse=True) advance = lambda memo, letter_score_tuple: "{}: {}, ".format(letter_score_tuple[0], letter_score_tuple[1]) \ diff --git a/lms/envs/common.py b/lms/envs/common.py index dd2e0408f1..42ae81cbab 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -195,17 +195,6 @@ FEATURES = { # .. toggle_creation_date: 2013-04-13 'ENABLE_MASQUERADE': True, - # .. toggle_name: FEATURES['ENABLE_SYSADMIN_DASHBOARD'] - # .. toggle_implementation: DjangoSetting - # .. toggle_default: False - # .. toggle_description: enables dashboard at /syadmin/ for django staff, for seeing overview of system status, for - # deleting and loading courses, for seeing log of git imports of courseware. Note that some views are noopen_edx - # .. toggle_use_cases: temporary, open_edx - # .. toggle_creation_date: 2013-12-12 - # .. toggle_target_removal_date: None - # .. toggle_warnings: This feature is not supported anymore and should have a target removal date. - 'ENABLE_SYSADMIN_DASHBOARD': False, # sysadmin dashboard, to see what courses are loaded, to delete & load courses - # .. toggle_name: FEATURES['DISABLE_LOGIN_BUTTON'] # .. toggle_implementation: DjangoSetting # .. toggle_default: False @@ -1268,8 +1257,6 @@ DATA_DIR = '/edx/var/edxapp/data' # The banner is only rendered when the switch is activated. MAINTENANCE_BANNER_TEXT = 'Sample banner message' -GIT_REPO_DIR = '/edx/var/edxapp/course_repos' - DJFS = { 'type': 'osfs', 'directory_root': '/edx/var/edxapp/django-pyfs/static/django-pyfs', @@ -2845,7 +2832,6 @@ INSTALLED_APPS = [ 'eventtracking.django.apps.EventTrackingConfig', 'common.djangoapps.util', 'lms.djangoapps.certificates.apps.CertificatesConfig', - 'lms.djangoapps.dashboard', 'lms.djangoapps.instructor_task', 'openedx.core.djangoapps.course_groups', 'lms.djangoapps.bulk_email', diff --git a/lms/envs/production.py b/lms/envs/production.py index 43d8290e88..bf32883127 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -334,12 +334,10 @@ COMMENTS_SERVICE_URL = ENV_TOKENS.get("COMMENTS_SERVICE_URL", '') COMMENTS_SERVICE_KEY = ENV_TOKENS.get("COMMENTS_SERVICE_KEY", '') CERT_QUEUE = ENV_TOKENS.get("CERT_QUEUE", 'test-pull') -# git repo loading environment -GIT_REPO_DIR = ENV_TOKENS.get('GIT_REPO_DIR', '/edx/var/edxapp/course_repos') -GIT_IMPORT_STATIC = ENV_TOKENS.get('GIT_IMPORT_STATIC', True) -GIT_IMPORT_PYTHON_LIB = ENV_TOKENS.get('GIT_IMPORT_PYTHON_LIB', True) +# Python lib settings PYTHON_LIB_FILENAME = ENV_TOKENS.get('PYTHON_LIB_FILENAME', 'python_lib.zip') +# Code jail settings for name, value in ENV_TOKENS.get("CODE_JAIL", {}).items(): oldvalue = CODE_JAIL.get(name) if isinstance(oldvalue, dict): diff --git a/lms/envs/test.py b/lms/envs/test.py index b9ccee0b2d..23a00d4b07 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -270,10 +270,6 @@ OAUTH_ENFORCE_SECURE = False FEATURES['ENABLE_MOBILE_REST_API'] = True FEATURES['ENABLE_VIDEO_ABSTRACTION_LAYER_API'] = True -########################### SYSADMIN DASHBOARD ################################ -FEATURES['ENABLE_SYSADMIN_DASHBOARD'] = True -GIT_REPO_DIR = TEST_ROOT / "course_repos" - ################################# CELERY ###################################### CELERY_ALWAYS_EAGER = True diff --git a/lms/templates/header/navbar-authenticated.html b/lms/templates/header/navbar-authenticated.html index 4c52282441..304a9b2a8b 100644 --- a/lms/templates/header/navbar-authenticated.html +++ b/lms/templates/header/navbar-authenticated.html @@ -12,7 +12,6 @@ from openedx.core.djangoapps.site_configuration import helpers as configuration_ <% show_explore_courses = settings.FEATURES.get('COURSES_ARE_BROWSABLE') - show_sysadmin_dashboard = settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD','') and user.is_staff self.real_user = getattr(user, 'real_user', user) enable_help_link = settings.FEATURES.get('ENABLE_HELP_LINK') @@ -53,12 +52,6 @@ from openedx.core.djangoapps.site_configuration import helpers as configuration_ % endif - % if show_sysadmin_dashboard: -
- ## Translators: This is short for "System administration". - ${_("Sysadmin")} -
- % endif
% if enable_help_link: diff --git a/lms/templates/instructor/instructor_dashboard_2/course_info.html b/lms/templates/instructor/instructor_dashboard_2/course_info.html index b521aefb07..6f2a20b622 100644 --- a/lms/templates/instructor/instructor_dashboard_2/course_info.html +++ b/lms/templates/instructor/instructor_dashboard_2/course_info.html @@ -114,16 +114,6 @@ from openedx.core.djangolib.markup import HTML, Text - %if settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD', '') and user.is_staff: -

- ## Translators: git is a version-control system; see http://git-scm.com/about - ${Text(_("View detailed Git import logs for this course {link_start}by clicking here{link_end}.")).format( - link_start=HTML('').format(section_data['detailed_gitlogs_url']), - link_end=HTML('') - )} -

- %endif -
%if settings.FEATURES.get('ENABLE_INSTRUCTOR_BACKGROUND_TASKS'): diff --git a/lms/templates/navigation/bootstrap/navbar-authenticated.html b/lms/templates/navigation/bootstrap/navbar-authenticated.html index d895f4d9a8..290920cf9d 100644 --- a/lms/templates/navigation/bootstrap/navbar-authenticated.html +++ b/lms/templates/navigation/bootstrap/navbar-authenticated.html @@ -47,13 +47,6 @@ from django.utils.translation import ugettext as _ % endif % endif - - % if settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD','') and user.is_staff: -
  • - ## Translators: This is short for "System administration". - ${_("Sysadmin")} -
    • - % endif