From cb2ac42426936b2324d5c338517ceae1f989f116 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Fri, 22 Apr 2016 13:28:49 -0400 Subject: [PATCH] Use separate git repos for separate TestCases --- lms/djangoapps/dashboard/git_import.py | 129 +++++++++++++----- .../commands/tests/test_git_add_course.py | 61 +++++---- lms/djangoapps/dashboard/sysadmin.py | 3 +- .../dashboard/tests/test_sysadmin.py | 10 +- 4 files changed, 141 insertions(+), 62 deletions(-) diff --git a/lms/djangoapps/dashboard/git_import.py b/lms/djangoapps/dashboard/git_import.py index d503606113..38226af5da 100644 --- a/lms/djangoapps/dashboard/git_import.py +++ b/lms/djangoapps/dashboard/git_import.py @@ -13,7 +13,7 @@ from django.conf import settings from django.core import management from django.core.management.base import CommandError from django.utils import timezone -from django.utils.translation import ugettext as _ +from django.utils.translation import ugettext_lazy as _ import mongoengine from dashboard.models import CourseImportLog @@ -23,33 +23,91 @@ from opaque_keys.edx.locations import SlashSeparatedCourseKey log = logging.getLogger(__name__) -GIT_REPO_DIR = getattr(settings, 'GIT_REPO_DIR', '/edx/var/app/edxapp/course_repos') -GIT_IMPORT_STATIC = getattr(settings, 'GIT_IMPORT_STATIC', True) +DEFAULT_GIT_REPO_DIR = '/edx/var/app/edxapp/course_repos' class GitImportError(Exception): """ Exception class for handling the typical errors in a git import. """ + MESSAGE = None - NO_DIR = _("Path {0} doesn't exist, please create it, " - "or configure a different path with " - "GIT_REPO_DIR").format(GIT_REPO_DIR) - URL_BAD = _('Non usable git url provided. Expecting something like:' - ' git@github.com:mitocw/edx4edx_lite.git') - BAD_REPO = _('Unable to get git log') - CANNOT_PULL = _('git clone or pull failed!') - XML_IMPORT_FAILED = _('Unable to run import command.') - UNSUPPORTED_STORE = _('The underlying module store does not support import.') + def __init__(self, message=None): + if message is None: + message = self.message + super(GitImportError, self).__init__(message) + + +class GitImportErrorNoDir(GitImportError): + """ + GitImportError when no directory exists at the specified path. + """ + def __init__(self, repo_dir): + super(GitImportErrorNoDir, self).__init__( + _( + "Path {0} doesn't exist, please create it, " + "or configure a different path with " + "GIT_REPO_DIR" + ).format(repo_dir) + ) + + +class GitImportErrorUrlBad(GitImportError): + """ + GitImportError when the git url provided wasn't usable. + """ + MESSAGE = _( + 'Non usable git url provided. Expecting something like:' + ' git@github.com:mitocw/edx4edx_lite.git' + ) + + +class GitImportErrorBadRepo(GitImportError): + """ + GitImportError when the cloned repository was malformed. + """ + MESSAGE = _('Unable to get git log') + + +class GitImportErrorCannotPull(GitImportError): + """ + GitImportError when the clone of the repository failed. + """ + MESSAGE = _('git clone or pull failed!') + + +class GitImportErrorXmlImportFailed(GitImportError): + """ + GitImportError when the course import command failed. + """ + MESSAGE = _('Unable to run import command.') + + +class GitImportErrorUnsupportedStore(GitImportError): + """ + GitImportError when the modulestore doesn't support imports. + """ + MESSAGE = _('The underlying module store does not support import.') + + +class GitImportErrorRemoteBranchMissing(GitImportError): + """ + GitImportError when the remote branch doesn't exist. + """ # Translators: This is an error message when they ask for a # particular version of a git repository and that version isn't # available from the remote source they specified - REMOTE_BRANCH_MISSING = _('The specified remote branch is not available.') + MESSAGE = _('The specified remote branch is not available.') + + +class GitImportErrorCannotBranch(GitImportError): + """ + GitImportError when the local branch doesn't exist. + """ # Translators: Error message shown when they have asked for a git # repository branch, a specific version within a repository, that # doesn't exist, or there is a problem changing to it. - CANNOT_BRANCH = _('Unable to switch to specified branch. Please check ' - 'your branch name.') + MESSAGE = _('Unable to switch to specified branch. Please check your branch name.') def cmd_log(cmd, cwd): @@ -78,7 +136,7 @@ def switch_branch(branch, rdir): cmd_log(['git', 'fetch', ], rdir) except subprocess.CalledProcessError as ex: log.exception('Unable to fetch remote: %r', ex.output) - raise GitImportError(GitImportError.CANNOT_BRANCH) + raise GitImportErrorCannotBranch() # Check if the branch is available from the remote. cmd = ['git', 'ls-remote', 'origin', '-h', 'refs/heads/{0}'.format(branch), ] @@ -86,16 +144,16 @@ def switch_branch(branch, rdir): output = cmd_log(cmd, rdir) except subprocess.CalledProcessError as ex: log.exception('Getting a list of remote branches failed: %r', ex.output) - raise GitImportError(GitImportError.CANNOT_BRANCH) + raise GitImportErrorCannotBranch() if branch not in output: - raise GitImportError(GitImportError.REMOTE_BRANCH_MISSING) + raise GitImportErrorRemoteBranchMissing() # Check it the remote branch has already been made locally cmd = ['git', 'branch', '-a', ] try: output = cmd_log(cmd, rdir) except subprocess.CalledProcessError as ex: log.exception('Getting a list of local branches failed: %r', ex.output) - raise GitImportError(GitImportError.CANNOT_BRANCH) + raise GitImportErrorCannotBranch() branches = [] for line in output.split('\n'): branches.append(line.replace('*', '').strip()) @@ -108,14 +166,14 @@ def switch_branch(branch, rdir): cmd_log(cmd, rdir) except subprocess.CalledProcessError as ex: log.exception('Unable to checkout remote branch: %r', ex.output) - raise GitImportError(GitImportError.CANNOT_BRANCH) + raise GitImportErrorCannotBranch() # Go ahead and reset hard to the newest version of the branch now that we know # it is local. try: cmd_log(['git', 'reset', '--hard', 'origin/{0}'.format(branch), ], rdir) except subprocess.CalledProcessError as ex: log.exception('Unable to reset to branch: %r', ex.output) - raise GitImportError(GitImportError.CANNOT_BRANCH) + raise GitImportErrorCannotBranch() def add_repo(repo, rdir_in, branch=None): @@ -126,6 +184,9 @@ def add_repo(repo, rdir_in, branch=None): """ # pylint: disable=too-many-statements + git_repo_dir = getattr(settings, 'GIT_REPO_DIR', DEFAULT_GIT_REPO_DIR) + git_import_static = getattr(settings, 'GIT_IMPORT_STATIC', True) + # Set defaults even if it isn't defined in settings mongo_db = { 'host': 'localhost', @@ -141,12 +202,12 @@ def add_repo(repo, rdir_in, branch=None): mongo_db[config_item] = settings.MONGODB_LOG.get( config_item, mongo_db[config_item]) - if not os.path.isdir(GIT_REPO_DIR): - raise GitImportError(GitImportError.NO_DIR) + if not os.path.isdir(git_repo_dir): + raise GitImportErrorNoDir(git_repo_dir) # pull from git if not (repo.endswith('.git') or repo.startswith(('http:', 'https:', 'git:', 'file:'))): - raise GitImportError(GitImportError.URL_BAD) + raise GitImportErrorUrlBad() if rdir_in: rdir = os.path.basename(rdir_in) @@ -154,7 +215,7 @@ def add_repo(repo, rdir_in, branch=None): rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0] log.debug('rdir = %s', rdir) - rdirp = '{0}/{1}'.format(GIT_REPO_DIR, rdir) + rdirp = '{0}/{1}'.format(git_repo_dir, rdir) if os.path.exists(rdirp): log.info('directory already exists, doing a git pull instead ' 'of git clone') @@ -162,14 +223,14 @@ def add_repo(repo, rdir_in, branch=None): cwd = rdirp else: cmd = ['git', 'clone', repo, ] - cwd = GIT_REPO_DIR + cwd = git_repo_dir cwd = os.path.abspath(cwd) try: ret_git = cmd_log(cmd, cwd=cwd) except subprocess.CalledProcessError as ex: log.exception('Error running git pull: %r', ex.output) - raise GitImportError(GitImportError.CANNOT_PULL) + raise GitImportErrorCannotPull() if branch: switch_branch(branch, rdirp) @@ -180,7 +241,7 @@ def add_repo(repo, rdir_in, branch=None): commit_id = cmd_log(cmd, cwd=rdirp) except subprocess.CalledProcessError as ex: log.exception('Unable to get git log: %r', ex.output) - raise GitImportError(GitImportError.BAD_REPO) + raise GitImportErrorBadRepo() ret_git += '\nCommit ID: {0}'.format(commit_id) @@ -192,7 +253,7 @@ def add_repo(repo, rdir_in, branch=None): # I can't discover a way to excercise this, but git is complex # so still logging and raising here in case. log.exception('Unable to determine branch: %r', ex.output) - raise GitImportError(GitImportError.BAD_REPO) + raise GitImportErrorBadRepo() ret_git += '{0}Branch: {1}'.format(' \n', branch) @@ -212,12 +273,12 @@ def add_repo(repo, rdir_in, branch=None): loggers.append(logger) try: - management.call_command('import', GIT_REPO_DIR, rdir, - nostatic=not GIT_IMPORT_STATIC) + management.call_command('import', git_repo_dir, rdir, + nostatic=not git_import_static) except CommandError: - raise GitImportError(GitImportError.XML_IMPORT_FAILED) + raise GitImportErrorXmlImportFailed() except NotImplementedError: - raise GitImportError(GitImportError.UNSUPPORTED_STORE) + raise GitImportErrorUnsupportedStore() ret_import = output.getvalue() @@ -238,7 +299,7 @@ def add_repo(repo, rdir_in, branch=None): course_key = CourseKey.from_string(course_id) except InvalidKeyError: course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id) - cdir = '{0}/{1}'.format(GIT_REPO_DIR, course_key.course) + cdir = '{0}/{1}'.format(git_repo_dir, course_key.course) log.debug('Studio course dir = %s', cdir) if os.path.exists(cdir) and not os.path.islink(cdir): diff --git a/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py b/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py index f5550cdfbf..dedf9217f8 100644 --- a/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py +++ b/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py @@ -7,7 +7,7 @@ import shutil import StringIO import subprocess import unittest - +from uuid import uuid4 from nose.plugins.attrib import attr from django.conf import settings @@ -17,7 +17,14 @@ from django.test.utils import override_settings from opaque_keys.edx.locations import SlashSeparatedCourseKey import dashboard.git_import as git_import -from dashboard.git_import import GitImportError +from dashboard.git_import import ( + GitImportError, + GitImportErrorNoDir, + GitImportErrorUrlBad, + GitImportErrorCannotPull, + GitImportErrorBadRepo, + GitImportErrorRemoteBranchMissing, +) from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import modulestore from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase @@ -37,7 +44,10 @@ FEATURES_WITH_SSL_AUTH['AUTH_USE_CERTIFICATES'] = True @attr('shard_3') -@override_settings(MONGODB_LOG=TEST_MONGODB_LOG) +@override_settings( + MONGODB_LOG=TEST_MONGODB_LOG, + GIT_REPO_DIR=settings.TEST_ROOT / "course_repos_{}".format(uuid4().hex) +) @unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'), "ENABLE_SYSADMIN_DASHBOARD not set") class TestGitAddCourse(SharedModuleStoreTestCase): @@ -49,10 +59,13 @@ class TestGitAddCourse(SharedModuleStoreTestCase): TEST_COURSE = 'MITx/edx4edx/edx4edx' TEST_BRANCH = 'testing_do_not_delete' TEST_BRANCH_COURSE = SlashSeparatedCourseKey('MITx', 'edx4edx_branch', 'edx4edx') - GIT_REPO_DIR = settings.GIT_REPO_DIR ENABLED_CACHES = ['default', 'mongo_metadata_inheritance', 'loc_cache'] + def setUp(self): + super(TestGitAddCourse, self).setUp() + self.git_repo_dir = settings.GIT_REPO_DIR + def assertCommandFailureRegexp(self, regex, *args): """ Convenience function for testing command failures @@ -72,40 +85,40 @@ class TestGitAddCourse(SharedModuleStoreTestCase): 'blah', 'blah', 'blah', 'blah') # Not a valid path. self.assertCommandFailureRegexp( - 'Path {0} doesn\'t exist, please create it,'.format(self.GIT_REPO_DIR), + 'Path {0} doesn\'t exist, please create it,'.format(self.git_repo_dir), 'blah') # Test successful import from command - if not os.path.isdir(self.GIT_REPO_DIR): - os.mkdir(self.GIT_REPO_DIR) - self.addCleanup(shutil.rmtree, self.GIT_REPO_DIR) + if not os.path.isdir(self.git_repo_dir): + os.mkdir(self.git_repo_dir) + self.addCleanup(shutil.rmtree, self.git_repo_dir) # Make a course dir that will be replaced with a symlink # while we are at it. - if not os.path.isdir(self.GIT_REPO_DIR / 'edx4edx'): - os.mkdir(self.GIT_REPO_DIR / 'edx4edx') + if not os.path.isdir(self.git_repo_dir / 'edx4edx'): + os.mkdir(self.git_repo_dir / 'edx4edx') call_command('git_add_course', self.TEST_REPO, - directory_path=self.GIT_REPO_DIR / 'edx4edx_lite') + directory_path=self.git_repo_dir / 'edx4edx_lite') # Test with all three args (branch) call_command('git_add_course', self.TEST_REPO, - directory_path=self.GIT_REPO_DIR / 'edx4edx_lite', + directory_path=self.git_repo_dir / 'edx4edx_lite', repository_branch=self.TEST_BRANCH) def test_add_repo(self): """ Various exit path tests for test_add_repo """ - with self.assertRaisesRegexp(GitImportError, GitImportError.NO_DIR): + with self.assertRaises(GitImportErrorNoDir): git_import.add_repo(self.TEST_REPO, None, None) - os.mkdir(self.GIT_REPO_DIR) - self.addCleanup(shutil.rmtree, self.GIT_REPO_DIR) + os.mkdir(self.git_repo_dir) + self.addCleanup(shutil.rmtree, self.git_repo_dir) - with self.assertRaisesRegexp(GitImportError, GitImportError.URL_BAD): + with self.assertRaises(GitImportErrorUrlBad): git_import.add_repo('foo', None, None) - with self.assertRaisesRegexp(GitImportError, GitImportError.CANNOT_PULL): + with self.assertRaises(GitImportErrorCannotPull): git_import.add_repo('file:///foobar.git', None, None) # Test git repo that exists, but is "broken" @@ -115,14 +128,14 @@ class TestGitAddCourse(SharedModuleStoreTestCase): subprocess.check_output(['git', '--bare', 'init', ], stderr=subprocess.STDOUT, cwd=bare_repo) - with self.assertRaisesRegexp(GitImportError, GitImportError.BAD_REPO): + with self.assertRaises(GitImportErrorBadRepo): git_import.add_repo('file://{0}'.format(bare_repo), None, None) def test_detached_repo(self): """ Test repo that is in detached head state. """ - repo_dir = self.GIT_REPO_DIR + repo_dir = self.git_repo_dir # Test successful import from command try: os.mkdir(repo_dir) @@ -133,21 +146,21 @@ class TestGitAddCourse(SharedModuleStoreTestCase): subprocess.check_output(['git', 'checkout', 'HEAD~2', ], stderr=subprocess.STDOUT, cwd=repo_dir / 'edx4edx_lite') - with self.assertRaisesRegexp(GitImportError, GitImportError.CANNOT_PULL): + with self.assertRaises(GitImportErrorCannotPull): git_import.add_repo(self.TEST_REPO, repo_dir / 'edx4edx_lite', None) def test_branching(self): """ Exercise branching code of import """ - repo_dir = self.GIT_REPO_DIR + repo_dir = self.git_repo_dir # Test successful import from command if not os.path.isdir(repo_dir): os.mkdir(repo_dir) self.addCleanup(shutil.rmtree, repo_dir) # Checkout non existent branch - with self.assertRaisesRegexp(GitImportError, GitImportError.REMOTE_BRANCH_MISSING): + with self.assertRaises(GitImportErrorRemoteBranchMissing): git_import.add_repo(self.TEST_REPO, repo_dir / 'edx4edx_lite', 'asdfasdfasdf') # Checkout new branch @@ -185,13 +198,13 @@ class TestGitAddCourse(SharedModuleStoreTestCase): cwd=bare_repo) # Build repo dir - repo_dir = self.GIT_REPO_DIR + repo_dir = self.git_repo_dir if not os.path.isdir(repo_dir): os.mkdir(repo_dir) self.addCleanup(shutil.rmtree, repo_dir) rdir = '{0}/bare'.format(repo_dir) - with self.assertRaisesRegexp(GitImportError, GitImportError.BAD_REPO): + with self.assertRaises(GitImportErrorBadRepo): git_import.add_repo('file://{0}'.format(bare_repo), None, None) # Get logger for checking strings in logs diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py index 4273035d9e..d8144107e8 100644 --- a/lms/djangoapps/dashboard/sysadmin.py +++ b/lms/djangoapps/dashboard/sysadmin.py @@ -346,7 +346,8 @@ class Courses(SysadminDashboardView): # Try the data dir, then try to find it in the git import dir if not gdir.exists(): - gdir = path(git_import.GIT_REPO_DIR) / cdir + git_repo_dir = getattr(settings, 'GIT_REPO_DIR', git_import.DEFAULT_GIT_REPO_DIR) + gdir = path(git_repo_dir / cdir) if not gdir.exists(): return info diff --git a/lms/djangoapps/dashboard/tests/test_sysadmin.py b/lms/djangoapps/dashboard/tests/test_sysadmin.py index 6a0d063d0f..efd56f0a9d 100644 --- a/lms/djangoapps/dashboard/tests/test_sysadmin.py +++ b/lms/djangoapps/dashboard/tests/test_sysadmin.py @@ -6,6 +6,7 @@ import os import re import shutil import unittest +from uuid import uuid4 from util.date_utils import get_time_display, DEFAULT_DATE_TIME_FORMAT from nose.plugins.attrib import attr @@ -18,7 +19,7 @@ import mongoengine from opaque_keys.edx.locations import SlashSeparatedCourseKey from dashboard.models import CourseImportLog -from dashboard.git_import import GitImportError +from dashboard.git_import import GitImportErrorNoDir from datetime import datetime from student.roles import CourseStaffRole, GlobalStaff from student.tests.factories import UserFactory @@ -109,7 +110,10 @@ class SysadminBaseTestCase(SharedModuleStoreTestCase): @attr('shard_1') -@override_settings(MONGODB_LOG=TEST_MONGODB_LOG) +@override_settings( + MONGODB_LOG=TEST_MONGODB_LOG, + GIT_REPO_DIR=settings.TEST_ROOT / "course_repos_{}".format(uuid4().hex) +) @unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'), "ENABLE_SYSADMIN_DASHBOARD not set") class TestSysAdminMongoCourseImport(SysadminBaseTestCase): @@ -149,7 +153,7 @@ class TestSysAdminMongoCourseImport(SysadminBaseTestCase): # Create git loaded course response = self._add_edx4edx() - self.assertIn(GitImportError.NO_DIR, + self.assertIn(GitImportErrorNoDir(settings.GIT_REPO_DIR).message, response.content.decode('UTF-8')) def test_mongo_course_add_delete(self):