Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Use separate git repos for separate TestCases

Browse Source
This commit is contained in:
Calen Pennington
2016-04-22 13:28:49 -04:00
parent 3402c657e9
commit cb2ac42426
4 changed files with 141 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
lms/djangoapps/dashboard/git_import.py
View File

@@ -13,7 +13,7 @@ from django.conf import settings
from django.core import management
from django.core.management.base import CommandError
from django.utils import timezone
from django.utils.translation import ugettext as _
from django.utils.translation import ugettext_lazy as _
import mongoengine
from dashboard.models import CourseImportLog
@@ -23,33 +23,91 @@ from opaque_keys.edx.locations import SlashSeparatedCourseKey
log = logging.getLogger(__name__)
GIT_REPO_DIR = getattr(settings, 'GIT_REPO_DIR', '/edx/var/app/edxapp/course_repos')
GIT_IMPORT_STATIC = getattr(settings, 'GIT_IMPORT_STATIC', True)
DEFAULT_GIT_REPO_DIR = '/edx/var/app/edxapp/course_repos'
class GitImportError(Exception):
"""
Exception class for handling the typical errors in a git import.
"""
MESSAGE = None
NO_DIR = _("Path {0} doesn't exist, please create it, "
"or configure a different path with "
"GIT_REPO_DIR").format(GIT_REPO_DIR)
URL_BAD = _('Non usable git url provided. Expecting something like:'
' git@github.com:mitocw/edx4edx_lite.git')
BAD_REPO = _('Unable to get git log')
CANNOT_PULL = _('git clone or pull failed!')
XML_IMPORT_FAILED = _('Unable to run import command.')
UNSUPPORTED_STORE = _('The underlying module store does not support import.')
def __init__(self, message=None):
if message is None:
message = self.message
super(GitImportError, self).__init__(message)
class GitImportErrorNoDir(GitImportError):
"""
GitImportError when no directory exists at the specified path.
"""
def __init__(self, repo_dir):
super(GitImportErrorNoDir, self).__init__(
_(
"Path {0} doesn't exist, please create it, "
"or configure a different path with "
"GIT_REPO_DIR"
).format(repo_dir)
)
class GitImportErrorUrlBad(GitImportError):
"""
GitImportError when the git url provided wasn't usable.
"""
MESSAGE = _(
'Non usable git url provided. Expecting something like:'
' git@github.com:mitocw/edx4edx_lite.git'
)
class GitImportErrorBadRepo(GitImportError):
"""
GitImportError when the cloned repository was malformed.
"""
MESSAGE = _('Unable to get git log')
class GitImportErrorCannotPull(GitImportError):
"""
GitImportError when the clone of the repository failed.
"""
MESSAGE = _('git clone or pull failed!')
class GitImportErrorXmlImportFailed(GitImportError):
"""
GitImportError when the course import command failed.
"""
MESSAGE = _('Unable to run import command.')
class GitImportErrorUnsupportedStore(GitImportError):
"""
GitImportError when the modulestore doesn't support imports.
"""
MESSAGE = _('The underlying module store does not support import.')
class GitImportErrorRemoteBranchMissing(GitImportError):
"""
GitImportError when the remote branch doesn't exist.
"""
# Translators: This is an error message when they ask for a
# particular version of a git repository and that version isn't
# available from the remote source they specified
REMOTE_BRANCH_MISSING = _('The specified remote branch is not available.')
MESSAGE = _('The specified remote branch is not available.')
class GitImportErrorCannotBranch(GitImportError):
"""
GitImportError when the local branch doesn't exist.
"""
# Translators: Error message shown when they have asked for a git
# repository branch, a specific version within a repository, that
# doesn't exist, or there is a problem changing to it.
CANNOT_BRANCH = _('Unable to switch to specified branch. Please check '
'your branch name.')
MESSAGE = _('Unable to switch to specified branch. Please check your branch name.')
def cmd_log(cmd, cwd):
@@ -78,7 +136,7 @@ def switch_branch(branch, rdir):
cmd_log(['git', 'fetch', ], rdir)
except subprocess.CalledProcessError as ex:
log.exception('Unable to fetch remote: %r', ex.output)
raise GitImportError(GitImportError.CANNOT_BRANCH)
raise GitImportErrorCannotBranch()
# Check if the branch is available from the remote.
cmd = ['git', 'ls-remote', 'origin', '-h', 'refs/heads/{0}'.format(branch), ]
@@ -86,16 +144,16 @@ def switch_branch(branch, rdir):
output = cmd_log(cmd, rdir)
except subprocess.CalledProcessError as ex:
log.exception('Getting a list of remote branches failed: %r', ex.output)
raise GitImportError(GitImportError.CANNOT_BRANCH)
raise GitImportErrorCannotBranch()
if branch not in output:
raise GitImportError(GitImportError.REMOTE_BRANCH_MISSING)
raise GitImportErrorRemoteBranchMissing()
# Check it the remote branch has already been made locally
cmd = ['git', 'branch', '-a', ]
try:
output = cmd_log(cmd, rdir)
except subprocess.CalledProcessError as ex:
log.exception('Getting a list of local branches failed: %r', ex.output)
raise GitImportError(GitImportError.CANNOT_BRANCH)
raise GitImportErrorCannotBranch()
branches = []
for line in output.split('\n'):
branches.append(line.replace('*', '').strip())
@@ -108,14 +166,14 @@ def switch_branch(branch, rdir):
cmd_log(cmd, rdir)
except subprocess.CalledProcessError as ex:
log.exception('Unable to checkout remote branch: %r', ex.output)
raise GitImportError(GitImportError.CANNOT_BRANCH)
raise GitImportErrorCannotBranch()
# Go ahead and reset hard to the newest version of the branch now that we know
# it is local.
try:
cmd_log(['git', 'reset', '--hard', 'origin/{0}'.format(branch), ], rdir)
except subprocess.CalledProcessError as ex:
log.exception('Unable to reset to branch: %r', ex.output)
raise GitImportError(GitImportError.CANNOT_BRANCH)
raise GitImportErrorCannotBranch()
def add_repo(repo, rdir_in, branch=None):
@@ -126,6 +184,9 @@ def add_repo(repo, rdir_in, branch=None):
"""
# pylint: disable=too-many-statements
git_repo_dir = getattr(settings, 'GIT_REPO_DIR', DEFAULT_GIT_REPO_DIR)
git_import_static = getattr(settings, 'GIT_IMPORT_STATIC', True)
# Set defaults even if it isn't defined in settings
mongo_db = {
'host': 'localhost',
@@ -141,12 +202,12 @@ def add_repo(repo, rdir_in, branch=None):
mongo_db[config_item] = settings.MONGODB_LOG.get(
config_item, mongo_db[config_item])
if not os.path.isdir(GIT_REPO_DIR):
raise GitImportError(GitImportError.NO_DIR)
if not os.path.isdir(git_repo_dir):
raise GitImportErrorNoDir(git_repo_dir)
# pull from git
if not (repo.endswith('.git') or
repo.startswith(('http:', 'https:', 'git:', 'file:'))):
raise GitImportError(GitImportError.URL_BAD)
raise GitImportErrorUrlBad()
if rdir_in:
rdir = os.path.basename(rdir_in)
@@ -154,7 +215,7 @@ def add_repo(repo, rdir_in, branch=None):
rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0]
log.debug('rdir = %s', rdir)
rdirp = '{0}/{1}'.format(GIT_REPO_DIR, rdir)
rdirp = '{0}/{1}'.format(git_repo_dir, rdir)
if os.path.exists(rdirp):
log.info('directory already exists, doing a git pull instead '
'of git clone')
@@ -162,14 +223,14 @@ def add_repo(repo, rdir_in, branch=None):
cwd = rdirp
else:
cmd = ['git', 'clone', repo, ]
cwd = GIT_REPO_DIR
cwd = git_repo_dir
cwd = os.path.abspath(cwd)
try:
ret_git = cmd_log(cmd, cwd=cwd)
except subprocess.CalledProcessError as ex:
log.exception('Error running git pull: %r', ex.output)
raise GitImportError(GitImportError.CANNOT_PULL)
raise GitImportErrorCannotPull()
if branch:
switch_branch(branch, rdirp)
@@ -180,7 +241,7 @@ def add_repo(repo, rdir_in, branch=None):
commit_id = cmd_log(cmd, cwd=rdirp)
except subprocess.CalledProcessError as ex:
log.exception('Unable to get git log: %r', ex.output)
raise GitImportError(GitImportError.BAD_REPO)
raise GitImportErrorBadRepo()
ret_git += '\nCommit ID: {0}'.format(commit_id)
@@ -192,7 +253,7 @@ def add_repo(repo, rdir_in, branch=None):
# I can't discover a way to excercise this, but git is complex
# so still logging and raising here in case.
log.exception('Unable to determine branch: %r', ex.output)
raise GitImportError(GitImportError.BAD_REPO)
raise GitImportErrorBadRepo()
ret_git += '{0}Branch: {1}'.format(' \n', branch)
@@ -212,12 +273,12 @@ def add_repo(repo, rdir_in, branch=None):
loggers.append(logger)
try:
management.call_command('import', GIT_REPO_DIR, rdir,
nostatic=not GIT_IMPORT_STATIC)
management.call_command('import', git_repo_dir, rdir,
nostatic=not git_import_static)
except CommandError:
raise GitImportError(GitImportError.XML_IMPORT_FAILED)
raise GitImportErrorXmlImportFailed()
except NotImplementedError:
raise GitImportError(GitImportError.UNSUPPORTED_STORE)
raise GitImportErrorUnsupportedStore()
ret_import = output.getvalue()
@@ -238,7 +299,7 @@ def add_repo(repo, rdir_in, branch=None):
course_key = CourseKey.from_string(course_id)
except InvalidKeyError:
course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id)
cdir = '{0}/{1}'.format(GIT_REPO_DIR, course_key.course)
cdir = '{0}/{1}'.format(git_repo_dir, course_key.course)
log.debug('Studio course dir = %s', cdir)
if os.path.exists(cdir) and not os.path.islink(cdir):

61
lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py
View File

@@ -7,7 +7,7 @@ import shutil
import StringIO
import subprocess
import unittest
from uuid import uuid4
from nose.plugins.attrib import attr
from django.conf import settings
@@ -17,7 +17,14 @@ from django.test.utils import override_settings
from opaque_keys.edx.locations import SlashSeparatedCourseKey
import dashboard.git_import as git_import
from dashboard.git_import import GitImportError
from dashboard.git_import import (
GitImportError,
GitImportErrorNoDir,
GitImportErrorUrlBad,
GitImportErrorCannotPull,
GitImportErrorBadRepo,
GitImportErrorRemoteBranchMissing,
)
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
@@ -37,7 +44,10 @@ FEATURES_WITH_SSL_AUTH['AUTH_USE_CERTIFICATES'] = True
@attr('shard_3')
@override_settings(MONGODB_LOG=TEST_MONGODB_LOG)
@override_settings(
MONGODB_LOG=TEST_MONGODB_LOG,
GIT_REPO_DIR=settings.TEST_ROOT / "course_repos_{}".format(uuid4().hex)
)
@unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'),
"ENABLE_SYSADMIN_DASHBOARD not set")
class TestGitAddCourse(SharedModuleStoreTestCase):
@@ -49,10 +59,13 @@ class TestGitAddCourse(SharedModuleStoreTestCase):
TEST_COURSE = 'MITx/edx4edx/edx4edx'
TEST_BRANCH = 'testing_do_not_delete'
TEST_BRANCH_COURSE = SlashSeparatedCourseKey('MITx', 'edx4edx_branch', 'edx4edx')
GIT_REPO_DIR = settings.GIT_REPO_DIR
ENABLED_CACHES = ['default', 'mongo_metadata_inheritance', 'loc_cache']
def setUp(self):
super(TestGitAddCourse, self).setUp()
self.git_repo_dir = settings.GIT_REPO_DIR
def assertCommandFailureRegexp(self, regex, *args):
"""
Convenience function for testing command failures
@@ -72,40 +85,40 @@ class TestGitAddCourse(SharedModuleStoreTestCase):
'blah', 'blah', 'blah', 'blah')
# Not a valid path.
self.assertCommandFailureRegexp(
'Path {0} doesn\'t exist, please create it,'.format(self.GIT_REPO_DIR),
'Path {0} doesn\'t exist, please create it,'.format(self.git_repo_dir),
'blah')
# Test successful import from command
if not os.path.isdir(self.GIT_REPO_DIR):
os.mkdir(self.GIT_REPO_DIR)
self.addCleanup(shutil.rmtree, self.GIT_REPO_DIR)
if not os.path.isdir(self.git_repo_dir):
os.mkdir(self.git_repo_dir)
self.addCleanup(shutil.rmtree, self.git_repo_dir)
# Make a course dir that will be replaced with a symlink
# while we are at it.
if not os.path.isdir(self.GIT_REPO_DIR / 'edx4edx'):
os.mkdir(self.GIT_REPO_DIR / 'edx4edx')
if not os.path.isdir(self.git_repo_dir / 'edx4edx'):
os.mkdir(self.git_repo_dir / 'edx4edx')
call_command('git_add_course', self.TEST_REPO,
directory_path=self.GIT_REPO_DIR / 'edx4edx_lite')
directory_path=self.git_repo_dir / 'edx4edx_lite')
# Test with all three args (branch)
call_command('git_add_course', self.TEST_REPO,
directory_path=self.GIT_REPO_DIR / 'edx4edx_lite',
directory_path=self.git_repo_dir / 'edx4edx_lite',
repository_branch=self.TEST_BRANCH)
def test_add_repo(self):
"""
Various exit path tests for test_add_repo
"""
with self.assertRaisesRegexp(GitImportError, GitImportError.NO_DIR):
with self.assertRaises(GitImportErrorNoDir):
git_import.add_repo(self.TEST_REPO, None, None)
os.mkdir(self.GIT_REPO_DIR)
self.addCleanup(shutil.rmtree, self.GIT_REPO_DIR)
os.mkdir(self.git_repo_dir)
self.addCleanup(shutil.rmtree, self.git_repo_dir)
with self.assertRaisesRegexp(GitImportError, GitImportError.URL_BAD):
with self.assertRaises(GitImportErrorUrlBad):
git_import.add_repo('foo', None, None)
with self.assertRaisesRegexp(GitImportError, GitImportError.CANNOT_PULL):
with self.assertRaises(GitImportErrorCannotPull):
git_import.add_repo('file:///foobar.git', None, None)
# Test git repo that exists, but is "broken"
@@ -115,14 +128,14 @@ class TestGitAddCourse(SharedModuleStoreTestCase):
subprocess.check_output(['git', '--bare', 'init', ], stderr=subprocess.STDOUT,
cwd=bare_repo)
with self.assertRaisesRegexp(GitImportError, GitImportError.BAD_REPO):
with self.assertRaises(GitImportErrorBadRepo):
git_import.add_repo('file://{0}'.format(bare_repo), None, None)
def test_detached_repo(self):
"""
Test repo that is in detached head state.
"""
repo_dir = self.GIT_REPO_DIR
repo_dir = self.git_repo_dir
# Test successful import from command
try:
os.mkdir(repo_dir)
@@ -133,21 +146,21 @@ class TestGitAddCourse(SharedModuleStoreTestCase):
subprocess.check_output(['git', 'checkout', 'HEAD~2', ],
stderr=subprocess.STDOUT,
cwd=repo_dir / 'edx4edx_lite')
with self.assertRaisesRegexp(GitImportError, GitImportError.CANNOT_PULL):
with self.assertRaises(GitImportErrorCannotPull):
git_import.add_repo(self.TEST_REPO, repo_dir / 'edx4edx_lite', None)
def test_branching(self):
"""
Exercise branching code of import
"""
repo_dir = self.GIT_REPO_DIR
repo_dir = self.git_repo_dir
# Test successful import from command
if not os.path.isdir(repo_dir):
os.mkdir(repo_dir)
self.addCleanup(shutil.rmtree, repo_dir)
# Checkout non existent branch
with self.assertRaisesRegexp(GitImportError, GitImportError.REMOTE_BRANCH_MISSING):
with self.assertRaises(GitImportErrorRemoteBranchMissing):
git_import.add_repo(self.TEST_REPO, repo_dir / 'edx4edx_lite', 'asdfasdfasdf')
# Checkout new branch
@@ -185,13 +198,13 @@ class TestGitAddCourse(SharedModuleStoreTestCase):
cwd=bare_repo)
# Build repo dir
repo_dir = self.GIT_REPO_DIR
repo_dir = self.git_repo_dir
if not os.path.isdir(repo_dir):
os.mkdir(repo_dir)
self.addCleanup(shutil.rmtree, repo_dir)
rdir = '{0}/bare'.format(repo_dir)
with self.assertRaisesRegexp(GitImportError, GitImportError.BAD_REPO):
with self.assertRaises(GitImportErrorBadRepo):
git_import.add_repo('file://{0}'.format(bare_repo), None, None)
# Get logger for checking strings in logs

3
lms/djangoapps/dashboard/sysadmin.py
View File

@@ -346,7 +346,8 @@ class Courses(SysadminDashboardView):
# Try the data dir, then try to find it in the git import dir
if not gdir.exists():
gdir = path(git_import.GIT_REPO_DIR) / cdir
git_repo_dir = getattr(settings, 'GIT_REPO_DIR', git_import.DEFAULT_GIT_REPO_DIR)
gdir = path(git_repo_dir / cdir)
if not gdir.exists():
return info

10
lms/djangoapps/dashboard/tests/test_sysadmin.py
View File

@@ -6,6 +6,7 @@ import os
import re
import shutil
import unittest
from uuid import uuid4
from util.date_utils import get_time_display, DEFAULT_DATE_TIME_FORMAT
from nose.plugins.attrib import attr
@@ -18,7 +19,7 @@ import mongoengine
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from dashboard.models import CourseImportLog
from dashboard.git_import import GitImportError
from dashboard.git_import import GitImportErrorNoDir
from datetime import datetime
from student.roles import CourseStaffRole, GlobalStaff
from student.tests.factories import UserFactory
@@ -109,7 +110,10 @@ class SysadminBaseTestCase(SharedModuleStoreTestCase):
@attr('shard_1')
@override_settings(MONGODB_LOG=TEST_MONGODB_LOG)
@override_settings(
MONGODB_LOG=TEST_MONGODB_LOG,
GIT_REPO_DIR=settings.TEST_ROOT / "course_repos_{}".format(uuid4().hex)
)
@unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'),
"ENABLE_SYSADMIN_DASHBOARD not set")
class TestSysAdminMongoCourseImport(SysadminBaseTestCase):
@@ -149,7 +153,7 @@ class TestSysAdminMongoCourseImport(SysadminBaseTestCase):
# Create git loaded course
response = self._add_edx4edx()
self.assertIn(GitImportError.NO_DIR,
self.assertIn(GitImportErrorNoDir(settings.GIT_REPO_DIR).message,
response.content.decode('UTF-8'))
def test_mongo_course_add_delete(self):