diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index 5f68faed15..a4c2116c60 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -36,7 +36,6 @@ from xmodule.modulestore.exceptions import ItemNotFoundError from xmodule.x_module import ModuleSystem from xmodule.error_module import ErrorDescriptor from xmodule.errortracker import exc_info_to_str -from github_sync import export_to_github from static_replace import replace_urls from external_auth.views import ssl_login_shortcut diff --git a/cms/djangoapps/github_sync/__init__.py b/cms/djangoapps/github_sync/__init__.py deleted file mode 100644 index a4dbe29fb6..0000000000 --- a/cms/djangoapps/github_sync/__init__.py +++ /dev/null @@ -1,125 +0,0 @@ -import logging -import os - -from django.conf import settings -from fs.osfs import OSFS -from git import Repo, PushInfo - -from xmodule.modulestore.xml_importer import import_from_xml -from xmodule.modulestore.django import modulestore -from collections import namedtuple - -from .exceptions import GithubSyncError, InvalidRepo - -log = logging.getLogger(__name__) - -RepoSettings = namedtuple('RepoSettings', 'path branch origin') - - -def sync_all_with_github(): - """ - Sync all defined repositories from github - """ - for repo_name in settings.REPOS: - sync_with_github(load_repo_settings(repo_name)) - - -def sync_with_github(repo_settings): - """ - Sync specified repository from github - - repo_settings: A RepoSettings defining which repo to sync - """ - revision, course = import_from_github(repo_settings) - export_to_github(course, "Changes from cms import of revision %s" % revision, "CMS ") - - -def setup_repo(repo_settings): - """ - Reset the local github repo specified by repo_settings - - repo_settings (RepoSettings): The settings for the repo to reset - """ - course_dir = repo_settings.path - repo_path = settings.GITHUB_REPO_ROOT / course_dir - - if not os.path.isdir(repo_path): - Repo.clone_from(repo_settings.origin, repo_path) - - git_repo = Repo(repo_path) - origin = git_repo.remotes.origin - origin.fetch() - - # Do a hard reset to the remote branch so that we have a clean import - git_repo.git.checkout(repo_settings.branch) - - return git_repo - - -def load_repo_settings(course_dir): - """ - Returns the repo_settings for the course stored in course_dir - """ - if course_dir not in settings.REPOS: - raise InvalidRepo(course_dir) - - return RepoSettings(course_dir, **settings.REPOS[course_dir]) - - -def import_from_github(repo_settings): - """ - Imports data into the modulestore based on the XML stored on github - """ - course_dir = repo_settings.path - git_repo = setup_repo(repo_settings) - git_repo.head.reset('origin/%s' % repo_settings.branch, index=True, working_tree=True) - - module_store = import_from_xml(modulestore(), - settings.GITHUB_REPO_ROOT, course_dirs=[course_dir]) - return git_repo.head.commit.hexsha, module_store.courses[course_dir] - - -def export_to_github(course, commit_message, author_str=None): - ''' - Commit any changes to the specified course with given commit message, - and push to github (if MITX_FEATURES['GITHUB_PUSH'] is True). - If author_str is specified, uses it in the commit. - ''' - course_dir = course.metadata.get('data_dir', course.location.course) - try: - repo_settings = load_repo_settings(course_dir) - except InvalidRepo: - log.warning("Invalid repo {0}, not exporting data to xml".format(course_dir)) - return - - git_repo = setup_repo(repo_settings) - - fs = OSFS(git_repo.working_dir) - xml = course.export_to_xml(fs) - - with fs.open('course.xml', 'w') as course_xml: - course_xml.write(xml) - - if git_repo.is_dirty(): - git_repo.git.add(A=True) - if author_str is not None: - git_repo.git.commit(m=commit_message, author=author_str) - else: - git_repo.git.commit(m=commit_message) - - origin = git_repo.remotes.origin - if settings.MITX_FEATURES['GITHUB_PUSH']: - push_infos = origin.push() - if len(push_infos) > 1: - log.error('Unexpectedly pushed multiple heads: {infos}'.format( - infos="\n".join(str(info.summary) for info in push_infos) - )) - - if push_infos[0].flags & PushInfo.ERROR: - log.error('Failed push: flags={p.flags}, local_ref={p.local_ref}, ' - 'remote_ref_string={p.remote_ref_string}, ' - 'remote_ref={p.remote_ref}, old_commit={p.old_commit}, ' - 'summary={p.summary})'.format(p=push_infos[0])) - raise GithubSyncError('Failed to push: {info}'.format( - info=str(push_infos[0].summary) - )) diff --git a/cms/djangoapps/github_sync/exceptions.py b/cms/djangoapps/github_sync/exceptions.py deleted file mode 100644 index 1fe8d1d73e..0000000000 --- a/cms/djangoapps/github_sync/exceptions.py +++ /dev/null @@ -1,6 +0,0 @@ -class GithubSyncError(Exception): - pass - - -class InvalidRepo(Exception): - pass diff --git a/cms/djangoapps/github_sync/management/__init__.py b/cms/djangoapps/github_sync/management/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/cms/djangoapps/github_sync/management/commands/__init__.py b/cms/djangoapps/github_sync/management/commands/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/cms/djangoapps/github_sync/management/commands/sync_with_github.py b/cms/djangoapps/github_sync/management/commands/sync_with_github.py deleted file mode 100644 index 4383871df3..0000000000 --- a/cms/djangoapps/github_sync/management/commands/sync_with_github.py +++ /dev/null @@ -1,14 +0,0 @@ -### -### Script for syncing CMS with defined github repos -### - -from django.core.management.base import NoArgsCommand -from github_sync import sync_all_with_github - - -class Command(NoArgsCommand): - help = \ -'''Sync the CMS with the defined github repos''' - - def handle_noargs(self, **options): - sync_all_with_github() diff --git a/cms/djangoapps/github_sync/tests/__init__.py b/cms/djangoapps/github_sync/tests/__init__.py deleted file mode 100644 index e2b9a909a7..0000000000 --- a/cms/djangoapps/github_sync/tests/__init__.py +++ /dev/null @@ -1,108 +0,0 @@ -from django.test import TestCase -from path import path -import shutil -from github_sync import ( - import_from_github, export_to_github, load_repo_settings, - sync_all_with_github, sync_with_github -) -from git import Repo -from django.conf import settings -from xmodule.modulestore.django import modulestore -from xmodule.modulestore import Location -from override_settings import override_settings -from github_sync.exceptions import GithubSyncError -from mock import patch, Mock - -REPO_DIR = settings.GITHUB_REPO_ROOT / 'local_repo' -WORKING_DIR = path(settings.TEST_ROOT) -REMOTE_DIR = WORKING_DIR / 'remote_repo' - - -@override_settings(REPOS={ - 'local_repo': { - 'origin': REMOTE_DIR, - 'branch': 'master', - } -}) -class GithubSyncTestCase(TestCase): - - def cleanup(self): - shutil.rmtree(REPO_DIR, ignore_errors=True) - shutil.rmtree(REMOTE_DIR, ignore_errors=True) - modulestore().collection.drop() - - def setUp(self): - # make sure there's no stale data lying around - self.cleanup() - - shutil.copytree('common/test/data/toy', REMOTE_DIR) - - remote = Repo.init(REMOTE_DIR) - remote.git.add(A=True) - remote.git.commit(m='Initial commit') - remote.git.config("receive.denyCurrentBranch", "ignore") - - self.import_revision, self.import_course = import_from_github(load_repo_settings('local_repo')) - - def tearDown(self): - self.cleanup() - - def test_initialize_repo(self): - """ - Test that importing from github will create a repo if the repo doesn't already exist - """ - self.assertEquals(1, len(Repo(REPO_DIR).head.reference.log())) - - def test_import_contents(self): - """ - Test that the import loads the correct course into the modulestore - """ - self.assertEquals('Toy Course', self.import_course.metadata['display_name']) - self.assertIn( - Location('i4x://edX/toy/chapter/Overview'), - [child.location for child in self.import_course.get_children()]) - self.assertEquals(2, len(self.import_course.get_children())) - - @patch('github_sync.sync_with_github') - def test_sync_all_with_github(self, sync_with_github): - sync_all_with_github() - sync_with_github.assert_called_with(load_repo_settings('local_repo')) - - def test_sync_with_github(self): - with patch('github_sync.import_from_github', Mock(return_value=(Mock(), Mock()))) as import_from_github: - with patch('github_sync.export_to_github') as export_to_github: - settings = load_repo_settings('local_repo') - sync_with_github(settings) - import_from_github.assert_called_with(settings) - export_to_github.assert_called - - @override_settings(MITX_FEATURES={'GITHUB_PUSH': False}) - def test_export_no_pash(self): - """ - Test that with the GITHUB_PUSH feature disabled, no content is pushed to the remote - """ - export_to_github(self.import_course, 'Test no-push') - self.assertEquals(1, Repo(REMOTE_DIR).head.commit.count()) - - @override_settings(MITX_FEATURES={'GITHUB_PUSH': True}) - def test_export_push(self): - """ - Test that with GITHUB_PUSH enabled, content is pushed to the remote - """ - self.import_course.metadata['display_name'] = 'Changed display name' - export_to_github(self.import_course, 'Test push') - self.assertEquals(2, Repo(REMOTE_DIR).head.commit.count()) - - @override_settings(MITX_FEATURES={'GITHUB_PUSH': True}) - def test_export_conflict(self): - """ - Test that if there is a conflict when pushing to the remote repo, nothing is pushed and an exception is raised - """ - self.import_course.metadata['display_name'] = 'Changed display name' - - remote = Repo(REMOTE_DIR) - remote.git.commit(allow_empty=True, m="Testing conflict commit") - - self.assertRaises(GithubSyncError, export_to_github, self.import_course, 'Test push') - self.assertEquals(2, remote.head.reference.commit.count()) - self.assertEquals("Testing conflict commit\n", remote.head.reference.commit.message) diff --git a/cms/djangoapps/github_sync/tests/test_views.py b/cms/djangoapps/github_sync/tests/test_views.py deleted file mode 100644 index 37030d6a1b..0000000000 --- a/cms/djangoapps/github_sync/tests/test_views.py +++ /dev/null @@ -1,43 +0,0 @@ -import json -from django.test.client import Client -from django.test import TestCase -from mock import patch -from override_settings import override_settings -from github_sync import load_repo_settings - - -@override_settings(REPOS={'repo': {'branch': 'branch', 'origin': 'origin'}}) -class PostReceiveTestCase(TestCase): - def setUp(self): - self.client = Client() - - @patch('github_sync.views.import_from_github') - def test_non_branch(self, import_from_github): - self.client.post('/github_service_hook', {'payload': json.dumps({ - 'ref': 'refs/tags/foo'}) - }) - self.assertFalse(import_from_github.called) - - @patch('github_sync.views.import_from_github') - def test_non_watched_repo(self, import_from_github): - self.client.post('/github_service_hook', {'payload': json.dumps({ - 'ref': 'refs/heads/branch', - 'repository': {'name': 'bad_repo'}}) - }) - self.assertFalse(import_from_github.called) - - @patch('github_sync.views.import_from_github') - def test_non_tracked_branch(self, import_from_github): - self.client.post('/github_service_hook', {'payload': json.dumps({ - 'ref': 'refs/heads/non_branch', - 'repository': {'name': 'repo'}}) - }) - self.assertFalse(import_from_github.called) - - @patch('github_sync.views.import_from_github') - def test_tracked_branch(self, import_from_github): - self.client.post('/github_service_hook', {'payload': json.dumps({ - 'ref': 'refs/heads/branch', - 'repository': {'name': 'repo'}}) - }) - import_from_github.assert_called_with(load_repo_settings('repo')) diff --git a/cms/djangoapps/github_sync/views.py b/cms/djangoapps/github_sync/views.py deleted file mode 100644 index c3b5172b29..0000000000 --- a/cms/djangoapps/github_sync/views.py +++ /dev/null @@ -1,51 +0,0 @@ -import logging -import json - -from django.http import HttpResponse -from django.conf import settings -from django_future.csrf import csrf_exempt - -from . import import_from_github, load_repo_settings - -log = logging.getLogger() - - -@csrf_exempt -def github_post_receive(request): - """ - This view recieves post-receive requests from github whenever one of - the watched repositiories changes. - - It is responsible for updating the relevant local git repo, - importing the new version of the course (if anything changed), - and then pushing back to github any changes that happened as part of the - import. - - The github request format is described here: https://help.github.com/articles/post-receive-hooks - """ - - payload = json.loads(request.POST['payload']) - - ref = payload['ref'] - - if not ref.startswith('refs/heads/'): - log.info('Ignore changes to non-branch ref %s' % ref) - return HttpResponse('Ignoring non-branch') - - branch_name = ref.replace('refs/heads/', '', 1) - - repo_name = payload['repository']['name'] - - if repo_name not in settings.REPOS: - log.info('No repository matching %s found' % repo_name) - return HttpResponse('No Repo Found') - - repo = load_repo_settings(repo_name) - - if repo.branch != branch_name: - log.info('Ignoring changes to non-tracked branch %s in repo %s' % (branch_name, repo_name)) - return HttpResponse('Ignoring non-tracked branch') - - import_from_github(repo) - - return HttpResponse('Push received') diff --git a/cms/envs/common.py b/cms/envs/common.py index 8b5e6c7655..b04c423732 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -284,7 +284,6 @@ INSTALLED_APPS = ( # For CMS 'contentstore', 'auth', - 'github_sync', 'student', # misleading name due to sharing with lms # For asset pipelining diff --git a/cms/urls.py b/cms/urls.py index d9f75d159e..7b3dd90a0b 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -23,7 +23,6 @@ urlpatterns = ('', url(r'^(?P[^/]+)/(?P[^/]+)/import/(?P[^/]+)$', 'contentstore.views.import_course', name='import_course'), - url(r'^github_service_hook$', 'github_sync.views.github_post_receive'), url(r'^preview/modx/(?P[^/]*)/(?P.*?)/(?P[^/]*)$', 'contentstore.views.preview_dispatch', name='preview_dispatch'), url(r'^(?P[^/]+)/(?P[^/]+)/course/(?P[^/]+)/upload_asset$',