Merge remote-tracking branch 'origin/release' into dj18-release-merge

Conflicts:
	common/djangoapps/util/testing.py
	lms/djangoapps/instructor/views/api.py
	lms/djangoapps/teams/tests/test_views.py
	openedx/core/djangoapps/programs/models.py
	openedx/core/djangoapps/user_api/accounts/tests/test_views.py
	requirements/edx/github.txt

pavelib/assets.py

@@ -110,8 +110,8 @@ class SassWatcher(PatternMatchingEventHandler):
     def on_modified(self, event):
         print('\tCHANGED:', event.src_path)
         try:
-            compile_sass()
-        except Exception:  # pylint: disable=broad-except
+            compile_sass()      # pylint: disable=no-value-for-parameter
+        except Exception:       # pylint: disable=broad-except
             traceback.print_exc()
 
 

pavelib/docs.py

@@ -4,7 +4,8 @@ Ties into Sphinx to generate files at the specified location(s)
 """
 from __future__ import print_function
 import sys
-from paver.easy import *
+
+from paver.easy import cmdopts, needs, sh, task
 
 
 DOC_PATHS = {

pavelib/js_test.py

@@ -61,7 +61,7 @@ def test_js_run(options):
     """
     Run the JavaScript tests and print results to the console
     """
-    setattr(options, 'mode', 'run')
+    options.mode = 'run'
     test_js(options)
 
 
@@ -74,5 +74,5 @@ def test_js_dev(options):
     """
     Run the JavaScript tests in your default browsers
     """
-    setattr(options, 'mode', 'dev')
+    options.mode = 'dev'
     test_js(options)

pavelib/paver_tests/test_servers.py

@@ -158,7 +158,7 @@ class TestPaverServerTasks(PaverTestCase):
         """
         settings = options.get("settings", "devstack")
         call_task("pavelib.servers.update_db", options=options)
-        db_command = "python manage.py {server} --settings={settings} syncdb --migrate --traceback --pythonpath=."
+        db_command = "python manage.py {server} --settings={settings} migrate --traceback --pythonpath=."
         self.assertEquals(
             self.task_messages,
             [

pavelib/prereqs.py

@@ -2,14 +2,17 @@
 Install Python, Ruby, and Node prerequisites.
 """
 
-import os
-import hashlib
 from distutils import sysconfig
-from paver.easy import *
+import hashlib
+import os
+
+from paver.easy import sh, task
+
 from .utils.envs import Env
+import sys
 
 
-PREREQS_MD5_DIR = os.getenv('PREREQ_CACHE_DIR', Env.REPO_ROOT / '.prereqs_cache')
+PREREQS_STATE_DIR = os.getenv('PREREQ_CACHE_DIR', Env.REPO_ROOT / '.prereqs_cache')
 NPM_REGISTRY = "http://registry.npmjs.org/"
 NO_PREREQ_MESSAGE = "NO_PREREQ_INSTALL is set, not installing prereqs"
 
@@ -86,7 +89,7 @@ def prereq_cache(cache_name, paths, install_func):
     """
     # Retrieve the old hash
     cache_filename = cache_name.replace(" ", "_")
-    cache_file_path = os.path.join(PREREQS_MD5_DIR, "{}.sha1".format(cache_filename))
+    cache_file_path = os.path.join(PREREQS_STATE_DIR, "{}.sha1".format(cache_filename))
     old_hash = None
     if os.path.isfile(cache_file_path):
         with open(cache_file_path) as cache_file:
@@ -103,9 +106,9 @@ def prereq_cache(cache_name, paths, install_func):
         # If the code executed within the context fails (throws an exception),
         # then this step won't get executed.
         try:
-            os.makedirs(PREREQS_MD5_DIR)
+            os.makedirs(PREREQS_STATE_DIR)
         except OSError:
-            if not os.path.isdir(PREREQS_MD5_DIR):
+            if not os.path.isdir(PREREQS_STATE_DIR):
                 raise
 
         with open(cache_file_path, "w") as cache_file:
@@ -166,16 +169,90 @@ def install_node_prereqs():
     prereq_cache("Node prereqs", ["package.json"], node_prereqs_installation)
 
 
+@task
+def uninstall_python_packages():
+    """
+    Uninstall Python packages that need explicit uninstallation.
+
+    Some Python packages that we no longer want need to be explicitly
+    uninstalled, notably, South.  Some other packages were once installed in
+    ways that were resistant to being upgraded, like edxval.  Also uninstall
+    them.
+
+    """
+    # So that we don't constantly uninstall things, use a version number of the
+    # uninstallation needs.  Check it, and skip this if we're up to date.
+    expected_version = 2
+    state_file_path = os.path.join(PREREQS_STATE_DIR, "python_uninstall_version.txt")
+    if os.path.isfile(state_file_path):
+        with open(state_file_path) as state_file:
+            version = int(state_file.read())
+        if version == expected_version:
+            return
+
+    # Run pip to find the packages we need to get rid of.  Believe it or not,
+    # edx-val is installed in a way that it is present twice, so we have a loop
+    # to really really get rid of it.
+    for _ in range(3):
+        uninstalled = False
+        frozen = sh("pip freeze", capture=True).splitlines()
+
+        # Uninstall South
+        if any(line.startswith("South") for line in frozen):
+            sh("pip uninstall -y South")
+            uninstalled = True
+
+        # Uninstall edx-val
+        if any("edxval" in line for line in frozen):
+            sh("pip uninstall -y edxval")
+            uninstalled = True
+
+        # Uninstall django-storages
+        if any("django-storages==" in line for line in frozen):
+            sh("pip uninstall -y django-storages")
+            uninstalled = True
+
+        if not uninstalled:
+            break
+    else:
+        # We tried three times and didn't manage to get rid of the pests.
+        print "Couldn't uninstall unwanted Python packages!"
+        return
+
+    # Write our version.
+    with open(state_file_path, "w") as state_file:
+        state_file.write(str(expected_version))
+
+
 @task
 def install_python_prereqs():
     """
-    Installs Python prerequisites
+    Installs Python prerequisites.
     """
     if no_prereq_install():
         print NO_PREREQ_MESSAGE
         return
 
-    prereq_cache("Python prereqs", PYTHON_REQ_FILES + [sysconfig.get_python_lib()], python_prereqs_installation)
+    # Include all of the requirements files in the fingerprint.
+    files_to_fingerprint = list(PYTHON_REQ_FILES)
+
+    # Also fingerprint the directories where packages get installed:
+    # ("/edx/app/edxapp/venvs/edxapp/lib/python2.7/site-packages")
+    files_to_fingerprint.append(sysconfig.get_python_lib())
+
+    # In a virtualenv, "-e installs" get put in a src directory.
+    src_dir = os.path.join(sys.prefix, "src")
+    if os.path.isdir(src_dir):
+        files_to_fingerprint.append(src_dir)
+
+    # Also fingerprint this source file, so that if the logic for installations
+    # changes, we will redo the installation.
+    this_file = __file__
+    if this_file.endswith(".pyc"):
+        this_file = this_file[:-1]      # use the .py file instead of the .pyc
+    files_to_fingerprint.append(this_file)
+
+    prereq_cache("Python prereqs", files_to_fingerprint, python_prereqs_installation)
 
 
 @task
@@ -189,4 +266,5 @@ def install_prereqs():
 
     install_ruby_prereqs()
     install_node_prereqs()
+    uninstall_python_packages()
     install_python_prereqs()

pavelib/servers.py

@@ -3,7 +3,9 @@ Run and manage servers for local development.
 """
 from __future__ import print_function
 import argparse
-from paver.easy import *
+import sys
+
+from paver.easy import call_task, cmdopts, consume_args, needs, sh, task
 
 from .assets import collect_assets
 from .utils.cmd import django_cmd
@@ -233,14 +235,16 @@ def run_all_servers(options):
 @needs('pavelib.prereqs.install_prereqs')
 @cmdopts([
     ("settings=", "s", "Django settings"),
+    ("fake-initial", None, "Fake the initial migrations"),
 ])
-def update_db():
+def update_db(options):
     """
     Runs syncdb and then migrate.
     """
     settings = getattr(options, 'settings', DEFAULT_SETTINGS)
+    fake = "--fake-initial" if getattr(options, 'fake_initial', False) else ""
     for system in ('lms', 'cms'):
-        sh(django_cmd(system, settings, 'syncdb', '--migrate', '--traceback', '--pythonpath=.'))
+        sh(django_cmd(system, settings, 'migrate', fake, '--traceback', '--pythonpath=.'))
 
 
 @task

pavelib/utils/cmd.py

@@ -7,7 +7,7 @@ def cmd(*args):
     """
     Concatenate the arguments into a space-separated shell command.
     """
-    return " ".join([str(arg) for arg in args])
+    return " ".join(str(arg) for arg in args if arg)
 
 
 def django_cmd(sys, settings, *args):

pavelib/utils/test/suites/acceptance_suite.py

@@ -130,8 +130,6 @@ class AcceptanceTestSuite(TestSuite):
             sh("./manage.py cms --settings acceptance migrate --traceback --noinput")
         else:
             # If no cached database exists, syncdb before migrating, then create the cache
-            sh("./manage.py lms --settings acceptance syncdb --traceback --noinput")
-            sh("./manage.py cms --settings acceptance syncdb --traceback --noinput")
             sh("./manage.py lms --settings acceptance migrate --traceback --noinput")
             sh("./manage.py cms --settings acceptance migrate --traceback --noinput")
 

pavelib/utils/test/suites/bokchoy_suite.py

@@ -66,7 +66,7 @@ class BokChoyTestSuite(TestSuite):
         self.log_dir.makedirs_p()
         self.har_dir.makedirs_p()
         self.report_dir.makedirs_p()
-        test_utils.clean_reports_dir()
+        test_utils.clean_reports_dir()      # pylint: disable=no-value-for-parameter
 
         if not (self.fasttest or self.skip_clean):
             test_utils.clean_test_files()

pavelib/utils/test/suites/nose_suite.py

@@ -120,7 +120,7 @@ class SystemTestSuite(NoseTestSuite):
     def cmd(self):
         cmd = (
             './manage.py {system} test --verbosity={verbosity} '
-            '{test_id} {test_opts} --traceback --settings=test {extra} '
+            '{test_id} {test_opts} --settings=test {extra} '
             '--with-xunit --xunit-file={xunit_report}'.format(
                 system=self.root,
                 verbosity=self.verbosity,