Merge pull request #11603 from edx/usman/migrations-setup

Update migrations setup during Python unit tests

cms/envs/test.py

@@ -23,6 +23,7 @@ import os
 from path import Path as path
 from warnings import filterwarnings, simplefilter
 from uuid import uuid4
+from util.db import NoOpMigrationModules
 
 # import settings from LMS for consistent behavior with CMS
 # pylint: disable=unused-import
@@ -42,7 +43,7 @@ MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost')
 THIS_UUID = uuid4().hex[:5]
 
 # Nose Test Runner
-TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
+TEST_RUNNER = 'openedx.core.djangolib.nose.NoseTestSuiteRunner'
 
 _SYSTEM = 'cms'
 
@@ -129,9 +130,10 @@ DATABASES = {
     },
 }
 
-# This hack disables migrations during tests. We want to create tables directly from the models for speed.
-# See https://groups.google.com/d/msg/django-developers/PWPj3etj3-U/kCl6pMsQYYoJ.
-MIGRATION_MODULES = {app: "app.migrations_not_used_in_tests" for app in INSTALLED_APPS}
+if os.environ.get('DISABLE_MIGRATIONS'):
+    # Create tables directly from apps' models. This can be removed once we upgrade
+    # to Django 1.9, which allows setting MIGRATION_MODULES to None in order to skip migrations.
+    MIGRATION_MODULES = NoOpMigrationModules()
 
 LMS_BASE = "localhost:8000"
 FEATURES['PREVIEW_LMS_BASE'] = "preview"

common/djangoapps/util/db.py

@@ -231,3 +231,16 @@ def generate_int_id(minimum=0, maximum=MYSQL_MAX_INT, used_ids=None):
         cid = random.randint(minimum, maximum)
 
     return cid
+
+
+class NoOpMigrationModules(object):
+    """
+    Return invalid migrations modules for apps. Used for disabling migrations during tests.
+    See https://groups.google.com/d/msg/django-developers/PWPj3etj3-U/kCl6pMsQYYoJ.
+    """
+
+    def __contains__(self, item):
+        return True
+
+    def __getitem__(self, item):
+        return "notmigrations"

lms/envs/test.py

@@ -24,6 +24,7 @@ from path import Path as path
 from uuid import uuid4
 from warnings import filterwarnings, simplefilter
 
+from util.db import NoOpMigrationModules
 from openedx.core.lib.tempdir import mkdtemp_clean
 
 # This patch disables the commit_on_success decorator during tests
@@ -86,7 +87,7 @@ PARENTAL_CONSENT_AGE_LIMIT = 13
 SOUTH_TESTS_MIGRATE = False  # To disable migrations and use syncdb instead
 
 # Nose Test Runner
-TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
+TEST_RUNNER = 'openedx.core.djangolib.nose.NoseTestSuiteRunner'
 
 _SYSTEM = 'lms'
 
@@ -188,9 +189,10 @@ DATABASES = {
 
 }
 
-# This hack disables migrations during tests. We want to create tables directly from the models for speed.
-# See https://groups.google.com/d/msg/django-developers/PWPj3etj3-U/kCl6pMsQYYoJ.
-MIGRATION_MODULES = {app: "app.migrations_not_used_in_tests" for app in INSTALLED_APPS}
+if os.environ.get('DISABLE_MIGRATIONS'):
+    # Create tables directly from apps' models. This can be removed once we upgrade
+    # to Django 1.9, which allows setting MIGRATION_MODULES to None in order to skip migrations.
+    MIGRATION_MODULES = NoOpMigrationModules()
 
 CACHES = {
     # This is the cache used for most things.

openedx/core/djangolib/nose.py

@@ -0,0 +1,30 @@
+"""
+Utilities related to nose.
+"""
+from django.core.management import call_command
+from django.db import DEFAULT_DB_ALIAS, connections, transaction
+import django_nose
+
+
+class NoseTestSuiteRunner(django_nose.NoseTestSuiteRunner):
+    """Custom NoseTestSuiteRunner."""
+
+    def setup_databases(self):
+        """ Setup databases and then flush to remove data added by migrations. """
+        return_value = super(NoseTestSuiteRunner, self).setup_databases()
+
+        # Delete all data added by data migrations. Unit tests should setup their own data using factories.
+        call_command('flush', verbosity=0, interactive=False, load_initial_data=False)
+
+        # Through Django 1.8, auto increment sequences are not reset when calling flush on a SQLite db.
+        # So we do it ourselves.
+        # http://sqlite.org/autoinc.html
+        connection = connections[DEFAULT_DB_ALIAS]
+        if connection.vendor == 'sqlite' and not connection.features.supports_sequence_reset:
+            with transaction.atomic(using=DEFAULT_DB_ALIAS):
+                cursor = connection.cursor()
+                cursor.execute(
+                    "delete from sqlite_sequence;"
+                )
+
+        return return_value

pavelib/tests.py

@@ -34,6 +34,12 @@ __test__ = False  # do not collect
     make_option("-q", "--quiet", action="store_const", const=0, dest="verbosity"),
     make_option("-v", "--verbosity", action="count", dest="verbosity", default=1),
     make_option("--pdb", action="store_true", help="Drop into debugger on failures or errors"),
+    make_option(
+        '--disable-migrations',
+        action='store_true',
+        dest='disable_migrations',
+        help="Create tables directly from apps' models. Can also be used by exporting DISABLE_MIGRATIONS=1."
+    ),
 ], share_with=['pavelib.utils.test.utils.clean_reports_dir'])
 def test_system(options):
     """
@@ -51,6 +57,7 @@ def test_system(options):
         'cov_args': getattr(options, 'cov_args', ''),
         'skip_clean': getattr(options, 'skip_clean', False),
         'pdb': getattr(options, 'pdb', False),
+        'disable_migrations': getattr(options, 'disable_migrations', False),
     }
 
     if test_id:
@@ -134,6 +141,12 @@ def test_lib(options):
     make_option("-q", "--quiet", action="store_const", const=0, dest="verbosity"),
     make_option("-v", "--verbosity", action="count", dest="verbosity", default=1),
     make_option("--pdb", action="store_true", help="Drop into debugger on failures or errors"),
+    make_option(
+        '--disable-migrations',
+        action='store_true',
+        dest='disable_migrations',
+        help="Create tables directly from apps' models. Can also be used by exporting DISABLE_MIGRATIONS=1."
+    ),
 ])
 def test_python(options):
     """
@@ -146,6 +159,7 @@ def test_python(options):
         'extra_args': getattr(options, 'extra_args', ''),
         'cov_args': getattr(options, 'cov_args', ''),
         'pdb': getattr(options, 'pdb', False),
+        'disable_migrations': getattr(options, 'disable_migrations', False),
     }
 
     python_suite = suites.PythonTestSuite('Python Tests', **opts)

pavelib/utils/test/suites/python_suite.py

@@ -1,6 +1,8 @@
 """
 Classes used for defining and running python test suites
 """
+import os
+
 from pavelib.utils.test import utils as test_utils
 from pavelib.utils.test.suites.suite import TestSuite
 from pavelib.utils.test.suites.nose_suite import LibTestSuite, SystemTestSuite
@@ -16,11 +18,16 @@ class PythonTestSuite(TestSuite):
     def __init__(self, *args, **kwargs):
         super(PythonTestSuite, self).__init__(*args, **kwargs)
         self.opts = kwargs
+        self.disable_migrations = kwargs.get('disable_migrations', False)
         self.fasttest = kwargs.get('fasttest', False)
         self.subsuites = kwargs.get('subsuites', self._default_subsuites)
 
     def __enter__(self):
         super(PythonTestSuite, self).__enter__()
+
+        if self.disable_migrations:
+            os.environ['DISABLE_MIGRATIONS'] = '1'
+
         if not (self.fasttest or self.skip_clean):
             test_utils.clean_test_files()