From 2586f09d7ac1b8034815f125d0a4a9b663206cea Mon Sep 17 00:00:00 2001 From: Usman Khalid <2200617@gmail.com> Date: Wed, 10 Feb 2016 21:24:48 +0500 Subject: [PATCH] Add option to disable migrations when running tests By default, migrations are applied as they always have been. Exporting DISABLE_MIGRATIONS=1 or passing --disable-migrations to Paver commands will create tables directly from apps' models. --- cms/envs/test.py | 10 +++++--- common/djangoapps/util/db.py | 13 ++++++++++ lms/envs/test.py | 10 +++++--- openedx/core/djangolib/nose.py | 30 +++++++++++++++++++++++ pavelib/tests.py | 14 +++++++++++ pavelib/utils/test/suites/python_suite.py | 7 ++++++ 6 files changed, 76 insertions(+), 8 deletions(-) create mode 100644 openedx/core/djangolib/nose.py diff --git a/cms/envs/test.py b/cms/envs/test.py index 8577e82a3b..ce63bc4d90 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -23,6 +23,7 @@ import os from path import Path as path from warnings import filterwarnings, simplefilter from uuid import uuid4 +from util.db import NoOpMigrationModules # import settings from LMS for consistent behavior with CMS # pylint: disable=unused-import @@ -42,7 +43,7 @@ MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost') THIS_UUID = uuid4().hex[:5] # Nose Test Runner -TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' +TEST_RUNNER = 'openedx.core.djangolib.nose.NoseTestSuiteRunner' _SYSTEM = 'cms' @@ -129,9 +130,10 @@ DATABASES = { }, } -# This hack disables migrations during tests. We want to create tables directly from the models for speed. -# See https://groups.google.com/d/msg/django-developers/PWPj3etj3-U/kCl6pMsQYYoJ. -MIGRATION_MODULES = {app: "app.migrations_not_used_in_tests" for app in INSTALLED_APPS} +if os.environ.get('DISABLE_MIGRATIONS'): + # Create tables directly from apps' models. This can be removed once we upgrade + # to Django 1.9, which allows setting MIGRATION_MODULES to None in order to skip migrations. + MIGRATION_MODULES = NoOpMigrationModules() LMS_BASE = "localhost:8000" FEATURES['PREVIEW_LMS_BASE'] = "preview" diff --git a/common/djangoapps/util/db.py b/common/djangoapps/util/db.py index ba8df7ae27..c246769063 100644 --- a/common/djangoapps/util/db.py +++ b/common/djangoapps/util/db.py @@ -231,3 +231,16 @@ def generate_int_id(minimum=0, maximum=MYSQL_MAX_INT, used_ids=None): cid = random.randint(minimum, maximum) return cid + + +class NoOpMigrationModules(object): + """ + Return invalid migrations modules for apps. Used for disabling migrations during tests. + See https://groups.google.com/d/msg/django-developers/PWPj3etj3-U/kCl6pMsQYYoJ. + """ + + def __contains__(self, item): + return True + + def __getitem__(self, item): + return "notmigrations" diff --git a/lms/envs/test.py b/lms/envs/test.py index eff52628c1..c16ac9d37e 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -24,6 +24,7 @@ from path import Path as path from uuid import uuid4 from warnings import filterwarnings, simplefilter +from util.db import NoOpMigrationModules from openedx.core.lib.tempdir import mkdtemp_clean # This patch disables the commit_on_success decorator during tests @@ -86,7 +87,7 @@ PARENTAL_CONSENT_AGE_LIMIT = 13 SOUTH_TESTS_MIGRATE = False # To disable migrations and use syncdb instead # Nose Test Runner -TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' +TEST_RUNNER = 'openedx.core.djangolib.nose.NoseTestSuiteRunner' _SYSTEM = 'lms' @@ -188,9 +189,10 @@ DATABASES = { } -# This hack disables migrations during tests. We want to create tables directly from the models for speed. -# See https://groups.google.com/d/msg/django-developers/PWPj3etj3-U/kCl6pMsQYYoJ. -MIGRATION_MODULES = {app: "app.migrations_not_used_in_tests" for app in INSTALLED_APPS} +if os.environ.get('DISABLE_MIGRATIONS'): + # Create tables directly from apps' models. This can be removed once we upgrade + # to Django 1.9, which allows setting MIGRATION_MODULES to None in order to skip migrations. + MIGRATION_MODULES = NoOpMigrationModules() CACHES = { # This is the cache used for most things. diff --git a/openedx/core/djangolib/nose.py b/openedx/core/djangolib/nose.py new file mode 100644 index 0000000000..fb79993986 --- /dev/null +++ b/openedx/core/djangolib/nose.py @@ -0,0 +1,30 @@ +""" +Utilities related to nose. +""" +from django.core.management import call_command +from django.db import DEFAULT_DB_ALIAS, connections, transaction +import django_nose + + +class NoseTestSuiteRunner(django_nose.NoseTestSuiteRunner): + """Custom NoseTestSuiteRunner.""" + + def setup_databases(self): + """ Setup databases and then flush to remove data added by migrations. """ + return_value = super(NoseTestSuiteRunner, self).setup_databases() + + # Delete all data added by data migrations. Unit tests should setup their own data using factories. + call_command('flush', verbosity=0, interactive=False, load_initial_data=False) + + # Through Django 1.8, auto increment sequences are not reset when calling flush on a SQLite db. + # So we do it ourselves. + # http://sqlite.org/autoinc.html + connection = connections[DEFAULT_DB_ALIAS] + if connection.vendor == 'sqlite' and not connection.features.supports_sequence_reset: + with transaction.atomic(using=DEFAULT_DB_ALIAS): + cursor = connection.cursor() + cursor.execute( + "delete from sqlite_sequence;" + ) + + return return_value diff --git a/pavelib/tests.py b/pavelib/tests.py index fc1c800141..9bce602df4 100644 --- a/pavelib/tests.py +++ b/pavelib/tests.py @@ -34,6 +34,12 @@ __test__ = False # do not collect make_option("-q", "--quiet", action="store_const", const=0, dest="verbosity"), make_option("-v", "--verbosity", action="count", dest="verbosity", default=1), make_option("--pdb", action="store_true", help="Drop into debugger on failures or errors"), + make_option( + '--disable-migrations', + action='store_true', + dest='disable_migrations', + help="Create tables directly from apps' models. Can also be used by exporting DISABLE_MIGRATIONS=1." + ), ], share_with=['pavelib.utils.test.utils.clean_reports_dir']) def test_system(options): """ @@ -51,6 +57,7 @@ def test_system(options): 'cov_args': getattr(options, 'cov_args', ''), 'skip_clean': getattr(options, 'skip_clean', False), 'pdb': getattr(options, 'pdb', False), + 'disable_migrations': getattr(options, 'disable_migrations', False), } if test_id: @@ -134,6 +141,12 @@ def test_lib(options): make_option("-q", "--quiet", action="store_const", const=0, dest="verbosity"), make_option("-v", "--verbosity", action="count", dest="verbosity", default=1), make_option("--pdb", action="store_true", help="Drop into debugger on failures or errors"), + make_option( + '--disable-migrations', + action='store_true', + dest='disable_migrations', + help="Create tables directly from apps' models. Can also be used by exporting DISABLE_MIGRATIONS=1." + ), ]) def test_python(options): """ @@ -146,6 +159,7 @@ def test_python(options): 'extra_args': getattr(options, 'extra_args', ''), 'cov_args': getattr(options, 'cov_args', ''), 'pdb': getattr(options, 'pdb', False), + 'disable_migrations': getattr(options, 'disable_migrations', False), } python_suite = suites.PythonTestSuite('Python Tests', **opts) diff --git a/pavelib/utils/test/suites/python_suite.py b/pavelib/utils/test/suites/python_suite.py index 85337d6e89..f205c6eae4 100644 --- a/pavelib/utils/test/suites/python_suite.py +++ b/pavelib/utils/test/suites/python_suite.py @@ -1,6 +1,8 @@ """ Classes used for defining and running python test suites """ +import os + from pavelib.utils.test import utils as test_utils from pavelib.utils.test.suites.suite import TestSuite from pavelib.utils.test.suites.nose_suite import LibTestSuite, SystemTestSuite @@ -16,11 +18,16 @@ class PythonTestSuite(TestSuite): def __init__(self, *args, **kwargs): super(PythonTestSuite, self).__init__(*args, **kwargs) self.opts = kwargs + self.disable_migrations = kwargs.get('disable_migrations', False) self.fasttest = kwargs.get('fasttest', False) self.subsuites = kwargs.get('subsuites', self._default_subsuites) def __enter__(self): super(PythonTestSuite, self).__enter__() + + if self.disable_migrations: + os.environ['DISABLE_MIGRATIONS'] = '1' + if not (self.fasttest or self.skip_clean): test_utils.clean_test_files()