Merge pull request #17053 from edx/estute/bokchoy-option-to-load-data-without-migrations

add bokchoy script option for running migrations

.gitignore

@@ -83,6 +83,7 @@ test_root/paver_logs/
 test_root/uploads/
 django-pyfs
 .tox/
+common/test/db_cache/bok_choy_*.yaml
 
 ### Installation artifacts
 *.egg-info

pavelib/database.py

@@ -7,7 +7,7 @@ import os
 from paver.easy import needs
 
 from pavelib.utils.db_utils import (
-    remove_files_from_folder, apply_migrations, compute_fingerprint_and_write_to_disk,
+    remove_files_from_folder, reset_test_db, compute_fingerprint_and_write_to_disk,
     fingerprint_bokchoy_db_files, does_fingerprint_on_disk_match, is_fingerprint_in_bucket,
     get_file_from_s3, extract_files_from_zip, create_tarfile_from_db_cache, upload_to_s3
 )
@@ -49,7 +49,7 @@ def update_bokchoy_db_cache():
     """
     print('Removing cached db files for bokchoy tests')
     remove_files_from_folder(BOKCHOY_DB_FILES, CACHE_FOLDER)
-    apply_migrations(BOKCHOY_DB_FILES, update_cache_files=True)
+    reset_test_db(BOKCHOY_DB_FILES, update_cache_files=True)
     compute_fingerprint_and_write_to_disk(MIGRATION_OUTPUT_FILES, ALL_DB_FILES)
 
 
@@ -71,14 +71,12 @@ def update_local_bokchoy_db_from_s3():
         print ("DB cache files match the current migrations.")
         # TODO: we don't really need to apply migrations, just to
         # load the db cache files into the database.
-        apply_migrations(BOKCHOY_DB_FILES, update_cache_files=False)
+        reset_test_db(BOKCHOY_DB_FILES, update_cache_files=False)
 
     elif is_fingerprint_in_bucket(fingerprint, CACHE_BUCKET_NAME):
         print ("Found updated bokchoy db files at S3.")
         refresh_bokchoy_db_cache_from_s3(fingerprint=fingerprint)
-        # TODO: we don't really need to apply migrations, just to
-        # load the db cache files into the database.
-        apply_migrations(BOKCHOY_DB_FILES, update_cache_files=False)
+        reset_test_db(BOKCHOY_DB_FILES, update_cache_files=False)
         # Write the new fingerprint to disk so that it reflects the
         # current state of the system.
         compute_fingerprint_and_write_to_disk(MIGRATION_OUTPUT_FILES, ALL_DB_FILES)
@@ -90,7 +88,7 @@ def update_local_bokchoy_db_from_s3():
             "and running migrations."
         )
         print (msg)
-        apply_migrations(BOKCHOY_DB_FILES, update_cache_files=True)
+        reset_test_db(BOKCHOY_DB_FILES, update_cache_files=True)
         # Write the new fingerprint to disk so that it reflects the
         # current state of the system.
         # E.g. you could have added a new migration in your PR.

pavelib/utils/db_utils.py

@@ -12,7 +12,7 @@ from pavelib.prereqs import compute_fingerprint
 from pavelib.utils.envs import Env
 
 CACHE_FOLDER = 'common/test/db_cache'
-FINGERPRINT_FILEPATH = '{}/{}/bokchoy_migrations.sha1'.format(Env.REPO_ROOT, CACHE_FOLDER)
+FINGERPRINT_FILEPATH = '{}/{}/bok_choy_migrations.sha1'.format(Env.REPO_ROOT, CACHE_FOLDER)
 
 
 def remove_files_from_folder(files, folder):
@@ -30,15 +30,14 @@ def remove_files_from_folder(files, folder):
             continue
 
 
-def apply_migrations(db_cache_files, update_cache_files=True):
+def reset_test_db(db_cache_files, update_cache_files=True):
     """
-    Apply migrations to the test database.
+    Reset the bokchoy test db for a new test run
 
     The called script will flush your db (or create it if it doesn't yet
     exist), load in the db cache files files if they exist on disk,
-    apply migrations, and then optionally write up-to-date cache files.
+    and optionally apply migrations and write up-to-date cache files.
     """
-    print ("Applying migrations.")
     cmd = '{}/scripts/reset-test-db.sh'.format(Env.REPO_ROOT)
     if update_cache_files:
         cmd = '{} --rebuild_cache'.format(cmd)

pavelib/utils/test/suites/bokchoy_suite.py

@@ -136,7 +136,7 @@ def reset_test_database():
     """
     Reset the database used by the bokchoy tests.
     """
-    sh("{}/scripts/reset-test-db.sh".format(Env.REPO_ROOT))
+    sh("{}/scripts/reset-test-db.sh --migrations".format(Env.REPO_ROOT))
 
 
 @task

scripts/reset-test-db.sh

@@ -40,6 +40,9 @@ for i in "$@"; do
         -r|--rebuild_cache)
             REBUILD_CACHE=true
             ;;
+        -m|--migrations)
+            APPLY_MIGRATIONS=true
+            ;;
     esac
 done
 
@@ -108,7 +111,7 @@ if [[ $REBUILD_CACHE ]]; then
     for db in "${database_order[@]}"; do
         rebuild_cache_for_db
     done
-else
+elif [[ $APPLY_MIGRATIONS ]]; then
     for db in "${database_order[@]}"; do
         run_migrations
     done