diff --git a/pavelib/database.py b/pavelib/database.py index 3fbd62abd3..06fe07fcaf 100644 --- a/pavelib/database.py +++ b/pavelib/database.py @@ -22,7 +22,7 @@ BOKCHOY_DB_FILES = [ 'bok_choy_schema_student_module_history.sql' ] -# Output files from scripts/calculate-bokchoy-migrations.sh +# Output files from scripts/reset-test-db.sh --calculate_migrations MIGRATION_OUTPUT_FILES = [ 'bok_choy_default_migrations.yaml', 'bok_choy_student_module_history_migrations.yaml' diff --git a/pavelib/paver_tests/test_database.py b/pavelib/paver_tests/test_database.py index 3ec5af6edf..5197f49720 100644 --- a/pavelib/paver_tests/test_database.py +++ b/pavelib/paver_tests/test_database.py @@ -114,7 +114,7 @@ class TestPaverDatabaseTasks(MockS3Mixin, TestCase): # Make sure that the local cache files are used - NOT downloaded from s3 self.assertFalse(_mock_get_file.called) calls = [ - call('{}/scripts/calculate-bokchoy-migrations.sh'.format(Env.REPO_ROOT)), + call('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)), call('{}/scripts/reset-test-db.sh'.format(Env.REPO_ROOT)) ] _mock_sh.assert_has_calls(calls) @@ -155,7 +155,7 @@ class TestPaverDatabaseTasks(MockS3Mixin, TestCase): 'moto_test_bucket', self.fingerprint_filename, db_utils.CACHE_FOLDER ) calls = [ - call('{}/scripts/calculate-bokchoy-migrations.sh'.format(Env.REPO_ROOT)), + call('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)), call('{}/scripts/reset-test-db.sh'.format(Env.REPO_ROOT)) ] _mock_sh.assert_has_calls(calls) @@ -183,7 +183,7 @@ class TestPaverDatabaseTasks(MockS3Mixin, TestCase): database.update_local_bokchoy_db_from_s3() calls = [ - call('{}/scripts/calculate-bokchoy-migrations.sh'.format(Env.REPO_ROOT)), + call('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)), call('{}/scripts/reset-test-db.sh --rebuild_cache'.format(Env.REPO_ROOT)) ] _mock_sh.assert_has_calls(calls) diff --git a/pavelib/utils/db_utils.py b/pavelib/utils/db_utils.py index e0fecede98..f3710f47b9 100644 --- a/pavelib/utils/db_utils.py +++ b/pavelib/utils/db_utils.py @@ -110,7 +110,7 @@ def calculate_bokchoy_migrations(migration_output_files): NOTE: the script first clears out the database, then calculates what migrations need to be run, which is all of them. """ - sh('{}/scripts/calculate-bokchoy-migrations.sh'.format(Env.REPO_ROOT)) + sh('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)) verify_files_exist(migration_output_files) diff --git a/pavelib/utils/test/suites/bokchoy_suite.py b/pavelib/utils/test/suites/bokchoy_suite.py index c4e3c96e78..8e13b78490 100644 --- a/pavelib/utils/test/suites/bokchoy_suite.py +++ b/pavelib/utils/test/suites/bokchoy_suite.py @@ -21,6 +21,7 @@ from pavelib.utils.test.bokchoy_options import ( ) from pavelib.utils.test import utils as test_utils from pavelib.utils.timer import timed +from pavelib.database import update_local_bokchoy_db_from_s3 import os @@ -135,8 +136,15 @@ def get_test_course(options): def reset_test_database(): """ Reset the database used by the bokchoy tests. + + If the tests are being run on Jenkins, use the database cache automation + defined in pavelib/database.py + If not, reset the test database and apply migrations """ - sh("{}/scripts/reset-test-db.sh --migrations".format(Env.REPO_ROOT)) + if os.environ.get('USER', None) == 'jenkins': + update_local_bokchoy_db_from_s3() + else: + sh("{}/scripts/reset-test-db.sh --migrations".format(Env.REPO_ROOT)) @task diff --git a/scripts/calculate-bokchoy-migrations.sh b/scripts/calculate-bokchoy-migrations.sh deleted file mode 100755 index c0fb58fae3..0000000000 --- a/scripts/calculate-bokchoy-migrations.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -############################################################################ -# -# Output all migrations that would be applied to an -# empty database for the bok-choy acceptance tests. -# -############################################################################ - -# Fail fast -set -e - -if [[ -z "$BOK_CHOY_HOSTNAME" ]]; then - MYSQL_HOST="" - SETTINGS="bok_choy" -else - MYSQL_HOST="--host=edx.devstack.mysql" - SETTINGS="bok_choy_docker" -fi - -declare -a database_order -database_order=("default" "student_module_history") - -for db in "${database_order[@]}"; do - # Use a different database than the one used for testing, - # because we will need to empty out the database to calculate - # the migrations fingerprint. - # Choosing an arbitrary name "calculate_migrations" for the db. - echo "DROP DATABASE IF EXISTS calculate_migrations;" | mysql $MYSQL_HOST -u root - echo "CREATE DATABASE calculate_migrations;" | mysql $MYSQL_HOST -u root - - # Now output all the migrations in the platform to a file. - echo "Calculating migrations for fingerprinting." - - output_file="common/test/db_cache/bok_choy_${db}_migrations.yaml" - # Redirect stdout to /dev/null because the script will print - # out all migrations to both stdout and the output file. - ./manage.py lms --settings $SETTINGS show_unapplied_migrations --database $db --output_file $output_file 1>/dev/null -done - -echo "DROP DATABASE IF EXISTS calculate_migrations;" | mysql $MYSQL_HOST -u root diff --git a/scripts/reset-test-db.sh b/scripts/reset-test-db.sh index 38fd53d889..6e23b20ad8 100755 --- a/scripts/reset-test-db.sh +++ b/scripts/reset-test-db.sh @@ -43,6 +43,9 @@ for i in "$@"; do -m|--migrations) APPLY_MIGRATIONS=true ;; + -c|--calculate_migrations) + CALCULATE_MIGRATIONS=true + ;; esac done @@ -51,6 +54,14 @@ declare -a database_order databases=(["default"]="edxtest" ["student_module_history"]="student_module_history_test") database_order=("default" "student_module_history") +calculate_migrations() { + echo "Calculating migrations for fingerprinting." + output_file="common/test/db_cache/bok_choy_${db}_migrations.yaml" + # Redirect stdout to /dev/null because the script will print + # out all migrations to both stdout and the output file. + ./manage.py lms --settings $SETTINGS show_unapplied_migrations --database $db --output_file $output_file 1>/dev/null +} + run_migrations() { echo "Running the lms migrations on the $db bok_choy DB." ./manage.py lms --settings $SETTINGS migrate --database $db --traceback --noinput @@ -115,4 +126,8 @@ elif [[ $APPLY_MIGRATIONS ]]; then for db in "${database_order[@]}"; do run_migrations done +elif [[ $CALCULATE_MIGRATIONS ]]; then + for db in "${database_order[@]}"; do + calculate_migrations + done fi