Merge pull request #17107 from edx/estute/enable-bokchoy-db-caching-on-jenkins

use db cache in jenkins bok choy builds

pavelib/database.py

@@ -22,7 +22,7 @@ BOKCHOY_DB_FILES = [
     'bok_choy_schema_student_module_history.sql'
 ]
 
-# Output files from scripts/calculate-bokchoy-migrations.sh
+# Output files from scripts/reset-test-db.sh --calculate_migrations
 MIGRATION_OUTPUT_FILES = [
     'bok_choy_default_migrations.yaml',
     'bok_choy_student_module_history_migrations.yaml'

pavelib/paver_tests/test_database.py

@@ -114,7 +114,7 @@ class TestPaverDatabaseTasks(MockS3Mixin, TestCase):
             # Make sure that the local cache files are used - NOT downloaded from s3
             self.assertFalse(_mock_get_file.called)
         calls = [
-            call('{}/scripts/calculate-bokchoy-migrations.sh'.format(Env.REPO_ROOT)),
+            call('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)),
             call('{}/scripts/reset-test-db.sh'.format(Env.REPO_ROOT))
         ]
         _mock_sh.assert_has_calls(calls)
@@ -155,7 +155,7 @@ class TestPaverDatabaseTasks(MockS3Mixin, TestCase):
                 'moto_test_bucket', self.fingerprint_filename, db_utils.CACHE_FOLDER
             )
         calls = [
-            call('{}/scripts/calculate-bokchoy-migrations.sh'.format(Env.REPO_ROOT)),
+            call('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)),
             call('{}/scripts/reset-test-db.sh'.format(Env.REPO_ROOT))
         ]
         _mock_sh.assert_has_calls(calls)
@@ -183,7 +183,7 @@ class TestPaverDatabaseTasks(MockS3Mixin, TestCase):
 
         database.update_local_bokchoy_db_from_s3()
         calls = [
-            call('{}/scripts/calculate-bokchoy-migrations.sh'.format(Env.REPO_ROOT)),
+            call('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)),
             call('{}/scripts/reset-test-db.sh --rebuild_cache'.format(Env.REPO_ROOT))
         ]
         _mock_sh.assert_has_calls(calls)

pavelib/utils/db_utils.py

@@ -110,7 +110,7 @@ def calculate_bokchoy_migrations(migration_output_files):
     NOTE: the script first clears out the database, then calculates
           what migrations need to be run, which is all of them.
     """
-    sh('{}/scripts/calculate-bokchoy-migrations.sh'.format(Env.REPO_ROOT))
+    sh('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT))
     verify_files_exist(migration_output_files)
 
 

pavelib/utils/test/suites/bokchoy_suite.py

@@ -21,6 +21,7 @@ from pavelib.utils.test.bokchoy_options import (
 )
 from pavelib.utils.test import utils as test_utils
 from pavelib.utils.timer import timed
+from pavelib.database import update_local_bokchoy_db_from_s3
 
 import os
 
@@ -135,8 +136,15 @@ def get_test_course(options):
 def reset_test_database():
     """
     Reset the database used by the bokchoy tests.
+
+    If the tests are being run on Jenkins, use the database cache automation
+    defined in pavelib/database.py
+    If not, reset the test database and apply migrations
     """
-    sh("{}/scripts/reset-test-db.sh --migrations".format(Env.REPO_ROOT))
+    if os.environ.get('USER', None) == 'jenkins':
+        update_local_bokchoy_db_from_s3()
+    else:
+        sh("{}/scripts/reset-test-db.sh --migrations".format(Env.REPO_ROOT))
 
 
 @task

scripts/calculate-bokchoy-migrations.sh

@@ -1,41 +0,0 @@
-#!/usr/bin/env bash
-
-############################################################################
-#
-#   Output all migrations that would be applied to an
-#   empty database for the bok-choy acceptance tests.
-#
-############################################################################
-
-# Fail fast
-set -e
-
-if [[ -z "$BOK_CHOY_HOSTNAME" ]]; then
-    MYSQL_HOST=""
-    SETTINGS="bok_choy"
-else
-    MYSQL_HOST="--host=edx.devstack.mysql"
-    SETTINGS="bok_choy_docker"
-fi
-
-declare -a database_order
-database_order=("default" "student_module_history")
-
-for db in "${database_order[@]}"; do
-    # Use a different database than the one used for testing,
-    # because we will need to empty out the database to calculate
-    # the migrations fingerprint.
-    # Choosing an arbitrary name "calculate_migrations" for the db.
-    echo "DROP DATABASE IF EXISTS calculate_migrations;" | mysql $MYSQL_HOST -u root
-    echo "CREATE DATABASE calculate_migrations;" | mysql $MYSQL_HOST -u root
-
-    # Now output all the migrations in the platform to a file.
-    echo "Calculating migrations for fingerprinting."
-
-    output_file="common/test/db_cache/bok_choy_${db}_migrations.yaml"
-    # Redirect stdout to /dev/null because the script will print
-    # out all migrations to both stdout and the output file.
-    ./manage.py lms --settings $SETTINGS show_unapplied_migrations --database $db --output_file $output_file 1>/dev/null
-done
-
-echo "DROP DATABASE IF EXISTS calculate_migrations;" | mysql $MYSQL_HOST -u root

scripts/reset-test-db.sh

@@ -43,6 +43,9 @@ for i in "$@"; do
         -m|--migrations)
             APPLY_MIGRATIONS=true
             ;;
+        -c|--calculate_migrations)
+            CALCULATE_MIGRATIONS=true
+            ;;
     esac
 done
 
@@ -51,6 +54,14 @@ declare -a database_order
 databases=(["default"]="edxtest" ["student_module_history"]="student_module_history_test")
 database_order=("default" "student_module_history")
 
+calculate_migrations() {
+    echo "Calculating migrations for fingerprinting."
+    output_file="common/test/db_cache/bok_choy_${db}_migrations.yaml"
+    # Redirect stdout to /dev/null because the script will print
+    # out all migrations to both stdout and the output file.
+    ./manage.py lms --settings $SETTINGS show_unapplied_migrations --database $db --output_file $output_file 1>/dev/null
+}
+
 run_migrations() {
     echo "Running the lms migrations on the $db bok_choy DB."
     ./manage.py lms --settings $SETTINGS migrate --database $db --traceback --noinput
@@ -115,4 +126,8 @@ elif [[ $APPLY_MIGRATIONS ]]; then
     for db in "${database_order[@]}"; do
         run_migrations
     done
+elif [[ $CALCULATE_MIGRATIONS ]]; then
+    for db in "${database_order[@]}"; do
+        calculate_migrations
+    done
 fi