diff --git a/pavelib/database.py b/pavelib/database.py index da317a566d..8bf663aa0e 100644 --- a/pavelib/database.py +++ b/pavelib/database.py @@ -3,6 +3,7 @@ tasks for controlling the databases used in tests """ from __future__ import print_function import os +import hashlib from paver.easy import sh, needs @@ -18,7 +19,9 @@ def update_bokchoy_db_cache(): """ Update and cache the MYSQL database for bokchoy testing. This command will remove any previously cached database files and apply migrations - on a fresh db. + on a fresh db. Additionally, the collective sha1 checksum for all of + these files will be written to file, for future comparisons/checking + for updates. You can commit the resulting files in common/test/db_cache into git to speed up test runs @@ -43,3 +46,59 @@ def update_bokchoy_db_cache(): continue sh('{}/scripts/reset-test-db.sh'.format(Env.REPO_ROOT)) + + # Write the fingerprint of the database files to disk for use in future + # comparisons + fingerprint = fingerprint_bokchoy_db_files() + with open('common/test/db_cache/bokchoy_migrations.sha1', 'w') as fingerprint_file: + fingerprint_file.write(fingerprint) + + +def compare_bokchoy_db_fingerprints(): + """ + Determine if the current state of the bokchoy databases and related files + have changed since the last time they were updated in the repository by + comparing their fingerprint to the fingerprint saved in the repo + """ + try: + fingerprint_filepath = '{}/common/test/db_cache/bokchoy_migrations.sha1'.format(Env.REPO_ROOT) + with open(fingerprint_filepath, 'r') as fingerprint_file: + cached_fingerprint = fingerprint_file.read().strip() + except IOError: + return False + current_fingerprint = fingerprint_bokchoy_db_files() + return current_fingerprint == cached_fingerprint + + +def fingerprint_bokchoy_db_files(): + """ + Generate a sha1 checksum for files used to configure the bokchoy databases. + This checksum will represent the current 'state' of the databases, + including schema, migrations to be run and data. It can be used to determine + if the databases need to be updated. + """ + # Run the calculate-bokchoy-migrations script, which will generate two + # yml files. These tell whether or not we need to run migrations + sh('{}/scripts/calculate-bokchoy-migrations.sh'.format(Env.REPO_ROOT)) + db_files = [ + # Bokchoy db schema and data fixtures + 'bok_choy_data_default.json', + 'bok_choy_data_student_module_history.json', + 'bok_choy_migrations_data_default.sql', + 'bok_choy_migrations_data_student_module_history.sql', + 'bok_choy_schema_default.sql', + 'bok_choy_schema_student_module_history.sql', + # Output files from scripts/calculate-bokchoy-migrations.sh + 'bok_choy_default_migrations.yaml', + 'bok_choy_student_module_history_migrations.yaml' + ] + hasher = hashlib.sha1() + file_paths = [ + os.path.join('common/test/db_cache', db_file) for db_file in db_files + ] + for file_path in file_paths: + with open(file_path, 'rb') as file_handle: + hasher.update(file_handle.read()) + fingerprint = hasher.hexdigest() + print("Computed fingerprint for bokchoy db files: {}".format(fingerprint)) + return fingerprint diff --git a/scripts/calculate-bokchoy-migrations.sh b/scripts/calculate-bokchoy-migrations.sh new file mode 100755 index 0000000000..f58f0aa3ad --- /dev/null +++ b/scripts/calculate-bokchoy-migrations.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +############################################################################ +# +# Output all migrations that would be applied to an +# empty database for the bok-choy acceptance tests. +# +############################################################################ + +# Fail fast +set -e + +if [[ -z "$BOK_CHOY_HOSTNAME" ]]; then + MYSQL_HOST="" + SETTINGS="bok_choy" +else + MYSQL_HOST="--host=edx.devstack.mysql" + SETTINGS="bok_choy_docker" +fi + +declare -A databases +declare -a database_order +databases=(["default"]="edxtest" ["student_module_history"]="student_module_history_test") +database_order=("default" "student_module_history") + +for db in "${database_order[@]}"; do + echo "CREATE DATABASE IF NOT EXISTS ${databases[$db]};" | mysql $MYSQL_HOST -u root + + # Clear out the test database using the reset_db command which uses "DROP DATABASE" and + # "CREATE DATABASE". This will result in an empty database. + echo "Clearing out the $db bok_choy MySQL database." + ./manage.py lms --settings $SETTINGS reset_db --traceback --router $db + # Now output all the migrations in the platform to a file. + echo "Calculating migrations." + + output_file="common/test/db_cache/bok_choy_${db}_migrations.yaml" + ./manage.py lms --settings $SETTINGS show_unapplied_migrations --database $db --output_file $output_file + +done