From 6ebb753823382da2ff79e6de80f199a15a7f2886 Mon Sep 17 00:00:00 2001 From: Jay Zoldak Date: Thu, 19 Sep 2013 14:40:27 -0400 Subject: [PATCH] Write xunit reports for acceptance tests Refactor rake tasks for acceptance tests Address PR comments --- docs/internal/testing.md | 15 +++++--- jenkins/test_acceptance.sh | 4 +- rakelib/acceptance_test.rake | 71 ++++++++++++++++++++++++++++++++++++ rakelib/deprecated.rake | 3 ++ rakelib/tests.rake | 54 --------------------------- 5 files changed, 84 insertions(+), 63 deletions(-) create mode 100644 rakelib/acceptance_test.rake diff --git a/docs/internal/testing.md b/docs/internal/testing.md index 5404c16d05..5d6c75bddb 100644 --- a/docs/internal/testing.md +++ b/docs/internal/testing.md @@ -205,24 +205,27 @@ with Chrome (not Chromium) version 28.0.1500.71 with ChromeDriver version 2.1.210398. To run all the acceptance tests: + rake test:acceptance - rake test_acceptance_lms - rake test_acceptance_cms +To run only for lms or cms: + + rake test:acceptance:lms + rake test:acceptance:cms To test only a specific feature: - rake test_acceptance_lms["lms/djangoapps/courseware/features/problems.feature"] + rake test:acceptance:lms["lms/djangoapps/courseware/features/problems.feature"] To test only a specific scenario - rake test_acceptance_lms["lms/djangoapps/courseware/features/problems.feature -s 3"] + rake test:acceptance:lms["lms/djangoapps/courseware/features/problems.feature -s 3"] To start the debugger on failure, add the `--pdb` option: - rake test_acceptance_lms["lms/djangoapps/courseware/features/problems.feature --pdb"] + rake test:acceptance:lms["lms/djangoapps/courseware/features/problems.feature --pdb"] To run tests faster by not collecting static files, you can use -`rake fasttest_acceptance_lms` and `rake fasttest_acceptance_cms`. +`rake test:acceptance:lms:fast` and `rake test:acceptance:cms:fast`. Acceptance tests will run on a randomized port and can be run in the background of rake cms and lms or unit tests. To specify the port, change the LETTUCE_SERVER_PORT constant in cms/envs/acceptance.py and lms/envs/acceptance.py diff --git a/jenkins/test_acceptance.sh b/jenkins/test_acceptance.sh index 8070576231..47da592863 100755 --- a/jenkins/test_acceptance.sh +++ b/jenkins/test_acceptance.sh @@ -44,8 +44,6 @@ if [ "$LETTUCE_SELENIUM_CLIENT" == saucelabs ]; then fi # Run the lms and cms acceptance tests -# (the -v flag turns off color in the output) -rake test_acceptance_lms["-v 3 $SKIP_TESTS"] || TESTS_FAILED=1 -rake test_acceptance_cms["-v 3 $SKIP_TESTS"] || TESTS_FAILED=1 +rake test:acceptance["$SKIP_TESTS"] || TESTS_FAILED=1 [ $TESTS_FAILED == '0' ] diff --git a/rakelib/acceptance_test.rake b/rakelib/acceptance_test.rake new file mode 100644 index 0000000000..111ba98615 --- /dev/null +++ b/rakelib/acceptance_test.rake @@ -0,0 +1,71 @@ +ACCEPTANCE_DB = 'test_root/db/test_edx.db' +ACCEPTANCE_REPORT_DIR = report_dir_path('acceptance') +directory ACCEPTANCE_REPORT_DIR + +def run_acceptance_tests(system, harvest_args) + # Create the acceptance report directory + # because if it doesn't exist then lettuce will give an IOError. + report_dir = report_dir_path('acceptance') + + report_file = File.join(ACCEPTANCE_REPORT_DIR, "#{system}.xml") + report_args = "--with-xunit --xunit-file #{report_file}" + test_sh(django_admin(system, 'acceptance', 'harvest', '--debug-mode', '--verbosity 2', '--tag -skip', report_args, harvest_args)) +end + +task :setup_acceptance_db do + # HACK: Since the CMS depends on the existence of some database tables + # that are now in common but used to be in LMS (Role/Permissions for Forums) + # we need to create/migrate the database tables defined in the LMS. + # We might be able to address this by moving out the migrations from + # lms/django_comment_client, but then we'd have to repair all the existing + # migrations from the upgrade tables in the DB. + # But for now for either system (lms or cms), use the lms + # definitions to sync and migrate. + if File.exists?(ACCEPTANCE_DB) + File.delete(ACCEPTANCE_DB) + end + + sh(django_admin('lms', 'acceptance', 'syncdb', '--noinput')) + sh(django_admin('lms', 'acceptance', 'migrate', '--noinput')) +end + +task :prep_for_acceptance_tests => [ + :clean_reports_dir, :clean_test_files, ACCEPTANCE_REPORT_DIR, + :install_prereqs, :setup_acceptance_db +] + +namespace :test do + namespace :acceptance do + task :all, [:harvest_args] => [ + :prep_for_acceptance_tests, + "^^lms:gather_assets:acceptance", + "^^cms:gather_assets:acceptance" + ] do |t, args| + run_acceptance_tests('lms', args.harvest_args) + run_acceptance_tests('cms', args.harvest_args) + end + + ['lms', 'cms'].each do |system| + desc "Run the acceptance tests for the #{system}" + task system, [:harvest_args] => [ + :prep_for_acceptance_tests, + "^^#{system}:gather_assets:acceptance" + ] do |t, args| + args.with_defaults(:harvest_args => '') + run_acceptance_tests(system, args.harvest_args) + end + + desc "Run acceptance tests for the #{system} without collectstatic or db migrations" + task "#{system}:fast", [:harvest_args] => [ + :clean_reports_dir, ACCEPTANCE_REPORT_DIR, + ] do |t, args| + args.with_defaults(:harvest_args => '') + run_acceptance_tests(system, args.harvest_args) + end + end + end + desc "Run the lettuce acceptance tests for lms and cms" + task :acceptance, [:harvest_args] do |t, args| + Rake::Task["test:acceptance:all"].invoke(args.harvest_args) + end +end diff --git a/rakelib/deprecated.rake b/rakelib/deprecated.rake index 55c033226c..fc7d67e43c 100644 --- a/rakelib/deprecated.rake +++ b/rakelib/deprecated.rake @@ -24,6 +24,8 @@ end deprecated("jasmine:#{system}:phantomjs", "test:js:run", system) deprecated("#{system}:check_settings:jasmine", "") deprecated("#{system}:gather_assets:jasmine", "") + deprecated("test_acceptance_#{system}", "test:acceptance:#{system}") + deprecated("fasttest_acceptance_#{system}", "test:acceptance:#{system}:fast") end Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib| @@ -49,3 +51,4 @@ deprecated("jasmine:common/static/coffee:phantomjs", "test:js:run", "common") deprecated("jasmine", "test:js") deprecated("jasmine:phantomjs", "test:js:run") deprecated("jasmine:browser", "test:js:dev") +deprecated("test_acceptance", "test:acceptance") diff --git a/rakelib/tests.rake b/rakelib/tests.rake index 04a32961d7..fb6cf575bd 100644 --- a/rakelib/tests.rake +++ b/rakelib/tests.rake @@ -4,8 +4,6 @@ CLOBBER.include(REPORT_DIR, 'test_root/*_repo', 'test_root/staticfiles') # Create the directory to hold coverage reports, if it doesn't already exist. directory REPORT_DIR -ACCEPTANCE_DB = 'test_root/db/test_edx.db' - def test_id_dir(path) return File.join(".testids", path.to_s) end @@ -32,38 +30,6 @@ def run_tests(system, report_dir, test_id=nil, stop_on_failure=true) test_sh(run_under_coverage(cmd, system)) end -def create_acceptance_db(system) - # HACK: Since now the CMS depends on the existence of some database tables - # that used to be in LMS (Role/Permissions for Forums) we need to make - # sure the acceptance tests create/migrate the database tables - # that are represented in the LMS. We might be able to address this by moving - # out the migrations from lms/django_comment_client, but then we'd have to - # repair all the existing migrations from the upgrade tables in the DB. - if system == :cms - sh(django_admin('lms', 'acceptance', 'syncdb', '--noinput')) - sh(django_admin('lms', 'acceptance', 'migrate', '--noinput')) - end - sh(django_admin(system, 'acceptance', 'syncdb', '--noinput')) - sh(django_admin(system, 'acceptance', 'migrate', '--noinput')) -end - -def setup_acceptance_db(system, fasttest=false) - # If running under fasttest mode and the database already - # exists, skip the migrations. - if File.exists?(ACCEPTANCE_DB) - if not fasttest - File.delete(ACCEPTANCE_DB) - create_acceptance_db(system) - end - else - create_acceptance_db(system) - end -end - -def run_acceptance_tests(system, report_dir, harvest_args) - test_sh(django_admin(system, 'acceptance', 'harvest', '--debug-mode', '--verbosity 2', '--tag -skip', harvest_args)) -end - # Run documentation tests desc "Run documentation tests" task :test_docs do @@ -110,26 +76,6 @@ TEST_TASK_DIRS = [] run_tests(system, report_dir, args.test_id) end - # Run acceptance tests - desc "Run acceptance tests" - task "test_acceptance_#{system}", [:harvest_args] => [ - :clean_test_files, :install_prereqs, - "#{system}:gather_assets:acceptance" - ] do |t, args| - setup_acceptance_db(system) - Rake::Task["fasttest_acceptance_#{system}"].invoke(args.harvest_args) - end - - desc "Run acceptance tests without collectstatic or database migrations" - task "fasttest_acceptance_#{system}", [:harvest_args] => [ - report_dir, :clean_reports_dir - ] do |t, args| - args.with_defaults(:harvest_args => '') - setup_acceptance_db(system, fasttest=true) - run_acceptance_tests(system, report_dir, args.harvest_args) - end - - task :fasttest => "fasttest_#{system}" TEST_TASK_DIRS << system