Merge pull request #802 from cpennington/run-failed-tests

Teach rake test tasks how to re-run failed tests

.gitignore

@@ -46,3 +46,4 @@ autodeploy.properties
 .ws_migrations_complete
 .vagrant/
 logs
+.testids/

docs/internal/testing.md

@@ -103,6 +103,10 @@ You can run tests using `rake` commands.  For example,
 
 runs all the tests.  It also runs `collectstatic`, which prepares the static files used by the site (for example, compiling Coffeescript to Javascript).
 
+You can re-run all failed python tests by running (all JS tests will still run)
+
+    rake test[--failed]
+
 You can also run the tests without `collectstatic`, which tends to be faster:
 
     rake fasttest_lms
@@ -128,6 +132,10 @@ To run a single django test:
 
     rake test_lms[courseware.tests.tests:TestViewAuth.test_dark_launch]
 
+To re-run all failing django tests from lms or cms:
+
+    rake test_lms[--failed]
+
 To run a single nose test file:
 
     nosetests common/lib/xmodule/xmodule/tests/test_stringify.py
@@ -142,18 +150,18 @@ To run a single test and get stdout, with proper env config:
 
 To run a single test and get stdout and get coverage:
 
-    python -m coverage run --rcfile=./common/lib/xmodule/.coveragerc which ./manage.py cms --settings test test --traceback --logging-clear-handlers --liveserver=localhost:8000-9000 contentstore.tests.test_import_nostatic -s # cms example 
+    python -m coverage run --rcfile=./common/lib/xmodule/.coveragerc which ./manage.py cms --settings test test --traceback --logging-clear-handlers --liveserver=localhost:8000-9000 contentstore.tests.test_import_nostatic -s # cms example
     python -m coverage run --rcfile=./lms/.coveragerc which ./manage.py lms --settings test test --traceback --logging-clear-handlers --liveserver=localhost:8000-9000  courseware.tests.test_module_render -s # lms example
 
 generate coverage report:
 
-    coverage report --rcfile=./common/lib/xmodule/.coveragerc 
+    coverage report --rcfile=./common/lib/xmodule/.coveragerc
 
 or to get html report:
 
     coverage html --rcfile=./common/lib/xmodule/.coveragerc
-    
-then browse reports/common/lib/xmodule/cover/index.html    
+
+then browse reports/common/lib/xmodule/cover/index.html
 
 
 Very handy: if you uncomment the `pdb=1` line in `setup.cfg`, it will drop you into pdb on error.  This lets you go up and down the stack and see what the values of the variables are.  Check out [the pdb documentation](http://docs.python.org/library/pdb.html)

rakelib/tests.rake

@@ -4,6 +4,10 @@ CLOBBER.include(REPORT_DIR, 'test_root/*_repo', 'test_root/staticfiles')
 # Create the directory to hold coverage reports, if it doesn't already exist.
 directory REPORT_DIR
 
+def test_id_dir(path)
+    return File.join(".testids", path.to_s)
+end
+
 def run_under_coverage(cmd, root)
     cmd0, cmd_rest = cmd.split(" ", 2)
     # We use "python -m coverage" so that the proper python will run the importable coverage
@@ -14,9 +18,15 @@ end
 
 def run_tests(system, report_dir, test_id=nil, stop_on_failure=true)
     ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
+    test_id_file = File.join(test_id_dir(system), "noseids")
     dirs = Dir["common/djangoapps/*"] + Dir["#{system}/djangoapps/*"]
     test_id = dirs.join(' ') if test_id.nil? or test_id == ''
-    cmd = django_admin(system, :test, 'test', '--logging-clear-handlers', '--liveserver=localhost:8000-9000', test_id)
+    cmd = django_admin(
+        system, :test, 'test',
+        '--logging-clear-handlers',
+        '--liveserver=localhost:8000-9000',
+        "--id-file=#{test_id_file}",
+        test_id)
     test_sh(run_under_coverage(cmd, system))
 end
 
@@ -64,14 +74,17 @@ TEST_TASK_DIRS = []
 
 [:lms, :cms].each do |system|
     report_dir = report_dir_path(system)
+    test_id_dir = test_id_dir(system)
 
-    # Per System tasks
+    directory test_id_dir
+
+    # Per System tasks/
     desc "Run all django tests on our djangoapps for the #{system}"
     task "test_#{system}", [:test_id] => [:clean_test_files, :predjango, "#{system}:gather_assets:test", "fasttest_#{system}"]
 
     # Have a way to run the tests without running collectstatic -- useful when debugging without
     # messing with static files.
-    task "fasttest_#{system}", [:test_id] => [report_dir, :clean_reports_dir, :install_prereqs, :predjango] do |t, args|
+    task "fasttest_#{system}", [:test_id] => [test_id_dir, report_dir, :clean_reports_dir, :install_prereqs, :predjango] do |t, args|
         args.with_defaults(:test_id => nil)
         run_tests(system, report_dir, args.test_id)
     end
@@ -97,12 +110,16 @@ end
 Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib|
 
     report_dir = report_dir_path(lib)
+    test_id_dir = test_id_dir(lib)
+    test_ids = File.join(test_id_dir(lib), '.noseids')
+
+    directory test_id_dir
 
     desc "Run tests for common lib #{lib}"
-    task "test_#{lib}", [:test_id] => [report_dir, :clean_reports_dir] do |t, args|
+    task "test_#{lib}", [:test_id] => [test_id_dir, report_dir, :clean_reports_dir] do |t, args|
         args.with_defaults(:test_id => lib)
         ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
-        cmd = "nosetests #{args.test_id}"
+        cmd = "nosetests --id-file=#{test_ids} #{args.test_id}"
         test_sh(run_under_coverage(cmd, lib))
     end
     TEST_TASK_DIRS << lib
@@ -126,7 +143,7 @@ TEST_TASK_DIRS.each do |dir|
 end
 
 desc "Run all tests"
-task :test => :test_docs
+task :test, [:test_id] => :test_docs
 
 desc "Build the html, xml, and diff coverage reports"
 task :coverage => :report_dirs do

requirements/edx/base.txt

@@ -89,5 +89,6 @@ django-jasmine==0.3.2
 django_debug_toolbar
 django-debug-toolbar-mongo
 nose-ignore-docstring
+nose-exclude
 
 git+https://github.com/mfogel/django-settings-context-processor.git

setup.cfg

@@ -3,6 +3,9 @@ logging-clear-handlers=1
 with-xunit=1
 rednose=1
 with-ignore-docstrings=1
+with-id=1
+exclude-dir=lms/envs
+            cms/envs
 
 # Uncomment the following line to open pdb when a test fails
 #pdb=1