Merge pull request #1026 from MITx/feature/ned/quieter-tests

Feature/ned/quieter tests

cms/.coveragerc

@@ -0,0 +1,12 @@
+# .coveragerc for cms
+[run]
+data_file = reports/cms/.coverage
+
+[report]
+ignore_errors = True
+
+[html]
+directory = reports/cms/cover
+
+[xml]
+output = reports/cms/coverage.xml

cms/envs/test.py

@@ -14,9 +14,7 @@ from path import path
 
 # Nose Test Runner
 INSTALLED_APPS += ('django_nose',)
-NOSE_ARGS = ['--cover-erase', '--with-xunit', '--with-xcoverage', '--cover-html', '--cover-inclusive']
-for app in os.listdir(PROJECT_ROOT / 'djangoapps'):
-    NOSE_ARGS += ['--cover-package', app]
+NOSE_ARGS = ['--with-xunit']
 TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
 
 TEST_ROOT = path('test_root')

common/lib/capa/.coveragerc

@@ -0,0 +1,12 @@
+# .coveragerc for common/lib/capa
+[run]
+data_file = reports/common/lib/capa/.coverage
+
+[report]
+ignore_errors = True
+
+[html]
+directory = reports/common/lib/capa/cover
+
+[xml]
+output = reports/common/lib/capa/coverage.xml

common/lib/xmodule/.coveragerc

@@ -0,0 +1,12 @@
+# .coveragerc for common/lib/xmodule
+[run]
+data_file = reports/common/lib/xmodule/.coverage
+
+[report]
+ignore_errors = True
+
+[html]
+directory = reports/common/lib/xmodule/cover
+
+[xml]
+output = reports/common/lib/xmodule/coverage.xml

github-requirements.txt

@@ -0,0 +1,5 @@
+# Python libraries to install directly from github
+-e git://github.com/MITx/django-staticfiles.git@6d2504e5c8#egg=django-staticfiles
+-e git://github.com/MITx/django-pipeline.git#egg=django-pipeline
+-e git://github.com/MITx/django-wiki.git@e2e84558#egg=django-wiki
+-e git://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev

jenkins/quality.sh

@@ -0,0 +1,14 @@
+#! /bin/bash
+
+set -e
+set -x
+
+# Reset the submodule, in case it changed
+git submodule foreach 'git reset --hard HEAD'
+
+# Set the IO encoding to UTF-8 so that askbot will start
+export PYTHONIOENCODING=UTF-8
+
+rake clobber
+rake pep8 || echo "pep8 failed, continuing"
+rake pylint || echo "pylint failed, continuing"

jenkins/test_edge.sh

@@ -0,0 +1,29 @@
+#! /bin/bash
+
+set -e
+set -x
+
+# Reset the submodule, in case it changed
+git submodule foreach 'git reset --hard HEAD'
+
+# Set the IO encoding to UTF-8 so that askbot will start
+export PYTHONIOENCODING=UTF-8
+
+GIT_BRANCH=${GIT_BRANCH/HEAD/master}
+
+pip install -q -r pre-requirements.txt
+yes w | pip install -q -r requirements.txt
+[ ! -d askbot ] || pip install -q -r askbot/askbot_requirements.txt
+
+rake clobber
+TESTS_FAILED=0
+rake test_cms[false] || TESTS_FAILED=1
+rake test_lms[false] || TESTS_FAILED=1
+rake test_common/lib/capa || TESTS_FAILED=1
+rake test_common/lib/xmodule || TESTS_FAILED=1
+rake phantomjs_jasmine_lms || true
+rake phantomjs_jasmine_cms || true
+rake coverage:xml coverage:html
+
+[ $TESTS_FAILED == '0' ]
+rake autodeploy_properties

jenkins/test_lms.sh

@@ -0,0 +1,27 @@
+#! /bin/bash
+
+set -e
+set -x
+
+# Reset the submodule, in case it changed
+git submodule foreach 'git reset --hard HEAD'
+
+# Set the IO encoding to UTF-8 so that askbot will start
+export PYTHONIOENCODING=UTF-8
+
+GIT_BRANCH=${GIT_BRANCH/HEAD/master}
+
+pip install -q -r pre-requirements.txt
+yes w | pip install -q -r requirements.txt
+[ ! -d askbot ] || pip install -q -r askbot/askbot_requirements.txt
+
+rake clobber
+TESTS_FAILED=0
+rake test_lms[false] || TESTS_FAILED=1
+rake test_common/lib/capa || TESTS_FAILED=1
+rake test_common/lib/xmodule || TESTS_FAILED=1
+rake phantomjs_jasmine_lms || true
+rake coverage:xml coverage:html
+
+[ $TESTS_FAILED == '0' ]
+rake autodeploy_properties

lms/.coveragerc

@@ -0,0 +1,12 @@
+# .coveragerc for lms
+[run]
+data_file = reports/lms/.coverage
+
+[report]
+ignore_errors = True
+
+[html]
+directory = reports/lms/cover
+
+[xml]
+output = reports/lms/coverage.xml

lms/envs/test.py

@@ -27,18 +27,11 @@ SOUTH_TESTS_MIGRATE = False # To disable migrations and use syncdb instead
 
 # Nose Test Runner
 INSTALLED_APPS += ('django_nose',)
-NOSE_ARGS = []
 
-# Turning off coverage speeds up tests dramatically... until we have better config,
-# leave it here for manual fiddling.
-_coverage = True
-if _coverage:
-    NOSE_ARGS = ['--cover-erase', '--with-xunit', '--with-xcoverage', '--cover-html',
-                 # '-v', '--pdb',     # When really stuck, uncomment to start debugger on error
-                 '--cover-inclusive', '--cover-html-dir',
-                 os.environ.get('NOSE_COVER_HTML_DIR', 'cover_html')]
-    for app in os.listdir(PROJECT_ROOT / 'djangoapps'):
-        NOSE_ARGS += ['--cover-package', app]
+NOSE_ARGS = [
+    '--with-xunit',
+    # '-v', '--pdb',     # When really stuck, uncomment to start debugger on error
+]
 TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
 
 # Local Directories

local-requirements.txt

@@ -0,0 +1,3 @@
+# Python libraries to install that are local to the mitx repo
+-e common/lib/capa
+-e common/lib/xmodule

rakefile

@@ -38,6 +38,10 @@ def django_admin(system, env, command, *args)
     return "#{django_admin} #{command} --settings=#{system}.envs.#{env} --pythonpath=. #{args.join(' ')}"
 end
 
+def report_dir_path(dir)
+    return File.join(REPORT_DIR, dir.to_s)
+end
+
 task :default => [:test, :pep8, :pylint]
 
 directory REPORT_DIR
@@ -49,16 +53,16 @@ default_options = {
 
 task :predjango do
     sh("find . -type f -name *.pyc -delete")
-    sh('pip install -e common/lib/xmodule -e common/lib/capa')
+    sh('pip install -q --upgrade -r local-requirements.txt')
     sh('git submodule update --init')
 end
 
 task :clean_test_files do
-    sh("git clean -fdx test_root")
+    sh("git clean -fqdx test_root")
 end
 
 [:lms, :cms, :common].each do |system|
-    report_dir = File.join(REPORT_DIR, system.to_s)
+    report_dir = report_dir_path(system)
     directory report_dir
 
     desc "Run pep8 on all #{system} code"
@@ -84,11 +88,19 @@ end
 
 $failed_tests = 0
 
+def run_under_coverage(cmd, root)
+    cmd0, cmd_rest = cmd.split(" ", 2)
+    # We use "python -m coverage" so that the proper python will run the importable coverage
+    # rather than the coverage that OS path finds.
+    cmd = "python -m coverage run --rcfile=#{root}/.coveragerc `which #{cmd0}` #{cmd_rest}"
+    return cmd
+end
+
 def run_tests(system, report_dir, stop_on_failure=true)
     ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
-    ENV['NOSE_COVER_HTML_DIR'] = File.join(report_dir, "cover")
     dirs = Dir["common/djangoapps/*"] + Dir["#{system}/djangoapps/*"]
-    sh(django_admin(system, :test, 'test', *dirs.each)) do |ok, res|
+    cmd = django_admin(system, :test, 'test', '--logging-clear-handlers', *dirs.each)
+    sh(run_under_coverage(cmd, system)) do |ok, res|
         if !ok and stop_on_failure
             abort "Test failed!"
         end
@@ -96,11 +108,10 @@ def run_tests(system, report_dir, stop_on_failure=true)
     end
 end
 
-TEST_TASKS = []
+TEST_TASK_DIRS = []
 
 [:lms, :cms].each do |system|
-    report_dir = File.join(REPORT_DIR, system.to_s)
-    directory report_dir
+    report_dir = report_dir_path(system)
 
     # Per System tasks
     desc "Run all django tests on our djangoapps for the #{system}"
@@ -113,7 +124,7 @@ TEST_TASKS = []
         run_tests(system, report_dir, args.stop_on_failure)
     end
 
-    TEST_TASKS << "test_#{system}"
+    TEST_TASK_DIRS << system
 
     desc <<-desc
         Start the #{system} locally with the specified environment (defaults to dev).
@@ -134,35 +145,48 @@ TEST_TASKS = []
 
         desc "Run collectstatic in the specified environment"
         task "#{system}:collectstatic:#{env}" => :predjango do
-            sh("#{django_admin(system, env, 'collectstatic', '--noinput')}")
+            sh("#{django_admin(system, env, 'collectstatic', '--noinput')} > /tmp/collectstatic.out") do |ok, status|
+                if !ok
+                    abort "collectstatic failed!"
+                end
+            end
         end
     end
 end
 
-Dir["common/lib/*"].each do |lib|
+Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib|
     task_name = "test_#{lib}"
 
-    report_dir = File.join(REPORT_DIR, task_name.gsub('/', '_'))
-    directory report_dir
+    report_dir = report_dir_path(lib)
 
     desc "Run tests for common lib #{lib}"
     task task_name => report_dir do
         ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
-        sh("nosetests #{lib} --cover-erase --with-xunit --with-xcoverage --cover-html --cover-inclusive --cover-package #{File.basename(lib)} --cover-html-dir #{File.join(report_dir, "cover")}")
+        cmd = "nosetests #{lib} --logging-clear-handlers --with-xunit"
+        sh(run_under_coverage(cmd, lib)) do |ok, res|
+            $failed_tests += 1 unless ok
+        end
     end
-    TEST_TASKS << task_name
+    TEST_TASK_DIRS << lib
 
     desc "Run tests for common lib #{lib} (without coverage)"
     task "fasttest_#{lib}" do 
         sh("nosetests #{lib}")
     end
 
+end
 
+task :report_dirs
+
+TEST_TASK_DIRS.each do |dir|
+    report_dir = report_dir_path(dir)
+    directory report_dir
+    task :report_dirs => [REPORT_DIR, report_dir]
 end
 
 task :test do
-    TEST_TASKS.each do |task|
-        Rake::Task[task].invoke(false)
+    TEST_TASK_DIRS.each do |dir|
+        Rake::Task["test_#{dir}"].invoke(false)
     end
 
     if $failed_tests > 0
@@ -170,6 +194,27 @@ task :test do
     end
 end
 
+namespace :coverage do
+    desc "Build the html coverage reports"
+    task :html => :report_dirs do
+        TEST_TASK_DIRS.each do |dir|
+            sh("coverage html --rcfile=#{dir}/.coveragerc || echo 'Unable to build coverage html for #{dir}'")
+        end
+    end
+
+    desc "Build the xml coverage reports"
+    task :xml => :report_dirs do
+        TEST_TASK_DIRS.each do |dir|
+            # Why doesn't the rcfile control the xml output file properly??
+            sh("coverage xml -o reports/#{dir}/coverage.xml --rcfile=#{dir}/.coveragerc || echo 'Unable to build coverage xml for #{dir}'")
+        end
+    end
+
+    TEST_TASK_DIRS.each do |dir|
+        CLOBBER.include("#{dir}/.coverage")
+    end
+end
+
 task :runserver => :lms
 
 desc "Run django-admin <action> against the specified system and environment"
@@ -270,4 +315,4 @@ task :doc => :builddocs do
     Dir.chdir('docs/build/html') do
         sh('open index.html')
     end
-end
+end

repo-requirements.txt

@@ -1,6 +1,2 @@
--e git://github.com/MITx/django-staticfiles.git@6d2504e5c8#egg=django-staticfiles
--e git://github.com/MITx/django-pipeline.git#egg=django-pipeline
--e git://github.com/MITx/django-wiki.git@e2e84558#egg=django-wiki
--e git://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev
--e common/lib/capa
--e common/lib/xmodule
+-r github-requirements.txt
+-r local-requirements.txt