Merge pull request #5552 from edx/benp/diff-cover-threshold2

Benp/diff cover threshold2

pavelib/quality.py

@@ -121,16 +121,29 @@ def _count_pep8_violations(report_file):
 
 @task
 @needs('pavelib.prereqs.install_python_prereqs')
-def run_quality():
+@cmdopts([
+    ("percentage=", "p", "fail if diff-quality is below this percentage"),
+])
+def run_quality(options):
     """
     Build the html diff quality reports, and print the reports to the console.
+    :param: p, diff-quality will fail if the quality percentage calculated is
+        below this percentage. For example, if p is set to 80, and diff-quality finds
+        quality of the branch vs master is less than 80%, then this task will fail.
+        This threshold would be applied to both pep8 and pylint.
     """
 
     # Directory to put the diff reports in.
     # This makes the folder if it doesn't already exist.
     dquality_dir = (Env.REPORT_DIR / "diff_quality").makedirs_p()
 
-    # Generage diff-quality html report for pep8, and print to console
+    # Set the string, if needed, to be used for the diff-quality --fail-under switch.
+    diff_threshold = int(getattr(options, 'percentage', -1))
+    threshold_string = ''
+    if diff_threshold > -1:
+        threshold_string = '--fail-under={0}'.format(diff_threshold)
+
+    # Generate diff-quality html report for pep8, and print to console
     # If pep8 reports exist, use those
     # Otherwise, `diff-quality` will call pep8 itself
 
@@ -149,8 +162,8 @@ def run_quality():
     )
 
     sh(
-        "diff-quality --violations=pep8 {pep8_reports}".format(
-            pep8_reports=pep8_reports)
+        "diff-quality --violations=pep8 {pep8_reports} {threshold_string}".format(
+            pep8_reports=pep8_reports, threshold_string=threshold_string)
     )
 
     # Generage diff-quality html report for pylint, and print to console
@@ -180,8 +193,9 @@ def run_quality():
     )
 
     sh(
-        "{pythonpath_prefix} diff-quality --violations=pylint {pylint_reports}".format(
+        "{pythonpath_prefix} diff-quality --violations=pylint {pylint_reports} {threshold_string}".format(
             pythonpath_prefix=pythonpath_prefix,
-            pylint_reports=pylint_reports
+            pylint_reports=pylint_reports,
+            threshold_string=threshold_string
         )
     )

requirements/edx/github.txt

@@ -22,7 +22,7 @@
 # Our libraries:
 -e git+https://github.com/edx/XBlock.git@81a6d713c98d4914af96a0ca624ee7fa4903625e#egg=XBlock
 -e git+https://github.com/edx/codejail.git@66dd5a45e5072666ff9a70c768576e9ffd1daa4b#egg=codejail
--e git+https://github.com/edx/diff-cover.git@v0.5.0#egg=diff_cover
+-e git+https://github.com/edx/diff-cover.git@9a44ae21369662a7d06bfc5111875fc0d119e03b#egg=diff_cover
 -e git+https://github.com/edx/js-test-tool.git@v0.1.5#egg=js_test_tool
 -e git+https://github.com/edx/event-tracking.git@0.1.0#egg=event-tracking
 -e git+https://github.com/edx/edx-analytics-data-api-client.git@0.1.0#egg=edx-analytics-data-api-client

scripts/all-tests.sh

@@ -109,7 +109,8 @@ case "$TEST_SUITE" in
     "quality")
         paver run_pep8 -l $PEP8_THRESHOLD > pep8.log || { cat pep8.log; EXIT=1; }
         paver run_pylint -l $PYLINT_THRESHOLD > pylint.log || { cat pylint.log; EXIT=1; }
-        paver run_quality
+        # Run quality task. Pass in the 'fail-under' percentage to diff-quality
+        paver run_quality -p 100
 
         # Need to create an empty test result so the post-build
         # action doesn't fail the build.