From 7b5e5794b0ff6cbadf664c6dfdd36a663d693bba Mon Sep 17 00:00:00 2001 From: Ben Patterson Date: Wed, 16 Jul 2014 07:21:36 -0400 Subject: [PATCH 01/11] add Solano CI. --- tddium.yml | 101 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 tddium.yml diff --git a/tddium.yml b/tddium.yml new file mode 100644 index 0000000000..873da74eb2 --- /dev/null +++ b/tddium.yml @@ -0,0 +1,101 @@ +tddium: + :firefox: 25.0.1 + :timeout: 3600 + :timeout_hook: 900 + :tool_config: + git: + :version: "1.8.5.5" + :hooks: + :pre_setup: "virtualenv $HOME/python-env && $HOME/python-env/bin/pip install -r requirements/edx/paver.txt && $HOME/python-env/bin/pip install -r requirements/edx/pre.txt && $HOME/python-env/bin/pip install -r requirements/edx/base.txt && $HOME/python-env/bin/pip install -r requirements/edx/github.txt && $HOME/python-env/bin/pip install -r requirements/edx/local.txt && $HOME/python-env/bin/pip install -r requirements/edx/post.txt" + :post_build: 'for i in $(find test_root/log -name *.png); do cp -R $i $HOME/results/$TDDIUM_SESSION_ID/session/; echo "copying file $i"; done' + # :post_build: "paver coverage; paver bokchoy_coverage; for i in $(find reports -name cover); do cp -R $i $HOME/results/$TDDIUM_SESSION_ID/session/; done" + :cache: + :key_paths: + - requirements/edx/paver.txt + - requirements/edx/pre.txt + - requirements/edx/base.txt + - requirements/edx/github.txt + - requirements/edx/local.txt + - requirements/edx/post.txt + - package.json + :mongodb: + :version: "2.4.7" + :mysql: + :version: "5.5" + :python: + :python_version: "2.7" + :headless: true + :java: + :java_version: "java-7-openjdk" + :test_pattern: 'none' + :tests: + - :type: custom + :command: paver run_quality + :invocation: single + :output: exit-status + :report_files: + - "reports/diff_quality/diff_quality_pylint.html" + - :type: junit + :mode: basic + :command: "mkdir -p jscover-dist && wget http://files.edx.org/testeng/JSCover-1.0.2.zip -P jscover-dist && unzip jscover-dist/JSCover-1.0.2.zip -d jscover-dist/ && cp jscover-dist/target/dist/JSCover-all.jar jscover-dist && export JSCOVER_JAR=$PWD/jscover-dist/JSCover-all.jar && paver test" + :invocation: single + :output: exit-status + :report_files: + - "reports/cms/nosetests.xml" + - "reports/common/lib/capa/nosetests.xml" + - "reports/common/lib/calc/nosetests.xml" + - "reports/common/lib/chem/nosetests.xml" + - "reports/common/lib/sandbox-packages/nosetests.xml" + - "reports/common/lib/symmath/nosetests.xml" + - "reports/common/lib/xmodule/nosetests.xml" + - "reports/lms/nosetests.xml" + - "reports/javascript/javascript_xunit.xml" + - :type: junit + :mode: basic + :command: paver test_acceptance -s lms --extra_args="-v 3 --tag shard_1" + :invocation: single + :output: exit-status + :report_files: + - "reports/acceptance/lms.xml" + - :type: junit + :mode: basic + :command: paver test_acceptance -s lms --extra_args="-v 3 --tag shard_2" + :invocation: single + :output: exit-status + :report_files: + - "reports/acceptance/lms.xml" + - :type: junit + :mode: basic + :command: paver test_acceptance -s cms --extra_args="-v 3 --tag shard_1" + :invocation: single + :output: exit-status + :report_files: + - "reports/acceptance/cms.xml" + - :type: junit + :mode: basic + :command: paver test_acceptance -s cms --extra_args="-v 3 --tag shard_2" + :invocation: single + :output: exit-status + :report_files: + - "reports/acceptance/cms.xml" + - :type: junit + :mode: basic + :command: paver test_acceptance -s cms --extra_args="-v 3 --tag shard_3" + :invocation: single + :output: exit-status + :report_files: + - "reports/acceptance/cms.xml" + - :type: junit + :mode: basic + :command: paver test_bokchoy --extra_args="-a shard_1" + :invocation: single + :output: exit-status + :report_files: + - "reports/bok_choy/xunit.xml" + - :type: junit + :mode: basic + :command: paver test_bokchoy --extra_args="-a '!shard_1'" + :invocation: single + :output: exit-status + :report_files: + - "reports/bok_choy/xunit.xml" From 2b40e262837d0386026e5e067e50dd0b07f75cb6 Mon Sep 17 00:00:00 2001 From: Ben Patterson Date: Thu, 14 Aug 2014 21:38:48 -0400 Subject: [PATCH 02/11] Merge PR#3. Solano: Add post worker hook for copying artifacts. --- scripts/post_worker.py | 30 ++++++++++++++++++++++++++++++ tddium.yml | 1 + 2 files changed, 31 insertions(+) create mode 100644 scripts/post_worker.py diff --git a/scripts/post_worker.py b/scripts/post_worker.py new file mode 100644 index 0000000000..d4994b382d --- /dev/null +++ b/scripts/post_worker.py @@ -0,0 +1,30 @@ +import tarfile +import os +import shutil + +full_path = os.path.realpath(__file__) +source_dir = full_path.replace("scripts/post_worker.py", "reports/") +output_filename = full_path.replace("post_worker.py", "reports.tar.gz") + +print "source dir:", source_dir + +count = 0 + +# walk through every subdirectory & add the folder if it is not empty +with tarfile.open(output_filename, "w:gz") as tar: + for (path, dirs, files) in os.walk(source_dir): + if len(files) > 0: + print "tarring:", path + tar.add(path, arcname=os.path.basename(path)) + count += 1 + +tar.close() + +file_dest = os.environ['HOME'] + '/results/' + os.environ['TDDIUM_SESSION_ID'] + '/session/reports.tar.gz' + +# if the tar file is not empty, copy it to the proper place +if count > 0: + shutil.copyfile(output_filename, file_dest) + print "done copying file" + +print "TDDIUM_SESSION_ID:", os.environ['TDDIUM_SESSION_ID'] \ No newline at end of file diff --git a/tddium.yml b/tddium.yml index 873da74eb2..98a33737af 100644 --- a/tddium.yml +++ b/tddium.yml @@ -9,6 +9,7 @@ tddium: :pre_setup: "virtualenv $HOME/python-env && $HOME/python-env/bin/pip install -r requirements/edx/paver.txt && $HOME/python-env/bin/pip install -r requirements/edx/pre.txt && $HOME/python-env/bin/pip install -r requirements/edx/base.txt && $HOME/python-env/bin/pip install -r requirements/edx/github.txt && $HOME/python-env/bin/pip install -r requirements/edx/local.txt && $HOME/python-env/bin/pip install -r requirements/edx/post.txt" :post_build: 'for i in $(find test_root/log -name *.png); do cp -R $i $HOME/results/$TDDIUM_SESSION_ID/session/; echo "copying file $i"; done' # :post_build: "paver coverage; paver bokchoy_coverage; for i in $(find reports -name cover); do cp -R $i $HOME/results/$TDDIUM_SESSION_ID/session/; done" + :post_worker: 'python ./scripts/post_worker.py' :cache: :key_paths: - requirements/edx/paver.txt From d6434676ca8af74062a3b8a756e0a36889476ddb Mon Sep 17 00:00:00 2001 From: Minh Tue Vo Date: Fri, 15 Aug 2014 11:25:39 -0400 Subject: [PATCH 03/11] fixed capturing screenshots --- scripts/post_worker.py | 6 ++++++ tddium.yml | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/post_worker.py b/scripts/post_worker.py index d4994b382d..7cba7640d4 100644 --- a/scripts/post_worker.py +++ b/scripts/post_worker.py @@ -27,4 +27,10 @@ if count > 0: shutil.copyfile(output_filename, file_dest) print "done copying file" +# TODO: fold the remaining bash script into the Python script +# finding if there is any screenshots +# print "checking for screenshots" +# for (path, dirs, files) in os.walk('test_root/log'): +# print files + print "TDDIUM_SESSION_ID:", os.environ['TDDIUM_SESSION_ID'] \ No newline at end of file diff --git a/tddium.yml b/tddium.yml index 98a33737af..c0d2a22527 100644 --- a/tddium.yml +++ b/tddium.yml @@ -7,9 +7,8 @@ tddium: :version: "1.8.5.5" :hooks: :pre_setup: "virtualenv $HOME/python-env && $HOME/python-env/bin/pip install -r requirements/edx/paver.txt && $HOME/python-env/bin/pip install -r requirements/edx/pre.txt && $HOME/python-env/bin/pip install -r requirements/edx/base.txt && $HOME/python-env/bin/pip install -r requirements/edx/github.txt && $HOME/python-env/bin/pip install -r requirements/edx/local.txt && $HOME/python-env/bin/pip install -r requirements/edx/post.txt" - :post_build: 'for i in $(find test_root/log -name *.png); do cp -R $i $HOME/results/$TDDIUM_SESSION_ID/session/; echo "copying file $i"; done' # :post_build: "paver coverage; paver bokchoy_coverage; for i in $(find reports -name cover); do cp -R $i $HOME/results/$TDDIUM_SESSION_ID/session/; done" - :post_worker: 'python ./scripts/post_worker.py' + :post_worker: 'python ./scripts/post_worker.py; echo "attaching failed screenshots (if any)"; for i in $(find test_root/log -name *.png); do cp -R $i $HOME/results/$TDDIUM_SESSION_ID/session/; echo "copying file $i"; done' :cache: :key_paths: - requirements/edx/paver.txt From bef4e9c5844e33912385f8dfa12396ca9933ac09 Mon Sep 17 00:00:00 2001 From: Minh Tue Vo Date: Mon, 25 Aug 2014 17:13:40 -0400 Subject: [PATCH 04/11] adding post worker script --- scripts/post_worker.py | 21 +++++++++++++-------- tddium.yml | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/scripts/post_worker.py b/scripts/post_worker.py index 7cba7640d4..5048595dc7 100644 --- a/scripts/post_worker.py +++ b/scripts/post_worker.py @@ -20,17 +20,22 @@ with tarfile.open(output_filename, "w:gz") as tar: tar.close() -file_dest = os.environ['HOME'] + '/results/' + os.environ['TDDIUM_SESSION_ID'] + '/session/reports.tar.gz' +session_path = os.environ['HOME'] + '/results/' + os.environ['TDDIUM_SESSION_ID'] + '/session/' +file_dest = session_path + 'reports.tar.gz' # if the tar file is not empty, copy it to the proper place if count > 0: shutil.copyfile(output_filename, file_dest) - print "done copying file" + print 'done copying file' -# TODO: fold the remaining bash script into the Python script -# finding if there is any screenshots -# print "checking for screenshots" -# for (path, dirs, files) in os.walk('test_root/log'): -# print files +# finding if there is any screenshot or log file +print 'attaching failed screenshots and logs (if any)' +for (path, dirs, files) in os.walk('test_root/log'): + for filename in files: + if filename.find('png') != -1 or filename.find('log') != -1: + filepath = path + filename + print 'copying file:', filepath + destpath = session_path + filename + shutil.copyfile(filepath, destpath) -print "TDDIUM_SESSION_ID:", os.environ['TDDIUM_SESSION_ID'] \ No newline at end of file +print 'TDDIUM_SESSION_ID:', os.environ['TDDIUM_SESSION_ID'] \ No newline at end of file diff --git a/tddium.yml b/tddium.yml index c0d2a22527..38bf58a05f 100644 --- a/tddium.yml +++ b/tddium.yml @@ -8,7 +8,7 @@ tddium: :hooks: :pre_setup: "virtualenv $HOME/python-env && $HOME/python-env/bin/pip install -r requirements/edx/paver.txt && $HOME/python-env/bin/pip install -r requirements/edx/pre.txt && $HOME/python-env/bin/pip install -r requirements/edx/base.txt && $HOME/python-env/bin/pip install -r requirements/edx/github.txt && $HOME/python-env/bin/pip install -r requirements/edx/local.txt && $HOME/python-env/bin/pip install -r requirements/edx/post.txt" # :post_build: "paver coverage; paver bokchoy_coverage; for i in $(find reports -name cover); do cp -R $i $HOME/results/$TDDIUM_SESSION_ID/session/; done" - :post_worker: 'python ./scripts/post_worker.py; echo "attaching failed screenshots (if any)"; for i in $(find test_root/log -name *.png); do cp -R $i $HOME/results/$TDDIUM_SESSION_ID/session/; echo "copying file $i"; done' + :post_worker: 'python ./scripts/post_worker.py' :cache: :key_paths: - requirements/edx/paver.txt From 07a723c08e996d75faf6dd8d1f50d7defa8d1e50 Mon Sep 17 00:00:00 2001 From: Minh Tue Vo Date: Wed, 27 Aug 2014 13:47:26 -0400 Subject: [PATCH 05/11] fixing file log error --- scripts/post_worker.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/scripts/post_worker.py b/scripts/post_worker.py index 5048595dc7..3230be6017 100644 --- a/scripts/post_worker.py +++ b/scripts/post_worker.py @@ -20,22 +20,28 @@ with tarfile.open(output_filename, "w:gz") as tar: tar.close() -session_path = os.environ['HOME'] + '/results/' + os.environ['TDDIUM_SESSION_ID'] + '/session/' -file_dest = session_path + 'reports.tar.gz' +session_path = os.path.join( + os.environ['HOME'], + 'results', + os.environ['TDDIUM_SESSION_ID'], + 'session') + +file_dest = os.path.join(session_path, 'reports.tar.gz') # if the tar file is not empty, copy it to the proper place if count > 0: + print 'copying tar file to:', file_dest shutil.copyfile(output_filename, file_dest) - print 'done copying file' # finding if there is any screenshot or log file print 'attaching failed screenshots and logs (if any)' for (path, dirs, files) in os.walk('test_root/log'): for filename in files: if filename.find('png') != -1 or filename.find('log') != -1: - filepath = path + filename + filepath = os.path.join(path, filename) print 'copying file:', filepath - destpath = session_path + filename + destpath = os.path.join(session_path, filename) + print 'destination:', destpath shutil.copyfile(filepath, destpath) print 'TDDIUM_SESSION_ID:', os.environ['TDDIUM_SESSION_ID'] \ No newline at end of file From eb0ada02d4ad36a93c776438c024d03f5e4537a6 Mon Sep 17 00:00:00 2001 From: Ben Patterson Date: Fri, 5 Sep 2014 13:54:14 -0400 Subject: [PATCH 06/11] introduce bok-choy shards, new firefox version --- tddium.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tddium.yml b/tddium.yml index 38bf58a05f..c148a9ebc0 100644 --- a/tddium.yml +++ b/tddium.yml @@ -1,5 +1,5 @@ tddium: - :firefox: 25.0.1 + :firefox: '28.0' :timeout: 3600 :timeout_hook: 900 :tool_config: @@ -52,14 +52,7 @@ tddium: - "reports/javascript/javascript_xunit.xml" - :type: junit :mode: basic - :command: paver test_acceptance -s lms --extra_args="-v 3 --tag shard_1" - :invocation: single - :output: exit-status - :report_files: - - "reports/acceptance/lms.xml" - - :type: junit - :mode: basic - :command: paver test_acceptance -s lms --extra_args="-v 3 --tag shard_2" + :command: paver test_acceptance -s lms :invocation: single :output: exit-status :report_files: @@ -94,7 +87,14 @@ tddium: - "reports/bok_choy/xunit.xml" - :type: junit :mode: basic - :command: paver test_bokchoy --extra_args="-a '!shard_1'" + :command: paver test_bokchoy --extra_args="-a shard_2" + :invocation: single + :output: exit-status + :report_files: + - "reports/bok_choy/xunit.xml" + - :type: junit + :mode: basic + :command: paver test_bokchoy --extra_args="-a shard_1=False,shard_2=False" :invocation: single :output: exit-status :report_files: From a90ba504f794ada87d09321fa9083dbdc0adbbcd Mon Sep 17 00:00:00 2001 From: Ben Patterson Date: Wed, 10 Sep 2014 14:29:19 -0400 Subject: [PATCH 07/11] re-shard lms to fix a failing test. --- tddium.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tddium.yml b/tddium.yml index c148a9ebc0..68d01a761b 100644 --- a/tddium.yml +++ b/tddium.yml @@ -52,7 +52,14 @@ tddium: - "reports/javascript/javascript_xunit.xml" - :type: junit :mode: basic - :command: paver test_acceptance -s lms + :command: paver test_acceptance -s lms --extra_args="-v 3 --tag shard_1" + :invocation: single + :output: exit-status + :report_files: + - "reports/acceptance/lms.xml" + - :type: junit + :mode: basic + :command: paver test_acceptance -s lms --extra_args="-v 3 --tag shard_2" :invocation: single :output: exit-status :report_files: @@ -66,14 +73,7 @@ tddium: - "reports/acceptance/cms.xml" - :type: junit :mode: basic - :command: paver test_acceptance -s cms --extra_args="-v 3 --tag shard_2" - :invocation: single - :output: exit-status - :report_files: - - "reports/acceptance/cms.xml" - - :type: junit - :mode: basic - :command: paver test_acceptance -s cms --extra_args="-v 3 --tag shard_3" + :command: paver test_acceptance -s cms --extra_args="-v 3 --tag shard_2 --tag shard_3" :invocation: single :output: exit-status :report_files: From 5524e0bdee534e4644cf39fe20b1544b46c2f1ad Mon Sep 17 00:00:00 2001 From: Ben Patterson Date: Mon, 15 Sep 2014 13:46:00 -0400 Subject: [PATCH 08/11] Add quality report capability. --- scripts/post_quality_build.sh | 15 +++++++++++++++ scripts/post_worker.py | 2 +- tddium.yml | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100755 scripts/post_quality_build.sh diff --git a/scripts/post_quality_build.sh b/scripts/post_quality_build.sh new file mode 100755 index 0000000000..0492ef48d6 --- /dev/null +++ b/scripts/post_quality_build.sh @@ -0,0 +1,15 @@ +current_path=`pwd` + +reports_path=$current_path/reports + +dest_path=$HOME/results/$TDDIUM_SESSION_ID/session/ + +echo "Getting Quality Reports... " + +pep8_rpt=$reports_path/diff_quality/diff_quality_pep8.html +pylint_rpt=$reports_path/diff_quality/diff_quality_pylint.html + +cp -f $pep8_rpt $dest_path +cp -f $pylint_rpt $dest_path + +echo "Reports can be found in "$dest_path diff --git a/scripts/post_worker.py b/scripts/post_worker.py index 3230be6017..7ffc7e0f24 100644 --- a/scripts/post_worker.py +++ b/scripts/post_worker.py @@ -44,4 +44,4 @@ for (path, dirs, files) in os.walk('test_root/log'): print 'destination:', destpath shutil.copyfile(filepath, destpath) -print 'TDDIUM_SESSION_ID:', os.environ['TDDIUM_SESSION_ID'] \ No newline at end of file +print 'TDDIUM_SESSION_ID:', os.environ['TDDIUM_SESSION_ID'] diff --git a/tddium.yml b/tddium.yml index 68d01a761b..4c97d149b4 100644 --- a/tddium.yml +++ b/tddium.yml @@ -30,7 +30,7 @@ tddium: :test_pattern: 'none' :tests: - :type: custom - :command: paver run_quality + :command: paver run_quality && bash ./scripts/post_quality_build.sh :invocation: single :output: exit-status :report_files: From ff0e8c3805269c8fc5f72f3d55e66cd48dc170e5 Mon Sep 17 00:00:00 2001 From: Muhammad Ammar Date: Fri, 12 Sep 2014 13:07:56 +0500 Subject: [PATCH 09/11] Merge Coverage Reports TE-465 --- scripts/cov_merge.py | 172 +++++++++++++++++++++++++++++++++++++++++++ scripts/coverage.sh | 25 +++++++ scripts/run_ut.sh | 21 ++++++ tddium.yml | 9 +-- 4 files changed, 222 insertions(+), 5 deletions(-) create mode 100644 scripts/cov_merge.py create mode 100755 scripts/coverage.sh create mode 100755 scripts/run_ut.sh diff --git a/scripts/cov_merge.py b/scripts/cov_merge.py new file mode 100644 index 0000000000..8b4cd1c6d5 --- /dev/null +++ b/scripts/cov_merge.py @@ -0,0 +1,172 @@ +import os +import sys +from textwrap import dedent +from bs4 import BeautifulSoup +import multiprocessing + + +FIRST = dedent( + ''' + + + + CMS Python Test Coverage Report + + + + + + + + + ''') + + +LAST = dedent( + ''' + + + ''') + + +class ReportMerge(object): + """Merge multiple html coverage reports""" + + DESTINATION = os.path.join(os.environ['HOME'], 'results', os.environ['TDDIUM_SESSION_ID'], 'session') + + def __init__(self): + self.reports_dir = os.path.realpath(__file__).replace("scripts/cov_merge.py", "reports/") + + def _files(self, cover_path): + """ + Return list of file paths in `cover_path`. `cover_path` will be something like */reports/cms/cover + """ + include = lambda f: f.endswith('.html') and os.path.basename(f) != 'index.html' + return [os.path.join(cover_path, f) for f in os.listdir(cover_path) if include(f)] + + def merge(self, modules, output_file=None): + """ + Merge reports for `modules` + + Arguments: + output_file (str): name of output report file -- only used for bok_choy reports + + """ + for module in modules: + for (path, _, _) in os.walk(os.path.join(self.reports_dir, module)): + if os.path.basename(path) == 'cover': + self.merge_report(path, output_file) + + def merge_report(self, path, output_file): + """ + Collect multiple parts of a report and join them to create a single report. + + Arguments: + path (str): path where multiple files are located to be merged + output_file (str): name of output report file -- only used for bok_choy reports + + """ + content = list() + + # Extract total coverage percentage and file links table + index_html = os.path.join(path, 'index.html') + with open(index_html) as index_file: + soup = BeautifulSoup(index_file) + total_percentage = soup.find('div', id='header') + total_percentage.find('img').decompose() + index_table = soup.find('div', id='index') + + # Extract file names + files = [os.path.join(path, name['href']) for name in index_table.find_all('a')] + if not files: + return + + print 'Merging Report for {}'.format(path) + + # Collect different parts of html report + content.append(FIRST) + content.append('') + content.append(str(total_percentage)) + content.append(str(index_table)) + for html in files: + content.append(self._html_content(html)) + + content.append(LAST) + + if output_file: + report_path = os.path.join(self.DESTINATION, output_file) + else: + report_filename = path.split('reports/')[1].split('/cover')[0].replace('/', '_') + report_path = os.path.join(self.DESTINATION, report_filename+'_coverage.html') + + # Write everything to single report file + with open(report_path, 'w') as report_file: + report_file.write('\n'.join(content)) + + print 'Report Merged for {}'.format(path) + + def _html_content(self, html): + """ + Returns html tags of interest for file specified by `html` + """ + # Create id for each link in file links table + navigate_div_id = os.path.basename(html).split('.')[0].replace('/', '_') + navigate_div_start = "
\n".format(navigate_div_id) + navigate_div_close = "\n
".format(navigate_div_id) + + content = list() + content.append(navigate_div_start) + + with open(html) as html_file: + soup = BeautifulSoup(html_file) + header = soup.find('div', id='header') + header.find('img').decompose() + source = soup.find('div', id='source') + source_img = source.find('img') + if source_img: + source_img.decompose() + + content.append(str(header)) + content.append(str(source)) + + content.append(navigate_div_close) + + return '\n'.join(content) + +if __name__ == '__main__': + args = sys.argv + + if 'bok_choy' in args[1]: + paths = ['bok_choy'] + rm = ReportMerge() + rm.merge(paths, output_file=args[2]) + elif 'unit' in args[1]: + paths = ['common', 'cms', 'lms'] + for pth in paths: + rm = ReportMerge() + mp = multiprocessing.Process(target=rm.merge, args=([pth],)) + mp.start() + else: + print 'Unsupported Test Suit' diff --git a/scripts/coverage.sh b/scripts/coverage.sh new file mode 100755 index 0000000000..3e91295277 --- /dev/null +++ b/scripts/coverage.sh @@ -0,0 +1,25 @@ +#!/bin/bash + + +case $1 in + "shard1") + echo "Collecting Coverage for Bok-Choy Shard1" + paver bokchoy_coverage + echo "Merging Coverage into a Single HTML File for Bok-Choy Shard1" + python ./scripts/cov_merge.py bok_choy bok_choy_shard1_coverage.html + ;; + "shard2") + echo "Collecting Coverage for Bok-Choy Shard2" + paver bokchoy_coverage + echo "Merging Coverage into a Single HTML File for Bok-Choy Shard2" + python ./scripts/cov_merge.py bok_choy bok_choy_shard2_coverage.html + ;; + "shard3") + echo "Collecting Coverage for Bok-Choy Shard3" + paver bokchoy_coverage + echo "Merging Coverage into a Single HTML File for Bok-Choy Shard3" + python ./scripts/cov_merge.py bok_choy bok_choy_shard3_coverage.html + ;; + *) + echo "Invalid Bok-Choy Shard Value!";; +esac diff --git a/scripts/run_ut.sh b/scripts/run_ut.sh new file mode 100755 index 0000000000..fddacc9f65 --- /dev/null +++ b/scripts/run_ut.sh @@ -0,0 +1,21 @@ +mkdir -p jscover-dist && wget http://files.edx.org/testeng/JSCover-1.0.2.zip -P jscover-dist && unzip jscover-dist/JSCover-1.0.2.zip -d jscover-dist/ && cp jscover-dist/target/dist/JSCover-all.jar jscover-dist && export JSCOVER_JAR=$PWD/jscover-dist/JSCover-all.jar && paver test + +echo '******************************************************' + +echo 'Collecting Coverage...' + +paver coverage + +echo 'Coverage Collection Completed' + + +current_path=`pwd` +reports_path=$current_path/reports +dest_path=$HOME/results/$TDDIUM_SESSION_ID/session/ +unit_combined_rpt=$reports_path/diff_coverage_combined.html + +echo 'Copying '$unit_combined_rpt' to '$dest_path + +cp -f $unit_combined_rpt $dest_path + +echo '******************************************************' diff --git a/tddium.yml b/tddium.yml index 4c97d149b4..19a218fe68 100644 --- a/tddium.yml +++ b/tddium.yml @@ -7,7 +7,6 @@ tddium: :version: "1.8.5.5" :hooks: :pre_setup: "virtualenv $HOME/python-env && $HOME/python-env/bin/pip install -r requirements/edx/paver.txt && $HOME/python-env/bin/pip install -r requirements/edx/pre.txt && $HOME/python-env/bin/pip install -r requirements/edx/base.txt && $HOME/python-env/bin/pip install -r requirements/edx/github.txt && $HOME/python-env/bin/pip install -r requirements/edx/local.txt && $HOME/python-env/bin/pip install -r requirements/edx/post.txt" - # :post_build: "paver coverage; paver bokchoy_coverage; for i in $(find reports -name cover); do cp -R $i $HOME/results/$TDDIUM_SESSION_ID/session/; done" :post_worker: 'python ./scripts/post_worker.py' :cache: :key_paths: @@ -37,7 +36,7 @@ tddium: - "reports/diff_quality/diff_quality_pylint.html" - :type: junit :mode: basic - :command: "mkdir -p jscover-dist && wget http://files.edx.org/testeng/JSCover-1.0.2.zip -P jscover-dist && unzip jscover-dist/JSCover-1.0.2.zip -d jscover-dist/ && cp jscover-dist/target/dist/JSCover-all.jar jscover-dist && export JSCOVER_JAR=$PWD/jscover-dist/JSCover-all.jar && paver test" + :command: bash ./scripts/run_ut.sh && python ./scripts/cov_merge.py unit :invocation: single :output: exit-status :report_files: @@ -80,21 +79,21 @@ tddium: - "reports/acceptance/cms.xml" - :type: junit :mode: basic - :command: paver test_bokchoy --extra_args="-a shard_1" + :command: paver test_bokchoy --extra_args="-a shard_1" && bash ./scripts/coverage.sh shard1 :invocation: single :output: exit-status :report_files: - "reports/bok_choy/xunit.xml" - :type: junit :mode: basic - :command: paver test_bokchoy --extra_args="-a shard_2" + :command: paver test_bokchoy --extra_args="-a shard_2" && bash ./scripts/coverage.sh shard2 :invocation: single :output: exit-status :report_files: - "reports/bok_choy/xunit.xml" - :type: junit :mode: basic - :command: paver test_bokchoy --extra_args="-a shard_1=False,shard_2=False" + :command: paver test_bokchoy --extra_args="-a shard_1=False,shard_2=False" && bash ./scripts/coverage.sh shard3 :invocation: single :output: exit-status :report_files: From 3a6a519bac5b327a02256a535d2d24db1a433d90 Mon Sep 17 00:00:00 2001 From: Ben Patterson Date: Fri, 26 Sep 2014 10:36:33 -0400 Subject: [PATCH 10/11] Upgrade to mongo 2.6.4 --- tddium.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tddium.yml b/tddium.yml index 19a218fe68..336d25a429 100644 --- a/tddium.yml +++ b/tddium.yml @@ -18,7 +18,7 @@ tddium: - requirements/edx/post.txt - package.json :mongodb: - :version: "2.4.7" + :version: "2.6.4" :mysql: :version: "5.5" :python: From 03b9ed744fb0ca608a47227510853eab8a615ce4 Mon Sep 17 00:00:00 2001 From: Muhammad Ammar Date: Fri, 26 Sep 2014 12:01:24 +0500 Subject: [PATCH 11/11] Publish datadog stats for build TE-453 --- scripts/metrics/__init__.py | 0 scripts/metrics/acceptance_test_groups.json | 6 + scripts/metrics/coverage_metrics.py | 221 ++++++++++++++++++++ scripts/metrics/publish.py | 53 +++++ scripts/metrics/unit_test_groups.json | 7 + tddium.yml | 2 +- 6 files changed, 288 insertions(+), 1 deletion(-) create mode 100644 scripts/metrics/__init__.py create mode 100644 scripts/metrics/acceptance_test_groups.json create mode 100644 scripts/metrics/coverage_metrics.py create mode 100644 scripts/metrics/publish.py create mode 100644 scripts/metrics/unit_test_groups.json diff --git a/scripts/metrics/__init__.py b/scripts/metrics/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/metrics/acceptance_test_groups.json b/scripts/metrics/acceptance_test_groups.json new file mode 100644 index 0000000000..00cadb69eb --- /dev/null +++ b/scripts/metrics/acceptance_test_groups.json @@ -0,0 +1,6 @@ +{ + "acceptance.lms": "lms/*.py", + "acceptance.studio": "cms/*.py", + "acceptance.common_lib": "common/lib/*.py", + "acceptance.common_app": "common/djangoapps/*.py" +} diff --git a/scripts/metrics/coverage_metrics.py b/scripts/metrics/coverage_metrics.py new file mode 100644 index 0000000000..44c252dfcc --- /dev/null +++ b/scripts/metrics/coverage_metrics.py @@ -0,0 +1,221 @@ +""" +Aggregate coverage data from XML reports. + +groups.json is a JSON-encoded dict mapping group names to source file glob patterns: +{ + "group_1": "group1/*.py", + "group_2": "group2/*.py" +} + +This would calculate line coverage percentages for source files in each group, +and send those metrics to DataDog: + +testeng.coverage.group_1 ==> 89.123 +testeng.coverage.group_2 ==> 45.523 + +The tool uses the *union* of covered lines across each of the input +coverage XML reports. If a line is covered *anywhere*, it's considered covered. +""" + +import fnmatch +import json +from lxml import etree + + +class CoverageParseError(Exception): + """ + Error occurred while parsing a coverage report. + """ + pass + + +class CoverageData(object): + """ + Aggregate coverage reports. + """ + + def __init__(self): + """ + Initialize the coverage data, which has no information until you add a report. + """ + self._coverage = dict() + + def add_report(self, report_str): + """ + Add the coverage information from the XML `report_str` to the aggregate data. + Raises a `CoverageParseError` if the report XML is not a valid coverage report. + """ + try: + root = etree.fromstring(report_str) + + except etree.XMLSyntaxError: + raise CoverageParseError("Warning: Could not parse report as XML") + + if root is not None: + + # Get all classes (source files) in the report + for class_node in root.xpath('//class'): + + class_filename = class_node.get('filename') + + if class_filename is None: + continue + + # If we haven't seen this source file before, create a dict + # to store its coverage information. + if class_filename not in self._coverage: + self._coverage[class_filename] = dict() + + # Store info for each line in the source file + for line in class_node.xpath('lines/line'): + + hits = line.get('hits') + line_num = line.get('number') + + # Ignore lines that do not have the right attributes + if line_num is not None: + + try: + line_num = int(line_num) + hits = int(hits) + + except ValueError: + pass + + else: + + # If any report says the line is covered, set it to covered + if hits > 0: + self._coverage[class_filename][line_num] = 1 + + # Otherwise if the line is not already covered, set it to uncovered + elif line_num not in self._coverage[class_filename]: + self._coverage[class_filename][line_num] = 0 + + def coverage(self, source_pattern="*"): + """ + Calculate line coverage percentage (float) for source files that match + `source_pattern` (a fnmatch-style glob pattern). + + If coverage could not be calculated (e.g. because no source files match + the pattern), returns None. + """ + num_covered = 0 + total = 0 + + # Find source files that match the pattern then calculate total lines and number covered + for filename in fnmatch.filter(self._coverage.keys(), source_pattern): + num_covered += sum(self._coverage[filename].values()) + total += len(self._coverage[filename]) + + # Calculate the percentage + if total > 0: + return float(num_covered) / float(total) * 100.0 + + else: + print u"Warning: No lines found in source files that match {}".format(source_pattern) + return None + + @staticmethod + def _parse_report(report_path): + """ + Parse the coverage report as XML and return the resulting tree. + If the report could not be found or parsed, return None. + """ + try: + return etree.parse(report_path) + + except IOError: + print u"Warning: Could not open report at '{path}'".format(path=report_path) + return None + + except ValueError: + print u"Warning: Could not parse report at '{path}' as XML".format(path=report_path) + return None + + +class CoverageMetrics(object): + """ + Collect Coverage Reports for DataDog. + """ + + def __init__(self, group_json_path, report_paths): + self._group_json_path = group_json_path + self._report_paths = report_paths + + def coverage_metrics(self): + """ + Find, parse, and create coverage metrics to be sent to DataDog. + """ + print "Loading group definitions..." + group_dict = self.load_group_defs(self._group_json_path) + + print "Parsing reports..." + metrics = self.parse_reports(self._report_paths) + + print "Creating metrics..." + stats = self.create_metrics(metrics, group_dict) + print "Done." + + return stats + + @staticmethod + def load_group_defs(group_json_path): + """ + Load the dictionary mapping group names to source file patterns + from the file located at `group_json_path`. + + Exits with an error message if the groups could not be parsed. + """ + try: + with open(group_json_path) as json_file: + return json.load(json_file) + + except IOError: + print u"Could not open group definition file at '{}'".format(group_json_path) + raise + + except ValueError: + print u"Could not parse group definitions in '{}'".format(group_json_path) + raise + + @staticmethod + def parse_reports(report_paths): + """ + Parses each coverage report in `report_paths` and returns + a `CoverageData` object containing the aggregate coverage information. + """ + data = CoverageData() + + for path in report_paths: + + try: + with open(path) as report_file: + data.add_report(report_file.read()) + + except IOError: + print u"Warning: could not open {}".format(path) + + except CoverageParseError: + print u"Warning: could not parse {} as an XML coverage report".format(path) + + return data + + @staticmethod + def create_metrics(data, groups): + """ + Given a `CoverageData` object, create coverage percentages for each group. + + `groups` is a dict mapping aggregate group names to source file patterns. + Group names are used in the name of the metric sent to DataDog. + """ + metrics = {} + for group_name, pattern in groups.iteritems(): + metric = 'test_eng.coverage.{group}'.format(group=group_name.replace(' ', '_')) + percent = data.coverage(pattern) + + if percent is not None: + print u"Sending {} ==> {}%".format(metric, percent) + metrics[metric] = percent + + return metrics diff --git a/scripts/metrics/publish.py b/scripts/metrics/publish.py new file mode 100644 index 0000000000..9344658d6c --- /dev/null +++ b/scripts/metrics/publish.py @@ -0,0 +1,53 @@ +""" +Publish Build Stats. +""" + +import os +import subprocess +from dogapi import dog_http_api +from coverage_metrics import CoverageMetrics + + +class PublishStats(object): + """ + Publish stats to DataDog. + """ + def __init__(self, api_key): + dog_http_api.api_key = api_key + + @staticmethod + def report_metrics(metrics): + """ + Send metrics to DataDog. + + Arguments: + metrics (dict): data to publish + + """ + for key, value in metrics.iteritems(): + print u"Sending {} ==> {}%".format(key, value) + dog_http_api.metric(key, value) + + +def main(api_key): + """ + Send Stats for everything to DataDog. + """ + dir_path = os.path.dirname(os.path.relpath(__file__)) + + unit_reports_cmd = ['find', 'reports', '-name', '"coverage.xml"'] + unit_report_paths = subprocess.check_output(unit_reports_cmd) + + cov_metrics = CoverageMetrics(os.path.join(dir_path, 'unit_test_groups.json'), unit_report_paths) + coverage_metrics = cov_metrics.coverage_metrics() + + # Publish Coverage Stats to DataDog + PublishStats(api_key).report_metrics(coverage_metrics) + + +if __name__ == "__main__": + API_KEY = os.environ.get('DATADOG_API_KEY') + if API_KEY: + main(API_KEY) + else: + print 'SKIP: Publish Stats to Datadog' diff --git a/scripts/metrics/unit_test_groups.json b/scripts/metrics/unit_test_groups.json new file mode 100644 index 0000000000..d856b95dd8 --- /dev/null +++ b/scripts/metrics/unit_test_groups.json @@ -0,0 +1,7 @@ +{ + "unit.lms": "lms/*.py", + "unit.studio": "cms/*.py", + "unit.javascript": "*.js", + "unit.common_lib": "common/lib/*.py", + "unit.common_app": "common/djangoapps/*.py" +} diff --git a/tddium.yml b/tddium.yml index 336d25a429..93e5949e2a 100644 --- a/tddium.yml +++ b/tddium.yml @@ -36,7 +36,7 @@ tddium: - "reports/diff_quality/diff_quality_pylint.html" - :type: junit :mode: basic - :command: bash ./scripts/run_ut.sh && python ./scripts/cov_merge.py unit + :command: bash ./scripts/run_ut.sh && python ./scripts/cov_merge.py unit && python ./scripts/metrics/publish.py :invocation: single :output: exit-status :report_files: