diff --git a/scripts/cov_merge.py b/scripts/cov_merge.py new file mode 100644 index 0000000000..8b4cd1c6d5 --- /dev/null +++ b/scripts/cov_merge.py @@ -0,0 +1,172 @@ +import os +import sys +from textwrap import dedent +from bs4 import BeautifulSoup +import multiprocessing + + +FIRST = dedent( + ''' + + + + CMS Python Test Coverage Report + + + + + + + + + ''') + + +LAST = dedent( + ''' + + + ''') + + +class ReportMerge(object): + """Merge multiple html coverage reports""" + + DESTINATION = os.path.join(os.environ['HOME'], 'results', os.environ['TDDIUM_SESSION_ID'], 'session') + + def __init__(self): + self.reports_dir = os.path.realpath(__file__).replace("scripts/cov_merge.py", "reports/") + + def _files(self, cover_path): + """ + Return list of file paths in `cover_path`. `cover_path` will be something like */reports/cms/cover + """ + include = lambda f: f.endswith('.html') and os.path.basename(f) != 'index.html' + return [os.path.join(cover_path, f) for f in os.listdir(cover_path) if include(f)] + + def merge(self, modules, output_file=None): + """ + Merge reports for `modules` + + Arguments: + output_file (str): name of output report file -- only used for bok_choy reports + + """ + for module in modules: + for (path, _, _) in os.walk(os.path.join(self.reports_dir, module)): + if os.path.basename(path) == 'cover': + self.merge_report(path, output_file) + + def merge_report(self, path, output_file): + """ + Collect multiple parts of a report and join them to create a single report. + + Arguments: + path (str): path where multiple files are located to be merged + output_file (str): name of output report file -- only used for bok_choy reports + + """ + content = list() + + # Extract total coverage percentage and file links table + index_html = os.path.join(path, 'index.html') + with open(index_html) as index_file: + soup = BeautifulSoup(index_file) + total_percentage = soup.find('div', id='header') + total_percentage.find('img').decompose() + index_table = soup.find('div', id='index') + + # Extract file names + files = [os.path.join(path, name['href']) for name in index_table.find_all('a')] + if not files: + return + + print 'Merging Report for {}'.format(path) + + # Collect different parts of html report + content.append(FIRST) + content.append('') + content.append(str(total_percentage)) + content.append(str(index_table)) + for html in files: + content.append(self._html_content(html)) + + content.append(LAST) + + if output_file: + report_path = os.path.join(self.DESTINATION, output_file) + else: + report_filename = path.split('reports/')[1].split('/cover')[0].replace('/', '_') + report_path = os.path.join(self.DESTINATION, report_filename+'_coverage.html') + + # Write everything to single report file + with open(report_path, 'w') as report_file: + report_file.write('\n'.join(content)) + + print 'Report Merged for {}'.format(path) + + def _html_content(self, html): + """ + Returns html tags of interest for file specified by `html` + """ + # Create id for each link in file links table + navigate_div_id = os.path.basename(html).split('.')[0].replace('/', '_') + navigate_div_start = "
\n".format(navigate_div_id) + navigate_div_close = "\n
".format(navigate_div_id) + + content = list() + content.append(navigate_div_start) + + with open(html) as html_file: + soup = BeautifulSoup(html_file) + header = soup.find('div', id='header') + header.find('img').decompose() + source = soup.find('div', id='source') + source_img = source.find('img') + if source_img: + source_img.decompose() + + content.append(str(header)) + content.append(str(source)) + + content.append(navigate_div_close) + + return '\n'.join(content) + +if __name__ == '__main__': + args = sys.argv + + if 'bok_choy' in args[1]: + paths = ['bok_choy'] + rm = ReportMerge() + rm.merge(paths, output_file=args[2]) + elif 'unit' in args[1]: + paths = ['common', 'cms', 'lms'] + for pth in paths: + rm = ReportMerge() + mp = multiprocessing.Process(target=rm.merge, args=([pth],)) + mp.start() + else: + print 'Unsupported Test Suit' diff --git a/scripts/coverage.sh b/scripts/coverage.sh new file mode 100755 index 0000000000..3e91295277 --- /dev/null +++ b/scripts/coverage.sh @@ -0,0 +1,25 @@ +#!/bin/bash + + +case $1 in + "shard1") + echo "Collecting Coverage for Bok-Choy Shard1" + paver bokchoy_coverage + echo "Merging Coverage into a Single HTML File for Bok-Choy Shard1" + python ./scripts/cov_merge.py bok_choy bok_choy_shard1_coverage.html + ;; + "shard2") + echo "Collecting Coverage for Bok-Choy Shard2" + paver bokchoy_coverage + echo "Merging Coverage into a Single HTML File for Bok-Choy Shard2" + python ./scripts/cov_merge.py bok_choy bok_choy_shard2_coverage.html + ;; + "shard3") + echo "Collecting Coverage for Bok-Choy Shard3" + paver bokchoy_coverage + echo "Merging Coverage into a Single HTML File for Bok-Choy Shard3" + python ./scripts/cov_merge.py bok_choy bok_choy_shard3_coverage.html + ;; + *) + echo "Invalid Bok-Choy Shard Value!";; +esac diff --git a/scripts/metrics/__init__.py b/scripts/metrics/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/metrics/acceptance_test_groups.json b/scripts/metrics/acceptance_test_groups.json new file mode 100644 index 0000000000..00cadb69eb --- /dev/null +++ b/scripts/metrics/acceptance_test_groups.json @@ -0,0 +1,6 @@ +{ + "acceptance.lms": "lms/*.py", + "acceptance.studio": "cms/*.py", + "acceptance.common_lib": "common/lib/*.py", + "acceptance.common_app": "common/djangoapps/*.py" +} diff --git a/scripts/metrics/coverage_metrics.py b/scripts/metrics/coverage_metrics.py new file mode 100644 index 0000000000..44c252dfcc --- /dev/null +++ b/scripts/metrics/coverage_metrics.py @@ -0,0 +1,221 @@ +""" +Aggregate coverage data from XML reports. + +groups.json is a JSON-encoded dict mapping group names to source file glob patterns: +{ + "group_1": "group1/*.py", + "group_2": "group2/*.py" +} + +This would calculate line coverage percentages for source files in each group, +and send those metrics to DataDog: + +testeng.coverage.group_1 ==> 89.123 +testeng.coverage.group_2 ==> 45.523 + +The tool uses the *union* of covered lines across each of the input +coverage XML reports. If a line is covered *anywhere*, it's considered covered. +""" + +import fnmatch +import json +from lxml import etree + + +class CoverageParseError(Exception): + """ + Error occurred while parsing a coverage report. + """ + pass + + +class CoverageData(object): + """ + Aggregate coverage reports. + """ + + def __init__(self): + """ + Initialize the coverage data, which has no information until you add a report. + """ + self._coverage = dict() + + def add_report(self, report_str): + """ + Add the coverage information from the XML `report_str` to the aggregate data. + Raises a `CoverageParseError` if the report XML is not a valid coverage report. + """ + try: + root = etree.fromstring(report_str) + + except etree.XMLSyntaxError: + raise CoverageParseError("Warning: Could not parse report as XML") + + if root is not None: + + # Get all classes (source files) in the report + for class_node in root.xpath('//class'): + + class_filename = class_node.get('filename') + + if class_filename is None: + continue + + # If we haven't seen this source file before, create a dict + # to store its coverage information. + if class_filename not in self._coverage: + self._coverage[class_filename] = dict() + + # Store info for each line in the source file + for line in class_node.xpath('lines/line'): + + hits = line.get('hits') + line_num = line.get('number') + + # Ignore lines that do not have the right attributes + if line_num is not None: + + try: + line_num = int(line_num) + hits = int(hits) + + except ValueError: + pass + + else: + + # If any report says the line is covered, set it to covered + if hits > 0: + self._coverage[class_filename][line_num] = 1 + + # Otherwise if the line is not already covered, set it to uncovered + elif line_num not in self._coverage[class_filename]: + self._coverage[class_filename][line_num] = 0 + + def coverage(self, source_pattern="*"): + """ + Calculate line coverage percentage (float) for source files that match + `source_pattern` (a fnmatch-style glob pattern). + + If coverage could not be calculated (e.g. because no source files match + the pattern), returns None. + """ + num_covered = 0 + total = 0 + + # Find source files that match the pattern then calculate total lines and number covered + for filename in fnmatch.filter(self._coverage.keys(), source_pattern): + num_covered += sum(self._coverage[filename].values()) + total += len(self._coverage[filename]) + + # Calculate the percentage + if total > 0: + return float(num_covered) / float(total) * 100.0 + + else: + print u"Warning: No lines found in source files that match {}".format(source_pattern) + return None + + @staticmethod + def _parse_report(report_path): + """ + Parse the coverage report as XML and return the resulting tree. + If the report could not be found or parsed, return None. + """ + try: + return etree.parse(report_path) + + except IOError: + print u"Warning: Could not open report at '{path}'".format(path=report_path) + return None + + except ValueError: + print u"Warning: Could not parse report at '{path}' as XML".format(path=report_path) + return None + + +class CoverageMetrics(object): + """ + Collect Coverage Reports for DataDog. + """ + + def __init__(self, group_json_path, report_paths): + self._group_json_path = group_json_path + self._report_paths = report_paths + + def coverage_metrics(self): + """ + Find, parse, and create coverage metrics to be sent to DataDog. + """ + print "Loading group definitions..." + group_dict = self.load_group_defs(self._group_json_path) + + print "Parsing reports..." + metrics = self.parse_reports(self._report_paths) + + print "Creating metrics..." + stats = self.create_metrics(metrics, group_dict) + print "Done." + + return stats + + @staticmethod + def load_group_defs(group_json_path): + """ + Load the dictionary mapping group names to source file patterns + from the file located at `group_json_path`. + + Exits with an error message if the groups could not be parsed. + """ + try: + with open(group_json_path) as json_file: + return json.load(json_file) + + except IOError: + print u"Could not open group definition file at '{}'".format(group_json_path) + raise + + except ValueError: + print u"Could not parse group definitions in '{}'".format(group_json_path) + raise + + @staticmethod + def parse_reports(report_paths): + """ + Parses each coverage report in `report_paths` and returns + a `CoverageData` object containing the aggregate coverage information. + """ + data = CoverageData() + + for path in report_paths: + + try: + with open(path) as report_file: + data.add_report(report_file.read()) + + except IOError: + print u"Warning: could not open {}".format(path) + + except CoverageParseError: + print u"Warning: could not parse {} as an XML coverage report".format(path) + + return data + + @staticmethod + def create_metrics(data, groups): + """ + Given a `CoverageData` object, create coverage percentages for each group. + + `groups` is a dict mapping aggregate group names to source file patterns. + Group names are used in the name of the metric sent to DataDog. + """ + metrics = {} + for group_name, pattern in groups.iteritems(): + metric = 'test_eng.coverage.{group}'.format(group=group_name.replace(' ', '_')) + percent = data.coverage(pattern) + + if percent is not None: + print u"Sending {} ==> {}%".format(metric, percent) + metrics[metric] = percent + + return metrics diff --git a/scripts/metrics/publish.py b/scripts/metrics/publish.py new file mode 100644 index 0000000000..9344658d6c --- /dev/null +++ b/scripts/metrics/publish.py @@ -0,0 +1,53 @@ +""" +Publish Build Stats. +""" + +import os +import subprocess +from dogapi import dog_http_api +from coverage_metrics import CoverageMetrics + + +class PublishStats(object): + """ + Publish stats to DataDog. + """ + def __init__(self, api_key): + dog_http_api.api_key = api_key + + @staticmethod + def report_metrics(metrics): + """ + Send metrics to DataDog. + + Arguments: + metrics (dict): data to publish + + """ + for key, value in metrics.iteritems(): + print u"Sending {} ==> {}%".format(key, value) + dog_http_api.metric(key, value) + + +def main(api_key): + """ + Send Stats for everything to DataDog. + """ + dir_path = os.path.dirname(os.path.relpath(__file__)) + + unit_reports_cmd = ['find', 'reports', '-name', '"coverage.xml"'] + unit_report_paths = subprocess.check_output(unit_reports_cmd) + + cov_metrics = CoverageMetrics(os.path.join(dir_path, 'unit_test_groups.json'), unit_report_paths) + coverage_metrics = cov_metrics.coverage_metrics() + + # Publish Coverage Stats to DataDog + PublishStats(api_key).report_metrics(coverage_metrics) + + +if __name__ == "__main__": + API_KEY = os.environ.get('DATADOG_API_KEY') + if API_KEY: + main(API_KEY) + else: + print 'SKIP: Publish Stats to Datadog' diff --git a/scripts/metrics/unit_test_groups.json b/scripts/metrics/unit_test_groups.json new file mode 100644 index 0000000000..d856b95dd8 --- /dev/null +++ b/scripts/metrics/unit_test_groups.json @@ -0,0 +1,7 @@ +{ + "unit.lms": "lms/*.py", + "unit.studio": "cms/*.py", + "unit.javascript": "*.js", + "unit.common_lib": "common/lib/*.py", + "unit.common_app": "common/djangoapps/*.py" +} diff --git a/scripts/post_quality_build.sh b/scripts/post_quality_build.sh new file mode 100755 index 0000000000..0492ef48d6 --- /dev/null +++ b/scripts/post_quality_build.sh @@ -0,0 +1,15 @@ +current_path=`pwd` + +reports_path=$current_path/reports + +dest_path=$HOME/results/$TDDIUM_SESSION_ID/session/ + +echo "Getting Quality Reports... " + +pep8_rpt=$reports_path/diff_quality/diff_quality_pep8.html +pylint_rpt=$reports_path/diff_quality/diff_quality_pylint.html + +cp -f $pep8_rpt $dest_path +cp -f $pylint_rpt $dest_path + +echo "Reports can be found in "$dest_path diff --git a/scripts/post_worker.py b/scripts/post_worker.py new file mode 100644 index 0000000000..7ffc7e0f24 --- /dev/null +++ b/scripts/post_worker.py @@ -0,0 +1,47 @@ +import tarfile +import os +import shutil + +full_path = os.path.realpath(__file__) +source_dir = full_path.replace("scripts/post_worker.py", "reports/") +output_filename = full_path.replace("post_worker.py", "reports.tar.gz") + +print "source dir:", source_dir + +count = 0 + +# walk through every subdirectory & add the folder if it is not empty +with tarfile.open(output_filename, "w:gz") as tar: + for (path, dirs, files) in os.walk(source_dir): + if len(files) > 0: + print "tarring:", path + tar.add(path, arcname=os.path.basename(path)) + count += 1 + +tar.close() + +session_path = os.path.join( + os.environ['HOME'], + 'results', + os.environ['TDDIUM_SESSION_ID'], + 'session') + +file_dest = os.path.join(session_path, 'reports.tar.gz') + +# if the tar file is not empty, copy it to the proper place +if count > 0: + print 'copying tar file to:', file_dest + shutil.copyfile(output_filename, file_dest) + +# finding if there is any screenshot or log file +print 'attaching failed screenshots and logs (if any)' +for (path, dirs, files) in os.walk('test_root/log'): + for filename in files: + if filename.find('png') != -1 or filename.find('log') != -1: + filepath = os.path.join(path, filename) + print 'copying file:', filepath + destpath = os.path.join(session_path, filename) + print 'destination:', destpath + shutil.copyfile(filepath, destpath) + +print 'TDDIUM_SESSION_ID:', os.environ['TDDIUM_SESSION_ID'] diff --git a/scripts/run_ut.sh b/scripts/run_ut.sh new file mode 100755 index 0000000000..fddacc9f65 --- /dev/null +++ b/scripts/run_ut.sh @@ -0,0 +1,21 @@ +mkdir -p jscover-dist && wget http://files.edx.org/testeng/JSCover-1.0.2.zip -P jscover-dist && unzip jscover-dist/JSCover-1.0.2.zip -d jscover-dist/ && cp jscover-dist/target/dist/JSCover-all.jar jscover-dist && export JSCOVER_JAR=$PWD/jscover-dist/JSCover-all.jar && paver test + +echo '******************************************************' + +echo 'Collecting Coverage...' + +paver coverage + +echo 'Coverage Collection Completed' + + +current_path=`pwd` +reports_path=$current_path/reports +dest_path=$HOME/results/$TDDIUM_SESSION_ID/session/ +unit_combined_rpt=$reports_path/diff_coverage_combined.html + +echo 'Copying '$unit_combined_rpt' to '$dest_path + +cp -f $unit_combined_rpt $dest_path + +echo '******************************************************' diff --git a/tddium.yml b/tddium.yml new file mode 100644 index 0000000000..93e5949e2a --- /dev/null +++ b/tddium.yml @@ -0,0 +1,100 @@ +tddium: + :firefox: '28.0' + :timeout: 3600 + :timeout_hook: 900 + :tool_config: + git: + :version: "1.8.5.5" + :hooks: + :pre_setup: "virtualenv $HOME/python-env && $HOME/python-env/bin/pip install -r requirements/edx/paver.txt && $HOME/python-env/bin/pip install -r requirements/edx/pre.txt && $HOME/python-env/bin/pip install -r requirements/edx/base.txt && $HOME/python-env/bin/pip install -r requirements/edx/github.txt && $HOME/python-env/bin/pip install -r requirements/edx/local.txt && $HOME/python-env/bin/pip install -r requirements/edx/post.txt" + :post_worker: 'python ./scripts/post_worker.py' + :cache: + :key_paths: + - requirements/edx/paver.txt + - requirements/edx/pre.txt + - requirements/edx/base.txt + - requirements/edx/github.txt + - requirements/edx/local.txt + - requirements/edx/post.txt + - package.json + :mongodb: + :version: "2.6.4" + :mysql: + :version: "5.5" + :python: + :python_version: "2.7" + :headless: true + :java: + :java_version: "java-7-openjdk" + :test_pattern: 'none' + :tests: + - :type: custom + :command: paver run_quality && bash ./scripts/post_quality_build.sh + :invocation: single + :output: exit-status + :report_files: + - "reports/diff_quality/diff_quality_pylint.html" + - :type: junit + :mode: basic + :command: bash ./scripts/run_ut.sh && python ./scripts/cov_merge.py unit && python ./scripts/metrics/publish.py + :invocation: single + :output: exit-status + :report_files: + - "reports/cms/nosetests.xml" + - "reports/common/lib/capa/nosetests.xml" + - "reports/common/lib/calc/nosetests.xml" + - "reports/common/lib/chem/nosetests.xml" + - "reports/common/lib/sandbox-packages/nosetests.xml" + - "reports/common/lib/symmath/nosetests.xml" + - "reports/common/lib/xmodule/nosetests.xml" + - "reports/lms/nosetests.xml" + - "reports/javascript/javascript_xunit.xml" + - :type: junit + :mode: basic + :command: paver test_acceptance -s lms --extra_args="-v 3 --tag shard_1" + :invocation: single + :output: exit-status + :report_files: + - "reports/acceptance/lms.xml" + - :type: junit + :mode: basic + :command: paver test_acceptance -s lms --extra_args="-v 3 --tag shard_2" + :invocation: single + :output: exit-status + :report_files: + - "reports/acceptance/lms.xml" + - :type: junit + :mode: basic + :command: paver test_acceptance -s cms --extra_args="-v 3 --tag shard_1" + :invocation: single + :output: exit-status + :report_files: + - "reports/acceptance/cms.xml" + - :type: junit + :mode: basic + :command: paver test_acceptance -s cms --extra_args="-v 3 --tag shard_2 --tag shard_3" + :invocation: single + :output: exit-status + :report_files: + - "reports/acceptance/cms.xml" + - :type: junit + :mode: basic + :command: paver test_bokchoy --extra_args="-a shard_1" && bash ./scripts/coverage.sh shard1 + :invocation: single + :output: exit-status + :report_files: + - "reports/bok_choy/xunit.xml" + - :type: junit + :mode: basic + :command: paver test_bokchoy --extra_args="-a shard_2" && bash ./scripts/coverage.sh shard2 + :invocation: single + :output: exit-status + :report_files: + - "reports/bok_choy/xunit.xml" + - :type: junit + :mode: basic + :command: paver test_bokchoy --extra_args="-a shard_1=False,shard_2=False" && bash ./scripts/coverage.sh shard3 + :invocation: single + :output: exit-status + :report_files: + - "reports/bok_choy/xunit.xml"