diff --git a/scripts/metrics/__init__.py b/scripts/metrics/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/metrics/acceptance_test_groups.json b/scripts/metrics/acceptance_test_groups.json new file mode 100644 index 0000000000..00cadb69eb --- /dev/null +++ b/scripts/metrics/acceptance_test_groups.json @@ -0,0 +1,6 @@ +{ + "acceptance.lms": "lms/*.py", + "acceptance.studio": "cms/*.py", + "acceptance.common_lib": "common/lib/*.py", + "acceptance.common_app": "common/djangoapps/*.py" +} diff --git a/scripts/metrics/coverage_metrics.py b/scripts/metrics/coverage_metrics.py new file mode 100644 index 0000000000..44c252dfcc --- /dev/null +++ b/scripts/metrics/coverage_metrics.py @@ -0,0 +1,221 @@ +""" +Aggregate coverage data from XML reports. + +groups.json is a JSON-encoded dict mapping group names to source file glob patterns: +{ + "group_1": "group1/*.py", + "group_2": "group2/*.py" +} + +This would calculate line coverage percentages for source files in each group, +and send those metrics to DataDog: + +testeng.coverage.group_1 ==> 89.123 +testeng.coverage.group_2 ==> 45.523 + +The tool uses the *union* of covered lines across each of the input +coverage XML reports. If a line is covered *anywhere*, it's considered covered. +""" + +import fnmatch +import json +from lxml import etree + + +class CoverageParseError(Exception): + """ + Error occurred while parsing a coverage report. + """ + pass + + +class CoverageData(object): + """ + Aggregate coverage reports. + """ + + def __init__(self): + """ + Initialize the coverage data, which has no information until you add a report. + """ + self._coverage = dict() + + def add_report(self, report_str): + """ + Add the coverage information from the XML `report_str` to the aggregate data. + Raises a `CoverageParseError` if the report XML is not a valid coverage report. + """ + try: + root = etree.fromstring(report_str) + + except etree.XMLSyntaxError: + raise CoverageParseError("Warning: Could not parse report as XML") + + if root is not None: + + # Get all classes (source files) in the report + for class_node in root.xpath('//class'): + + class_filename = class_node.get('filename') + + if class_filename is None: + continue + + # If we haven't seen this source file before, create a dict + # to store its coverage information. + if class_filename not in self._coverage: + self._coverage[class_filename] = dict() + + # Store info for each line in the source file + for line in class_node.xpath('lines/line'): + + hits = line.get('hits') + line_num = line.get('number') + + # Ignore lines that do not have the right attributes + if line_num is not None: + + try: + line_num = int(line_num) + hits = int(hits) + + except ValueError: + pass + + else: + + # If any report says the line is covered, set it to covered + if hits > 0: + self._coverage[class_filename][line_num] = 1 + + # Otherwise if the line is not already covered, set it to uncovered + elif line_num not in self._coverage[class_filename]: + self._coverage[class_filename][line_num] = 0 + + def coverage(self, source_pattern="*"): + """ + Calculate line coverage percentage (float) for source files that match + `source_pattern` (a fnmatch-style glob pattern). + + If coverage could not be calculated (e.g. because no source files match + the pattern), returns None. + """ + num_covered = 0 + total = 0 + + # Find source files that match the pattern then calculate total lines and number covered + for filename in fnmatch.filter(self._coverage.keys(), source_pattern): + num_covered += sum(self._coverage[filename].values()) + total += len(self._coverage[filename]) + + # Calculate the percentage + if total > 0: + return float(num_covered) / float(total) * 100.0 + + else: + print u"Warning: No lines found in source files that match {}".format(source_pattern) + return None + + @staticmethod + def _parse_report(report_path): + """ + Parse the coverage report as XML and return the resulting tree. + If the report could not be found or parsed, return None. + """ + try: + return etree.parse(report_path) + + except IOError: + print u"Warning: Could not open report at '{path}'".format(path=report_path) + return None + + except ValueError: + print u"Warning: Could not parse report at '{path}' as XML".format(path=report_path) + return None + + +class CoverageMetrics(object): + """ + Collect Coverage Reports for DataDog. + """ + + def __init__(self, group_json_path, report_paths): + self._group_json_path = group_json_path + self._report_paths = report_paths + + def coverage_metrics(self): + """ + Find, parse, and create coverage metrics to be sent to DataDog. + """ + print "Loading group definitions..." + group_dict = self.load_group_defs(self._group_json_path) + + print "Parsing reports..." + metrics = self.parse_reports(self._report_paths) + + print "Creating metrics..." + stats = self.create_metrics(metrics, group_dict) + print "Done." + + return stats + + @staticmethod + def load_group_defs(group_json_path): + """ + Load the dictionary mapping group names to source file patterns + from the file located at `group_json_path`. + + Exits with an error message if the groups could not be parsed. + """ + try: + with open(group_json_path) as json_file: + return json.load(json_file) + + except IOError: + print u"Could not open group definition file at '{}'".format(group_json_path) + raise + + except ValueError: + print u"Could not parse group definitions in '{}'".format(group_json_path) + raise + + @staticmethod + def parse_reports(report_paths): + """ + Parses each coverage report in `report_paths` and returns + a `CoverageData` object containing the aggregate coverage information. + """ + data = CoverageData() + + for path in report_paths: + + try: + with open(path) as report_file: + data.add_report(report_file.read()) + + except IOError: + print u"Warning: could not open {}".format(path) + + except CoverageParseError: + print u"Warning: could not parse {} as an XML coverage report".format(path) + + return data + + @staticmethod + def create_metrics(data, groups): + """ + Given a `CoverageData` object, create coverage percentages for each group. + + `groups` is a dict mapping aggregate group names to source file patterns. + Group names are used in the name of the metric sent to DataDog. + """ + metrics = {} + for group_name, pattern in groups.iteritems(): + metric = 'test_eng.coverage.{group}'.format(group=group_name.replace(' ', '_')) + percent = data.coverage(pattern) + + if percent is not None: + print u"Sending {} ==> {}%".format(metric, percent) + metrics[metric] = percent + + return metrics diff --git a/scripts/metrics/publish.py b/scripts/metrics/publish.py new file mode 100644 index 0000000000..9344658d6c --- /dev/null +++ b/scripts/metrics/publish.py @@ -0,0 +1,53 @@ +""" +Publish Build Stats. +""" + +import os +import subprocess +from dogapi import dog_http_api +from coverage_metrics import CoverageMetrics + + +class PublishStats(object): + """ + Publish stats to DataDog. + """ + def __init__(self, api_key): + dog_http_api.api_key = api_key + + @staticmethod + def report_metrics(metrics): + """ + Send metrics to DataDog. + + Arguments: + metrics (dict): data to publish + + """ + for key, value in metrics.iteritems(): + print u"Sending {} ==> {}%".format(key, value) + dog_http_api.metric(key, value) + + +def main(api_key): + """ + Send Stats for everything to DataDog. + """ + dir_path = os.path.dirname(os.path.relpath(__file__)) + + unit_reports_cmd = ['find', 'reports', '-name', '"coverage.xml"'] + unit_report_paths = subprocess.check_output(unit_reports_cmd) + + cov_metrics = CoverageMetrics(os.path.join(dir_path, 'unit_test_groups.json'), unit_report_paths) + coverage_metrics = cov_metrics.coverage_metrics() + + # Publish Coverage Stats to DataDog + PublishStats(api_key).report_metrics(coverage_metrics) + + +if __name__ == "__main__": + API_KEY = os.environ.get('DATADOG_API_KEY') + if API_KEY: + main(API_KEY) + else: + print 'SKIP: Publish Stats to Datadog' diff --git a/scripts/metrics/unit_test_groups.json b/scripts/metrics/unit_test_groups.json new file mode 100644 index 0000000000..d856b95dd8 --- /dev/null +++ b/scripts/metrics/unit_test_groups.json @@ -0,0 +1,7 @@ +{ + "unit.lms": "lms/*.py", + "unit.studio": "cms/*.py", + "unit.javascript": "*.js", + "unit.common_lib": "common/lib/*.py", + "unit.common_app": "common/djangoapps/*.py" +} diff --git a/tddium.yml b/tddium.yml index 336d25a429..93e5949e2a 100644 --- a/tddium.yml +++ b/tddium.yml @@ -36,7 +36,7 @@ tddium: - "reports/diff_quality/diff_quality_pylint.html" - :type: junit :mode: basic - :command: bash ./scripts/run_ut.sh && python ./scripts/cov_merge.py unit + :command: bash ./scripts/run_ut.sh && python ./scripts/cov_merge.py unit && python ./scripts/metrics/publish.py :invocation: single :output: exit-status :report_files: