Merge pull request #3162 from cpennington/problem-check-metrics

Capture metrics for problem attempts and histograms of problem grades

common/lib/xmodule/xmodule/capa_base.py

@@ -9,6 +9,13 @@ import traceback
 import struct
 import sys
 
+# We don't want to force a dependency on datadog, so make the import conditional
+try:
+    from dogapi import dog_stats_api
+except ImportError:
+    # pylint: disable=invalid-name
+    dog_stats_api = None
+
 from pkg_resources import resource_string
 
 from capa.capa_problem import LoncapaProblem, LoncapaSystem
@@ -869,18 +876,24 @@ class CapaMixin(CapaFields):
         answers_without_files = convert_files_to_filenames(answers)
         event_info['answers'] = answers_without_files
 
+        metric_name = 'capa.check_problem'
+
         _ = self.runtime.service(self, "i18n").ugettext
 
         # Too late. Cannot submit
         if self.closed():
             event_info['failure'] = 'closed'
             self.runtime.track_function('problem_check_fail', event_info)
+            if dog_stats_api:
+                dog_stats_api.increment(metric_name, [u'result:failed', u'failure:closed'])
             raise NotFoundError(_("Problem is closed."))
 
         # Problem submitted. Student should reset before checking again
         if self.done and self.rerandomize == "always":
             event_info['failure'] = 'unreset'
             self.runtime.track_function('problem_check_fail', event_info)
+            if dog_stats_api:
+                dog_stats_api.increment(metric_name, [u'result:failed', u'failure:unreset'])
             raise NotFoundError(_("Problem must be reset before it can be checked again."))
 
         # Problem queued. Students must wait a specified waittime before they are allowed to submit
@@ -948,6 +961,17 @@ class CapaMixin(CapaFields):
         event_info['submission'] = self.get_submission_metadata_safe(answers_without_files, correct_map)
         self.runtime.track_function('problem_check', event_info)
 
+        if dog_stats_api:
+            dog_stats_api.increment(metric_name, [u'result:success'])
+            dog_stats_api.histogram(
+                u'{}.correct_pct'.format(metric_name),
+                float(published_grade['grade']) / published_grade['max_grade'],
+            )
+            dog_stats_api.histogram(
+                u'{}.attempts'.format(metric_name),
+                self.attempts,
+            )
+
         if hasattr(self.runtime, 'psychometrics_handler'):  # update PsychometricsData using callback
             self.runtime.psychometrics_handler(self.get_state_for_lcp())