diff --git a/common/lib/xmodule/xmodule/js/src/peergrading/peer_grading_problem.coffee b/common/lib/xmodule/xmodule/js/src/peergrading/peer_grading_problem.coffee
index 6785ab0541..7c2e9695b1 100644
--- a/common/lib/xmodule/xmodule/js/src/peergrading/peer_grading_problem.coffee
+++ b/common/lib/xmodule/xmodule/js/src/peergrading/peer_grading_problem.coffee
@@ -185,6 +185,11 @@ class @PeerGradingProblem
     @question_header = $('.question-header')
     @question_header.click @collapse_question
     @flag_submission_confirmation = $('.flag-submission-confirmation')
+    @flag_submission_confirmation_button = $('.flag-submission-confirmation-button')
+    @flag_submission_removal_button = $('.flag-submission-removal-button')
+
+    @flag_submission_confirmation_button.click @close_dialog_box
+    @flag_submission_removal_button.click @remove_flag
 
     @grading_wrapper =$('.grading-wrapper')
     @calibration_feedback_panel = $('.calibration-feedback')
@@ -290,9 +295,16 @@ class @PeerGradingProblem
   #
   ##########
 
+  remove_flag: () =>
+    @flag_student_checkbox.removeAttr("checked")
+    @close_dialog_box()
+
+  close_dialog_box: () =>
+    $( ".flag-submission-confirmation" ).dialog('close')
+
   flag_box_checked: () =>
     if @flag_student_checkbox.is(':checked')
-      $( ".flag-submission-confirmation" ).dialog()
+      $( ".flag-submission-confirmation" ).dialog({ height: 400, width: 400 })
 
   # called after we perform an is_student_calibrated check
   calibration_check_callback: (response) =>
diff --git a/lms/templates/peer_grading/peer_grading_problem.html b/lms/templates/peer_grading/peer_grading_problem.html
index aa74fd9a6b..b4b900b874 100644
--- a/lms/templates/peer_grading/peer_grading_problem.html
+++ b/lms/templates/peer_grading/peer_grading_problem.html
@@ -85,8 +85,13 @@
     <!-- Flag submission confirmation dialog -->
     <section class="flag-submission-confirmation">
         <h4> Are you sure that you want to flag this submission?</h4>
-        <br/>
-        Please only flag explicit or pornographic content.
+        <p>
+        Please only flag explicit or pornographic content.  Click the button below if you understand this and still want to flag the submission.
+        </p>
+        <div>
+        <input type="button" class="flag-submission-removal-button" value="Remove Flag" name="calibration-interstitial-page-button" />
+        <input type="button" class="flag-submission-confirmation-button" value="Keep Flag" name="calibration-interstitial-page-button" />
+        </div>
     </section>