Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1865 from MITx/feature/will/more_capa_tests

Browse Source 
Feature/will/more capa tests
This commit is contained in:
Will Daly
2013-04-17 12:38:59 -07:00
parent a3233b67e5 52c2f3ae37
commit e5e0dc4bfb
4 changed files with 405 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

269
common/lib/capa/capa/tests/test_input_templates.py Normal file
View File

@@ -0,0 +1,269 @@
"""Tests for the logic in input type mako templates."""
import unittest
import capa
import os.path
from lxml import etree
from mako.template import Template as MakoTemplate
class TemplateTestCase(unittest.TestCase):
"""Utilitites for testing templates"""
# Subclasses override this to specify the file name of the template
# to be loaded from capa/templates.
# The template name should include the .html extension:
# for example: choicegroup.html
TEMPLATE_NAME = None
def setUp(self):
"""Load the template"""
capa_path = capa.__path__[0]
self.template_path = os.path.join(capa_path,
'templates',
self.TEMPLATE_NAME)
template_file = open(self.template_path)
self.template = MakoTemplate(template_file.read())
template_file.close()
def render_to_xml(self, context_dict):
"""Render the template using the `context_dict` dict.
Returns an `etree` XML element."""
xml_str = self.template.render_unicode(**context_dict)
return etree.fromstring(xml_str)
def assert_has_xpath(self, xml_root, xpath, context_dict, exact_num=1):
"""Asserts that the xml tree has an element satisfying `xpath`.
`xml_root` is an etree XML element
`xpath` is an XPath string, such as `'/foo/bar'`
`context` is used to print a debugging message
`exact_num` is the exact number of matches to expect.
"""
message = ("XML does not have %d match(es) for xpath '%s'\nXML: %s\nContext: %s"
% (exact_num, str(xpath), etree.tostring(xml_root), str(context_dict)))
self.assertEqual(len(xml_root.xpath(xpath)), exact_num, msg=message)
def assert_no_xpath(self, xml_root, xpath, context_dict):
"""Asserts that the xml tree does NOT have an element
satisfying `xpath`.
`xml_root` is an etree XML element
`xpath` is an XPath string, such as `'/foo/bar'`
`context` is used to print a debugging message
"""
self.assert_has_xpath(xml_root, xpath, context_dict, exact_num=0)
class TestChoiceGroupTemplate(TemplateTestCase):
"""Test mako template for `<choicegroup>` input"""
TEMPLATE_NAME = 'choicegroup.html'
def setUp(self):
choices = [('1', 'choice 1'), ('2', 'choice 2'), ('3', 'choice 3')]
self.context = {'id': '1',
'choices': choices,
'status': 'correct',
'input_type': 'checkbox',
'name_array_suffix': '1',
'value': '3'}
super(TestChoiceGroupTemplate, self).setUp()
def test_problem_marked_correct(self):
"""Test conditions under which the entire problem
(not a particular option) is marked correct"""
self.context['status'] = 'correct'
self.context['input_type'] = 'checkbox'
self.context['value'] = ['1', '2']
# Should mark the entire problem correct
xml = self.render_to_xml(self.context)
xpath = "//div[@class='indicator_container']/span[@class='correct']"
self.assert_has_xpath(xml, xpath, self.context)
# Should NOT mark individual options
self.assert_no_xpath(xml, "//label[@class='choicegroup_incorrect']",
self.context)
self.assert_no_xpath(xml, "//label[@class='choicegroup_correct']",
self.context)
def test_problem_marked_incorrect(self):
"""Test all conditions under which the entire problem
(not a particular option) is marked incorrect"""
conditions = [
{'status': 'incorrect', 'input_type': 'radio', 'value': ''},
{'status': 'incorrect', 'input_type': 'checkbox', 'value': []},
{'status': 'incorrect', 'input_type': 'checkbox', 'value': ['2']},
{'status': 'incorrect', 'input_type': 'checkbox', 'value': ['2', '3']},
{'status': 'incomplete', 'input_type': 'radio', 'value': ''},
{'status': 'incomplete', 'input_type': 'checkbox', 'value': []},
{'status': 'incomplete', 'input_type': 'checkbox', 'value': ['2']},
{'status': 'incomplete', 'input_type': 'checkbox', 'value': ['2', '3']}]
for test_conditions in conditions:
self.context.update(test_conditions)
xml = self.render_to_xml(self.context)
xpath = "//div[@class='indicator_container']/span[@class='incorrect']"
self.assert_has_xpath(xml, xpath, self.context)
# Should NOT mark individual options
self.assert_no_xpath(xml,
"//label[@class='choicegroup_incorrect']",
self.context)
self.assert_no_xpath(xml,
"//label[@class='choicegroup_correct']",
self.context)
def test_problem_marked_unanswered(self):
"""Test all conditions under which the entire problem
(not a particular option) is marked unanswered"""
conditions = [
{'status': 'unsubmitted', 'input_type': 'radio', 'value': ''},
{'status': 'unsubmitted', 'input_type': 'radio', 'value': []},
{'status': 'unsubmitted', 'input_type': 'checkbox', 'value': []},
{'input_type': 'radio', 'value': ''},
{'input_type': 'radio', 'value': []},
{'input_type': 'checkbox', 'value': []},
{'input_type': 'checkbox', 'value': ['1']},
{'input_type': 'checkbox', 'value': ['1', '2']}]
self.context['status'] = 'unanswered'
for test_conditions in conditions:
self.context.update(test_conditions)
xml = self.render_to_xml(self.context)
xpath = "//div[@class='indicator_container']/span[@class='unanswered']"
self.assert_has_xpath(xml, xpath, self.context)
# Should NOT mark individual options
self.assert_no_xpath(xml,
"//label[@class='choicegroup_incorrect']",
self.context)
self.assert_no_xpath(xml,
"//label[@class='choicegroup_correct']",
self.context)
def test_option_marked_correct(self):
"""Test conditions under which a particular option
(not the entire problem) is marked correct."""
conditions = [
{'input_type': 'radio', 'value': '2'},
{'input_type': 'radio', 'value': ['2']}]
self.context['status'] = 'correct'
for test_conditions in conditions:
self.context.update(test_conditions)
xml = self.render_to_xml(self.context)
xpath = "//label[@class='choicegroup_correct']"
self.assert_has_xpath(xml, xpath, self.context)
# Should NOT mark the whole problem
xpath = "//div[@class='indicator_container']/span"
self.assert_no_xpath(xml, xpath, self.context)
def test_option_marked_incorrect(self):
"""Test conditions under which a particular option
(not the entire problem) is marked incorrect."""
conditions = [
{'input_type': 'radio', 'value': '2'},
{'input_type': 'radio', 'value': ['2']}]
self.context['status'] = 'incorrect'
for test_conditions in conditions:
self.context.update(test_conditions)
xml = self.render_to_xml(self.context)
xpath = "//label[@class='choicegroup_incorrect']"
self.assert_has_xpath(xml, xpath, self.context)
# Should NOT mark the whole problem
xpath = "//div[@class='indicator_container']/span"
self.assert_no_xpath(xml, xpath, self.context)
def test_never_show_correctness(self):
"""Test conditions under which we tell the template to
NOT show correct/incorrect, but instead show a message.
This is used, for example, by the Justice course to ask
questions without specifying a correct answer. When
the student responds, the problem displays "Thank you
for your response"
"""
conditions = [
{'input_type': 'radio', 'status': 'correct', 'value': ''},
{'input_type': 'radio', 'status': 'correct', 'value': '2'},
{'input_type': 'radio', 'status': 'correct', 'value': ['2']},
{'input_type': 'radio', 'status': 'incorrect', 'value': '2'},
{'input_type': 'radio', 'status': 'incorrect', 'value': []},
{'input_type': 'radio', 'status': 'incorrect', 'value': ['2']},
{'input_type': 'checkbox', 'status': 'correct', 'value': []},
{'input_type': 'checkbox', 'status': 'correct', 'value': ['2']},
{'input_type': 'checkbox', 'status': 'incorrect', 'value': []},
{'input_type': 'checkbox', 'status': 'incorrect', 'value': ['2']}]
self.context['show_correctness'] = 'never'
self.context['submitted_message'] = 'Test message'
for test_conditions in conditions:
self.context.update(test_conditions)
xml = self.render_to_xml(self.context)
# Should NOT mark the entire problem correct/incorrect
xpath = "//div[@class='indicator_container']/span[@class='correct']"
self.assert_no_xpath(xml, xpath, self.context)
xpath = "//div[@class='indicator_container']/span[@class='incorrect']"
self.assert_no_xpath(xml, xpath, self.context)
# Should NOT mark individual options
self.assert_no_xpath(xml,
"//label[@class='choicegroup_incorrect']",
self.context)
self.assert_no_xpath(xml,
"//label[@class='choicegroup_correct']",
self.context)
# Expect to see the message
message_elements = xml.xpath("//div[@class='capa_alert']")
self.assertEqual(len(message_elements), 1)
self.assertEqual(message_elements[0].text,
self.context['submitted_message'])
def test_no_message_before_submission(self):
"""Ensure that we don't show the `submitted_message`
before submitting"""
conditions = [
{'input_type': 'radio', 'status': 'unsubmitted', 'value': ''},
{'input_type': 'radio', 'status': 'unsubmitted', 'value': []},
{'input_type': 'checkbox', 'status': 'unsubmitted', 'value': []},
# These tests expose bug #365
# When the bug is fixed, uncomment these cases.
#{'input_type': 'radio', 'status': 'unsubmitted', 'value': '2'},
#{'input_type': 'radio', 'status': 'unsubmitted', 'value': ['2']},
#{'input_type': 'radio', 'status': 'unsubmitted', 'value': '2'},
#{'input_type': 'radio', 'status': 'unsubmitted', 'value': ['2']},
#{'input_type': 'checkbox', 'status': 'unsubmitted', 'value': ['2']},
#{'input_type': 'checkbox', 'status': 'unsubmitted', 'value': ['2']}]
]
self.context['show_correctness'] = 'never'
self.context['submitted_message'] = 'Test message'
for test_conditions in conditions:
self.context.update(test_conditions)
xml = self.render_to_xml(self.context)
# Expect that we do NOT see the message yet
self.assert_no_xpath(xml, "//div[@class='capa_alert']", self.context)

155
common/lib/xmodule/xmodule/tests/test_capa_module.py
View File

@@ -1,17 +1,19 @@
"""Tests of the Capa XModule"""
#pylint: disable=C0111
#pylint: disable=R0904
#pylint: disable=C0103
#pylint: disable=C0302
import datetime
import json
from mock import Mock, MagicMock, patch
from pprint import pprint
from mock import Mock, patch
import unittest
import random
import xmodule
import capa
from capa.responsetypes import StudentInputError, \
LoncapaProblemError, ResponseError
LoncapaProblemError, ResponseError
from xmodule.capa_module import CapaModule
from xmodule.modulestore import Location
from lxml import etree
from django.http import QueryDict
@@ -384,7 +386,7 @@ class CapaModuleTest(unittest.TestCase):
# what the input is, by patching CorrectMap.is_correct()
# Also simulate rendering the HTML
# TODO: pep8 thinks the following line has invalid syntax
with patch('capa.correctmap.CorrectMap.is_correct') as mock_is_correct,\
with patch('capa.correctmap.CorrectMap.is_correct') as mock_is_correct, \
patch('xmodule.capa_module.CapaModule.get_problem_html') as mock_html:
mock_is_correct.return_value = True
mock_html.return_value = "Test HTML"
@@ -435,32 +437,38 @@ class CapaModuleTest(unittest.TestCase):
self.assertEqual(module.attempts, 3)
def test_check_problem_resubmitted_with_randomize(self):
# Randomize turned on
module = CapaFactory.create(rerandomize='always', attempts=0)
rerandomize_values = ['always', 'true']
# Simulate that the problem is completed
module.done = True
for rerandomize in rerandomize_values:
# Randomize turned on
module = CapaFactory.create(rerandomize=rerandomize, attempts=0)
# Expect that we cannot submit
with self.assertRaises(xmodule.exceptions.NotFoundError):
get_request_dict = {CapaFactory.input_key(): '3.14'}
module.check_problem(get_request_dict)
# Simulate that the problem is completed
module.done = True
# Expect that number of attempts NOT incremented
self.assertEqual(module.attempts, 0)
# Expect that we cannot submit
with self.assertRaises(xmodule.exceptions.NotFoundError):
get_request_dict = {CapaFactory.input_key(): '3.14'}
module.check_problem(get_request_dict)
# Expect that number of attempts NOT incremented
self.assertEqual(module.attempts, 0)
def test_check_problem_resubmitted_no_randomize(self):
# Randomize turned off
module = CapaFactory.create(rerandomize='never', attempts=0, done=True)
rerandomize_values = ['never', 'false', 'per_student']
# Expect that we can submit successfully
get_request_dict = {CapaFactory.input_key(): '3.14'}
result = module.check_problem(get_request_dict)
for rerandomize in rerandomize_values:
# Randomize turned off
module = CapaFactory.create(rerandomize=rerandomize, attempts=0, done=True)
self.assertEqual(result['success'], 'correct')
# Expect that we can submit successfully
get_request_dict = {CapaFactory.input_key(): '3.14'}
result = module.check_problem(get_request_dict)
# Expect that number of attempts IS incremented
self.assertEqual(module.attempts, 1)
self.assertEqual(result['success'], 'correct')
# Expect that number of attempts IS incremented
self.assertEqual(module.attempts, 1)
def test_check_problem_queued(self):
module = CapaFactory.create(attempts=1)
@@ -615,24 +623,34 @@ class CapaModuleTest(unittest.TestCase):
self.assertTrue('success' in result and not result['success'])
def test_save_problem_submitted_with_randomize(self):
module = CapaFactory.create(rerandomize='always', done=True)
# Try to save
get_request_dict = {CapaFactory.input_key(): '3.14'}
result = module.save_problem(get_request_dict)
# Capa XModule treats 'always' and 'true' equivalently
rerandomize_values = ['always', 'true']
# Expect that we cannot save
self.assertTrue('success' in result and not result['success'])
for rerandomize in rerandomize_values:
module = CapaFactory.create(rerandomize=rerandomize, done=True)
# Try to save
get_request_dict = {CapaFactory.input_key(): '3.14'}
result = module.save_problem(get_request_dict)
# Expect that we cannot save
self.assertTrue('success' in result and not result['success'])
def test_save_problem_submitted_no_randomize(self):
module = CapaFactory.create(rerandomize='never', done=True)
# Try to save
get_request_dict = {CapaFactory.input_key(): '3.14'}
result = module.save_problem(get_request_dict)
# Capa XModule treats 'false' and 'per_student' equivalently
rerandomize_values = ['never', 'false', 'per_student']
# Expect that we succeed
self.assertTrue('success' in result and result['success'])
for rerandomize in rerandomize_values:
module = CapaFactory.create(rerandomize=rerandomize, done=True)
# Try to save
get_request_dict = {CapaFactory.input_key(): '3.14'}
result = module.save_problem(get_request_dict)
# Expect that we succeed
self.assertTrue('success' in result and result['success'])
def test_check_button_name(self):
@@ -681,21 +699,30 @@ class CapaModuleTest(unittest.TestCase):
# If user submitted a problem but hasn't reset,
# do NOT show the check button
# Note: we can only reset when rerandomize="always"
# Note: we can only reset when rerandomize="always" or "true"
module = CapaFactory.create(rerandomize="always", done=True)
self.assertFalse(module.should_show_check_button())
module = CapaFactory.create(rerandomize="true", done=True)
self.assertFalse(module.should_show_check_button())
# Otherwise, DO show the check button
module = CapaFactory.create()
self.assertTrue(module.should_show_check_button())
# If the user has submitted the problem
# and we do NOT have a reset button, then we can show the check button
# Setting rerandomize to "never" ensures that the reset button
# Setting rerandomize to "never" or "false" ensures that the reset button
# is not shown
module = CapaFactory.create(rerandomize="never", done=True)
self.assertTrue(module.should_show_check_button())
module = CapaFactory.create(rerandomize="false", done=True)
self.assertTrue(module.should_show_check_button())
module = CapaFactory.create(rerandomize="per_student", done=True)
self.assertTrue(module.should_show_check_button())
def test_should_show_reset_button(self):
attempts = random.randint(1, 10)
@@ -712,6 +739,14 @@ class CapaModuleTest(unittest.TestCase):
module = CapaFactory.create(rerandomize="never", done=True)
self.assertFalse(module.should_show_reset_button())
# If we're NOT randomizing, then do NOT show the reset button
module = CapaFactory.create(rerandomize="per_student", done=True)
self.assertFalse(module.should_show_reset_button())
# If we're NOT randomizing, then do NOT show the reset button
module = CapaFactory.create(rerandomize="false", done=True)
self.assertFalse(module.should_show_reset_button())
# If the user hasn't submitted an answer yet,
# then do NOT show the reset button
module = CapaFactory.create(done=False)
@@ -742,13 +777,19 @@ class CapaModuleTest(unittest.TestCase):
module = CapaFactory.create(rerandomize="always", done=True)
self.assertFalse(module.should_show_save_button())
module = CapaFactory.create(rerandomize="true", done=True)
self.assertFalse(module.should_show_save_button())
# If the user has unlimited attempts and we are not randomizing,
# then do NOT show a save button
# because they can keep using "Check"
module = CapaFactory.create(max_attempts=None, rerandomize="never", done=False)
self.assertFalse(module.should_show_save_button())
module = CapaFactory.create(max_attempts=None, rerandomize="never", done=True)
module = CapaFactory.create(max_attempts=None, rerandomize="false", done=True)
self.assertFalse(module.should_show_save_button())
module = CapaFactory.create(max_attempts=None, rerandomize="per_student", done=True)
self.assertFalse(module.should_show_save_button())
# Otherwise, DO show the save button
@@ -759,6 +800,12 @@ class CapaModuleTest(unittest.TestCase):
module = CapaFactory.create(rerandomize="never", max_attempts=2, done=True)
self.assertTrue(module.should_show_save_button())
module = CapaFactory.create(rerandomize="false", max_attempts=2, done=True)
self.assertTrue(module.should_show_save_button())
module = CapaFactory.create(rerandomize="per_student", max_attempts=2, done=True)
self.assertTrue(module.should_show_save_button())
# If survey question for capa (max_attempts = 0),
# DO show the save button
module = CapaFactory.create(max_attempts=0, done=False)
@@ -788,9 +835,15 @@ class CapaModuleTest(unittest.TestCase):
done=True)
self.assertTrue(module.should_show_save_button())
module = CapaFactory.create(force_save_button="true",
rerandomize="true",
done=True)
self.assertTrue(module.should_show_save_button())
def test_no_max_attempts(self):
module = CapaFactory.create(max_attempts='')
html = module.get_problem_html()
self.assertTrue(html is not None)
# assert that we got here without exploding
def test_get_problem_html(self):
@@ -875,6 +928,8 @@ class CapaModuleTest(unittest.TestCase):
# Try to render the module with DEBUG turned off
html = module.get_problem_html()
self.assertTrue(html is not None)
# Check the rendering context
render_args, _ = module.system.render_template.call_args
context = render_args[1]
@@ -886,7 +941,9 @@ class CapaModuleTest(unittest.TestCase):
def test_random_seed_no_change(self):
# Run the test for each possible rerandomize value
for rerandomize in ['never', 'per_student', 'always', 'onreset']:
for rerandomize in ['false', 'never',
'per_student', 'always',
'true', 'onreset']:
module = CapaFactory.create(rerandomize=rerandomize)
# Get the seed
@@ -896,8 +953,9 @@ class CapaModuleTest(unittest.TestCase):
# If we're not rerandomizing, the seed is always set
# to the same value (1)
if rerandomize == 'never':
self.assertEqual(seed, 1)
if rerandomize in ['never']:
self.assertEqual(seed, 1,
msg="Seed should always be 1 when rerandomize='%s'" % rerandomize)
# Check the problem
get_request_dict = {CapaFactory.input_key(): '3.14'}
@@ -947,7 +1005,8 @@ class CapaModuleTest(unittest.TestCase):
return success
# Run the test for each possible rerandomize value
for rerandomize in ['never', 'per_student', 'always', 'onreset']:
for rerandomize in ['never', 'false', 'per_student',
'always', 'true', 'onreset']:
module = CapaFactory.create(rerandomize=rerandomize)
# Get the seed
@@ -959,7 +1018,7 @@ class CapaModuleTest(unittest.TestCase):
# is set to 'never' -- it should still be 1
# The seed also stays the same if we're randomizing
# 'per_student': the same student should see the same problem
if rerandomize in ['never', 'per_student']:
if rerandomize in ['never', 'false', 'per_student']:
self.assertEqual(seed, _reset_and_get_seed(module))
# Otherwise, we expect the seed to change
@@ -969,10 +1028,8 @@ class CapaModuleTest(unittest.TestCase):
# Since there's a small chance we might get the
# same seed again, give it 5 chances
# to generate a different seed
success = _retry_and_check(5,
lambda: _reset_and_get_seed(module) != seed)
success = _retry_and_check(5, lambda: _reset_and_get_seed(module) != seed)
# TODO: change this comparison to module.seed is not None?
self.assertTrue(module.seed != None)
self.assertTrue(module.seed is not None)
msg = 'Could not get a new seed from reset after 5 tries'
self.assertTrue(success, msg)

5
lms/djangoapps/courseware/features/problems.feature
View File

@@ -15,6 +15,7 @@ Feature: Answer problems
| drop down |
| multiple choice |
| checkbox |
| radio |
| string |
| numerical |
| formula |
@@ -33,6 +34,7 @@ Feature: Answer problems
| drop down |
| multiple choice |
| checkbox |
| radio |
| string |
| numerical |
| formula |
@@ -50,6 +52,7 @@ Feature: Answer problems
| drop down |
| multiple choice |
| checkbox |
| radio |
| string |
| numerical |
| formula |
@@ -71,6 +74,8 @@ Feature: Answer problems
| multiple choice | incorrect |
| checkbox | correct |
| checkbox | incorrect |
| radio | correct |
| radio | incorrect |
| string | correct |
| string | incorrect |
| numerical | correct |

26
lms/djangoapps/courseware/features/problems.py
View File

@@ -42,7 +42,13 @@ PROBLEM_FACTORY_DICT = {
'choice_type': 'checkbox',
'choices': [True, False, True, False, False],
'choice_names': ['Choice 1', 'Choice 2', 'Choice 3', 'Choice 4']}},
'radio': {
'factory': ChoiceResponseXMLFactory(),
'kwargs': {
'question_text': 'The correct answer is Choice 3',
'choice_type': 'radio',
'choices': [False, False, True, False],
'choice_names': ['Choice 1', 'Choice 2', 'Choice 3', 'Choice 4']}},
'string': {
'factory': StringResponseXMLFactory(),
'kwargs': {
@@ -174,6 +180,12 @@ def answer_problem(step, problem_type, correctness):
else:
inputfield('checkbox', choice='choice_3').check()
elif problem_type == 'radio':
if correctness == 'correct':
inputfield('radio', choice='choice_2').check()
else:
inputfield('radio', choice='choice_1').check()
elif problem_type == 'string':
textvalue = 'correct string' if correctness == 'correct' \
else 'incorrect'
@@ -252,6 +264,14 @@ def assert_problem_has_answer(step, problem_type, answer_class):
else:
assert_checked('checkbox', [])
elif problem_type == "radio":
if answer_class == 'correct':
assert_checked('radio', ['choice_2'])
elif answer_class == 'incorrect':
assert_checked('radio', ['choice_1'])
else:
assert_checked('radio', [])
elif problem_type == 'string':
if answer_class == 'blank':
expected = ''
@@ -298,6 +318,7 @@ CORRECTNESS_SELECTORS = {
'correct': {'drop down': ['span.correct'],
'multiple choice': ['label.choicegroup_correct'],
'checkbox': ['span.correct'],
'radio': ['label.choicegroup_correct'],
'string': ['div.correct'],
'numerical': ['div.correct'],
'formula': ['div.correct'],
@@ -308,6 +329,8 @@ CORRECTNESS_SELECTORS = {
'multiple choice': ['label.choicegroup_incorrect',
'span.incorrect'],
'checkbox': ['span.incorrect'],
'radio': ['label.choicegroup_incorrect',
'span.incorrect'],
'string': ['div.incorrect'],
'numerical': ['div.incorrect'],
'formula': ['div.incorrect'],
@@ -317,6 +340,7 @@ CORRECTNESS_SELECTORS = {
'unanswered': {'drop down': ['span.unanswered'],
'multiple choice': ['span.unanswered'],
'checkbox': ['span.unanswered'],
'radio': ['span.unanswered'],
'string': ['div.unanswered'],
'numerical': ['div.unanswered'],
'formula': ['div.unanswered'],