diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index 150b3b3c9b..7dcd7b925e 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -470,6 +470,7 @@ class LoncapaProblem(object): python_path=python_path, cache=self.system.cache, slug=self.problem_id, + unsafely=self.system.can_execute_unsafe_code(), ) except Exception as err: log.exception("Error while execing script code: " + all_code) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 0fa50079de..6183ca2ade 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -288,7 +288,14 @@ class LoncapaResponse(object): } try: - safe_exec.safe_exec(code, globals_dict, python_path=self.context['python_path'], slug=self.id) + safe_exec.safe_exec( + code, + globals_dict, + python_path=self.context['python_path'], + slug=self.id, + random_seed=self.context['seed'], + unsafely=self.system.can_execute_unsafe_code(), + ) except Exception as err: msg = 'Error %s in evaluating hint function %s' % (err, hintfn) msg += "\nSee XML source line %s" % getattr( @@ -973,7 +980,14 @@ class CustomResponse(LoncapaResponse): 'ans': ans, } globals_dict.update(kwargs) - safe_exec.safe_exec(code, globals_dict, python_path=self.context['python_path'], slug=self.id) + safe_exec.safe_exec( + code, + globals_dict, + python_path=self.context['python_path'], + slug=self.id, + random_seed=self.context['seed'], + unsafely=self.system.can_execute_unsafe_code(), + ) return globals_dict['cfn_return'] return check_function @@ -1090,7 +1104,14 @@ class CustomResponse(LoncapaResponse): # exec the check function if isinstance(self.code, basestring): try: - safe_exec.safe_exec(self.code, self.context, cache=self.system.cache, slug=self.id) + safe_exec.safe_exec( + self.code, + self.context, + cache=self.system.cache, + slug=self.id, + random_seed=self.context['seed'], + unsafely=self.system.can_execute_unsafe_code(), + ) except Exception as err: self._handle_exec_exception(err) @@ -1814,7 +1835,14 @@ class SchematicResponse(LoncapaResponse): ] self.context.update({'submission': submission}) try: - safe_exec.safe_exec(self.code, self.context, cache=self.system.cache, slug=self.id) + safe_exec.safe_exec( + self.code, + self.context, + cache=self.system.cache, + slug=self.id, + random_seed=self.context['seed'], + unsafely=self.system.can_execute_unsafe_code(), + ) except Exception as err: msg = 'Error %s in evaluating SchematicResponse' % err raise ResponseError(msg) diff --git a/common/lib/capa/capa/safe_exec/safe_exec.py b/common/lib/capa/capa/safe_exec/safe_exec.py index 67e93be46f..3ab8f0bf9e 100644 --- a/common/lib/capa/capa/safe_exec/safe_exec.py +++ b/common/lib/capa/capa/safe_exec/safe_exec.py @@ -1,6 +1,7 @@ """Capa's specialized use of codejail.safe_exec.""" from codejail.safe_exec import safe_exec as codejail_safe_exec +from codejail.safe_exec import not_safe_exec as codejail_not_safe_exec from codejail.safe_exec import json_safe, SafeExecException from . import lazymod from statsd import statsd @@ -71,7 +72,7 @@ def update_hash(hasher, obj): @statsd.timed('capa.safe_exec.time') -def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None, slug=None): +def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None, slug=None, unsafely=False): """ Execute python code safely. @@ -90,6 +91,8 @@ def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None `slug` is an arbitrary string, a description that's meaningful to the caller, that will be used in log messages. + If `unsafely` is true, then the code will actually be executed without sandboxing. + """ # Check the cache for a previous result. if cache: @@ -111,9 +114,15 @@ def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None # Create the complete code we'll run. code_prolog = CODE_PROLOG % random_seed + # Decide which code executor to use. + if unsafely: + exec_fn = codejail_not_safe_exec + else: + exec_fn = codejail_safe_exec + # Run the code! Results are side effects in globals_dict. try: - codejail_safe_exec( + exec_fn( code_prolog + LAZY_IMPORTS + code, globals_dict, python_path=python_path, slug=slug, ) diff --git a/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py b/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py index 4592af8305..f8a8a32297 100644 --- a/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py +++ b/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py @@ -1,13 +1,17 @@ """Test safe_exec.py""" import hashlib +import os import os.path import random import textwrap import unittest +from nose.plugins.skip import SkipTest + from capa.safe_exec import safe_exec, update_hash from codejail.safe_exec import SafeExecException +from codejail.jail_code import is_configured class TestSafeExec(unittest.TestCase): @@ -68,6 +72,24 @@ class TestSafeExec(unittest.TestCase): self.assertIn("ZeroDivisionError", cm.exception.message) +class TestSafeOrNot(unittest.TestCase): + def test_cant_do_something_forbidden(self): + # Can't test for forbiddenness if CodeJail isn't configured for python. + if not is_configured("python"): + raise SkipTest + + g = {} + with self.assertRaises(SafeExecException) as cm: + safe_exec("import os; files = os.listdir('/')", g) + self.assertIn("OSError", cm.exception.message) + self.assertIn("Permission denied", cm.exception.message) + + def test_can_do_something_forbidden_if_run_unsafely(self): + g = {} + safe_exec("import os; files = os.listdir('/')", g, unsafely=True) + self.assertEqual(g['files'], os.listdir('/')) + + class DictCache(object): """A cache implementation over a simple dict, for testing.""" diff --git a/common/lib/capa/capa/tests/test_responsetypes.py b/common/lib/capa/capa/tests/test_responsetypes.py index 780c475b09..20de19f567 100644 --- a/common/lib/capa/capa/tests/test_responsetypes.py +++ b/common/lib/capa/capa/tests/test_responsetypes.py @@ -640,6 +640,23 @@ class StringResponseTest(ResponseTest): correct_map = problem.grade_answers(input_dict) self.assertEquals(correct_map.get_hint('1_2_1'), "Hello??") + def test_hint_function_randomization(self): + # The hint function should get the seed from the problem. + problem = self.build_problem( + answer="1", + hintfn="gimme_a_random_hint", + script=textwrap.dedent(""" + def gimme_a_random_hint(answer_ids, student_answers, new_cmap, old_cmap): + answer = str(random.randint(0, 1e9)) + new_cmap.set_hint_and_mode(answer_ids[0], answer, "always") + + """) + ) + correct_map = problem.grade_answers({'1_2_1': '2'}) + hint = correct_map.get_hint('1_2_1') + r = random.Random(problem.seed) + self.assertEqual(hint, str(r.randint(0, 1e9))) + class CodeResponseTest(ResponseTest): from response_xml_factory import CodeResponseXMLFactory @@ -948,7 +965,6 @@ class CustomResponseTest(ResponseTest): xml_factory_class = CustomResponseXMLFactory def test_inline_code(self): - # For inline code, we directly modify global context variables # 'answers' is a list of answers provided to us # 'correct' is a list we fill in with True/False @@ -961,15 +977,14 @@ class CustomResponseTest(ResponseTest): self.assert_grade(problem, '0', 'incorrect') def test_inline_message(self): - # Inline code can update the global messages list # to pass messages to the CorrectMap for a particular input # The code can also set the global overall_message (str) # to pass a message that applies to the whole response inline_script = textwrap.dedent(""" - messages[0] = "Test Message" - overall_message = "Overall message" - """) + messages[0] = "Test Message" + overall_message = "Overall message" + """) problem = self.build_problem(answer=inline_script) input_dict = {'1_2_1': '0'} @@ -983,8 +998,19 @@ class CustomResponseTest(ResponseTest): overall_msg = correctmap.get_overall_message() self.assertEqual(overall_msg, "Overall message") - def test_function_code_single_input(self): + def test_inline_randomization(self): + # Make sure the seed from the problem gets fed into the script execution. + inline_script = """messages[0] = str(random.randint(0, 1e9))""" + problem = self.build_problem(answer=inline_script) + input_dict = {'1_2_1': '0'} + correctmap = problem.grade_answers(input_dict) + + input_msg = correctmap.get_msg('1_2_1') + r = random.Random(problem.seed) + self.assertEqual(input_msg, str(r.randint(0, 1e9))) + + def test_function_code_single_input(self): # For function code, we pass in these arguments: # # 'expect' is the expect attribute of the @@ -1212,6 +1238,29 @@ class CustomResponseTest(ResponseTest): with self.assertRaises(ResponseError): problem.grade_answers({'1_2_1': '42'}) + def test_setup_randomization(self): + # Ensure that the problem setup script gets the random seed from the problem. + script = textwrap.dedent(""" + num = random.randint(0, 1e9) + """) + problem = self.build_problem(script=script) + r = random.Random(problem.seed) + self.assertEqual(r.randint(0, 1e9), problem.context['num']) + + def test_check_function_randomization(self): + # The check function should get random-seeded from the problem. + script = textwrap.dedent(""" + def check_func(expect, answer_given): + return {'ok': True, 'msg': str(random.randint(0, 1e9))} + """) + + problem = self.build_problem(script=script, cfn="check_func", expect="42") + input_dict = {'1_2_1': '42'} + correct_map = problem.grade_answers(input_dict) + msg = correct_map.get_msg('1_2_1') + r = random.Random(problem.seed) + self.assertEqual(msg, str(r.randint(0, 1e9))) + def test_module_imports_inline(self): ''' Check that the correct modules are available to custom @@ -1275,7 +1324,6 @@ class SchematicResponseTest(ResponseTest): xml_factory_class = SchematicResponseXMLFactory def test_grade(self): - # Most of the schematic-specific work is handled elsewhere # (in client-side JavaScript) # The is responsible only for executing the @@ -1290,7 +1338,7 @@ class SchematicResponseTest(ResponseTest): # The actual dictionary would contain schematic information # sent from the JavaScript simulation - submission_dict = {'test': 'test'} + submission_dict = {'test': 'the_answer'} input_dict = {'1_2_1': json.dumps(submission_dict)} correct_map = problem.grade_answers(input_dict) @@ -1299,8 +1347,19 @@ class SchematicResponseTest(ResponseTest): # is what we expect) self.assertEqual(correct_map.get_correctness('1_2_1'), 'correct') - def test_script_exception(self): + def test_check_function_randomization(self): + # The check function should get a random seed from the problem. + script = "correct = ['correct' if (submission[0]['num'] == random.randint(0, 1e9)) else 'incorrect']" + problem = self.build_problem(answer=script) + r = random.Random(problem.seed) + submission_dict = {'num': r.randint(0, 1e9)} + input_dict = {'1_2_1': json.dumps(submission_dict)} + correct_map = problem.grade_answers(input_dict) + + self.assertEqual(correct_map.get_correctness('1_2_1'), 'correct') + + def test_script_exception(self): # Construct a script that will raise an exception script = "raise Exception('test')" problem = self.build_problem(answer=script) diff --git a/requirements/edx/github.txt b/requirements/edx/github.txt index 367b22fb46..8f3d4594ac 100644 --- a/requirements/edx/github.txt +++ b/requirements/edx/github.txt @@ -9,5 +9,5 @@ # Our libraries: -e git+https://github.com/edx/XBlock.git@2144a25d#egg=XBlock --e git+https://github.com/edx/codejail.git@5fb5fa0#egg=codejail +-e git+https://github.com/edx/codejail.git@0a1b468#egg=codejail -e git+https://github.com/edx/diff-cover.git@v0.1.1#egg=diff_cover