From 2717360de9ffbdfd267a0157159d2538bad46f4e Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 6 Feb 2013 10:12:26 -0500 Subject: [PATCH 001/162] Make this work with non-Django test suites also. --- runone.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/runone.py b/runone.py index 2227ae0adf..a644aa077b 100755 --- a/runone.py +++ b/runone.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -from django.core import management import argparse import os @@ -42,21 +41,34 @@ def main(argv): test_py_path = find_full_path(test_py_path) test_spec = "%s:%s.%s" % (test_py_path, test_class, test_method) + settings = None if test_py_path.startswith('cms'): settings = 'cms.envs.test' elif test_py_path.startswith('lms'): settings = 'lms.envs.test' + + if settings: + # Run as a django test suite + from django.core import management + + django_args = ["django-admin.py", "test", "--pythonpath=."] + django_args.append("--settings=%s" % settings) + if args.nocapture: + django_args.append("-s") + django_args.append(test_spec) + + print " ".join(django_args) + management.execute_from_command_line(django_args) else: - raise Exception("Couldn't determine settings to use!") + # Run as a nose test suite + import nose.core + nose_args = ["nosetests"] + if args.nocapture: + nose_args.append("-s") + nose_args.append(test_spec) + print " ".join(nose_args) + nose.core.main(argv=nose_args) - django_args = ["django-admin.py", "test", "--pythonpath=."] - django_args.append("--settings=%s" % settings) - if args.nocapture: - django_args.append("-s") - django_args.append(test_spec) - - print " ".join(django_args) - management.execute_from_command_line(django_args) if __name__ == "__main__": main(sys.argv[1:]) From 70c37130ac9561f2cb1b843959eace77e6015d80 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 6 Feb 2013 16:15:43 -0500 Subject: [PATCH 002/162] A codejail package to run code securely. --- common/lib/codejail/codejail/__init__.py | 0 common/lib/codejail/codejail/jailpy.py | 122 ++++++++++++++++++ common/lib/codejail/codejail/safe_exec.py | 34 +++++ .../lib/codejail/codejail/tests/__init__.py | 0 .../codejail/codejail/tests/test_jailpy.py | 48 +++++++ common/lib/codejail/codejail/util.py | 30 +++++ common/lib/codejail/jailed_code.py | 1 + common/lib/codejail/setup.py | 7 + local-requirements.txt | 1 + 9 files changed, 243 insertions(+) create mode 100644 common/lib/codejail/codejail/__init__.py create mode 100644 common/lib/codejail/codejail/jailpy.py create mode 100644 common/lib/codejail/codejail/safe_exec.py create mode 100644 common/lib/codejail/codejail/tests/__init__.py create mode 100644 common/lib/codejail/codejail/tests/test_jailpy.py create mode 100644 common/lib/codejail/codejail/util.py create mode 100644 common/lib/codejail/jailed_code.py create mode 100644 common/lib/codejail/setup.py diff --git a/common/lib/codejail/codejail/__init__.py b/common/lib/codejail/codejail/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/common/lib/codejail/codejail/jailpy.py b/common/lib/codejail/codejail/jailpy.py new file mode 100644 index 0000000000..198c8bffdb --- /dev/null +++ b/common/lib/codejail/codejail/jailpy.py @@ -0,0 +1,122 @@ +"""Run a python process in a jail.""" + +# Instructions: +# - AppArmor.md from xserver +# XXX- apt-get install timelimit + +import os, os.path +import resource +import shutil +import subprocess +import threading +import time + +from .util import temp_directory + +# TODO: limit too much stdout data? + +DEBUG = False +STRICT = True + +# Configure the Python command + +SANDBOX_PYTHON = "/usr/bin/python-sandbox" + +if os.path.exists(SANDBOX_PYTHON): + # Python -S inhibits loading site.py, which prevent Ubuntu from adding + # specialized traceback handlers that fail in the sandbox. + PYTHON_CMD = [ + #'timelimit', '-t', '1', '-s', '9', + 'sudo', '-u', 'sandbox', + SANDBOX_PYTHON, '-S' + ] +elif STRICT: + raise Exception("Couldn't find Python sandbox") +else: + PYTHON_CMD = ['python', '-S'] + + +class JailResult(object): + """A passive object for us to return from jailpy.""" + pass + +def jailpy(code, files=None, argv=None, stdin=None): + """ + Run Python code in a jailed subprocess. + + `code` is a string containing the Python code to run. + + `files` is a list of file paths. + + Return an object with: + + .stdout: stdout of the program, a string + .stderr: stderr of the program, a string + .status: return status of the process: an int, 0 for successful + + """ + with temp_directory(delete_when_done=True) as tmpdir: + + # All the supporting files are copied into our directory. + for filename in files or (): + dest = os.path.join(tmpdir, os.path.basename(filename)) + shutil.copyfile(filename, dest) + + # Create the main file. + with open(os.path.join(tmpdir, "jailed_code.py"), "w") as jailed: + jailed.write(code) + + cmd = PYTHON_CMD + ['jailed_code.py'] + (argv or []) + + subproc = subprocess.Popen( + cmd, preexec_fn=set_process_limits, cwd=tmpdir, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + + # TODO: time limiting + + killer = ProcessKillerThread(subproc) + killer.start() + result = JailResult() + result.stdout, result.stderr = subproc.communicate(stdin) + result.status = subproc.returncode + killer.join() + + return result + + +def set_process_limits(): + """ + Set limits on this processs, to be used first in a child process. + """ + resource.setrlimit(resource.RLIMIT_CPU, (1, 1)) # 1 second of CPU--not wall clock time + resource.setrlimit(resource.RLIMIT_NPROC, (0, 0)) # no subprocesses + resource.setrlimit(resource.RLIMIT_FSIZE, (0, 0)) # no files + + mem = 32 * 2**20 # 32 MB should be enough for anyone, right? :) + resource.setrlimit(resource.RLIMIT_STACK, (mem, mem)) + resource.setrlimit(resource.RLIMIT_RSS, (mem, mem)) + resource.setrlimit(resource.RLIMIT_DATA, (mem, mem)) + + +class ProcessKillerThread(threading.Thread): + def __init__(self, subproc, limit=1): + super(ProcessKillerThread, self).__init__() + self.subproc = subproc + self.limit = limit + + def run(self): + time.sleep(self.limit) + if self.subproc.poll() is None: + # Can't use subproc.kill because we launched the subproc with sudo. + #killargs = ["sudo", "-u", "sandbox", "kill", "-9", str(self.subproc.pid)] + killargs = ["sudo", "-u", "sandbox", "ps", str(self.subproc.pid)] + print killargs + kill = subprocess.Popen(killargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = kill.communicate() + print out + print err + print "Return status: %r" % kill.returncode + #if ret: + #print "Couldn't kill: %r" % ret + #os.system("sudo -u sandbox kill -9 %s" % self.subproc.pid) diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py new file mode 100644 index 0000000000..64700fbbce --- /dev/null +++ b/common/lib/codejail/codejail/safe_exec.py @@ -0,0 +1,34 @@ +"""Safe execution of untrusted Python code.""" + +import json + +def straw(v): + return json.loads(json.dumps(jsonable_dict(v))) + +def jsonable_dict(d): + jd = {} + for k,v in d.iteritems(): + try: + json.dumps(v) + except TypeError: + continue + else: + jd[k] = v + return jd + +def safe_exec(code, globals_dict, locals_dict=None, future_division=False, assumed_imports=None): + if future_division: + code = "from __future__ import division\n" + code + + g_dict = straw(globals_dict) + + if locals_dict is None: + l_dict = g_dict + else: + l_dict = straw(locals_dict) + + exec code in g_dict, l_dict + + globals_dict.update(straw(g_dict)) + if locals_dict is not None: + locals_dict.update(straw(l_dict)) diff --git a/common/lib/codejail/codejail/tests/__init__.py b/common/lib/codejail/codejail/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/common/lib/codejail/codejail/tests/test_jailpy.py b/common/lib/codejail/codejail/tests/test_jailpy.py new file mode 100644 index 0000000000..c7a56d7bda --- /dev/null +++ b/common/lib/codejail/codejail/tests/test_jailpy.py @@ -0,0 +1,48 @@ +import textwrap +import unittest + +from codejail.jailpy import jailpy + +dedent = textwrap.dedent + +class TestFeatures(unittest.TestCase): + def test_hello_world(self): + res = jailpy("print 'Hello, world!'") + self.assertEqual(res.status, 0) + self.assertEqual(res.stdout, 'Hello, world!\n') + + def test_argv(self): + res = jailpy( + "import sys; print ':'.join(sys.argv[1:])", + argv=["Hello", "world", "-x"] + ) + self.assertEqual(res.status, 0) + self.assertEqual(res.stdout, "Hello:world:-x\n") + + def test_ends_with_exception(self): + res = jailpy("""raise Exception('FAIL')""") + self.assertNotEqual(res.status, 0) + self.assertEqual(res.stdout, "") + self.assertEqual(res.stderr, dedent("""\ + Traceback (most recent call last): + File "jailed_code.py", line 1, in + raise Exception('FAIL') + Exception: FAIL + """)) + + +class TestLimits(unittest.TestCase): + def test_cant_use_too_much_memory(self): + res = jailpy("print sum(range(100000000))") + self.assertNotEqual(res.status, 0) + self.assertEqual(res.stdout, "") + + def test_cant_use_too_much_cpu(self): + res = jailpy("print sum(xrange(100000000))") + self.assertNotEqual(res.status, 0) + self.assertEqual(res.stdout, "") + + def test_cant_use_too_much_time(self): + res = jailpy("import time; time.sleep(5); print 'Done!'") + self.assertNotEqual(res.status, 0) + self.assertEqual(res.stdout, "") diff --git a/common/lib/codejail/codejail/util.py b/common/lib/codejail/codejail/util.py new file mode 100644 index 0000000000..7ce0c7051b --- /dev/null +++ b/common/lib/codejail/codejail/util.py @@ -0,0 +1,30 @@ +"""Helpers for codejail.""" + +import contextlib +import os +import shutil +import tempfile + +class TempDirectory(object): + def __init__(self, delete_when_done=True): + self.delete_when_done = delete_when_done + self.temp_dir = tempfile.mkdtemp(prefix="codejail-") + # Make directory readable by other users ('sandbox' user needs to be able to read it) + os.chmod(self.temp_dir, 0775) + + def clean_up(self): + if self.delete_when_done: + # if this errors, something is genuinely wrong, so don't ignore errors. + shutil.rmtree(self.temp_dir) + +@contextlib.contextmanager +def temp_directory(delete_when_done=True): + """ + A context manager to make and use a temp directory. If `delete_when_done` + is true (the default), the directory will be removed when done. + """ + tmp = TempDirectory(delete_when_done) + try: + yield tmp.temp_dir + finally: + tmp.clean_up() diff --git a/common/lib/codejail/jailed_code.py b/common/lib/codejail/jailed_code.py new file mode 100644 index 0000000000..f439268c4c --- /dev/null +++ b/common/lib/codejail/jailed_code.py @@ -0,0 +1 @@ +import time; time.sleep(5); print 'Done!' \ No newline at end of file diff --git a/common/lib/codejail/setup.py b/common/lib/codejail/setup.py new file mode 100644 index 0000000000..c4dcf2b0f7 --- /dev/null +++ b/common/lib/codejail/setup.py @@ -0,0 +1,7 @@ +from setuptools import setup + +setup( + name="codejail", + version="0.1", + packages=['codejail'], +) diff --git a/local-requirements.txt b/local-requirements.txt index 201467d11e..053a223605 100644 --- a/local-requirements.txt +++ b/local-requirements.txt @@ -1,4 +1,5 @@ # Python libraries to install that are local to the mitx repo -e common/lib/capa -e common/lib/xmodule +-e common/lib/codejail -e . From a9979b8aae4504bd49fbc121afba0b19f66e6220 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 7 Feb 2013 11:36:52 -0500 Subject: [PATCH 003/162] Killing processes isn't working. --- common/lib/codejail/codejail/jailpy.py | 28 ++++++++----------- .../codejail/codejail/tests/test_jailpy.py | 8 +++++- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/common/lib/codejail/codejail/jailpy.py b/common/lib/codejail/codejail/jailpy.py index 198c8bffdb..45842d715f 100644 --- a/common/lib/codejail/codejail/jailpy.py +++ b/common/lib/codejail/codejail/jailpy.py @@ -2,7 +2,6 @@ # Instructions: # - AppArmor.md from xserver -# XXX- apt-get install timelimit import os, os.path import resource @@ -26,8 +25,7 @@ if os.path.exists(SANDBOX_PYTHON): # Python -S inhibits loading site.py, which prevent Ubuntu from adding # specialized traceback handlers that fail in the sandbox. PYTHON_CMD = [ - #'timelimit', '-t', '1', '-s', '9', - 'sudo', '-u', 'sandbox', + 'sudo', '-u', 'sandbox', SANDBOX_PYTHON, '-S' ] elif STRICT: @@ -46,9 +44,9 @@ def jailpy(code, files=None, argv=None, stdin=None): `code` is a string containing the Python code to run. - `files` is a list of file paths. + `files` is a list of file paths. - Return an object with: + Return an object with: .stdout: stdout of the program, a string .stderr: stderr of the program, a string @@ -80,7 +78,6 @@ def jailpy(code, files=None, argv=None, stdin=None): result = JailResult() result.stdout, result.stderr = subproc.communicate(stdin) result.status = subproc.returncode - killer.join() return result @@ -106,17 +103,16 @@ class ProcessKillerThread(threading.Thread): self.limit = limit def run(self): - time.sleep(self.limit) + start = time.time() + while (time.time() - start) < self.limit: + time.sleep(.1) + if self.subproc.poll() is not None: + # Process ended, no need for us any more. + return + if self.subproc.poll() is None: # Can't use subproc.kill because we launched the subproc with sudo. - #killargs = ["sudo", "-u", "sandbox", "kill", "-9", str(self.subproc.pid)] - killargs = ["sudo", "-u", "sandbox", "ps", str(self.subproc.pid)] - print killargs + killargs = ["sudo", "kill", "-9", str(self.subproc.pid)] kill = subprocess.Popen(killargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = kill.communicate() - print out - print err - print "Return status: %r" % kill.returncode - #if ret: - #print "Couldn't kill: %r" % ret - #os.system("sudo -u sandbox kill -9 %s" % self.subproc.pid) + # TODO: This doesn't actually kill the process.... :( diff --git a/common/lib/codejail/codejail/tests/test_jailpy.py b/common/lib/codejail/codejail/tests/test_jailpy.py index c7a56d7bda..408bb2ba30 100644 --- a/common/lib/codejail/codejail/tests/test_jailpy.py +++ b/common/lib/codejail/codejail/tests/test_jailpy.py @@ -1,5 +1,6 @@ import textwrap import unittest +from nose.plugins.skip import SkipTest from codejail.jailpy import jailpy @@ -43,6 +44,11 @@ class TestLimits(unittest.TestCase): self.assertEqual(res.stdout, "") def test_cant_use_too_much_time(self): - res = jailpy("import time; time.sleep(5); print 'Done!'") + raise SkipTest # TODO: test this once we can kill sleeping processes. + res = jailpy(dedent("""\ + import time + time.sleep(5) + print 'Done!' + """)) self.assertNotEqual(res.status, 0) self.assertEqual(res.stdout, "") From f8c53053528d7a4137126690e4e635a0176aa64c Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 7 Feb 2013 11:37:06 -0500 Subject: [PATCH 004/162] Add some malware tests --- .../codejail/codejail/tests/test_jailpy.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/common/lib/codejail/codejail/tests/test_jailpy.py b/common/lib/codejail/codejail/tests/test_jailpy.py index 408bb2ba30..ff9be079d1 100644 --- a/common/lib/codejail/codejail/tests/test_jailpy.py +++ b/common/lib/codejail/codejail/tests/test_jailpy.py @@ -52,3 +52,51 @@ class TestLimits(unittest.TestCase): """)) self.assertNotEqual(res.status, 0) self.assertEqual(res.stdout, "") + + # TODO: write files + # TODO: read network + # TODO: fork + +class TestMalware(unittest.TestCase): + def test_crash_cpython(self): + # http://nedbatchelder.com/blog/201206/eval_really_is_dangerous.html + res = jailpy(dedent("""\ + import new, sys + crash_me = new.function(new.code(0,0,0,0,"KABOOM",(),(),(),"","",0,""), {}) + print "Here we go..." + sys.stdout.flush() + crash_me() + print "The afterlife!" + """)) + self.assertNotEqual(res.status, 0) + self.assertEqual(res.stdout, "Here we go...\n") + self.assertEqual(res.stderr, "") + + def test_read_etc_passwd(self): + res = jailpy(dedent("""\ + bytes = len(open('/etc/passwd').read()) + print 'Gotcha', bytes + """)) + self.assertNotEqual(res.status, 0) + self.assertEqual(res.stdout, "") + self.assertIn("ermission denied", res.stderr) + + def test_find_other_sandboxes(self): + res = jailpy(dedent(""" + import os; + places = [ + "..", "/tmp", "/", "/home", "/etc", + "/var" + ] + for place in places: + try: + files = os.listdir(place) + except Exception: + # darn + pass + else: + print "Files in %r: %r" % (place, files) + print "Done." + """)) + self.assertEqual(res.status, 0) + self.assertEqual(res.stdout, "Done.\n") From 17f9e4b27df59b6226446e9a1f6b3497e2826227 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 7 Feb 2013 12:15:34 -0500 Subject: [PATCH 005/162] A turd left over from a test --- common/lib/codejail/jailed_code.py | 1 - 1 file changed, 1 deletion(-) delete mode 100644 common/lib/codejail/jailed_code.py diff --git a/common/lib/codejail/jailed_code.py b/common/lib/codejail/jailed_code.py deleted file mode 100644 index f439268c4c..0000000000 --- a/common/lib/codejail/jailed_code.py +++ /dev/null @@ -1 +0,0 @@ -import time; time.sleep(5); print 'Done!' \ No newline at end of file From 6c609afdb1fb122e3f0ab0aa459f600122302865 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 7 Feb 2013 14:06:01 -0500 Subject: [PATCH 006/162] LazyModule for lazily proxying module imports. --- common/lib/codejail/codejail/lazymod.py | 41 +++++++++++++++++++ .../codejail/codejail/tests/test_lazymod.py | 26 ++++++++++++ common/lib/codejail/codejail/util.py | 21 ++++++++++ 3 files changed, 88 insertions(+) create mode 100644 common/lib/codejail/codejail/lazymod.py create mode 100644 common/lib/codejail/codejail/tests/test_lazymod.py diff --git a/common/lib/codejail/codejail/lazymod.py b/common/lib/codejail/codejail/lazymod.py new file mode 100644 index 0000000000..936a8d263a --- /dev/null +++ b/common/lib/codejail/codejail/lazymod.py @@ -0,0 +1,41 @@ +"""A module proxy for delayed importing of modules. + +Lifted from http://barnesc.blogspot.com/2006/06/automatic-python-imports-with-autoimp.html + +""" + +import sys + +class LazyModule(object): + """A lazy module proxy.""" + + def __init__(self, modname): + self.__dict__['__name__'] = modname + self._set_mod(None) + + def _set_mod(self, mod): + if mod is not None: + self.__dict__ = mod.__dict__ + self.__dict__['_lazymod_mod'] = mod + + def _load_mod(self): + __import__(self.__name__) + self._set_mod(sys.modules[self.__name__]) + + def __getattr__(self, name): + if self.__dict__['_lazymod_mod'] is None: + self._load_mod() + + mod = self.__dict__['_lazymod_mod'] + + if hasattr(mod, name): + return getattr(mod, name) + else: + try: + subname = '%s.%s' % (self.__name__, name) + __import__(subname) + submod = getattr(mod, name) + except ImportError: + raise AttributeError("'module' object has no attribute %r" % name) + self.__dict__[name] = LazyModule(subname, submod) + return self.__dict__[name] diff --git a/common/lib/codejail/codejail/tests/test_lazymod.py b/common/lib/codejail/codejail/tests/test_lazymod.py new file mode 100644 index 0000000000..eb853060d0 --- /dev/null +++ b/common/lib/codejail/codejail/tests/test_lazymod.py @@ -0,0 +1,26 @@ +"""Test lazymod.py""" + +import sys +import unittest + +from codejail.lazymod import LazyModule +from codejail.util import ModuleIsolation + + +class TestLazyMod(unittest.TestCase): + + def setUp(self): + # Each test will remove modules that it imported. + self.addCleanup(ModuleIsolation().clean_up) + + def test_simple(self): + # Import some stdlib module that has not been imported before + self.assertNotIn("colorsys", sys.modules) + colorsys = LazyModule("colorsys") + hsv = colorsys.rgb_to_hsv(.3, .4, .2) + self.assertEqual(hsv[0], 0.25) + + def test_dotted(self): + self.assertNotIn("email.utils", sys.modules) + email_utils = LazyModule("email.utils") + self.assertEqual(email_utils.quote('"hi"'), r'\"hi\"') diff --git a/common/lib/codejail/codejail/util.py b/common/lib/codejail/codejail/util.py index 7ce0c7051b..88ecc1b319 100644 --- a/common/lib/codejail/codejail/util.py +++ b/common/lib/codejail/codejail/util.py @@ -3,8 +3,10 @@ import contextlib import os import shutil +import sys import tempfile + class TempDirectory(object): def __init__(self, delete_when_done=True): self.delete_when_done = delete_when_done @@ -28,3 +30,22 @@ def temp_directory(delete_when_done=True): yield tmp.temp_dir finally: tmp.clean_up() + + +class ModuleIsolation(object): + """ + Manage changes to sys.modules so that we can roll back imported modules. + + Create this object, it will snapshot the currently imported modules. When + you call `clean_up()`, it will delete any module imported since its creation. + """ + def __init__(self): + # Save all the names of all the imported modules. + self.mods = set(sys.modules) + + def clean_up(self): + # Get a list of modules that didn't exist when we were created + new_mods = [m for m in sys.modules if m not in self.mods] + # and delete them all so another import will run code for real again. + for m in new_mods: + del sys.modules[m] From 9cc43f1d9b4f53662386ad2c20b30d91281070d8 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 8 Feb 2013 10:31:24 -0500 Subject: [PATCH 007/162] Simplify this test setup. --- lms/djangoapps/courseware/tests/tests.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lms/djangoapps/courseware/tests/tests.py b/lms/djangoapps/courseware/tests/tests.py index 4c9f592797..eb132abc59 100644 --- a/lms/djangoapps/courseware/tests/tests.py +++ b/lms/djangoapps/courseware/tests/tests.py @@ -814,11 +814,7 @@ class TestCourseGrader(LoginEnrollmentTestCase): xmodule.modulestore.django._MODULESTORES = {} courses = modulestore().get_courses() - def find_course(course_id): - """Assumes the course is present""" - return [c for c in courses if c.id == course_id][0] - - self.graded_course = find_course("edX/graded/2012_Fall") + self.graded_course = modulestore().get_course("edX/graded/2012_Fall") # create a test student self.student = 'view@test.com' From 4bb5d14f700c313dfaa3b5301611cbdeb31b3031 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 8 Feb 2013 10:34:03 -0500 Subject: [PATCH 008/162] Test that we can't write files --- common/lib/codejail/codejail/tests/test_jailpy.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/common/lib/codejail/codejail/tests/test_jailpy.py b/common/lib/codejail/codejail/tests/test_jailpy.py index ff9be079d1..d1133d745d 100644 --- a/common/lib/codejail/codejail/tests/test_jailpy.py +++ b/common/lib/codejail/codejail/tests/test_jailpy.py @@ -1,3 +1,5 @@ +"""Test jailpy.py""" + import textwrap import unittest from nose.plugins.skip import SkipTest @@ -53,6 +55,18 @@ class TestLimits(unittest.TestCase): self.assertNotEqual(res.status, 0) self.assertEqual(res.stdout, "") + def test_cant_write_files(self): + res = jailpy(dedent("""\ + print "Trying" + with open("mydata.txt", "w") as f: + f.write("hello") + with open("mydata.txt") as f2: + print "Got this:", f2.read() + """)) + self.assertNotEqual(res.status, 0) + self.assertEqual(res.stdout, "Trying\n") + self.assertIn("ermission denied", res.stderr) + # TODO: write files # TODO: read network # TODO: fork From 9827a0e218414328192fbf1a7240df7423c61884 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 8 Feb 2013 11:02:04 -0500 Subject: [PATCH 009/162] Oops, this line can go too. --- lms/djangoapps/courseware/tests/tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lms/djangoapps/courseware/tests/tests.py b/lms/djangoapps/courseware/tests/tests.py index eb132abc59..5345630957 100644 --- a/lms/djangoapps/courseware/tests/tests.py +++ b/lms/djangoapps/courseware/tests/tests.py @@ -812,7 +812,6 @@ class TestCourseGrader(LoginEnrollmentTestCase): def setUp(self): xmodule.modulestore.django._MODULESTORES = {} - courses = modulestore().get_courses() self.graded_course = modulestore().get_course("edX/graded/2012_Fall") From 0c47f1e0b9f8bb280ea52d0cf42d9aa703b77b34 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 8 Feb 2013 15:52:43 -0500 Subject: [PATCH 010/162] safe_exec can load modules for you. --- common/lib/codejail/codejail/safe_exec.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index 64700fbbce..eeb62d5bbd 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -2,6 +2,8 @@ import json +from .lazymod import LazyModule + def straw(v): return json.loads(json.dumps(jsonable_dict(v))) @@ -27,6 +29,13 @@ def safe_exec(code, globals_dict, locals_dict=None, future_division=False, assum else: l_dict = straw(locals_dict) + for modname in assumed_imports or (): + if isinstance(modname, tuple): + name, modname = modname + else: + name = modname + g_dict[name] = LazyModule(modname) + exec code in g_dict, l_dict globals_dict.update(straw(g_dict)) From e69a073161d37f7b62a3a904c06ebb4067195b70 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 11 Feb 2013 14:08:11 -0500 Subject: [PATCH 011/162] Add a test for the Python in schemaresponse, and refactor the tests while I was in there. --- common/test/data/embedded_python/course.xml | 1 + .../embedded_python/course/2013_Spring.xml | 95 ++++++++++++++++ .../embedded_python/roots/2013_Spring.xml | 1 + lms/djangoapps/courseware/tests/tests.py | 104 +++++++++++++----- 4 files changed, 171 insertions(+), 30 deletions(-) create mode 100644 common/test/data/embedded_python/course.xml create mode 100644 common/test/data/embedded_python/course/2013_Spring.xml create mode 100644 common/test/data/embedded_python/roots/2013_Spring.xml diff --git a/common/test/data/embedded_python/course.xml b/common/test/data/embedded_python/course.xml new file mode 100644 index 0000000000..1662543b4d --- /dev/null +++ b/common/test/data/embedded_python/course.xml @@ -0,0 +1 @@ + diff --git a/common/test/data/embedded_python/course/2013_Spring.xml b/common/test/data/embedded_python/course/2013_Spring.xml new file mode 100644 index 0000000000..35e0990d18 --- /dev/null +++ b/common/test/data/embedded_python/course/2013_Spring.xml @@ -0,0 +1,95 @@ + + + + + +
+# for a schematic response, submission[i] is the json representation +# of the diagram and analysis results for the i-th schematic tag + +def get_tran(json,signal): + for element in json: + if element[0] == 'transient': + return element[1].get(signal,[]) + return [] + +def get_value(at,output): + for (t,v) in output: + if at == t: return v + return None + +output = get_tran(submission[0],'Z') +okay = True + +# output should be 1, 1, 1, 1, 1, 0, 0, 0 +if get_value(0.0000004,output) < 2.7: okay = False; +if get_value(0.0000009,output) < 2.7: okay = False; +if get_value(0.0000014,output) < 2.7: okay = False; +if get_value(0.0000019,output) < 2.7: okay = False; +if get_value(0.0000024,output) < 2.7: okay = False; +if get_value(0.0000029,output) > 0.25: okay = False; +if get_value(0.0000034,output) > 0.25: okay = False; +if get_value(0.0000039,output) > 0.25: okay = False; + +correct = ['correct' if okay else 'incorrect'] + + + + + + + + + + + + + + diff --git a/common/test/data/embedded_python/roots/2013_Spring.xml b/common/test/data/embedded_python/roots/2013_Spring.xml new file mode 100644 index 0000000000..1662543b4d --- /dev/null +++ b/common/test/data/embedded_python/roots/2013_Spring.xml @@ -0,0 +1 @@ + diff --git a/lms/djangoapps/courseware/tests/tests.py b/lms/djangoapps/courseware/tests/tests.py index 5345630957..3a25b2b62d 100644 --- a/lms/djangoapps/courseware/tests/tests.py +++ b/lms/djangoapps/courseware/tests/tests.py @@ -372,6 +372,7 @@ class TestCoursesLoadTestCase_XmlModulestore(PageLoaderTestCase): '''Check that all pages in test courses load properly from XML''' def setUp(self): + super(TestCoursesLoadTestCase_XmlModulestore, self).setUp() self.setup_viewtest_user() xmodule.modulestore.django._MODULESTORES = {} @@ -390,6 +391,7 @@ class TestCoursesLoadTestCase_MongoModulestore(PageLoaderTestCase): '''Check that all pages in test courses load properly from Mongo''' def setUp(self): + super(TestCoursesLoadTestCase_MongoModulestore, self).setUp() self.setup_viewtest_user() xmodule.modulestore.django._MODULESTORES = {} modulestore().collection.drop() @@ -479,9 +481,6 @@ class TestDraftModuleStore(TestCase): class TestViewAuth(LoginEnrollmentTestCase): """Check that view authentication works properly""" - # NOTE: setUpClass() runs before override_settings takes effect, so - # can't do imports there without manually hacking settings. - def setUp(self): xmodule.modulestore.django._MODULESTORES = {} @@ -804,38 +803,61 @@ class TestViewAuth(LoginEnrollmentTestCase): @override_settings(MODULESTORE=TEST_DATA_XML_MODULESTORE) -class TestCourseGrader(LoginEnrollmentTestCase): +class TestSubmittingProblems(LoginEnrollmentTestCase): """Check that a course gets graded properly""" - # NOTE: setUpClass() runs before override_settings takes effect, so - # can't do imports there without manually hacking settings. + # Subclasses should specify the course slug + course_slug = "UNKNOWN" + course_when = "UNKNOWN" def setUp(self): xmodule.modulestore.django._MODULESTORES = {} - self.graded_course = modulestore().get_course("edX/graded/2012_Fall") + course_name = "edX/%s/%s" % (self.course_slug, self.course_when) + self.course = modulestore().get_course(course_name) + assert self.course, "Couldn't load course %r" % course_name # create a test student self.student = 'view@test.com' self.password = 'foo' self.create_account('u1', self.student, self.password) self.activate_user(self.student) - self.enroll(self.graded_course) + self.enroll(self.course) self.student_user = get_user(self.student) self.factory = RequestFactory() + def problem_location(self, problem_url_name): + return "i4x://edX/{}/problem/{}".format(self.course_slug, problem_url_name) + + def modx_url(self, problem_location, dispatch): + return reverse( + 'modx_dispatch', + kwargs={ + 'course_id': self.course.id, + 'location': problem_location, + 'dispatch': dispatch, + } + ) + + +class TestCourseGrader(TestSubmittingProblems): + """Check that a course gets graded properly""" + + course_slug = "graded" + course_when = "2012_Fall" + def get_grade_summary(self): '''calls grades.grade for current user and course''' model_data_cache = ModelDataCache.cache_for_descriptor_descendents( - self.graded_course.id, self.student_user, self.graded_course) + self.course.id, self.student_user, self.course) fake_request = self.factory.get(reverse('progress', - kwargs={'course_id': self.graded_course.id})) + kwargs={'course_id': self.course.id})) return grades.grade(self.student_user, fake_request, - self.graded_course, model_data_cache) + self.course, model_data_cache) def get_homework_scores(self): '''get scores for homeworks''' @@ -844,10 +866,10 @@ class TestCourseGrader(LoginEnrollmentTestCase): def get_progress_summary(self): '''return progress summary structure for current user and course''' model_data_cache = ModelDataCache.cache_for_descriptor_descendents( - self.graded_course.id, self.student_user, self.graded_course) + self.course.id, self.student_user, self.course) fake_request = self.factory.get(reverse('progress', - kwargs={'course_id': self.graded_course.id})) + kwargs={'course_id': self.course.id})) progress_summary = grades.progress_summary(self.student_user, fake_request, @@ -868,13 +890,8 @@ class TestCourseGrader(LoginEnrollmentTestCase): input_i4x-edX-graded-problem-H1P3_2_1 input_i4x-edX-graded-problem-H1P3_2_2 """ - problem_location = "i4x://edX/graded/problem/%s" % problem_url_name - - modx_url = reverse('modx_dispatch', - kwargs={'course_id': self.graded_course.id, - 'location': problem_location, - 'dispatch': 'problem_check', }) - + problem_location = self.problem_location(problem_url_name) + modx_url = self.modx_url(problem_location, 'problem_check') resp = self.client.post(modx_url, { 'input_i4x-edX-graded-problem-%s_2_1' % problem_url_name: responses[0], 'input_i4x-edX-graded-problem-%s_2_2' % problem_url_name: responses[1], @@ -884,19 +901,10 @@ class TestCourseGrader(LoginEnrollmentTestCase): return resp - def problem_location(self, problem_url_name): - '''Get location string for problem, assuming hardcoded course_id''' - return "i4x://edX/graded/problem/{0}".format(problem_url_name) - def reset_question_answer(self, problem_url_name): '''resets specified problem for current user''' problem_location = self.problem_location(problem_url_name) - - modx_url = reverse('modx_dispatch', - kwargs={'course_id': self.graded_course.id, - 'location': problem_location, - 'dispatch': 'problem_reset', }) - + modx_url = self.modx_url(problem_location, 'problem_reset') resp = self.client.post(modx_url) return resp @@ -962,3 +970,39 @@ class TestCourseGrader(LoginEnrollmentTestCase): # Now we answer the final question (worth half of the grade) self.submit_question_answer('FinalQuestion', ['Correct', 'Correct']) self.check_grade_percent(1.0) # Hooray! We got 100% + + +@override_settings(MODULESTORE=TEST_DATA_XML_MODULESTORE) +class TestSchematicResponse(TestSubmittingProblems): + """Check that a course gets graded properly""" + + course_slug = "embedded_python" + course_when = "2013_Spring" + + def submit_question_answer(self, problem_url_name, responses): + """Particular to the embedded_python/2013_Spring course.""" + problem_location = self.problem_location(problem_url_name) + modx_url = self.modx_url(problem_location, 'problem_check') + resp = self.client.post(modx_url, { + 'input_i4x-edX-embedded_python-problem-{0}_2_1'.format(problem_url_name): json.dumps(responses), + }) + print "modx_url", modx_url, "responses", responses + print "resp", resp + + return resp + + def test_get_graded(self): + resp = self.submit_question_answer('H1P1', + [['transient', {'Z': [ + [0.0000004, 2.8], + [0.0000009, 2.8], + [0.0000014, 2.8], + [0.0000019, 2.8], + [0.0000024, 2.8], + [0.0000029, 0.2], + [0.0000034, 0.2], + [0.0000039, 0.2] + ]}]] + ) + respdata = json.loads(resp.content) + self.assertEqual(respdata['success'], 'correct') From 9249bafd001a79c140327f417d4e6af2b5d2eed2 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 11 Feb 2013 15:37:23 -0500 Subject: [PATCH 012/162] Add a test of a bad answer also. --- lms/djangoapps/courseware/tests/tests.py | 38 +++++++++++++++--------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/lms/djangoapps/courseware/tests/tests.py b/lms/djangoapps/courseware/tests/tests.py index 3a25b2b62d..2f469fe750 100644 --- a/lms/djangoapps/courseware/tests/tests.py +++ b/lms/djangoapps/courseware/tests/tests.py @@ -841,6 +841,13 @@ class TestSubmittingProblems(LoginEnrollmentTestCase): } ) + def reset_question_answer(self, problem_url_name): + '''resets specified problem for current user''' + problem_location = self.problem_location(problem_url_name) + modx_url = self.modx_url(problem_location, 'problem_reset') + resp = self.client.post(modx_url) + return resp + class TestCourseGrader(TestSubmittingProblems): """Check that a course gets graded properly""" @@ -896,16 +903,6 @@ class TestCourseGrader(TestSubmittingProblems): 'input_i4x-edX-graded-problem-%s_2_1' % problem_url_name: responses[0], 'input_i4x-edX-graded-problem-%s_2_2' % problem_url_name: responses[1], }) - print "modx_url", modx_url, "responses", responses - print "resp", resp - - return resp - - def reset_question_answer(self, problem_url_name): - '''resets specified problem for current user''' - problem_location = self.problem_location(problem_url_name) - modx_url = self.modx_url(problem_location, 'problem_reset') - resp = self.client.post(modx_url) return resp def test_get_graded(self): @@ -974,7 +971,7 @@ class TestCourseGrader(TestSubmittingProblems): @override_settings(MODULESTORE=TEST_DATA_XML_MODULESTORE) class TestSchematicResponse(TestSubmittingProblems): - """Check that a course gets graded properly""" + """Check that we can submit a schematic response, and it answers properly.""" course_slug = "embedded_python" course_when = "2013_Spring" @@ -986,9 +983,6 @@ class TestSchematicResponse(TestSubmittingProblems): resp = self.client.post(modx_url, { 'input_i4x-edX-embedded_python-problem-{0}_2_1'.format(problem_url_name): json.dumps(responses), }) - print "modx_url", modx_url, "responses", responses - print "resp", resp - return resp def test_get_graded(self): @@ -1006,3 +1000,19 @@ class TestSchematicResponse(TestSubmittingProblems): ) respdata = json.loads(resp.content) self.assertEqual(respdata['success'], 'correct') + + self.reset_question_answer('H1P1') + resp = self.submit_question_answer('H1P1', + [['transient', {'Z': [ + [0.0000004, 2.8], + [0.0000009, 0.0], # wrong. + [0.0000014, 2.8], + [0.0000019, 2.8], + [0.0000024, 2.8], + [0.0000029, 0.2], + [0.0000034, 0.2], + [0.0000039, 0.2] + ]}]] + ) + respdata = json.loads(resp.content) + self.assertEqual(respdata['success'], 'incorrect') From 0a6761c9a5ebdd50b932b60133e3558f8a381984 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 12 Feb 2013 11:41:33 -0500 Subject: [PATCH 013/162] Clean up this xml data file. --- .../embedded_python/course/2013_Spring.xml | 60 +++---------------- 1 file changed, 7 insertions(+), 53 deletions(-) diff --git a/common/test/data/embedded_python/course/2013_Spring.xml b/common/test/data/embedded_python/course/2013_Spring.xml index 35e0990d18..cfa67ff0f3 100644 --- a/common/test/data/embedded_python/course/2013_Spring.xml +++ b/common/test/data/embedded_python/course/2013_Spring.xml @@ -1,9 +1,13 @@ - + -
+ +
+ +
+ # for a schematic response, submission[i] is the json representation # of the diagram and analysis results for the i-th schematic tag @@ -39,57 +43,7 @@ correct = ['correct' if okay else 'incorrect'] - + - - - From 33abe54e0dd6dd11b4e1734cdfd835cf10ba4738 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 14 Feb 2013 10:17:07 -0500 Subject: [PATCH 014/162] Work in progress to sandbox the uses of eval in LMS. --- common/lib/capa/capa/capa_problem.py | 44 +++++++++---- common/lib/capa/capa/responsetypes.py | 66 ++++++++----------- common/lib/codejail/codejail/safe_exec.py | 5 ++ .../embedded_python/course/2013_Spring.xml | 54 ++++++++++++++- lms/djangoapps/courseware/tests/tests.py | 34 +++++++++- 5 files changed, 147 insertions(+), 56 deletions(-) diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index 6580114bcc..6973deb4a0 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -22,7 +22,6 @@ import numpy import os import random import re -import scipy import struct import sys @@ -30,6 +29,7 @@ from lxml import etree from xml.sax.saxutils import unescape from copy import deepcopy +<<<<<<< HEAD import chem import chem.miller import chem.chemcalc @@ -38,8 +38,9 @@ import verifiers import verifiers.draganddrop import calc +======= +>>>>>>> Work in progress to sandbox the uses of eval in LMS. from .correctmap import CorrectMap -import eia import inputtypes import customrender from .util import contextualize_text, convert_files_to_filenames @@ -48,6 +49,8 @@ import xqueue_interface # to be replaced with auto-registering import responsetypes +from codejail.safe_exec import safe_exec + # dict of tagname, Response Class -- this should come from auto-registering response_tag_dict = dict([(x.response_tag, x) for x in responsetypes.__all__]) @@ -63,6 +66,7 @@ html_transforms = {'problem': {'tag': 'div'}, "math": {'tag': 'span'}, } +<<<<<<< HEAD global_context = {'random': random, 'numpy': numpy, 'math': math, @@ -73,6 +77,20 @@ global_context = {'random': random, 'chemtools': chem.chemtools, 'miller': chem.miller, 'draganddrop': verifiers.draganddrop} +======= +safe_exec_assumed_imports = [ + "random", + "numpy", + "math", + "scipy", + "calc", + "eia", + ("chemcalc", "chem.chemcalc"), + ("chemtools", "chem.chemtools"), + ("miller", "chem.miller"), + ("draganddrop", "verifiers.draganddrop"), +] +>>>>>>> Work in progress to sandbox the uses of eval in LMS. # These should be removed from HTML output, including all subelements html_problem_semantics = ["codeparam", "responseparam", "answer", "script", "hintgroup", "openendedparam", "openendedrubric"] @@ -144,7 +162,7 @@ class LoncapaProblem(object): self._process_includes() # construct script processor context (eg for customresponse problems) - self.context = self._extract_context(self.tree, seed=self.seed) + self.context = self._extract_context(self.tree) # Pre-parse the XML tree: modifies it to add ID's and perform some in-place # transformations. This also creates the dict (self.responders) of Response @@ -451,7 +469,7 @@ class LoncapaProblem(object): return path - def _extract_context(self, tree, seed=struct.unpack('i', os.urandom(4))[0]): # private + def _extract_context(self, tree): ''' Extract content of from the problem.xml file, and exec it in the context of this problem. Provides ability to randomize problems, and also set @@ -460,14 +478,18 @@ class LoncapaProblem(object): Problem XML goes to Python execution context. Runs everything in script tags. ''' random.seed(self.seed) - # save global context in here also - context = {'global_context': global_context} - - # initialize context to have stuff in global_context - context.update(global_context) + # TODO: REMOVE THIS COMMENTED OUT CODE. + ## save global context in here also + #context = {'global_context': global_context} + # + ## initialize context to have stuff in global_context + #context.update(global_context) + # # put globals there also - context['__builtins__'] = globals()['__builtins__'] + #context['__builtins__'] = globals()['__builtins__'] + + context = {} # pass instance of LoncapaProblem in context['the_lcp'] = self @@ -501,7 +523,7 @@ class LoncapaProblem(object): context['script_code'] += code try: # use "context" for global context; thus defs in code are global within code - exec code in context, context + safe_exec(code, context, future_division=True, assumed_imports=safe_exec_assumed_imports) except Exception as err: log.exception("Error while execing script code: " + code) msg = "Error while executing script code: %s" % str(err).replace('<', '<') diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index b06a62ffc6..f732c9fc84 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -37,6 +37,8 @@ from lxml import etree from lxml.html.soupparser import fromstring as fromstring_bs # uses Beautiful Soup!!! FIXME? import xqueue_interface +from codejail.safe_exec import safe_exec + log = logging.getLogger(__name__) @@ -968,14 +970,20 @@ def sympy_check2(): cfn = xml.get('cfn') if cfn: log.debug("cfn = %s" % cfn) - if cfn in self.context: - self.code = self.context[cfn] - else: - msg = "%s: can't find cfn %s in context" % ( - unicode(self), cfn) - msg += "\nSee XML source line %s" % getattr(self.xml, 'sourceline', - '') - raise LoncapaProblemError(msg) + + def make_check_function(script_code, cfn): + def check_function(expect, ans): + code = (script_code + "\n" + + "cfn_return = %s(expect, ans)\n" % cfn) + globals_dict = { + 'expect': expect, + 'ans': ans, + } + safe_exec(code, globals_dict) + return globals_dict['cfn_return'] + return check_function + + self.code = make_check_function(self.context['script_code'], cfn) if not self.code: if answer is None: @@ -1074,6 +1082,7 @@ def sympy_check2(): # exec the check function if isinstance(self.code, basestring): try: + raise Exception("exec 1") exec self.code in self.context['global_context'], self.context correct = self.context['correct'] messages = self.context['messages'] @@ -1083,32 +1092,15 @@ def sympy_check2(): self._handle_exec_exception(err) else: - # self.code is not a string; assume its a function + # self.code is not a string; it's a function we created earlier. # this is an interface to the Tutor2 check functions fn = self.code ret = None log.debug(" submission = %s" % submission) try: - answer_given = submission[0] if ( - len(idset) == 1) else submission - # handle variable number of arguments in check function, for backwards compatibility - # with various Tutor2 check functions - args = [self.expect, answer_given, - student_answers, self.answer_ids[0]] - argspec = inspect.getargspec(fn) - nargs = len(argspec.args) - len(argspec.defaults or []) - kwargs = {} - for argname in argspec.args[nargs:]: - kwargs[argname] = self.context[ - argname] if argname in self.context else None - - log.debug('[customresponse] answer_given=%s' % answer_given) - log.debug('nargs=%d, args=%s, kwargs=%s' % ( - nargs, args, kwargs)) - - ret = fn(*args[:nargs], **kwargs) - + answer_given = submission[0] if (len(idset) == 1) else submission + ret = fn(self.expect, answer_given) except Exception as err: self._handle_exec_exception(err) @@ -1265,6 +1257,7 @@ class SymbolicResponse(CustomResponse): def setup_response(self): self.xml.set('cfn', 'symmath_check') code = "from symmath import *" + raise Exception("exec 2") exec code in self.context, self.context CustomResponse.setup_response(self) @@ -1378,6 +1371,7 @@ class CodeResponse(LoncapaResponse): penv = {} penv['__builtins__'] = globals()['__builtins__'] try: + raise Exception("exec 3") exec(code, penv, penv) except Exception as err: log.error( @@ -1925,18 +1919,12 @@ class SchematicResponse(LoncapaResponse): self.code = answer.text def get_score(self, student_answers): - from capa_problem import global_context - submission = [json.loads(student_answers[ - k]) for k in sorted(self.answer_ids)] + #from capa_problem import global_context + submission = [ + json.loads(student_answers[k]) for k in sorted(self.answer_ids) + ] self.context.update({'submission': submission}) - - try: - exec self.code in global_context, self.context - - except Exception as err: - _, _, traceback_obj = sys.exc_info() - raise ResponseError, ResponseError(err.message), traceback_obj - + safe_exec(self.code, {}, self.context) cmap = CorrectMap() cmap.set_dict(dict(zip(sorted( self.answer_ids), self.context['correct']))) diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index eeb62d5bbd..92beba98cc 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -19,6 +19,11 @@ def jsonable_dict(d): return jd def safe_exec(code, globals_dict, locals_dict=None, future_division=False, assumed_imports=None): + """Execute code safely. + + Returns None. The code can modify globals in `global_dict`. + + """ if future_division: code = "from __future__ import division\n" + code diff --git a/common/test/data/embedded_python/course/2013_Spring.xml b/common/test/data/embedded_python/course/2013_Spring.xml index cfa67ff0f3..15e11befd1 100644 --- a/common/test/data/embedded_python/course/2013_Spring.xml +++ b/common/test/data/embedded_python/course/2013_Spring.xml @@ -1,11 +1,14 @@ - + - +
- +
# for a schematic response, submission[i] is the json representation @@ -44,6 +47,51 @@ correct = ['correct' if okay else 'incorrect'] + + + + + +
    +
  1. + 
    +num = 0
+while num <= 5:
+    print(num)
+    num += 1
+
+print("Outside of loop")
+print(num)
+
    +

    + + + +

    +
    2. +
+ + + diff --git a/lms/djangoapps/courseware/tests/tests.py b/lms/djangoapps/courseware/tests/tests.py index 2f469fe750..acad3e9c84 100644 --- a/lms/djangoapps/courseware/tests/tests.py +++ b/lms/djangoapps/courseware/tests/tests.py @@ -986,7 +986,7 @@ class TestSchematicResponse(TestSubmittingProblems): return resp def test_get_graded(self): - resp = self.submit_question_answer('H1P1', + resp = self.submit_question_answer('schematic_problem', [['transient', {'Z': [ [0.0000004, 2.8], [0.0000009, 2.8], @@ -1001,8 +1001,8 @@ class TestSchematicResponse(TestSubmittingProblems): respdata = json.loads(resp.content) self.assertEqual(respdata['success'], 'correct') - self.reset_question_answer('H1P1') - resp = self.submit_question_answer('H1P1', + self.reset_question_answer('schematic_problem') + resp = self.submit_question_answer('schematic_problem', [['transient', {'Z': [ [0.0000004, 2.8], [0.0000009, 0.0], # wrong. @@ -1016,3 +1016,31 @@ class TestSchematicResponse(TestSubmittingProblems): ) respdata = json.loads(resp.content) self.assertEqual(respdata['success'], 'incorrect') + + +@override_settings(MODULESTORE=TEST_DATA_XML_MODULESTORE) +class TestCustomResponseCfnFunction(TestSubmittingProblems): + """Check that cfn functions work properly.""" + + course_slug = "embedded_python" + course_when = "2013_Spring" + + def submit_question_answer(self, problem_url_name, responses): + """Particular to the embedded_python/2013_Spring course.""" + problem_location = self.problem_location(problem_url_name) + modx_url = self.modx_url(problem_location, 'problem_check') + resp = self.client.post(modx_url, { + 'input_i4x-edX-embedded_python-problem-{0}_2_1'.format(problem_url_name): responses, + }) + return resp + + def test_get_graded(self): + resp = self.submit_question_answer('cfn_problem', "0, 1, 2, 3, 4, 5, 'Outside of loop', 6") + respdata = json.loads(resp.content) + self.assertEqual(respdata['success'], 'correct') + + self.reset_question_answer('cfn_problem') + + resp = self.submit_question_answer('cfn_problem', "xyzzy!") + respdata = json.loads(resp.content) + self.assertEqual(respdata['success'], 'incorrect') From ff1df569cb15d0c7f5f2c907cd0bcd0aea4b311d Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 14 Feb 2013 12:40:47 -0500 Subject: [PATCH 015/162] Refactor submitting problems so we don't need custom code for each test. --- lms/djangoapps/courseware/tests/tests.py | 123 ++++++++++------------- 1 file changed, 55 insertions(+), 68 deletions(-) diff --git a/lms/djangoapps/courseware/tests/tests.py b/lms/djangoapps/courseware/tests/tests.py index acad3e9c84..39914d5b8d 100644 --- a/lms/djangoapps/courseware/tests/tests.py +++ b/lms/djangoapps/courseware/tests/tests.py @@ -841,6 +841,23 @@ class TestSubmittingProblems(LoginEnrollmentTestCase): } ) + def submit_question_answer(self, problem_url_name, responses): + """ + Submit answers to a question. + + Responses is a dict mapping problem ids (not sure of the right term) + to answers: + {'2_1': 'Correct', '2_2': 'Incorrect'} + + """ + problem_location = self.problem_location(problem_url_name) + modx_url = self.modx_url(problem_location, 'problem_check') + answer_key_prefix = 'input_i4x-edX-{}-problem-{}_'.format(self.course_slug, problem_url_name) + resp = self.client.post(modx_url, + { (answer_key_prefix + k): v for k,v in responses.items() } + ) + return resp + def reset_question_answer(self, problem_url_name): '''resets specified problem for current user''' problem_location = self.problem_location(problem_url_name) @@ -889,22 +906,6 @@ class TestCourseGrader(TestSubmittingProblems): grade_summary = self.get_grade_summary() self.assertEqual(grade_summary['percent'], percent) - def submit_question_answer(self, problem_url_name, responses): - """ - The field names of a problem are hard to determine. This method only works - for the problems used in the edX/graded course, which has fields named in the - following form: - input_i4x-edX-graded-problem-H1P3_2_1 - input_i4x-edX-graded-problem-H1P3_2_2 - """ - problem_location = self.problem_location(problem_url_name) - modx_url = self.modx_url(problem_location, 'problem_check') - resp = self.client.post(modx_url, { - 'input_i4x-edX-graded-problem-%s_2_1' % problem_url_name: responses[0], - 'input_i4x-edX-graded-problem-%s_2_2' % problem_url_name: responses[1], - }) - return resp - def test_get_graded(self): #### Check that the grader shows we have 0% in the course self.check_grade_percent(0) @@ -922,27 +923,27 @@ class TestCourseGrader(TestSubmittingProblems): return [s.earned for s in hw_section['scores']] # Only get half of the first problem correct - self.submit_question_answer('H1P1', ['Correct', 'Incorrect']) + self.submit_question_answer('H1P1', {'2_1': 'Correct', '2_2': 'Incorrect'}) self.check_grade_percent(0.06) self.assertEqual(earned_hw_scores(), [1.0, 0, 0]) # Order matters self.assertEqual(score_for_hw('Homework1'), [1.0, 0.0]) # Get both parts of the first problem correct self.reset_question_answer('H1P1') - self.submit_question_answer('H1P1', ['Correct', 'Correct']) + self.submit_question_answer('H1P1', {'2_1': 'Correct', '2_2': 'Correct'}) self.check_grade_percent(0.13) self.assertEqual(earned_hw_scores(), [2.0, 0, 0]) self.assertEqual(score_for_hw('Homework1'), [2.0, 0.0]) # This problem is shown in an ABTest - self.submit_question_answer('H1P2', ['Correct', 'Correct']) + self.submit_question_answer('H1P2', {'2_1': 'Correct', '2_2': 'Correct'}) self.check_grade_percent(0.25) self.assertEqual(earned_hw_scores(), [4.0, 0.0, 0]) self.assertEqual(score_for_hw('Homework1'), [2.0, 2.0]) # This problem is hidden in an ABTest. # Getting it correct doesn't change total grade - self.submit_question_answer('H1P3', ['Correct', 'Correct']) + self.submit_question_answer('H1P3', {'2_1': 'Correct', '2_2': 'Correct'}) self.check_grade_percent(0.25) self.assertEqual(score_for_hw('Homework1'), [2.0, 2.0]) @@ -951,21 +952,21 @@ class TestCourseGrader(TestSubmittingProblems): # This problem is also weighted to be 4 points (instead of default of 2) # If the problem was unweighted the percent would have been 0.38 so we # know it works. - self.submit_question_answer('H2P1', ['Correct', 'Correct']) + self.submit_question_answer('H2P1', {'2_1': 'Correct', '2_2': 'Correct'}) self.check_grade_percent(0.42) self.assertEqual(earned_hw_scores(), [4.0, 4.0, 0]) # Third homework - self.submit_question_answer('H3P1', ['Correct', 'Correct']) + self.submit_question_answer('H3P1', {'2_1': 'Correct', '2_2': 'Correct'}) self.check_grade_percent(0.42) # Score didn't change self.assertEqual(earned_hw_scores(), [4.0, 4.0, 2.0]) - self.submit_question_answer('H3P2', ['Correct', 'Correct']) + self.submit_question_answer('H3P2', {'2_1': 'Correct', '2_2': 'Correct'}) self.check_grade_percent(0.5) # Now homework2 dropped. Score changes self.assertEqual(earned_hw_scores(), [4.0, 4.0, 4.0]) # Now we answer the final question (worth half of the grade) - self.submit_question_answer('FinalQuestion', ['Correct', 'Correct']) + self.submit_question_answer('FinalQuestion', {'2_1': 'Correct', '2_2': 'Correct'}) self.check_grade_percent(1.0) # Hooray! We got 100% @@ -976,44 +977,39 @@ class TestSchematicResponse(TestSubmittingProblems): course_slug = "embedded_python" course_when = "2013_Spring" - def submit_question_answer(self, problem_url_name, responses): - """Particular to the embedded_python/2013_Spring course.""" - problem_location = self.problem_location(problem_url_name) - modx_url = self.modx_url(problem_location, 'problem_check') - resp = self.client.post(modx_url, { - 'input_i4x-edX-embedded_python-problem-{0}_2_1'.format(problem_url_name): json.dumps(responses), - }) - return resp - - def test_get_graded(self): + def test_schematic(self): resp = self.submit_question_answer('schematic_problem', - [['transient', {'Z': [ - [0.0000004, 2.8], - [0.0000009, 2.8], - [0.0000014, 2.8], - [0.0000019, 2.8], - [0.0000024, 2.8], - [0.0000029, 0.2], - [0.0000034, 0.2], - [0.0000039, 0.2] - ]}]] - ) + { '2_1': json.dumps( + [['transient', {'Z': [ + [0.0000004, 2.8], + [0.0000009, 2.8], + [0.0000014, 2.8], + [0.0000019, 2.8], + [0.0000024, 2.8], + [0.0000029, 0.2], + [0.0000034, 0.2], + [0.0000039, 0.2] + ]}]] + ) + }) respdata = json.loads(resp.content) self.assertEqual(respdata['success'], 'correct') self.reset_question_answer('schematic_problem') resp = self.submit_question_answer('schematic_problem', - [['transient', {'Z': [ - [0.0000004, 2.8], - [0.0000009, 0.0], # wrong. - [0.0000014, 2.8], - [0.0000019, 2.8], - [0.0000024, 2.8], - [0.0000029, 0.2], - [0.0000034, 0.2], - [0.0000039, 0.2] - ]}]] - ) + { '2_1': json.dumps( + [['transient', {'Z': [ + [0.0000004, 2.8], + [0.0000009, 0.0], # wrong. + [0.0000014, 2.8], + [0.0000019, 2.8], + [0.0000024, 2.8], + [0.0000029, 0.2], + [0.0000034, 0.2], + [0.0000039, 0.2] + ]}]] + ) + }) respdata = json.loads(resp.content) self.assertEqual(respdata['success'], 'incorrect') @@ -1025,22 +1021,13 @@ class TestCustomResponseCfnFunction(TestSubmittingProblems): course_slug = "embedded_python" course_when = "2013_Spring" - def submit_question_answer(self, problem_url_name, responses): - """Particular to the embedded_python/2013_Spring course.""" - problem_location = self.problem_location(problem_url_name) - modx_url = self.modx_url(problem_location, 'problem_check') - resp = self.client.post(modx_url, { - 'input_i4x-edX-embedded_python-problem-{0}_2_1'.format(problem_url_name): responses, - }) - return resp - - def test_get_graded(self): - resp = self.submit_question_answer('cfn_problem', "0, 1, 2, 3, 4, 5, 'Outside of loop', 6") + def test_check_function(self): + resp = self.submit_question_answer('cfn_problem', {'2_1': "0, 1, 2, 3, 4, 5, 'Outside of loop', 6"}) respdata = json.loads(resp.content) self.assertEqual(respdata['success'], 'correct') self.reset_question_answer('cfn_problem') - resp = self.submit_question_answer('cfn_problem', "xyzzy!") + resp = self.submit_question_answer('cfn_problem', {'2_1': "xyzzy!"}) respdata = json.loads(resp.content) self.assertEqual(respdata['success'], 'incorrect') From 6297d64528afa1cae676aa25a7aac9ee97616aea Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 14 Feb 2013 12:43:06 -0500 Subject: [PATCH 016/162] Now these can be in the same test class --- lms/djangoapps/courseware/tests/tests.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lms/djangoapps/courseware/tests/tests.py b/lms/djangoapps/courseware/tests/tests.py index 39914d5b8d..ba9799b0df 100644 --- a/lms/djangoapps/courseware/tests/tests.py +++ b/lms/djangoapps/courseware/tests/tests.py @@ -1013,14 +1013,6 @@ class TestSchematicResponse(TestSubmittingProblems): respdata = json.loads(resp.content) self.assertEqual(respdata['success'], 'incorrect') - -@override_settings(MODULESTORE=TEST_DATA_XML_MODULESTORE) -class TestCustomResponseCfnFunction(TestSubmittingProblems): - """Check that cfn functions work properly.""" - - course_slug = "embedded_python" - course_when = "2013_Spring" - def test_check_function(self): resp = self.submit_question_answer('cfn_problem', {'2_1': "0, 1, 2, 3, 4, 5, 'Outside of loop', 6"}) respdata = json.loads(resp.content) From 37ca6bf77ef3b4479d5c4d53a141e6466cdf6d90 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 14 Feb 2013 13:14:15 -0500 Subject: [PATCH 017/162] Move our specialization of safe_exec into a new module to avoid circular imports. --- common/lib/capa/capa/capa_problem.py | 54 +-------------------------- common/lib/capa/capa/responsetypes.py | 11 ++++-- common/lib/capa/capa/safe_exec.py | 20 ++++++++++ 3 files changed, 30 insertions(+), 55 deletions(-) create mode 100644 common/lib/capa/capa/safe_exec.py diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index 6973deb4a0..5518da8318 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -13,8 +13,6 @@ Main module which shows problems (of "capa" type). This is used by capa_module. ''' -from __future__ import division - from datetime import datetime import logging import math @@ -29,17 +27,6 @@ from lxml import etree from xml.sax.saxutils import unescape from copy import deepcopy -<<<<<<< HEAD -import chem -import chem.miller -import chem.chemcalc -import chem.chemtools -import verifiers -import verifiers.draganddrop - -import calc -======= ->>>>>>> Work in progress to sandbox the uses of eval in LMS. from .correctmap import CorrectMap import inputtypes import customrender @@ -48,8 +35,7 @@ import xqueue_interface # to be replaced with auto-registering import responsetypes - -from codejail.safe_exec import safe_exec +import safe_exec # dict of tagname, Response Class -- this should come from auto-registering response_tag_dict = dict([(x.response_tag, x) for x in responsetypes.__all__]) @@ -66,32 +52,6 @@ html_transforms = {'problem': {'tag': 'div'}, "math": {'tag': 'span'}, } -<<<<<<< HEAD -global_context = {'random': random, - 'numpy': numpy, - 'math': math, - 'scipy': scipy, - 'calc': calc, - 'eia': eia, - 'chemcalc': chem.chemcalc, - 'chemtools': chem.chemtools, - 'miller': chem.miller, - 'draganddrop': verifiers.draganddrop} -======= -safe_exec_assumed_imports = [ - "random", - "numpy", - "math", - "scipy", - "calc", - "eia", - ("chemcalc", "chem.chemcalc"), - ("chemtools", "chem.chemtools"), - ("miller", "chem.miller"), - ("draganddrop", "verifiers.draganddrop"), -] ->>>>>>> Work in progress to sandbox the uses of eval in LMS. - # These should be removed from HTML output, including all subelements html_problem_semantics = ["codeparam", "responseparam", "answer", "script", "hintgroup", "openendedparam", "openendedrubric"] @@ -479,16 +439,6 @@ class LoncapaProblem(object): ''' random.seed(self.seed) - # TODO: REMOVE THIS COMMENTED OUT CODE. - ## save global context in here also - #context = {'global_context': global_context} - # - ## initialize context to have stuff in global_context - #context.update(global_context) - # - # put globals there also - #context['__builtins__'] = globals()['__builtins__'] - context = {} # pass instance of LoncapaProblem in @@ -523,7 +473,7 @@ class LoncapaProblem(object): context['script_code'] += code try: # use "context" for global context; thus defs in code are global within code - safe_exec(code, context, future_division=True, assumed_imports=safe_exec_assumed_imports) + safe_exec.safe_exec(code, context) except Exception as err: log.exception("Error while execing script code: " + code) msg = "Error while executing script code: %s" % str(err).replace('<', '<') diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index f732c9fc84..1b40742419 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -37,7 +37,7 @@ from lxml import etree from lxml.html.soupparser import fromstring as fromstring_bs # uses Beautiful Soup!!! FIXME? import xqueue_interface -from codejail.safe_exec import safe_exec +import safe_exec log = logging.getLogger(__name__) @@ -971,6 +971,11 @@ def sympy_check2(): if cfn: log.debug("cfn = %s" % cfn) + # This is a bit twisty. We used to grab the cfn function from + # the context, but now that we sandbox Python execution, we + # can't get functions from previous executions. So we make an + # actual function that will re-execute the original script, + # and invoke the function with the data needed. def make_check_function(script_code, cfn): def check_function(expect, ans): code = (script_code + "\n" + @@ -979,7 +984,7 @@ def sympy_check2(): 'expect': expect, 'ans': ans, } - safe_exec(code, globals_dict) + safe_exec.safe_exec(code, globals_dict) return globals_dict['cfn_return'] return check_function @@ -1924,7 +1929,7 @@ class SchematicResponse(LoncapaResponse): json.loads(student_answers[k]) for k in sorted(self.answer_ids) ] self.context.update({'submission': submission}) - safe_exec(self.code, {}, self.context) + safe_exec.safe_exec(self.code, {}, self.context) cmap = CorrectMap() cmap.set_dict(dict(zip(sorted( self.answer_ids), self.context['correct']))) diff --git a/common/lib/capa/capa/safe_exec.py b/common/lib/capa/capa/safe_exec.py new file mode 100644 index 0000000000..0f57ece529 --- /dev/null +++ b/common/lib/capa/capa/safe_exec.py @@ -0,0 +1,20 @@ +"""Capa's specialized use of codejail.safe_exec.""" + +import codejail.safe_exec + +def safe_exec(code, globals_dict, locals_dict=None): + codejail.safe_exec.safe_exec( + code, globals_dict, locals_dict, future_division=True, + assumed_imports=[ + "random", + "numpy", + "math", + "scipy", + "calc", + "eia", + ("chemcalc", "chem.chemcalc"), + ("chemtools", "chem.chemtools"), + ("miller", "chem.miller"), + ("draganddrop", "verifiers.draganddrop"), + ], + ) From a6677aa0a8cf8507844b89f2c5f0fc7d933ba729 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 14 Feb 2013 13:15:29 -0500 Subject: [PATCH 018/162] Computed answers are run through safe_exec. --- common/lib/capa/capa/responsetypes.py | 3 +-- .../data/embedded_python/course/2013_Spring.xml | 14 ++++++++++++++ lms/djangoapps/courseware/tests/tests.py | 5 +++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 1b40742419..6c8af09946 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -1087,8 +1087,7 @@ def sympy_check2(): # exec the check function if isinstance(self.code, basestring): try: - raise Exception("exec 1") - exec self.code in self.context['global_context'], self.context + safe_exec.safe_exec(self.code, self.context) correct = self.context['correct'] messages = self.context['messages'] overall_message = self.context['overall_message'] diff --git a/common/test/data/embedded_python/course/2013_Spring.xml b/common/test/data/embedded_python/course/2013_Spring.xml index 15e11befd1..fa6881c37b 100644 --- a/common/test/data/embedded_python/course/2013_Spring.xml +++ b/common/test/data/embedded_python/course/2013_Spring.xml @@ -92,6 +92,20 @@ print(num) + + + + + +if submission[0] == "Xyzzy": + correct = ['correct'] +else: + correct = ['incorrect'] + + + + + diff --git a/lms/djangoapps/courseware/tests/tests.py b/lms/djangoapps/courseware/tests/tests.py index ba9799b0df..9d3538d19b 100644 --- a/lms/djangoapps/courseware/tests/tests.py +++ b/lms/djangoapps/courseware/tests/tests.py @@ -1023,3 +1023,8 @@ class TestSchematicResponse(TestSubmittingProblems): resp = self.submit_question_answer('cfn_problem', {'2_1': "xyzzy!"}) respdata = json.loads(resp.content) self.assertEqual(respdata['success'], 'incorrect') + + def test_computed_answer(self): + resp = self.submit_question_answer('computed_answer', {'2_1': "Xyzzy"}) + respdata = json.loads(resp.content) + self.assertEqual(respdata['success'], 'correct') From 248017b4ea4cf4304550d001b12abc26f7cca603 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 14 Feb 2013 13:46:48 -0500 Subject: [PATCH 019/162] No longer need to support without , so scrap the code. --- common/lib/capa/capa/responsetypes.py | 63 +-------------------------- 1 file changed, 2 insertions(+), 61 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 6c8af09946..1b06ce63db 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -1318,10 +1318,8 @@ class CodeResponse(LoncapaResponse): # Check if XML uses the ExternalResponse format or the generic # CodeResponse format codeparam = self.xml.find('codeparam') - if codeparam is None: - self._parse_externalresponse_xml() - else: - self._parse_coderesponse_xml(codeparam) + assert codeparam is not None, "Unsupported old format! without " + self._parse_coderesponse_xml(codeparam) def _parse_coderesponse_xml(self, codeparam): ''' @@ -1341,63 +1339,6 @@ class CodeResponse(LoncapaResponse): self.answer = find_with_default(codeparam, 'answer_display', 'No answer provided.') - def _parse_externalresponse_xml(self): - ''' - VS[compat]: Suppport for old ExternalResponse XML format. When successful, sets: - self.initial_display - self.answer (an answer to display to the student in the LMS) - self.payload - ''' - answer = self.xml.find('answer') - - if answer is not None: - answer_src = answer.get('src') - if answer_src is not None: - code = self.system.filesystem.open('src/' + answer_src).read() - else: - code = answer.text - else: # no stanza; get code from ''' - snippets = [{'snippet': r""" - -
- Suppose that \(I(t)\) rises from \(0\) to \(I_S\) at a time \(t_0 \neq 0\) - In the space provided below write an algebraic expression for \(I(t)\). -
- - - - correct=['correct'] - try: - r = str(submission[0]) - except ValueError: - correct[0] ='incorrect' - r = '0' - if not(r=="IS*u(t-t0)"): - correct[0] ='incorrect' - - """}, - {'snippet': """ - - - - - """}] response_tag = 'customresponse' @@ -1245,16 +1195,6 @@ class SymbolicResponse(CustomResponse): """ Symbolic math response checking, using symmath library. """ - snippets = [{'snippet': r''' - Compute \[ \exp\left(-i \frac{\theta}{2} \left[ \begin{matrix} 0 & 1 \\ 1 & 0 \end{matrix} \right] \right) \] - and give the resulting \(2\times 2\) matrix:
- - - -
- Your input should be typed in as a list of lists, eg [[1,2],[3,4]]. - - '''}] response_tag = 'symbolicresponse' @@ -1518,44 +1458,6 @@ class ExternalResponse(LoncapaResponse): Typically used by coding problems. ''' - snippets = [{'snippet': ''' - - - - '''}] response_tag = 'externalresponse' allowed_inputfields = ['textline', 'textbox'] @@ -1701,23 +1603,6 @@ class FormulaResponse(LoncapaResponse): ''' Checking of symbolic math response using numerical sampling. ''' - snippets = [{'snippet': ''' - - - - -
- Give an equation for the relativistic energy of an object with mass m. - - - - - - - '''}] response_tag = 'formularesponse' hint_tag = 'formulahint' @@ -1908,19 +1793,6 @@ class ImageResponse(LoncapaResponse): Returns: True, if click is inside any region or rectangle. Otherwise False. """ - snippets = [{'snippet': ''' - - - - - - '''}] response_tag = 'imageresponse' allowed_inputfields = ['imageinput'] From 19e3a0ceb9faaabda80c82c06575af6947ebae06 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 14 Feb 2013 16:06:05 -0500 Subject: [PATCH 022/162] Implement safe_exec on top of jailpy (old unsafe safe_exec is still here); Remove some crazy stuff from the context; always pass globals and locals, locals are the things that can be changed. --- common/lib/capa/capa/capa_problem.py | 7 +- common/lib/capa/capa/responsetypes.py | 18 ++--- common/lib/capa/capa/safe_exec.py | 2 +- common/lib/codejail/codejail/safe_exec.py | 68 +++++++++++++++---- .../codejail/codejail/tests/test_jailpy.py | 7 ++ 5 files changed, 73 insertions(+), 29 deletions(-) diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index 5518da8318..202d197428 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -440,9 +440,6 @@ class LoncapaProblem(object): random.seed(self.seed) context = {} - - # pass instance of LoncapaProblem in - context['the_lcp'] = self context['script_code'] = '' self._execute_scripts(tree.findall('.//script'), context) @@ -473,7 +470,9 @@ class LoncapaProblem(object): context['script_code'] += code try: # use "context" for global context; thus defs in code are global within code - safe_exec.safe_exec(code, context) + locals_dict = {} + safe_exec.safe_exec(code, context, locals_dict) + context.update(locals_dict) except Exception as err: log.exception("Error while execing script code: " + code) msg = "Error while executing script code: %s" % str(err).replace('<', '<') diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 3aabc4adbd..1471093a98 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -934,8 +934,9 @@ class CustomResponse(LoncapaResponse): 'expect': expect, 'ans': ans, } - safe_exec.safe_exec(code, globals_dict) - return globals_dict['cfn_return'] + locals_dict = {} + safe_exec.safe_exec(code, globals_dict, locals_dict) + return locals_dict['cfn_return'] return check_function self.code = make_check_function(self.context['script_code'], cfn) @@ -995,9 +996,6 @@ class CustomResponse(LoncapaResponse): # put these in the context of the check function evaluator # note that this doesn't help the "cfn" version - only the exec version self.context.update({ - # our subtree - 'xml': self.xml, - # my ID 'response_id': self.myid, @@ -1037,7 +1035,9 @@ class CustomResponse(LoncapaResponse): # exec the check function if isinstance(self.code, basestring): try: - safe_exec.safe_exec(self.code, self.context) + locals_dict = {} + safe_exec.safe_exec(self.code, self.context, locals_dict) + self.context.update(locals_dict) correct = self.context['correct'] messages = self.context['messages'] overall_message = self.context['overall_message'] @@ -1754,10 +1754,10 @@ class SchematicResponse(LoncapaResponse): json.loads(student_answers[k]) for k in sorted(self.answer_ids) ] self.context.update({'submission': submission}) - safe_exec.safe_exec(self.code, {}, self.context) + locals_dict = {} + safe_exec.safe_exec(self.code, self.context, locals_dict) cmap = CorrectMap() - cmap.set_dict(dict(zip(sorted( - self.answer_ids), self.context['correct']))) + cmap.set_dict(dict(zip(sorted(self.answer_ids), locals_dict['correct']))) return cmap def get_answers(self): diff --git a/common/lib/capa/capa/safe_exec.py b/common/lib/capa/capa/safe_exec.py index 0f57ece529..e6fa0e7bb5 100644 --- a/common/lib/capa/capa/safe_exec.py +++ b/common/lib/capa/capa/safe_exec.py @@ -2,7 +2,7 @@ import codejail.safe_exec -def safe_exec(code, globals_dict, locals_dict=None): +def safe_exec(code, globals_dict, locals_dict): codejail.safe_exec.safe_exec( code, globals_dict, locals_dict, future_division=True, assumed_imports=[ diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index 92beba98cc..9de8398030 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -1,13 +1,15 @@ """Safe execution of untrusted Python code.""" import json +import textwrap -from .lazymod import LazyModule +import lazymod +import jailpy -def straw(v): - return json.loads(json.dumps(jsonable_dict(v))) - -def jsonable_dict(d): +# If we aren't running safe, then we need to artificially pass the values +# through a JSON straw to ensure we aren't passing something that won't +# be executable in the safe context. +def straw(d): jd = {} for k,v in d.iteritems(): try: @@ -16,9 +18,9 @@ def jsonable_dict(d): continue else: jd[k] = v - return jd + return json.loads(json.dumps(jd)) -def safe_exec(code, globals_dict, locals_dict=None, future_division=False, assumed_imports=None): +def safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_imports=None): """Execute code safely. Returns None. The code can modify globals in `global_dict`. @@ -28,21 +30,57 @@ def safe_exec(code, globals_dict, locals_dict=None, future_division=False, assum code = "from __future__ import division\n" + code g_dict = straw(globals_dict) - - if locals_dict is None: - l_dict = g_dict - else: - l_dict = straw(locals_dict) + l_dict = straw(locals_dict) for modname in assumed_imports or (): if isinstance(modname, tuple): name, modname = modname else: name = modname - g_dict[name] = LazyModule(modname) + g_dict[name] = lazymod.LazyModule(modname) exec code in g_dict, l_dict globals_dict.update(straw(g_dict)) - if locals_dict is not None: - locals_dict.update(straw(l_dict)) + locals_dict.update(straw(l_dict)) + +# We'll need the code from lazymod.py for use in jailpy, so read it now. +lazymod_py_file = lazymod.__file__ +if lazymod_py_file.endswith("c"): + lazymod_py_file = lazymod_py_file[:-1] + +lazymod_py = open(lazymod_py_file).read() + + +def xxxsafe_exec(code, globals_dict, locals_dict, future_division=False, assumed_imports=None): + the_code = [] + + the_code.append(textwrap.dedent("""\ + import json + import sys + code, g_dict, l_dict = json.load(sys.stdin) + """)) + + if assumed_imports: + the_code.append(lazymod_py) + for modname in assumed_imports: + if isinstance(modname, tuple): + name, modname = modname + else: + name = modname + the_code.append("g_dict['{}'] = LazyModule('{}')\n".format(name, modname)) + + the_code.append(textwrap.dedent("""\ + exec code in g_dict, l_dict + print >>sys.stderr, l_dict.keys() + ok_types = (int, long, float, str, unicode, list, tuple, dict) + l_dict = {k:v for k,v in l_dict.iteritems() if isinstance(v, ok_types)} + json.dump(l_dict, sys.stdout) + """)) + + print "".join(the_code) + stdin = json.dumps([code, globals_dict, locals_dict]) + res = jailpy.jailpy("".join(the_code), stdin=stdin) + if res.status != 0: + raise Exception("Couldn't excecute jailed code: %s" % res.stderr) + locals_dict.update(json.loads(res.stdout)) diff --git a/common/lib/codejail/codejail/tests/test_jailpy.py b/common/lib/codejail/codejail/tests/test_jailpy.py index d1133d745d..a04bf3bf0f 100644 --- a/common/lib/codejail/codejail/tests/test_jailpy.py +++ b/common/lib/codejail/codejail/tests/test_jailpy.py @@ -33,6 +33,13 @@ class TestFeatures(unittest.TestCase): Exception: FAIL """)) + def test_stdin_is_provided(self): + res = jailpy( + "import json,sys; print sum(json.load(sys.stdin))", + stdin="[1, 2.5, 33]" + ) + self.assertEqual(res.stdout.strip(), "36.5") + class TestLimits(unittest.TestCase): def test_cant_use_too_much_memory(self): From 5db5426e05f443ab4e29f4750be23f01ae9fb8ef Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 15 Feb 2013 10:22:21 -0500 Subject: [PATCH 023/162] Use the real safe_exec; make the seed available in the context. --- common/lib/capa/capa/capa_problem.py | 1 + common/lib/capa/capa/responsetypes.py | 3 +-- common/lib/codejail/codejail/safe_exec.py | 14 +++++++++++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index 202d197428..888501e8b5 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -440,6 +440,7 @@ class LoncapaProblem(object): random.seed(self.seed) context = {} + context['seed'] = self.seed context['script_code'] = '' self._execute_scripts(tree.findall('.//script'), context) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 1471093a98..7213a766ce 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -489,7 +489,7 @@ class JavascriptResponse(LoncapaResponse): output = self.call_node([generator_file, self.generator, json.dumps(self.generator_dependencies), - json.dumps(str(self.context['the_lcp'].seed)), + json.dumps(str(self.context['seed'])), json.dumps(self.params)]).strip() return json.loads(output) @@ -1201,7 +1201,6 @@ class SymbolicResponse(CustomResponse): def setup_response(self): self.xml.set('cfn', 'symmath_check') code = "from symmath import *" - raise Exception("exec 2") exec code in self.context, self.context CustomResponse.setup_response(self) diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index 9de8398030..ba5d3fb5bc 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -52,9 +52,12 @@ if lazymod_py_file.endswith("c"): lazymod_py = open(lazymod_py_file).read() -def xxxsafe_exec(code, globals_dict, locals_dict, future_division=False, assumed_imports=None): +def safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_imports=None): the_code = [] + if future_division: + the_code.append("from __future__ import division\n") + the_code.append(textwrap.dedent("""\ import json import sys @@ -73,12 +76,17 @@ def xxxsafe_exec(code, globals_dict, locals_dict, future_division=False, assumed the_code.append(textwrap.dedent("""\ exec code in g_dict, l_dict print >>sys.stderr, l_dict.keys() - ok_types = (int, long, float, str, unicode, list, tuple, dict) + ok_types = (type(None), int, long, float, str, unicode, list, tuple, dict) l_dict = {k:v for k,v in l_dict.iteritems() if isinstance(v, ok_types)} json.dump(l_dict, sys.stdout) """)) - print "".join(the_code) + if 0: + print "-- {:-<40}".format("jailed ") + print "".join(the_code) + print "-- {:-<40}".format("exec ") + print code + stdin = json.dumps([code, globals_dict, locals_dict]) res = jailpy.jailpy("".join(the_code), stdin=stdin) if res.status != 0: From 70930c25c10438c45569aa3fa4c79191e8c3f3f6 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 15 Feb 2013 11:58:36 -0500 Subject: [PATCH 024/162] Remove the unsafe version of safe_exec, and document the safe one. --- common/lib/codejail/codejail/safe_exec.py | 54 +++++++---------------- 1 file changed, 16 insertions(+), 38 deletions(-) diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index ba5d3fb5bc..7a39a57690 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -6,44 +6,6 @@ import textwrap import lazymod import jailpy -# If we aren't running safe, then we need to artificially pass the values -# through a JSON straw to ensure we aren't passing something that won't -# be executable in the safe context. -def straw(d): - jd = {} - for k,v in d.iteritems(): - try: - json.dumps(v) - except TypeError: - continue - else: - jd[k] = v - return json.loads(json.dumps(jd)) - -def safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_imports=None): - """Execute code safely. - - Returns None. The code can modify globals in `global_dict`. - - """ - if future_division: - code = "from __future__ import division\n" + code - - g_dict = straw(globals_dict) - l_dict = straw(locals_dict) - - for modname in assumed_imports or (): - if isinstance(modname, tuple): - name, modname = modname - else: - name = modname - g_dict[name] = lazymod.LazyModule(modname) - - exec code in g_dict, l_dict - - globals_dict.update(straw(g_dict)) - locals_dict.update(straw(l_dict)) - # We'll need the code from lazymod.py for use in jailpy, so read it now. lazymod_py_file = lazymod.__file__ if lazymod_py_file.endswith("c"): @@ -53,6 +15,22 @@ lazymod_py = open(lazymod_py_file).read() def safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_imports=None): + """Execute code as "exec" does, but safely. + + `code` is a string of Python code. `globals_dict` and `locals_dict` are + dictionaries to use as the globals and locals. Modifications the code + makes to `locals_dict` are reflected in the dictionary on return. + + `future_division` determines whether Python-3-style division is used. + + `assumed_imports` is a list of module to make available as implicit + imports for the code. Entries are either a name, "mod", which makes + "import mod" part of the code, or a pair, ("fooey", "f"), which makes + "import fooey as f" part of the code. The module name can be dotted. + + Returns None, changes made by `code` are visible in `locals_dict`. + + """ the_code = [] if future_division: From 7c498be606daea6177ce17e29d90457c9d912e84 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 19 Feb 2013 11:07:11 -0500 Subject: [PATCH 025/162] Move packages around so we can install packages into the sandbox. --- common/djangoapps/util/views.py | 4 ++-- common/lib/{capa/capa => calc}/calc.py | 0 common/lib/calc/setup.py | 12 +++++++++++ common/lib/capa/setup.py | 2 +- .../lib/{capa/capa => chem}/chem/__init__.py | 0 .../lib/{capa/capa => chem}/chem/chemcalc.py | 0 .../lib/{capa/capa => chem}/chem/chemtools.py | 0 common/lib/{capa/capa => chem}/chem/miller.py | 0 common/lib/{capa/capa => chem}/chem/tests.py | 0 common/lib/chem/setup.py | 13 ++++++++++++ common/lib/codejail/README | 21 +++++++++++++++++++ common/lib/sandbox-packages/README | 1 + .../{capa/capa => sandbox-packages}/eia.py | 0 common/lib/sandbox-packages/setup.py | 17 +++++++++++++++ .../lib/sandbox-packages}/symmath/README.md | 0 .../lib/sandbox-packages}/symmath/__init__.py | 0 .../lib/sandbox-packages}/symmath/formula.py | 0 .../symmath/symmath_check.py | 0 .../verifiers/__init__.py | 0 .../verifiers/draganddrop.py | 0 .../verifiers/tests_draganddrop.py | 0 common/lib/xmodule/xmodule/tests/__init__.py | 2 +- local-requirements.txt | 4 +++- sandbox-requirements.txt | 6 ++++++ 24 files changed, 77 insertions(+), 5 deletions(-) rename common/lib/{capa/capa => calc}/calc.py (100%) create mode 100644 common/lib/calc/setup.py rename common/lib/{capa/capa => chem}/chem/__init__.py (100%) rename common/lib/{capa/capa => chem}/chem/chemcalc.py (100%) rename common/lib/{capa/capa => chem}/chem/chemtools.py (100%) rename common/lib/{capa/capa => chem}/chem/miller.py (100%) rename common/lib/{capa/capa => chem}/chem/tests.py (100%) create mode 100644 common/lib/chem/setup.py create mode 100644 common/lib/codejail/README create mode 100644 common/lib/sandbox-packages/README rename common/lib/{capa/capa => sandbox-packages}/eia.py (100%) create mode 100644 common/lib/sandbox-packages/setup.py rename {lms/lib => common/lib/sandbox-packages}/symmath/README.md (100%) rename {lms/lib => common/lib/sandbox-packages}/symmath/__init__.py (100%) rename {lms/lib => common/lib/sandbox-packages}/symmath/formula.py (100%) rename {lms/lib => common/lib/sandbox-packages}/symmath/symmath_check.py (100%) rename common/lib/{capa/capa => sandbox-packages}/verifiers/__init__.py (100%) rename common/lib/{capa/capa => sandbox-packages}/verifiers/draganddrop.py (100%) rename common/lib/{capa/capa => sandbox-packages}/verifiers/tests_draganddrop.py (100%) create mode 100644 sandbox-requirements.txt diff --git a/common/djangoapps/util/views.py b/common/djangoapps/util/views.py index cece37757b..75fdfac210 100644 --- a/common/djangoapps/util/views.py +++ b/common/djangoapps/util/views.py @@ -12,7 +12,7 @@ from django.http import HttpResponse from django.shortcuts import redirect from mitxmako.shortcuts import render_to_response, render_to_string -import capa.calc +import calc import track.views @@ -20,7 +20,7 @@ def calculate(request): ''' Calculator in footer of every page. ''' equation = request.GET['equation'] try: - result = capa.calc.evaluator({}, {}, equation) + result = calc.evaluator({}, {}, equation) except: event = {'error': map(str, sys.exc_info()), 'equation': equation} diff --git a/common/lib/capa/capa/calc.py b/common/lib/calc/calc.py similarity index 100% rename from common/lib/capa/capa/calc.py rename to common/lib/calc/calc.py diff --git a/common/lib/calc/setup.py b/common/lib/calc/setup.py new file mode 100644 index 0000000000..f7bb1708af --- /dev/null +++ b/common/lib/calc/setup.py @@ -0,0 +1,12 @@ +from setuptools import setup + +setup( + name="calc", + version="0.1", + py_modules=["calc"], + install_requires=[ + "pyparsing==1.5.6", + "numpy", + "scipy" + ], +) diff --git a/common/lib/capa/setup.py b/common/lib/capa/setup.py index d9c813f55c..5f1731fdc0 100644 --- a/common/lib/capa/setup.py +++ b/common/lib/capa/setup.py @@ -4,5 +4,5 @@ setup( name="capa", version="0.1", packages=find_packages(exclude=["tests"]), - install_requires=['distribute==0.6.30', 'pyparsing==1.5.6'], + install_requires=["distribute==0.6.34"], ) diff --git a/common/lib/capa/capa/chem/__init__.py b/common/lib/chem/chem/__init__.py similarity index 100% rename from common/lib/capa/capa/chem/__init__.py rename to common/lib/chem/chem/__init__.py diff --git a/common/lib/capa/capa/chem/chemcalc.py b/common/lib/chem/chem/chemcalc.py similarity index 100% rename from common/lib/capa/capa/chem/chemcalc.py rename to common/lib/chem/chem/chemcalc.py diff --git a/common/lib/capa/capa/chem/chemtools.py b/common/lib/chem/chem/chemtools.py similarity index 100% rename from common/lib/capa/capa/chem/chemtools.py rename to common/lib/chem/chem/chemtools.py diff --git a/common/lib/capa/capa/chem/miller.py b/common/lib/chem/chem/miller.py similarity index 100% rename from common/lib/capa/capa/chem/miller.py rename to common/lib/chem/chem/miller.py diff --git a/common/lib/capa/capa/chem/tests.py b/common/lib/chem/chem/tests.py similarity index 100% rename from common/lib/capa/capa/chem/tests.py rename to common/lib/chem/chem/tests.py diff --git a/common/lib/chem/setup.py b/common/lib/chem/setup.py new file mode 100644 index 0000000000..4f2b24ddee --- /dev/null +++ b/common/lib/chem/setup.py @@ -0,0 +1,13 @@ +from setuptools import setup + +setup( + name="chem", + version="0.1", + packages=["chem"], + install_requires=[ + "pyparsing==1.5.6", + "numpy", + "scipy", + "nltk==2.0.4", + ], +) diff --git a/common/lib/codejail/README b/common/lib/codejail/README new file mode 100644 index 0000000000..7b1849e18c --- /dev/null +++ b/common/lib/codejail/README @@ -0,0 +1,21 @@ +Choose a place for the virtualenv, call it + +Create a virtualenv: + + virtualenv + +Install the sandbox requirements + + +Edit an AppArmor profile: + + /bin/python { + ... + } + +Parse the profiles + + $ apparmor_parser + $ aaenforce /bin/python + + diff --git a/common/lib/sandbox-packages/README b/common/lib/sandbox-packages/README new file mode 100644 index 0000000000..706998b08e --- /dev/null +++ b/common/lib/sandbox-packages/README @@ -0,0 +1 @@ +This directory is in the Python path for sandboxed Python execution. diff --git a/common/lib/capa/capa/eia.py b/common/lib/sandbox-packages/eia.py similarity index 100% rename from common/lib/capa/capa/eia.py rename to common/lib/sandbox-packages/eia.py diff --git a/common/lib/sandbox-packages/setup.py b/common/lib/sandbox-packages/setup.py new file mode 100644 index 0000000000..f5cfa91b8b --- /dev/null +++ b/common/lib/sandbox-packages/setup.py @@ -0,0 +1,17 @@ +from setuptools import setup + +setup( + name="sandbox-packages", + version="0.1", + packages=[ + "symmath", + "verifiers", + ], + py_modules=[ + "eia", + ], + install_requires=[ + # symmath needs: + "sympy", "requests", "lxml", + ], +) diff --git a/lms/lib/symmath/README.md b/common/lib/sandbox-packages/symmath/README.md similarity index 100% rename from lms/lib/symmath/README.md rename to common/lib/sandbox-packages/symmath/README.md diff --git a/lms/lib/symmath/__init__.py b/common/lib/sandbox-packages/symmath/__init__.py similarity index 100% rename from lms/lib/symmath/__init__.py rename to common/lib/sandbox-packages/symmath/__init__.py diff --git a/lms/lib/symmath/formula.py b/common/lib/sandbox-packages/symmath/formula.py similarity index 100% rename from lms/lib/symmath/formula.py rename to common/lib/sandbox-packages/symmath/formula.py diff --git a/lms/lib/symmath/symmath_check.py b/common/lib/sandbox-packages/symmath/symmath_check.py similarity index 100% rename from lms/lib/symmath/symmath_check.py rename to common/lib/sandbox-packages/symmath/symmath_check.py diff --git a/common/lib/capa/capa/verifiers/__init__.py b/common/lib/sandbox-packages/verifiers/__init__.py similarity index 100% rename from common/lib/capa/capa/verifiers/__init__.py rename to common/lib/sandbox-packages/verifiers/__init__.py diff --git a/common/lib/capa/capa/verifiers/draganddrop.py b/common/lib/sandbox-packages/verifiers/draganddrop.py similarity index 100% rename from common/lib/capa/capa/verifiers/draganddrop.py rename to common/lib/sandbox-packages/verifiers/draganddrop.py diff --git a/common/lib/capa/capa/verifiers/tests_draganddrop.py b/common/lib/sandbox-packages/verifiers/tests_draganddrop.py similarity index 100% rename from common/lib/capa/capa/verifiers/tests_draganddrop.py rename to common/lib/sandbox-packages/verifiers/tests_draganddrop.py diff --git a/common/lib/xmodule/xmodule/tests/__init__.py b/common/lib/xmodule/xmodule/tests/__init__.py index 1a10654f6c..79e1fec0f0 100644 --- a/common/lib/xmodule/xmodule/tests/__init__.py +++ b/common/lib/xmodule/xmodule/tests/__init__.py @@ -14,7 +14,7 @@ import fs.osfs import numpy -import capa.calc as calc +import calc import xmodule from xmodule.x_module import ModuleSystem from mock import Mock diff --git a/local-requirements.txt b/local-requirements.txt index 053a223605..4ad1ef6636 100644 --- a/local-requirements.txt +++ b/local-requirements.txt @@ -1,5 +1,7 @@ # Python libraries to install that are local to the mitx repo +-e common/lib/calc -e common/lib/capa --e common/lib/xmodule +-e common/lib/chem -e common/lib/codejail +-e common/lib/xmodule -e . diff --git a/sandbox-requirements.txt b/sandbox-requirements.txt new file mode 100644 index 0000000000..9b18fba9e9 --- /dev/null +++ b/sandbox-requirements.txt @@ -0,0 +1,6 @@ +# Packages to install in the Python sandbox for secured execution. +numpy==1.6.2 +scipy==0.11.0 +-e common/lib/calc +-e common/lib/chem +-e common/lib/sandbox-packages From ebb2624719307b674ccbfff99a606dc87a4e083e Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 19 Feb 2013 11:08:09 -0500 Subject: [PATCH 026/162] Make jailpy tests more convenient and informative. --- .../codejail/codejail/tests/test_jailpy.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/common/lib/codejail/codejail/tests/test_jailpy.py b/common/lib/codejail/codejail/tests/test_jailpy.py index a04bf3bf0f..74c475b63d 100644 --- a/common/lib/codejail/codejail/tests/test_jailpy.py +++ b/common/lib/codejail/codejail/tests/test_jailpy.py @@ -8,10 +8,17 @@ from codejail.jailpy import jailpy dedent = textwrap.dedent -class TestFeatures(unittest.TestCase): +class JailPyHelpers(object): + """Assert helpers for jailpy tests.""" + def assertResultOk(self, res): + self.assertEqual(res.stderr, "") + self.assertEqual(res.status, 0) + + +class TestFeatures(JailPyHelpers, unittest.TestCase): def test_hello_world(self): res = jailpy("print 'Hello, world!'") - self.assertEqual(res.status, 0) + self.assertResultOk(res) self.assertEqual(res.stdout, 'Hello, world!\n') def test_argv(self): @@ -19,7 +26,7 @@ class TestFeatures(unittest.TestCase): "import sys; print ':'.join(sys.argv[1:])", argv=["Hello", "world", "-x"] ) - self.assertEqual(res.status, 0) + self.assertResultOk(res) self.assertEqual(res.stdout, "Hello:world:-x\n") def test_ends_with_exception(self): @@ -38,10 +45,11 @@ class TestFeatures(unittest.TestCase): "import json,sys; print sum(json.load(sys.stdin))", stdin="[1, 2.5, 33]" ) + self.assertResultOk(res) self.assertEqual(res.stdout.strip(), "36.5") -class TestLimits(unittest.TestCase): +class TestLimits(JailPyHelpers, unittest.TestCase): def test_cant_use_too_much_memory(self): res = jailpy("print sum(range(100000000))") self.assertNotEqual(res.status, 0) @@ -78,7 +86,7 @@ class TestLimits(unittest.TestCase): # TODO: read network # TODO: fork -class TestMalware(unittest.TestCase): +class TestMalware(JailPyHelpers, unittest.TestCase): def test_crash_cpython(self): # http://nedbatchelder.com/blog/201206/eval_really_is_dangerous.html res = jailpy(dedent("""\ @@ -119,5 +127,5 @@ class TestMalware(unittest.TestCase): print "Files in %r: %r" % (place, files) print "Done." """)) - self.assertEqual(res.status, 0) + self.assertResultOk(res) self.assertEqual(res.stdout, "Done.\n") From 3316aeb0325a5981bef61dbec32c717276240c9e Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 19 Feb 2013 11:48:50 -0500 Subject: [PATCH 027/162] Add back the not_safe_exec implementation, for debugging. --- common/lib/codejail/codejail/safe_exec.py | 58 ++++++++++++++++++++--- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index 7a39a57690..824a6c2d57 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -23,12 +23,12 @@ def safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_im `future_division` determines whether Python-3-style division is used. - `assumed_imports` is a list of module to make available as implicit + `assumed_imports` is a list of modules to make available as implicit imports for the code. Entries are either a name, "mod", which makes - "import mod" part of the code, or a pair, ("fooey", "f"), which makes + "import mod" part of the code, or a pair, ("f", "fooey"), which makes "import fooey as f" part of the code. The module name can be dotted. - Returns None, changes made by `code` are visible in `locals_dict`. + Returns None. Changes made by `code` are visible in `locals_dict`. """ the_code = [] @@ -53,16 +53,15 @@ def safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_im the_code.append(textwrap.dedent("""\ exec code in g_dict, l_dict - print >>sys.stderr, l_dict.keys() ok_types = (type(None), int, long, float, str, unicode, list, tuple, dict) l_dict = {k:v for k,v in l_dict.iteritems() if isinstance(v, ok_types)} json.dump(l_dict, sys.stdout) """)) - if 0: - print "-- {:-<40}".format("jailed ") + if 1: + print "--{:-<40}".format(" jailed ") print "".join(the_code) - print "-- {:-<40}".format("exec ") + print "--{:-<40}".format(" exec ") print code stdin = json.dumps([code, globals_dict, locals_dict]) @@ -70,3 +69,48 @@ def safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_im if res.status != 0: raise Exception("Couldn't excecute jailed code: %s" % res.stderr) locals_dict.update(json.loads(res.stdout)) + + +def not_safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_imports=None): + """Another implementation of `safe_exec`, but not safe. + + This can be swapped in for debugging problems in sandboxed Python code. + + """ + def straw(d): + """Return only the JSON-safe part of d. + + Used to emulate reading data through a serialization straw. + + """ + jd = {} + for k,v in d.iteritems(): + try: + json.dumps(v) + except TypeError: + continue + else: + jd[k] = v + return json.loads(json.dumps(jd)) + + if future_division: + code = "from __future__ import division\n" + code + + g_dict = straw(globals_dict) + l_dict = straw(locals_dict) + + for modname in assumed_imports or (): + if isinstance(modname, tuple): + name, modname = modname + else: + name = modname + g_dict[name] = lazymod.LazyModule(modname) + + exec code in g_dict, l_dict + + locals_dict.update(straw(l_dict)) + +# Running Python code in the sandbox makes it difficult to debug. +# Turn this on to run the code directly. +if 1: + safe_exec = not_safe_exec From e8da1b8f61e1ed40209e8a4c5088f92f9af17b39 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 19 Feb 2013 12:43:58 -0500 Subject: [PATCH 028/162] Turn off our debugging levers --- common/lib/codejail/codejail/safe_exec.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index 824a6c2d57..89e04c17f3 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -58,7 +58,8 @@ def safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_im json.dump(l_dict, sys.stdout) """)) - if 1: + # Turn this on to see what's being executed. + if 0: print "--{:-<40}".format(" jailed ") print "".join(the_code) print "--{:-<40}".format(" exec ") @@ -112,5 +113,5 @@ def not_safe_exec(code, globals_dict, locals_dict, future_division=False, assume # Running Python code in the sandbox makes it difficult to debug. # Turn this on to run the code directly. -if 1: +if 0: safe_exec = not_safe_exec From 716a97ea59dc2a12af3b03340fe6b25a6e461a37 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 19 Feb 2013 13:42:47 -0500 Subject: [PATCH 029/162] Symbolic response doesn't need to pre-import symmath any more, I think? --- common/lib/capa/capa/responsetypes.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 7213a766ce..743edb358a 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -1200,8 +1200,6 @@ class SymbolicResponse(CustomResponse): def setup_response(self): self.xml.set('cfn', 'symmath_check') - code = "from symmath import *" - exec code in self.context, self.context CustomResponse.setup_response(self) #----------------------------------------------------------------------------- From 30748a06ff9494621b2e04a2eac198293b48f539 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 19 Feb 2013 13:51:45 -0500 Subject: [PATCH 030/162] Try to find the sandbox in a few places. --- common/lib/codejail/codejail/jailpy.py | 27 +++++++++++++------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/common/lib/codejail/codejail/jailpy.py b/common/lib/codejail/codejail/jailpy.py index 45842d715f..2a8fd61b98 100644 --- a/common/lib/codejail/codejail/jailpy.py +++ b/common/lib/codejail/codejail/jailpy.py @@ -14,24 +14,23 @@ from .util import temp_directory # TODO: limit too much stdout data? -DEBUG = False -STRICT = True - # Configure the Python command -SANDBOX_PYTHON = "/usr/bin/python-sandbox" +SANDBOX_POSSIBILITIES = [ + "~/mitx_all/python-sandbox/bin/python", + "/usr/bin/python-sandbox", +] -if os.path.exists(SANDBOX_PYTHON): - # Python -S inhibits loading site.py, which prevent Ubuntu from adding - # specialized traceback handlers that fail in the sandbox. - PYTHON_CMD = [ - 'sudo', '-u', 'sandbox', - SANDBOX_PYTHON, '-S' - ] -elif STRICT: - raise Exception("Couldn't find Python sandbox") +for sandbox_python in SANDBOX_POSSIBILITIES: + sandbox_python = os.path.expanduser(sandbox_python) + if os.path.exists(sandbox_python): + PYTHON_CMD = [ + 'sudo', '-u', 'sandbox', + sandbox_python, '-E', + ] + break else: - PYTHON_CMD = ['python', '-S'] + raise Exception("Couldn't find Python sandbox") class JailResult(object): From 5d4b61c7f588923663020da6856e529c126bc5a5 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 19 Feb 2013 15:47:46 -0500 Subject: [PATCH 031/162] Better configuration for codejail. --- .../codejail/codejail/django_integration.py | 16 +++++++++ common/lib/codejail/codejail/jailpy.py | 35 +++++++++++-------- .../codejail/codejail/tests/test_jailpy.py | 7 +++- lms/envs/common.py | 11 ++++++ 4 files changed, 54 insertions(+), 15 deletions(-) create mode 100644 common/lib/codejail/codejail/django_integration.py diff --git a/common/lib/codejail/codejail/django_integration.py b/common/lib/codejail/codejail/django_integration.py new file mode 100644 index 0000000000..52306ba6f1 --- /dev/null +++ b/common/lib/codejail/codejail/django_integration.py @@ -0,0 +1,16 @@ +"""Django integration for codejail""" + +from django.core.exceptions import MiddlewareNotUsed +from django.conf import settings + +import codejail.jailpy + +class ConfigureCodeJailMiddleware(object): + """Middleware to configure codejail on startup.""" + + def __init__(self): + python_bin = settings.CODE_JAIL.get('python_bin') + if python_bin: + user = settings.CODE_JAIL['user'] + codejail.jailpy.configure(python_bin, user=user) + raise MiddlewareNotUsed diff --git a/common/lib/codejail/codejail/jailpy.py b/common/lib/codejail/codejail/jailpy.py index 2a8fd61b98..8f68ab33de 100644 --- a/common/lib/codejail/codejail/jailpy.py +++ b/common/lib/codejail/codejail/jailpy.py @@ -7,6 +7,7 @@ import os, os.path import resource import shutil import subprocess +import sys import threading import time @@ -16,21 +17,24 @@ from .util import temp_directory # Configure the Python command -SANDBOX_POSSIBILITIES = [ - "~/mitx_all/python-sandbox/bin/python", - "/usr/bin/python-sandbox", -] +PYTHON_CMD = None -for sandbox_python in SANDBOX_POSSIBILITIES: - sandbox_python = os.path.expanduser(sandbox_python) - if os.path.exists(sandbox_python): - PYTHON_CMD = [ - 'sudo', '-u', 'sandbox', - sandbox_python, '-E', - ] - break -else: - raise Exception("Couldn't find Python sandbox") +def configure(python_bin, user=None): + """Configure the jailpy module.""" + global PYTHON_CMD + PYTHON_CMD = [] + if user: + PYTHON_CMD.extend(['sudo', '-u', 'sandbox']) + PYTHON_CMD.extend([python_bin, '-E']) + +def is_configured(): + return bool(PYTHON_CMD) + +# By default, look where our current Python is, and maybe there's a +# python-sandbox alongside. Only do this if running in a virtualenv. +if hasattr(sys, 'real_prefix'): + if os.path.isdir(sys.prefix + "-sandbox"): + configure(sys.prefix + "-sandbox/bin/python", "sandbox") class JailResult(object): @@ -52,6 +56,9 @@ def jailpy(code, files=None, argv=None, stdin=None): .status: return status of the process: an int, 0 for successful """ + if not PYTHON_CMD: + raise Exception("jailpy needs to be configured") + with temp_directory(delete_when_done=True) as tmpdir: # All the supporting files are copied into our directory. diff --git a/common/lib/codejail/codejail/tests/test_jailpy.py b/common/lib/codejail/codejail/tests/test_jailpy.py index 74c475b63d..3c6d13b714 100644 --- a/common/lib/codejail/codejail/tests/test_jailpy.py +++ b/common/lib/codejail/codejail/tests/test_jailpy.py @@ -4,12 +4,17 @@ import textwrap import unittest from nose.plugins.skip import SkipTest -from codejail.jailpy import jailpy +from codejail.jailpy import jailpy, is_configured dedent = textwrap.dedent class JailPyHelpers(object): """Assert helpers for jailpy tests.""" + def setUp(self): + super(JailPyHelpers, self).setUp() + if not is_configured(): + raise SkipTest + def assertResultOk(self, res): self.assertEqual(res.stderr, "") self.assertEqual(res.status, 0) diff --git a/lms/envs/common.py b/lms/envs/common.py index 32a213f06e..b99efd60c3 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -242,6 +242,16 @@ MODULESTORE = { } CONTENTSTORE = None +#################### Python sandbox ############################################ + +CODE_JAIL = { + # Path to a sandboxed Python executable. None means don't bother. + 'python_bin': None, + # User to run as in the sandbox. + 'user': 'sandbox', +} + + ############################ SIGNAL HANDLERS ################################ # This is imported to register the exception signal handling that logs exceptions import monitoring.exceptions # noqa @@ -385,6 +395,7 @@ MIDDLEWARE_CLASSES = ( # 'debug_toolbar.middleware.DebugToolbarMiddleware', 'django_comment_client.utils.ViewNameMiddleware', + 'codejail.django_integration.ConfigureCodeJailMiddleware', ) ############################### Pipeline ####################################### From eb8569634738ec93b8ca1cf9676863a10a44712e Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 20 Feb 2013 10:59:40 -0500 Subject: [PATCH 032/162] Tests for safe_exec --- .../codejail/codejail/tests/test_safe_exec.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 common/lib/codejail/codejail/tests/test_safe_exec.py diff --git a/common/lib/codejail/codejail/tests/test_safe_exec.py b/common/lib/codejail/codejail/tests/test_safe_exec.py new file mode 100644 index 0000000000..685e2b8c39 --- /dev/null +++ b/common/lib/codejail/codejail/tests/test_safe_exec.py @@ -0,0 +1,53 @@ +"""Test safe_exec.py""" + +import textwrap +import unittest +from nose.plugins.skip import SkipTest + +from codejail.safe_exec import safe_exec, not_safe_exec + +dedent = textwrap.dedent + +class SafeExecTests(object): + """The tests for `safe_exec`, will be mixed into specific test classes below.""" + def test_set_values(self): + g, l = {}, {} + self.safe_exec("a = 17", g, l) + self.assertEqual(l['a'], 17) + + def test_division(self): + g, l = {}, {} + # No future division: 1/2 is 0. + self.safe_exec("a = 1/2", g, l) + self.assertEqual(l['a'], 0) + # Future division: 1/2 is 0.5. + self.safe_exec("a = 1/2", g, l, future_division=True) + self.assertEqual(l['a'], 0.5) + + def test_assumed_imports(self): + g, l = {}, {} + # Using string without importing it is bad. + with self.assertRaises(Exception): + self.safe_exec("a = string.ascii_lowercase[0]", g, l) + # Using string with an assumed import is fine. + self.safe_exec("a = string.ascii_lowercase[0]", g, l, assumed_imports=["string"]) + self.assertEqual(l['a'], 'a') + # Can also import with a shorthand. + self.safe_exec("a = op.join('x', 'y')", g, l, assumed_imports=[("op", "os.path")]) + self.assertEqual(l['a'][0], 'x') + self.assertEqual(l['a'][-1], 'y') + + +class TestSafeExec(SafeExecTests, unittest.TestCase): + """Run SafeExecTests, with the real safe_exec.""" + def safe_exec(self, *args, **kwargs): + safe_exec(*args, **kwargs) + +class TestNotSafeExec(SafeExecTests, unittest.TestCase): + """Run SafeExecTests, with not_safe_exec.""" + def setUp(self): + if safe_exec is not_safe_exec: + raise SkipTest + + def safe_exec(self, *args, **kwargs): + not_safe_exec(*args, **kwargs) From d99eadc0a2059ca1852c49c9f39021150cdf18d9 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 20 Feb 2013 11:00:10 -0500 Subject: [PATCH 033/162] Refactor the assumed_imports handling --- common/lib/codejail/codejail/safe_exec.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index 89e04c17f3..b6469996ab 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -14,6 +14,15 @@ if lazymod_py_file.endswith("c"): lazymod_py = open(lazymod_py_file).read() +def names_and_modules(assumed_imports): + """Get uniform names and modules from assumed_imports.""" + for modname in assumed_imports: + if isinstance(modname, tuple): + yield modname + else: + yield modname, modname + + def safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_imports=None): """Execute code as "exec" does, but safely. @@ -44,11 +53,7 @@ def safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_im if assumed_imports: the_code.append(lazymod_py) - for modname in assumed_imports: - if isinstance(modname, tuple): - name, modname = modname - else: - name = modname + for name, modname in names_and_modules(assumed_imports): the_code.append("g_dict['{}'] = LazyModule('{}')\n".format(name, modname)) the_code.append(textwrap.dedent("""\ @@ -100,11 +105,7 @@ def not_safe_exec(code, globals_dict, locals_dict, future_division=False, assume g_dict = straw(globals_dict) l_dict = straw(locals_dict) - for modname in assumed_imports or (): - if isinstance(modname, tuple): - name, modname = modname - else: - name = modname + for name, modname in names_and_modules(assumed_imports or ()): g_dict[name] = lazymod.LazyModule(modname) exec code in g_dict, l_dict From ab8a3050fd46dded98f5dc51fe54b6867f52f442 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 20 Feb 2013 11:07:24 -0500 Subject: [PATCH 034/162] Don't use jailpy if it hasn't been configured. --- common/lib/codejail/codejail/safe_exec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index b6469996ab..04045fb782 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -113,6 +113,6 @@ def not_safe_exec(code, globals_dict, locals_dict, future_division=False, assume locals_dict.update(straw(l_dict)) # Running Python code in the sandbox makes it difficult to debug. -# Turn this on to run the code directly. -if 0: +# Change 0 to 1 to run the code directly. +if 0 or not jailpy.is_configured(): safe_exec = not_safe_exec From 12b6876753fc1fcac330b83839a1586091ec1033 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 20 Feb 2013 12:41:40 -0500 Subject: [PATCH 035/162] safe_exec seeds the random module, and now we have tests for it. --- common/lib/capa/capa/safe_exec.py | 12 +++++-- common/lib/capa/capa/tests/test_safe_exec.py | 37 ++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 common/lib/capa/capa/tests/test_safe_exec.py diff --git a/common/lib/capa/capa/safe_exec.py b/common/lib/capa/capa/safe_exec.py index e6fa0e7bb5..32f6049453 100644 --- a/common/lib/capa/capa/safe_exec.py +++ b/common/lib/capa/capa/safe_exec.py @@ -2,11 +2,17 @@ import codejail.safe_exec -def safe_exec(code, globals_dict, locals_dict): +CODE_PROLOG = """\ +import random as random_module +random = random_module.Random(%r) +random.Random = random_module.Random +""" + +def safe_exec(code, globals_dict, locals_dict, random_seed=None): + code_prolog = CODE_PROLOG % random_seed codejail.safe_exec.safe_exec( - code, globals_dict, locals_dict, future_division=True, + code_prolog + code, globals_dict, locals_dict, future_division=True, assumed_imports=[ - "random", "numpy", "math", "scipy", diff --git a/common/lib/capa/capa/tests/test_safe_exec.py b/common/lib/capa/capa/tests/test_safe_exec.py new file mode 100644 index 0000000000..035b15b24d --- /dev/null +++ b/common/lib/capa/capa/tests/test_safe_exec.py @@ -0,0 +1,37 @@ +"""Test safe_exec.py""" + +import random +import unittest + +from capa.safe_exec import safe_exec + +class TestSafeExec(unittest.TestCase): + def test_set_values(self): + g, l = {}, {} + safe_exec("a = 17", g, l) + self.assertEqual(l['a'], 17) + + def test_division(self): + g, l = {}, {} + # Future division: 1/2 is 0.5. + safe_exec("a = 1/2", g, l) + self.assertEqual(l['a'], 0.5) + + def test_assumed_imports(self): + g, l = {}, {} + # Math is always available. + safe_exec("a = int(math.pi)", g, l) + self.assertEqual(l['a'], 3) + + def test_random_seeding(self): + g, l = {}, {} + r = random.Random(17) + rnums = [r.randint(0, 999) for _ in xrange(100)] + + # Without a seed, the results are unpredictable + safe_exec("rnums = [random.randint(0, 999) for _ in xrange(100)]", g, l) + self.assertNotEqual(l['rnums'], rnums) + + # With a seed, the results are predictable + safe_exec("rnums = [random.randint(0, 999) for _ in xrange(100)]", g, l, random_seed=17) + self.assertEqual(l['rnums'], rnums) From abb91745596f97f1b497257d34042909e21c3c5c Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 20 Feb 2013 13:14:51 -0500 Subject: [PATCH 036/162] Refactor how script chunks are run. --- common/lib/capa/capa/capa_problem.py | 38 +++++++++++----------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index 888501e8b5..2e23d38380 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -18,7 +18,6 @@ import logging import math import numpy import os -import random import re import struct import sys @@ -437,49 +436,42 @@ class LoncapaProblem(object): Problem XML goes to Python execution context. Runs everything in script tags. ''' - random.seed(self.seed) - context = {} context['seed'] = self.seed - context['script_code'] = '' + all_code = '' - self._execute_scripts(tree.findall('.//script'), context) + python_path = [] - return context - - def _execute_scripts(self, scripts, context): - ''' - Executes scripts in the given context. - ''' - original_path = sys.path - - for script in scripts: - sys.path = original_path + self._extract_system_path(script) + for script in tree.findall('.//script'): stype = script.get('type') - if stype: if 'javascript' in stype: continue # skip javascript if 'perl' in stype: continue # skip perl # TODO: evaluate only python + + python_path.extend(self._extract_system_path(script)) + code = script.text XMLESC = {"'": "'", """: '"'} code = unescape(code, XMLESC) - # store code source in context - context['script_code'] += code + all_code += code + + if all_code: try: - # use "context" for global context; thus defs in code are global within code locals_dict = {} - safe_exec.safe_exec(code, context, locals_dict) + safe_exec.safe_exec(all_code, context, locals_dict, random_seed=self.seed) context.update(locals_dict) except Exception as err: - log.exception("Error while execing script code: " + code) + log.exception("Error while execing script code: " + all_code) msg = "Error while executing script code: %s" % str(err).replace('<', '<') raise responsetypes.LoncapaProblemError(msg) - finally: - sys.path = original_path + + # store code source in context + context['script_code'] = all_code + return context From a04317b31d4b0e8894127c93fbb0ab230570381f Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 20 Feb 2013 15:16:43 -0500 Subject: [PATCH 037/162] Files are properly copied in both implementations of safe_exec, and a new python_path argument adds to the python path. --- common/lib/capa/capa/capa_problem.py | 3 +- common/lib/capa/capa/safe_exec.py | 9 +++- .../capa/tests/test_files/pylib/constant.py | 1 + common/lib/capa/capa/tests/test_safe_exec.py | 9 ++++ common/lib/codejail/codejail/jailpy.py | 7 ++- common/lib/codejail/codejail/safe_exec.py | 43 ++++++++++++++++--- common/lib/codejail/codejail/tests/hello.txt | 1 + .../codejail/codejail/tests/pylib/module.py | 1 + .../codejail/codejail/tests/test_jailpy.py | 9 ++++ .../codejail/codejail/tests/test_safe_exec.py | 20 +++++++-- common/lib/codejail/codejail/util.py | 20 +++++++++ 11 files changed, 108 insertions(+), 15 deletions(-) create mode 100644 common/lib/capa/capa/tests/test_files/pylib/constant.py create mode 100644 common/lib/codejail/codejail/tests/hello.txt create mode 100644 common/lib/codejail/codejail/tests/pylib/module.py diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index 2e23d38380..b8e146d773 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -454,9 +454,8 @@ class LoncapaProblem(object): python_path.extend(self._extract_system_path(script)) - code = script.text XMLESC = {"'": "'", """: '"'} - code = unescape(code, XMLESC) + code = unescape(script.text, XMLESC) all_code += code if all_code: diff --git a/common/lib/capa/capa/safe_exec.py b/common/lib/capa/capa/safe_exec.py index 32f6049453..e10da30196 100644 --- a/common/lib/capa/capa/safe_exec.py +++ b/common/lib/capa/capa/safe_exec.py @@ -2,16 +2,23 @@ import codejail.safe_exec +# This will set up the name "random" as a properly-seeded stand-in for the +# random module. CODE_PROLOG = """\ import random as random_module random = random_module.Random(%r) random.Random = random_module.Random +del random_module """ -def safe_exec(code, globals_dict, locals_dict, random_seed=None): +def safe_exec(code, globals_dict, locals_dict, random_seed=None, python_path=None): + """Exec python code safely. + + """ code_prolog = CODE_PROLOG % random_seed codejail.safe_exec.safe_exec( code_prolog + code, globals_dict, locals_dict, future_division=True, + python_path=python_path, assumed_imports=[ "numpy", "math", diff --git a/common/lib/capa/capa/tests/test_files/pylib/constant.py b/common/lib/capa/capa/tests/test_files/pylib/constant.py new file mode 100644 index 0000000000..0769d528ba --- /dev/null +++ b/common/lib/capa/capa/tests/test_files/pylib/constant.py @@ -0,0 +1 @@ +THE_CONST = 23 diff --git a/common/lib/capa/capa/tests/test_safe_exec.py b/common/lib/capa/capa/tests/test_safe_exec.py index 035b15b24d..3a436ac3ce 100644 --- a/common/lib/capa/capa/tests/test_safe_exec.py +++ b/common/lib/capa/capa/tests/test_safe_exec.py @@ -1,5 +1,6 @@ """Test safe_exec.py""" +import os.path import random import unittest @@ -35,3 +36,11 @@ class TestSafeExec(unittest.TestCase): # With a seed, the results are predictable safe_exec("rnums = [random.randint(0, 999) for _ in xrange(100)]", g, l, random_seed=17) self.assertEqual(l['rnums'], rnums) + + def test_python_lib(self): + pylib = os.path.dirname(__file__) + "/test_files/pylib" + g, l = {}, {} + safe_exec( + "import constant; a = constant.THE_CONST", + g, l, python_path=[pylib] + ) diff --git a/common/lib/codejail/codejail/jailpy.py b/common/lib/codejail/codejail/jailpy.py index 8f68ab33de..ed13a0d3f0 100644 --- a/common/lib/codejail/codejail/jailpy.py +++ b/common/lib/codejail/codejail/jailpy.py @@ -63,8 +63,11 @@ def jailpy(code, files=None, argv=None, stdin=None): # All the supporting files are copied into our directory. for filename in files or (): - dest = os.path.join(tmpdir, os.path.basename(filename)) - shutil.copyfile(filename, dest) + if os.path.isfile(filename): + shutil.copy(filename, tmpdir) + else: + dest = os.path.join(tmpdir, os.path.basename(filename)) + shutil.copytree(filename, dest) # Create the main file. with open(os.path.join(tmpdir, "jailed_code.py"), "w") as jailed: diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index 04045fb782..06bc4c60d3 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -1,11 +1,16 @@ """Safe execution of untrusted Python code.""" import json +import os.path +import shutil +import sys import textwrap import lazymod import jailpy +from util import temp_directory, change_directory, TempDirectory + # We'll need the code from lazymod.py for use in jailpy, so read it now. lazymod_py_file = lazymod.__file__ if lazymod_py_file.endswith("c"): @@ -23,7 +28,7 @@ def names_and_modules(assumed_imports): yield modname, modname -def safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_imports=None): +def safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_imports=None, files=None, python_path=None): """Execute code as "exec" does, but safely. `code` is a string of Python code. `globals_dict` and `locals_dict` are @@ -41,6 +46,7 @@ def safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_im """ the_code = [] + files = list(files or ()) if future_division: the_code.append("from __future__ import division\n") @@ -51,6 +57,11 @@ def safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_im code, g_dict, l_dict = json.load(sys.stdin) """)) + for pydir in python_path or (): + pybase = os.path.basename(pydir) + the_code.append("sys.path.append(%r)\n" % pybase) + files.append(pydir) + if assumed_imports: the_code.append(lazymod_py) for name, modname in names_and_modules(assumed_imports): @@ -63,25 +74,30 @@ def safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_im json.dump(l_dict, sys.stdout) """)) + stdin = json.dumps([code, globals_dict, locals_dict]) + jailed_code = "".join(the_code) + # Turn this on to see what's being executed. - if 0: + if 1: print "--{:-<40}".format(" jailed ") - print "".join(the_code) + print jailed_code print "--{:-<40}".format(" exec ") print code - stdin = json.dumps([code, globals_dict, locals_dict]) - res = jailpy.jailpy("".join(the_code), stdin=stdin) + res = jailpy.jailpy(jailed_code, stdin=stdin, files=files) if res.status != 0: raise Exception("Couldn't excecute jailed code: %s" % res.stderr) locals_dict.update(json.loads(res.stdout)) -def not_safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_imports=None): +def not_safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_imports=None, files=None, python_path=None): """Another implementation of `safe_exec`, but not safe. This can be swapped in for debugging problems in sandboxed Python code. + This is not thread-safe, due to temporarily changing the current directory + and modifying sys.path. + """ def straw(d): """Return only the JSON-safe part of d. @@ -108,7 +124,20 @@ def not_safe_exec(code, globals_dict, locals_dict, future_division=False, assume for name, modname in names_and_modules(assumed_imports or ()): g_dict[name] = lazymod.LazyModule(modname) - exec code in g_dict, l_dict + with temp_directory(delete_when_done=True) as tmpdir: + with change_directory(tmpdir): + # Copy the files here. + for filename in files or (): + dest = os.path.join(tmpdir, os.path.basename(filename)) + shutil.copyfile(filename, dest) + + original_path = sys.path + if python_path: + sys.path.extend(python_path) + try: + exec code in g_dict, l_dict + finally: + sys.path = original_path locals_dict.update(straw(l_dict)) diff --git a/common/lib/codejail/codejail/tests/hello.txt b/common/lib/codejail/codejail/tests/hello.txt new file mode 100644 index 0000000000..c12abce9f3 --- /dev/null +++ b/common/lib/codejail/codejail/tests/hello.txt @@ -0,0 +1 @@ +Hello there. diff --git a/common/lib/codejail/codejail/tests/pylib/module.py b/common/lib/codejail/codejail/tests/pylib/module.py new file mode 100644 index 0000000000..8cb5cded29 --- /dev/null +++ b/common/lib/codejail/codejail/tests/pylib/module.py @@ -0,0 +1 @@ +const = 42 diff --git a/common/lib/codejail/codejail/tests/test_jailpy.py b/common/lib/codejail/codejail/tests/test_jailpy.py index 3c6d13b714..fb59bac31d 100644 --- a/common/lib/codejail/codejail/tests/test_jailpy.py +++ b/common/lib/codejail/codejail/tests/test_jailpy.py @@ -1,5 +1,6 @@ """Test jailpy.py""" +import os.path import textwrap import unittest from nose.plugins.skip import SkipTest @@ -53,6 +54,14 @@ class TestFeatures(JailPyHelpers, unittest.TestCase): self.assertResultOk(res) self.assertEqual(res.stdout.strip(), "36.5") + def test_files_are_copied(self): + res = jailpy( + "print 'Look:', open('hello.txt').read()", + files=[os.path.dirname(__file__) + "/hello.txt"] + ) + self.assertResultOk(res) + self.assertEqual(res.stdout, 'Look: Hello there.\n\n') + class TestLimits(JailPyHelpers, unittest.TestCase): def test_cant_use_too_much_memory(self): diff --git a/common/lib/codejail/codejail/tests/test_safe_exec.py b/common/lib/codejail/codejail/tests/test_safe_exec.py index 685e2b8c39..00987f7787 100644 --- a/common/lib/codejail/codejail/tests/test_safe_exec.py +++ b/common/lib/codejail/codejail/tests/test_safe_exec.py @@ -1,13 +1,11 @@ """Test safe_exec.py""" -import textwrap +import os.path import unittest from nose.plugins.skip import SkipTest from codejail.safe_exec import safe_exec, not_safe_exec -dedent = textwrap.dedent - class SafeExecTests(object): """The tests for `safe_exec`, will be mixed into specific test classes below.""" def test_set_values(self): @@ -37,6 +35,22 @@ class SafeExecTests(object): self.assertEqual(l['a'][0], 'x') self.assertEqual(l['a'][-1], 'y') + def test_files_are_copied(self): + g, l = {}, {} + self.safe_exec( + "a = 'Look: ' + open('hello.txt').read()", g, l, + files=[os.path.dirname(__file__) + "/hello.txt"] + ) + self.assertEqual(l['a'], 'Look: Hello there.\n') + + def test_python_path(self): + g, l = {}, {} + self.safe_exec( + "import module; a = module.const", g, l, + python_path=[os.path.dirname(__file__) + "/pylib"] + ) + self.assertEqual(l['a'], 42) + class TestSafeExec(SafeExecTests, unittest.TestCase): """Run SafeExecTests, with the real safe_exec.""" diff --git a/common/lib/codejail/codejail/util.py b/common/lib/codejail/codejail/util.py index 88ecc1b319..e293ce052f 100644 --- a/common/lib/codejail/codejail/util.py +++ b/common/lib/codejail/codejail/util.py @@ -49,3 +49,23 @@ class ModuleIsolation(object): # and delete them all so another import will run code for real again. for m in new_mods: del sys.modules[m] + + +class ChangeDirectory(object): + def __init__(self, new_dir): + self.old_dir = os.getcwd() + os.chdir(new_dir) + + def clean_up(self): + os.chdir(self.old_dir) + +@contextlib.contextmanager +def change_directory(new_dir): + """ + A context manager to change the directory, and then change it back. + """ + cd = ChangeDirectory(new_dir) + try: + yield new_dir + finally: + cd.clean_up() From 7187b10f9ce786f9c56b65fa007ecb85ad65256f Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 20 Feb 2013 15:26:24 -0500 Subject: [PATCH 038/162] Use the python_path argument to safe_exec --- common/lib/capa/capa/capa_problem.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index b8e146d773..9665721293 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -17,7 +17,7 @@ from datetime import datetime import logging import math import numpy -import os +import os, os.path import re import struct import sys @@ -452,7 +452,9 @@ class LoncapaProblem(object): continue # skip perl # TODO: evaluate only python - python_path.extend(self._extract_system_path(script)) + for d in self._extract_system_path(script): + if d not in python_path and os.path.exists(d): + python_path.append(d) XMLESC = {"'": "'", """: '"'} code = unescape(script.text, XMLESC) @@ -461,7 +463,7 @@ class LoncapaProblem(object): if all_code: try: locals_dict = {} - safe_exec.safe_exec(all_code, context, locals_dict, random_seed=self.seed) + safe_exec.safe_exec(all_code, context, locals_dict, random_seed=self.seed, python_path=python_path) context.update(locals_dict) except Exception as err: log.exception("Error while execing script code: " + all_code) From 4b234a63a3bc3f35cda85b27ee009829f35361b6 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 20 Feb 2013 15:29:52 -0500 Subject: [PATCH 039/162] Future division is really a capa concern, not a general-purpose codejail concern. Move it. --- common/lib/capa/capa/safe_exec.py | 5 +++-- common/lib/codejail/codejail/safe_exec.py | 12 ++---------- common/lib/codejail/codejail/tests/test_safe_exec.py | 9 --------- 3 files changed, 5 insertions(+), 21 deletions(-) diff --git a/common/lib/capa/capa/safe_exec.py b/common/lib/capa/capa/safe_exec.py index e10da30196..355350eb5d 100644 --- a/common/lib/capa/capa/safe_exec.py +++ b/common/lib/capa/capa/safe_exec.py @@ -3,8 +3,9 @@ import codejail.safe_exec # This will set up the name "random" as a properly-seeded stand-in for the -# random module. +# random module. Also, capa assumes float-friendly division always. CODE_PROLOG = """\ +from __future__ import division import random as random_module random = random_module.Random(%r) random.Random = random_module.Random @@ -17,7 +18,7 @@ def safe_exec(code, globals_dict, locals_dict, random_seed=None, python_path=Non """ code_prolog = CODE_PROLOG % random_seed codejail.safe_exec.safe_exec( - code_prolog + code, globals_dict, locals_dict, future_division=True, + code_prolog + code, globals_dict, locals_dict, python_path=python_path, assumed_imports=[ "numpy", diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index 06bc4c60d3..aaffe8757f 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -28,15 +28,13 @@ def names_and_modules(assumed_imports): yield modname, modname -def safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_imports=None, files=None, python_path=None): +def safe_exec(code, globals_dict, locals_dict, assumed_imports=None, files=None, python_path=None): """Execute code as "exec" does, but safely. `code` is a string of Python code. `globals_dict` and `locals_dict` are dictionaries to use as the globals and locals. Modifications the code makes to `locals_dict` are reflected in the dictionary on return. - `future_division` determines whether Python-3-style division is used. - `assumed_imports` is a list of modules to make available as implicit imports for the code. Entries are either a name, "mod", which makes "import mod" part of the code, or a pair, ("f", "fooey"), which makes @@ -48,9 +46,6 @@ def safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_im the_code = [] files = list(files or ()) - if future_division: - the_code.append("from __future__ import division\n") - the_code.append(textwrap.dedent("""\ import json import sys @@ -90,7 +85,7 @@ def safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_im locals_dict.update(json.loads(res.stdout)) -def not_safe_exec(code, globals_dict, locals_dict, future_division=False, assumed_imports=None, files=None, python_path=None): +def not_safe_exec(code, globals_dict, locals_dict, assumed_imports=None, files=None, python_path=None): """Another implementation of `safe_exec`, but not safe. This can be swapped in for debugging problems in sandboxed Python code. @@ -115,9 +110,6 @@ def not_safe_exec(code, globals_dict, locals_dict, future_division=False, assume jd[k] = v return json.loads(json.dumps(jd)) - if future_division: - code = "from __future__ import division\n" + code - g_dict = straw(globals_dict) l_dict = straw(locals_dict) diff --git a/common/lib/codejail/codejail/tests/test_safe_exec.py b/common/lib/codejail/codejail/tests/test_safe_exec.py index 00987f7787..1bdcceaf44 100644 --- a/common/lib/codejail/codejail/tests/test_safe_exec.py +++ b/common/lib/codejail/codejail/tests/test_safe_exec.py @@ -13,15 +13,6 @@ class SafeExecTests(object): self.safe_exec("a = 17", g, l) self.assertEqual(l['a'], 17) - def test_division(self): - g, l = {}, {} - # No future division: 1/2 is 0. - self.safe_exec("a = 1/2", g, l) - self.assertEqual(l['a'], 0) - # Future division: 1/2 is 0.5. - self.safe_exec("a = 1/2", g, l, future_division=True) - self.assertEqual(l['a'], 0.5) - def test_assumed_imports(self): g, l = {}, {} # Using string without importing it is bad. From 4fb73248ba25818b7f0cfcb0c9fbcc5190b1f225 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 20 Feb 2013 17:31:56 -0500 Subject: [PATCH 040/162] Try to get test running. (Not yet) --- common/lib/capa/capa/responsetypes.py | 2 ++ common/lib/xmodule/test_files/symbolicresponse.xml | 13 ++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 743edb358a..1a788a9f54 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -1199,6 +1199,8 @@ class SymbolicResponse(CustomResponse): response_tag = 'symbolicresponse' def setup_response(self): + # No, this is not pretty. + self.context['script_code'] += "from symmath import symmath_check\n" self.xml.set('cfn', 'symmath_check') CustomResponse.setup_response(self) diff --git a/common/lib/xmodule/test_files/symbolicresponse.xml b/common/lib/xmodule/test_files/symbolicresponse.xml index 4dc2bc9d7b..7d5de7a101 100644 --- a/common/lib/xmodule/test_files/symbolicresponse.xml +++ b/common/lib/xmodule/test_files/symbolicresponse.xml @@ -13,16 +13,15 @@ real time, next to the input box.

This is a correct answer which may be entered below:

cos(theta)*[[1,0],[0,1]] + i*sin(theta)*[[0,1],[1,0]]

- Compute [mathjax] U = \exp\left( i \theta \left[ \begin{matrix} 0 & 1 \\ 1 & 0 \end{matrix} \right] \right) [/mathjax] and give the resulting \(2 \times 2\) matrix.
Your input should be typed in as a list of lists, eg [[1,2],[3,4]].
- [mathjax]U=[/mathjax] - - -
+ [mathjax]U=[/mathjax] + + + + +
From a40aed58d0345f9608e770c3cfd45dc0faabfe0e Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 20 Feb 2013 17:32:07 -0500 Subject: [PATCH 041/162] Clean up --- common/lib/capa/capa/safe_exec.py | 6 ++++-- common/lib/codejail/codejail/safe_exec.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/common/lib/capa/capa/safe_exec.py b/common/lib/capa/capa/safe_exec.py index 355350eb5d..31f47b5c36 100644 --- a/common/lib/capa/capa/safe_exec.py +++ b/common/lib/capa/capa/safe_exec.py @@ -2,10 +2,12 @@ import codejail.safe_exec -# This will set up the name "random" as a properly-seeded stand-in for the -# random module. Also, capa assumes float-friendly division always. +# Establish the Python environment for Capa. +# Capa assumes float-friendly division always. +# The name "random" is a properly-seeded stand-in for the random module. CODE_PROLOG = """\ from __future__ import division + import random as random_module random = random_module.Random(%r) random.Random = random_module.Random diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index aaffe8757f..5a4e48d39e 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -73,7 +73,7 @@ def safe_exec(code, globals_dict, locals_dict, assumed_imports=None, files=None, jailed_code = "".join(the_code) # Turn this on to see what's being executed. - if 1: + if 0: print "--{:-<40}".format(" jailed ") print jailed_code print "--{:-<40}".format(" exec ") From 42eee48ec9b07e2c98018d59648c86f6813087fa Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 21 Feb 2013 13:24:32 -0500 Subject: [PATCH 042/162] A few places we used an option of 'imaginaryi', which isn't a real option. Doesn't change any behavior. --- common/lib/sandbox-packages/symmath/formula.py | 2 +- common/lib/xmodule/test_files/symbolicresponse.xml | 10 ++++------ .../data/full/problem/test_files/symbolicresponse.xml | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/common/lib/sandbox-packages/symmath/formula.py b/common/lib/sandbox-packages/symmath/formula.py index 604941ffdd..8369baa27c 100644 --- a/common/lib/sandbox-packages/symmath/formula.py +++ b/common/lib/sandbox-packages/symmath/formula.py @@ -736,4 +736,4 @@ def test6(): # imaginary numbers ''' - return formula(xmlstr, options='imaginaryi') + return formula(xmlstr, options='imaginary') diff --git a/common/lib/xmodule/test_files/symbolicresponse.xml b/common/lib/xmodule/test_files/symbolicresponse.xml index 7d5de7a101..f174bc986c 100644 --- a/common/lib/xmodule/test_files/symbolicresponse.xml +++ b/common/lib/xmodule/test_files/symbolicresponse.xml @@ -16,12 +16,10 @@ real time, next to the input box. Compute [mathjax] U = \exp\left( i \theta \left[ \begin{matrix} 0 & 1 \\ 1 & 0 \end{matrix} \right] \right) [/mathjax] and give the resulting \(2 \times 2\) matrix.
Your input should be typed in as a list of lists, eg [[1,2],[3,4]].
- [mathjax]U=[/mathjax] - - - - -
+ [mathjax]U=[/mathjax] + + +
diff --git a/common/test/data/full/problem/test_files/symbolicresponse.xml b/common/test/data/full/problem/test_files/symbolicresponse.xml index 4dc2bc9d7b..3573c34076 100644 --- a/common/test/data/full/problem/test_files/symbolicresponse.xml +++ b/common/test/data/full/problem/test_files/symbolicresponse.xml @@ -19,7 +19,7 @@ from symmath import * Compute [mathjax] U = \exp\left( i \theta \left[ \begin{matrix} 0 & 1 \\ 1 & 0 \end{matrix} \right] \right) [/mathjax] and give the resulting \(2 \times 2\) matrix.
Your input should be typed in as a list of lists, eg [[1,2],[3,4]].
- [mathjax]U=[/mathjax] + [mathjax]U=[/mathjax]
From 81c4e4f74ff0e9f5a835f40f5895bbff3e2d9059 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 21 Feb 2013 14:13:30 -0500 Subject: [PATCH 043/162] Make check_function more flexible so symbolicresponse can pass in more information. --- common/lib/capa/capa/responsetypes.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 1a788a9f54..ae6361ecd3 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -927,15 +927,19 @@ class CustomResponse(LoncapaResponse): # actual function that will re-execute the original script, # and invoke the function with the data needed. def make_check_function(script_code, cfn): - def check_function(expect, ans): - code = (script_code + "\n" + - "cfn_return = %s(expect, ans)\n" % cfn) + def check_function(expect, ans, **kwargs): + code = [script_code, "kwargs = {}"] + for name, val in kwargs.iteritems(): + if isinstance(val, (str, list, tuple, int, long, float, dict)): + code.append("kwargs[%r] = %r" % (name, val)) + code.append("cfn_return = %s(expect, ans, **kwargs)" % cfn) + code.append("") # a final newline globals_dict = { 'expect': expect, 'ans': ans, } locals_dict = {} - safe_exec.safe_exec(code, globals_dict, locals_dict) + safe_exec.safe_exec("\n".join(code), globals_dict, locals_dict) return locals_dict['cfn_return'] return check_function @@ -954,6 +958,8 @@ class CustomResponse(LoncapaResponse): else: self.code = answer.text + self.cfn_kwargs_keys = [] + def get_score(self, student_answers): ''' student_answers is a dict with everything from request.POST, but with the first part @@ -1054,7 +1060,8 @@ class CustomResponse(LoncapaResponse): log.debug(" submission = %s" % submission) try: answer_given = submission[0] if (len(idset) == 1) else submission - ret = fn(self.expect, answer_given) + kwargs = {n:self.context.get(n) for n in self.cfn_kwargs_keys} + ret = fn(self.expect, answer_given, **kwargs) except Exception as err: self._handle_exec_exception(err) @@ -1203,6 +1210,7 @@ class SymbolicResponse(CustomResponse): self.context['script_code'] += "from symmath import symmath_check\n" self.xml.set('cfn', 'symmath_check') CustomResponse.setup_response(self) + self.cfn_kwargs_keys.extend(['dynamath', 'options', 'debug']) #----------------------------------------------------------------------------- From 070f184ee0d848cc2eb42437487fa74c269ab00f Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 21 Feb 2013 14:14:19 -0500 Subject: [PATCH 044/162] Not sure why these had capital-I's in them, since the text just above shows lowercase-i's, and uppercase doesn't work properly. --- common/lib/xmodule/test_files/symbolicresponse.xml | 2 +- common/test/data/full/problem/test_files/symbolicresponse.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/lib/xmodule/test_files/symbolicresponse.xml b/common/lib/xmodule/test_files/symbolicresponse.xml index f174bc986c..8443366ffe 100644 --- a/common/lib/xmodule/test_files/symbolicresponse.xml +++ b/common/lib/xmodule/test_files/symbolicresponse.xml @@ -16,7 +16,7 @@ real time, next to the input box. Compute [mathjax] U = \exp\left( i \theta \left[ \begin{matrix} 0 & 1 \\ 1 & 0 \end{matrix} \right] \right) [/mathjax] and give the resulting \(2 \times 2\) matrix.
Your input should be typed in as a list of lists, eg [[1,2],[3,4]].
- [mathjax]U=[/mathjax] + [mathjax]U=[/mathjax]
diff --git a/common/test/data/full/problem/test_files/symbolicresponse.xml b/common/test/data/full/problem/test_files/symbolicresponse.xml index 3573c34076..85945b1d8c 100644 --- a/common/test/data/full/problem/test_files/symbolicresponse.xml +++ b/common/test/data/full/problem/test_files/symbolicresponse.xml @@ -19,7 +19,7 @@ from symmath import * Compute [mathjax] U = \exp\left( i \theta \left[ \begin{matrix} 0 & 1 \\ 1 & 0 \end{matrix} \right] \right) [/mathjax] and give the resulting \(2 \times 2\) matrix.
Your input should be typed in as a list of lists, eg [[1,2],[3,4]].
- [mathjax]U=[/mathjax] + [mathjax]U=[/mathjax]
From 9dbfca129c384572726b3e5619c8198866f60922 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 21 Feb 2013 14:36:14 -0500 Subject: [PATCH 045/162] Check functions now can only return serializable data, and 'ex' and 'got' weren't used later anyway. --- common/lib/sandbox-packages/symmath/symmath_check.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/lib/sandbox-packages/symmath/symmath_check.py b/common/lib/sandbox-packages/symmath/symmath_check.py index 151debee71..65a17883f5 100644 --- a/common/lib/sandbox-packages/symmath/symmath_check.py +++ b/common/lib/sandbox-packages/symmath/symmath_check.py @@ -324,4 +324,5 @@ def symmath_check(expect, ans, dynamath=None, options=None, debug=None, xml=None msg += "

Difference: %s

" % to_latex(diff) msg += '
' - return {'ok': False, 'msg': msg, 'ex': fexpect, 'got': fsym} + # Used to return more keys: 'ex': fexpect, 'got': fsym + return {'ok': False, 'msg': msg} From c04f3e09c09d830927487e1e045bee6294121b4e Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 21 Feb 2013 15:52:37 -0500 Subject: [PATCH 046/162] Test that the sandbox can't get to the network. --- common/lib/codejail/codejail/tests/test_jailpy.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/common/lib/codejail/codejail/tests/test_jailpy.py b/common/lib/codejail/codejail/tests/test_jailpy.py index fb59bac31d..15c548663b 100644 --- a/common/lib/codejail/codejail/tests/test_jailpy.py +++ b/common/lib/codejail/codejail/tests/test_jailpy.py @@ -96,6 +96,18 @@ class TestLimits(JailPyHelpers, unittest.TestCase): self.assertEqual(res.stdout, "Trying\n") self.assertIn("ermission denied", res.stderr) + def test_cant_use_network(self): + res = jailpy(dedent("""\ + import urllib + print "Reading google" + u = urllib.urlopen("http://google.com") + google = u.read() + print len(google) + """)) + self.assertNotEqual(res.status, 0) + self.assertEqual(res.stdout, "Reading google\n") + self.assertIn("IOError", res.stderr) + # TODO: write files # TODO: read network # TODO: fork From 94f6e685df4b3f069cdf5ea522bc9ca26f37f16f Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 21 Feb 2013 17:29:44 -0500 Subject: [PATCH 047/162] Mock the response from the snuggletex server, and unskip the SymbolicResponse test. --- .../tests/test_files/snuggletex_correct.html | 480 ++++++++++++++++++ .../tests/test_files/snuggletex_wrong.html | 187 +++++++ .../lib/capa/capa/tests/test_responsetypes.py | 29 +- 3 files changed, 687 insertions(+), 9 deletions(-) create mode 100644 common/lib/capa/capa/tests/test_files/snuggletex_correct.html create mode 100644 common/lib/capa/capa/tests/test_files/snuggletex_wrong.html diff --git a/common/lib/capa/capa/tests/test_files/snuggletex_correct.html b/common/lib/capa/capa/tests/test_files/snuggletex_correct.html new file mode 100644 index 0000000000..0d10f7f56d --- /dev/null +++ b/common/lib/capa/capa/tests/test_files/snuggletex_correct.html @@ -0,0 +1,480 @@ + + + + + + + + + + + + SnuggleTeX - ASCIIMathML Enrichment Demo + + + + + + + +

SnuggleTeX (1.2.2)

+
+ + +
+ +
+

ASCIIMathML Enrichment Demo

+

Input

+

+ This demo is similar to the + MathML Semantic Enrichnment Demo + but uses + ASCIIMathML as + an alternative input format, which provides real-time feedback as you + type but can often generate MathML with odd semantics in it. + SnuggleTeX includes some functionality that can to convert this raw MathML into + something equivalent to its own MathML output, thereby allowing you to + semantically enrich it in + certain simple cases, making ASCIIMathML a possibly viable input format + for simple semantic maths. + +

+

+ To try the demo, simply enter some some ASCIIMathML into the box below. + You should see a real time preview of this while you type. + Then hit Go! to use SnuggleTeX to semantically enrich your + input. + +

+
+
+ ASCIIMath Input: +
+
+

Live Preview

+

+ This is a MathML rendering of your input, generated by ASCIIMathML as you type. + +

+
+
+
+

+ This is the underlying MathML source generated by ASCIIMathML, again updated in real time. + +

+
+

Enhanced Presentation MathML

+

+ This shows the result of attempting to enrich the raw Presentation MathML + generated by ASCIIMathML: + + 

<math xmlns="http://www.w3.org/1998/Math/MathML">
+   <mrow>
+      <mrow>
+         <mrow>
+            <mi>cos</mi>
+            <mo>&ApplyFunction;</mo>
+            <mfenced close=")" open="(">
+               <mi>theta</mi>
+            </mfenced>
+         </mrow>
+         <mo>&sdot;</mo>
+         <mfenced close="]" open="[">
+            <mtable>
+               <mtr>
+                  <mtd>
+                     <mn>1</mn>
+                  </mtd>
+                  <mtd>
+                     <mn>0</mn>
+                  </mtd>
+               </mtr>
+               <mtr>
+                  <mtd>
+                     <mn>0</mn>
+                  </mtd>
+                  <mtd>
+                     <mn>1</mn>
+                  </mtd>
+               </mtr>
+            </mtable>
+         </mfenced>
+      </mrow>
+      <mo>+</mo>
+      <mrow>
+         <mi>i</mi>
+         <mo>&sdot;</mo>
+         <mrow>
+            <mi>sin</mi>
+            <mo>&ApplyFunction;</mo>
+            <mfenced close=")" open="(">
+               <mi>theta</mi>
+            </mfenced>
+         </mrow>
+         <mo>&sdot;</mo>
+         <mfenced close="]" open="[">
+            <mtable>
+               <mtr>
+                  <mtd>
+                     <mn>0</mn>
+                  </mtd>
+                  <mtd>
+                     <mn>1</mn>
+                  </mtd>
+               </mtr>
+               <mtr>
+                  <mtd>
+                     <mn>1</mn>
+                  </mtd>
+                  <mtd>
+                     <mn>0</mn>
+                  </mtd>
+               </mtr>
+            </mtable>
+         </mfenced>
+      </mrow>
+   </mrow>
+</math>

Content MathML

+

+ This shows the result of an attempted + conversion to Content MathML: + + 

<math xmlns="http://www.w3.org/1998/Math/MathML">
+   <apply>
+      <plus/>
+      <apply>
+         <times/>
+         <apply>
+            <cos/>
+            <ci>theta</ci>
+         </apply>
+         <list>
+            <matrix>
+               <vector>
+                  <cn>1</cn>
+                  <cn>0</cn>
+               </vector>
+               <vector>
+                  <cn>0</cn>
+                  <cn>1</cn>
+               </vector>
+            </matrix>
+         </list>
+      </apply>
+      <apply>
+         <times/>
+         <ci>i</ci>
+         <apply>
+            <sin/>
+            <ci>theta</ci>
+         </apply>
+         <list>
+            <matrix>
+               <vector>
+                  <cn>0</cn>
+                  <cn>1</cn>
+               </vector>
+               <vector>
+                  <cn>1</cn>
+                  <cn>0</cn>
+               </vector>
+            </matrix>
+         </list>
+      </apply>
+   </apply>
+</math>

Maxima Input Form

+

+ This shows the result of an attempted + conversion to Maxima Input syntax: + +

+

+ The conversion from Content MathML to Maxima Input was not successful for + this input. + +

+ + + + + + + + + + + + + + + + + + + + + + + +
Failure CodeMessageXPathContext
UMFG00Content MathML element matrix not supportedapply[1]/apply[1]/list[1]/matrix[1]
<matrix>
+   <vector>
+      <cn>1</cn>
+      <cn>0</cn>
+   </vector>
+   <vector>
+      <cn>0</cn>
+      <cn>1</cn>
+   </vector>
+</matrix>
UMFG00Content MathML element matrix not supportedapply[1]/apply[2]/list[1]/matrix[1]
<matrix>
+   <vector>
+      <cn>0</cn>
+      <cn>1</cn>
+   </vector>
+   <vector>
+      <cn>1</cn>
+      <cn>0</cn>
+   </vector>
+</matrix>
+

MathML Parallel Markup

+

+ This shows the enhanced Presentation MathML with other forms encapsulated + as annotations: + + 

<math xmlns="http://www.w3.org/1998/Math/MathML">
+   <semantics>
+      <mrow>
+         <mrow>
+            <mrow>
+               <mi>cos</mi>
+               <mo>&ApplyFunction;</mo>
+               <mfenced close=")" open="(">
+                  <mi>theta</mi>
+               </mfenced>
+            </mrow>
+            <mo>&sdot;</mo>
+            <mfenced close="]" open="[">
+               <mtable>
+                  <mtr>
+                     <mtd>
+                        <mn>1</mn>
+                     </mtd>
+                     <mtd>
+                        <mn>0</mn>
+                     </mtd>
+                  </mtr>
+                  <mtr>
+                     <mtd>
+                        <mn>0</mn>
+                     </mtd>
+                     <mtd>
+                        <mn>1</mn>
+                     </mtd>
+                  </mtr>
+               </mtable>
+            </mfenced>
+         </mrow>
+         <mo>+</mo>
+         <mrow>
+            <mi>i</mi>
+            <mo>&sdot;</mo>
+            <mrow>
+               <mi>sin</mi>
+               <mo>&ApplyFunction;</mo>
+               <mfenced close=")" open="(">
+                  <mi>theta</mi>
+               </mfenced>
+            </mrow>
+            <mo>&sdot;</mo>
+            <mfenced close="]" open="[">
+               <mtable>
+                  <mtr>
+                     <mtd>
+                        <mn>0</mn>
+                     </mtd>
+                     <mtd>
+                        <mn>1</mn>
+                     </mtd>
+                  </mtr>
+                  <mtr>
+                     <mtd>
+                        <mn>1</mn>
+                     </mtd>
+                     <mtd>
+                        <mn>0</mn>
+                     </mtd>
+                  </mtr>
+               </mtable>
+            </mfenced>
+         </mrow>
+      </mrow>
+      <annotation-xml encoding="MathML-Content">
+         <apply>
+            <plus/>
+            <apply>
+               <times/>
+               <apply>
+                  <cos/>
+                  <ci>theta</ci>
+               </apply>
+               <list>
+                  <matrix>
+                     <vector>
+                        <cn>1</cn>
+                        <cn>0</cn>
+                     </vector>
+                     <vector>
+                        <cn>0</cn>
+                        <cn>1</cn>
+                     </vector>
+                  </matrix>
+               </list>
+            </apply>
+            <apply>
+               <times/>
+               <ci>i</ci>
+               <apply>
+                  <sin/>
+                  <ci>theta</ci>
+               </apply>
+               <list>
+                  <matrix>
+                     <vector>
+                        <cn>0</cn>
+                        <cn>1</cn>
+                     </vector>
+                     <vector>
+                        <cn>1</cn>
+                        <cn>0</cn>
+                     </vector>
+                  </matrix>
+               </list>
+            </apply>
+         </apply>
+      </annotation-xml>
+      <annotation encoding="ASCIIMathInput"/>
+      <annotation-xml encoding="Maxima-upconversion-failures">
+         <s:fail xmlns:s="http://www.ph.ed.ac.uk/snuggletex" code="UMFG00"
+                 message="Content MathML element matrix not supported">
+            <s:arg>matrix</s:arg>
+            <s:xpath>apply[1]/apply[1]/list[1]/matrix[1]</s:xpath>
+            <s:context>
+               <matrix>
+                  <vector>
+                     <cn>1</cn>
+                     <cn>0</cn>
+                  </vector>
+                  <vector>
+                     <cn>0</cn>
+                     <cn>1</cn>
+                  </vector>
+               </matrix>
+            </s:context>
+         </s:fail>
+         <s:fail xmlns:s="http://www.ph.ed.ac.uk/snuggletex" code="UMFG00"
+                 message="Content MathML element matrix not supported">
+            <s:arg>matrix</s:arg>
+            <s:xpath>apply[1]/apply[2]/list[1]/matrix[1]</s:xpath>
+            <s:context>
+               <matrix>
+                  <vector>
+                     <cn>0</cn>
+                     <cn>1</cn>
+                  </vector>
+                  <vector>
+                     <cn>1</cn>
+                     <cn>0</cn>
+                  </vector>
+               </matrix>
+            </s:context>
+         </s:fail>
+      </annotation-xml>
+   </semantics>
+</math>
+
+
+
+ + + \ No newline at end of file diff --git a/common/lib/capa/capa/tests/test_files/snuggletex_wrong.html b/common/lib/capa/capa/tests/test_files/snuggletex_wrong.html new file mode 100644 index 0000000000..abd62ca4d2 --- /dev/null +++ b/common/lib/capa/capa/tests/test_files/snuggletex_wrong.html @@ -0,0 +1,187 @@ + + + + + + + + + + + + SnuggleTeX - ASCIIMathML Enrichment Demo + + + + + + + +

SnuggleTeX (1.2.2)

+
+ + +
+ +
+

ASCIIMathML Enrichment Demo

+

Input

+

+ This demo is similar to the + MathML Semantic Enrichnment Demo + but uses + ASCIIMathML as + an alternative input format, which provides real-time feedback as you + type but can often generate MathML with odd semantics in it. + SnuggleTeX includes some functionality that can to convert this raw MathML into + something equivalent to its own MathML output, thereby allowing you to + semantically enrich it in + certain simple cases, making ASCIIMathML a possibly viable input format + for simple semantic maths. + +

+

+ To try the demo, simply enter some some ASCIIMathML into the box below. + You should see a real time preview of this while you type. + Then hit Go! to use SnuggleTeX to semantically enrich your + input. + +

+
+
+ ASCIIMath Input: +
+
+

Live Preview

+

+ This is a MathML rendering of your input, generated by ASCIIMathML as you type. + +

+
+
+
+

+ This is the underlying MathML source generated by ASCIIMathML, again updated in real time. + +

+
+

Enhanced Presentation MathML

+

+ This shows the result of attempting to enrich the raw Presentation MathML + generated by ASCIIMathML: + + 

<math xmlns="http://www.w3.org/1998/Math/MathML">
+   <mn>2</mn>
+</math>

Content MathML

+

+ This shows the result of an attempted + conversion to Content MathML: + + 

<math xmlns="http://www.w3.org/1998/Math/MathML">
+   <cn>2</cn>
+</math>

Maxima Input Form

+

+ This shows the result of an attempted + conversion to Maxima Input syntax: + + 

2

MathML Parallel Markup

+

+ This shows the enhanced Presentation MathML with other forms encapsulated + as annotations: + + 

<math xmlns="http://www.w3.org/1998/Math/MathML">
+   <semantics>
+      <mn>2</mn>
+      <annotation-xml encoding="MathML-Content">
+         <cn>2</cn>
+      </annotation-xml>
+      <annotation encoding="ASCIIMathInput"/>
+      <annotation encoding="Maxima">2</annotation>
+   </semantics>
+</math>
+
+
+
+ + + \ No newline at end of file diff --git a/common/lib/capa/capa/tests/test_responsetypes.py b/common/lib/capa/capa/tests/test_responsetypes.py index 7a43fff4c9..3d93af0c75 100644 --- a/common/lib/capa/capa/tests/test_responsetypes.py +++ b/common/lib/capa/capa/tests/test_responsetypes.py @@ -10,6 +10,7 @@ import os import random import unittest import textwrap +import mock from . import test_system @@ -186,7 +187,6 @@ class ImageResponseTest(ResponseTest): class SymbolicResponseTest(unittest.TestCase): def test_sr_grade(self): - raise SkipTest() # This test fails due to dependencies on a local copy of snuggletex-webapp. Until we have figured that out, we'll just skip this test symbolicresponse_file = os.path.dirname(__file__) + "/test_files/symbolicresponse.xml" test_lcp = lcp.LoncapaProblem(open(symbolicresponse_file).read(), '1', system=test_system) correct_answers = {'1_2_1': 'cos(theta)*[[1,0],[0,1]] + i*sin(theta)*[[0,1],[1,0]]', @@ -264,14 +264,25 @@ class SymbolicResponseTest(unittest.TestCase): } wrong_answers = {'1_2_1': '2', '1_2_1_dynamath': ''' - - - 2 - - ''', - } - self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct') - self.assertEquals(test_lcp.grade_answers(wrong_answers).get_correctness('1_2_1'), 'incorrect') + + + 2 + + ''', + } + + import requests + d = os.path.dirname(__file__) + correct_snuggletex_response = open(os.path.join(d, "test_files/snuggletex_correct.html")).read().decode('utf8') + wrong_snuggletex_response = open(os.path.join(d, "test_files/snuggletex_wrong.html")).read().decode('utf8') + + with mock.patch.object(requests, 'post') as mock_post: + mock_post.return_value.text = correct_snuggletex_response + self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct') + + with mock.patch.object(requests, 'post') as mock_post: + mock_post.return_value.text = wrong_snuggletex_response + self.assertEquals(test_lcp.grade_answers(wrong_answers).get_correctness('1_2_1'), 'incorrect') class OptionResponseTest(ResponseTest): From 67d0670b2e93e25678fbf2f0469cbfba720c5d4b Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 21 Feb 2013 17:30:08 -0500 Subject: [PATCH 048/162] Symbolic response no longer runs its checker in the Python sandbox. --- common/lib/capa/capa/responsetypes.py | 111 +++++++++++++------------- 1 file changed, 56 insertions(+), 55 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index ae6361ecd3..8f1ca25fa8 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -927,19 +927,17 @@ class CustomResponse(LoncapaResponse): # actual function that will re-execute the original script, # and invoke the function with the data needed. def make_check_function(script_code, cfn): - def check_function(expect, ans, **kwargs): - code = [script_code, "kwargs = {}"] - for name, val in kwargs.iteritems(): - if isinstance(val, (str, list, tuple, int, long, float, dict)): - code.append("kwargs[%r] = %r" % (name, val)) - code.append("cfn_return = %s(expect, ans, **kwargs)" % cfn) - code.append("") # a final newline + def check_function(expect, ans): + code = ( + script_code + "\n" + + "cfn_return = %s(expect, ans)\n" % cfn + ) globals_dict = { 'expect': expect, 'ans': ans, } locals_dict = {} - safe_exec.safe_exec("\n".join(code), globals_dict, locals_dict) + safe_exec.safe_exec(code, globals_dict, locals_dict) return locals_dict['cfn_return'] return check_function @@ -958,8 +956,6 @@ class CustomResponse(LoncapaResponse): else: self.code = answer.text - self.cfn_kwargs_keys = [] - def get_score(self, student_answers): ''' student_answers is a dict with everything from request.POST, but with the first part @@ -1038,16 +1034,30 @@ class CustomResponse(LoncapaResponse): # pass self.system.debug to cfn self.context['debug'] = self.system.DEBUG + # Run the check function + self.execute_check_function(idset, submission) + + # build map giving "correct"ness of the answer(s) + correct = self.context['correct'] + messages = self.context['messages'] + correct_map = CorrectMap() + + overall_message = self.clean_message_html(self.context['overall_message'])) + correct_map.set_overall_message(overall_message) + + for k in range(len(idset)): + npoints = self.maxpoints[idset[k]] if correct[k] == 'correct' else 0 + correct_map.set(idset[k], correct[k], msg=messages[k], + npoints=npoints) + return correct_map + + def execute_check_function(self, idset, submission): # exec the check function if isinstance(self.code, basestring): try: locals_dict = {} safe_exec.safe_exec(self.code, self.context, locals_dict) self.context.update(locals_dict) - correct = self.context['correct'] - messages = self.context['messages'] - overall_message = self.context['overall_message'] - except Exception as err: self._handle_exec_exception(err) @@ -1056,33 +1066,27 @@ class CustomResponse(LoncapaResponse): # this is an interface to the Tutor2 check functions fn = self.code - ret = None log.debug(" submission = %s" % submission) try: answer_given = submission[0] if (len(idset) == 1) else submission - kwargs = {n:self.context.get(n) for n in self.cfn_kwargs_keys} - ret = fn(self.expect, answer_given, **kwargs) + ret = fn(self.expect, answer_given) except Exception as err: - self._handle_exec_exception(err) - - if type(ret) == dict: - + log.error("oops in customresponse (cfn) error %s" % err) + # print "context = ",self.context + log.error(traceback.format_exc()) + raise Exception("oops in customresponse (cfn) error %s" % err) + log.debug( + "[courseware.capa.responsetypes.customresponse.get_score] ret = %s", + ret + ) + if isinstance(ret, dict): # One kind of dictionary the check function can return has the # form {'ok': BOOLEAN, 'msg': STRING} # If there are multiple inputs, they all get marked # to the same correct/incorrect value if 'ok' in ret: - correct = ['correct'] * len(idset) if ret[ - 'ok'] else ['incorrect'] * len(idset) - msg = ret.get('msg', None) - msg = self.clean_message_html(msg) - - # If there is only one input, apply the message to that input - # Otherwise, apply the message to the whole problem - if len(idset) > 1: - overall_message = msg - else: - messages[0] = msg + correct = ['correct' if ret['ok'] else 'incorrect'] * len(idset) + self.context['messages'][0] = self.clean_message_html(ret['msg']) # Another kind of dictionary the check function can return has # the form: @@ -1104,6 +1108,7 @@ class CustomResponse(LoncapaResponse): msg = (self.clean_message_html(input_dict['msg']) if 'msg' in input_dict else None) messages.append(msg) + self.context['messages'] = messages # Otherwise, we do not recognize the dictionary # Raise an exception @@ -1112,25 +1117,10 @@ class CustomResponse(LoncapaResponse): raise ResponseError( "CustomResponse: check function returned an invalid dict") - # The check function can return a boolean value, - # indicating whether all inputs should be marked - # correct or incorrect else: - n = len(idset) - correct = ['correct'] * n if ret else ['incorrect'] * n + correct = ['correct' if ret else 'incorrect'] * len(idset) - # build map giving "correct"ness of the answer(s) - correct_map = CorrectMap() - - overall_message = self.clean_message_html(overall_message) - correct_map.set_overall_message(overall_message) - - for k in range(len(idset)): - npoints = (self.maxpoints[idset[k]] - if correct[k] == 'correct' else 0) - correct_map.set(idset[k], correct[k], msg=messages[k], - npoints=npoints) - return correct_map + self.context['correct'] = correct def clean_message_html(self, msg): @@ -1205,12 +1195,23 @@ class SymbolicResponse(CustomResponse): response_tag = 'symbolicresponse' - def setup_response(self): - # No, this is not pretty. - self.context['script_code'] += "from symmath import symmath_check\n" - self.xml.set('cfn', 'symmath_check') - CustomResponse.setup_response(self) - self.cfn_kwargs_keys.extend(['dynamath', 'options', 'debug']) + def execute_check_function(self, idset, submission): + from symmath import symmath_check + fn = self.code + try: + answer_given = submission[0] if (len(idset) == 1) else submission + ret = symmath_check( + self.expect, answer_given, + dynamath=self.context.get('dynamath'), + options=self.context.get('options'), + debug=self.context.get('debug'), + ) + except Exception as err: + log.error("oops in symbolicresponse (cfn) error %s" % err) + log.error(traceback.format_exc()) + raise Exception("oops in symbolicresponse (cfn) error %s" % err) + self.context['messages'][0] = self.clean_message_html(ret['msg']) + self.context['correct'] = ['correct' if ret['ok'] else 'incorrect'] * len(idset) #----------------------------------------------------------------------------- From df17c0c7dde48e46ecf403e0412c5cf09087f689 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 22 Feb 2013 06:27:36 -0500 Subject: [PATCH 049/162] Move symmath into capa so that it's available where needed --- common/lib/{sandbox-packages => capa}/symmath/README.md | 0 common/lib/{sandbox-packages => capa}/symmath/__init__.py | 0 common/lib/{sandbox-packages => capa}/symmath/formula.py | 0 common/lib/{sandbox-packages => capa}/symmath/symmath_check.py | 0 common/lib/sandbox-packages/setup.py | 3 --- 5 files changed, 3 deletions(-) rename common/lib/{sandbox-packages => capa}/symmath/README.md (100%) rename common/lib/{sandbox-packages => capa}/symmath/__init__.py (100%) rename common/lib/{sandbox-packages => capa}/symmath/formula.py (100%) rename common/lib/{sandbox-packages => capa}/symmath/symmath_check.py (100%) diff --git a/common/lib/sandbox-packages/symmath/README.md b/common/lib/capa/symmath/README.md similarity index 100% rename from common/lib/sandbox-packages/symmath/README.md rename to common/lib/capa/symmath/README.md diff --git a/common/lib/sandbox-packages/symmath/__init__.py b/common/lib/capa/symmath/__init__.py similarity index 100% rename from common/lib/sandbox-packages/symmath/__init__.py rename to common/lib/capa/symmath/__init__.py diff --git a/common/lib/sandbox-packages/symmath/formula.py b/common/lib/capa/symmath/formula.py similarity index 100% rename from common/lib/sandbox-packages/symmath/formula.py rename to common/lib/capa/symmath/formula.py diff --git a/common/lib/sandbox-packages/symmath/symmath_check.py b/common/lib/capa/symmath/symmath_check.py similarity index 100% rename from common/lib/sandbox-packages/symmath/symmath_check.py rename to common/lib/capa/symmath/symmath_check.py diff --git a/common/lib/sandbox-packages/setup.py b/common/lib/sandbox-packages/setup.py index f5cfa91b8b..1b99118aca 100644 --- a/common/lib/sandbox-packages/setup.py +++ b/common/lib/sandbox-packages/setup.py @@ -4,14 +4,11 @@ setup( name="sandbox-packages", version="0.1", packages=[ - "symmath", "verifiers", ], py_modules=[ "eia", ], install_requires=[ - # symmath needs: - "sympy", "requests", "lxml", ], ) From ec7a04fdb3cb26d8fc8b2de2b10b82e17167f94f Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 22 Feb 2013 13:31:31 -0500 Subject: [PATCH 050/162] A /debug/run_python endpoint for staff to test the sandboxing of Python code. --- lms/djangoapps/debug/__init__.py | 0 lms/djangoapps/debug/models.py | 3 +++ lms/djangoapps/debug/views.py | 29 ++++++++++++++++++++++++ lms/envs/common.py | 1 + lms/templates/debug/run_python_form.html | 19 ++++++++++++++++ lms/urls.py | 4 ++++ 6 files changed, 56 insertions(+) create mode 100644 lms/djangoapps/debug/__init__.py create mode 100644 lms/djangoapps/debug/models.py create mode 100644 lms/djangoapps/debug/views.py create mode 100644 lms/templates/debug/run_python_form.html diff --git a/lms/djangoapps/debug/__init__.py b/lms/djangoapps/debug/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lms/djangoapps/debug/models.py b/lms/djangoapps/debug/models.py new file mode 100644 index 0000000000..71a8362390 --- /dev/null +++ b/lms/djangoapps/debug/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/lms/djangoapps/debug/views.py b/lms/djangoapps/debug/views.py new file mode 100644 index 0000000000..5d58436ed6 --- /dev/null +++ b/lms/djangoapps/debug/views.py @@ -0,0 +1,29 @@ +"""Views for debugging and diagnostics""" + +import pprint + +from django.http import Http404 +from django.contrib.auth.decorators import login_required +from django_future.csrf import ensure_csrf_cookie, csrf_exempt +from mitxmako.shortcuts import render_to_response + +from codejail.safe_exec import safe_exec + +@login_required +@ensure_csrf_cookie +def run_python(request): + if not request.user.is_staff: + raise Http404 + c = {} + c['code'] = '' + c['results'] = None + if request.method == 'POST': + py_code = c['code'] = request.POST.get('code') + g, l = {}, {} + try: + safe_exec(py_code, g, l) + except Exception as e: + c['results'] = str(e) + else: + c['results'] = pprint.pformat(l) + return render_to_response("debug/run_python_form.html", c) diff --git a/lms/envs/common.py b/lms/envs/common.py index b99efd60c3..1b492a3c56 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -589,6 +589,7 @@ INSTALLED_APPS = ( # For testing 'django.contrib.admin', # only used in DEBUG mode + 'debug', # Discussion forums 'django_comment_client', diff --git a/lms/templates/debug/run_python_form.html b/lms/templates/debug/run_python_form.html new file mode 100644 index 0000000000..daecdf2abd --- /dev/null +++ b/lms/templates/debug/run_python_form.html @@ -0,0 +1,19 @@ + +
+

Python:

+
+ +
+ +
+ +
+
+%if results: +
+

Results:

+ 
+${results|h}
+
+
+%endif diff --git a/lms/urls.py b/lms/urls.py index 126d68c73e..dc558d6a54 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -358,6 +358,10 @@ urlpatterns += ( url(r'^comm/foldit_ops', 'foldit.views.foldit_ops', name="foldit_ops"), ) +urlpatterns += ( + url(r'^debug/run_python', 'debug.views.run_python'), +) + urlpatterns = patterns(*urlpatterns) if settings.DEBUG: From 771de938c7ed6d4b01ca0150797e4f0d5e5651e0 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 25 Feb 2013 10:48:14 -0500 Subject: [PATCH 051/162] Update the instructions for setting up the sandbox. --- common/lib/codejail/README | 49 ++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/common/lib/codejail/README b/common/lib/codejail/README index 7b1849e18c..75862d69e3 100644 --- a/common/lib/codejail/README +++ b/common/lib/codejail/README @@ -1,21 +1,56 @@ -Choose a place for the virtualenv, call it +Choose a place for the virtualenv, call it . It will be automatically +detected and used if you put it right alongside your existing virtualenv, but +with -sandbox appended. So if your existing virtualenv is in ~/mitx_all/python, +make be ~/mitx_all/python-sandbox (but you'll need to spell out your +home directory instead of ~). + +Other details here that depend on your configuration: + + - Your mitx working tree is , for example, ~/mitx_all/mitx + + - The user running the LMS is , for example, you on a dev machine, + or www-data on a server. Create a virtualenv: - virtualenv + $ sudo virtualenv Install the sandbox requirements + $ source /bin/activate + $ sudo pip install -r sandbox-requirements.txt -Edit an AppArmor profile: +Add a sandbox user: + + $ sudo addgroup sandbox + $ sudo adduser --disabled-login sandbox --ingroup sandbox + +Let the web server run the sandboxed Python as sandbox. Create the file +/etc/sudoers.d/01-sandbox: + + $ visudo -f /etc/sudoers.d/01-sandbox + + ALL=(sandbox) NOPASSWD:/bin/python + ALL=(ALL) NOPASSWD:/bin/kill + +Edit an AppArmor profile. The file must be named for the python executable, +but with slashes changed to dots: + + #include /bin/python { - ... + #include + + /** mr, + /common/lib/sandbox-packages/** r, + /usr/local/lib/python2.7/** r, + /usr/lib/python2.7/** rix, + + /tmp/** rix, } Parse the profiles - $ apparmor_parser - $ aaenforce /bin/python - + $ sudo apparmor_parser +Reactivate your real virtualenv again From d9df65eef06aa8bd0db521e108d97741060503c3 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 25 Feb 2013 16:04:59 -0500 Subject: [PATCH 052/162] Add some logging to codejail --- common/lib/codejail/codejail/jailpy.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/common/lib/codejail/codejail/jailpy.py b/common/lib/codejail/codejail/jailpy.py index ed13a0d3f0..c3e98a518a 100644 --- a/common/lib/codejail/codejail/jailpy.py +++ b/common/lib/codejail/codejail/jailpy.py @@ -3,6 +3,7 @@ # Instructions: # - AppArmor.md from xserver +import logging import os, os.path import resource import shutil @@ -13,6 +14,8 @@ import time from .util import temp_directory +log = logging.getLogger(__name__) + # TODO: limit too much stdout data? # Configure the Python command @@ -61,6 +64,8 @@ def jailpy(code, files=None, argv=None, stdin=None): with temp_directory(delete_when_done=True) as tmpdir: + log.debug("Executing jailed code: %r", code) + # All the supporting files are copied into our directory. for filename in files or (): if os.path.isfile(filename): From 1473fe377a8ef6a48940986810724ba375ef9b43 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 26 Feb 2013 13:19:19 -0500 Subject: [PATCH 053/162] A unit test that demonstrates the problem we're having with some sandboxed code. --- common/lib/codejail/codejail/tests/test_safe_exec.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/common/lib/codejail/codejail/tests/test_safe_exec.py b/common/lib/codejail/codejail/tests/test_safe_exec.py index 1bdcceaf44..f9b496b8e9 100644 --- a/common/lib/codejail/codejail/tests/test_safe_exec.py +++ b/common/lib/codejail/codejail/tests/test_safe_exec.py @@ -1,6 +1,7 @@ """Test safe_exec.py""" import os.path +import textwrap import unittest from nose.plugins.skip import SkipTest @@ -42,6 +43,17 @@ class SafeExecTests(object): ) self.assertEqual(l['a'], 42) + def test_functions_calling_each_other(self): + g, l = {}, {} + self.safe_exec(textwrap.dedent("""\ + def f(): + return 1723 + def g(): + return f() + x = g() + """), g, l) + self.assertEqual(l['x'], 1723) + class TestSafeExec(SafeExecTests, unittest.TestCase): """Run SafeExecTests, with the real safe_exec.""" From 839c5684746391a8c2173f23662cf60050a32a1e Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 26 Feb 2013 14:17:08 -0500 Subject: [PATCH 054/162] Hmm, turns out exec wants just one dict to properly simulate Python module execution. --- common/lib/capa/capa/capa_problem.py | 4 +- common/lib/capa/capa/responsetypes.py | 14 +++---- common/lib/capa/capa/safe_exec.py | 4 +- common/lib/capa/capa/tests/test_safe_exec.py | 32 ++++++++-------- common/lib/codejail/codejail/safe_exec.py | 30 +++++++-------- .../codejail/codejail/tests/test_safe_exec.py | 38 +++++++++---------- lms/djangoapps/debug/views.py | 6 +-- 7 files changed, 61 insertions(+), 67 deletions(-) diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index 9665721293..b5e773ebc7 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -462,9 +462,7 @@ class LoncapaProblem(object): if all_code: try: - locals_dict = {} - safe_exec.safe_exec(all_code, context, locals_dict, random_seed=self.seed, python_path=python_path) - context.update(locals_dict) + safe_exec.safe_exec(all_code, context, random_seed=self.seed, python_path=python_path) except Exception as err: log.exception("Error while execing script code: " + all_code) msg = "Error while executing script code: %s" % str(err).replace('<', '<') diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 8f1ca25fa8..c441539aee 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -936,9 +936,8 @@ class CustomResponse(LoncapaResponse): 'expect': expect, 'ans': ans, } - locals_dict = {} - safe_exec.safe_exec(code, globals_dict, locals_dict) - return locals_dict['cfn_return'] + safe_exec.safe_exec(code, globals_dict) + return globals_dict['cfn_return'] return check_function self.code = make_check_function(self.context['script_code'], cfn) @@ -1055,9 +1054,7 @@ class CustomResponse(LoncapaResponse): # exec the check function if isinstance(self.code, basestring): try: - locals_dict = {} - safe_exec.safe_exec(self.code, self.context, locals_dict) - self.context.update(locals_dict) + safe_exec.safe_exec(self.code, self.context) except Exception as err: self._handle_exec_exception(err) @@ -1762,10 +1759,9 @@ class SchematicResponse(LoncapaResponse): json.loads(student_answers[k]) for k in sorted(self.answer_ids) ] self.context.update({'submission': submission}) - locals_dict = {} - safe_exec.safe_exec(self.code, self.context, locals_dict) + safe_exec.safe_exec(self.code, self.context) cmap = CorrectMap() - cmap.set_dict(dict(zip(sorted(self.answer_ids), locals_dict['correct']))) + cmap.set_dict(dict(zip(sorted(self.answer_ids), self.context['correct']))) return cmap def get_answers(self): diff --git a/common/lib/capa/capa/safe_exec.py b/common/lib/capa/capa/safe_exec.py index 31f47b5c36..dd52c3571a 100644 --- a/common/lib/capa/capa/safe_exec.py +++ b/common/lib/capa/capa/safe_exec.py @@ -14,13 +14,13 @@ random.Random = random_module.Random del random_module """ -def safe_exec(code, globals_dict, locals_dict, random_seed=None, python_path=None): +def safe_exec(code, globals_dict, random_seed=None, python_path=None): """Exec python code safely. """ code_prolog = CODE_PROLOG % random_seed codejail.safe_exec.safe_exec( - code_prolog + code, globals_dict, locals_dict, + code_prolog + code, globals_dict, python_path=python_path, assumed_imports=[ "numpy", diff --git a/common/lib/capa/capa/tests/test_safe_exec.py b/common/lib/capa/capa/tests/test_safe_exec.py index 3a436ac3ce..8a65cb6c38 100644 --- a/common/lib/capa/capa/tests/test_safe_exec.py +++ b/common/lib/capa/capa/tests/test_safe_exec.py @@ -8,39 +8,39 @@ from capa.safe_exec import safe_exec class TestSafeExec(unittest.TestCase): def test_set_values(self): - g, l = {}, {} - safe_exec("a = 17", g, l) - self.assertEqual(l['a'], 17) + g = {} + safe_exec("a = 17", g) + self.assertEqual(g['a'], 17) def test_division(self): - g, l = {}, {} + g = {} # Future division: 1/2 is 0.5. - safe_exec("a = 1/2", g, l) - self.assertEqual(l['a'], 0.5) + safe_exec("a = 1/2", g) + self.assertEqual(g['a'], 0.5) def test_assumed_imports(self): - g, l = {}, {} + g = {} # Math is always available. - safe_exec("a = int(math.pi)", g, l) - self.assertEqual(l['a'], 3) + safe_exec("a = int(math.pi)", g) + self.assertEqual(g['a'], 3) def test_random_seeding(self): - g, l = {}, {} + g = {} r = random.Random(17) rnums = [r.randint(0, 999) for _ in xrange(100)] # Without a seed, the results are unpredictable - safe_exec("rnums = [random.randint(0, 999) for _ in xrange(100)]", g, l) - self.assertNotEqual(l['rnums'], rnums) + safe_exec("rnums = [random.randint(0, 999) for _ in xrange(100)]", g) + self.assertNotEqual(g['rnums'], rnums) # With a seed, the results are predictable - safe_exec("rnums = [random.randint(0, 999) for _ in xrange(100)]", g, l, random_seed=17) - self.assertEqual(l['rnums'], rnums) + safe_exec("rnums = [random.randint(0, 999) for _ in xrange(100)]", g, random_seed=17) + self.assertEqual(g['rnums'], rnums) def test_python_lib(self): pylib = os.path.dirname(__file__) + "/test_files/pylib" - g, l = {}, {} + g = {} safe_exec( "import constant; a = constant.THE_CONST", - g, l, python_path=[pylib] + g, python_path=[pylib] ) diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index 5a4e48d39e..c4c6a8145b 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -28,19 +28,19 @@ def names_and_modules(assumed_imports): yield modname, modname -def safe_exec(code, globals_dict, locals_dict, assumed_imports=None, files=None, python_path=None): +def safe_exec(code, globals_dict, assumed_imports=None, files=None, python_path=None): """Execute code as "exec" does, but safely. - `code` is a string of Python code. `globals_dict` and `locals_dict` are - dictionaries to use as the globals and locals. Modifications the code - makes to `locals_dict` are reflected in the dictionary on return. + `code` is a string of Python code. `globals_dict` is used as the globals + during execution. Modifications the code makes to `globals_dict` are + reflected in the dictionary on return. `assumed_imports` is a list of modules to make available as implicit imports for the code. Entries are either a name, "mod", which makes "import mod" part of the code, or a pair, ("f", "fooey"), which makes "import fooey as f" part of the code. The module name can be dotted. - Returns None. Changes made by `code` are visible in `locals_dict`. + Returns None. Changes made by `code` are visible in `globals_dict`. """ the_code = [] @@ -49,7 +49,7 @@ def safe_exec(code, globals_dict, locals_dict, assumed_imports=None, files=None, the_code.append(textwrap.dedent("""\ import json import sys - code, g_dict, l_dict = json.load(sys.stdin) + code, g_dict = json.load(sys.stdin) """)) for pydir in python_path or (): @@ -63,13 +63,14 @@ def safe_exec(code, globals_dict, locals_dict, assumed_imports=None, files=None, the_code.append("g_dict['{}'] = LazyModule('{}')\n".format(name, modname)) the_code.append(textwrap.dedent("""\ - exec code in g_dict, l_dict + exec code in g_dict ok_types = (type(None), int, long, float, str, unicode, list, tuple, dict) - l_dict = {k:v for k,v in l_dict.iteritems() if isinstance(v, ok_types)} - json.dump(l_dict, sys.stdout) + bad_keys = ("__builtins__",) + g_dict = {k:v for k,v in g_dict.iteritems() if isinstance(v, ok_types) and k not in bad_keys} + json.dump(g_dict, sys.stdout) """)) - stdin = json.dumps([code, globals_dict, locals_dict]) + stdin = json.dumps([code, globals_dict]) jailed_code = "".join(the_code) # Turn this on to see what's being executed. @@ -82,10 +83,10 @@ def safe_exec(code, globals_dict, locals_dict, assumed_imports=None, files=None, res = jailpy.jailpy(jailed_code, stdin=stdin, files=files) if res.status != 0: raise Exception("Couldn't excecute jailed code: %s" % res.stderr) - locals_dict.update(json.loads(res.stdout)) + globals_dict.update(json.loads(res.stdout)) -def not_safe_exec(code, globals_dict, locals_dict, assumed_imports=None, files=None, python_path=None): +def not_safe_exec(code, globals_dict, assumed_imports=None, files=None, python_path=None): """Another implementation of `safe_exec`, but not safe. This can be swapped in for debugging problems in sandboxed Python code. @@ -111,7 +112,6 @@ def not_safe_exec(code, globals_dict, locals_dict, assumed_imports=None, files=N return json.loads(json.dumps(jd)) g_dict = straw(globals_dict) - l_dict = straw(locals_dict) for name, modname in names_and_modules(assumed_imports or ()): g_dict[name] = lazymod.LazyModule(modname) @@ -127,11 +127,11 @@ def not_safe_exec(code, globals_dict, locals_dict, assumed_imports=None, files=N if python_path: sys.path.extend(python_path) try: - exec code in g_dict, l_dict + exec code in g_dict finally: sys.path = original_path - locals_dict.update(straw(l_dict)) + globals_dict.update(straw(g_dict)) # Running Python code in the sandbox makes it difficult to debug. # Change 0 to 1 to run the code directly. diff --git a/common/lib/codejail/codejail/tests/test_safe_exec.py b/common/lib/codejail/codejail/tests/test_safe_exec.py index f9b496b8e9..bf4a0408cd 100644 --- a/common/lib/codejail/codejail/tests/test_safe_exec.py +++ b/common/lib/codejail/codejail/tests/test_safe_exec.py @@ -10,49 +10,49 @@ from codejail.safe_exec import safe_exec, not_safe_exec class SafeExecTests(object): """The tests for `safe_exec`, will be mixed into specific test classes below.""" def test_set_values(self): - g, l = {}, {} - self.safe_exec("a = 17", g, l) - self.assertEqual(l['a'], 17) + g = {} + self.safe_exec("a = 17", g) + self.assertEqual(g['a'], 17) def test_assumed_imports(self): - g, l = {}, {} + g = {} # Using string without importing it is bad. with self.assertRaises(Exception): - self.safe_exec("a = string.ascii_lowercase[0]", g, l) + self.safe_exec("a = string.ascii_lowercase[0]", g) # Using string with an assumed import is fine. - self.safe_exec("a = string.ascii_lowercase[0]", g, l, assumed_imports=["string"]) - self.assertEqual(l['a'], 'a') + self.safe_exec("a = string.ascii_lowercase[0]", g, assumed_imports=["string"]) + self.assertEqual(g['a'], 'a') # Can also import with a shorthand. - self.safe_exec("a = op.join('x', 'y')", g, l, assumed_imports=[("op", "os.path")]) - self.assertEqual(l['a'][0], 'x') - self.assertEqual(l['a'][-1], 'y') + self.safe_exec("a = op.join('x', 'y')", g, assumed_imports=[("op", "os.path")]) + self.assertEqual(g['a'][0], 'x') + self.assertEqual(g['a'][-1], 'y') def test_files_are_copied(self): - g, l = {}, {} + g = {} self.safe_exec( - "a = 'Look: ' + open('hello.txt').read()", g, l, + "a = 'Look: ' + open('hello.txt').read()", g, files=[os.path.dirname(__file__) + "/hello.txt"] ) - self.assertEqual(l['a'], 'Look: Hello there.\n') + self.assertEqual(g['a'], 'Look: Hello there.\n') def test_python_path(self): - g, l = {}, {} + g = {} self.safe_exec( - "import module; a = module.const", g, l, + "import module; a = module.const", g, python_path=[os.path.dirname(__file__) + "/pylib"] ) - self.assertEqual(l['a'], 42) + self.assertEqual(g['a'], 42) def test_functions_calling_each_other(self): - g, l = {}, {} + g = {} self.safe_exec(textwrap.dedent("""\ def f(): return 1723 def g(): return f() x = g() - """), g, l) - self.assertEqual(l['x'], 1723) + """), g) + self.assertEqual(g['x'], 1723) class TestSafeExec(SafeExecTests, unittest.TestCase): diff --git a/lms/djangoapps/debug/views.py b/lms/djangoapps/debug/views.py index 5d58436ed6..3bf0c81240 100644 --- a/lms/djangoapps/debug/views.py +++ b/lms/djangoapps/debug/views.py @@ -19,11 +19,11 @@ def run_python(request): c['results'] = None if request.method == 'POST': py_code = c['code'] = request.POST.get('code') - g, l = {}, {} + g = {} try: - safe_exec(py_code, g, l) + safe_exec(py_code, g) except Exception as e: c['results'] = str(e) else: - c['results'] = pprint.pformat(l) + c['results'] = pprint.pformat(g) return render_to_response("debug/run_python_form.html", c) From 5acb2258164ce5f8d7c8c297b32534547e861450 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 26 Feb 2013 17:03:39 -0500 Subject: [PATCH 055/162] Print the full traceback when execution fails. --- lms/djangoapps/debug/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lms/djangoapps/debug/views.py b/lms/djangoapps/debug/views.py index 3bf0c81240..c1d4155fdd 100644 --- a/lms/djangoapps/debug/views.py +++ b/lms/djangoapps/debug/views.py @@ -1,6 +1,7 @@ """Views for debugging and diagnostics""" import pprint +import traceback from django.http import Http404 from django.contrib.auth.decorators import login_required @@ -12,6 +13,7 @@ from codejail.safe_exec import safe_exec @login_required @ensure_csrf_cookie def run_python(request): + """A page to allow testing the Python sandbox on a production server.""" if not request.user.is_staff: raise Http404 c = {} @@ -23,7 +25,7 @@ def run_python(request): try: safe_exec(py_code, g) except Exception as e: - c['results'] = str(e) + c['results'] = traceback.format_exc() else: c['results'] = pprint.pformat(g) return render_to_response("debug/run_python_form.html", c) From b95ea4422bdb6c161a8ff466403d48f92ed88ce6 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 26 Feb 2013 17:27:31 -0500 Subject: [PATCH 056/162] Prevent a print statement from accidentally borking the sandbox. --- common/lib/codejail/codejail/safe_exec.py | 31 ++++++++++++++++--- .../codejail/codejail/tests/test_safe_exec.py | 5 +++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index c4c6a8145b..7b4037a2a9 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -46,9 +46,24 @@ def safe_exec(code, globals_dict, assumed_imports=None, files=None, python_path= the_code = [] files = list(files or ()) - the_code.append(textwrap.dedent("""\ + the_code.append(textwrap.dedent( + """ import json import sys + """ + # We need to prevent the sandboxed code from printing to stdout, + # or it will pollute the json we print there. This isn't a + # security concern (they can put any values in the json output + # anyway, either by writing to sys.__stdout__, or just by defining + # global values), but keeps accidents from happening. + """ + class DevNull(object): + def write(self, *args, **kwargs): + pass + sys.stdout = DevNull() + """ + # Read the code and the globals from the stdin. + """ code, g_dict = json.load(sys.stdin) """)) @@ -62,12 +77,20 @@ def safe_exec(code, globals_dict, assumed_imports=None, files=None, python_path= for name, modname in names_and_modules(assumed_imports): the_code.append("g_dict['{}'] = LazyModule('{}')\n".format(name, modname)) - the_code.append(textwrap.dedent("""\ + the_code.append(textwrap.dedent( + # Execute the sandboxed code. + """ exec code in g_dict + """ + # Clean the globals for sending back as JSON over stdout. + """ ok_types = (type(None), int, long, float, str, unicode, list, tuple, dict) bad_keys = ("__builtins__",) g_dict = {k:v for k,v in g_dict.iteritems() if isinstance(v, ok_types) and k not in bad_keys} - json.dump(g_dict, sys.stdout) + """ + # Write the globals back to the calling process. + """ + json.dump(g_dict, sys.__stdout__) """)) stdin = json.dumps([code, globals_dict]) @@ -82,7 +105,7 @@ def safe_exec(code, globals_dict, assumed_imports=None, files=None, python_path= res = jailpy.jailpy(jailed_code, stdin=stdin, files=files) if res.status != 0: - raise Exception("Couldn't excecute jailed code: %s" % res.stderr) + raise Exception("Couldn't execute jailed code: %s" % res.stderr) globals_dict.update(json.loads(res.stdout)) diff --git a/common/lib/codejail/codejail/tests/test_safe_exec.py b/common/lib/codejail/codejail/tests/test_safe_exec.py index bf4a0408cd..b4f3627ad6 100644 --- a/common/lib/codejail/codejail/tests/test_safe_exec.py +++ b/common/lib/codejail/codejail/tests/test_safe_exec.py @@ -54,6 +54,11 @@ class SafeExecTests(object): """), g) self.assertEqual(g['x'], 1723) + def test_printing_stuff_when_you_shouldnt(self): + g = {} + self.safe_exec("a = 17; print 'hi!'", g) + self.assertEqual(g['a'], 17) + class TestSafeExec(SafeExecTests, unittest.TestCase): """Run SafeExecTests, with the real safe_exec.""" From 478f967af46b9808354d8b8b69242bead2fc5f35 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 27 Feb 2013 14:01:31 -0500 Subject: [PATCH 057/162] We would fail if a global was defined with a non-jsonable value inside a jsonable one. Now we don't/ --- common/lib/codejail/codejail/safe_exec.py | 20 ++++++++++++++++++- .../codejail/codejail/tests/test_safe_exec.py | 8 ++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index 7b4037a2a9..b77081a2a2 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -43,6 +43,10 @@ def safe_exec(code, globals_dict, assumed_imports=None, files=None, python_path= Returns None. Changes made by `code` are visible in `globals_dict`. """ + print "--- Executing: -------" + print code + print + the_code = [] files = list(files or ()) @@ -86,7 +90,15 @@ def safe_exec(code, globals_dict, assumed_imports=None, files=None, python_path= """ ok_types = (type(None), int, long, float, str, unicode, list, tuple, dict) bad_keys = ("__builtins__",) - g_dict = {k:v for k,v in g_dict.iteritems() if isinstance(v, ok_types) and k not in bad_keys} + def jsonable(v): + if not isinstance(v, ok_types): + return False + try: + json.dumps(v) + except Exception: + return False + return True + g_dict = {k:v for k,v in g_dict.iteritems() if jsonable(v) and k not in bad_keys} """ # Write the globals back to the calling process. """ @@ -124,8 +136,14 @@ def not_safe_exec(code, globals_dict, assumed_imports=None, files=None, python_p Used to emulate reading data through a serialization straw. """ + ok_types = (type(None), int, long, float, str, unicode, list, tuple, dict) + bad_keys = ("__builtins__",) jd = {} for k,v in d.iteritems(): + if not isinstance(v, ok_types): + continue + if k in bad_keys: + continue try: json.dumps(v) except TypeError: diff --git a/common/lib/codejail/codejail/tests/test_safe_exec.py b/common/lib/codejail/codejail/tests/test_safe_exec.py index b4f3627ad6..8a565f8cfb 100644 --- a/common/lib/codejail/codejail/tests/test_safe_exec.py +++ b/common/lib/codejail/codejail/tests/test_safe_exec.py @@ -59,6 +59,14 @@ class SafeExecTests(object): self.safe_exec("a = 17; print 'hi!'", g) self.assertEqual(g['a'], 17) + def test_importing_lots_of_crap(self): + g = {} + self.safe_exec(textwrap.dedent("""\ + from numpy import * + a = 1723 + """), g) + self.assertEqual(g['a'], 1723) + class TestSafeExec(SafeExecTests, unittest.TestCase): """Run SafeExecTests, with the real safe_exec.""" From f3e8d5bb7abb6862b293e0ee5a6f555c5f8d635b Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 27 Feb 2013 14:04:18 -0500 Subject: [PATCH 058/162] Didn't mean to put this in --- common/lib/codejail/codejail/safe_exec.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index b77081a2a2..f2ffa41523 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -43,10 +43,6 @@ def safe_exec(code, globals_dict, assumed_imports=None, files=None, python_path= Returns None. Changes made by `code` are visible in `globals_dict`. """ - print "--- Executing: -------" - print code - print - the_code = [] files = list(files or ()) From 283fc47a95f2f0be95b5ba38e6dc2056b1621230 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 27 Feb 2013 14:13:45 -0500 Subject: [PATCH 059/162] Jailed code importing random explicitly would get the wrong seed. --- common/lib/capa/capa/safe_exec.py | 2 ++ common/lib/capa/capa/tests/test_safe_exec.py | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/common/lib/capa/capa/safe_exec.py b/common/lib/capa/capa/safe_exec.py index dd52c3571a..3d481495d4 100644 --- a/common/lib/capa/capa/safe_exec.py +++ b/common/lib/capa/capa/safe_exec.py @@ -9,9 +9,11 @@ CODE_PROLOG = """\ from __future__ import division import random as random_module +import sys random = random_module.Random(%r) random.Random = random_module.Random del random_module +sys.modules['random'] = random """ def safe_exec(code, globals_dict, random_seed=None, python_path=None): diff --git a/common/lib/capa/capa/tests/test_safe_exec.py b/common/lib/capa/capa/tests/test_safe_exec.py index 8a65cb6c38..7ed44a69a1 100644 --- a/common/lib/capa/capa/tests/test_safe_exec.py +++ b/common/lib/capa/capa/tests/test_safe_exec.py @@ -37,6 +37,18 @@ class TestSafeExec(unittest.TestCase): safe_exec("rnums = [random.randint(0, 999) for _ in xrange(100)]", g, random_seed=17) self.assertEqual(g['rnums'], rnums) + def test_random_is_still_importable(self): + g = {} + r = random.Random(17) + rnums = [r.randint(0, 999) for _ in xrange(100)] + + # With a seed, the results are predictable even from the random module + safe_exec( + "import random\n" + "rnums = [random.randint(0, 999) for _ in xrange(100)]\n", + g, random_seed=17) + self.assertEqual(g['rnums'], rnums) + def test_python_lib(self): pylib = os.path.dirname(__file__) + "/test_files/pylib" g = {} From 7aa493ec85f68f48004d49d32f2ed9a50ec1201e Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 1 Mar 2013 10:36:20 -0500 Subject: [PATCH 060/162] A start on getting these tests to run again. --- .../capa/capa/tests/response_xml_factory.py | 21 +++++++++++++++++++ .../lib/capa/capa/tests/test_responsetypes.py | 5 ++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/common/lib/capa/capa/tests/response_xml_factory.py b/common/lib/capa/capa/tests/response_xml_factory.py index aa401b70cd..1d04b7897d 100644 --- a/common/lib/capa/capa/tests/response_xml_factory.py +++ b/common/lib/capa/capa/tests/response_xml_factory.py @@ -241,6 +241,27 @@ class CustomResponseXMLFactory(ResponseXMLFactory): return ResponseXMLFactory.textline_input_xml(**kwargs) +class SymbolicResponseXMLFactory(ResponseXMLFactory): + """ Factory for creating XML trees """ + + def create_response_element(self, **kwargs): + cfn = kwargs.get('cfn', None) + answer = kwargs.get('answer', None) + options = kwargs.get('options', None) + + response_element = etree.Element("symbolicresponse") + if cfn: + response_element.set('cfn', str(cfn)) + if answer: + response_element.set('answer', str(answer)) + if options: + response_element.set('options', str(options)) + return response_element + + def create_input_element(self, **kwargs): + return ResponseXMLFactory.textline_input_xml(**kwargs) + + class SchematicResponseXMLFactory(ResponseXMLFactory): """ Factory for creating XML trees """ diff --git a/common/lib/capa/capa/tests/test_responsetypes.py b/common/lib/capa/capa/tests/test_responsetypes.py index 3d93af0c75..018d9240d7 100644 --- a/common/lib/capa/capa/tests/test_responsetypes.py +++ b/common/lib/capa/capa/tests/test_responsetypes.py @@ -186,7 +186,10 @@ class ImageResponseTest(ResponseTest): class SymbolicResponseTest(unittest.TestCase): - def test_sr_grade(self): + from response_xml_factory import SymbolicResponseXMLFactory + xml_factory_class = SymbolicResponseXMLFactory + + def test_symbolic_response_grade(self): symbolicresponse_file = os.path.dirname(__file__) + "/test_files/symbolicresponse.xml" test_lcp = lcp.LoncapaProblem(open(symbolicresponse_file).read(), '1', system=test_system) correct_answers = {'1_2_1': 'cos(theta)*[[1,0],[0,1]] + i*sin(theta)*[[0,1],[1,0]]', From c49b0c50270c93d9c771f296538dbf9d753527bf Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 1 Mar 2013 11:16:50 -0500 Subject: [PATCH 061/162] Have to make the globals json-safe before sending them to the sandbox. --- common/lib/codejail/codejail/safe_exec.py | 51 ++++++++++++----------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index f2ffa41523..b11ef60616 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -101,7 +101,7 @@ def safe_exec(code, globals_dict, assumed_imports=None, files=None, python_path= json.dump(g_dict, sys.__stdout__) """)) - stdin = json.dumps([code, globals_dict]) + stdin = json.dumps([code, json_safe(globals_dict)]) jailed_code = "".join(the_code) # Turn this on to see what's being executed. @@ -117,6 +117,29 @@ def safe_exec(code, globals_dict, assumed_imports=None, files=None, python_path= globals_dict.update(json.loads(res.stdout)) +def json_safe(d): + """Return only the JSON-safe part of d. + + Used to emulate reading data through a serialization straw. + + """ + ok_types = (type(None), int, long, float, str, unicode, list, tuple, dict) + bad_keys = ("__builtins__",) + jd = {} + for k,v in d.iteritems(): + if not isinstance(v, ok_types): + continue + if k in bad_keys: + continue + try: + json.dumps(v) + except TypeError: + continue + else: + jd[k] = v + return json.loads(json.dumps(jd)) + + def not_safe_exec(code, globals_dict, assumed_imports=None, files=None, python_path=None): """Another implementation of `safe_exec`, but not safe. @@ -126,29 +149,7 @@ def not_safe_exec(code, globals_dict, assumed_imports=None, files=None, python_p and modifying sys.path. """ - def straw(d): - """Return only the JSON-safe part of d. - - Used to emulate reading data through a serialization straw. - - """ - ok_types = (type(None), int, long, float, str, unicode, list, tuple, dict) - bad_keys = ("__builtins__",) - jd = {} - for k,v in d.iteritems(): - if not isinstance(v, ok_types): - continue - if k in bad_keys: - continue - try: - json.dumps(v) - except TypeError: - continue - else: - jd[k] = v - return json.loads(json.dumps(jd)) - - g_dict = straw(globals_dict) + g_dict = json_safe(globals_dict) for name, modname in names_and_modules(assumed_imports or ()): g_dict[name] = lazymod.LazyModule(modname) @@ -168,7 +169,7 @@ def not_safe_exec(code, globals_dict, assumed_imports=None, files=None, python_p finally: sys.path = original_path - globals_dict.update(straw(g_dict)) + globals_dict.update(json_safe(g_dict)) # Running Python code in the sandbox makes it difficult to debug. # Change 0 to 1 to run the code directly. From e61a6fe787e6213b2dd8e64821ca5ede7f71de6c Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 1 Mar 2013 15:08:49 -0500 Subject: [PATCH 062/162] Make it possible for customresponse check functions to get extra arguments, though they need to be declared in the XML. --- common/lib/capa/capa/responsetypes.py | 12 ++++--- .../capa/capa/tests/response_xml_factory.py | 8 +++++ .../lib/capa/capa/tests/test_responsetypes.py | 35 ++++++++++++++++--- 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index c441539aee..9dd1e74409 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -927,15 +927,17 @@ class CustomResponse(LoncapaResponse): # actual function that will re-execute the original script, # and invoke the function with the data needed. def make_check_function(script_code, cfn): - def check_function(expect, ans): + def check_function(expect, ans, **kwargs): + extra_args = "".join(", {0}={0}".format(k) for k in kwargs) code = ( script_code + "\n" + - "cfn_return = %s(expect, ans)\n" % cfn + "cfn_return = %s(expect, ans%s)\n" % (cfn, extra_args) ) globals_dict = { 'expect': expect, 'ans': ans, } + globals_dict.update(kwargs) safe_exec.safe_exec(code, globals_dict) return globals_dict['cfn_return'] return check_function @@ -1063,10 +1065,12 @@ class CustomResponse(LoncapaResponse): # this is an interface to the Tutor2 check functions fn = self.code + answer_given = submission[0] if (len(idset) == 1) else submission + kwnames = self.xml.get("cfn_extra_args", "").split() + kwargs = {n:self.context.get(n) for n in kwnames} log.debug(" submission = %s" % submission) try: - answer_given = submission[0] if (len(idset) == 1) else submission - ret = fn(self.expect, answer_given) + ret = fn(self.expect, answer_given, **kwargs) except Exception as err: log.error("oops in customresponse (cfn) error %s" % err) # print "context = ",self.context diff --git a/common/lib/capa/capa/tests/response_xml_factory.py b/common/lib/capa/capa/tests/response_xml_factory.py index 1d04b7897d..ee1fdfafc9 100644 --- a/common/lib/capa/capa/tests/response_xml_factory.py +++ b/common/lib/capa/capa/tests/response_xml_factory.py @@ -221,6 +221,8 @@ class CustomResponseXMLFactory(ResponseXMLFactory): cfn = kwargs.get('cfn', None) expect = kwargs.get('expect', None) answer = kwargs.get('answer', None) + options = kwargs.get('options', None) + cfn_extra_args = kwargs.get('cfn_extra_args', None) # Create the response element response_element = etree.Element("customresponse") @@ -235,6 +237,12 @@ class CustomResponseXMLFactory(ResponseXMLFactory): answer_element = etree.SubElement(response_element, "answer") answer_element.text = str(answer) + if options: + response_element.set('options', str(options)) + + if cfn_extra_args: + response_element.set('cfn_extra_args', str(cfn_extra_args)) + return response_element def create_input_element(self, **kwargs): diff --git a/common/lib/capa/capa/tests/test_responsetypes.py b/common/lib/capa/capa/tests/test_responsetypes.py index 018d9240d7..c2ee62ed7a 100644 --- a/common/lib/capa/capa/tests/test_responsetypes.py +++ b/common/lib/capa/capa/tests/test_responsetypes.py @@ -2,7 +2,6 @@ Tests of responsetypes """ - from datetime import datetime import json from nose.plugins.skip import SkipTest @@ -806,9 +805,8 @@ class CustomResponseTest(ResponseTest): # # 'answer_given' is the answer the student gave (if there is just one input) # or an ordered list of answers (if there are multiple inputs) - # - # - # The function should return a dict of the form + # + # The function should return a dict of the form # { 'ok': BOOL, 'msg': STRING } # script = textwrap.dedent(""" @@ -917,6 +915,35 @@ class CustomResponseTest(ResponseTest): self.assertEqual(correct_map.get_msg('1_2_2'), 'Feedback 2') self.assertEqual(correct_map.get_msg('1_2_3'), 'Feedback 3') + def test_function_code_with_extra_args(self): + script = textwrap.dedent("""\ + def check_func(expect, answer_given, options, dynamath): + assert options == "xyzzy", "Options was %r" % options + return {'ok': answer_given == expect, 'msg': 'Message text'} + """) + + problem = self.build_problem(script=script, cfn="check_func", expect="42", options="xyzzy", cfn_extra_args="options dynamath") + + # Correct answer + input_dict = {'1_2_1': '42'} + correct_map = problem.grade_answers(input_dict) + + correctness = correct_map.get_correctness('1_2_1') + msg = correct_map.get_msg('1_2_1') + + self.assertEqual(correctness, 'correct') + self.assertEqual(msg, "Message text\n") + + # Incorrect answer + input_dict = {'1_2_1': '0'} + correct_map = problem.grade_answers(input_dict) + + correctness = correct_map.get_correctness('1_2_1') + msg = correct_map.get_msg('1_2_1') + + self.assertEqual(correctness, 'incorrect') + self.assertEqual(msg, "Message text\n") + def test_multiple_inputs_return_one_status(self): # When given multiple inputs, the 'answer_given' argument # to the check_func() is a list of inputs From f62dad2f5710a41453bb9c71f6bef2313b58820e Mon Sep 17 00:00:00 2001 From: Will Daly Date: Tue, 5 Mar 2013 16:31:02 -0500 Subject: [PATCH 063/162] Added symbolic response tests --- common/lib/capa/capa/inputtypes.py | 2 +- common/lib/capa/capa/responsetypes.py | 20 +- .../capa/capa/tests/response_xml_factory.py | 35 +++ .../lib/capa/capa/tests/test_responsetypes.py | 225 +++++++++++------- common/lib/capa/capa/util.py | 2 +- 5 files changed, 188 insertions(+), 96 deletions(-) diff --git a/common/lib/capa/capa/inputtypes.py b/common/lib/capa/capa/inputtypes.py index e253b61948..65280d6d29 100644 --- a/common/lib/capa/capa/inputtypes.py +++ b/common/lib/capa/capa/inputtypes.py @@ -46,7 +46,7 @@ import sys import pyparsing from .registry import TagRegistry -from capa.chem import chemcalc +from chem import chemcalc import xqueue_interface from datetime import datetime diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 9dd1e74409..ddf184c9be 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -29,7 +29,7 @@ from collections import namedtuple from shapely.geometry import Point, MultiPoint # specific library imports -from .calc import evaluator, UndefinedVariable +from calc import evaluator, UndefinedVariable from .correctmap import CorrectMap from datetime import datetime from .util import * @@ -1043,7 +1043,7 @@ class CustomResponse(LoncapaResponse): messages = self.context['messages'] correct_map = CorrectMap() - overall_message = self.clean_message_html(self.context['overall_message'])) + overall_message = self.clean_message_html(self.context['overall_message']) correct_map.set_overall_message(overall_message) for k in range(len(idset)): @@ -1195,12 +1195,24 @@ class SymbolicResponse(CustomResponse): """ response_tag = 'symbolicresponse' + max_inputfields = 1 + + def setup_response(self): + # Symbolic response always uses symmath_check() + # If the XML did not specify this, then set it now + # Otherwise, we get an error from the superclass + self.xml.set('cfn', 'symmath_check') + + # Let CustomResponse do its setup + super(SymbolicResponse, self).setup_response() def execute_check_function(self, idset, submission): from symmath import symmath_check - fn = self.code try: - answer_given = submission[0] if (len(idset) == 1) else submission + # Since we have limited max_inputfields to 1, + # we can assume that there is only one submission + answer_given = submission[0] + ret = symmath_check( self.expect, answer_given, dynamath=self.context.get('dynamath'), diff --git a/common/lib/capa/capa/tests/response_xml_factory.py b/common/lib/capa/capa/tests/response_xml_factory.py index ee1fdfafc9..ee9a7e6530 100644 --- a/common/lib/capa/capa/tests/response_xml_factory.py +++ b/common/lib/capa/capa/tests/response_xml_factory.py @@ -734,3 +734,38 @@ class AnnotationResponseXMLFactory(ResponseXMLFactory): option_element.text = description return input_element + + +class SymbolicResponseXMLFactory(ResponseXMLFactory): + """ Factory for producing xml """ + + def create_response_element(self, **kwargs): + """ Build the XML element. + + Uses **kwargs: + + *expect*: The correct answer (a sympy string) + + *options*: list of option strings to pass to symmath_check + (e.g. 'matrix', 'qbit', 'imaginary', 'numerical')""" + + # Retrieve **kwargs + expect = kwargs.get('expect', '') + options = kwargs.get('options', []) + + # Symmath check expects a string of options + options_str = ",".join(options) + + # Construct the element + response_element = etree.Element('symbolicresponse') + + if expect: + response_element.set('expect', str(expect)) + + if options_str: + response_element.set('options', str(options_str)) + + return response_element + + def create_input_element(self, **kwargs): + return ResponseXMLFactory.textline_input_xml(**kwargs) diff --git a/common/lib/capa/capa/tests/test_responsetypes.py b/common/lib/capa/capa/tests/test_responsetypes.py index c2ee62ed7a..b76854c744 100644 --- a/common/lib/capa/capa/tests/test_responsetypes.py +++ b/common/lib/capa/capa/tests/test_responsetypes.py @@ -10,6 +10,7 @@ import random import unittest import textwrap import mock +import textwrap from . import test_system @@ -184,107 +185,151 @@ class ImageResponseTest(ResponseTest): self.assert_answer_format(problem) -class SymbolicResponseTest(unittest.TestCase): +class SymbolicResponseTest(ResponseTest): from response_xml_factory import SymbolicResponseXMLFactory xml_factory_class = SymbolicResponseXMLFactory - def test_symbolic_response_grade(self): - symbolicresponse_file = os.path.dirname(__file__) + "/test_files/symbolicresponse.xml" - test_lcp = lcp.LoncapaProblem(open(symbolicresponse_file).read(), '1', system=test_system) - correct_answers = {'1_2_1': 'cos(theta)*[[1,0],[0,1]] + i*sin(theta)*[[0,1],[1,0]]', - '1_2_1_dynamath': ''' - - - - cos - - ( - θ - ) - - - ⋅ - - [ - - - - 1 - - - 0 - - - - - 0 - - - 1 - - - - ] - - + - i - ⋅ - - sin - - ( - θ - ) - - - ⋅ - - [ - - - - 0 - - - 1 - - - - - 1 - - - 0 - - - - ] - - - - ''', - } - wrong_answers = {'1_2_1': '2', - '1_2_1_dynamath': ''' - - - 2 - - ''', - } + def test_grade_single_input(self): + problem = self.build_problem(math_display=True, + expect="2*x+3*y") + # Correct answers + correct_inputs = [ + ('2x+3y', textwrap.dedent(""" + + + 2*x+3*y + """)), + + ('x+x+3y', textwrap.dedent(""" + + + x+x+3*y + """)), + ] + + for (input_str, input_mathml) in correct_inputs: + self._assert_symbolic_grade(problem, input_str, input_mathml, 'correct') + + # Incorrect answers + incorrect_inputs = [ + ('0', ''), + ('4x+3y', textwrap.dedent(""" + + + 4*x+3*y + """)), + ] + + for (input_str, input_mathml) in incorrect_inputs: + self._assert_symbolic_grade(problem, input_str, input_mathml, 'incorrect') + + + def test_complex_number_grade(self): + problem = self.build_problem(math_display=True, + expect="[[cos(theta),i*sin(theta)],[i*sin(theta),cos(theta)]]", + options=["matrix", "imaginary"]) + + # For LaTeX-style inputs, symmath_check() will try to contact + # a server to convert the input to MathML. + # We mock out the server, simulating the response that it would give + # for this input. import requests - d = os.path.dirname(__file__) - correct_snuggletex_response = open(os.path.join(d, "test_files/snuggletex_correct.html")).read().decode('utf8') - wrong_snuggletex_response = open(os.path.join(d, "test_files/snuggletex_wrong.html")).read().decode('utf8') + dirpath = os.path.dirname(__file__) + correct_snuggletex_response = open(os.path.join(dirpath, "test_files/snuggletex_correct.html")).read().decode('utf8') + wrong_snuggletex_response = open(os.path.join(dirpath, "test_files/snuggletex_wrong.html")).read().decode('utf8') + # Correct answer with mock.patch.object(requests, 'post') as mock_post: + + # Simulate what the LaTeX-to-MathML server would + # send for the correct response input mock_post.return_value.text = correct_snuggletex_response - self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct') + self._assert_symbolic_grade(problem, + "cos(theta)*[[1,0],[0,1]] + i*sin(theta)*[[0,1],[1,0]]", + textwrap.dedent(""" + + + + cos + (θ) + + ⋅ + + [ + + + 10 + + + 01 + + + ] + + + + i + ⋅ + + sin + + (θ) + + + ⋅ + + [ + + + 01 + + + 10 + + + ] + + + + """), + 'correct') + + # Incorrect answer with mock.patch.object(requests, 'post') as mock_post: + + # Simulate what the LaTeX-to-MathML server would + # send for the incorrect response input mock_post.return_value.text = wrong_snuggletex_response - self.assertEquals(test_lcp.grade_answers(wrong_answers).get_correctness('1_2_1'), 'incorrect') + + self._assert_symbolic_grade(problem, "2", + textwrap.dedent(""" + + 2 + + """), + 'incorrect') + + def test_multiple_inputs_exception(self): + + # Should not allow multiple inputs, since we specify + # only one "expect" value + with self.assertRaises(Exception): + problem = self.build_problem(math_display=True, + expect="2*x+3*y", + num_inputs=3) + + def _assert_symbolic_grade(self, problem, + student_input, + dynamath_input, + expected_correctness): + input_dict = {'1_2_1': str(student_input), + '1_2_1_dynamath': str(dynamath_input) } + + correct_map = problem.grade_answers(input_dict) + + self.assertEqual(correct_map.get_correctness('1_2_1'), + expected_correctness) class OptionResponseTest(ResponseTest): diff --git a/common/lib/capa/capa/util.py b/common/lib/capa/capa/util.py index 9f3e8bd3a0..cd694f1137 100644 --- a/common/lib/capa/capa/util.py +++ b/common/lib/capa/capa/util.py @@ -1,4 +1,4 @@ -from .calc import evaluator, UndefinedVariable +from calc import evaluator #----------------------------------------------------------------------------- # From be79810ff68bdeba2e0b4b79e6cb109e09e413e8 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 6 Mar 2013 10:54:09 -0500 Subject: [PATCH 064/162] Fix one problem from the merges --- common/lib/capa/capa/responsetypes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index ddf184c9be..b9ea3944c0 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -1041,9 +1041,8 @@ class CustomResponse(LoncapaResponse): # build map giving "correct"ness of the answer(s) correct = self.context['correct'] messages = self.context['messages'] - correct_map = CorrectMap() - overall_message = self.clean_message_html(self.context['overall_message']) + correct_map = CorrectMap() correct_map.set_overall_message(overall_message) for k in range(len(idset)): @@ -1110,6 +1109,7 @@ class CustomResponse(LoncapaResponse): if 'msg' in input_dict else None) messages.append(msg) self.context['messages'] = messages + self.context['overall_message'] = overall_message # Otherwise, we do not recognize the dictionary # Raise an exception From efaa0eea037d80f63fd811523dac6a8e8593a599 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 6 Mar 2013 11:15:02 -0500 Subject: [PATCH 065/162] More fixes to the merge, now all tests pass. --- common/lib/capa/capa/responsetypes.py | 10 +++++++++- common/lib/capa/capa/tests/test_responsetypes.py | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index b9ea3944c0..caefbe0fd0 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -1086,7 +1086,15 @@ class CustomResponse(LoncapaResponse): # to the same correct/incorrect value if 'ok' in ret: correct = ['correct' if ret['ok'] else 'incorrect'] * len(idset) - self.context['messages'][0] = self.clean_message_html(ret['msg']) + msg = ret.get('msg', None) + msg = self.clean_message_html(msg) + + # If there is only one input, apply the message to that input + # Otherwise, apply the message to the whole problem + if len(idset) > 1: + self.context['overall_message'] = msg + else: + self.context['messages'][0] = msg # Another kind of dictionary the check function can return has # the form: diff --git a/common/lib/capa/capa/tests/test_responsetypes.py b/common/lib/capa/capa/tests/test_responsetypes.py index b76854c744..3f88734884 100644 --- a/common/lib/capa/capa/tests/test_responsetypes.py +++ b/common/lib/capa/capa/tests/test_responsetypes.py @@ -977,7 +977,7 @@ class CustomResponseTest(ResponseTest): msg = correct_map.get_msg('1_2_1') self.assertEqual(correctness, 'correct') - self.assertEqual(msg, "Message text\n") + self.assertEqual(msg, "Message text") # Incorrect answer input_dict = {'1_2_1': '0'} @@ -987,7 +987,7 @@ class CustomResponseTest(ResponseTest): msg = correct_map.get_msg('1_2_1') self.assertEqual(correctness, 'incorrect') - self.assertEqual(msg, "Message text\n") + self.assertEqual(msg, "Message text") def test_multiple_inputs_return_one_status(self): # When given multiple inputs, the 'answer_given' argument From d925604113b78d1926cc5713d46d25610a17210e Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 26 Mar 2013 13:49:45 -0400 Subject: [PATCH 066/162] Clarify provenance --- common/lib/codejail/codejail/lazymod.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/lib/codejail/codejail/lazymod.py b/common/lib/codejail/codejail/lazymod.py index 936a8d263a..cdd8410f2c 100644 --- a/common/lib/codejail/codejail/lazymod.py +++ b/common/lib/codejail/codejail/lazymod.py @@ -1,6 +1,7 @@ """A module proxy for delayed importing of modules. -Lifted from http://barnesc.blogspot.com/2006/06/automatic-python-imports-with-autoimp.html +From http://barnesc.blogspot.com/2006/06/automatic-python-imports-with-autoimp.html, +in the public domain. """ From 89f6ef840710727a55b4e7129256484b6278431e Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 27 Mar 2013 17:24:56 -0400 Subject: [PATCH 067/162] Move capa/safe_exec into its own directory, in prep for moving code here. --- common/lib/capa/capa/safe_exec/__init__.py | 3 +++ common/lib/capa/capa/{ => safe_exec}/safe_exec.py | 0 .../capa/{ => safe_exec}/tests/test_files/pylib/constant.py | 0 common/lib/capa/capa/{ => safe_exec}/tests/test_safe_exec.py | 0 4 files changed, 3 insertions(+) create mode 100644 common/lib/capa/capa/safe_exec/__init__.py rename common/lib/capa/capa/{ => safe_exec}/safe_exec.py (100%) rename common/lib/capa/capa/{ => safe_exec}/tests/test_files/pylib/constant.py (100%) rename common/lib/capa/capa/{ => safe_exec}/tests/test_safe_exec.py (100%) diff --git a/common/lib/capa/capa/safe_exec/__init__.py b/common/lib/capa/capa/safe_exec/__init__.py new file mode 100644 index 0000000000..bbfea1c138 --- /dev/null +++ b/common/lib/capa/capa/safe_exec/__init__.py @@ -0,0 +1,3 @@ +"""Capa's specialized use of codejail.safe_exec.""" + +from .safe_exec import safe_exec diff --git a/common/lib/capa/capa/safe_exec.py b/common/lib/capa/capa/safe_exec/safe_exec.py similarity index 100% rename from common/lib/capa/capa/safe_exec.py rename to common/lib/capa/capa/safe_exec/safe_exec.py diff --git a/common/lib/capa/capa/tests/test_files/pylib/constant.py b/common/lib/capa/capa/safe_exec/tests/test_files/pylib/constant.py similarity index 100% rename from common/lib/capa/capa/tests/test_files/pylib/constant.py rename to common/lib/capa/capa/safe_exec/tests/test_files/pylib/constant.py diff --git a/common/lib/capa/capa/tests/test_safe_exec.py b/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py similarity index 100% rename from common/lib/capa/capa/tests/test_safe_exec.py rename to common/lib/capa/capa/safe_exec/tests/test_safe_exec.py From 0021b0acb32ccdeaf77cd954f6475346e808ec6f Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 28 Mar 2013 10:03:33 -0400 Subject: [PATCH 068/162] Refactor to move assumed_imports into capa, so that code_jail is more pure. --- .../capa/safe_exec}/lazymod.py | 0 common/lib/capa/capa/safe_exec/safe_exec.py | 41 ++++++++++++----- .../capa/capa/safe_exec/tests/test_lazymod.py | 44 +++++++++++++++++++ common/lib/codejail/codejail/safe_exec.py | 37 ++-------------- .../codejail/codejail/tests/test_lazymod.py | 26 ----------- .../codejail/codejail/tests/test_safe_exec.py | 13 ------ common/lib/codejail/codejail/util.py | 19 -------- 7 files changed, 76 insertions(+), 104 deletions(-) rename common/lib/{codejail/codejail => capa/capa/safe_exec}/lazymod.py (100%) create mode 100644 common/lib/capa/capa/safe_exec/tests/test_lazymod.py delete mode 100644 common/lib/codejail/codejail/tests/test_lazymod.py diff --git a/common/lib/codejail/codejail/lazymod.py b/common/lib/capa/capa/safe_exec/lazymod.py similarity index 100% rename from common/lib/codejail/codejail/lazymod.py rename to common/lib/capa/capa/safe_exec/lazymod.py diff --git a/common/lib/capa/capa/safe_exec/safe_exec.py b/common/lib/capa/capa/safe_exec/safe_exec.py index 3d481495d4..3ae9567fa9 100644 --- a/common/lib/capa/capa/safe_exec/safe_exec.py +++ b/common/lib/capa/capa/safe_exec/safe_exec.py @@ -1,6 +1,7 @@ """Capa's specialized use of codejail.safe_exec.""" import codejail.safe_exec +from . import lazymod # Establish the Python environment for Capa. # Capa assumes float-friendly division always. @@ -16,23 +17,39 @@ del random_module sys.modules['random'] = random """ +ASSUMED_IMPORTS=[ + ("numpy", "numpy"), + ("math", "math"), + ("scipy", "scipy"), + ("calc", "calc"), + ("eia", "eia"), + ("chemcalc", "chem.chemcalc"), + ("chemtools", "chem.chemtools"), + ("miller", "chem.miller"), + ("draganddrop", "verifiers.draganddrop"), +] + +# We'll need the code from lazymod.py for use in jailpy, so read it now. +lazymod_py_file = lazymod.__file__ +if lazymod_py_file.endswith("c"): + lazymod_py_file = lazymod_py_file[:-1] + +lazymod_py = open(lazymod_py_file).read() + +LAZY_IMPORTS = [lazymod_py] +for name, modname in ASSUMED_IMPORTS: + LAZY_IMPORTS.append("{} = LazyModule('{}')\n".format(name, modname)) + +LAZY_IMPORTS = "".join(LAZY_IMPORTS) + + def safe_exec(code, globals_dict, random_seed=None, python_path=None): """Exec python code safely. """ code_prolog = CODE_PROLOG % random_seed + codejail.safe_exec.safe_exec( - code_prolog + code, globals_dict, + code_prolog + LAZY_IMPORTS + code, globals_dict, python_path=python_path, - assumed_imports=[ - "numpy", - "math", - "scipy", - "calc", - "eia", - ("chemcalc", "chem.chemcalc"), - ("chemtools", "chem.chemtools"), - ("miller", "chem.miller"), - ("draganddrop", "verifiers.draganddrop"), - ], ) diff --git a/common/lib/capa/capa/safe_exec/tests/test_lazymod.py b/common/lib/capa/capa/safe_exec/tests/test_lazymod.py new file mode 100644 index 0000000000..68dcd81ea7 --- /dev/null +++ b/common/lib/capa/capa/safe_exec/tests/test_lazymod.py @@ -0,0 +1,44 @@ +"""Test lazymod.py""" + +import sys +import unittest + +from capa.safe_exec.lazymod import LazyModule + + +class ModuleIsolation(object): + """ + Manage changes to sys.modules so that we can roll back imported modules. + + Create this object, it will snapshot the currently imported modules. When + you call `clean_up()`, it will delete any module imported since its creation. + """ + def __init__(self): + # Save all the names of all the imported modules. + self.mods = set(sys.modules) + + def clean_up(self): + # Get a list of modules that didn't exist when we were created + new_mods = [m for m in sys.modules if m not in self.mods] + # and delete them all so another import will run code for real again. + for m in new_mods: + del sys.modules[m] + + +class TestLazyMod(unittest.TestCase): + + def setUp(self): + # Each test will remove modules that it imported. + self.addCleanup(ModuleIsolation().clean_up) + + def test_simple(self): + # Import some stdlib module that has not been imported before + self.assertNotIn("colorsys", sys.modules) + colorsys = LazyModule("colorsys") + hsv = colorsys.rgb_to_hsv(.3, .4, .2) + self.assertEqual(hsv[0], 0.25) + + def test_dotted(self): + self.assertNotIn("email.utils", sys.modules) + email_utils = LazyModule("email.utils") + self.assertEqual(email_utils.quote('"hi"'), r'\"hi\"') diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index b11ef60616..07b1ad378c 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -6,40 +6,17 @@ import shutil import sys import textwrap -import lazymod import jailpy -from util import temp_directory, change_directory, TempDirectory +from util import temp_directory, change_directory -# We'll need the code from lazymod.py for use in jailpy, so read it now. -lazymod_py_file = lazymod.__file__ -if lazymod_py_file.endswith("c"): - lazymod_py_file = lazymod_py_file[:-1] - -lazymod_py = open(lazymod_py_file).read() - - -def names_and_modules(assumed_imports): - """Get uniform names and modules from assumed_imports.""" - for modname in assumed_imports: - if isinstance(modname, tuple): - yield modname - else: - yield modname, modname - - -def safe_exec(code, globals_dict, assumed_imports=None, files=None, python_path=None): +def safe_exec(code, globals_dict, files=None, python_path=None): """Execute code as "exec" does, but safely. `code` is a string of Python code. `globals_dict` is used as the globals during execution. Modifications the code makes to `globals_dict` are reflected in the dictionary on return. - `assumed_imports` is a list of modules to make available as implicit - imports for the code. Entries are either a name, "mod", which makes - "import mod" part of the code, or a pair, ("f", "fooey"), which makes - "import fooey as f" part of the code. The module name can be dotted. - Returns None. Changes made by `code` are visible in `globals_dict`. """ @@ -72,11 +49,6 @@ def safe_exec(code, globals_dict, assumed_imports=None, files=None, python_path= the_code.append("sys.path.append(%r)\n" % pybase) files.append(pydir) - if assumed_imports: - the_code.append(lazymod_py) - for name, modname in names_and_modules(assumed_imports): - the_code.append("g_dict['{}'] = LazyModule('{}')\n".format(name, modname)) - the_code.append(textwrap.dedent( # Execute the sandboxed code. """ @@ -140,7 +112,7 @@ def json_safe(d): return json.loads(json.dumps(jd)) -def not_safe_exec(code, globals_dict, assumed_imports=None, files=None, python_path=None): +def not_safe_exec(code, globals_dict, files=None, python_path=None): """Another implementation of `safe_exec`, but not safe. This can be swapped in for debugging problems in sandboxed Python code. @@ -151,9 +123,6 @@ def not_safe_exec(code, globals_dict, assumed_imports=None, files=None, python_p """ g_dict = json_safe(globals_dict) - for name, modname in names_and_modules(assumed_imports or ()): - g_dict[name] = lazymod.LazyModule(modname) - with temp_directory(delete_when_done=True) as tmpdir: with change_directory(tmpdir): # Copy the files here. diff --git a/common/lib/codejail/codejail/tests/test_lazymod.py b/common/lib/codejail/codejail/tests/test_lazymod.py deleted file mode 100644 index eb853060d0..0000000000 --- a/common/lib/codejail/codejail/tests/test_lazymod.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Test lazymod.py""" - -import sys -import unittest - -from codejail.lazymod import LazyModule -from codejail.util import ModuleIsolation - - -class TestLazyMod(unittest.TestCase): - - def setUp(self): - # Each test will remove modules that it imported. - self.addCleanup(ModuleIsolation().clean_up) - - def test_simple(self): - # Import some stdlib module that has not been imported before - self.assertNotIn("colorsys", sys.modules) - colorsys = LazyModule("colorsys") - hsv = colorsys.rgb_to_hsv(.3, .4, .2) - self.assertEqual(hsv[0], 0.25) - - def test_dotted(self): - self.assertNotIn("email.utils", sys.modules) - email_utils = LazyModule("email.utils") - self.assertEqual(email_utils.quote('"hi"'), r'\"hi\"') diff --git a/common/lib/codejail/codejail/tests/test_safe_exec.py b/common/lib/codejail/codejail/tests/test_safe_exec.py index 8a565f8cfb..05a27020b3 100644 --- a/common/lib/codejail/codejail/tests/test_safe_exec.py +++ b/common/lib/codejail/codejail/tests/test_safe_exec.py @@ -14,19 +14,6 @@ class SafeExecTests(object): self.safe_exec("a = 17", g) self.assertEqual(g['a'], 17) - def test_assumed_imports(self): - g = {} - # Using string without importing it is bad. - with self.assertRaises(Exception): - self.safe_exec("a = string.ascii_lowercase[0]", g) - # Using string with an assumed import is fine. - self.safe_exec("a = string.ascii_lowercase[0]", g, assumed_imports=["string"]) - self.assertEqual(g['a'], 'a') - # Can also import with a shorthand. - self.safe_exec("a = op.join('x', 'y')", g, assumed_imports=[("op", "os.path")]) - self.assertEqual(g['a'][0], 'x') - self.assertEqual(g['a'][-1], 'y') - def test_files_are_copied(self): g = {} self.safe_exec( diff --git a/common/lib/codejail/codejail/util.py b/common/lib/codejail/codejail/util.py index e293ce052f..ce41f9d5d4 100644 --- a/common/lib/codejail/codejail/util.py +++ b/common/lib/codejail/codejail/util.py @@ -32,25 +32,6 @@ def temp_directory(delete_when_done=True): tmp.clean_up() -class ModuleIsolation(object): - """ - Manage changes to sys.modules so that we can roll back imported modules. - - Create this object, it will snapshot the currently imported modules. When - you call `clean_up()`, it will delete any module imported since its creation. - """ - def __init__(self): - # Save all the names of all the imported modules. - self.mods = set(sys.modules) - - def clean_up(self): - # Get a list of modules that didn't exist when we were created - new_mods = [m for m in sys.modules if m not in self.mods] - # and delete them all so another import will run code for real again. - for m in new_mods: - del sys.modules[m] - - class ChangeDirectory(object): def __init__(self, new_dir): self.old_dir = os.getcwd() From 182a1a189938234a92b1bbb07c7a92a4b181c3ba Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 28 Mar 2013 14:47:12 -0400 Subject: [PATCH 069/162] Cleanups --- common/lib/codejail/codejail/django_integration.py | 1 + common/lib/codejail/codejail/jailpy.py | 11 ++++++++--- common/lib/codejail/codejail/safe_exec.py | 6 +++--- common/lib/codejail/codejail/tests/test_jailpy.py | 2 ++ common/lib/codejail/codejail/tests/test_safe_exec.py | 2 ++ common/lib/codejail/codejail/util.py | 3 ++- 6 files changed, 18 insertions(+), 7 deletions(-) diff --git a/common/lib/codejail/codejail/django_integration.py b/common/lib/codejail/codejail/django_integration.py index 52306ba6f1..b720ffcff1 100644 --- a/common/lib/codejail/codejail/django_integration.py +++ b/common/lib/codejail/codejail/django_integration.py @@ -5,6 +5,7 @@ from django.conf import settings import codejail.jailpy + class ConfigureCodeJailMiddleware(object): """Middleware to configure codejail on startup.""" diff --git a/common/lib/codejail/codejail/jailpy.py b/common/lib/codejail/codejail/jailpy.py index c3e98a518a..df1bd8df12 100644 --- a/common/lib/codejail/codejail/jailpy.py +++ b/common/lib/codejail/codejail/jailpy.py @@ -4,7 +4,8 @@ # - AppArmor.md from xserver import logging -import os, os.path +import os +import os.path import resource import shutil import subprocess @@ -22,6 +23,7 @@ log = logging.getLogger(__name__) PYTHON_CMD = None + def configure(python_bin, user=None): """Configure the jailpy module.""" global PYTHON_CMD @@ -30,6 +32,7 @@ def configure(python_bin, user=None): PYTHON_CMD.extend(['sudo', '-u', 'sandbox']) PYTHON_CMD.extend([python_bin, '-E']) + def is_configured(): return bool(PYTHON_CMD) @@ -42,7 +45,9 @@ if hasattr(sys, 'real_prefix'): class JailResult(object): """A passive object for us to return from jailpy.""" - pass + def __init__(self): + self.stdout = self.stderr = self.status = None + def jailpy(code, files=None, argv=None, stdin=None): """ @@ -104,7 +109,7 @@ def set_process_limits(): resource.setrlimit(resource.RLIMIT_NPROC, (0, 0)) # no subprocesses resource.setrlimit(resource.RLIMIT_FSIZE, (0, 0)) # no files - mem = 32 * 2**20 # 32 MB should be enough for anyone, right? :) + mem = 32 * (2 ** 20) # 32 MB should be enough for anyone, right? :) resource.setrlimit(resource.RLIMIT_STACK, (mem, mem)) resource.setrlimit(resource.RLIMIT_RSS, (mem, mem)) resource.setrlimit(resource.RLIMIT_DATA, (mem, mem)) diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index 07b1ad378c..5db9651438 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -6,9 +6,9 @@ import shutil import sys import textwrap -import jailpy +from codejail import jailpy +from codejail.util import temp_directory, change_directory -from util import temp_directory, change_directory def safe_exec(code, globals_dict, files=None, python_path=None): """Execute code as "exec" does, but safely. @@ -98,7 +98,7 @@ def json_safe(d): ok_types = (type(None), int, long, float, str, unicode, list, tuple, dict) bad_keys = ("__builtins__",) jd = {} - for k,v in d.iteritems(): + for k, v in d.iteritems(): if not isinstance(v, ok_types): continue if k in bad_keys: diff --git a/common/lib/codejail/codejail/tests/test_jailpy.py b/common/lib/codejail/codejail/tests/test_jailpy.py index 15c548663b..c0d51ba684 100644 --- a/common/lib/codejail/codejail/tests/test_jailpy.py +++ b/common/lib/codejail/codejail/tests/test_jailpy.py @@ -9,6 +9,7 @@ from codejail.jailpy import jailpy, is_configured dedent = textwrap.dedent + class JailPyHelpers(object): """Assert helpers for jailpy tests.""" def setUp(self): @@ -112,6 +113,7 @@ class TestLimits(JailPyHelpers, unittest.TestCase): # TODO: read network # TODO: fork + class TestMalware(JailPyHelpers, unittest.TestCase): def test_crash_cpython(self): # http://nedbatchelder.com/blog/201206/eval_really_is_dangerous.html diff --git a/common/lib/codejail/codejail/tests/test_safe_exec.py b/common/lib/codejail/codejail/tests/test_safe_exec.py index 05a27020b3..46d6a59a98 100644 --- a/common/lib/codejail/codejail/tests/test_safe_exec.py +++ b/common/lib/codejail/codejail/tests/test_safe_exec.py @@ -7,6 +7,7 @@ from nose.plugins.skip import SkipTest from codejail.safe_exec import safe_exec, not_safe_exec + class SafeExecTests(object): """The tests for `safe_exec`, will be mixed into specific test classes below.""" def test_set_values(self): @@ -60,6 +61,7 @@ class TestSafeExec(SafeExecTests, unittest.TestCase): def safe_exec(self, *args, **kwargs): safe_exec(*args, **kwargs) + class TestNotSafeExec(SafeExecTests, unittest.TestCase): """Run SafeExecTests, with not_safe_exec.""" def setUp(self): diff --git a/common/lib/codejail/codejail/util.py b/common/lib/codejail/codejail/util.py index ce41f9d5d4..d88a05b4dd 100644 --- a/common/lib/codejail/codejail/util.py +++ b/common/lib/codejail/codejail/util.py @@ -3,7 +3,6 @@ import contextlib import os import shutil -import sys import tempfile @@ -19,6 +18,7 @@ class TempDirectory(object): # if this errors, something is genuinely wrong, so don't ignore errors. shutil.rmtree(self.temp_dir) + @contextlib.contextmanager def temp_directory(delete_when_done=True): """ @@ -40,6 +40,7 @@ class ChangeDirectory(object): def clean_up(self): os.chdir(self.old_dir) + @contextlib.contextmanager def change_directory(new_dir): """ From ceb6cedaaee1a59bd4f0e4bbd3d154e8a69b8a19 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 29 Mar 2013 16:41:01 -0400 Subject: [PATCH 070/162] Fix merge --- lms/djangoapps/courseware/tests/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/courseware/tests/tests.py b/lms/djangoapps/courseware/tests/tests.py index 2d3db03074..df141fe284 100644 --- a/lms/djangoapps/courseware/tests/tests.py +++ b/lms/djangoapps/courseware/tests/tests.py @@ -897,7 +897,7 @@ class TestCourseGrader(TestSubmittingProblems): progress_summary = grades.progress_summary(self.student_user, fake_request, - self.graded_course, + self.course, model_data_cache) return progress_summary From 5e8e31b2d14622033a214406ad0f40dde5a5d176 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 1 Apr 2013 12:34:11 -0400 Subject: [PATCH 071/162] Add a cache attribute to ModuleSystem --- common/lib/xmodule/xmodule/x_module.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index 1fd0b8e138..31a32eb6dc 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -700,7 +700,9 @@ class ModuleSystem(object): anonymous_student_id='', course_id=None, open_ended_grading_interface=None, - s3_interface=None): + s3_interface=None, + cache=None, + ): ''' Create a closure around the system environment. @@ -742,6 +744,11 @@ class ModuleSystem(object): xblock_model_data - A dict-like object containing the all data available to this xblock + + cache - A cache object with two methods: + .get(key) returns an object from the cache or None. + .set(key, value, timeout_secs=None) stores a value in the cache with a timeout. + ''' self.ajax_url = ajax_url self.xqueue = xqueue @@ -766,6 +773,8 @@ class ModuleSystem(object): self.open_ended_grading_interface = open_ended_grading_interface self.s3_interface = s3_interface + self.cache = cache or DoNothingCache() + def get(self, attr): ''' provide uniform access to attributes (like etree).''' return self.__dict__.get(attr) @@ -779,3 +788,12 @@ class ModuleSystem(object): def __str__(self): return str(self.__dict__) + + +class DoNothingCache(object): + """A duck-compatible object to use in ModuleSystem when there's no cache.""" + def get(self, key): + return None + + def set(self, key, value, timeout=None): + pass From c8b908a24461ce7603cb9e57a6d401503e675fc9 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 1 Apr 2013 13:54:25 -0400 Subject: [PATCH 072/162] capa.safe_exec can use a cache. --- common/lib/capa/capa/safe_exec/safe_exec.py | 26 ++++++++++++-- .../capa/safe_exec/tests/test_safe_exec.py | 34 +++++++++++++++++++ 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/common/lib/capa/capa/safe_exec/safe_exec.py b/common/lib/capa/capa/safe_exec/safe_exec.py index 3ae9567fa9..da74f3aaf5 100644 --- a/common/lib/capa/capa/safe_exec/safe_exec.py +++ b/common/lib/capa/capa/safe_exec/safe_exec.py @@ -1,6 +1,7 @@ """Capa's specialized use of codejail.safe_exec.""" -import codejail.safe_exec +from codejail.safe_exec import safe_exec as codejail_safe_exec +from codejail.safe_exec import json_safe from . import lazymod # Establish the Python environment for Capa. @@ -43,13 +44,32 @@ for name, modname in ASSUMED_IMPORTS: LAZY_IMPORTS = "".join(LAZY_IMPORTS) -def safe_exec(code, globals_dict, random_seed=None, python_path=None): +def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None): """Exec python code safely. + `cache` is an object with .get(key) and .set(key, value) methods. + """ + # Check the cache for a previous result. + if cache: + canonical_globals = sorted(json_safe(globals_dict).iteritems()) + key = "safe_exec %r %s %r" % (random_seed, code, canonical_globals) + cached = cache.get(key) + if cached is not None: + globals_dict.update(cached) + return + + # Create the complete code we'll run. code_prolog = CODE_PROLOG % random_seed - codejail.safe_exec.safe_exec( + # Run the code! Results are side effects in globals_dict. + codejail_safe_exec( code_prolog + LAZY_IMPORTS + code, globals_dict, python_path=python_path, ) + + # Put the result back in the cache. This is complicated by the fact that + # the globals dict might not be entirely serializable. + if cache: + cleaned_results = json_safe(globals_dict) + cache.set(key, cleaned_results) diff --git a/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py b/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py index 7ed44a69a1..37f86383c2 100644 --- a/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py +++ b/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py @@ -56,3 +56,37 @@ class TestSafeExec(unittest.TestCase): "import constant; a = constant.THE_CONST", g, python_path=[pylib] ) + + +class DictCache(object): + """A cache implementation over a simple dict, for testing.""" + + def __init__(self, d): + self.cache = d + + def get(self, key): + return self.cache.get(key) + + def set(self, key, value): + self.cache[key] = value + + +class TestSafeExecCaching(unittest.TestCase): + """Test that caching works on safe_exec.""" + + def test_cache_miss_then_hit(self): + g = {} + cache = {} + + # Cache miss + safe_exec("a = int(math.pi)", g, cache=DictCache(cache)) + self.assertEqual(g['a'], 3) + # A result has been cached + self.assertEqual(cache.values(), [{'a': 3}]) + + # Fiddle with the cache, then try it again. + cache[cache.keys()[0]] = {'a': 17} + + g = {} + safe_exec("a = int(math.pi)", g, cache=DictCache(cache)) + self.assertEqual(g['a'], 17) From 5e7d328e7f2581247fc86d7617fa6fdbbeefb3db Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 1 Apr 2013 16:33:59 -0400 Subject: [PATCH 073/162] Use the Django cache for sandboxed code execution. --- common/lib/capa/capa/capa_problem.py | 8 +++++++- common/lib/capa/capa/responsetypes.py | 6 +++--- common/lib/capa/capa/tests/__init__.py | 3 ++- lms/djangoapps/courseware/module_render.py | 2 ++ 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index b5e773ebc7..45200b8607 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -462,7 +462,13 @@ class LoncapaProblem(object): if all_code: try: - safe_exec.safe_exec(all_code, context, random_seed=self.seed, python_path=python_path) + safe_exec.safe_exec( + all_code, + context, + random_seed=self.seed, + python_path=python_path, + cache=self.system.cache, + ) except Exception as err: log.exception("Error while execing script code: " + all_code) msg = "Error while executing script code: %s" % str(err).replace('<', '<') diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index caefbe0fd0..829a554b4d 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -938,7 +938,7 @@ class CustomResponse(LoncapaResponse): 'ans': ans, } globals_dict.update(kwargs) - safe_exec.safe_exec(code, globals_dict) + safe_exec.safe_exec(code, globals_dict, cache=self.system.cache) return globals_dict['cfn_return'] return check_function @@ -1055,7 +1055,7 @@ class CustomResponse(LoncapaResponse): # exec the check function if isinstance(self.code, basestring): try: - safe_exec.safe_exec(self.code, self.context) + safe_exec.safe_exec(self.code, self.context, cache=self.system.cache) except Exception as err: self._handle_exec_exception(err) @@ -1783,7 +1783,7 @@ class SchematicResponse(LoncapaResponse): json.loads(student_answers[k]) for k in sorted(self.answer_ids) ] self.context.update({'submission': submission}) - safe_exec.safe_exec(self.code, self.context) + safe_exec.safe_exec(self.code, self.context, cache=self.system.cache) cmap = CorrectMap() cmap.set_dict(dict(zip(sorted(self.answer_ids), self.context['correct']))) return cmap diff --git a/common/lib/capa/capa/tests/__init__.py b/common/lib/capa/capa/tests/__init__.py index 72d82c683b..59c87780b5 100644 --- a/common/lib/capa/capa/tests/__init__.py +++ b/common/lib/capa/capa/tests/__init__.py @@ -33,5 +33,6 @@ test_system = Mock( debug=True, xqueue={'interface': xqueue_interface, 'construct_callback': calledback_url, 'default_queuename': 'testqueue', 'waittime': 10}, node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules"), - anonymous_student_id='student' + anonymous_student_id='student', + cache=None, ) diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 6f05b32778..dc71216adb 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -8,6 +8,7 @@ from functools import partial from django.conf import settings from django.contrib.auth.models import User +from django.core.cache import cache from django.core.exceptions import PermissionDenied from django.core.urlresolvers import reverse from django.http import Http404 @@ -299,6 +300,7 @@ def get_module_for_descriptor(user, request, descriptor, model_data_cache, cours course_id=course_id, open_ended_grading_interface=open_ended_grading_interface, s3_interface=s3_interface, + cache=cache, ) # pass position specified in URL to module through ModuleSystem system.set('position', position) From ed13f0a0f164d67154b7ad5493008416abdbec5f Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 3 Apr 2013 09:31:57 -0400 Subject: [PATCH 074/162] Catch up to new exception handling in responses. --- common/lib/capa/capa/responsetypes.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 829a554b4d..345716a236 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -1071,10 +1071,7 @@ class CustomResponse(LoncapaResponse): try: ret = fn(self.expect, answer_given, **kwargs) except Exception as err: - log.error("oops in customresponse (cfn) error %s" % err) - # print "context = ",self.context - log.error(traceback.format_exc()) - raise Exception("oops in customresponse (cfn) error %s" % err) + self._handle_exec_exception(err) log.debug( "[courseware.capa.responsetypes.customresponse.get_score] ret = %s", ret @@ -1783,7 +1780,11 @@ class SchematicResponse(LoncapaResponse): json.loads(student_answers[k]) for k in sorted(self.answer_ids) ] self.context.update({'submission': submission}) - safe_exec.safe_exec(self.code, self.context, cache=self.system.cache) + try: + safe_exec.safe_exec(self.code, self.context, cache=self.system.cache) + except Exception as err: + msg = 'Error %s in evaluating SchematicResponse' % err + raise ResponseError(msg) cmap = CorrectMap() cmap.set_dict(dict(zip(sorted(self.answer_ids), self.context['correct']))) return cmap From bcdc11c3a5271ef1dde7ab2297dae2f6bb2de775 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 5 Apr 2013 16:14:11 -0400 Subject: [PATCH 075/162] Hint functions are now run in the sandbox. --- common/lib/capa/capa/responsetypes.py | 42 +++++++++++++++---- .../capa/capa/tests/response_xml_factory.py | 30 ++++++++----- .../lib/capa/capa/tests/test_responsetypes.py | 16 +++++++ common/lib/codejail/codejail/safe_exec.py | 10 +++-- 4 files changed, 76 insertions(+), 22 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 345716a236..de73dcda30 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -22,6 +22,7 @@ import random import re import requests import subprocess +import textwrap import traceback import xml.sax.saxutils as saxutils @@ -30,7 +31,7 @@ from shapely.geometry import Point, MultiPoint # specific library imports from calc import evaluator, UndefinedVariable -from .correctmap import CorrectMap +from . import correctmap from datetime import datetime from .util import * from lxml import etree @@ -42,6 +43,10 @@ import safe_exec log = logging.getLogger(__name__) +CorrectMap = correctmap.CorrectMap +CORRECTMAP_PY = None + + #----------------------------------------------------------------------------- # Exceptions @@ -253,20 +258,41 @@ class LoncapaResponse(object): # We may extend this in the future to add another argument which provides a # callback procedure to a social hint generation system. - if not hintfn in self.context: - msg = 'missing specified hint function %s in script context' % hintfn - msg += "\nSee XML source line %s" % getattr( - self.xml, 'sourceline', '') - raise LoncapaProblemError(msg) + + global CORRECTMAP_PY + if CORRECTMAP_PY is None: + # We need the CorrectMap code for hint functions. No, this is not great. + CORRECTMAP_PY = inspect.getsource(correctmap) + + code = ( + CORRECTMAP_PY + "\n" + + self.context['script_code'] + "\n" + + textwrap.dedent(""" + new_cmap = CorrectMap() + new_cmap.set_dict(new_cmap_dict) + old_cmap = CorrectMap() + old_cmap.set_dict(old_cmap_dict) + {hintfn}(answer_ids, student_answers, new_cmap, old_cmap) + new_cmap_dict.update(new_cmap.get_dict()) + old_cmap_dict.update(old_cmap.get_dict()) + """).format(hintfn=hintfn) + ) + globals_dict = { + 'answer_ids': self.answer_ids, + 'student_answers': student_answers, + 'new_cmap_dict': new_cmap.get_dict(), + 'old_cmap_dict': old_cmap.get_dict(), + } try: - self.context[hintfn]( - self.answer_ids, student_answers, new_cmap, old_cmap) + safe_exec.safe_exec(code, globals_dict) except Exception as err: msg = 'Error %s in evaluating hint function %s' % (err, hintfn) msg += "\nSee XML source line %s" % getattr( self.xml, 'sourceline', '') raise ResponseError(msg) + + new_cmap.set_dict(globals_dict['new_cmap_dict']) return # hint specified by conditions and text dependent on conditions (a-la Loncapa design) diff --git a/common/lib/capa/capa/tests/response_xml_factory.py b/common/lib/capa/capa/tests/response_xml_factory.py index ee9a7e6530..35c12800ae 100644 --- a/common/lib/capa/capa/tests/response_xml_factory.py +++ b/common/lib/capa/capa/tests/response_xml_factory.py @@ -667,12 +667,16 @@ class StringResponseXMLFactory(ResponseXMLFactory): Where *hint_prompt* is the string for which we show the hint, *hint_name* is an internal identifier for the hint, and *hint_text* is the text we show for the hint. + + *hintfn*: The name of a function in the script to use for hints. + """ # Retrieve the **kwargs answer = kwargs.get("answer", None) case_sensitive = kwargs.get("case_sensitive", True) hint_list = kwargs.get('hints', None) - assert(answer) + hint_fn = kwargs.get('hintfn', None) + assert answer # Create the element response_element = etree.Element("stringresponse") @@ -684,18 +688,24 @@ class StringResponseXMLFactory(ResponseXMLFactory): response_element.set("type", "cs" if case_sensitive else "ci") # Add the hints if specified - if hint_list: + if hint_list or hint_fn: hintgroup_element = etree.SubElement(response_element, "hintgroup") - for (hint_prompt, hint_name, hint_text) in hint_list: - stringhint_element = etree.SubElement(hintgroup_element, "stringhint") - stringhint_element.set("answer", str(hint_prompt)) - stringhint_element.set("name", str(hint_name)) + if hint_list: + assert not hint_fn + for (hint_prompt, hint_name, hint_text) in hint_list: + stringhint_element = etree.SubElement(hintgroup_element, "stringhint") + stringhint_element.set("answer", str(hint_prompt)) + stringhint_element.set("name", str(hint_name)) - hintpart_element = etree.SubElement(hintgroup_element, "hintpart") - hintpart_element.set("on", str(hint_name)) + hintpart_element = etree.SubElement(hintgroup_element, "hintpart") + hintpart_element.set("on", str(hint_name)) - hint_text_element = etree.SubElement(hintpart_element, "text") - hint_text_element.text = str(hint_text) + hint_text_element = etree.SubElement(hintpart_element, "text") + hint_text_element.text = str(hint_text) + + if hint_fn: + assert not hint_list + hintgroup_element.set("hintfn", hint_fn) return response_element diff --git a/common/lib/capa/capa/tests/test_responsetypes.py b/common/lib/capa/capa/tests/test_responsetypes.py index 3f88734884..8e41ff39de 100644 --- a/common/lib/capa/capa/tests/test_responsetypes.py +++ b/common/lib/capa/capa/tests/test_responsetypes.py @@ -552,6 +552,22 @@ class StringResponseTest(ResponseTest): correct_map = problem.grade_answers(input_dict) self.assertEquals(correct_map.get_hint('1_2_1'), "") + def test_computed_hints(self): + problem = self.build_problem( + answer="Michigan", + hintfn="gimme_a_hint", + script = textwrap.dedent(""" + def gimme_a_hint(answer_ids, student_answers, new_cmap, old_cmap): + aid = answer_ids[0] + answer = student_answers[aid] + new_cmap.set_hint_and_mode(aid, answer+"??", "always") + """) + ) + + input_dict = {'1_2_1': 'Hello'} + correct_map = problem.grade_answers(input_dict) + self.assertEquals(correct_map.get_hint('1_2_1'), "Hello??") + class CodeResponseTest(ResponseTest): from response_xml_factory import CodeResponseXMLFactory diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index 5db9651438..682ae809af 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -1,6 +1,7 @@ """Safe execution of untrusted Python code.""" import json +import logging import os.path import shutil import sys @@ -9,6 +10,8 @@ import textwrap from codejail import jailpy from codejail.util import temp_directory, change_directory +log = logging.getLogger(__name__) + def safe_exec(code, globals_dict, files=None, python_path=None): """Execute code as "exec" does, but safely. @@ -78,10 +81,9 @@ def safe_exec(code, globals_dict, files=None, python_path=None): # Turn this on to see what's being executed. if 0: - print "--{:-<40}".format(" jailed ") - print jailed_code - print "--{:-<40}".format(" exec ") - print code + log.debug("Jailed code: %s", jailed_code) + log.debug("Exec: %s", code) + log.debug("Stdin: %s", stdin) res = jailpy.jailpy(jailed_code, stdin=stdin, files=files) if res.status != 0: From 55e910aafc04a4be704f933e72fbff14b895b3ea Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 9 Apr 2013 14:15:50 -0400 Subject: [PATCH 076/162] Not sure why my branch was ahead of master for the version of distribute. Make them the same. --- common/lib/capa/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/lib/capa/setup.py b/common/lib/capa/setup.py index 5f1731fdc0..d0a174a15e 100644 --- a/common/lib/capa/setup.py +++ b/common/lib/capa/setup.py @@ -4,5 +4,5 @@ setup( name="capa", version="0.1", packages=find_packages(exclude=["tests"]), - install_requires=["distribute==0.6.34"], + install_requires=["distribute==0.6.30"], ) From bde976dad21757451398ddeba4e42a9a2b2f053f Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 9 Apr 2013 17:07:57 -0400 Subject: [PATCH 077/162] Refactor code_jail to accommodate non-Python code. --- common/lib/capa/capa/safe_exec/safe_exec.py | 2 +- .../codejail/codejail/django_integration.py | 4 +- .../codejail/{jailpy.py => jail_code.py} | 46 +++++++++++-------- common/lib/codejail/codejail/safe_exec.py | 6 +-- .../codejail/codejail/tests/test_jailpy.py | 22 +++++---- 5 files changed, 47 insertions(+), 33 deletions(-) rename common/lib/codejail/codejail/{jailpy.py => jail_code.py} (74%) diff --git a/common/lib/capa/capa/safe_exec/safe_exec.py b/common/lib/capa/capa/safe_exec/safe_exec.py index da74f3aaf5..a07fe359fd 100644 --- a/common/lib/capa/capa/safe_exec/safe_exec.py +++ b/common/lib/capa/capa/safe_exec/safe_exec.py @@ -30,7 +30,7 @@ ASSUMED_IMPORTS=[ ("draganddrop", "verifiers.draganddrop"), ] -# We'll need the code from lazymod.py for use in jailpy, so read it now. +# We'll need the code from lazymod.py for use in safe_exec, so read it now. lazymod_py_file = lazymod.__file__ if lazymod_py_file.endswith("c"): lazymod_py_file = lazymod_py_file[:-1] diff --git a/common/lib/codejail/codejail/django_integration.py b/common/lib/codejail/codejail/django_integration.py index b720ffcff1..56c47f88d9 100644 --- a/common/lib/codejail/codejail/django_integration.py +++ b/common/lib/codejail/codejail/django_integration.py @@ -3,7 +3,7 @@ from django.core.exceptions import MiddlewareNotUsed from django.conf import settings -import codejail.jailpy +import codejail.jail_code class ConfigureCodeJailMiddleware(object): @@ -13,5 +13,5 @@ class ConfigureCodeJailMiddleware(object): python_bin = settings.CODE_JAIL.get('python_bin') if python_bin: user = settings.CODE_JAIL['user'] - codejail.jailpy.configure(python_bin, user=user) + codejail.jail_code.configure("python", python_bin, user=user) raise MiddlewareNotUsed diff --git a/common/lib/codejail/codejail/jailpy.py b/common/lib/codejail/codejail/jail_code.py similarity index 74% rename from common/lib/codejail/codejail/jailpy.py rename to common/lib/codejail/codejail/jail_code.py index df1bd8df12..a44004d585 100644 --- a/common/lib/codejail/codejail/jailpy.py +++ b/common/lib/codejail/codejail/jail_code.py @@ -19,39 +19,49 @@ log = logging.getLogger(__name__) # TODO: limit too much stdout data? -# Configure the Python command +# Configure the commands -PYTHON_CMD = None +# COMMANDS is a map from an abstract command name to a list of command-line +# pieces, such as subprocess.Popen wants. +COMMANDS = {} -def configure(python_bin, user=None): - """Configure the jailpy module.""" - global PYTHON_CMD - PYTHON_CMD = [] +def configure(command, bin_path, user=None): + """Configure a command for jail_code to use. + + `command` is the abstract command you're configuring, such as "python" or + "node". `bin_path` is the path to the binary. `user`, if provided, is + the user name to run the command under. + + """ + cmd_argv = [] if user: - PYTHON_CMD.extend(['sudo', '-u', 'sandbox']) - PYTHON_CMD.extend([python_bin, '-E']) + cmd_argv.extend(['sudo', '-u', 'sandbox']) + cmd_argv.extend([bin_path, '-E']) + COMMANDS[command] = cmd_argv -def is_configured(): - return bool(PYTHON_CMD) +def is_configured(command): + return command in COMMANDS # By default, look where our current Python is, and maybe there's a # python-sandbox alongside. Only do this if running in a virtualenv. if hasattr(sys, 'real_prefix'): if os.path.isdir(sys.prefix + "-sandbox"): - configure(sys.prefix + "-sandbox/bin/python", "sandbox") + configure("python", sys.prefix + "-sandbox/bin/python", "sandbox") class JailResult(object): - """A passive object for us to return from jailpy.""" + """A passive object for us to return from jail_code.""" def __init__(self): self.stdout = self.stderr = self.status = None -def jailpy(code, files=None, argv=None, stdin=None): - """ - Run Python code in a jailed subprocess. +def jail_code(command, code, files=None, argv=None, stdin=None): + """Run code in a jailed subprocess. + + `command` is an abstract command ("python", "node", ...) that must have + been configured using `configure`. `code` is a string containing the Python code to run. @@ -64,8 +74,8 @@ def jailpy(code, files=None, argv=None, stdin=None): .status: return status of the process: an int, 0 for successful """ - if not PYTHON_CMD: - raise Exception("jailpy needs to be configured") + if not is_configured(command): + raise Exception("jail_code needs to be configured for %r" % command) with temp_directory(delete_when_done=True) as tmpdir: @@ -83,7 +93,7 @@ def jailpy(code, files=None, argv=None, stdin=None): with open(os.path.join(tmpdir, "jailed_code.py"), "w") as jailed: jailed.write(code) - cmd = PYTHON_CMD + ['jailed_code.py'] + (argv or []) + cmd = COMMANDS[command] + ['jailed_code.py'] + (argv or []) subproc = subprocess.Popen( cmd, preexec_fn=set_process_limits, cwd=tmpdir, diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index 682ae809af..5379052ce0 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -7,7 +7,7 @@ import shutil import sys import textwrap -from codejail import jailpy +from codejail import jail_code from codejail.util import temp_directory, change_directory log = logging.getLogger(__name__) @@ -85,7 +85,7 @@ def safe_exec(code, globals_dict, files=None, python_path=None): log.debug("Exec: %s", code) log.debug("Stdin: %s", stdin) - res = jailpy.jailpy(jailed_code, stdin=stdin, files=files) + res = jail_code.jail_code("python", jailed_code, stdin=stdin, files=files) if res.status != 0: raise Exception("Couldn't execute jailed code: %s" % res.stderr) globals_dict.update(json.loads(res.stdout)) @@ -144,5 +144,5 @@ def not_safe_exec(code, globals_dict, files=None, python_path=None): # Running Python code in the sandbox makes it difficult to debug. # Change 0 to 1 to run the code directly. -if 0 or not jailpy.is_configured(): +if 0 or not jail_code.is_configured("python"): safe_exec = not_safe_exec diff --git a/common/lib/codejail/codejail/tests/test_jailpy.py b/common/lib/codejail/codejail/tests/test_jailpy.py index c0d51ba684..395cfc4d53 100644 --- a/common/lib/codejail/codejail/tests/test_jailpy.py +++ b/common/lib/codejail/codejail/tests/test_jailpy.py @@ -1,20 +1,24 @@ -"""Test jailpy.py""" +"""Test jail_code.py""" import os.path import textwrap import unittest from nose.plugins.skip import SkipTest -from codejail.jailpy import jailpy, is_configured +from codejail.jail_code import jail_code, is_configured dedent = textwrap.dedent -class JailPyHelpers(object): - """Assert helpers for jailpy tests.""" +def jailpy(*args, **kwargs): + return jail_code("python", *args, **kwargs) + + +class JailCodeHelpers(object): + """Assert helpers for jail_code tests.""" def setUp(self): - super(JailPyHelpers, self).setUp() - if not is_configured(): + super(JailCodeHelpers, self).setUp() + if not is_configured("python"): raise SkipTest def assertResultOk(self, res): @@ -22,7 +26,7 @@ class JailPyHelpers(object): self.assertEqual(res.status, 0) -class TestFeatures(JailPyHelpers, unittest.TestCase): +class TestFeatures(JailCodeHelpers, unittest.TestCase): def test_hello_world(self): res = jailpy("print 'Hello, world!'") self.assertResultOk(res) @@ -64,7 +68,7 @@ class TestFeatures(JailPyHelpers, unittest.TestCase): self.assertEqual(res.stdout, 'Look: Hello there.\n\n') -class TestLimits(JailPyHelpers, unittest.TestCase): +class TestLimits(JailCodeHelpers, unittest.TestCase): def test_cant_use_too_much_memory(self): res = jailpy("print sum(range(100000000))") self.assertNotEqual(res.status, 0) @@ -114,7 +118,7 @@ class TestLimits(JailPyHelpers, unittest.TestCase): # TODO: fork -class TestMalware(JailPyHelpers, unittest.TestCase): +class TestMalware(JailCodeHelpers, unittest.TestCase): def test_crash_cpython(self): # http://nedbatchelder.com/blog/201206/eval_really_is_dangerous.html res = jailpy(dedent("""\ From 9683098f3da394150fe0d163145af316487e8bcc Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 10 Apr 2013 17:53:05 -0400 Subject: [PATCH 078/162] Python should have -E, not sure of a clean way to do it, but this at least only applies it to python. --- common/lib/codejail/codejail/jail_code.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/common/lib/codejail/codejail/jail_code.py b/common/lib/codejail/codejail/jail_code.py index a44004d585..9b48568bd8 100644 --- a/common/lib/codejail/codejail/jail_code.py +++ b/common/lib/codejail/codejail/jail_code.py @@ -27,7 +27,7 @@ COMMANDS = {} def configure(command, bin_path, user=None): - """Configure a command for jail_code to use. + """Configure a command for `jail_code` to use. `command` is the abstract command you're configuring, such as "python" or "node". `bin_path` is the path to the binary. `user`, if provided, is @@ -37,11 +37,22 @@ def configure(command, bin_path, user=None): cmd_argv = [] if user: cmd_argv.extend(['sudo', '-u', 'sandbox']) - cmd_argv.extend([bin_path, '-E']) + cmd_argv.append(bin_path) + + # Command-specific arguments + if command == "python": + cmd_argv.append('-E') + COMMANDS[command] = cmd_argv def is_configured(command): + """Has `jail_code` been configured for `command`? + + Returns true if the abstract command `command` has been configured for use + in the `jail_code` function. + + """ return command in COMMANDS # By default, look where our current Python is, and maybe there's a From fb5343237a4729f79d9c449f1b24202dc76745bd Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 12 Apr 2013 14:57:37 -0400 Subject: [PATCH 079/162] jail_code can execute a provided file also. --- common/lib/codejail/codejail/jail_code.py | 22 +++++++--- common/lib/codejail/codejail/safe_exec.py | 2 +- common/lib/codejail/codejail/tests/doit.py | 4 ++ .../codejail/codejail/tests/test_jailpy.py | 44 ++++++++++++------- 4 files changed, 50 insertions(+), 22 deletions(-) create mode 100644 common/lib/codejail/codejail/tests/doit.py diff --git a/common/lib/codejail/codejail/jail_code.py b/common/lib/codejail/codejail/jail_code.py index 9b48568bd8..2e1b78fe0a 100644 --- a/common/lib/codejail/codejail/jail_code.py +++ b/common/lib/codejail/codejail/jail_code.py @@ -68,15 +68,20 @@ class JailResult(object): self.stdout = self.stderr = self.status = None -def jail_code(command, code, files=None, argv=None, stdin=None): +def jail_code(command, code=None, files=None, argv=None, stdin=None): """Run code in a jailed subprocess. `command` is an abstract command ("python", "node", ...) that must have been configured using `configure`. - `code` is a string containing the Python code to run. + `code` is a string containing the code to run. If no code is supplied, + then the code to run must be in one of the `files` copied, and must be + named in the `argv` list. - `files` is a list of file paths. + `files` is a list of file paths, they are all copied to the jailed + directory. + + `argv` is the command-line arguments to supply. Return an object with: @@ -92,6 +97,8 @@ def jail_code(command, code, files=None, argv=None, stdin=None): log.debug("Executing jailed code: %r", code) + argv = argv or [] + # All the supporting files are copied into our directory. for filename in files or (): if os.path.isfile(filename): @@ -101,10 +108,13 @@ def jail_code(command, code, files=None, argv=None, stdin=None): shutil.copytree(filename, dest) # Create the main file. - with open(os.path.join(tmpdir, "jailed_code.py"), "w") as jailed: - jailed.write(code) + if code: + with open(os.path.join(tmpdir, "jailed_code"), "w") as jailed: + jailed.write(code) - cmd = COMMANDS[command] + ['jailed_code.py'] + (argv or []) + argv = ["jailed_code"] + argv + + cmd = COMMANDS[command] + argv subproc = subprocess.Popen( cmd, preexec_fn=set_process_limits, cwd=tmpdir, diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py index 5379052ce0..79729565f7 100644 --- a/common/lib/codejail/codejail/safe_exec.py +++ b/common/lib/codejail/codejail/safe_exec.py @@ -85,7 +85,7 @@ def safe_exec(code, globals_dict, files=None, python_path=None): log.debug("Exec: %s", code) log.debug("Stdin: %s", stdin) - res = jail_code.jail_code("python", jailed_code, stdin=stdin, files=files) + res = jail_code.jail_code("python", code=jailed_code, stdin=stdin, files=files) if res.status != 0: raise Exception("Couldn't execute jailed code: %s" % res.stderr) globals_dict.update(json.loads(res.stdout)) diff --git a/common/lib/codejail/codejail/tests/doit.py b/common/lib/codejail/codejail/tests/doit.py new file mode 100644 index 0000000000..5786635067 --- /dev/null +++ b/common/lib/codejail/codejail/tests/doit.py @@ -0,0 +1,4 @@ +import sys + +print "This is doit.py!" +print "My args are %r" % (sys.argv,) diff --git a/common/lib/codejail/codejail/tests/test_jailpy.py b/common/lib/codejail/codejail/tests/test_jailpy.py index 395cfc4d53..21cb780946 100644 --- a/common/lib/codejail/codejail/tests/test_jailpy.py +++ b/common/lib/codejail/codejail/tests/test_jailpy.py @@ -11,9 +11,15 @@ dedent = textwrap.dedent def jailpy(*args, **kwargs): + """Run `jail_code` on Python.""" return jail_code("python", *args, **kwargs) +def file_here(fname): + """Return the full path to a file alongside this code.""" + return os.path.join(os.path.dirname(__file__), fname) + + class JailCodeHelpers(object): """Assert helpers for jail_code tests.""" def setUp(self): @@ -28,32 +34,32 @@ class JailCodeHelpers(object): class TestFeatures(JailCodeHelpers, unittest.TestCase): def test_hello_world(self): - res = jailpy("print 'Hello, world!'") + res = jailpy(code="print 'Hello, world!'") self.assertResultOk(res) self.assertEqual(res.stdout, 'Hello, world!\n') def test_argv(self): res = jailpy( - "import sys; print ':'.join(sys.argv[1:])", + code="import sys; print ':'.join(sys.argv[1:])", argv=["Hello", "world", "-x"] ) self.assertResultOk(res) self.assertEqual(res.stdout, "Hello:world:-x\n") def test_ends_with_exception(self): - res = jailpy("""raise Exception('FAIL')""") + res = jailpy(code="""raise Exception('FAIL')""") self.assertNotEqual(res.status, 0) self.assertEqual(res.stdout, "") self.assertEqual(res.stderr, dedent("""\ Traceback (most recent call last): - File "jailed_code.py", line 1, in + File "jailed_code", line 1, in raise Exception('FAIL') Exception: FAIL """)) def test_stdin_is_provided(self): res = jailpy( - "import json,sys; print sum(json.load(sys.stdin))", + code="import json,sys; print sum(json.load(sys.stdin))", stdin="[1, 2.5, 33]" ) self.assertResultOk(res) @@ -61,27 +67,35 @@ class TestFeatures(JailCodeHelpers, unittest.TestCase): def test_files_are_copied(self): res = jailpy( - "print 'Look:', open('hello.txt').read()", - files=[os.path.dirname(__file__) + "/hello.txt"] + code="print 'Look:', open('hello.txt').read()", + files=[file_here("hello.txt")] ) self.assertResultOk(res) self.assertEqual(res.stdout, 'Look: Hello there.\n\n') + def test_executing_a_copied_file(self): + res = jailpy( + files=[file_here("doit.py")], + argv=["doit.py", "1", "2", "3"] + ) + self.assertResultOk(res) + self.assertEqual(res.stdout, "This is doit.py!\nMy args are ['doit.py', '1', '2', '3']\n") + class TestLimits(JailCodeHelpers, unittest.TestCase): def test_cant_use_too_much_memory(self): - res = jailpy("print sum(range(100000000))") + res = jailpy(code="print sum(range(100000000))") self.assertNotEqual(res.status, 0) self.assertEqual(res.stdout, "") def test_cant_use_too_much_cpu(self): - res = jailpy("print sum(xrange(100000000))") + res = jailpy(code="print sum(xrange(100000000))") self.assertNotEqual(res.status, 0) self.assertEqual(res.stdout, "") def test_cant_use_too_much_time(self): raise SkipTest # TODO: test this once we can kill sleeping processes. - res = jailpy(dedent("""\ + res = jailpy(code=dedent("""\ import time time.sleep(5) print 'Done!' @@ -90,7 +104,7 @@ class TestLimits(JailCodeHelpers, unittest.TestCase): self.assertEqual(res.stdout, "") def test_cant_write_files(self): - res = jailpy(dedent("""\ + res = jailpy(code=dedent("""\ print "Trying" with open("mydata.txt", "w") as f: f.write("hello") @@ -102,7 +116,7 @@ class TestLimits(JailCodeHelpers, unittest.TestCase): self.assertIn("ermission denied", res.stderr) def test_cant_use_network(self): - res = jailpy(dedent("""\ + res = jailpy(code=dedent("""\ import urllib print "Reading google" u = urllib.urlopen("http://google.com") @@ -121,7 +135,7 @@ class TestLimits(JailCodeHelpers, unittest.TestCase): class TestMalware(JailCodeHelpers, unittest.TestCase): def test_crash_cpython(self): # http://nedbatchelder.com/blog/201206/eval_really_is_dangerous.html - res = jailpy(dedent("""\ + res = jailpy(code=dedent("""\ import new, sys crash_me = new.function(new.code(0,0,0,0,"KABOOM",(),(),(),"","",0,""), {}) print "Here we go..." @@ -134,7 +148,7 @@ class TestMalware(JailCodeHelpers, unittest.TestCase): self.assertEqual(res.stderr, "") def test_read_etc_passwd(self): - res = jailpy(dedent("""\ + res = jailpy(code=dedent("""\ bytes = len(open('/etc/passwd').read()) print 'Gotcha', bytes """)) @@ -143,7 +157,7 @@ class TestMalware(JailCodeHelpers, unittest.TestCase): self.assertIn("ermission denied", res.stderr) def test_find_other_sandboxes(self): - res = jailpy(dedent(""" + res = jailpy(code=dedent(""" import os; places = [ "..", "/tmp", "/", "/home", "/etc", From 726e8db13e9b248c7a07d5d74e9f36b08886dff2 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 24 Apr 2013 15:14:07 -0400 Subject: [PATCH 080/162] Add more docs --- common/lib/codejail/README | 75 ++++++++++++++++++++++++++++++-------- 1 file changed, 60 insertions(+), 15 deletions(-) diff --git a/common/lib/codejail/README b/common/lib/codejail/README index 75862d69e3..889534a0c3 100644 --- a/common/lib/codejail/README +++ b/common/lib/codejail/README @@ -1,40 +1,75 @@ -Choose a place for the virtualenv, call it . It will be automatically -detected and used if you put it right alongside your existing virtualenv, but -with -sandbox appended. So if your existing virtualenv is in ~/mitx_all/python, -make be ~/mitx_all/python-sandbox (but you'll need to spell out your -home directory instead of ~). +CodeJail +======== + +CodeJail manages execution of untrusted code in secure sandboxes. It is +designed primarily for Python execution, but can be used for other languages as +well. + +Security is enforced with AppArmor. If your operating system doesn't support +AppArmor, then CodeJail won't protect the execution. + +CodeJail is designed to be configurable, and will auto-configure itself for +Python execution if you install it properly. The configuration is designed to +be flexible: it can run in safe more or unsafe mode. This helps support large +development groups where only some of the developers are involved enough with +secure execution to configure AppArmor on their development machines. + +If CodeJail is not configured for safe execution, it will execution Python +using the same API, but will not guard against malicious code. This allows the +same code to be used on safe-configured or non-safe-configured developer's +machines. + + +Installation +------------ + +These instructions detail how to configure your operating system so that +CodeJail can execute Python code safely. You can run CodeJail without these +steps, and you will have an unsafe CodeJail. This is fine for developers' +machines who are unconcerned with security, and simplifies the integration of +CodeJail into your project. + +To secure Python execution, you'll be creating a new virtualenv. This means +you'll have two: the main virtualenv for your project, and the new one for +sandboxed Python code. + +Choose a place for the new virtualenv, call it . It will be +automatically detected and used if you put it right alongside your existing +virtualenv, but with -sandbox appended. So if your existing virtualenv is in +~/ve/myproj, make be ~/ve/myproj-sandbox (but you'll need to spell +out your home directory instead of ~). Other details here that depend on your configuration: - Your mitx working tree is , for example, ~/mitx_all/mitx - - The user running the LMS is , for example, you on a dev machine, + - The user running the LMS is , for example, you on your dev machine, or www-data on a server. -Create a virtualenv: +1. Create the new virtualenv:: $ sudo virtualenv -Install the sandbox requirements +2. Install the sandbox requirements:: $ source /bin/activate $ sudo pip install -r sandbox-requirements.txt -Add a sandbox user: +3. Add a sandbox user:: $ sudo addgroup sandbox $ sudo adduser --disabled-login sandbox --ingroup sandbox -Let the web server run the sandboxed Python as sandbox. Create the file -/etc/sudoers.d/01-sandbox: +4. Let the web server run the sandboxed Python as sandbox. Create the file +/etc/sudoers.d/01-sandbox:: $ visudo -f /etc/sudoers.d/01-sandbox ALL=(sandbox) NOPASSWD:/bin/python ALL=(ALL) NOPASSWD:/bin/kill -Edit an AppArmor profile. The file must be named for the python executable, -but with slashes changed to dots: +5. Edit an AppArmor profile. The file must be named for the python executable, +but with slashes changed to dots:: #include @@ -49,8 +84,18 @@ but with slashes changed to dots: /tmp/** rix, } -Parse the profiles +6. Parse the profiles:: $ sudo apparmor_parser -Reactivate your real virtualenv again +7. Reactivate your project's main virtualenv again. + + +Tests +===== + +The tests run under nose in the standard fashion. + +If CodeJail is running unsafely, many of the tests will be automatically +skipped, or will fail, depending on whether CodeJail thinks it should be in +safe mode or not. From adde93983149444fe47899a995a794b65ab81e79 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 24 Apr 2013 15:14:24 -0400 Subject: [PATCH 081/162] Clarify some comments in tests. --- common/lib/codejail/codejail/tests/test_jailpy.py | 2 -- common/lib/codejail/codejail/tests/test_safe_exec.py | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/lib/codejail/codejail/tests/test_jailpy.py b/common/lib/codejail/codejail/tests/test_jailpy.py index 21cb780946..76bf2b1c1d 100644 --- a/common/lib/codejail/codejail/tests/test_jailpy.py +++ b/common/lib/codejail/codejail/tests/test_jailpy.py @@ -127,8 +127,6 @@ class TestLimits(JailCodeHelpers, unittest.TestCase): self.assertEqual(res.stdout, "Reading google\n") self.assertIn("IOError", res.stderr) - # TODO: write files - # TODO: read network # TODO: fork diff --git a/common/lib/codejail/codejail/tests/test_safe_exec.py b/common/lib/codejail/codejail/tests/test_safe_exec.py index 46d6a59a98..f7dac45b03 100644 --- a/common/lib/codejail/codejail/tests/test_safe_exec.py +++ b/common/lib/codejail/codejail/tests/test_safe_exec.py @@ -65,6 +65,8 @@ class TestSafeExec(SafeExecTests, unittest.TestCase): class TestNotSafeExec(SafeExecTests, unittest.TestCase): """Run SafeExecTests, with not_safe_exec.""" def setUp(self): + # If safe_exec is actually an alias to not_safe_exec, then there's no + # point running these tests. if safe_exec is not_safe_exec: raise SkipTest From 09fbbe7bfa2c16a33074b72de9e5fcf2eeb02de1 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 29 Apr 2013 17:31:19 -0400 Subject: [PATCH 082/162] Codejail is in its own repo now. --- common/lib/codejail/README | 101 ---------- common/lib/codejail/codejail/__init__.py | 0 .../codejail/codejail/django_integration.py | 17 -- common/lib/codejail/codejail/jail_code.py | 168 ----------------- common/lib/codejail/codejail/safe_exec.py | 148 --------------- .../lib/codejail/codejail/tests/__init__.py | 0 common/lib/codejail/codejail/tests/doit.py | 4 - common/lib/codejail/codejail/tests/hello.txt | 1 - .../codejail/codejail/tests/pylib/module.py | 1 - .../codejail/codejail/tests/test_jailpy.py | 175 ------------------ .../codejail/codejail/tests/test_safe_exec.py | 74 -------- common/lib/codejail/codejail/util.py | 53 ------ common/lib/codejail/setup.py | 7 - local-requirements.txt | 2 +- 14 files changed, 1 insertion(+), 750 deletions(-) delete mode 100644 common/lib/codejail/README delete mode 100644 common/lib/codejail/codejail/__init__.py delete mode 100644 common/lib/codejail/codejail/django_integration.py delete mode 100644 common/lib/codejail/codejail/jail_code.py delete mode 100644 common/lib/codejail/codejail/safe_exec.py delete mode 100644 common/lib/codejail/codejail/tests/__init__.py delete mode 100644 common/lib/codejail/codejail/tests/doit.py delete mode 100644 common/lib/codejail/codejail/tests/hello.txt delete mode 100644 common/lib/codejail/codejail/tests/pylib/module.py delete mode 100644 common/lib/codejail/codejail/tests/test_jailpy.py delete mode 100644 common/lib/codejail/codejail/tests/test_safe_exec.py delete mode 100644 common/lib/codejail/codejail/util.py delete mode 100644 common/lib/codejail/setup.py diff --git a/common/lib/codejail/README b/common/lib/codejail/README deleted file mode 100644 index 889534a0c3..0000000000 --- a/common/lib/codejail/README +++ /dev/null @@ -1,101 +0,0 @@ -CodeJail -======== - -CodeJail manages execution of untrusted code in secure sandboxes. It is -designed primarily for Python execution, but can be used for other languages as -well. - -Security is enforced with AppArmor. If your operating system doesn't support -AppArmor, then CodeJail won't protect the execution. - -CodeJail is designed to be configurable, and will auto-configure itself for -Python execution if you install it properly. The configuration is designed to -be flexible: it can run in safe more or unsafe mode. This helps support large -development groups where only some of the developers are involved enough with -secure execution to configure AppArmor on their development machines. - -If CodeJail is not configured for safe execution, it will execution Python -using the same API, but will not guard against malicious code. This allows the -same code to be used on safe-configured or non-safe-configured developer's -machines. - - -Installation ------------- - -These instructions detail how to configure your operating system so that -CodeJail can execute Python code safely. You can run CodeJail without these -steps, and you will have an unsafe CodeJail. This is fine for developers' -machines who are unconcerned with security, and simplifies the integration of -CodeJail into your project. - -To secure Python execution, you'll be creating a new virtualenv. This means -you'll have two: the main virtualenv for your project, and the new one for -sandboxed Python code. - -Choose a place for the new virtualenv, call it . It will be -automatically detected and used if you put it right alongside your existing -virtualenv, but with -sandbox appended. So if your existing virtualenv is in -~/ve/myproj, make be ~/ve/myproj-sandbox (but you'll need to spell -out your home directory instead of ~). - -Other details here that depend on your configuration: - - - Your mitx working tree is , for example, ~/mitx_all/mitx - - - The user running the LMS is , for example, you on your dev machine, - or www-data on a server. - -1. Create the new virtualenv:: - - $ sudo virtualenv - -2. Install the sandbox requirements:: - - $ source /bin/activate - $ sudo pip install -r sandbox-requirements.txt - -3. Add a sandbox user:: - - $ sudo addgroup sandbox - $ sudo adduser --disabled-login sandbox --ingroup sandbox - -4. Let the web server run the sandboxed Python as sandbox. Create the file -/etc/sudoers.d/01-sandbox:: - - $ visudo -f /etc/sudoers.d/01-sandbox - - ALL=(sandbox) NOPASSWD:/bin/python - ALL=(ALL) NOPASSWD:/bin/kill - -5. Edit an AppArmor profile. The file must be named for the python executable, -but with slashes changed to dots:: - - #include - - /bin/python { - #include - - /** mr, - /common/lib/sandbox-packages/** r, - /usr/local/lib/python2.7/** r, - /usr/lib/python2.7/** rix, - - /tmp/** rix, - } - -6. Parse the profiles:: - - $ sudo apparmor_parser - -7. Reactivate your project's main virtualenv again. - - -Tests -===== - -The tests run under nose in the standard fashion. - -If CodeJail is running unsafely, many of the tests will be automatically -skipped, or will fail, depending on whether CodeJail thinks it should be in -safe mode or not. diff --git a/common/lib/codejail/codejail/__init__.py b/common/lib/codejail/codejail/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/common/lib/codejail/codejail/django_integration.py b/common/lib/codejail/codejail/django_integration.py deleted file mode 100644 index 56c47f88d9..0000000000 --- a/common/lib/codejail/codejail/django_integration.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Django integration for codejail""" - -from django.core.exceptions import MiddlewareNotUsed -from django.conf import settings - -import codejail.jail_code - - -class ConfigureCodeJailMiddleware(object): - """Middleware to configure codejail on startup.""" - - def __init__(self): - python_bin = settings.CODE_JAIL.get('python_bin') - if python_bin: - user = settings.CODE_JAIL['user'] - codejail.jail_code.configure("python", python_bin, user=user) - raise MiddlewareNotUsed diff --git a/common/lib/codejail/codejail/jail_code.py b/common/lib/codejail/codejail/jail_code.py deleted file mode 100644 index 2e1b78fe0a..0000000000 --- a/common/lib/codejail/codejail/jail_code.py +++ /dev/null @@ -1,168 +0,0 @@ -"""Run a python process in a jail.""" - -# Instructions: -# - AppArmor.md from xserver - -import logging -import os -import os.path -import resource -import shutil -import subprocess -import sys -import threading -import time - -from .util import temp_directory - -log = logging.getLogger(__name__) - -# TODO: limit too much stdout data? - -# Configure the commands - -# COMMANDS is a map from an abstract command name to a list of command-line -# pieces, such as subprocess.Popen wants. -COMMANDS = {} - - -def configure(command, bin_path, user=None): - """Configure a command for `jail_code` to use. - - `command` is the abstract command you're configuring, such as "python" or - "node". `bin_path` is the path to the binary. `user`, if provided, is - the user name to run the command under. - - """ - cmd_argv = [] - if user: - cmd_argv.extend(['sudo', '-u', 'sandbox']) - cmd_argv.append(bin_path) - - # Command-specific arguments - if command == "python": - cmd_argv.append('-E') - - COMMANDS[command] = cmd_argv - - -def is_configured(command): - """Has `jail_code` been configured for `command`? - - Returns true if the abstract command `command` has been configured for use - in the `jail_code` function. - - """ - return command in COMMANDS - -# By default, look where our current Python is, and maybe there's a -# python-sandbox alongside. Only do this if running in a virtualenv. -if hasattr(sys, 'real_prefix'): - if os.path.isdir(sys.prefix + "-sandbox"): - configure("python", sys.prefix + "-sandbox/bin/python", "sandbox") - - -class JailResult(object): - """A passive object for us to return from jail_code.""" - def __init__(self): - self.stdout = self.stderr = self.status = None - - -def jail_code(command, code=None, files=None, argv=None, stdin=None): - """Run code in a jailed subprocess. - - `command` is an abstract command ("python", "node", ...) that must have - been configured using `configure`. - - `code` is a string containing the code to run. If no code is supplied, - then the code to run must be in one of the `files` copied, and must be - named in the `argv` list. - - `files` is a list of file paths, they are all copied to the jailed - directory. - - `argv` is the command-line arguments to supply. - - Return an object with: - - .stdout: stdout of the program, a string - .stderr: stderr of the program, a string - .status: return status of the process: an int, 0 for successful - - """ - if not is_configured(command): - raise Exception("jail_code needs to be configured for %r" % command) - - with temp_directory(delete_when_done=True) as tmpdir: - - log.debug("Executing jailed code: %r", code) - - argv = argv or [] - - # All the supporting files are copied into our directory. - for filename in files or (): - if os.path.isfile(filename): - shutil.copy(filename, tmpdir) - else: - dest = os.path.join(tmpdir, os.path.basename(filename)) - shutil.copytree(filename, dest) - - # Create the main file. - if code: - with open(os.path.join(tmpdir, "jailed_code"), "w") as jailed: - jailed.write(code) - - argv = ["jailed_code"] + argv - - cmd = COMMANDS[command] + argv - - subproc = subprocess.Popen( - cmd, preexec_fn=set_process_limits, cwd=tmpdir, - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - ) - - # TODO: time limiting - - killer = ProcessKillerThread(subproc) - killer.start() - result = JailResult() - result.stdout, result.stderr = subproc.communicate(stdin) - result.status = subproc.returncode - - return result - - -def set_process_limits(): - """ - Set limits on this processs, to be used first in a child process. - """ - resource.setrlimit(resource.RLIMIT_CPU, (1, 1)) # 1 second of CPU--not wall clock time - resource.setrlimit(resource.RLIMIT_NPROC, (0, 0)) # no subprocesses - resource.setrlimit(resource.RLIMIT_FSIZE, (0, 0)) # no files - - mem = 32 * (2 ** 20) # 32 MB should be enough for anyone, right? :) - resource.setrlimit(resource.RLIMIT_STACK, (mem, mem)) - resource.setrlimit(resource.RLIMIT_RSS, (mem, mem)) - resource.setrlimit(resource.RLIMIT_DATA, (mem, mem)) - - -class ProcessKillerThread(threading.Thread): - def __init__(self, subproc, limit=1): - super(ProcessKillerThread, self).__init__() - self.subproc = subproc - self.limit = limit - - def run(self): - start = time.time() - while (time.time() - start) < self.limit: - time.sleep(.1) - if self.subproc.poll() is not None: - # Process ended, no need for us any more. - return - - if self.subproc.poll() is None: - # Can't use subproc.kill because we launched the subproc with sudo. - killargs = ["sudo", "kill", "-9", str(self.subproc.pid)] - kill = subprocess.Popen(killargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = kill.communicate() - # TODO: This doesn't actually kill the process.... :( diff --git a/common/lib/codejail/codejail/safe_exec.py b/common/lib/codejail/codejail/safe_exec.py deleted file mode 100644 index 79729565f7..0000000000 --- a/common/lib/codejail/codejail/safe_exec.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Safe execution of untrusted Python code.""" - -import json -import logging -import os.path -import shutil -import sys -import textwrap - -from codejail import jail_code -from codejail.util import temp_directory, change_directory - -log = logging.getLogger(__name__) - - -def safe_exec(code, globals_dict, files=None, python_path=None): - """Execute code as "exec" does, but safely. - - `code` is a string of Python code. `globals_dict` is used as the globals - during execution. Modifications the code makes to `globals_dict` are - reflected in the dictionary on return. - - Returns None. Changes made by `code` are visible in `globals_dict`. - - """ - the_code = [] - files = list(files or ()) - - the_code.append(textwrap.dedent( - """ - import json - import sys - """ - # We need to prevent the sandboxed code from printing to stdout, - # or it will pollute the json we print there. This isn't a - # security concern (they can put any values in the json output - # anyway, either by writing to sys.__stdout__, or just by defining - # global values), but keeps accidents from happening. - """ - class DevNull(object): - def write(self, *args, **kwargs): - pass - sys.stdout = DevNull() - """ - # Read the code and the globals from the stdin. - """ - code, g_dict = json.load(sys.stdin) - """)) - - for pydir in python_path or (): - pybase = os.path.basename(pydir) - the_code.append("sys.path.append(%r)\n" % pybase) - files.append(pydir) - - the_code.append(textwrap.dedent( - # Execute the sandboxed code. - """ - exec code in g_dict - """ - # Clean the globals for sending back as JSON over stdout. - """ - ok_types = (type(None), int, long, float, str, unicode, list, tuple, dict) - bad_keys = ("__builtins__",) - def jsonable(v): - if not isinstance(v, ok_types): - return False - try: - json.dumps(v) - except Exception: - return False - return True - g_dict = {k:v for k,v in g_dict.iteritems() if jsonable(v) and k not in bad_keys} - """ - # Write the globals back to the calling process. - """ - json.dump(g_dict, sys.__stdout__) - """)) - - stdin = json.dumps([code, json_safe(globals_dict)]) - jailed_code = "".join(the_code) - - # Turn this on to see what's being executed. - if 0: - log.debug("Jailed code: %s", jailed_code) - log.debug("Exec: %s", code) - log.debug("Stdin: %s", stdin) - - res = jail_code.jail_code("python", code=jailed_code, stdin=stdin, files=files) - if res.status != 0: - raise Exception("Couldn't execute jailed code: %s" % res.stderr) - globals_dict.update(json.loads(res.stdout)) - - -def json_safe(d): - """Return only the JSON-safe part of d. - - Used to emulate reading data through a serialization straw. - - """ - ok_types = (type(None), int, long, float, str, unicode, list, tuple, dict) - bad_keys = ("__builtins__",) - jd = {} - for k, v in d.iteritems(): - if not isinstance(v, ok_types): - continue - if k in bad_keys: - continue - try: - json.dumps(v) - except TypeError: - continue - else: - jd[k] = v - return json.loads(json.dumps(jd)) - - -def not_safe_exec(code, globals_dict, files=None, python_path=None): - """Another implementation of `safe_exec`, but not safe. - - This can be swapped in for debugging problems in sandboxed Python code. - - This is not thread-safe, due to temporarily changing the current directory - and modifying sys.path. - - """ - g_dict = json_safe(globals_dict) - - with temp_directory(delete_when_done=True) as tmpdir: - with change_directory(tmpdir): - # Copy the files here. - for filename in files or (): - dest = os.path.join(tmpdir, os.path.basename(filename)) - shutil.copyfile(filename, dest) - - original_path = sys.path - if python_path: - sys.path.extend(python_path) - try: - exec code in g_dict - finally: - sys.path = original_path - - globals_dict.update(json_safe(g_dict)) - -# Running Python code in the sandbox makes it difficult to debug. -# Change 0 to 1 to run the code directly. -if 0 or not jail_code.is_configured("python"): - safe_exec = not_safe_exec diff --git a/common/lib/codejail/codejail/tests/__init__.py b/common/lib/codejail/codejail/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/common/lib/codejail/codejail/tests/doit.py b/common/lib/codejail/codejail/tests/doit.py deleted file mode 100644 index 5786635067..0000000000 --- a/common/lib/codejail/codejail/tests/doit.py +++ /dev/null @@ -1,4 +0,0 @@ -import sys - -print "This is doit.py!" -print "My args are %r" % (sys.argv,) diff --git a/common/lib/codejail/codejail/tests/hello.txt b/common/lib/codejail/codejail/tests/hello.txt deleted file mode 100644 index c12abce9f3..0000000000 --- a/common/lib/codejail/codejail/tests/hello.txt +++ /dev/null @@ -1 +0,0 @@ -Hello there. diff --git a/common/lib/codejail/codejail/tests/pylib/module.py b/common/lib/codejail/codejail/tests/pylib/module.py deleted file mode 100644 index 8cb5cded29..0000000000 --- a/common/lib/codejail/codejail/tests/pylib/module.py +++ /dev/null @@ -1 +0,0 @@ -const = 42 diff --git a/common/lib/codejail/codejail/tests/test_jailpy.py b/common/lib/codejail/codejail/tests/test_jailpy.py deleted file mode 100644 index 76bf2b1c1d..0000000000 --- a/common/lib/codejail/codejail/tests/test_jailpy.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Test jail_code.py""" - -import os.path -import textwrap -import unittest -from nose.plugins.skip import SkipTest - -from codejail.jail_code import jail_code, is_configured - -dedent = textwrap.dedent - - -def jailpy(*args, **kwargs): - """Run `jail_code` on Python.""" - return jail_code("python", *args, **kwargs) - - -def file_here(fname): - """Return the full path to a file alongside this code.""" - return os.path.join(os.path.dirname(__file__), fname) - - -class JailCodeHelpers(object): - """Assert helpers for jail_code tests.""" - def setUp(self): - super(JailCodeHelpers, self).setUp() - if not is_configured("python"): - raise SkipTest - - def assertResultOk(self, res): - self.assertEqual(res.stderr, "") - self.assertEqual(res.status, 0) - - -class TestFeatures(JailCodeHelpers, unittest.TestCase): - def test_hello_world(self): - res = jailpy(code="print 'Hello, world!'") - self.assertResultOk(res) - self.assertEqual(res.stdout, 'Hello, world!\n') - - def test_argv(self): - res = jailpy( - code="import sys; print ':'.join(sys.argv[1:])", - argv=["Hello", "world", "-x"] - ) - self.assertResultOk(res) - self.assertEqual(res.stdout, "Hello:world:-x\n") - - def test_ends_with_exception(self): - res = jailpy(code="""raise Exception('FAIL')""") - self.assertNotEqual(res.status, 0) - self.assertEqual(res.stdout, "") - self.assertEqual(res.stderr, dedent("""\ - Traceback (most recent call last): - File "jailed_code", line 1, in - raise Exception('FAIL') - Exception: FAIL - """)) - - def test_stdin_is_provided(self): - res = jailpy( - code="import json,sys; print sum(json.load(sys.stdin))", - stdin="[1, 2.5, 33]" - ) - self.assertResultOk(res) - self.assertEqual(res.stdout.strip(), "36.5") - - def test_files_are_copied(self): - res = jailpy( - code="print 'Look:', open('hello.txt').read()", - files=[file_here("hello.txt")] - ) - self.assertResultOk(res) - self.assertEqual(res.stdout, 'Look: Hello there.\n\n') - - def test_executing_a_copied_file(self): - res = jailpy( - files=[file_here("doit.py")], - argv=["doit.py", "1", "2", "3"] - ) - self.assertResultOk(res) - self.assertEqual(res.stdout, "This is doit.py!\nMy args are ['doit.py', '1', '2', '3']\n") - - -class TestLimits(JailCodeHelpers, unittest.TestCase): - def test_cant_use_too_much_memory(self): - res = jailpy(code="print sum(range(100000000))") - self.assertNotEqual(res.status, 0) - self.assertEqual(res.stdout, "") - - def test_cant_use_too_much_cpu(self): - res = jailpy(code="print sum(xrange(100000000))") - self.assertNotEqual(res.status, 0) - self.assertEqual(res.stdout, "") - - def test_cant_use_too_much_time(self): - raise SkipTest # TODO: test this once we can kill sleeping processes. - res = jailpy(code=dedent("""\ - import time - time.sleep(5) - print 'Done!' - """)) - self.assertNotEqual(res.status, 0) - self.assertEqual(res.stdout, "") - - def test_cant_write_files(self): - res = jailpy(code=dedent("""\ - print "Trying" - with open("mydata.txt", "w") as f: - f.write("hello") - with open("mydata.txt") as f2: - print "Got this:", f2.read() - """)) - self.assertNotEqual(res.status, 0) - self.assertEqual(res.stdout, "Trying\n") - self.assertIn("ermission denied", res.stderr) - - def test_cant_use_network(self): - res = jailpy(code=dedent("""\ - import urllib - print "Reading google" - u = urllib.urlopen("http://google.com") - google = u.read() - print len(google) - """)) - self.assertNotEqual(res.status, 0) - self.assertEqual(res.stdout, "Reading google\n") - self.assertIn("IOError", res.stderr) - - # TODO: fork - - -class TestMalware(JailCodeHelpers, unittest.TestCase): - def test_crash_cpython(self): - # http://nedbatchelder.com/blog/201206/eval_really_is_dangerous.html - res = jailpy(code=dedent("""\ - import new, sys - crash_me = new.function(new.code(0,0,0,0,"KABOOM",(),(),(),"","",0,""), {}) - print "Here we go..." - sys.stdout.flush() - crash_me() - print "The afterlife!" - """)) - self.assertNotEqual(res.status, 0) - self.assertEqual(res.stdout, "Here we go...\n") - self.assertEqual(res.stderr, "") - - def test_read_etc_passwd(self): - res = jailpy(code=dedent("""\ - bytes = len(open('/etc/passwd').read()) - print 'Gotcha', bytes - """)) - self.assertNotEqual(res.status, 0) - self.assertEqual(res.stdout, "") - self.assertIn("ermission denied", res.stderr) - - def test_find_other_sandboxes(self): - res = jailpy(code=dedent(""" - import os; - places = [ - "..", "/tmp", "/", "/home", "/etc", - "/var" - ] - for place in places: - try: - files = os.listdir(place) - except Exception: - # darn - pass - else: - print "Files in %r: %r" % (place, files) - print "Done." - """)) - self.assertResultOk(res) - self.assertEqual(res.stdout, "Done.\n") diff --git a/common/lib/codejail/codejail/tests/test_safe_exec.py b/common/lib/codejail/codejail/tests/test_safe_exec.py deleted file mode 100644 index f7dac45b03..0000000000 --- a/common/lib/codejail/codejail/tests/test_safe_exec.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Test safe_exec.py""" - -import os.path -import textwrap -import unittest -from nose.plugins.skip import SkipTest - -from codejail.safe_exec import safe_exec, not_safe_exec - - -class SafeExecTests(object): - """The tests for `safe_exec`, will be mixed into specific test classes below.""" - def test_set_values(self): - g = {} - self.safe_exec("a = 17", g) - self.assertEqual(g['a'], 17) - - def test_files_are_copied(self): - g = {} - self.safe_exec( - "a = 'Look: ' + open('hello.txt').read()", g, - files=[os.path.dirname(__file__) + "/hello.txt"] - ) - self.assertEqual(g['a'], 'Look: Hello there.\n') - - def test_python_path(self): - g = {} - self.safe_exec( - "import module; a = module.const", g, - python_path=[os.path.dirname(__file__) + "/pylib"] - ) - self.assertEqual(g['a'], 42) - - def test_functions_calling_each_other(self): - g = {} - self.safe_exec(textwrap.dedent("""\ - def f(): - return 1723 - def g(): - return f() - x = g() - """), g) - self.assertEqual(g['x'], 1723) - - def test_printing_stuff_when_you_shouldnt(self): - g = {} - self.safe_exec("a = 17; print 'hi!'", g) - self.assertEqual(g['a'], 17) - - def test_importing_lots_of_crap(self): - g = {} - self.safe_exec(textwrap.dedent("""\ - from numpy import * - a = 1723 - """), g) - self.assertEqual(g['a'], 1723) - - -class TestSafeExec(SafeExecTests, unittest.TestCase): - """Run SafeExecTests, with the real safe_exec.""" - def safe_exec(self, *args, **kwargs): - safe_exec(*args, **kwargs) - - -class TestNotSafeExec(SafeExecTests, unittest.TestCase): - """Run SafeExecTests, with not_safe_exec.""" - def setUp(self): - # If safe_exec is actually an alias to not_safe_exec, then there's no - # point running these tests. - if safe_exec is not_safe_exec: - raise SkipTest - - def safe_exec(self, *args, **kwargs): - not_safe_exec(*args, **kwargs) diff --git a/common/lib/codejail/codejail/util.py b/common/lib/codejail/codejail/util.py deleted file mode 100644 index d88a05b4dd..0000000000 --- a/common/lib/codejail/codejail/util.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Helpers for codejail.""" - -import contextlib -import os -import shutil -import tempfile - - -class TempDirectory(object): - def __init__(self, delete_when_done=True): - self.delete_when_done = delete_when_done - self.temp_dir = tempfile.mkdtemp(prefix="codejail-") - # Make directory readable by other users ('sandbox' user needs to be able to read it) - os.chmod(self.temp_dir, 0775) - - def clean_up(self): - if self.delete_when_done: - # if this errors, something is genuinely wrong, so don't ignore errors. - shutil.rmtree(self.temp_dir) - - -@contextlib.contextmanager -def temp_directory(delete_when_done=True): - """ - A context manager to make and use a temp directory. If `delete_when_done` - is true (the default), the directory will be removed when done. - """ - tmp = TempDirectory(delete_when_done) - try: - yield tmp.temp_dir - finally: - tmp.clean_up() - - -class ChangeDirectory(object): - def __init__(self, new_dir): - self.old_dir = os.getcwd() - os.chdir(new_dir) - - def clean_up(self): - os.chdir(self.old_dir) - - -@contextlib.contextmanager -def change_directory(new_dir): - """ - A context manager to change the directory, and then change it back. - """ - cd = ChangeDirectory(new_dir) - try: - yield new_dir - finally: - cd.clean_up() diff --git a/common/lib/codejail/setup.py b/common/lib/codejail/setup.py deleted file mode 100644 index c4dcf2b0f7..0000000000 --- a/common/lib/codejail/setup.py +++ /dev/null @@ -1,7 +0,0 @@ -from setuptools import setup - -setup( - name="codejail", - version="0.1", - packages=['codejail'], -) diff --git a/local-requirements.txt b/local-requirements.txt index 4ad1ef6636..5407398a75 100644 --- a/local-requirements.txt +++ b/local-requirements.txt @@ -2,6 +2,6 @@ -e common/lib/calc -e common/lib/capa -e common/lib/chem --e common/lib/codejail +#-e common/lib/codejail -e common/lib/xmodule -e . From baa6b4e3e4f9e69b3c63706b996c02661ffe6aff Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 30 Apr 2013 11:23:11 -0400 Subject: [PATCH 083/162] The cache key for safe_exec has to be hashed to keep it a reasonable size. --- common/lib/capa/capa/safe_exec/safe_exec.py | 7 ++++++- .../capa/capa/safe_exec/tests/test_safe_exec.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/common/lib/capa/capa/safe_exec/safe_exec.py b/common/lib/capa/capa/safe_exec/safe_exec.py index a07fe359fd..3fd6c215b7 100644 --- a/common/lib/capa/capa/safe_exec/safe_exec.py +++ b/common/lib/capa/capa/safe_exec/safe_exec.py @@ -4,6 +4,8 @@ from codejail.safe_exec import safe_exec as codejail_safe_exec from codejail.safe_exec import json_safe from . import lazymod +import hashlib + # Establish the Python environment for Capa. # Capa assumes float-friendly division always. # The name "random" is a properly-seeded stand-in for the random module. @@ -53,7 +55,10 @@ def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None # Check the cache for a previous result. if cache: canonical_globals = sorted(json_safe(globals_dict).iteritems()) - key = "safe_exec %r %s %r" % (random_seed, code, canonical_globals) + md5er = hashlib.md5() + md5er.update(code) + md5er.update(repr(canonical_globals)) + key = "safe_exec %r %s" % (random_seed, md5er.hexdigest()) cached = cache.get(key) if cached is not None: globals_dict.update(cached) diff --git a/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py b/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py index 37f86383c2..bd960f331c 100644 --- a/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py +++ b/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py @@ -65,9 +65,13 @@ class DictCache(object): self.cache = d def get(self, key): + # Actual cache implementations have limits on key length + assert len(key) <= 250 return self.cache.get(key) def set(self, key, value): + # Actual cache implementations have limits on key length + assert len(key) <= 250 self.cache[key] = value @@ -90,3 +94,13 @@ class TestSafeExecCaching(unittest.TestCase): g = {} safe_exec("a = int(math.pi)", g, cache=DictCache(cache)) self.assertEqual(g['a'], 17) + + def test_cache_large_code_chunk(self): + # Caching used to die on memcache with more than 250 bytes of code. + # Check that it doesn't any more. + code = "a = 0\n" + ("a += 1\n" * 12345) + + g = {} + cache = {} + safe_exec(code, g, cache=DictCache(cache)) + self.assertEqual(g['a'], 12345) From 05021377d481417ac12431ecbfa7b396856d9358 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 30 Apr 2013 11:51:52 -0400 Subject: [PATCH 084/162] Make the correct link to the codejail repo --- github-requirements.txt | 1 + local-requirements.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/github-requirements.txt b/github-requirements.txt index 3b71d228e7..2111d3f597 100644 --- a/github-requirements.txt +++ b/github-requirements.txt @@ -8,3 +8,4 @@ # Our libraries: -e git+https://github.com/edx/XBlock.git@5ce6f70a#egg=XBlock +-e git+https://github.com/edx/codejail.git#egg=codejail diff --git a/local-requirements.txt b/local-requirements.txt index 5407398a75..a72f1f6dea 100644 --- a/local-requirements.txt +++ b/local-requirements.txt @@ -2,6 +2,5 @@ -e common/lib/calc -e common/lib/capa -e common/lib/chem -#-e common/lib/codejail -e common/lib/xmodule -e . From 0b2aedb4fea8d689bfe343cd8f15b426fbc66233 Mon Sep 17 00:00:00 2001 From: Will Daly Date: Mon, 29 Apr 2013 10:47:26 -0400 Subject: [PATCH 085/162] Added datadog monitoring of safe_exec() time --- common/lib/capa/capa/safe_exec/safe_exec.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/lib/capa/capa/safe_exec/safe_exec.py b/common/lib/capa/capa/safe_exec/safe_exec.py index 3fd6c215b7..d8b7f561cc 100644 --- a/common/lib/capa/capa/safe_exec/safe_exec.py +++ b/common/lib/capa/capa/safe_exec/safe_exec.py @@ -3,6 +3,7 @@ from codejail.safe_exec import safe_exec as codejail_safe_exec from codejail.safe_exec import json_safe from . import lazymod +from statsd import statsd import hashlib @@ -46,6 +47,7 @@ for name, modname in ASSUMED_IMPORTS: LAZY_IMPORTS = "".join(LAZY_IMPORTS) +@statsd.timed('capa.safe_exec.time') def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None): """Exec python code safely. From ac660ead3e2400899e3a9680a494d52c38211158 Mon Sep 17 00:00:00 2001 From: Will Daly Date: Fri, 26 Apr 2013 11:11:34 -0400 Subject: [PATCH 086/162] Added load test of CustomResponse --- .../courseware/tests/load_tests/README.md | 4 + .../load_tests/custom_response/README.md | 47 +++++++ .../load_tests/custom_response/config.cfg | 22 ++++ .../custom_response/test_scripts/v_user.py | 115 ++++++++++++++++++ 4 files changed, 188 insertions(+) create mode 100644 lms/djangoapps/courseware/tests/load_tests/README.md create mode 100644 lms/djangoapps/courseware/tests/load_tests/custom_response/README.md create mode 100644 lms/djangoapps/courseware/tests/load_tests/custom_response/config.cfg create mode 100644 lms/djangoapps/courseware/tests/load_tests/custom_response/test_scripts/v_user.py diff --git a/lms/djangoapps/courseware/tests/load_tests/README.md b/lms/djangoapps/courseware/tests/load_tests/README.md new file mode 100644 index 0000000000..09d8797947 --- /dev/null +++ b/lms/djangoapps/courseware/tests/load_tests/README.md @@ -0,0 +1,4 @@ +# Load Testing + +Scripts for load testing the courseware app, +mostly using [multimechanize](http://testutils.org/multi-mechanize/) diff --git a/lms/djangoapps/courseware/tests/load_tests/custom_response/README.md b/lms/djangoapps/courseware/tests/load_tests/custom_response/README.md new file mode 100644 index 0000000000..2120b06b7e --- /dev/null +++ b/lms/djangoapps/courseware/tests/load_tests/custom_response/README.md @@ -0,0 +1,47 @@ +# Custom Response Load Test + +## Optional Installations + +* [memcached](http://pypi.python.org/pypi/python-memcached/): Install this +and make sure it is running, or the Capa problem will not cache results. + +* [AppArmor](http://wiki.apparmor.net): Follow the instructions in +`common/lib/codejail/README` to set up the Python sandbox environment. +If you do not set up the sandbox, the tests will still execute code in the CustomResponse, +so you can still run the tests. + +* [matplotlib](http://matplotlib.org): Multi-mechanize uses this to create graphs. + + +## Running the Tests + +This test simulates student submissions for a custom response problem. + +You can run the test using: + + multimech-run custom_response + +You can configure the parameters in `customresponse/config.cfg`, +and you can change the CustomResponse script and student submissions +in `customresponse/test_scripts/v_user.py`. + +## Components Under Test + +Components under test: + +* Python sandbox (see `common/lib/codejail`), which uses `AppArmor` +* Caching (see `common/lib/capa/capa/safe_exec/`), which uses `memcache` in production + +Components NOT under test: + +* Django views +* `XModule` +* gunicorn + +This allows us to avoid creating courses in mongo, logging in, using CSRF tokens, +and other inconveniences. Instead, we create a capa problem (from the capa package), +pass it Django's memcache backend, and pass the problem student submissions. + +Even though the test uses `capa.capa_problem.LoncapaProblem` directly, +the `capa` should not depend on Django. For this reason, we put the +test in the `courseware` Django app. diff --git a/lms/djangoapps/courseware/tests/load_tests/custom_response/config.cfg b/lms/djangoapps/courseware/tests/load_tests/custom_response/config.cfg new file mode 100644 index 0000000000..c75f02a669 --- /dev/null +++ b/lms/djangoapps/courseware/tests/load_tests/custom_response/config.cfg @@ -0,0 +1,22 @@ + +[global] +run_time = 240 +rampup = 30 +results_ts_interval = 10 +progress_bar = on +console_logging = off +xml_report = off + + +[user_group-1] +threads = 10 +script = v_user.py + +[user_group-2] +threads = 10 +script = v_user.py + +[user_group-3] +threads = 10 +script = v_user.py + diff --git a/lms/djangoapps/courseware/tests/load_tests/custom_response/test_scripts/v_user.py b/lms/djangoapps/courseware/tests/load_tests/custom_response/test_scripts/v_user.py new file mode 100644 index 0000000000..9bfc39e55b --- /dev/null +++ b/lms/djangoapps/courseware/tests/load_tests/custom_response/test_scripts/v_user.py @@ -0,0 +1,115 @@ +""" User script for load testing CustomResponse """ + +from capa.tests.response_xml_factory import CustomResponseXMLFactory +import capa.capa_problem as lcp +from xmodule.x_module import ModuleSystem +import mock +import fs.osfs +import random +import textwrap + +# Use memcache running locally +CACHE_SETTINGS = { + 'default': { + 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', + 'LOCATION': '127.0.0.1:11211' + }, +} + +# Configure settings so Django will let us import its cache wrapper +# Caching is the only part of Django being tested +from django.conf import settings +settings.configure(CACHES=CACHE_SETTINGS) + +from django.core.cache import cache + +# Script to install as the checker for the CustomResponse +TEST_SCRIPT = textwrap.dedent(""" + def check_func(expect, answer_given): + return {'ok': answer_given == expect, 'msg': 'Message text'} +""") + +# Submissions submitted by the student +TEST_SUBMISSIONS = [random.randint(-100, 100) for i in range(100)] + +class TestContext(object): + """ One-time set up for the test that is shared across transactions. + Uses a Singleton design pattern.""" + + SINGLETON = None + NUM_UNIQUE_SEEDS = 20 + + @classmethod + def singleton(cls): + """ Return the singleton, creating one if it does not already exist.""" + + # If we haven't created the singleton yet, create it now + if cls.SINGLETON is None: + + # Create a mock ModuleSystem, installing our cache + system = mock.MagicMock(ModuleSystem) + system.render_template = lambda template, context: "
%s
" % template + system.cache = cache + system.filestore = mock.MagicMock(fs.osfs.OSFS) + system.filestore.root_path = "" + system.DEBUG = True + + # Create a custom response problem + xml_factory = CustomResponseXMLFactory() + xml = xml_factory.build_xml(script=TEST_SCRIPT, cfn="check_func", expect="42") + + # Create and store the context + cls.SINGLETON = cls(system, xml) + + else: + pass + + # Return the singleton + return cls.SINGLETON + + def __init__(self, system, xml): + """ Store context needed for the test across transactions """ + self.system = system + self.xml = xml + + # Construct a small pool of unique seeds + # To keep our implementation in line with the one capa actually uses, + # construct the problems, then use the seeds they generate + self.seeds = [lcp.LoncapaProblem(self.xml, 'problem_id', system=self.system).seed + for i in range(self.NUM_UNIQUE_SEEDS)] + + def random_seed(self): + """ Return one of a small number of unique random seeds """ + return random.choice(self.seeds) + + def student_submission(self): + """ Return one of a small number of student submissions """ + return random.choice(TEST_SUBMISSIONS) + +class Transaction(object): + """ User script that submits a response to a CustomResponse problem """ + + def __init__(self): + """ Create the problem """ + + # Get the context (re-used across transactions) + self.context = TestContext.singleton() + + # Create a new custom response problem + # using one of a small number of unique seeds + # We're assuming that the capa module is limiting the number + # of seeds (currently not the case for certain settings) + self.problem = lcp.LoncapaProblem(self.context.xml, + '1', + state=None, + seed=self.context.random_seed(), + system=self.context.system) + + def run(self): + """ Submit a response to the CustomResponse problem """ + answers = {'1_2_1': self.context.student_submission()} + self.problem.grade_answers(answers) + +if __name__ == '__main__': + trans = Transaction() + trans.run() From 7b26c50e3208a4a711d6a5bce47b2f21e49f86c1 Mon Sep 17 00:00:00 2001 From: Will Daly Date: Fri, 26 Apr 2013 11:14:19 -0400 Subject: [PATCH 087/162] Added instructions for clearing the cache before running tests --- .../courseware/tests/load_tests/custom_response/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lms/djangoapps/courseware/tests/load_tests/custom_response/README.md b/lms/djangoapps/courseware/tests/load_tests/custom_response/README.md index 2120b06b7e..e3fae8c817 100644 --- a/lms/djangoapps/courseware/tests/load_tests/custom_response/README.md +++ b/lms/djangoapps/courseware/tests/load_tests/custom_response/README.md @@ -17,7 +17,11 @@ so you can still run the tests. This test simulates student submissions for a custom response problem. -You can run the test using: +First, clear the cache: + + /etc/init.d/memcached restart + +Then, run the test: multimech-run custom_response From 477fe670dd1b30ae97de3258cc8710255941c4a4 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 1 May 2013 10:08:37 -0400 Subject: [PATCH 088/162] All re-randomization has to be bucketed to get a reasonable cache hit rate. --- common/lib/capa/capa/capa_problem.py | 14 +++---- common/lib/capa/capa/tests/__init__.py | 8 +++- .../lib/capa/capa/tests/test_html_render.py | 19 +++++----- .../lib/capa/capa/tests/test_responsetypes.py | 5 +-- common/lib/xmodule/xmodule/capa_module.py | 38 +++++++++++++------ .../xmodule/xmodule/tests/test_capa_module.py | 14 ++++++- 6 files changed, 61 insertions(+), 37 deletions(-) diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index 45200b8607..1c0189d9aa 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -17,9 +17,8 @@ from datetime import datetime import logging import math import numpy -import os, os.path +import os.path import re -import struct import sys from lxml import etree @@ -73,7 +72,7 @@ class LoncapaProblem(object): - problem_text (string): xml defining the problem - id (string): identifier for this problem; often a filename (no spaces) - - seed (int): random number generator seed (int) + - seed (int): random number generator seed (int) - state (dict): containing the following keys: - 'seed' - (int) random number generator seed - 'student_answers' - (dict) maps input id to the stored answer for that input @@ -92,23 +91,20 @@ class LoncapaProblem(object): if self.system is None: raise Exception() - state = state if state else {} + state = state or {} # Set seed according to the following priority: # 1. Contained in problem's state # 2. Passed into capa_problem via constructor - # 3. Assign from the OS's random number generator self.seed = state.get('seed', seed) - if self.seed is None: - self.seed = struct.unpack('i', os.urandom(4))[0] + assert self.seed is not None, "Seed must be provided for LoncapaProblem." + self.student_answers = state.get('student_answers', {}) if 'correct_map' in state: self.correct_map.set_dict(state['correct_map']) self.done = state.get('done', False) self.input_state = state.get('input_state', {}) - - # Convert startouttext and endouttext to proper problem_text = re.sub("startouttext\s*/", "text", problem_text) problem_text = re.sub("endouttext\s*/", "/text", problem_text) diff --git a/common/lib/capa/capa/tests/__init__.py b/common/lib/capa/capa/tests/__init__.py index 59c87780b5..1997e4d055 100644 --- a/common/lib/capa/capa/tests/__init__.py +++ b/common/lib/capa/capa/tests/__init__.py @@ -1,7 +1,7 @@ -import fs import fs.osfs -import os +import os, os.path +from capa.capa_problem import LoncapaProblem from mock import Mock, MagicMock import xml.sax.saxutils as saxutils @@ -36,3 +36,7 @@ test_system = Mock( anonymous_student_id='student', cache=None, ) + +def new_loncapa_problem(xml): + """Construct a `LoncapaProblem` suitable for unit tests.""" + return LoncapaProblem(xml, id='1', seed=723, system=test_system) diff --git a/common/lib/capa/capa/tests/test_html_render.py b/common/lib/capa/capa/tests/test_html_render.py index 492fcb2743..e0edf344f2 100644 --- a/common/lib/capa/capa/tests/test_html_render.py +++ b/common/lib/capa/capa/tests/test_html_render.py @@ -6,9 +6,8 @@ import json import mock -from capa.capa_problem import LoncapaProblem from .response_xml_factory import StringResponseXMLFactory, CustomResponseXMLFactory -from . import test_system +from . import test_system, new_loncapa_problem class CapaHtmlRenderTest(unittest.TestCase): @@ -20,7 +19,7 @@ class CapaHtmlRenderTest(unittest.TestCase): xml_str = " " # Create the problem - problem = LoncapaProblem(xml_str, '1', system=test_system) + problem = new_loncapa_problem(xml_str) # Render the HTML rendered_html = etree.XML(problem.get_html()) @@ -39,7 +38,7 @@ class CapaHtmlRenderTest(unittest.TestCase): """) # Create the problem - problem = LoncapaProblem(xml_str, '1', system=test_system) + problem = new_loncapa_problem(xml_str) # Render the HTML rendered_html = etree.XML(problem.get_html()) @@ -61,7 +60,7 @@ class CapaHtmlRenderTest(unittest.TestCase): """) # Create the problem - problem = LoncapaProblem(xml_str, '1', system=test_system) + problem = new_loncapa_problem(xml_str) # Render the HTML rendered_html = etree.XML(problem.get_html()) @@ -80,7 +79,7 @@ class CapaHtmlRenderTest(unittest.TestCase): """) # Create the problem - problem = LoncapaProblem(xml_str, '1', system=test_system) + problem = new_loncapa_problem(xml_str) # Render the HTML rendered_html = etree.XML(problem.get_html()) @@ -98,7 +97,7 @@ class CapaHtmlRenderTest(unittest.TestCase): """) # Create the problem - problem = LoncapaProblem(xml_str, '1', system=test_system) + problem = new_loncapa_problem(xml_str) # Render the HTML rendered_html = etree.XML(problem.get_html()) @@ -121,7 +120,7 @@ class CapaHtmlRenderTest(unittest.TestCase): test_system.render_template.return_value = "
Input Template Render
" # Create the problem and render the HTML - problem = LoncapaProblem(xml_str, '1', system=test_system) + problem = new_loncapa_problem(xml_str) rendered_html = etree.XML(problem.get_html()) # Expect problem has been turned into a
@@ -184,7 +183,7 @@ class CapaHtmlRenderTest(unittest.TestCase): xml_str = CustomResponseXMLFactory().build_xml(**kwargs) # Create the problem and render the html - problem = LoncapaProblem(xml_str, '1', system=test_system) + problem = new_loncapa_problem(xml_str) # Grade the problem correctmap = problem.grade_answers({'1_2_1': 'test'}) @@ -219,7 +218,7 @@ class CapaHtmlRenderTest(unittest.TestCase): """) # Create the problem and render the HTML - problem = LoncapaProblem(xml_str, '1', system=test_system) + problem = new_loncapa_problem(xml_str) rendered_html = etree.XML(problem.get_html()) # Expect that the variable $test has been replaced with its value diff --git a/common/lib/capa/capa/tests/test_responsetypes.py b/common/lib/capa/capa/tests/test_responsetypes.py index 8e41ff39de..9cafef23d6 100644 --- a/common/lib/capa/capa/tests/test_responsetypes.py +++ b/common/lib/capa/capa/tests/test_responsetypes.py @@ -12,9 +12,8 @@ import textwrap import mock import textwrap -from . import test_system +from . import new_loncapa_problem -import capa.capa_problem as lcp from capa.responsetypes import LoncapaProblemError, \ StudentInputError, ResponseError from capa.correctmap import CorrectMap @@ -33,7 +32,7 @@ class ResponseTest(unittest.TestCase): def build_problem(self, **kwargs): xml = self.xml_factory.build_xml(**kwargs) - return lcp.LoncapaProblem(xml, '1', system=test_system) + return new_loncapa_problem(xml) def assert_grade(self, problem, submission, expected_correctness, msg=None): input_dict = {'1_2_1': submission} diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py index 4143345196..ec0c3c8619 100644 --- a/common/lib/xmodule/xmodule/capa_module.py +++ b/common/lib/xmodule/xmodule/capa_module.py @@ -3,7 +3,9 @@ import datetime import hashlib import json import logging +import os import traceback +import struct import sys from pkg_resources import resource_string @@ -23,8 +25,10 @@ from xmodule.util.date_utils import time_to_datetime log = logging.getLogger("mitx.courseware") -# Generated this many different variants of problems with rerandomize=per_student +# Generate this many different variants of problems with rerandomize=per_student NUM_RANDOMIZATION_BINS = 20 +# Never produce more than this many different seeds, no matter what. +MAX_RANDOMIZATION_BINS = 1000 def randomization_bin(seed, problem_id): @@ -108,11 +112,7 @@ class CapaModule(CapaFields, XModule): self.close_date = due_date if self.seed is None: - if self.rerandomize == 'never': - self.seed = 1 - elif self.rerandomize == "per_student" and hasattr(self.system, 'seed'): - # see comment on randomization_bin - self.seed = randomization_bin(system.seed, self.location.url) + self.choose_new_seed() # Need the problem location in openendedresponse to send out. Adding # it to the system here seems like the least clunky way to get it @@ -156,6 +156,22 @@ class CapaModule(CapaFields, XModule): self.set_state_from_lcp() + assert self.seed is not None + + def choose_new_seed(self): + """Choose a new seed.""" + if self.rerandomize == 'never': + self.seed = 1 + elif self.rerandomize == "per_student" and hasattr(self.system, 'seed'): + # see comment on randomization_bin + self.seed = randomization_bin(self.system.seed, self.location.url) + else: + self.seed = struct.unpack('i', os.urandom(4))[0] + + # So that sandboxed code execution can be cached, but still have an interesting + # number of possibilities, cap the number of different random seeds. + self.seed %= MAX_RANDOMIZATION_BINS + def new_lcp(self, state, text=None): if text is None: text = self.data @@ -164,6 +180,7 @@ class CapaModule(CapaFields, XModule): problem_text=text, id=self.location.html_id(), state=state, + seed=self.seed, system=self.system, ) @@ -831,14 +848,11 @@ class CapaModule(CapaFields, XModule): 'error': "Refresh the page and make an attempt before resetting."} if self.rerandomize in ["always", "onreset"]: - # reset random number generator seed (note the self.lcp.get_state() - # in next line) - seed = None - else: - seed = self.lcp.seed + # Reset random number generator seed. + self.choose_new_seed() # Generate a new problem with either the previous seed or a new seed - self.lcp = self.new_lcp({'seed': seed}) + self.lcp = self.new_lcp(None) # Pull in the new problem seed self.set_state_from_lcp() diff --git a/common/lib/xmodule/xmodule/tests/test_capa_module.py b/common/lib/xmodule/xmodule/tests/test_capa_module.py index f948f5bdfe..61de21b129 100644 --- a/common/lib/xmodule/xmodule/tests/test_capa_module.py +++ b/common/lib/xmodule/xmodule/tests/test_capa_module.py @@ -550,6 +550,7 @@ class CapaModuleTest(unittest.TestCase): def test_reset_problem(self): module = CapaFactory.create(done=True) module.new_lcp = Mock(wraps=module.new_lcp) + module.choose_new_seed = Mock(wraps=module.choose_new_seed) # Stub out HTML rendering with patch('xmodule.capa_module.CapaModule.get_problem_html') as mock_html: @@ -567,7 +568,8 @@ class CapaModuleTest(unittest.TestCase): self.assertEqual(result['html'], "
Test HTML
") # Expect that the problem was reset - module.new_lcp.assert_called_once_with({'seed': None}) + module.new_lcp.assert_called_once_with(None) + module.choose_new_seed.assert_called_once_with() def test_reset_problem_closed(self): module = CapaFactory.create() @@ -1033,3 +1035,13 @@ class CapaModuleTest(unittest.TestCase): self.assertTrue(module.seed is not None) msg = 'Could not get a new seed from reset after 5 tries' self.assertTrue(success, msg) + + def test_random_seed_bins(self): + # Assert that we are limiting the number of possible seeds. + + # Check the conditions that generate random seeds + for rerandomize in ['always', 'per_student', 'true', 'onreset']: + # Get a bunch of seeds, they should all be in 0-999. + for i in range(200): + module = CapaFactory.create(rerandomize=rerandomize) + assert 0 <= module.seed < 1000 From 0ba4b680f9ece6e9a8986ca5837176079b2cf14c Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 1 May 2013 16:03:28 -0400 Subject: [PATCH 089/162] Minor fixes of test_system in xmodule tests --- common/lib/xmodule/xmodule/tests/test_progress.py | 2 +- common/lib/xmodule/xmodule/tests/test_randomize_module.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/common/lib/xmodule/xmodule/tests/test_progress.py b/common/lib/xmodule/xmodule/tests/test_progress.py index 0114ba4ad3..4bb663ad85 100644 --- a/common/lib/xmodule/xmodule/tests/test_progress.py +++ b/common/lib/xmodule/xmodule/tests/test_progress.py @@ -134,6 +134,6 @@ class ModuleProgressTest(unittest.TestCase): ''' def test_xmodule_default(self): '''Make sure default get_progress exists, returns None''' - xm = x_module.XModule(test_system, 'a://b/c/d/e', None, {}) + xm = x_module.XModule(test_system(), 'a://b/c/d/e', None, {}) p = xm.get_progress() self.assertEqual(p, None) diff --git a/common/lib/xmodule/xmodule/tests/test_randomize_module.py b/common/lib/xmodule/xmodule/tests/test_randomize_module.py index 59cf5a59f3..81935c4013 100644 --- a/common/lib/xmodule/xmodule/tests/test_randomize_module.py +++ b/common/lib/xmodule/xmodule/tests/test_randomize_module.py @@ -14,7 +14,6 @@ START = '2013-01-01T01:00:00' from .test_course_module import DummySystem as DummyImportSystem -from . import test_system class RandomizeModuleTestCase(unittest.TestCase): From f4d84e67e1ab153e4238d648ffc6f57ba9c75487 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 2 May 2013 10:15:23 -0400 Subject: [PATCH 090/162] Build the XModuleSystem anew for each test so we can fiddle with it safely. --- common/lib/capa/capa/tests/__init__.py | 38 ++++++++------- .../lib/capa/capa/tests/test_customrender.py | 8 ++-- .../lib/capa/capa/tests/test_html_render.py | 20 ++++---- common/lib/capa/capa/tests/test_inputtypes.py | 47 +++++++++---------- common/lib/xmodule/xmodule/tests/__init__.py | 13 +++-- 5 files changed, 66 insertions(+), 60 deletions(-) diff --git a/common/lib/capa/capa/tests/__init__.py b/common/lib/capa/capa/tests/__init__.py index 1997e4d055..d989405951 100644 --- a/common/lib/capa/capa/tests/__init__.py +++ b/common/lib/capa/capa/tests/__init__.py @@ -22,21 +22,27 @@ def calledback_url(dispatch = 'score_update'): xqueue_interface = MagicMock() xqueue_interface.send_to_queue.return_value = (0, 'Success!') -test_system = Mock( - ajax_url='courses/course_id/modx/a_location', - track_function=Mock(), - get_module=Mock(), - render_template=tst_render_template, - replace_urls=Mock(), - user=Mock(), - filestore=fs.osfs.OSFS(os.path.join(TEST_DIR, "test_files")), - debug=True, - xqueue={'interface': xqueue_interface, 'construct_callback': calledback_url, 'default_queuename': 'testqueue', 'waittime': 10}, - node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules"), - anonymous_student_id='student', - cache=None, -) +def test_system(): + """ + Construct a mock ModuleSystem instance. -def new_loncapa_problem(xml): + """ + the_system = Mock( + ajax_url='courses/course_id/modx/a_location', + track_function=Mock(), + get_module=Mock(), + render_template=tst_render_template, + replace_urls=Mock(), + user=Mock(), + filestore=fs.osfs.OSFS(os.path.join(TEST_DIR, "test_files")), + debug=True, + xqueue={'interface': xqueue_interface, 'construct_callback': calledback_url, 'default_queuename': 'testqueue', 'waittime': 10}, + node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules"), + anonymous_student_id='student', + cache=None, + ) + return the_system + +def new_loncapa_problem(xml, system=None): """Construct a `LoncapaProblem` suitable for unit tests.""" - return LoncapaProblem(xml, id='1', seed=723, system=test_system) + return LoncapaProblem(xml, id='1', seed=723, system=system or test_system()) diff --git a/common/lib/capa/capa/tests/test_customrender.py b/common/lib/capa/capa/tests/test_customrender.py index eece275b05..8012804a40 100644 --- a/common/lib/capa/capa/tests/test_customrender.py +++ b/common/lib/capa/capa/tests/test_customrender.py @@ -26,7 +26,7 @@ class HelperTest(unittest.TestCase): Make sure that our helper function works! ''' def check(self, d): - xml = etree.XML(test_system.render_template('blah', d)) + xml = etree.XML(test_system().render_template('blah', d)) self.assertEqual(d, extract_context(xml)) def test_extract_context(self): @@ -46,11 +46,11 @@ class SolutionRenderTest(unittest.TestCase): xml_str = """{s}""".format(s=solution) element = etree.fromstring(xml_str) - renderer = lookup_tag('solution')(test_system, element) + renderer = lookup_tag('solution')(test_system(), element) self.assertEqual(renderer.id, 'solution_12') - # our test_system "renders" templates to a div with the repr of the context + # Our test_system "renders" templates to a div with the repr of the context. xml = renderer.get_html() context = extract_context(xml) self.assertEqual(context, {'id': 'solution_12'}) @@ -65,7 +65,7 @@ class MathRenderTest(unittest.TestCase): xml_str = """{tex}""".format(tex=latex_in) element = etree.fromstring(xml_str) - renderer = lookup_tag('math')(test_system, element) + renderer = lookup_tag('math')(test_system(), element) self.assertEqual(renderer.mathstr, mathjax_out) diff --git a/common/lib/capa/capa/tests/test_html_render.py b/common/lib/capa/capa/tests/test_html_render.py index e0edf344f2..62605b48f5 100644 --- a/common/lib/capa/capa/tests/test_html_render.py +++ b/common/lib/capa/capa/tests/test_html_render.py @@ -11,6 +11,10 @@ from . import test_system, new_loncapa_problem class CapaHtmlRenderTest(unittest.TestCase): + def setUp(self): + super(CapaHtmlRenderTest, self).setUp() + self.system = test_system() + def test_blank_problem(self): """ It's important that blank problems don't break, since that's @@ -38,7 +42,7 @@ class CapaHtmlRenderTest(unittest.TestCase): """) # Create the problem - problem = new_loncapa_problem(xml_str) + problem = new_loncapa_problem(xml_str, system=self.system) # Render the HTML rendered_html = etree.XML(problem.get_html()) @@ -48,9 +52,6 @@ class CapaHtmlRenderTest(unittest.TestCase): self.assertEqual(test_element.tag, "test") self.assertEqual(test_element.text, "Test include") - - - def test_process_outtext(self): # Generate some XML with and xml_str = textwrap.dedent(""" @@ -116,11 +117,12 @@ class CapaHtmlRenderTest(unittest.TestCase): xml_str = StringResponseXMLFactory().build_xml(**kwargs) # Mock out the template renderer - test_system.render_template = mock.Mock() - test_system.render_template.return_value = "
Input Template Render
" + the_system = test_system() + the_system.render_template = mock.Mock() + the_system.render_template.return_value = "
Input Template Render
" # Create the problem and render the HTML - problem = new_loncapa_problem(xml_str) + problem = new_loncapa_problem(xml_str, system=the_system) rendered_html = etree.XML(problem.get_html()) # Expect problem has been turned into a
@@ -165,7 +167,7 @@ class CapaHtmlRenderTest(unittest.TestCase): mock.call('textline.html', expected_textline_context), mock.call('solutionspan.html', expected_solution_context)] - self.assertEqual(test_system.render_template.call_args_list, + self.assertEqual(the_system.render_template.call_args_list, expected_calls) @@ -226,7 +228,7 @@ class CapaHtmlRenderTest(unittest.TestCase): self.assertEqual(span_element.get('attr'), "TEST") def _create_test_file(self, path, content_str): - test_fp = test_system.filestore.open(path, "w") + test_fp = self.system.filestore.open(path, "w") test_fp.write(content_str) test_fp.close() diff --git a/common/lib/capa/capa/tests/test_inputtypes.py b/common/lib/capa/capa/tests/test_inputtypes.py index 54edb5bf9f..313eb28249 100644 --- a/common/lib/capa/capa/tests/test_inputtypes.py +++ b/common/lib/capa/capa/tests/test_inputtypes.py @@ -45,7 +45,7 @@ class OptionInputTest(unittest.TestCase): state = {'value': 'Down', 'id': 'sky_input', 'status': 'answered'} - option_input = lookup_tag('optioninput')(test_system, element, state) + option_input = lookup_tag('optioninput')(test_system(), element, state) context = option_input._get_render_context() @@ -92,7 +92,7 @@ class ChoiceGroupTest(unittest.TestCase): 'id': 'sky_input', 'status': 'answered'} - the_input = lookup_tag(tag)(test_system, element, state) + the_input = lookup_tag(tag)(test_system(), element, state) context = the_input._get_render_context() @@ -142,7 +142,7 @@ class JavascriptInputTest(unittest.TestCase): element = etree.fromstring(xml_str) state = {'value': '3', } - the_input = lookup_tag('javascriptinput')(test_system, element, state) + the_input = lookup_tag('javascriptinput')(test_system(), element, state) context = the_input._get_render_context() @@ -170,7 +170,7 @@ class TextLineTest(unittest.TestCase): element = etree.fromstring(xml_str) state = {'value': 'BumbleBee', } - the_input = lookup_tag('textline')(test_system, element, state) + the_input = lookup_tag('textline')(test_system(), element, state) context = the_input._get_render_context() @@ -198,7 +198,7 @@ class TextLineTest(unittest.TestCase): element = etree.fromstring(xml_str) state = {'value': 'BumbleBee', } - the_input = lookup_tag('textline')(test_system, element, state) + the_input = lookup_tag('textline')(test_system(), element, state) context = the_input._get_render_context() @@ -236,7 +236,7 @@ class TextLineTest(unittest.TestCase): element = etree.fromstring(xml_str) state = {'value': 'BumbleBee', } - the_input = lookup_tag('textline')(test_system, element, state) + the_input = lookup_tag('textline')(test_system(), element, state) context = the_input._get_render_context() @@ -274,7 +274,7 @@ class FileSubmissionTest(unittest.TestCase): 'status': 'incomplete', 'feedback': {'message': '3'}, } input_class = lookup_tag('filesubmission') - the_input = input_class(test_system, element, state) + the_input = input_class(test_system(), element, state) context = the_input._get_render_context() @@ -319,7 +319,7 @@ class CodeInputTest(unittest.TestCase): 'feedback': {'message': '3'}, } input_class = lookup_tag('codeinput') - the_input = input_class(test_system, element, state) + the_input = input_class(test_system(), element, state) context = the_input._get_render_context() @@ -368,7 +368,7 @@ class MatlabTest(unittest.TestCase): 'feedback': {'message': '3'}, } self.input_class = lookup_tag('matlabinput') - self.the_input = self.input_class(test_system, elt, state) + self.the_input = self.input_class(test_system(), elt, state) def test_rendering(self): context = self.the_input._get_render_context() @@ -396,7 +396,7 @@ class MatlabTest(unittest.TestCase): 'feedback': {'message': '3'}, } elt = etree.fromstring(self.xml) - the_input = self.input_class(test_system, elt, state) + the_input = self.input_class(test_system(), elt, state) context = the_input._get_render_context() expected = {'id': 'prob_1_2', @@ -423,7 +423,7 @@ class MatlabTest(unittest.TestCase): } elt = etree.fromstring(self.xml) - the_input = self.input_class(test_system, elt, state) + the_input = self.input_class(test_system(), elt, state) context = the_input._get_render_context() expected = {'id': 'prob_1_2', 'value': 'print "good evening"', @@ -448,7 +448,7 @@ class MatlabTest(unittest.TestCase): } elt = etree.fromstring(self.xml) - the_input = self.input_class(test_system, elt, state) + the_input = self.input_class(test_system(), elt, state) context = the_input._get_render_context() expected = {'id': 'prob_1_2', 'value': 'print "good evening"', @@ -470,7 +470,7 @@ class MatlabTest(unittest.TestCase): get = {'submission': 'x = 1234;'} response = self.the_input.handle_ajax("plot", get) - test_system.xqueue['interface'].send_to_queue.assert_called_with(header=ANY, body=ANY) + test_system().xqueue['interface'].send_to_queue.assert_called_with(header=ANY, body=ANY) self.assertTrue(response['success']) self.assertTrue(self.the_input.input_state['queuekey'] is not None) @@ -479,13 +479,12 @@ class MatlabTest(unittest.TestCase): def test_plot_data_failure(self): get = {'submission': 'x = 1234;'} error_message = 'Error message!' - test_system.xqueue['interface'].send_to_queue.return_value = (1, error_message) + test_system().xqueue['interface'].send_to_queue.return_value = (1, error_message) response = self.the_input.handle_ajax("plot", get) self.assertFalse(response['success']) self.assertEqual(response['message'], error_message) self.assertTrue('queuekey' not in self.the_input.input_state) self.assertTrue('queuestate' not in self.the_input.input_state) - test_system.xqueue['interface'].send_to_queue.return_value = (0, 'Success!') def test_ungraded_response_success(self): queuekey = 'abcd' @@ -496,7 +495,7 @@ class MatlabTest(unittest.TestCase): 'feedback': {'message': '3'}, } elt = etree.fromstring(self.xml) - the_input = self.input_class(test_system, elt, state) + the_input = self.input_class(test_system(), elt, state) inner_msg = 'hello!' queue_msg = json.dumps({'msg': inner_msg}) @@ -514,7 +513,7 @@ class MatlabTest(unittest.TestCase): 'feedback': {'message': '3'}, } elt = etree.fromstring(self.xml) - the_input = self.input_class(test_system, elt, state) + the_input = self.input_class(test_system(), elt, state) inner_msg = 'hello!' queue_msg = json.dumps({'msg': inner_msg}) @@ -553,7 +552,7 @@ class SchematicTest(unittest.TestCase): state = {'value': value, 'status': 'unsubmitted'} - the_input = lookup_tag('schematic')(test_system, element, state) + the_input = lookup_tag('schematic')(test_system(), element, state) context = the_input._get_render_context() @@ -592,7 +591,7 @@ class ImageInputTest(unittest.TestCase): state = {'value': value, 'status': 'unsubmitted'} - the_input = lookup_tag('imageinput')(test_system, element, state) + the_input = lookup_tag('imageinput')(test_system(), element, state) context = the_input._get_render_context() @@ -643,7 +642,7 @@ class CrystallographyTest(unittest.TestCase): state = {'value': value, 'status': 'unsubmitted'} - the_input = lookup_tag('crystallography')(test_system, element, state) + the_input = lookup_tag('crystallography')(test_system(), element, state) context = the_input._get_render_context() @@ -681,7 +680,7 @@ class VseprTest(unittest.TestCase): state = {'value': value, 'status': 'unsubmitted'} - the_input = lookup_tag('vsepr_input')(test_system, element, state) + the_input = lookup_tag('vsepr_input')(test_system(), element, state) context = the_input._get_render_context() @@ -708,7 +707,7 @@ class ChemicalEquationTest(unittest.TestCase): element = etree.fromstring(xml_str) state = {'value': 'H2OYeah', } - self.the_input = lookup_tag('chemicalequationinput')(test_system, element, state) + self.the_input = lookup_tag('chemicalequationinput')(test_system(), element, state) def test_rendering(self): ''' Verify that the render context matches the expected render context''' @@ -783,7 +782,7 @@ class DragAndDropTest(unittest.TestCase): ] } - the_input = lookup_tag('drag_and_drop_input')(test_system, element, state) + the_input = lookup_tag('drag_and_drop_input')(test_system(), element, state) context = the_input._get_render_context() expected = {'id': 'prob_1_2', @@ -832,7 +831,7 @@ class AnnotationInputTest(unittest.TestCase): tag = 'annotationinput' - the_input = lookup_tag(tag)(test_system, element, state) + the_input = lookup_tag(tag)(test_system(), element, state) context = the_input._get_render_context() diff --git a/common/lib/xmodule/xmodule/tests/__init__.py b/common/lib/xmodule/xmodule/tests/__init__.py index 79e1fec0f0..bcf333d7d9 100644 --- a/common/lib/xmodule/xmodule/tests/__init__.py +++ b/common/lib/xmodule/xmodule/tests/__init__.py @@ -33,15 +33,14 @@ def test_system(): """ Construct a test ModuleSystem instance. - By default, the render_template() method simply returns - the context it is passed as a string. - You can override this behavior by monkey patching: + By default, the render_template() method simply returns the context it is + passed as a string. You can override this behavior by monkey patching:: - system = test_system() - system.render_template = my_render_func + system = test_system() + system.render_template = my_render_func + + where `my_render_func` is a function of the form my_render_func(template, context). - where my_render_func is a function of the form - my_render_func(template, context) """ return ModuleSystem( ajax_url='courses/course_id/modx/a_location', From f1fac732cf6399484a24b63068e65d86ac2c086c Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 2 May 2013 10:50:56 -0400 Subject: [PATCH 091/162] A new boolean on XModuleSystem that determines whether to allow execution of untrusted unsandboxed code. --- common/lib/capa/capa/responsetypes.py | 4 ++ common/lib/capa/capa/tests/__init__.py | 1 + .../lib/capa/capa/tests/test_responsetypes.py | 37 +++++++++++++++---- common/lib/xmodule/xmodule/x_module.py | 5 +++ 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index de73dcda30..f7d285887e 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -502,6 +502,10 @@ class JavascriptResponse(LoncapaResponse): return tmp_env def call_node(self, args): + # Node.js code is un-sandboxed. If the XModuleSystem says we aren't + # allowed to run unsafe code, then stop now. + if not self.system.can_execute_unsafe_code: + raise LoncapaProblemError("Execution of unsafe Javascript code is not allowed.") subprocess_args = ["node"] subprocess_args.extend(args) diff --git a/common/lib/capa/capa/tests/__init__.py b/common/lib/capa/capa/tests/__init__.py index d989405951..da3a3524f4 100644 --- a/common/lib/capa/capa/tests/__init__.py +++ b/common/lib/capa/capa/tests/__init__.py @@ -40,6 +40,7 @@ def test_system(): node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules"), anonymous_student_id='student', cache=None, + can_execute_unsafe_code=False, ) return the_system diff --git a/common/lib/capa/capa/tests/test_responsetypes.py b/common/lib/capa/capa/tests/test_responsetypes.py index 9cafef23d6..af6bdf823c 100644 --- a/common/lib/capa/capa/tests/test_responsetypes.py +++ b/common/lib/capa/capa/tests/test_responsetypes.py @@ -12,7 +12,7 @@ import textwrap import mock import textwrap -from . import new_loncapa_problem +from . import new_loncapa_problem, test_system from capa.responsetypes import LoncapaProblemError, \ StudentInputError, ResponseError @@ -30,9 +30,9 @@ class ResponseTest(unittest.TestCase): if self.xml_factory_class: self.xml_factory = self.xml_factory_class() - def build_problem(self, **kwargs): + def build_problem(self, system=None, **kwargs): xml = self.xml_factory.build_xml(**kwargs) - return new_loncapa_problem(xml) + return new_loncapa_problem(xml, system=system) def assert_grade(self, problem, submission, expected_correctness, msg=None): input_dict = {'1_2_1': submission} @@ -746,16 +746,37 @@ class JavascriptResponseTest(ResponseTest): coffee_file_path = os.path.dirname(__file__) + "/test_files/js/*.coffee" os.system("coffee -c %s" % (coffee_file_path)) - problem = self.build_problem(generator_src="test_problem_generator.js", - grader_src="test_problem_grader.js", - display_class="TestProblemDisplay", - display_src="test_problem_display.js", - param_dict={'value': '4'}) + system = test_system() + system.can_execute_unsafe_code = True + problem = self.build_problem( + system=system, + generator_src="test_problem_generator.js", + grader_src="test_problem_grader.js", + display_class="TestProblemDisplay", + display_src="test_problem_display.js", + param_dict={'value': '4'}, + ) # Test that we get graded correctly self.assert_grade(problem, json.dumps({0: 4}), "correct") self.assert_grade(problem, json.dumps({0: 5}), "incorrect") + def test_cant_execute_javascript(self): + # If the system says to disallow unsafe code execution, then making + # this problem will raise an exception. + system = test_system() + system.can_execute_unsafe_code = False + + with self.assertRaises(LoncapaProblemError): + problem = self.build_problem( + system=system, + generator_src="test_problem_generator.js", + grader_src="test_problem_grader.js", + display_class="TestProblemDisplay", + display_src="test_problem_display.js", + param_dict={'value': '4'}, + ) + class NumericalResponseTest(ResponseTest): from response_xml_factory import NumericalResponseXMLFactory diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index 31a32eb6dc..268eb4b40d 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -702,6 +702,7 @@ class ModuleSystem(object): open_ended_grading_interface=None, s3_interface=None, cache=None, + can_execute_unsafe_code=False, ): ''' Create a closure around the system environment. @@ -749,6 +750,9 @@ class ModuleSystem(object): .get(key) returns an object from the cache or None. .set(key, value, timeout_secs=None) stores a value in the cache with a timeout. + can_execute_unsafe_code - A boolean, whether or not to allow the execution + of unsafe, unsandboxed code. + ''' self.ajax_url = ajax_url self.xqueue = xqueue @@ -774,6 +778,7 @@ class ModuleSystem(object): self.s3_interface = s3_interface self.cache = cache or DoNothingCache() + self.can_execute_unsafe_code = can_execute_unsafe_code def get(self, attr): ''' provide uniform access to attributes (like etree).''' From d7ea1dafe8a7fbea4d0e0dd0e2a90d5b4e242b2e Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 2 May 2013 10:58:38 -0400 Subject: [PATCH 092/162] On second thought, make can_execute_unsafe_code a function returning a boolean. --- common/lib/capa/capa/responsetypes.py | 2 +- common/lib/capa/capa/tests/__init__.py | 2 +- common/lib/capa/capa/tests/test_responsetypes.py | 4 ++-- common/lib/xmodule/xmodule/x_module.py | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index f7d285887e..1f71c828c3 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -504,7 +504,7 @@ class JavascriptResponse(LoncapaResponse): def call_node(self, args): # Node.js code is un-sandboxed. If the XModuleSystem says we aren't # allowed to run unsafe code, then stop now. - if not self.system.can_execute_unsafe_code: + if not self.system.can_execute_unsafe_code(): raise LoncapaProblemError("Execution of unsafe Javascript code is not allowed.") subprocess_args = ["node"] diff --git a/common/lib/capa/capa/tests/__init__.py b/common/lib/capa/capa/tests/__init__.py index da3a3524f4..ac81ff66c4 100644 --- a/common/lib/capa/capa/tests/__init__.py +++ b/common/lib/capa/capa/tests/__init__.py @@ -40,7 +40,7 @@ def test_system(): node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules"), anonymous_student_id='student', cache=None, - can_execute_unsafe_code=False, + can_execute_unsafe_code=lambda: False, ) return the_system diff --git a/common/lib/capa/capa/tests/test_responsetypes.py b/common/lib/capa/capa/tests/test_responsetypes.py index af6bdf823c..b9c2bfa3ae 100644 --- a/common/lib/capa/capa/tests/test_responsetypes.py +++ b/common/lib/capa/capa/tests/test_responsetypes.py @@ -747,7 +747,7 @@ class JavascriptResponseTest(ResponseTest): os.system("coffee -c %s" % (coffee_file_path)) system = test_system() - system.can_execute_unsafe_code = True + system.can_execute_unsafe_code = lambda: True problem = self.build_problem( system=system, generator_src="test_problem_generator.js", @@ -765,7 +765,7 @@ class JavascriptResponseTest(ResponseTest): # If the system says to disallow unsafe code execution, then making # this problem will raise an exception. system = test_system() - system.can_execute_unsafe_code = False + system.can_execute_unsafe_code = lambda: False with self.assertRaises(LoncapaProblemError): problem = self.build_problem( diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index 268eb4b40d..3a2bdb798e 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -702,7 +702,7 @@ class ModuleSystem(object): open_ended_grading_interface=None, s3_interface=None, cache=None, - can_execute_unsafe_code=False, + can_execute_unsafe_code=None, ): ''' Create a closure around the system environment. @@ -750,8 +750,8 @@ class ModuleSystem(object): .get(key) returns an object from the cache or None. .set(key, value, timeout_secs=None) stores a value in the cache with a timeout. - can_execute_unsafe_code - A boolean, whether or not to allow the execution - of unsafe, unsandboxed code. + can_execute_unsafe_code - A function returning a boolean, whether or + not to allow the execution of unsafe, unsandboxed code. ''' self.ajax_url = ajax_url @@ -778,7 +778,7 @@ class ModuleSystem(object): self.s3_interface = s3_interface self.cache = cache or DoNothingCache() - self.can_execute_unsafe_code = can_execute_unsafe_code + self.can_execute_unsafe_code = can_execute_unsafe_code or (lambda: False) def get(self, attr): ''' provide uniform access to attributes (like etree).''' From d8c22dbeb3895a22eef9aa1b9694d34558e9f897 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 2 May 2013 11:21:38 -0400 Subject: [PATCH 093/162] Add a Django setting for course allowed to run unsafe code. --- lms/djangoapps/courseware/module_render.py | 9 +++++++++ lms/envs/common.py | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index dc71216adb..622800382e 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -274,6 +274,14 @@ def get_module_for_descriptor(user, request, descriptor, model_data_cache, cours statsd.increment("lms.courseware.question_answered", tags=tags) + def can_execute_unsafe_code(): + # To decide if we can run unsafe code, we check the course id against + # a list of regexes configured on the server. + for regex in settings.COURSES_WITH_UNSAFE_CODE: + if re.match(regex, course_id): + return True + return False + # TODO (cpennington): When modules are shared between courses, the static # prefix is going to have to be specific to the module, not the directory # that the xml was loaded from @@ -301,6 +309,7 @@ def get_module_for_descriptor(user, request, descriptor, model_data_cache, cours open_ended_grading_interface=open_ended_grading_interface, s3_interface=s3_interface, cache=cache, + can_execute_unsafe_code=can_execute_unsafe_code, ) # pass position specified in URL to module through ModuleSystem system.set('position', position) diff --git a/lms/envs/common.py b/lms/envs/common.py index 1b492a3c56..12f50233a2 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -251,6 +251,15 @@ CODE_JAIL = { 'user': 'sandbox', } +# Some courses are allowed to run unsafe code. This is a list of regexes, one +# of them must match the course id for that course to run unsafe code. +# +# For example: +# +# COURSES_WITH_UNSAFE_CODE = [ +# r"Harvard/XY123.1/.*" +# ] +COURSES_WITH_UNSAFE_CODE = [] ############################ SIGNAL HANDLERS ################################ # This is imported to register the exception signal handling that logs exceptions From 001ef7b0fe8e505a3b374a66b8e09948466d180d Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 2 May 2013 11:42:06 -0400 Subject: [PATCH 094/162] Use only safe characters for the cache key --- common/lib/capa/capa/safe_exec/safe_exec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/lib/capa/capa/safe_exec/safe_exec.py b/common/lib/capa/capa/safe_exec/safe_exec.py index d8b7f561cc..745704373b 100644 --- a/common/lib/capa/capa/safe_exec/safe_exec.py +++ b/common/lib/capa/capa/safe_exec/safe_exec.py @@ -60,7 +60,7 @@ def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None md5er = hashlib.md5() md5er.update(code) md5er.update(repr(canonical_globals)) - key = "safe_exec %r %s" % (random_seed, md5er.hexdigest()) + key = "safe_exec.%r.%s" % (random_seed, md5er.hexdigest()) cached = cache.get(key) if cached is not None: globals_dict.update(cached) From 403218ec6b5e5eb46b0a134f7641b6385df3bdb5 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 2 May 2013 12:46:35 -0400 Subject: [PATCH 095/162] If sandboxed code raises an exception, the exception will be cached. --- common/lib/capa/capa/safe_exec/safe_exec.py | 28 +++++++++---- .../capa/safe_exec/tests/test_safe_exec.py | 41 ++++++++++++++++++- 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/common/lib/capa/capa/safe_exec/safe_exec.py b/common/lib/capa/capa/safe_exec/safe_exec.py index 745704373b..34cca0593e 100644 --- a/common/lib/capa/capa/safe_exec/safe_exec.py +++ b/common/lib/capa/capa/safe_exec/safe_exec.py @@ -1,7 +1,7 @@ """Capa's specialized use of codejail.safe_exec.""" from codejail.safe_exec import safe_exec as codejail_safe_exec -from codejail.safe_exec import json_safe +from codejail.safe_exec import json_safe, SafeExecException from . import lazymod from statsd import statsd @@ -63,20 +63,34 @@ def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None key = "safe_exec.%r.%s" % (random_seed, md5er.hexdigest()) cached = cache.get(key) if cached is not None: - globals_dict.update(cached) + # We have a cached result. The result is a pair: the exception + # message, if any, else None; and the resulting globals dictionary. + emsg, cleaned_results = cached + globals_dict.update(cleaned_results) + if emsg: + raise SafeExecException(emsg) return # Create the complete code we'll run. code_prolog = CODE_PROLOG % random_seed # Run the code! Results are side effects in globals_dict. - codejail_safe_exec( - code_prolog + LAZY_IMPORTS + code, globals_dict, - python_path=python_path, - ) + try: + codejail_safe_exec( + code_prolog + LAZY_IMPORTS + code, globals_dict, + python_path=python_path, + ) + except SafeExecException as e: + emsg = e.message + else: + emsg = None # Put the result back in the cache. This is complicated by the fact that # the globals dict might not be entirely serializable. if cache: cleaned_results = json_safe(globals_dict) - cache.set(key, cleaned_results) + cache.set(key, (emsg, cleaned_results)) + + # If an exception happened, raise it now. + if emsg: + raise e diff --git a/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py b/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py index bd960f331c..b648672daf 100644 --- a/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py +++ b/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py @@ -5,6 +5,8 @@ import random import unittest from capa.safe_exec import safe_exec +from codejail.safe_exec import SafeExecException + class TestSafeExec(unittest.TestCase): def test_set_values(self): @@ -57,6 +59,12 @@ class TestSafeExec(unittest.TestCase): g, python_path=[pylib] ) + def test_raising_exceptions(self): + g = {} + with self.assertRaises(SafeExecException) as cm: + safe_exec("1/0", g) + self.assertIn("ZeroDivisionError", cm.exception.message) + class DictCache(object): """A cache implementation over a simple dict, for testing.""" @@ -86,10 +94,10 @@ class TestSafeExecCaching(unittest.TestCase): safe_exec("a = int(math.pi)", g, cache=DictCache(cache)) self.assertEqual(g['a'], 3) # A result has been cached - self.assertEqual(cache.values(), [{'a': 3}]) + self.assertEqual(cache.values()[0], (None, {'a': 3})) # Fiddle with the cache, then try it again. - cache[cache.keys()[0]] = {'a': 17} + cache[cache.keys()[0]] = (None, {'a': 17}) g = {} safe_exec("a = int(math.pi)", g, cache=DictCache(cache)) @@ -104,3 +112,32 @@ class TestSafeExecCaching(unittest.TestCase): cache = {} safe_exec(code, g, cache=DictCache(cache)) self.assertEqual(g['a'], 12345) + + def test_cache_exceptions(self): + # Used to be that running code that raised an exception didn't cache + # the result. Check that now it does. + code = "1/0" + g = {} + cache = {} + with self.assertRaises(SafeExecException): + safe_exec(code, g, cache=DictCache(cache)) + + # The exception should be in the cache now. + self.assertEqual(len(cache), 1) + cache_exc_msg, cache_globals = cache.values()[0] + self.assertIn("ZeroDivisionError", cache_exc_msg) + + # Change the value stored in the cache, the result should change. + cache[cache.keys()[0]] = ("Hey there!", {}) + + with self.assertRaises(SafeExecException): + safe_exec(code, g, cache=DictCache(cache)) + + self.assertEqual(len(cache), 1) + cache_exc_msg, cache_globals = cache.values()[0] + self.assertEqual("Hey there!", cache_exc_msg) + + # Change it again, now no exception! + cache[cache.keys()[0]] = (None, {'a': 17}) + safe_exec(code, g, cache=DictCache(cache)) + self.assertEqual(g['a'], 17) From 686eb64949b84d16a91e562206723f18431d0478 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 2 May 2013 17:39:29 -0400 Subject: [PATCH 096/162] An example of how to set the CPU limit for codejail. --- lms/envs/common.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lms/envs/common.py b/lms/envs/common.py index 12f50233a2..324bb82216 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -249,6 +249,12 @@ CODE_JAIL = { 'python_bin': None, # User to run as in the sandbox. 'user': 'sandbox', + + # Configurable limits. + 'limits': { + # How many CPU seconds can jailed code use? + 'CPU': 1, + }, } # Some courses are allowed to run unsafe code. This is a list of regexes, one From 7cb3987f949b4391c88e4831de5f6f4964f07a92 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 3 May 2013 14:10:31 -0400 Subject: [PATCH 097/162] Ugh, missing import. --- lms/djangoapps/courseware/module_render.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 622800382e..d6c104a83c 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -1,6 +1,7 @@ import json import logging import pyparsing +import re import sys import static_replace From e3de0dc847d8756ffc23a019d80608b619797fc3 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 6 May 2013 10:15:27 -0400 Subject: [PATCH 098/162] A fuller unit test with a real 8.02x problem. --- .../capa/safe_exec/tests/test_safe_exec.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py b/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py index b648672daf..b8a70a09c6 100644 --- a/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py +++ b/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py @@ -2,6 +2,7 @@ import os.path import random +import textwrap import unittest from capa.safe_exec import safe_exec @@ -141,3 +142,71 @@ class TestSafeExecCaching(unittest.TestCase): cache[cache.keys()[0]] = (None, {'a': 17}) safe_exec(code, g, cache=DictCache(cache)) self.assertEqual(g['a'], 17) + + +class TestRealProblems(unittest.TestCase): + def test_802x(self): + code = textwrap.dedent("""\ + import math + import random + import numpy + e=1.602e-19 #C + me=9.1e-31 #kg + mp=1.672e-27 #kg + eps0=8.854e-12 #SI units + mu0=4e-7*math.pi #SI units + + Rd1=random.randrange(1,30,1) + Rd2=random.randrange(30,50,1) + Rd3=random.randrange(50,70,1) + Rd4=random.randrange(70,100,1) + Rd5=random.randrange(100,120,1) + + Vd1=random.randrange(1,20,1) + Vd2=random.randrange(20,40,1) + Vd3=random.randrange(40,60,1) + + #R=[0,10,30,50,70,100] #Ohm + #V=[0,12,24,36] # Volt + + R=[0,Rd1,Rd2,Rd3,Rd4,Rd5] #Ohms + V=[0,Vd1,Vd2,Vd3] #Volts + #here the currents IL and IR are defined as in figure ps3_p3_fig2 + a=numpy.array([ [ R[1]+R[4]+R[5],R[4] ],[R[4], R[2]+R[3]+R[4] ] ]) + b=numpy.array([V[1]-V[2],-V[3]-V[2]]) + x=numpy.linalg.solve(a,b) + IL='%.2e' % x[0] + IR='%.2e' % x[1] + ILR='%.2e' % (x[0]+x[1]) + def sign(x): + return abs(x)/x + + RW="Rightwards" + LW="Leftwards" + UW="Upwards" + DW="Downwards" + I1='%.2e' % abs(x[0]) + I1d=LW if sign(x[0])==1 else RW + I1not=LW if I1d==RW else RW + I2='%.2e' % abs(x[1]) + I2d=RW if sign(x[1])==1 else LW + I2not=LW if I2d==RW else RW + I3='%.2e' % abs(x[1]) + I3d=DW if sign(x[1])==1 else UW + I3not=DW if I3d==UW else UW + I4='%.2e' % abs(x[0]+x[1]) + I4d=UW if sign(x[1]+x[0])==1 else DW + I4not=DW if I4d==UW else UW + I5='%.2e' % abs(x[0]) + I5d=RW if sign(x[0])==1 else LW + I5not=LW if I5d==RW else RW + VAP=-x[0]*R[1]-(x[0]+x[1])*R[4] + VPN=-V[2] + VGD=+V[1]-x[0]*R[1]+V[3]+x[1]*R[2] + aVAP='%.2e' % VAP + aVPN='%.2e' % VPN + aVGD='%.2e' % VGD + """) + g = {} + safe_exec(code, g) + self.assertIn("aVAP", g) From 4f33b8e0c07e9b72813d53a63cf5ab84546751f0 Mon Sep 17 00:00:00 2001 From: Will Daly Date: Mon, 6 May 2013 14:20:43 -0400 Subject: [PATCH 099/162] Fixed a UnicodeEncodeError that occurred when generating cache keys from non-ASCII unicode code submissions. --- common/lib/capa/capa/safe_exec/safe_exec.py | 2 +- .../lib/capa/capa/safe_exec/tests/test_safe_exec.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/common/lib/capa/capa/safe_exec/safe_exec.py b/common/lib/capa/capa/safe_exec/safe_exec.py index 34cca0593e..07e1430688 100644 --- a/common/lib/capa/capa/safe_exec/safe_exec.py +++ b/common/lib/capa/capa/safe_exec/safe_exec.py @@ -58,7 +58,7 @@ def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None if cache: canonical_globals = sorted(json_safe(globals_dict).iteritems()) md5er = hashlib.md5() - md5er.update(code) + md5er.update(repr(code)) md5er.update(repr(canonical_globals)) key = "safe_exec.%r.%s" % (random_seed, md5er.hexdigest()) cached = cache.get(key) diff --git a/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py b/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py index b8a70a09c6..d79140f33c 100644 --- a/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py +++ b/common/lib/capa/capa/safe_exec/tests/test_safe_exec.py @@ -143,6 +143,19 @@ class TestSafeExecCaching(unittest.TestCase): safe_exec(code, g, cache=DictCache(cache)) self.assertEqual(g['a'], 17) + def test_unicode_submission(self): + # Check that using non-ASCII unicode does not raise an encoding error. + + # Try several non-ASCII unicode characters + for code in [129, 500, 2**8 - 1, 2**16 - 1]: + + code_with_unichr = unicode("# ") + unichr(code) + + try: + safe_exec(code_with_unichr, {}, cache=DictCache({})) + except UnicodeEncodeError: + self.fail("Tried executing code with non-ASCII unicode: {0}".format(code)) + class TestRealProblems(unittest.TestCase): def test_802x(self): From 3da841ea2be46f019050a7637f8e663c02323a87 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 6 May 2013 17:16:35 -0400 Subject: [PATCH 100/162] Pin codejail to a specific commit so it will get installed properly on deploying. --- github-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-requirements.txt b/github-requirements.txt index 2111d3f597..7c06f679ad 100644 --- a/github-requirements.txt +++ b/github-requirements.txt @@ -8,4 +8,4 @@ # Our libraries: -e git+https://github.com/edx/XBlock.git@5ce6f70a#egg=XBlock --e git+https://github.com/edx/codejail.git#egg=codejail +-e git+https://github.com/edx/codejail.git@eb0803a#egg=codejail From 91db1eed4cbdcce097ff5e8851cae4470f3757cc Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 6 May 2013 18:06:08 -0400 Subject: [PATCH 101/162] The new settings will only be read from .json files if aws.py handles them. --- lms/envs/aws.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lms/envs/aws.py b/lms/envs/aws.py index df81c1e3a9..c7e6c0e50f 100644 --- a/lms/envs/aws.py +++ b/lms/envs/aws.py @@ -89,6 +89,16 @@ COMMENTS_SERVICE_URL = ENV_TOKENS.get("COMMENTS_SERVICE_URL", '') COMMENTS_SERVICE_KEY = ENV_TOKENS.get("COMMENTS_SERVICE_KEY", '') CERT_QUEUE = ENV_TOKENS.get("CERT_QUEUE", 'test-pull') +for name, value in ENV_TOKENS.get("CODE_JAIL", {}).items(): + oldvalue = CODE_JAIL.get(name) + if isinstance(oldvalue, dict): + for subname, subvalue in value.items(): + oldvalue[subname] = subvalue + else: + CODE_JAIL[name] = value + +COURSES_WITH_UNSAFE_CODE = ENV_TOKENS.get("COURSES_WITH_UNSAFE_CODE", []) + ############################## SECURE AUTH ITEMS ############### # Secret things: passwords, access keys, etc. with open(ENV_ROOT / CONFIG_PREFIX + "auth.json") as auth_file: From 6264d41f9244fe5531adf7d1e9510b91ebffa43b Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 8 May 2013 14:48:46 -0400 Subject: [PATCH 102/162] Very minor docstring tweak :) --- common/lib/capa/capa/safe_exec/safe_exec.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/lib/capa/capa/safe_exec/safe_exec.py b/common/lib/capa/capa/safe_exec/safe_exec.py index 07e1430688..2424fd4dc4 100644 --- a/common/lib/capa/capa/safe_exec/safe_exec.py +++ b/common/lib/capa/capa/safe_exec/safe_exec.py @@ -49,7 +49,8 @@ LAZY_IMPORTS = "".join(LAZY_IMPORTS) @statsd.timed('capa.safe_exec.time') def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None): - """Exec python code safely. + """ + Exec python code safely. `cache` is an object with .get(key) and .set(key, value) methods. From ce3e733d1f0e828dcc399322eeb97c3ca95ffc85 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 8 May 2013 14:49:02 -0400 Subject: [PATCH 103/162] Problems in 7.02x need lxml for their sandbox. --- sandbox-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/sandbox-requirements.txt b/sandbox-requirements.txt index 9b18fba9e9..d7c703e08f 100644 --- a/sandbox-requirements.txt +++ b/sandbox-requirements.txt @@ -1,6 +1,7 @@ # Packages to install in the Python sandbox for secured execution. numpy==1.6.2 scipy==0.11.0 +lxml==3.0.1 -e common/lib/calc -e common/lib/chem -e common/lib/sandbox-packages From c98ab686e63c65993fe248a075db9817d3aa28c4 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Mon, 13 May 2013 14:11:28 -0400 Subject: [PATCH 104/162] adding pre-requirements for sandbox-requirements.txt --- pre-sandbox-requirements.txt | 1 + sandbox-requirements.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 pre-sandbox-requirements.txt diff --git a/pre-sandbox-requirements.txt b/pre-sandbox-requirements.txt new file mode 100644 index 0000000000..d801f46c8e --- /dev/null +++ b/pre-sandbox-requirements.txt @@ -0,0 +1 @@ +numpy==1.6.2 diff --git a/sandbox-requirements.txt b/sandbox-requirements.txt index d7c703e08f..f99e8a8c4b 100644 --- a/sandbox-requirements.txt +++ b/sandbox-requirements.txt @@ -1,5 +1,4 @@ # Packages to install in the Python sandbox for secured execution. -numpy==1.6.2 scipy==0.11.0 lxml==3.0.1 -e common/lib/calc From 72714408284ee93c92f663c347fc30efa1621860 Mon Sep 17 00:00:00 2001 From: James Tauber Date: Thu, 9 May 2013 16:09:21 -0400 Subject: [PATCH 105/162] added initial AUTHORS file --- AUTHORS | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 AUTHORS diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000000..004f869f73 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,75 @@ +Piotr Mitros +Kyle Fiedler +Ernie Park +Bridger Maxwell +Lyla Fischer +David Ormsbee +Chris Terman +Reda Lemeden +Anant Agarwal +Jean-Michel Claus +Calen Pennington +JM Van Thong +Prem Sichanugrist +Isaac Chuang +Galen Frechette +Edward Loveall +Matt Jankowski +John Jarvis +Victor Shnayder +Matthew Mongeau +Tony Kim +Arjun Singh +John Hess +Carlos Andrés Rocha +Mike Chen +Rocky Duan +Sidhanth Rao +Brittany Cheng +Dhaval Adjodah +Tom Giannattasio +Ibrahim Awwal +Sarina Canelake +Mark L. Chang +Dean Dieker +Tommy MacWilliam +Nate Hardison +Chris Dodge +Kevin Chugh +Ned Batchelder +Alexander Kryklia +Vik Paruchuri +Louis Sobel +Brian Wilson +Ashley Penney +Don Mitchell +Aaron Culich +Brian Talbot +Jay Zoldak +Valera Rozuvan +Diana Huang +Marco Morales +Christina Roberts +Valera Rozuvan +Robert Chirwa +Ed Zarecor +Deena Wang +Jean Manuel-Nater +Emily Zhang <1800.ehz.hang@gmail.com> +Jennifer Akana +Peter Baratta +Julian Arni +Arthur Barrett +Vasyl Nakvasiuk +Will Daly +James Tauber +Greg Price +Joe Blaylock +Sef Kloninger +Anto Stupak +David Adams +Steve Strassmann +Giulio Gratta +David Baumgold +Jason Bau +Giulio Gratta From 9cc07d2345d954fcef08f9c7f57fb62ba2f499d3 Mon Sep 17 00:00:00 2001 From: James Tauber Date: Mon, 13 May 2013 16:20:37 -0400 Subject: [PATCH 106/162] removed duplicates --- AUTHORS | 2 -- 1 file changed, 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 004f869f73..711670bd93 100644 --- a/AUTHORS +++ b/AUTHORS @@ -50,7 +50,6 @@ Valera Rozuvan Diana Huang Marco Morales Christina Roberts -Valera Rozuvan Robert Chirwa Ed Zarecor Deena Wang @@ -72,4 +71,3 @@ Steve Strassmann Giulio Gratta David Baumgold Jason Bau -Giulio Gratta From b682e1c0d35a4b66c2257185d3521950619bfcf4 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 13 May 2013 17:07:13 -0400 Subject: [PATCH 107/162] Put a feature flag around the staff-only debugging page. --- lms/envs/common.py | 6 +++++- lms/urls.py | 7 ++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lms/envs/common.py b/lms/envs/common.py index 324bb82216..c9c53fd034 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -90,7 +90,11 @@ MITX_FEATURES = { # Give a UI to show a student's submission history in a problem by the # Staff Debug tool. - 'ENABLE_STUDENT_HISTORY_VIEW': True + 'ENABLE_STUDENT_HISTORY_VIEW': True, + + # Turn on a page that lets staff enter Python code to be run in the + # sandbox, for testing whether it's enabled properly. + 'ENABLE_DEBUG_RUN_PYTHON': False, } # Used for A/B testing diff --git a/lms/urls.py b/lms/urls.py index dc558d6a54..dc0d820278 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -358,9 +358,10 @@ urlpatterns += ( url(r'^comm/foldit_ops', 'foldit.views.foldit_ops', name="foldit_ops"), ) -urlpatterns += ( - url(r'^debug/run_python', 'debug.views.run_python'), -) +if settings.MITX_FEATURES.get('ENABLE_DEBUG_RUN_PYTHON'): + urlpatterns += ( + url(r'^debug/run_python', 'debug.views.run_python'), + ) urlpatterns = patterns(*urlpatterns) From 29f4566a0032308dd879e3a710aa77f6de29d3b4 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 13 May 2013 17:27:21 -0400 Subject: [PATCH 108/162] Change the XBlock target hash, not because we need new code, but because the old one was tangled up in a eponymous branch that was causing trouble. --- github-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-requirements.txt b/github-requirements.txt index 7c06f679ad..9838f5d089 100644 --- a/github-requirements.txt +++ b/github-requirements.txt @@ -7,5 +7,5 @@ -e git://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev # Our libraries: --e git+https://github.com/edx/XBlock.git@5ce6f70a#egg=XBlock +-e git+https://github.com/edx/XBlock.git@c33a0ce#egg=XBlock -e git+https://github.com/edx/codejail.git@eb0803a#egg=codejail From 768665067f3d7465e2b59c94bb89de042c5b20bf Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 14 May 2013 14:52:30 -0400 Subject: [PATCH 109/162] Update the CodeJail we want. --- github-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-requirements.txt b/github-requirements.txt index 9838f5d089..c4cd7bd532 100644 --- a/github-requirements.txt +++ b/github-requirements.txt @@ -8,4 +8,4 @@ # Our libraries: -e git+https://github.com/edx/XBlock.git@c33a0ce#egg=XBlock --e git+https://github.com/edx/codejail.git@eb0803a#egg=codejail +-e git+https://github.com/edx/codejail.git@de65688#egg=codejail From 4ea76076481d1af79087bd97a8fb3d089ad81392 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 14 May 2013 14:59:19 -0400 Subject: [PATCH 110/162] Check that the directory being added to the Python path is really inside the course. --- common/lib/capa/capa/capa_problem.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index 1c0189d9aa..7ead599d67 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -413,12 +413,17 @@ class LoncapaProblem(object): path = [] for dir in raw_path: - if not dir: continue # path is an absolute path or a path relative to the data dir dir = os.path.join(self.system.filestore.root_path, dir) + # Check that we are within the filestore tree. + reldir = os.path.relpath(dir, self.system.filestore.root_path) + if ".." in reldir: + log.warning("Ignoring Python directory outside of course: %r" % dir) + continue + abs_dir = os.path.normpath(dir) path.append(abs_dir) From b32878e6c781d5b862c9de2e01bfb64781774899 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 15 May 2013 12:59:56 -0400 Subject: [PATCH 111/162] Added instructions for getting capa sandboxed. --- common/lib/capa/capa/safe_exec/README.rst | 32 +++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 common/lib/capa/capa/safe_exec/README.rst diff --git a/common/lib/capa/capa/safe_exec/README.rst b/common/lib/capa/capa/safe_exec/README.rst new file mode 100644 index 0000000000..57eac6b9fd --- /dev/null +++ b/common/lib/capa/capa/safe_exec/README.rst @@ -0,0 +1,32 @@ +Configuring Capa sandboxed execution +==================================== + +Capa problems can contain code authored by the course author. We need to +execute that code in a sandbox. We use CodeJail as the sandboxing facility, +but it needs to be configured specifically for Capa's use. + +As a developer, you don't have to do anything to configure sandboxing if you +don't want to, and everything will operate properly, you just won't have +protection on that code. + +If you want to configure sandboxing, you're going to use the `README from +CodeJail`__, with a few customized tweaks. + +__ https://github.com/edx/codejail/blob/master/README.rst + + +1. At the instruction to install packages into the sandboxed code, you'll + need to install both `pre-sandbox-requirements.txt` and + `sandbox-requirements.txt`:: + + $ sudo pip install -r pre-sandbox-requirements.txt + $ sudo pip install -r sandbox-requirements.txt + +2. At the instruction to create the AppArmor profile, you'll need a line in + the profile for the sandbox packages. is the full path to + your edx_platform repo:: + + /common/lib/sandbox-packages/** r, + +That's it. Once you've finished the CodeJail configuration instructions, +your course-hosted Python code should be run securely. From 8f3c1db1ef8cc52ab6e79ecdb873380bc6750c5a Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 15 May 2013 17:00:39 -0400 Subject: [PATCH 112/162] Add some notes for the future maintainer. --- common/lib/xmodule/xmodule/tests/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/lib/xmodule/xmodule/tests/__init__.py b/common/lib/xmodule/xmodule/tests/__init__.py index bcf333d7d9..d26352bdae 100644 --- a/common/lib/xmodule/xmodule/tests/__init__.py +++ b/common/lib/xmodule/xmodule/tests/__init__.py @@ -85,10 +85,12 @@ class ModelsTest(unittest.TestCase): self.assertTrue(abs(calc.evaluator(variables, functions, "e^(j*pi)") + 1) < 0.00001) self.assertTrue(abs(calc.evaluator(variables, functions, "j||1") - 0.5 - 0.5j) < 0.00001) variables['t'] = 1.0 + # Use self.assertAlmostEqual here... self.assertTrue(abs(calc.evaluator(variables, functions, "t") - 1.0) < 0.00001) self.assertTrue(abs(calc.evaluator(variables, functions, "T") - 1.0) < 0.00001) self.assertTrue(abs(calc.evaluator(variables, functions, "t", cs=True) - 1.0) < 0.00001) self.assertTrue(abs(calc.evaluator(variables, functions, "T", cs=True) - 298) < 0.2) + # Use self.assertRaises here... exception_happened = False try: calc.evaluator({}, {}, "5+7 QWSEKO") From 1c36564f453a25fe72ec36384ccc7f060df27ab3 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 16 May 2013 07:47:16 -0400 Subject: [PATCH 113/162] Use newer CodeJail. --- github-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-requirements.txt b/github-requirements.txt index c4cd7bd532..154a172534 100644 --- a/github-requirements.txt +++ b/github-requirements.txt @@ -8,4 +8,4 @@ # Our libraries: -e git+https://github.com/edx/XBlock.git@c33a0ce#egg=XBlock --e git+https://github.com/edx/codejail.git@de65688#egg=codejail +-e git+https://github.com/edx/codejail.git@94dc7ce#egg=codejail From c410da91ded5f3b48d42a6a0fdf16740e6ff408f Mon Sep 17 00:00:00 2001 From: Kevin Chugh Date: Thu, 25 Apr 2013 07:02:57 -0400 Subject: [PATCH 114/162] fix coffeescript error --- lms/djangoapps/django_comment_client/base/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lms/djangoapps/django_comment_client/base/views.py b/lms/djangoapps/django_comment_client/base/views.py index 69609dcf01..296eb65b66 100644 --- a/lms/djangoapps/django_comment_client/base/views.py +++ b/lms/djangoapps/django_comment_client/base/views.py @@ -27,6 +27,7 @@ from django_comment_client.utils import JsonResponse, JsonError, extract, get_co from django_comment_client.permissions import check_permissions_by_view, cached_has_permission from django_comment_client.models import Role +from courseware.access import has_access log = logging.getLogger(__name__) From ab08fa94e4ea4d9c7a80f476ecad7de4c33ba72a Mon Sep 17 00:00:00 2001 From: Kevin Chugh Date: Thu, 25 Apr 2013 11:36:12 -0400 Subject: [PATCH 115/162] coffeescript fixes --- common/static/coffee/src/discussion/utils.coffee | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/common/static/coffee/src/discussion/utils.coffee b/common/static/coffee/src/discussion/utils.coffee index 41f52f1711..0da84a3709 100644 --- a/common/static/coffee/src/discussion/utils.coffee +++ b/common/static/coffee/src/discussion/utils.coffee @@ -18,6 +18,9 @@ class @DiscussionUtil @loadRoles: (roles)-> @roleIds = roles + @loadFlagModerator: (what)-> + @isFlagModerator = (what=="True") + @loadRolesFromContainer: -> @loadRoles($("#discussion-container").data("roles")) @@ -50,7 +53,7 @@ class @DiscussionUtil delete_thread : "/courses/#{$$course_id}/discussion/threads/#{param}/delete" upvote_thread : "/courses/#{$$course_id}/discussion/threads/#{param}/upvote" downvote_thread : "/courses/#{$$course_id}/discussion/threads/#{param}/downvote" - pin_thread : "/courses/#{$$course_id}/discussion/threads/#{param}/pin" + pin_thread : "/courses/#{$$course_id}/discussion/threads/#{param}/pin" un_pin_thread : "/courses/#{$$course_id}/discussion/threads/#{param}/unpin" undo_vote_for_thread : "/courses/#{$$course_id}/discussion/threads/#{param}/unvote" follow_thread : "/courses/#{$$course_id}/discussion/threads/#{param}/follow" From 8568ce7ad08cf4c77d2a1d71905cfce7388b9735 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 25 Apr 2013 15:00:26 -0400 Subject: [PATCH 116/162] server django version generating 1 instead of True so test for both --- common/static/coffee/src/discussion/utils.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/static/coffee/src/discussion/utils.coffee b/common/static/coffee/src/discussion/utils.coffee index 0da84a3709..304af48031 100644 --- a/common/static/coffee/src/discussion/utils.coffee +++ b/common/static/coffee/src/discussion/utils.coffee @@ -19,7 +19,7 @@ class @DiscussionUtil @roleIds = roles @loadFlagModerator: (what)-> - @isFlagModerator = (what=="True") + @isFlagModerator = ((what=="True") or (what == "1")) @loadRolesFromContainer: -> @loadRoles($("#discussion-container").data("roles")) From 4b806768ec8c627fc083390562f495a6ebeb9b5a Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 25 Apr 2013 15:28:01 -0400 Subject: [PATCH 117/162] more server trial and error --- common/static/coffee/src/discussion/utils.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/static/coffee/src/discussion/utils.coffee b/common/static/coffee/src/discussion/utils.coffee index 304af48031..5955d07d20 100644 --- a/common/static/coffee/src/discussion/utils.coffee +++ b/common/static/coffee/src/discussion/utils.coffee @@ -19,7 +19,7 @@ class @DiscussionUtil @roleIds = roles @loadFlagModerator: (what)-> - @isFlagModerator = ((what=="True") or (what == "1")) + @isFlagModerator = ((what=="True") or (what == 1)) @loadRolesFromContainer: -> @loadRoles($("#discussion-container").data("roles")) From ed5ad46192c62761762e35a608ab79c6923a6851 Mon Sep 17 00:00:00 2001 From: Jay Zoldak Date: Mon, 29 Apr 2013 13:17:16 -0400 Subject: [PATCH 118/162] Revert "Revert "fix merge conflict"" This reverts commit 2df3fe93440cb8b58bf86fbcc796eaad09be4ce8. Conflicts: common/static/coffee/src/discussion/utils.coffee --- .../coffee/src/discussion/content.coffee | 34 ++++++--- .../coffee/src/discussion/discussion.coffee | 3 + .../static/coffee/src/discussion/utils.coffee | 7 +- .../views/discussion_content_view.coffee | 46 ++++++++++++ .../views/discussion_thread_list_view.coffee | 11 +++ .../views/discussion_thread_show_view.coffee | 17 ++++- .../views/discussion_thread_view.coffee | 2 +- .../views/response_comment_show_view.coffee | 22 ++++++ .../views/thread_response_show_view.coffee | 17 +++++ .../views/thread_response_view.coffee | 2 +- .../django_comment_client/base/urls.py | 5 +- .../django_comment_client/base/views.py | 65 +++++++++++++---- .../django_comment_client/forum/views.py | 19 ++--- .../management/commands/reload_forum_users.py | 8 +-- .../django_comment_client/permissions.py | 5 +- .../tests/test_mustache_helpers.py | 1 - lms/djangoapps/django_comment_client/utils.py | 34 ++++----- lms/lib/comment_client/comment.py | 38 +++++++++- lms/lib/comment_client/comment_client.py | 1 - lms/lib/comment_client/thread.py | 67 +++++++++++++----- lms/static/images/flagged.png | Bin 0 -> 40840 bytes lms/static/images/notflagged.png | Bin 0 -> 39000 bytes lms/static/images/resolvedflag.png | Bin 0 -> 362 bytes lms/static/sass/_discussion.scss | 44 +++++++++++- .../discussion/_filter_dropdown.html | 8 +++ .../discussion/_underscore_templates.html | 15 +++- lms/templates/discussion/index.html | 2 +- lms/templates/discussion/single_thread.html | 2 +- 28 files changed, 388 insertions(+), 87 deletions(-) create mode 100644 lms/static/images/flagged.png create mode 100644 lms/static/images/notflagged.png create mode 100644 lms/static/images/resolvedflag.png diff --git a/common/static/coffee/src/discussion/content.coffee b/common/static/coffee/src/discussion/content.coffee index 00c34df686..6361a4b76e 100644 --- a/common/static/coffee/src/discussion/content.coffee +++ b/common/static/coffee/src/discussion/content.coffee @@ -88,20 +88,32 @@ if Backbone? pinned = @get("pinned") @set("pinned",pinned) @trigger "change", @ + + flagAbuse: -> + temp_array = @get("abuse_flaggers") + temp_array.push(window.user.get('id')) + @set("abuse_flaggers",temp_array) + @trigger "change", @ + unflagAbuse: -> + @get("abuse_flaggers").pop(window.user.get('id')) + @trigger "change", @ + class @Thread extends @Content urlMappers: - 'retrieve' : -> DiscussionUtil.urlFor('retrieve_single_thread', @discussion.id, @id) - 'reply' : -> DiscussionUtil.urlFor('create_comment', @id) - 'unvote' : -> DiscussionUtil.urlFor("undo_vote_for_#{@get('type')}", @id) - 'upvote' : -> DiscussionUtil.urlFor("upvote_#{@get('type')}", @id) - 'downvote' : -> DiscussionUtil.urlFor("downvote_#{@get('type')}", @id) - 'close' : -> DiscussionUtil.urlFor('openclose_thread', @id) - 'update' : -> DiscussionUtil.urlFor('update_thread', @id) - 'delete' : -> DiscussionUtil.urlFor('delete_thread', @id) - 'follow' : -> DiscussionUtil.urlFor('follow_thread', @id) - 'unfollow' : -> DiscussionUtil.urlFor('unfollow_thread', @id) + 'retrieve' : -> DiscussionUtil.urlFor('retrieve_single_thread', @discussion.id, @id) + 'reply' : -> DiscussionUtil.urlFor('create_comment', @id) + 'unvote' : -> DiscussionUtil.urlFor("undo_vote_for_#{@get('type')}", @id) + 'upvote' : -> DiscussionUtil.urlFor("upvote_#{@get('type')}", @id) + 'downvote' : -> DiscussionUtil.urlFor("downvote_#{@get('type')}", @id) + 'close' : -> DiscussionUtil.urlFor('openclose_thread', @id) + 'update' : -> DiscussionUtil.urlFor('update_thread', @id) + 'delete' : -> DiscussionUtil.urlFor('delete_thread', @id) + 'follow' : -> DiscussionUtil.urlFor('follow_thread', @id) + 'unfollow' : -> DiscussionUtil.urlFor('unfollow_thread', @id) + 'flagAbuse' : -> DiscussionUtil.urlFor("flagAbuse_#{@get('type')}", @id) + 'unFlagAbuse' : -> DiscussionUtil.urlFor("unFlagAbuse_#{@get('type')}", @id) 'pinThread' : -> DiscussionUtil.urlFor("pin_thread", @id) 'unPinThread' : -> DiscussionUtil.urlFor("un_pin_thread", @id) @@ -157,6 +169,8 @@ if Backbone? 'endorse': -> DiscussionUtil.urlFor('endorse_comment', @id) 'update': -> DiscussionUtil.urlFor('update_comment', @id) 'delete': -> DiscussionUtil.urlFor('delete_comment', @id) + 'flagAbuse' : -> DiscussionUtil.urlFor("flagAbuse_#{@get('type')}", @id) + 'unFlagAbuse' : -> DiscussionUtil.urlFor("unFlagAbuse_#{@get('type')}", @id) getCommentsCount: -> count = 0 diff --git a/common/static/coffee/src/discussion/discussion.coffee b/common/static/coffee/src/discussion/discussion.coffee index 83e25e1da7..5a52cd4de0 100644 --- a/common/static/coffee/src/discussion/discussion.coffee +++ b/common/static/coffee/src/discussion/discussion.coffee @@ -37,6 +37,9 @@ if Backbone? data['commentable_ids'] = options.commentable_ids when 'all' url = DiscussionUtil.urlFor 'threads' + when 'flagged' + data['flagged'] = true + url = DiscussionUtil.urlFor 'search' when 'followed' url = DiscussionUtil.urlFor 'followed_threads', options.user_id if options['group_id'] diff --git a/common/static/coffee/src/discussion/utils.coffee b/common/static/coffee/src/discussion/utils.coffee index 5955d07d20..b7b7cb2550 100644 --- a/common/static/coffee/src/discussion/utils.coffee +++ b/common/static/coffee/src/discussion/utils.coffee @@ -23,6 +23,7 @@ class @DiscussionUtil @loadRolesFromContainer: -> @loadRoles($("#discussion-container").data("roles")) + @loadFlagModerator($("#discussion-container").data("flag-moderator")) @isStaff: (user_id) -> staff = _.union(@roleIds['Staff'], @roleIds['Moderator'], @roleIds['Administrator']) @@ -51,6 +52,10 @@ class @DiscussionUtil update_thread : "/courses/#{$$course_id}/discussion/threads/#{param}/update" create_comment : "/courses/#{$$course_id}/discussion/threads/#{param}/reply" delete_thread : "/courses/#{$$course_id}/discussion/threads/#{param}/delete" + flagAbuse_thread : "/courses/#{$$course_id}/discussion/threads/#{param}/flagAbuse" + unFlagAbuse_thread : "/courses/#{$$course_id}/discussion/threads/#{param}/unFlagAbuse" + flagAbuse_comment : "/courses/#{$$course_id}/discussion/comments/#{param}/flagAbuse" + unFlagAbuse_comment : "/courses/#{$$course_id}/discussion/comments/#{param}/unFlagAbuse" upvote_thread : "/courses/#{$$course_id}/discussion/threads/#{param}/upvote" downvote_thread : "/courses/#{$$course_id}/discussion/threads/#{param}/downvote" pin_thread : "/courses/#{$$course_id}/discussion/threads/#{param}/pin" @@ -75,7 +80,7 @@ class @DiscussionUtil permanent_link_thread : "/courses/#{$$course_id}/discussion/forum/#{param}/threads/#{param1}" permanent_link_comment : "/courses/#{$$course_id}/discussion/forum/#{param}/threads/#{param1}##{param2}" user_profile : "/courses/#{$$course_id}/discussion/forum/users/#{param}" - followed_threads : "/courses/#{$$course_id}/discussion/forum/users/#{param}/followed" + followed_threads : "/courses/#{$$course_id}/discussion/forum/users/#{param}/followed" threads : "/courses/#{$$course_id}/discussion/forum" }[name] diff --git a/common/static/coffee/src/discussion/views/discussion_content_view.coffee b/common/static/coffee/src/discussion/views/discussion_content_view.coffee index 9399d95398..9b2de1b198 100644 --- a/common/static/coffee/src/discussion/views/discussion_content_view.coffee +++ b/common/static/coffee/src/discussion/views/discussion_content_view.coffee @@ -1,6 +1,11 @@ if Backbone? class @DiscussionContentView extends Backbone.View + + events: + "click .discussion-flag-abuse": "toggleFlagAbuse" + + attrRenderer: endorsed: (endorsed) -> if endorsed @@ -94,7 +99,48 @@ if Backbone? setWmdContent: (cls_identifier, text) => DiscussionUtil.setWmdContent @$el, $.proxy(@$, @), cls_identifier, text + initialize: -> @initLocal() @model.bind('change', @renderPartialAttrs, @) + + + + toggleFlagAbuse: (event) -> + event.preventDefault() + if window.user.id in @model.get("abuse_flaggers") or (DiscussionUtil.isFlagModerator and @model.get("abuse_flaggers").length > 0) + @unFlagAbuse() + else + @flagAbuse() + + flagAbuse: -> + url = @model.urlFor("flagAbuse") + DiscussionUtil.safeAjax + $elem: @$(".discussion-flag-abuse") + url: url + type: "POST" + success: (response, textStatus) => + if textStatus == 'success' + ### + note, we have to clone the array in order to trigger a change event + ### + temp_array = _.clone(@model.get('abuse_flaggers')); + temp_array.push(window.user.id) + @model.set('abuse_flaggers', temp_array) + + unFlagAbuse: -> + url = @model.urlFor("unFlagAbuse") + DiscussionUtil.safeAjax + $elem: @$(".discussion-flag-abuse") + url: url + type: "POST" + success: (response, textStatus) => + if textStatus == 'success' + temp_array = _.clone(@model.get('abuse_flaggers')); + temp_array.pop(window.user.id) + # if you're an admin, clear this + if DiscussionUtil.isFlagModerator + temp_array = [] + + @model.set('abuse_flaggers', temp_array) diff --git a/common/static/coffee/src/discussion/views/discussion_thread_list_view.coffee b/common/static/coffee/src/discussion/views/discussion_thread_list_view.coffee index 8364963218..9aa4ba869d 100644 --- a/common/static/coffee/src/discussion/views/discussion_thread_list_view.coffee +++ b/common/static/coffee/src/discussion/views/discussion_thread_list_view.coffee @@ -276,6 +276,11 @@ if Backbone? @$(".post-search-field").val("") @$('.cohort').show() @retrieveAllThreads() + else if discussionId == "#flagged" + @discussionIds = "" + @$(".post-search-field").val("") + @$('.cohort').hide() + @retrieveFlaggedThreads() else if discussionId == "#following" @retrieveFollowed(event) @$('.cohort').hide() @@ -321,6 +326,12 @@ if Backbone? @collection.reset() @loadMorePages(event) + retrieveFlaggedThreads: (event)-> + @collection.current_page = 0 + @collection.reset() + @mode = 'flagged' + @loadMorePages(event) + sortThreads: (event) -> @$(".sort-bar a").removeClass("active") $(event.target).addClass("active") diff --git a/common/static/coffee/src/discussion/views/discussion_thread_show_view.coffee b/common/static/coffee/src/discussion/views/discussion_thread_show_view.coffee index 56525af347..49936c46e8 100644 --- a/common/static/coffee/src/discussion/views/discussion_thread_show_view.coffee +++ b/common/static/coffee/src/discussion/views/discussion_thread_show_view.coffee @@ -3,6 +3,7 @@ if Backbone? events: "click .discussion-vote": "toggleVote" + "click .discussion-flag-abuse": "toggleFlagAbuse" "click .admin-pin": "togglePin" "click .action-follow": "toggleFollowing" "click .action-edit": "edit" @@ -25,6 +26,7 @@ if Backbone? @delegateEvents() @renderDogear() @renderVoted() + @renderFlagged() @renderPinned() @renderAttrs() @$("span.timeago").timeago() @@ -42,6 +44,16 @@ if Backbone? @$("[data-role=discussion-vote]").addClass("is-cast") else @$("[data-role=discussion-vote]").removeClass("is-cast") + + renderFlagged: => + if window.user.id in @model.get("abuse_flaggers") or (DiscussionUtil.isFlagModerator and @model.get("abuse_flaggers").length > 0) + @$("[data-role=thread-flag]").addClass("flagged") + @$("[data-role=thread-flag]").removeClass("notflagged") + @$(".discussion-flag-abuse .flag-label").html("Misuse Reported") + else + @$("[data-role=thread-flag]").removeClass("flagged") + @$("[data-role=thread-flag]").addClass("notflagged") + @$(".discussion-flag-abuse .flag-label").html("Report Misuse") renderPinned: => if @model.get("pinned") @@ -56,6 +68,7 @@ if Backbone? updateModelDetails: => @renderVoted() + @renderFlagged() @renderPinned() @$("[data-role=discussion-vote] .votes-count-number").html(@model.get("votes")["up_count"]) @@ -96,6 +109,7 @@ if Backbone? if textStatus == 'success' @model.set(response, {silent: true}) + unvote: -> window.user.unvote(@model) url = @model.urlFor("unvote") @@ -107,6 +121,7 @@ if Backbone? if textStatus == 'success' @model.set(response, {silent: true}) + edit: (event) -> @trigger "thread:edit", event @@ -182,4 +197,4 @@ if Backbone? params = $.extend(params, user:{username: @model.username, user_url: @model.user_url}) Mustache.render(@template, params) - \ No newline at end of file + diff --git a/common/static/coffee/src/discussion/views/discussion_thread_view.coffee b/common/static/coffee/src/discussion/views/discussion_thread_view.coffee index cb549f1088..c3a793b478 100644 --- a/common/static/coffee/src/discussion/views/discussion_thread_view.coffee +++ b/common/static/coffee/src/discussion/views/discussion_thread_view.coffee @@ -91,7 +91,7 @@ if Backbone? body = @getWmdContent("reply-body") return if not body.trim().length @setWmdContent("reply-body", "") - comment = new Comment(body: body, created_at: (new Date()).toISOString(), username: window.user.get("username"), votes: { up_count: 0 }, endorsed: false, user_id: window.user.get("id")) + comment = new Comment(body: body, created_at: (new Date()).toISOString(), username: window.user.get("username"), votes: { up_count: 0 }, abuse_flaggers:[], endorsed: false, user_id: window.user.get("id")) comment.set('thread', @model.get('thread')) @renderResponse(comment) @model.addComment() diff --git a/common/static/coffee/src/discussion/views/response_comment_show_view.coffee b/common/static/coffee/src/discussion/views/response_comment_show_view.coffee index 84e7357e1f..18d405fdb4 100644 --- a/common/static/coffee/src/discussion/views/response_comment_show_view.coffee +++ b/common/static/coffee/src/discussion/views/response_comment_show_view.coffee @@ -1,7 +1,14 @@ if Backbone? class @ResponseCommentShowView extends DiscussionContentView + + events: + "click .discussion-flag-abuse": "toggleFlagAbuse" tagName: "li" + + initialize: -> + super() + @model.on "change", @updateModelDetails render: -> @template = _.template($("#response-comment-show-template").html()) @@ -11,6 +18,7 @@ if Backbone? @initLocal() @delegateEvents() @renderAttrs() + @renderFlagged() @markAsStaff() @$el.find(".timeago").timeago() @convertMath() @@ -34,3 +42,17 @@ if Backbone? @$el.find("a.profile-link").after('staff') else if DiscussionUtil.isTA(@model.get("user_id")) @$el.find("a.profile-link").after('Community  TA') + + + renderFlagged: => + if window.user.id in @model.get("abuse_flaggers") or (DiscussionUtil.isFlagModerator and @model.get("abuse_flaggers").length > 0) + @$("[data-role=thread-flag]").addClass("flagged") + @$("[data-role=thread-flag]").removeClass("notflagged") + else + @$("[data-role=thread-flag]").removeClass("flagged") + @$("[data-role=thread-flag]").addClass("notflagged") + + updateModelDetails: => + @renderFlagged() + + diff --git a/common/static/coffee/src/discussion/views/thread_response_show_view.coffee b/common/static/coffee/src/discussion/views/thread_response_show_view.coffee index 1f305ddf34..0e42b79b9a 100644 --- a/common/static/coffee/src/discussion/views/thread_response_show_view.coffee +++ b/common/static/coffee/src/discussion/views/thread_response_show_view.coffee @@ -5,6 +5,7 @@ if Backbone? "click .action-endorse": "toggleEndorse" "click .action-delete": "delete" "click .action-edit": "edit" + "click .discussion-flag-abuse": "toggleFlagAbuse" $: (selector) -> @$el.find(selector) @@ -23,6 +24,7 @@ if Backbone? if window.user.voted(@model) @$(".vote-btn").addClass("is-cast") @renderAttrs() + @renderFlagged() @$el.find(".posted-details").timeago() @convertMath() @markAsStaff() @@ -70,6 +72,7 @@ if Backbone? success: (response, textStatus) => if textStatus == 'success' @model.set(response) + edit: (event) -> @trigger "response:edit", event @@ -92,3 +95,17 @@ if Backbone? url: url data: data type: "POST" + + + renderFlagged: => + if window.user.id in @model.get("abuse_flaggers") or (DiscussionUtil.isFlagModerator and @model.get("abuse_flaggers").length > 0) + @$("[data-role=thread-flag]").addClass("flagged") + @$("[data-role=thread-flag]").removeClass("notflagged") + @$(".discussion-flag-abuse .flag-label").html("Misuse Reported") + else + @$("[data-role=thread-flag]").removeClass("flagged") + @$("[data-role=thread-flag]").addClass("notflagged") + @$(".discussion-flag-abuse .flag-label").html("Report Misuse") + + updateModelDetails: => + @renderFlagged() diff --git a/common/static/coffee/src/discussion/views/thread_response_view.coffee b/common/static/coffee/src/discussion/views/thread_response_view.coffee index 9b6800cdde..46a96a55ec 100644 --- a/common/static/coffee/src/discussion/views/thread_response_view.coffee +++ b/common/static/coffee/src/discussion/views/thread_response_view.coffee @@ -77,7 +77,7 @@ if Backbone? body = @getWmdContent("comment-body") return if not body.trim().length @setWmdContent("comment-body", "") - comment = new Comment(body: body, created_at: (new Date()).toISOString(), username: window.user.get("username"), user_id: window.user.get("id"), id:"unsaved") + comment = new Comment(body: body, created_at: (new Date()).toISOString(), username: window.user.get("username"), abuse_flaggers:[], user_id: window.user.get("id"), id:"unsaved") view = @renderComment(comment) @hideEditorChrome() @trigger "comment:add", comment diff --git a/lms/djangoapps/django_comment_client/base/urls.py b/lms/djangoapps/django_comment_client/base/urls.py index 5a43030565..41bf568012 100644 --- a/lms/djangoapps/django_comment_client/base/urls.py +++ b/lms/djangoapps/django_comment_client/base/urls.py @@ -9,6 +9,8 @@ urlpatterns = patterns('django_comment_client.base.views', # nopep8 url(r'threads/(?P[\w\-]+)/delete', 'delete_thread', name='delete_thread'), url(r'threads/(?P[\w\-]+)/upvote$', 'vote_for_thread', {'value': 'up'}, name='upvote_thread'), url(r'threads/(?P[\w\-]+)/downvote$', 'vote_for_thread', {'value': 'down'}, name='downvote_thread'), + url(r'threads/(?P[\w\-]+)/flagAbuse$', 'flag_abuse_for_thread', name='flag_abuse_for_thread'), + url(r'threads/(?P[\w\-]+)/unFlagAbuse$', 'un_flag_abuse_for_thread', name='un_flag_abuse_for_thread'), url(r'threads/(?P[\w\-]+)/unvote$', 'undo_vote_for_thread', name='undo_vote_for_thread'), url(r'threads/(?P[\w\-]+)/pin$', 'pin_thread', name='pin_thread'), url(r'threads/(?P[\w\-]+)/unpin$', 'un_pin_thread', name='un_pin_thread'), @@ -23,7 +25,8 @@ urlpatterns = patterns('django_comment_client.base.views', # nopep8 url(r'comments/(?P[\w\-]+)/upvote$', 'vote_for_comment', {'value': 'up'}, name='upvote_comment'), url(r'comments/(?P[\w\-]+)/downvote$', 'vote_for_comment', {'value': 'down'}, name='downvote_comment'), url(r'comments/(?P[\w\-]+)/unvote$', 'undo_vote_for_comment', name='undo_vote_for_comment'), - + url(r'comments/(?P[\w\-]+)/flagAbuse$', 'flag_abuse_for_comment', name='flag_abuse_for_comment'), + url(r'comments/(?P[\w\-]+)/unFlagAbuse$', 'un_flag_abuse_for_comment', name='un_flag_abuse_for_comment'), url(r'^(?P[\w\-.]+)/threads/create$', 'create_thread', name='create_thread'), # TODO should we search within the board? url(r'^(?P[\w\-.]+)/threads/search_similar$', 'search_similar_threads', name='search_similar_threads'), diff --git a/lms/djangoapps/django_comment_client/base/views.py b/lms/djangoapps/django_comment_client/base/views.py index 296eb65b66..8f7ea2f06b 100644 --- a/lms/djangoapps/django_comment_client/base/views.py +++ b/lms/djangoapps/django_comment_client/base/views.py @@ -20,7 +20,7 @@ from django.utils.translation import ugettext as _ from django.contrib.auth.models import User from mitxmako.shortcuts import render_to_response, render_to_string -from courseware.courses import get_course_with_access +from courseware.courses import get_course_with_access, get_course_by_id from course_groups.cohorts import get_cohort_id, is_commentable_cohorted from django_comment_client.utils import JsonResponse, JsonError, extract, get_courseware_context @@ -120,7 +120,7 @@ def create_thread(request, course_id, commentable_id): #patch for backward compatibility to comments service if not 'pinned' in thread.attributes: thread['pinned'] = False - + if post.get('auto_subscribe', 'false').lower() == 'true': user = cc.User.from_django_user(request.user) user.follow(thread) @@ -285,6 +285,50 @@ def vote_for_thread(request, course_id, thread_id, value): return JsonResponse(utils.safe_content(thread.to_dict())) +@require_POST +@login_required +@permitted +def flag_abuse_for_thread(request, course_id, thread_id): + user = cc.User.from_django_user(request.user) + thread = cc.Thread.find(thread_id) + thread.flagAbuse(user, thread) + return JsonResponse(utils.safe_content(thread.to_dict())) + + +@require_POST +@login_required +@permitted +def un_flag_abuse_for_thread(request, course_id, thread_id): + user = cc.User.from_django_user(request.user) + course = get_course_by_id(course_id) + thread = cc.Thread.find(thread_id) + removeAll = cached_has_permission(request.user, 'openclose_thread', course_id) or has_access(request.user, course, 'staff') + thread.unFlagAbuse(user, thread, removeAll) + return JsonResponse(utils.safe_content(thread.to_dict())) + + +@require_POST +@login_required +@permitted +def flag_abuse_for_comment(request, course_id, comment_id): + user = cc.User.from_django_user(request.user) + comment = cc.Comment.find(comment_id) + comment.flagAbuse(user, comment) + return JsonResponse(utils.safe_content(comment.to_dict())) + + +@require_POST +@login_required +@permitted +def un_flag_abuse_for_comment(request, course_id, comment_id): + user = cc.User.from_django_user(request.user) + course = get_course_by_id(course_id) + removeAll = cached_has_permission(request.user, 'openclose_thread', course_id) or has_access(request.user, course, 'staff') + comment = cc.Comment.find(comment_id) + comment.unFlagAbuse(user, comment, removeAll) + return JsonResponse(utils.safe_content(comment.to_dict())) + + @require_POST @login_required @permitted @@ -294,19 +338,21 @@ def undo_vote_for_thread(request, course_id, thread_id): user.unvote(thread) return JsonResponse(utils.safe_content(thread.to_dict())) + @require_POST @login_required @permitted def pin_thread(request, course_id, thread_id): user = cc.User.from_django_user(request.user) thread = cc.Thread.find(thread_id) - thread.pin(user,thread_id) + thread.pin(user, thread_id) return JsonResponse(utils.safe_content(thread.to_dict())) + def un_pin_thread(request, course_id, thread_id): user = cc.User.from_django_user(request.user) thread = cc.Thread.find(thread_id) - thread.un_pin(user,thread_id) + thread.un_pin(user, thread_id) return JsonResponse(utils.safe_content(thread.to_dict())) @@ -453,16 +499,11 @@ def upload(request, course_id): # ajax upload file to a question or answer if not file_extension in cc_settings.ALLOWED_UPLOAD_FILE_TYPES: file_types = "', '".join(cc_settings.ALLOWED_UPLOAD_FILE_TYPES) msg = _("allowed file types are '%(file_types)s'") % \ - {'file_types': file_types} + {'file_types': file_types} raise exceptions.PermissionDenied(msg) # generate new file name - new_file_name = str( - time.time() - ).replace( - '.', - str(random.randint(0, 100000)) - ) + file_extension + new_file_name = str(time.time()).replace('.', str(random.randint(0, 100000))) + file_extension file_storage = get_storage_class()() # use default storage to store file @@ -473,7 +514,7 @@ def upload(request, course_id): # ajax upload file to a question or answer if size > cc_settings.MAX_UPLOAD_FILE_SIZE: file_storage.delete(new_file_name) msg = _("maximum upload file size is %(file_size)sK") % \ - {'file_size': cc_settings.MAX_UPLOAD_FILE_SIZE} + {'file_size': cc_settings.MAX_UPLOAD_FILE_SIZE} raise exceptions.PermissionDenied(msg) except exceptions.PermissionDenied, e: diff --git a/lms/djangoapps/django_comment_client/forum/views.py b/lms/djangoapps/django_comment_client/forum/views.py index 6498ea8370..a94b9a07ad 100644 --- a/lms/djangoapps/django_comment_client/forum/views.py +++ b/lms/djangoapps/django_comment_client/forum/views.py @@ -9,9 +9,10 @@ from django.contrib.auth.models import User from mitxmako.shortcuts import render_to_response, render_to_string from courseware.courses import get_course_with_access -from course_groups.cohorts import (is_course_cohorted, get_cohort_id, is_commentable_cohorted, +from course_groups.cohorts import (is_course_cohorted, get_cohort_id, is_commentable_cohorted, get_cohorted_commentables, get_course_cohorts, get_cohort_by_id) from courseware.access import has_access +from django_comment_client.models import Role from django_comment_client.permissions import cached_has_permission from django_comment_client.utils import (merge_dict, extract, strip_none, get_courseware_context) @@ -79,7 +80,7 @@ def get_threads(request, course_id, discussion_id=None, per_page=THREADS_PER_PAG strip_none(extract(request.GET, ['page', 'sort_key', 'sort_order', 'text', - 'tags', 'commentable_ids']))) + 'tags', 'commentable_ids', 'flagged']))) threads, page, num_pages = cc.Thread.search(query_params) @@ -92,7 +93,7 @@ def get_threads(request, course_id, discussion_id=None, per_page=THREADS_PER_PAG else: thread['group_name'] = "" thread['group_string'] = "This post visible to everyone." - + #patch for backward compatibility to comments service if not 'pinned' in thread: thread['pinned'] = False @@ -108,7 +109,6 @@ def inline_discussion(request, course_id, discussion_id): """ Renders JSON for DiscussionModules """ - course = get_course_with_access(request.user, course_id, 'load') try: @@ -219,6 +219,7 @@ def forum_form_discussion(request, course_id): 'threads': saxutils.escape(json.dumps(threads), escapedict), 'thread_pages': query_params['num_pages'], 'user_info': saxutils.escape(json.dumps(user_info), escapedict), + 'flag_moderator': cached_has_permission(request.user, 'openclose_thread', course.id) or has_access(request.user, course, 'staff'), 'annotated_content_info': saxutils.escape(json.dumps(annotated_content_info), escapedict), 'course_id': course.id, 'category_map': category_map, @@ -241,19 +242,12 @@ def single_thread(request, course_id, discussion_id, thread_id): try: thread = cc.Thread.find(thread_id).retrieve(recursive=True, user_id=request.user.id) - - #patch for backward compatibility with comments service - if not 'pinned' in thread.attributes: - thread['pinned'] = False - except (cc.utils.CommentClientError, cc.utils.CommentClientUnknownError) as err: log.error("Error loading single thread.") raise Http404 if request.is_ajax(): - courseware_context = get_courseware_context(thread, course) - annotated_content_info = utils.get_annotated_content_infos(course_id, thread, request.user, user_info=user_info) context = {'thread': thread.to_dict(), 'course_id': course_id} # TODO: Remove completely or switch back to server side rendering @@ -325,6 +319,7 @@ def single_thread(request, course_id, discussion_id, thread_id): 'thread_pages': query_params['num_pages'], 'is_course_cohorted': is_course_cohorted(course_id), 'is_moderator': cached_has_permission(request.user, "see_all_cohorts", course_id), + 'flag_moderator': cached_has_permission(request.user, 'openclose_thread', course.id) or has_access(request.user, course, 'staff'), 'cohorts': cohorts, 'user_cohort': get_cohort_id(request.user, course_id), 'cohorted_commentables': cohorted_commentables @@ -412,7 +407,7 @@ def followed_threads(request, course_id, user_id): 'user_info': saxutils.escape(json.dumps(user_info), escapedict), 'annotated_content_info': saxutils.escape(json.dumps(annotated_content_info), escapedict), # 'content': content, - } + } return render_to_response('discussion/user_profile.html', context) except (cc.utils.CommentClientError, cc.utils.CommentClientUnknownError): diff --git a/lms/djangoapps/django_comment_client/management/commands/reload_forum_users.py b/lms/djangoapps/django_comment_client/management/commands/reload_forum_users.py index 5e7e268270..e84771d615 100644 --- a/lms/djangoapps/django_comment_client/management/commands/reload_forum_users.py +++ b/lms/djangoapps/django_comment_client/management/commands/reload_forum_users.py @@ -6,10 +6,11 @@ from django.core.management.base import BaseCommand, CommandError from django.contrib.auth.models import User import comment_client as cc + class Command(BaseCommand): help = 'Reload forum (comment client) users from existing users' - def adduser(self,user): + def adduser(self, user): print user try: cc_user = cc.User.from_django_user(user) @@ -22,8 +23,7 @@ class Command(BaseCommand): uset = [User.objects.get(username=x) for x in args] else: uset = User.objects.all() - + for user in uset: self.adduser(user) - - \ No newline at end of file + diff --git a/lms/djangoapps/django_comment_client/permissions.py b/lms/djangoapps/django_comment_client/permissions.py index 7d21cc9783..cc3ead53e7 100644 --- a/lms/djangoapps/django_comment_client/permissions.py +++ b/lms/djangoapps/django_comment_client/permissions.py @@ -73,7 +73,6 @@ def check_conditions_permissions(user, permissions, course_id, **kwargs): return True in results elif operator == "and": return not False in results - return test(user, permissions, operator="or") @@ -89,6 +88,10 @@ VIEW_PERMISSIONS = { 'vote_for_comment' : [['vote', 'is_open']], 'undo_vote_for_comment': [['unvote', 'is_open']], 'vote_for_thread' : [['vote', 'is_open']], + 'flag_abuse_for_thread': [['vote', 'is_open']], + 'un_flag_abuse_for_thread': [['vote', 'is_open']], + 'flag_abuse_for_comment': [['vote', 'is_open']], + 'un_flag_abuse_for_comment': [['vote', 'is_open']], 'undo_vote_for_thread': [['unvote', 'is_open']], 'pin_thread': ['create_comment'], 'un_pin_thread': ['create_comment'], diff --git a/lms/djangoapps/django_comment_client/tests/test_mustache_helpers.py b/lms/djangoapps/django_comment_client/tests/test_mustache_helpers.py index 7db3ba6e86..d5a403ecb8 100644 --- a/lms/djangoapps/django_comment_client/tests/test_mustache_helpers.py +++ b/lms/djangoapps/django_comment_client/tests/test_mustache_helpers.py @@ -39,4 +39,3 @@ class CloseThreadTextTest(TestCase): self.assertEqual(mustache_helpers.close_thread_text(self.contentOpen), 'Close thread') ######################################################################################### - diff --git a/lms/djangoapps/django_comment_client/utils.py b/lms/djangoapps/django_comment_client/utils.py index 9bfb9a9d0d..c79cc4cb89 100644 --- a/lms/djangoapps/django_comment_client/utils.py +++ b/lms/djangoapps/django_comment_client/utils.py @@ -1,3 +1,4 @@ +import time from collections import defaultdict import logging import time @@ -104,12 +105,12 @@ def filter_unstarted_categories(category_map): result_map = {} unfiltered_queue = [category_map] - filtered_queue = [result_map] + filtered_queue = [result_map] while len(unfiltered_queue) > 0: unfiltered_map = unfiltered_queue.pop() - filtered_map = filtered_queue.pop() + filtered_map = filtered_queue.pop() filtered_map["children"] = [] filtered_map["entries"] = {} @@ -174,8 +175,7 @@ def initialize_discussion_info(course): category = " / ".join([x.strip() for x in category.split("/")]) last_category = category.split("/")[-1] discussion_id_map[id] = {"location": module.location, "title": last_category + " / " + title} - unexpanded_category_map[category].append({"title": title, "id": id, - "sort_key": sort_key, "start_date": module.lms.start}) + unexpanded_category_map[category].append({"title": title, "id": id, "sort_key": sort_key, "start_date": module.lms.start}) category_map = {"entries": defaultdict(dict), "subcategories": defaultdict(dict)} for category_path, entries in unexpanded_category_map.items(): @@ -202,9 +202,9 @@ def initialize_discussion_info(course): level = path[-1] if level not in node: node[level] = {"subcategories": defaultdict(dict), - "entries": defaultdict(dict), - "sort_key": level, - "start_date": category_start_date} + "entries": defaultdict(dict), + "sort_key": level, + "start_date": category_start_date} else: if node[level]["start_date"] > category_start_date: node[level]["start_date"] = category_start_date @@ -284,12 +284,12 @@ class QueryCountDebugMiddleware(object): def get_ability(course_id, content, user): return { - 'editable': check_permissions_by_view(user, course_id, content, "update_thread" if content['type'] == 'thread' else "update_comment"), - 'can_reply': check_permissions_by_view(user, course_id, content, "create_comment" if content['type'] == 'thread' else "create_sub_comment"), - 'can_endorse': check_permissions_by_view(user, course_id, content, "endorse_comment") if content['type'] == 'comment' else False, - 'can_delete': check_permissions_by_view(user, course_id, content, "delete_thread" if content['type'] == 'thread' else "delete_comment"), - 'can_openclose': check_permissions_by_view(user, course_id, content, "openclose_thread") if content['type'] == 'thread' else False, - 'can_vote': check_permissions_by_view(user, course_id, content, "vote_for_thread" if content['type'] == 'thread' else "vote_for_comment"), + 'editable': check_permissions_by_view(user, course_id, content, "update_thread" if content['type'] == 'thread' else "update_comment"), + 'can_reply': check_permissions_by_view(user, course_id, content, "create_comment" if content['type'] == 'thread' else "create_sub_comment"), + 'can_endorse': check_permissions_by_view(user, course_id, content, "endorse_comment") if content['type'] == 'comment' else False, + 'can_delete': check_permissions_by_view(user, course_id, content, "delete_thread" if content['type'] == 'thread' else "delete_comment"), + 'can_openclose': check_permissions_by_view(user, course_id, content, "openclose_thread") if content['type'] == 'thread' else False, + 'can_vote': check_permissions_by_view(user, course_id, content, "vote_for_thread" if content['type'] == 'thread' else "vote_for_comment"), } #TODO: RENAME @@ -318,6 +318,7 @@ def get_annotated_content_infos(course_id, thread, user, user_info): Get metadata for a thread and its children """ infos = {} + def annotate(content): infos[str(content['id'])] = get_annotated_content_info(course_id, content, user, user_info) for child in content.get('children', []): @@ -382,8 +383,8 @@ def get_courseware_context(content, course): location = id_map[id]["location"].url() title = id_map[id]["title"] - url = reverse('jump_to', kwargs={"course_id":course.location.course_id, - "location": location}) + url = reverse('jump_to', kwargs={"course_id": course.location.course_id, + "location": location}) content_info = {"courseware_url": url, "courseware_title": title} return content_info @@ -396,7 +397,8 @@ def safe_content(content): 'updated_at', 'depth', 'type', 'commentable_id', 'comments_count', 'at_position_list', 'children', 'highlighted_title', 'highlighted_body', 'courseware_title', 'courseware_url', 'tags', 'unread_comments_count', - 'read', 'group_id', 'group_name', 'group_string', 'pinned' + 'read', 'group_id', 'group_name', 'group_string', 'pinned', 'abuse_flaggers' + ] if (content.get('anonymous') is False) and (content.get('anonymous_to_peers') is False): diff --git a/lms/lib/comment_client/comment.py b/lms/lib/comment_client/comment.py index 2f93aff6b3..324de7923f 100644 --- a/lms/lib/comment_client/comment.py +++ b/lms/lib/comment_client/comment.py @@ -11,12 +11,12 @@ class Comment(models.Model): 'id', 'body', 'anonymous', 'anonymous_to_peers', 'course_id', 'endorsed', 'parent_id', 'thread_id', 'username', 'votes', 'user_id', 'closed', 'created_at', 'updated_at', 'depth', 'at_position_list', - 'type', 'commentable_id', + 'type', 'commentable_id', 'abuse_flaggers' ] updatable_fields = [ 'body', 'anonymous', 'anonymous_to_peers', 'course_id', 'closed', - 'user_id', 'endorsed', + 'user_id', 'endorsed' ] initializable_fields = updatable_fields @@ -42,6 +42,32 @@ class Comment(models.Model): else: return super(Comment, cls).url(action, params) + def flagAbuse(self, user, voteable): + if voteable.type == 'thread': + url = _url_for_flag_abuse_thread(voteable.id) + elif voteable.type == 'comment': + url = _url_for_flag_abuse_comment(voteable.id) + else: + raise CommentClientError("Can only flag/unflag threads or comments") + params = {'user_id': user.id} + request = perform_request('put', url, params) + voteable.update_attributes(request) + + def unFlagAbuse(self, user, voteable, removeAll): + if voteable.type == 'thread': + url = _url_for_unflag_abuse_thread(voteable.id) + elif voteable.type == 'comment': + url = _url_for_unflag_abuse_comment(voteable.id) + else: + raise CommentClientError("Can flag/unflag for threads or comments") + params = {'user_id': user.id} + + if removeAll: + params['all'] = True + + request = perform_request('put', url, params) + voteable.update_attributes(request) + def _url_for_thread_comments(thread_id): return "{prefix}/threads/{thread_id}/comments".format(prefix=settings.PREFIX, thread_id=thread_id) @@ -49,3 +75,11 @@ def _url_for_thread_comments(thread_id): def _url_for_comment(comment_id): return "{prefix}/comments/{comment_id}".format(prefix=settings.PREFIX, comment_id=comment_id) + + +def _url_for_flag_abuse_comment(comment_id): + return "{prefix}/comments/{comment_id}/abuse_flags".format(prefix=settings.PREFIX, comment_id=comment_id) + + +def _url_for_unflag_abuse_comment(comment_id): + return "{prefix}/comments/{comment_id}/abuse_unflags".format(prefix=settings.PREFIX, comment_id=comment_id) diff --git a/lms/lib/comment_client/comment_client.py b/lms/lib/comment_client/comment_client.py index 862483a75b..9b1a0baee2 100644 --- a/lms/lib/comment_client/comment_client.py +++ b/lms/lib/comment_client/comment_client.py @@ -29,7 +29,6 @@ def search_trending_tags(course_id, query_params={}, *args, **kwargs): def tags_autocomplete(value, *args, **kwargs): return perform_request('get', _url_for_threads_tags_autocomplete(), {'value': value}, *args, **kwargs) - def _url_for_search_similar_threads(): return "{prefix}/search/threads/more_like_this".format(prefix=settings.PREFIX) diff --git a/lms/lib/comment_client/thread.py b/lms/lib/comment_client/thread.py index 8911d5a2c6..60a68dc3ae 100644 --- a/lms/lib/comment_client/thread.py +++ b/lms/lib/comment_client/thread.py @@ -1,5 +1,4 @@ from .utils import * - import models import settings @@ -11,7 +10,7 @@ class Thread(models.Model): 'closed', 'tags', 'votes', 'commentable_id', 'username', 'user_id', 'created_at', 'updated_at', 'comments_count', 'unread_comments_count', 'at_position_list', 'children', 'type', 'highlighted_title', - 'highlighted_body', 'endorsed', 'read', 'group_id', 'group_name', 'pinned' + 'highlighted_body', 'endorsed', 'read', 'group_id', 'group_name', 'pinned', 'abuse_flaggers' ] updatable_fields = [ @@ -27,11 +26,13 @@ class Thread(models.Model): @classmethod def search(cls, query_params, *args, **kwargs): + default_params = {'page': 1, 'per_page': 20, 'course_id': query_params['course_id'], 'recursive': False} params = merge_dict(default_params, strip_blank(strip_none(query_params))) + if query_params.get('text') or query_params.get('tags') or query_params.get('commentable_ids'): url = cls.url(action='search') else: @@ -54,6 +55,7 @@ class Thread(models.Model): @classmethod def url(cls, action, params={}): + if action in ['get_all', 'post']: return cls.url_for_threads(params) elif action == 'search': @@ -66,12 +68,11 @@ class Thread(models.Model): # that subclasses don't need to override for this. def _retrieve(self, *args, **kwargs): url = self.url(action='get', params=self.attributes) - request_params = { - 'recursive': kwargs.get('recursive'), - 'user_id': kwargs.get('user_id'), - 'mark_as_read': kwargs.get('mark_as_read', True), - } + 'recursive': kwargs.get('recursive'), + 'user_id': kwargs.get('user_id'), + 'mark_as_read': kwargs.get('mark_as_read', True), + } # user_id may be none, in which case it shouldn't be part of the # request. @@ -79,23 +80,57 @@ class Thread(models.Model): response = perform_request('get', url, request_params) self.update_attributes(**response) - + + def flagAbuse(self, user, voteable): + if voteable.type == 'thread': + url = _url_for_flag_abuse_thread(voteable.id) + elif voteable.type == 'comment': + url = _url_for_flag_comment(voteable.id) + else: + raise CommentClientError("Can only flag/unflag threads or comments") + params = {'user_id': user.id} + request = perform_request('put', url, params) + voteable.update_attributes(request) + + def unFlagAbuse(self, user, voteable, removeAll): + if voteable.type == 'thread': + url = _url_for_unflag_abuse_thread(voteable.id) + elif voteable.type == 'comment': + url = _url_for_unflag_comment(voteable.id) + else: + raise CommentClientError("Can only flag/unflag for threads or comments") + params = {'user_id': user.id} + #if you're an admin, when you unflag, remove ALL flags + if removeAll: + params['all'] = True + + request = perform_request('put', url, params) + voteable.update_attributes(request) + def pin(self, user, thread_id): url = _url_for_pin_thread(thread_id) params = {'user_id': user.id} request = perform_request('put', url, params) - self.update_attributes(request) + self.update_attributes(request) def un_pin(self, user, thread_id): url = _url_for_un_pin_thread(thread_id) params = {'user_id': user.id} request = perform_request('put', url, params) - self.update_attributes(request) - - + self.update_attributes(request) + + +def _url_for_flag_abuse_thread(thread_id): + return "{prefix}/threads/{thread_id}/abuse_flags".format(prefix=settings.PREFIX, thread_id=thread_id) + + +def _url_for_unflag_abuse_thread(thread_id): + return "{prefix}/threads/{thread_id}/abuse_unflags".format(prefix=settings.PREFIX, thread_id=thread_id) + + def _url_for_pin_thread(thread_id): - return "{prefix}/threads/{thread_id}/pin".format(prefix=settings.PREFIX, thread_id=thread_id) - + return "{prefix}/threads/{thread_id}/pin".format(prefix=settings.PREFIX, thread_id=thread_id) + + def _url_for_un_pin_thread(thread_id): - return "{prefix}/threads/{thread_id}/unpin".format(prefix=settings.PREFIX, thread_id=thread_id) - \ No newline at end of file + return "{prefix}/threads/{thread_id}/unpin".format(prefix=settings.PREFIX, thread_id=thread_id) diff --git a/lms/static/images/flagged.png b/lms/static/images/flagged.png new file mode 100644 index 0000000000000000000000000000000000000000..c3de8577330a8719d91375a7b23b85c6e29fdc00 GIT binary patch literal 40840 zcmc%P1#I0=_aOMBNdpZta~f!vnVFL&4Kp({Q^U;6&@eMIGvf<0GrZ(&`^`wZTAE$` zceIwSeQcjAAKO>5t)Igc?>?yPKQ>g;CVXaXi=Y-eafEN!RNum7&IXz8Bnpi6^-lIEoA}+hcN?r zVf0Fn6mN9ipj>9SSj>D~P1_QhFuxh6bZFD3-cNA>UenVFP642g6~Do7PLEKNuNxgp zXIrSc6K?lQ9j}gJYHE!&sxqsp6Ni@Fntf{bua?c)Q%5vRdcx^7DL4GN3}zFD@G%YF zwXqsrt@(4Ojtu#+8h)pRQP$>$`8fQT9?XAf;6#OUkiZX1cLzIu-(%D6q9Kf=TV;QA zrPvNjwFd;agZB(~DD0%p3~9A4Y(Ew{b;CYip!JLLqQ~xpx%Di)pw&UcoUONYyXxt^ z6KcA`^n5@!k*ET66#*ZTK8bKE95$`p8W4A%2d8?szc;g#Y<5G~a{8X#0m9x8yq5zt zhye=%eGVkuf&2K?AIhD93VtXfd(f?SY+JES+jChX$V7r2NT+V^ds>`?14<-6a2W17 zu20+mrvN^9U#G3#F0Noz!bq)GKTR9!$uq}>m&NC$l^;=Ibvv0qt&G6UeP$}`RBrIJ zHvL7d!;JUGU!ZH5pzmFPWR2!qq8F~;poLDr)1Ja{4r`-!|PS z7V-TVT<^SF`34Dg4OVWb`^;LnFJm_r1M}VzQ6ZRKy=o^m_F>r#RP-VAi3UOm0&}DE ziN3sn9oW$XY@9|nZhh(1n$415247PGx+-?Xe%5P#Zhe^pYgCk5{mLJQHam_uR>O{t zsVUem0%yBx)>#A#Q!Y^8ZQezk)Kkg|cCjBa8S6AV1j&$h_X{F~OIiTzQ(zA02ahYs z^7yKj=Z`K0Q_k>=YWOCB;Oy22gi(-i0XG?e@;@O532JC-;en-nIW7G8>8L%O55Afglx;Hss$yR!5)a3tUMF1(vK3VHX;VWRuOX-$s2|DO+g^{M|THP>{rb_`C)4>#bywE@^u7eDWOyF z?*ud$0HOT3 zi(wvwgA8O;wp@|sASx;VtCIrZ#<1pzg|bI|8yr?s^sxc8@SM1`#QP0&7?UQ6FY@d> zXuL|FN<%!fVWWU^&Z*Z|i`DZ(f-Mvd8WE95Dw6HP0`SL6Z#FD#f!+zYt(3>@X`bGO`<{?g)w+kl6 z*0&z?J5|hQ>ZQCFCCT~sV)%Y_p#}$f3}G}B;NIQK4-np@7u)eci-i$-Vo)A zGUz>&$Un2e&+_+w=L5^ZcR)~)RoRUAsLp6`F>o;?<=CiW0B5AmyT;3K&5zbD+iyKgt`e zI)oK=GEVV+OsQKxf1g%GUn`H$+gGGmQ2*i5R}qV*JGlcV*;)Zb;K*Gr&-)GQbkCEY zKqXHW&xhcG^oaZ}PdVu@NW&tAZ@lNuFLTz~dnVmzs_PWGb>08kQ__&5-|zB8=Dz08 ztW5efwjrL)osJ72rx|q{OU7{Vnl+pCn)(ZfRODiNwq`ENal8(1P6k%$Fa4=;-4lRt zFBld}KpZFFmQcwiBJkWA-sb?>76Q7n_6dtMu+n-KFs82d7+@^K7@=s@NCv(y6P^0S z9Sf6Xy;ud`8`kY3`bze?>DR7^T1gIpncxv+8=caXbH57+Z8c1L91j2oF%5KY|Jsx~i z)TE?DSN)U*)uigRY@SZP~dHp*mzjBbx*ZY6HnF3f|NKAAu?pIsM5t@H?C zS)wM;&`Ym+`Sg>#)?ot1pIxn2&aP=F988DR4x_@VcY5zY31*P$D`In^0}$rqIjZ8- z?OnH~;LYu21efP>I$Hj{ki8T1W!uZrJ1st2W`yW1M(W^?;18ikt$sgUoIS5wy(bic zCk1C3bZ*6>o#H0A8Py-R*Z$LHd;ItggnPD-fI+|FIajD&99HnBCx(r%Xqz|>g}tkJ z+j{aRP`v>r5vct_LD-w`b3#2Qw(?U6jn7gK>2<7F0G~-j?4U>bGpPMG^iWTuEk0}8 zPJac)_9Q>o&I@5DP0&VhUSh;Aw)1$Fq`O)B$uR{V5bq~^{dU(-vwEU#<(E8@I|pKN ztdnXuc!>c%Gn@Ej_?_9^5Rd*4k|Nse>HJ1cbpg7#%W&IN^37Ti!m3?n$R5-$a47~J zvcXursPUDlmN(zXE2EO^AbK|?M2j{q^OaU)gPJ~T4pwfKkO*OE-6h%Ei_O~OD+>ko(=BO`!j^!ipW z4+u*q@P~@-QTQ$&VNA~3vS%2q70|~GbIS+1UO6MGOZrrv0%^W7B!QwqK~P~UY;UEZ zcj3$+^g26=^Ou?_5c)QBI|zxSOV#(O2U4%;$uIuJ?j022r|UN>`WiIdsi%mUCVJ!X zDdF-!uMbN7@wks<_9^i$z#w2zu=vz-2~^1c=u!GudG474#!8!D)%Hy6ZeX|xh#lvy zf?R$mWFF?`b`U9IMjQ&9gN*$V?)~7o)#gFSGd;_oU)S9Rp!b|*d_sQ&{=8oaLJl89 zp@azML|y_6;5f(mR}nL$FP^}W`)Y&<5Ql?2X0Mfn$lp7?KUXSjHw)5Dd@S2GTLLny z_TBU_-~bbf9uj8Qv^1 z|9*LCuEZ`vir-SU9VdtY{0^eQwUzrbGrcC{@w(bam zm!8uX(BwpSB22fy=gQ|w+NaKjKvb0gbu0*P85CGBb1aS|L_Gs4nR?Uf`Xu`V3+ zAajs){LDV|raIX-K|IAuvDsh0MVy07KMT2DC;S4?lb+K+ZL+&PUVEkXB;AJsAh5fY z8ozp=708bDvztoHKX3LytUkT*Axc=JRh#q4kuZu?(0UhLSQ^c;ssc0=r04#$D{&IU z8*vCyA*W->PhqA1dk5c1^B-xwIp(0Hi_3f~8_lcgm3O@ug`Yh&x3A=EIYnuEN zyWTp`>feT6)mRiPKei&(fdJ7rEz&tOO}UBK*3zR_5o6$OKx983^Lbu%2Yy7)*(0=D z9!BmQ74is#x2vuPg2bfdBl--WtmCTh-H9*L($DOA+v!S`PvQB>$&7D0ASba^LUpfx zuwv=sTs3mZ@3+N!rnjpk&S*>@VjvSq?9>H!G~a#;ubUeR-LT6I`>c?Lk+Oo3$;U8G zz2-qO+X(R#O4Z}Wg68JJo0v`4JpW6`1GmG~&*vNNDL%8tQQlTScIGuEB350G(>C1a zw1=|Ha8EzG>|o95=>y(WT8HXbc8j)l2ZAhM0*c-U%Oyv}3~MQ;QO&T{`{S*Vv!j4g zj^R+srS7;shtd`4#%6 zyJB@-tn<2i&T8|$WTkot0Hrzj3(Ri?v?b&{>+a%mooPWgO}ZTl;CJmdt1*fVf8V6d z*z%P>^dtAkDCBUo2mbx;lC-fB7Ju6w|NI+6{%!xIiMog-9}J8a-o{Er**)v>@AG&= zJcXY-doUg#|7MKYFvqpf15|ZDQwS_o^A1Zelr}_5qBGlv2n?b<3D=lez{=fz8lB^@Uxe*ZEIlvj+OmRc^m^_KNwP=J#sVzK7C zJxT5-+*{Z7luZSa-}=<^iGued0!V1RO$$)`Z2iI8@ikNVRsAvPriEFlD*?!ud~}{*g7>3H=HCYT$Qr>|z>@@#7&8V(^`3&pi_SQ2hVc4g(3asxxbg|a z7&}Lp@B>{x;2tg^@N-Me+9{M9*96NE5R2LGz6qYL!Ar0AFf%rZ^D`1TGH&GAW8HA6 z=|X5%-*w!Hzm9@$g;O7Dy?p9=2%yO(KWqM)k$bnCigVjokFvE|#!eW74lBAox@ybF z#*^7v#!hMHrjOdXK-3{(IkAo<;m}ABQOY!-Z%QyuMXIj{`+#^mit5xLz*sWuwnZTswijj>%aSZp1yj*%DTvW*4no{H3FAKLNVhCuPpL4QU@Nbf zYp~-IXXwS$mYFMq`$54;&;VrSWebRHcJR*fjjZcgkdFWCf~ny+Sn$eY^3TeXv#HON zd?j+4;~SA#;-xnHvb)l?_rpXTo;lU1BsO}pxxu?M7|*g~!e3Nb8GnpF9>kXCp#&oZ z7&YMfm~|`=h({X@+u)2lNTel_2%drMLLuikN(t6O);LNH)`R*@(wzuTAJHqrZe+_T zbf%3BfPI#|F@|h4pu^h%-zk@)fM=OApB%1?3ZkVHo60&R_R_rNCu|!j8O}4pJ{pkE zJ}4QMyk!*t-;#$U(2R>UmYEnd9h;334o+x`PwNu`$G;Zcj;H{#Q%V@JpyR=dInEhUEn|w!=~FRdB=(!P!%Nse#h6By zg5&Zc&F)A#4mGqMX^3N{Su4QH+H*%zA0m5l|0%lDir3AOp}F^%acnA@jtcQ?bp9lR z5b@!JbVP1poqU>9n?ZCr;u!QBbN+Q9c_q&rUVTKB)$RSbDXm$~sl?_TRT1a^U zc1on|+wEN1-dWZbDB0( z;)K3qakn4VfE=zZ4(W^^D<@?+H>g3k|J$0#oGqrPAr-UO&NRcwjj|TaF`} zPKd^^B@PH%tY6?9qJkdjpwA&g;WOwe)fuwn`zMAG1<$U4z^+QhZ}_9kY!!$#^s(l)>t=3 z+ZV$&@c0`BbRVyJGaYIL@-LVV7@#$(7*gf*9rcdbXbYf4)6Qm4>CYP zL^V**%5Dy36;X;{#c_a4L-*-L7hvdgBXh5zE)7gns=j*>*v)h0ToZPo;6 zB2UM7UCw`XT-(tUXR;UBQ4tuANA_R$E_o5k5jk35y1TXOA_pBYEAuuTcl6_fBbG8r z&(@F>0+8k7DP;*q(s|>8Y#ismP6#7l=S}-7vw)%nE4q^s24G5&6uX1$EL?si=kA_x z3G%zqQgg}ca=-J^JkjhxkYg;-XL3rC-PqWre) zSPH_d!am)p2LpzlOkMNl^c^3^Hz%ApdWd;>uCvX@s2O${FkI*xaOj(_9;|5mtWw-m z@a6{-+c5V}1+CnAU!X&=;Vg{~?&BsJXKRbl$IK_axo5@OB5o#j$W5EACNp59Gl=%R zgTyoF7L-%KG)en+)8YwZ-ZNgoQ$~{7ttZ1gT~z@LP)6~2o5$OQxetUs9MEAN{ZnFS zf{qwV#xV7NcnQCX1=;MukHs-SZw~m@bYi@q%FXf&jYw&jSDRp$m-I0?Ilaj#?Ge(wibdG&u-0q2pJocm|(kR8coJP!s~v#?f&ki|nzMkOIv z@}DrmtIgJ}`W4x6ga|ZmdCEhPrhoTl`~BMm}X z;tL>1Y#m1`vx-uU`e;Igf@^-=T^td1nPv44op|2$7fP8-c2Nz4iu`xQ@m6xBpr3bw?< zyht<(jPtDX-m19OAmlG$h(9S6ZCiIKEEX>=2Wh&($5GgI1K5$`+S{@nt zDlzkZ#Cos@iUKfSh|KKLJ?-%mqEO!0_)j%P{d+w0eF%k=(YirGh9^V6X)=M6;(hEF z@wy0Gj9heV?!fzk?x0y0e#wsM`#dkywM{t9MRj z47xg%SK(MwWCYA2g0xP87sw|)<$kN}y9ElKd0~*= zyZBjBz!;K*OJ!c^-RiZ(!#{32x2is||LphLIp3<87m+nwXz0}XTuXYhXZ7k@i4+743 zC5rABxvj%xTHW`as%CyCN}47#8j6^>Iu^lyeB^DMZ*IEoE8yNp(IS37m^Mn3^g$hA zini$dE|hIbxVv}4EZ81}*A5?gMQGvRHk(fx1kU5y%Wl>R+kQ(G^D9i?37m{QU;37; zG+ko+mz2@3z$ssqLP|v<2HaR|C!5X~gja1b?TcG3BJ(%iLynzz`-g%K5Wc^FZ9^nc z|MLewe^~JZVcpL{P~a*F8}DFlPvD5|UfufMijs^jDKBIXY&$VYyO*}Jjc0~GG{K87 zX*W1DQ@IIEP0WK1=GK7xherAWzR5W3vPqx|O0;(psK+-T#5>jJmEJ_YDOKG zN({RDdKP7!49hz2OFP4oWciF=WEE3+noM!DG{wTYOyNsK#w7^(@jWth-Df3vzW{ylV`% zlxnivi%V&wr!M1-k}?U7Aw-aBF^>?<5~KBabjIs&@BGN%<11&|^t}6H zARik;d#i1=T=s}lT5cFEy5{S-1gO>JJ@TxRaEda2h^LsmSfG{Rw92tHXWNodWkRgx zwp?6N!fZVoyg^`r(4NS4XJdogeE9vVPQ1<8sG_3BU<>9WW^^OXllUGOGfSPO)As5s z(S2@>T`GZJ3Qp%7=@}EO||ff z<{#e#4>SKVNT5xfTX{x-H~dOCvPJ@!D#W-TN@}^k@R#jtd=Z+9z-6L2n`W#lunauM zFbkFCJ*2%4U-B7fzY#rgs@rCd-O7I6St95~VfC zJh#EuP~Q0Wn^V@Sy#$8hkpPH)wT5i_znM{Ir5|!b6Gt;n; z%lWbC9!Nxu`RBk@*Qse~36PL>2sfTgkKVAex%b@Ie4@#LJWQQPMo=_P944?UA9^Qv-#*ctE51S-+RNwosCi*~Kwt)D zjBluv?dXJtC~(R2zG>z)F@opvSn`;$PrK~PBoMej1FD$ z4|xDuB zT2^*YgBzzANZRLGt##x)BA+HimrLXvLqrW;hshZ+q&w5p{KoD_Of{$H626TJzK(go|>HOsfcVE*Xq?a!5Aa3Kf52bGr#_JeJICSFUM{-Jc-DMV|2DY zsQnkIP0y09sQ%lVIys9f_epBDU53G92hO9$a{a^gb92q3JUg=1iS536@8WXj!tzMx zUyaAR7YufP!`Y4A*`>*jeYRqA#T@bVrvLS53CRDK_V1*E^Lw)Xg?11k|Bf0`jqN!;OF;GejVRIV!C7#4cA=E2 zTRKUTL93(J4q?cD2D{pf;dhmpRo$PUYQ~Wr+r0~q`^tTjH8kzg3qzPqWQNRQr$-z* zDW@Vtm^7=oF2!6|wDzAW&;1v;EkE?Ksx^ek>v(k<;LdS9$<`?=ZIIfUhc07IqJ2jm z5zMLswq7odan1vWjo%CQCd(C8&@eXYEEF0uOn}s$1?96XGzzSlxz`G>e_xV9p1#2I z`SWC<736Y=M0T4F&Hm?%4!@;L%p#R|mq(d7J`@I{yOrr&5CmX_X9bANbCnw1vgGVh z4z1mLxIxCNBgC+W2~ZApKs_eMxiRrO?e0UYgv4{FH1D<`$#R6;_2JfLtvtCSd=jj$ z2YS7zKnx?+?C7QrB9Bt~^n~$LVQ)hDVqwF2wU3nV+Seglz7o8orpvbcBj-gfO99PPnjw24n zk#y)JFT)sHK*YQkAx(BDf(t3nYc)kbscLv7ZVzBcoh9tmsAYxf^<@#XN zJ>H&EYJoTlA8V3twr(_-95BD^rPsglCFL<*$HW#%9<*(ALE@gihv5)#k&^xHN>$e7 zK^d_?NIV`pdln2FzUbgfGGqeDyP-*jJu9{ZX6I5tKM?Nq1R#ZLq$3SFx)U`rPSgSR zM}4id%zEw{1*jJ|RQO|uhIBK#O>!Ib0U^bc`d^oGl>@@a7_Ucy6weIA;9TAnj%eqn1xq=zwOWefRaDW?j1RT|HC_GJ5&gXAgO*9hbIu<7mLE25R$2mJDoot2O8 z_nQ}OzY2O%mxIAqv?g8Y3$Mu_&RJzbnQFyXTBOkIwb#kzdI_%*>7&MYn8Fae&px&@ zSPfJs)M{V>5>Wxq3YLLsU7z2OrO*12(#=3<&DDw|nFUKlhRRZ4wY@K(G7tOAs}t1;8w9pxYw;NX|#@cCHTBqQ8?oHl%BuflZ!lB@UZSRgBYv=Pow zE^KQj8*lI^Qs&msiIvcQSt-Jh+P%H#SmNHIRMO0+w zyQAgc)7L)oc1>lyW#Aa%{&EojQ9kiL zbX#m$MS{mxv5f1>Z-rnZ^v>Q9xOX1)3(XM|F&D{Ac0~fO=L%(dCN?~>=2e#ut#pMz zRa@KP_IPCIRjr}QNX`Tk%MXr9FHTJ$gb=q{GxC}S&F7u9xtE!MaE6!NkOh40+ZFE3 zk*S(JrB?f!`a9twS*IR1Za*RF@ND}HZ8U-5Vl2yaziU5xsCRDMG_y8+`1jhDOM0qD zCX@7YV>~=^mq>9eT-dT$b6{C~9QWsa*@dTg!CfQBY4P{9Maq=KwraEH8<{spE2};G zg>F)!Ow2L@td_OG>n(L=$hhv<>HM zz0)Vchug#D-p^NC?sNMg=CUC2GgfB3QWQ_1ReTle7>{PC?0l6@sOS~S6IupeB%wPe z5r=j$fHLU)Io5F51@(|%=GkIqeoCK<89^mWb?zCa~JZh>EToo1^TTFe#YV`Ug+4^8O{x-dD@|2r8wHj*|(yv zX7{xU*BP4N2Qzl95wx{a2;OX8!{?OEvw0Y0qZh>HgO{^zkCfaQfl(hn;m>@&4-yc< z^RFP&o)LD$?SeJyg&NaPc0_H{FW-6G`}uVnF@f`-k=Lqlt@d zS?6e9oJ$v?i=Il7fVs?qS5O0eG9ORD8xpiL7bhBrSvG}Erxz5KV&Fd6{t@P97C;l+ z#=BW*t%Sl_e`Dy5n3g3EVYaVM*=WU4rF-c`we)h^0dE2!ldoJJ^woB{mc(+_Hv4C~ zYozg}Et`d1sZBP)of1{A6w9_RIU#W{vBTx+UIxkT$I7SIVP3slU9)x__y~F zbkawRRC5Qc(es-y$Yp_xIt0uQ8hMEPk>?YAUDj{LF2ob;L^~DW(*&f@aNM{dT%0I{ za!HDm`?J3l@Vok7HxdNQ;b_MRdk~F?@(V#QbAnnE=jMSuac%`^MX@F7jgMjY2N5f_CF6&}}1g*boB zApW|kxMV&YRPozY{E^SK)f3L~uEFOj4$#MU^$s3;2HLH{6rGVytn5xKzGMRLnT8;{ z)2Y}byO`;pr?ePkm!gD@l|{KuGt;knR3B&_hYD!vles5EM`H8uLwuGo%=LT%b6quH z$@V%A3bQ3Xr1kb)i-5InT)P*fUVedHe!pYYgL-71{wP+Dehk8odprbJv(AGWl!|Tv z8-uHT64dJ+iqZgsVi@SY9c+9E7sK`pD0H6?SzQ*Jmp9ZN_Qf4zd&%mb9g(hi*ZL@O z6;=v((e2f^wa_xlzs@b>e%U}oTogxKgp0M8k?Q1+YNTdm8Z?CC^IDf5w-A0VPTL`@ z=J=}_K~L=xzM6|L<837x5e9Lhf7QL`B+xL$Z|o4^x7{g({jNMcg=|#WPgtD*9L1Dx>BFa5bstKuo7*O%d0}`~h3X!RitA z&;v|Dx3EBN${(twP6fLr+?3ol?LW%BODpQ4V={@M*<*jM!V4>;0qee^{1osB7QTcX zdBU0{T;h;Ek8LOBKC^r@i3bLf^%% z3AXLt-BYYmh<Y$y=I`nHPN9SOR{;I;~S#GI@u0T=LfwqFIGCY)O3L0b{T-GNBf4M5Snh zgQ>GWe=zS|B4L<9(CVr8M5UX)N`z4Ox-Pm^=^#b-+$1G`p8y|c(@jGuX_|BJmh15w6T;JApd_3!!V>6K*E*7zf=F`ovJ& z$4n;?C?&Ju_IY-5U~{6tI4Vsbc)_%r#)1tfaHH4Kk{#j>i~gju5Ev`aY>$vVuP$Oc z!w-0`mL+a3VpHcFQRZ!g=XDg3Z@im}+19Pn;$$d=DKbCDmy%hrKmVkn8VgUU9{Z0A zi%b~mq@xh~;=Bs0iQ}sBq!MN(_Hu%cbuA znd+UZyk!*(ItItZm#TKhT;ZJ<{gQYY9snzE##D?yzfZt-*h1RHnW}nppL1v_m5qb& zXnqQmfC_oDhdCg&wMe^6ugM`eo3t|wnlfnr;Y#jSnre#DOsrF!BCBRn_@Ex~GZfX! zdi6tjP)Z9;UIh0F^pQ9urN=GLgCcKj+4qJz(BNLVi}O8*)fUwfJ_9%CFN~KlnOkz@ zdrh*U=)~3VGRpnG( zefs^(Fm62>SwkJqj&e>(%^ULnScK)($joRm(){j?`A{2{(;L~M=VZteRZGkqdVDja zgcXvRrJjnwa@E9Q@uf9H6aK0pBSos1@n`~ri zhW0%(`l~@fqN@w4rF0}yibPkg``d z%_Pc>J#Hc72QKp)!Y;Z;|A9l+7zEcU1*J|c?V>;U89XQ3JpiWhOoo~(%Kq(}mS3)_ znzqzh=m#i6ASDT(k|EnC2jd;%20S6cGF1-0DE^fLhxs8U#Eo?VXx9C-bXXy=9Jc28 zQS#joGvoC%%WK(X|GUX9{BOE+`VuYPI>p|;zvI~vT$dwaR=^6l!TuZ$FE|+ zO(`y$(u-_zBXaM|8x|kmkoHe>{EP_wg784XPx3;h`%^)@!yo+&L5XQ=)l62TR2IHb z_XxNa&9Vv_U(CZ{LXE^dIj^japw$VL&+3`T&ek-1da$F!0v&U`Vtwbm9=7m_k6<~l zcipM($ULR-Yo~;PPq)E~EU{vTsG^FrfCZk(b8EcCqrNtOD49-D+#IRSaD z8c0X7+M(1l2>!*{ey=Zv2#*&gHhY>=Ro)*8El&y7J|G9DJM5J2RUV%`-)wv$>xilW z`LNvb`x?je=R(xSVX!Gn{#x>^G|VM9);ggd=G3*QrvvB^94oPM z@N^UPP;_0N)JBDrj%1!kyNC`qvMWN|{Np45IaP@W{W%3Kb!71H*WM43>TD{i!JL5< zjmTTzLm2jqLTnP+R$92NcDOBg8B^&v9dw)20hB$F{;?ngcya>aA0k(rXBJ;xo{ z{(r~oT-aF!l?&e@xtIAkM5zM=gv;MXFZoYlBMQg(3~ExzX0Z+%O8a(~C+6XLo_neB zFKotNR!fWAacs7B*Vju{n1J&Ce^G8I18sUKo~GP$!mp^HiWpoMZ-fARZ5P9rkaY0s zL;22(jIXwD{pIR~+U<3JfwI;%*jM7fMwTzm!&l-2eX@GfyN3+m!XC9i7PH`w*)GN> z(O=!(8EyFW_QOI6<5th~(geS?JkuIh*+GSCox#GCJX4Vy`dAYJTVukXI9g$|FZph) z+_v5z>)Ih}KrO5uE3+%x4BGfx$q-V6>SCCNl3PeP#xFGO8crVLJ!-Rbg~}_u<=>RT zxsi{}9v4r}nyD!M2c|CJbx5I!(0F_4EMD6Qc8vyu*N%fmNG>5pPVYG;8Yy`_@g-|) zJXO6Y^eNvpSJR;O!(Ddu`NYlKJwv{&?1DD8>>?|AzQ6TW>a@?8xXZ0A?LA4MPkvch zV-)6z@9M|m-Gq6CD(h!^fLaug*@Z_C(D=r|Eb%5erN}uUZFQ%zJB$XDtP}j+a-UIA zRsESVNUNtO!b0Nq%Kqx!{-N1w^yx1q%Lw)Y)PGg zUiRPMe_0>f$=+UpF@F6kvy{3OJan&tDL-{+>=GENVw3+*NcM%o-g_Fn z>Y_LmPLFCoikmMl^tzKG8$s{wDQ3DOVp_8M=LeiOzn$_WrLkf@5by zJX)MAPZ5Tv>2(=!tCzQK@c}7Ekh&~^N?-8gyvZxvOn zg0p?^B-yD^=^vqDNMMaRz?*Z49}1~mbb7k7|A$ZwkH2umfairg5AdroISJ|1n)S_a&z3>VM{%%+fg8^v?!A zzOyunCiaY`yIBrRYxPe5ICHHQaw zd*^=!dlFF?dBE8n^_NR|>>&Od(*AW`$(a3#^uIkG*ctz&{j2Z(KhggdZLx04{|nB# zv$Z9c#b=k}(n;$Tc%GyKAAq-K(%@XpU-;c3W)U3RiLM>%Y{6DoE#gdPYNAfeA!8wJZbw{cFhm~%W%?hO&rCpq&mpVb z-0oj>Qt%+cm`Bo(o*(&Gqli%W&(wR0;W{dDW$_W>*40?_b;x%!*q`~KtLl-u#!Xyf z#A^2V*df3UwN2EnKtN5hQVB;6?=iyjTcX(q&boEU#a-IPjmtC-Us&>tUn&7-3}@%> zp1sGVtg2jNalHmj0X`bP^M>oBrMERV6fL42-}=W`7EO>QEPyZj(8-L1IET6dAOidC z{^~_CLNupiOC~SAI~JR190iB^qRa1YJ)BeCDveh`wFerN)f8}rC&_JE5aLty)t@5=Egy_!9=k^ zZ;h zccDICMJ7d9Wa5ZVFQP??c935#P;hKa&{b14mBPH<4rm9;Ymoef$uPHhjlOL4BRCg} zEaPTjWyWn%ufwdjt6#MMXu;?Afy|8tT|B_z zOM*?{DgPAfnl2Evs5%^{%tZJ8bMcP;vHe+?r+Kl$HexP^3VJc6M+iQYYX+(4yE~Eb z;i36E=e(=Uc_W{~dt%00$b@fqvT!b&&lIm2f5~WjD_a+mnYQ8@RR+d`XbxyQ>B$FA zuSw|#{qiVgxwe~c$;F0x9l3);8r_<#c9!)iyM7N^?2OFwgR!Y_Bs1F4vI&LpWmP@u zFV%u^X%&s|N3-wU;1mNck%?ikME9IL+5@c<4S(cX@2=DI82YBoFjAX-ORdxbmGhNf5>2soDpenV)<1TCoR58-`H!{UECT+UwbBNL?3jK2v(~k* zhACWCun^TPX|WpO012pN3kn1y{pf1DM2a2rM5zpazD-IU>_i4bWW1Nf_bxVz{>qdZ zU9`U=9^yK85t6Cr34=Ik@i!6k%qOL_&LJri`z^-Z1-@Ks4X}JC;-y7qHV{g0AOv1t zj7v58g0b2T4~2->NS!s1m6ZO*LRPAwLk$wFV_1nRNhbNu?-v9Ikn(1ogYll=?uOdT zZvo_eIjft9dPB9wH$R-5$I9~w>=ktE=MkJ^y&268@&lI`iz3XCuKrH|c|{8k-%m{d zMG+V0Gag-~ghn&&tERw0JW6hSzR0=C-RjyKgogXJ1k4;=7&zp#p_M%pNWh9j@{FDF}5nrzd>QB-~ar$ zj1%PTzNV?Z2zY93{6^$s^V~1&mYH|EW%3r&@ht?ob@6c1|Oy(2B4=Jb%0`(5zcxtG&d=jF#4l3IV ztzi75x^H{o9S~Cu`8tgKW|aMvcnD#B#A+P%E)yk_^ZbfD4F1S5}H_&(>Q$tZW_{3zYVeyB&g^u&ES+sX@ zW-t6?ryD*?K{k9*SMZ5^WCFg9i-pv*1vNcQeo{;7klB8VswCs^JhBd6axc=bI?KB3 z=1+Wd`0Su^RTK$z$XaJgJ5sweCxFWBVY<;B+o9NVCDXQ+?XyM(Jn3g7>}#<95!!2B z93k{|uuHq{zM*gS5jpTdioffaR0{FVyju_|Yva(V$j2fZDZN4i$v-zL<7|M*62$TE zV*S-Zo@|g_9`QlNw`gbOUfh|ZMVXLs?9ZAz?G$Ej{a4%LNgnKON_+ji=F4wfgiU&J zPkQb0J37?tI7zOt(+#<6wvW!*tRYTRC*^A%rFejdA*gCcJk&m-3M`lKHLfRBP z_1wGQAF4e_n@k4W^m|{8C2ZqQ>(Xa#1U(Sf-CMBBD)NYA9b4GLweg&HPkI5&$Q<~2 zS5+7`I+@+|{z)KyWbH-oP?y2MZ`hSbSLR8&ivjr*}3~d&Z(fO}w*e7Y^@nOKHE=d#H6T_4vmCNog%wK2h#K{ z9yOR4?9kQOxFzSxWvi~c@9@tX-7>a1I*t+|B$7q8HNsp2JGRlKmEXDg&$_0W5-ip+ zZZuZh0p)!BLW>@)iOlHf;wA6NFeel$o%w$@7ESX?9UH5T{< z+h1wkcK_IRN-J?*Y)#k~)21w^tc`cpd);>bW)XY%Rw3ZoTpI>YY;0+}<&q+yECGIJ zBJKT5W=YfDFhhM4(|mJ!NA%h|G|RLxhDLu_QISgZ-8aHv-cwsM{DVsu(uIt??+{y8~XT4tCe0lKiJooY7{S48)lA>RA-#u z%rSlB{{6gy-ecE}u&C6m;ksz=Zl}EpXA|}IzTe){FCV$XS;3yp|xDs=Ao4{)t ztyA~^qj0l$F^^Rf&2Nse3#|d!Xkdp zNAsJ{94IC#rN>>nsQBwc5G#Sn5wT?{9N704u9=IkUpnpt3mJhdkl7vTV#&4#0z z{WE1OdQ>cG(80mFz0NcAyc{~f;c(P0`e%ng(g0TuegsGagTbi&HEU)k3Qd2e+U+@yXH{{FF*t0h3<#_$~dFU=du0*~-z57m5= zTUcq|9y*S`hFyxcvM1!HCbk;d$k3eKn--OAGce}KHj?Kn&1(dw+g6gq0%m7t=c67X zgEE-7%ks!mNpe-4qsa^-ZmOkpvvyTNi~|!1j&#s$9W1=(BqzENO;8D%=-wLg ztP?drSb9CB};BWYTS-9tf~faDpT^=i#eMimTHs) zX9u8kKXdMP1fCKc-4vQzGLI%2DPpc>7^tcDsG~Gq zcBB#_FxS++?bTk6UL)C6vmNfiM9ac`qiL@I7W%;AktNO$P^f{)a^~a9rS<8k7yc?s z3yR1v?89b*5{qT^O4#0#ah@K@sos0^CaLHIB4t^Vc_cbM4DGhC-$TKpDYdTwNYlV9 zsy3i(dlZpI#I}l?(H+)?gO~G21YRHK>uc|CFQ?<3euS03=V+5?9ol>GOwzgFrDOdk ziBB#|ia{1rzu%Si#`-({*OwUmaoNa(s5{yj_J`N35iV z5Hd%g4wUfNEld)Yhs|Tl5SUraleyi2oMgjpj5NXR@1n7pzakd?d&}}k$ZxFdGLZR4 zK{&&0gn#g2g8sb>{1mwqVbBezWuX>hyn=vk=-2l6zSPzk$m5|8Iw z=iRvjt90GIb5na*YC^)-8^`vFYz+JB*9e5mUYeO8;W6mYQ9b4-(kMJVdku7%`4uoF zvmWy`LjgW-qASzn_rn^YEZPJSuUdoTF!ZUPwV~_j9HVnTI7O-PsU}OnTdkQs{Xgd5SnyvPe7^I9}`!zX@T zFmx%3gMpy=OvIE*CvSOSj9DENQSm`-%UQi@8M5YCe(GV`!>V#|XqLlyal%sY^cz=s zH4`IgdIFcr)KlE7!T?9S$*EpPPzWOPc^ zy%zqo*9SApK#Ej|C9*?xG)%1>`~h{(z|uFM;v~3fKZLp4B8A6#j&COJ1E}iaWp(^1 z1GfCq+;}=ao`;kk{Zi5k;-9mX6I>!HBAag7|K#-I+bfBx^QEdvvoYlkP_)YrQZ@Io zmy)eroeCpU+O3j0zSV_cg^r}QvX3B#c+t~Zb3Q}7gEjJnkdSt*m*mr3A6s3;8>kE% zgm#%bv9DD-cmJ3s#l+DR*^UwNU!qQX?ITLPmPsVrvO%foqJD1a4No}B#$SM#dg^`! zq}sZr`bd|Gc3%xvqa*GNI})H4eBkOOe58{rr2jG#JsoDk5W1D-^%Ye1KuJes@i?+> zAoZK`#P@V-yo=90#7ztSbl+}FkI$>sQa_(OR```+dk1`E$l){xwfn# zqf-+-ATftbC13vqE~0qmvgJfKm%o{0kq%J=EWtUQv5OIHifow;q@d^Fp~>rP$l0Cu z>UUSdWL(kov1GftVAnwh=Sd587tQNu*HO@pv@Kr>zb)=xY~p{PHQy6o*kzNxt{CEb ziw5HZO){^=I9ACC@Sdh>OQ}9j){mv57|TA{Ict{u#4Zrfa;{;JhyJtFWh;<;?Up;+Ug~Y7b4!+O`B2%vi8v zf0bTixDC3lNzoFjsSubOM~Zf3#g5EGHn~e?ZIXuK(O62U*%rzy%-CMqrJ#PvPAjocx zwio`8S6^De$b&hb&to^IlJ?EHR;Mr@Bjyt5EvGUxQ_1uT=tZ3f-0ebIAg#%zrPH4f z*xW!b3$u{Az#O+C`NW0I>*UOJ6G#`*z``j?326%_Cn#~E-idT6(>l!0#Rem1{`G`U z+!UH!{il{`c2w1ONsD^1Yzi@@biI8`CrfQvZx)zp!fU#a^c%DEV5fB(uBAL{pi@JQ ztWfa@bA-TsSsGb0`m|di2B&b*F~r9(3dNC(Cb{omu``>+gWGvg_)dOm-Dn{Vl$TQY zz#_4GNjg{)lqx3)#M{!f9A`P@{eY&u1<$d0jzX29e*)Zw?SZ^Wv^_v5_#^M(JHXv2 z6>!8)j9(~uSt-Hw1V3xg6S$lDGph{)tGM236Kb%eekHgrJJe8{@5cV)N|>J^OSwHj zh?-l{AOSpX@EQ+*p0&jwuV7jyD)-)N3umPY?c;(c{3on}b6m2UvUBq~T|%V(x_ORH z4cbAhbiNp+y=W%L$_n8}*VB7#d0iWq0TLx_N-Y zj8*aOe68wKAJ1M4B30OT%Aa%~u*{&-+>PF*nkYlJd*k~)zQ%Jx(qx1~?FL&=qF;%j zoFnDUA!!RrxihK33@W0^l=V$HTG=7R=bW^fbsZ-_r$V*oBTVckK@&OKP&T^OZ__ zSQ7vpoLC#!PPXM_vxD+xwi$h(7PqjmQAB%ym=A&JFDa8eiJjH!`n>{mD(rI$I|=dp zJUsCxnh7+_jX~Hr6H9Uxh*fsI&vCJ>9+2jw$zubcT*%LsKXrYA1?!@2+ip_%J9Ln`tL288W|qJHj@Frj#- zN9Ap}xu?eSAZpJ%GP`CQ&TA?pUURP?Qp{G5OJiH6vc>`|>f8AC4wbG&S1Lq@C(5j( zPcbUxiAP!4tbQk1)qM9$4*di z=%q#4azI@cl>2DFj`zy9;c?1wp<)b978I4VfUue_V-Kl!ayaW#^g%m&Ae=%wUgS`bDt{Yb;=={^m$GnWOVIqAtZSX zIe?Ovzo$pvfVqbre7{uAYftpSD|70Sq@sr)J!1ORIm@x#LaF|kG=n0XoWbENE_Pb; zQdY3onU7_A)x=rHU#3a<;uKv$#r&gcp>0tP&+VH{%dfPjsQW}2m2Y~M9tf^2M@+>3 z++uL<)6(MNv$n660&ndCjcU)nVRtW}_gQZyE;Qex#4D*}t-`3!O-jz}yRIkAbG|7z zplHf8^j1EwJ9U3M6X5pL5r3%z zAGesJb6QP!=xfSt)|4T>`;BaZ!V}%;KnVJherJ##El6^{VuQg;O6bWF(E;be@A&H6 z-I19rOZG_cX#H;CN@qbMc5>ill$lImtIla*sd$=CZtWak&q)&pMUypz>F&{|yP#T@ zNGO9@{xCHsi*}87q_me@+x&BQvn8JhZDL$EmwG$%;P?xA$&&8)&iPwl!pd~t7%)grbn@YaMMgIZ>Y4aBpEwzz77 zei$=6AGqNT`K#6Z>mm26KQYhzxdYMB$Dzl3MTr5~jOs#MC((7s4X?|L;0IuTuaC7v zS393aluL`DNMI9nO{XYFJGId)z+bo3h&j9Pk&=gX{#GN;=p%zmmGO7H=5uI)h^6ib zw#@d(9?7AC$jhAdEKrt*)zmbtLBrqiJ>yw6pwpZ8a)OMx2k5mW`8eP_(Y`MGbiy_7!KTBWk@w=kYoFGpA;j(?<5O%s=R zz6uOx^_(3Eoufc{;g@V~Q-%+uKW+(8J-e6=`Y>n}+SQ2i0Xe>{Srh!J#<+5Dfz;P0 z)QV%i%p1o=yYYqmGO0JsKpK;r(DcrHd~nf~en+H9-W-&`ikpn79*Aa)@aFw4k0hIE zVSH|!0cKn7NZsNUX}*+iCJ%lVbOO57w*chX^Rsz;d*k%Fjr$jjLCY#r;G=Obq$oYvNC93wfwN**%qLYEQgsl_G0NlGNas zu(oCA0E8( z4&rCQ5=7}Q{-=wZhx*r@*MINmA6?u59g%Zp|LJSY^PbO>0vG4DoNu(rlO71tOzobI ze7|YuyEV_*Eco6Cr@z;r3pHqfKS0Gr%L3As3Kmg_gr9w^`Jc63*kAp{Pt5)r7|%-g zlSnP+UKM_nG5?ZZURe^=?WKUuHL}`@0He6QJ>3`ibAUUVO(zqGUc?$6mp0LiX1`;I z7w6FwkE*pz>?Orq;o0T(qwyxJ5yYrp!7u!Etvo+<#8g< zbv156Nap;8U5Ny~yExsg?GU6?tbgrUs6kuKn;OTfkDo>ti^RZ-cQb9qaVJK= zHF^=?HBHVSbxL0&kj*z?1yV((g2x@uSt@db{t{(gi*+M0WX%*OKGnar9d68$C%mw* zp8=nDWWM?mylnFV7f`W(;L+042}h6bt|GuS|4}}?NY=j9I@v=u(-Fo|w6Pgh6lHr% z+7p766AH3D`sG3*)ElQ?|H(4Qa4(J+YxJ3r2`qZ4QEg~LY#F<0w+gML+@J@|Q?qx3 z@sqIgDf2HGHI*-!iz!@u??QBFL~tt8zvjDYbvze{;4kj6$8i4ILffBJ{9d%3_m|J&6g zH6D;CB*Ok_yDF*y*&g7?1TN~&1d+U2U>6xPbQ}8@!-X^GlO^PJs+26n#GhV1=rb?5 z*jNaX2tJA9SYreMJYLZhRJd!#yBdKb)mAYOJcFH7kYDwLUN<04Vh@uy(jN_MuBfZn z8^EuR!{TSgO_G}u>DTzl1Xe9SfMR%nro3yU!UbTY_9DG_HBFrFu8I zmRk#h!MdioBR!j^w&5O-2EWi%PCLvjx;erig5dCZ=OD>xg;{by{174FSZHW(RJ&v^ z8idO{kG)zyHvl8SDWzj-xZAS5F$)*A;{14nC~*QJhDKt0I%Fd6l~AuwE{j$KpG}=f zSj1CG7f(z*Bm*StMszx06);av&oq&iRGdA)$#*{1Ce<8ZGZNcdw?zaXU8hk)Q6uG! zht%*Jl-uZNZ@atAjpe~@p9M$M+8fFGYC#d=JwMyvOI%0t#!_7fAf0u!Wiao=P^I0h zZS`1i=y2cKy|tvJ@6U8ne&C0eF|ak`9Bgw_{$<2C%n+Gyd3?P=j1W;AZH(=(;A7cFmW+jM`JRINon2 zLdprf6XO1z>2PK&p132xEhoFTT0VC~00YW-KBov2X&Puc5QGOLOUl2w&1bu~vz*ai z50~D#q?D9A;e|c`rLhfE$C^@aBz>KBIHDQSLc=qr{C~rI&2S%3(yC+=1B&L73=$&< z+iUY8x7N66+XHIuUCxrWH+Uxf3|6}*vYJ4my?&5I-4bKeM4XAr-}svFJ;d+%jxvV; z;<$z)8lrFIQSZBSB84mMHSWxiEyVCIgis*tnpe>=CUeY?Am&p<`QcvNeoi>FJVcfM zkz2`2;dY7ts5{V+C+Iq$>08T()VaHQ7RSFX7{Lg@EJCR>dR^%YG(pCl6KEjB10juu zsww^F>OVYpU@OWlrQ9o_gj|sJ9yh|qbGPJ;U81bX?n!P#v?BT%iO>e>7S3y{1Ja8|X zb$uXqUd2c4C4^YG0K!fi>}YO5l{`&~l;;uqCUbzk$x3iZSKm1|;OW2tAO3R4XCdDl ztm%~3X5ISlz8T8P59e|+JzH&T^`&Mry(aa96wxO|%T#UHQZqFNRV_7U3EUJg`{HjUbP5MpX_vLsLJ>%i`+1Z0q zdj>r2^Byqc7d^WiySyE@$)Es2+G*hVF1jPnb3gD63}TXo3}{taIJ7=%43|ZRbD2~8 z!M`6@h>@V?r-^oT=;6mhf&fj@L8KDMF~XB7ae3fGDsf~vWD?gAofzhfwa+LlcpJ3M zfj#XWNvnDjFWQbfB1X8y$_MmNvY3qKuD&z}A@?_~qII666B4?4g&)OJt#Ei$sh$3= z0Z0JfGb&-Wl0kZ*1#R(7CwB0#ixxM{Pywt=ewsr$(?Y~V+vEs2R9}f@h9?`*xyM5x zy|-p%i-2}!$x#T-H!6W?g~Ds#z3Q&;U-n^|1g(J?Ul$;tT%w$^NUun=xt_?uMxw{A z^G~!G6pt^~fGsd|ksr=`j%tIej_vGU7o6MyJogAY-dAEWI7{6fdRhj^ap9)Yn9FLU zP4Z-}OWva#BxBQ=N;vX&Ju`1Q?SN9~KMAysmkM^{wpER-?#bCz= zFEz~%ZzuX*vIbri!!^*O`A3N9G*U&>#TiQJo(i3~sHY0x(HGVyM+M<<+Dx-KraXz8 zX(RnU^r#eCV9=v2l$%V&SnSMPnPbzX(D>+>fHrZt;X!p$RJU=J)^@i|Pz+Q>1D8Z) zpI~8h6BeZ*mmH!DE8e3X>C*7|rqujgn_R-Va-Ekm=pCY0AQLEkJ7u(X3@3$yS}Br4)S=^R4Vd$N3#rJ);mS1V4cTdiGH3KOCdFssK= z;^Ba_#R6l*uvLN#Pw?;PHsyKOnK@0<#qVw$Dt)?jw)4HaZghCnPE!1AYDuD0cOG8e z1aA?dH1*X2fauTh#8~hs(xA(0luW?F$d%Hh$g?RV?W3wTq%29L0KKkYs6KsJrx}@n zA_GIA;iUyiwK2otSNOWYSpi5ZSiGD~(-R#(YhR!>cqjW-N+Jz*`PIz&kQB8% zrQT{=L{uVCy25vU=nUOB*M{Jef%-XRTPcNz4t`RVmP>05me}mPp&{fHHglL>Nsk%U z@0tswKFG?yi_=ZW77J;Fhgf0bIOlVd7b%ODkxoIHz?wJr*>X9Av~PkTu;>50z}89r zd#|YfS|xaa?NL7xZo2ob)04)VK9z@Jp1Q;|${h$hct~T@qq7BjMIM{(c=GjjT<6wx zh9WPN_bO|=5;=Tm(TD%yNUR-m5?tFw=5QkOhBcGlZ9m;@XX6Gpm)KpG_$}s9YwgWX z^2+J@wV|J8b=xvKPXFMcwv~$mJ>3CAK{_8KIG-JLB}$HNtUVX(Wql#=5UDQ2-!YeY zP(N35Db|sVmS-st5bq%DGzgZ6<`kfHeGr{GoDgfxg>si^UAGUNG8{D~c zR$eVI`U%qwVP=BUHV*!*&^Jr0?gKBf5I%8_FVb!p@oOOAr+{x6XeY^_UR{&n;Ex8q z;n;*KYLW22M)ZF@I+8iOMqX<9r%_g{V8Sj930HG#KWpR5dd^-Rs!qQq{!$I{@cuou z0VK%V$=MC+CwCJHbM*urpjKiJ$ho?J4p^NqK4$EF2724|e26d9EaZy0bBLR>uFC-k z_&0+fJwXN@P(LTRAP;v>xLy$G!0-9$3ATUjRy!c~yTs28bl{Jf#;lF6%AN7@h02{! z)l_jlrmilhrKPHV{DhX4y0VHPrQvAJbFUx&9y0{~`J>$&I~STyF*ccgfZDG}Zr1{vSktCO1&~1?4|L`n?8z zUEhD%FZVydb8-HgowvWQ`|nlc;;aUBhk8Ig{on$A>i?Oai?iM>FJBKQKd`HZ6AY^6 z?Flnb`)B9BEZ-lc7MO^@I@SI*&%cWPceVb1+4zV1|Cj9l$aM82D}uX=D-y3-*D9_R5m=FU)w))3t%$&i#H-e|ifcs#RwQ1vu2ozs zBCsOys&%d6S`mR2iC3*_71xRgtVq0SU8}fOL|{eYRqI;CwITv560cgj+mh`@@(tJbxOYefWBBwn?yRa`3~up;rQb*srONA_6NC|4Zxo?H`8^sHecoBT(SZ@&4U|wE_>G-!7azWqu?kCkAwBo#}Q% z`~Wgq`9;~I&tIF|_wTuXz%oFn>o*2}vxnLBzUwTdv+MV#+&J`blWi~Sl=9h|C)4KM zUC`3sQz5!z%ag%dlWkI}B57-0pORmHTCSfg!(86KL+Ip*u==-|Efu8OquC}$`?hQs zQXu+ozCANR?z>Jlc0YYI5#tgTqK3%BrGX7gPhR(KEAlD2|8_&M@4USzap_BFEBInP zd^xjTbh;|>dg6fac~AYtuuPpezvE6_PbF>Vi$_Z&5VoX~yN}?kL?^DyywfT_*0t4A zTz<kWp4{DYR2Af5b~MJ>rWqKXJj)m^0!x7Q(!3d-fB0)^PofpJH2+ XY(MU5$SVKkB6Gp;%Guo0H}3x*@%T+T literal 0 HcmV?d00001 diff --git a/lms/static/images/notflagged.png b/lms/static/images/notflagged.png new file mode 100644 index 0000000000000000000000000000000000000000..42bc0b3aff36ecf2dc79801e5c670037d09ad198 GIT binary patch literal 39000 zcmcHAb97`+zbO1@lF7uFOpJ*qwr$(CZQJPBb|$uM+vwP~bMt$id(L`s-*wMfcfEV9 zs{QTms;=*@Ke~Fa&#o|8X%V;|m_I;3K;Xng1?9ia|HSa$A-`(0{HU_86NCf5nBw=Z z!}Ggg=+|p#TTwNKuXd;Z6j1*NIQ6eLu^oj}9TjYh9bNS7jX?MeZS;-sh0W}ZTx{$u z9PkAcn2*|PBfr}H{HL9Oy^)@ynT<8RqM4Nu2pv5QBMlvWTJq;C2neK11ZUzQaO z!9aX}4+TU=1YAM1JNzupGbtNt0)9lh@~n$WZyzPC4K1$tqs~!^ zc)XA21B8a6MdAFJi;IiJ#l_;&TT*B5)bn)a+mo!t zOT(Ve7LqSbou&B$~NFlqcx^(;cn;Pre6ZAGlt8ez&riR4LaY#2$ z{vY-&|Eav5BAy>FRUK4MvOzxEzV;BlGB&FObHSB+fN2i3FzLRbj8S@ zjv_h_Zg(!~B_AzdLEwBZ$?6ktCfhWJJhHPdCa`;96>2UOzsV>(8;Nd2=oI^ZUWi6l z!3f#^Vh8hf`=EOfGVh7eb+s~@@&3TliR^^yb;)yhVgA^4pB@>C8+=Z8puSDDNw^!p zjJzW)NZbOx zB<-l?6ZgR$0h<01f1ZCBH?y5glYzs{#!9gZUNlayEehp;N`)VtedL3?r_1}+!S(R# zOt<2xE*~A1I|^({E~UKTYTwntQCvq<7H?L!>hWm^JG(Z2SvT@rz&zOTtxDd9%nMyr z=1LE?G-w_VY^(@v5Z^9=B~ePC%wO6ePtgaeldwA%K9VJ~9T@2&<;W_4W8QU!LMok{1qE5 z^WN-Q5^OV5!_t#2jsS87v}7H|cda{#8LSh8Ami5EErXHY$5Rc7c?3}+GugvY$Os2` zV<_dq1|Le(X8Mdv*~4N=d^N&lwJjgsGqEiCqL?Q(VkK|sF=wgzqA=2O2$;t@R}DH{ zrcb{8nMc%bZ1)~M3fZkhnlJ`qa6M~Pgs$Gdp)$6Ct~?zF89E`(H#A2c+gf+GS|$70 z@9RLG?z0(ngJ-_8W3X5ih=zvAw%hlJ(_5Aa=i)Bz%RQZH-ZLZ6<((#Oc`Wa{{+-lc zGE^j9*Xm?ivR59uFhXdPMSsEx4;Og2 zh0#4;hL23e2O$(ms7&;y)&8gM;Zi&WcDA6d`aPZxNcz)@$X&6*=TBt(xQ&o|pS0}H zB9%{vyHB(C;(eFddmXv2jzI5spX#@_E+MG)`%m0jW^BrePbv+w_a?Ms_en2HD`8#C z)x(2|&-j}jrB4J)u!oq6PnhgN;1E^mEYRTFO7F*sF2k(EQWKnxfk8HJhLgN$?IKFz*^zi{l~-A40`8Ax-;vuNlbmv z`O`hmvzG|{9FoGJjraTDO-NDC3xdyQ=3A1_RJ+OfMA=t9@P6-cueS<-V2YIojQR#V zNoQsIslp29NYX`>i;C=V$PTuYYq;^kcDt4|)(_kw}9PrQ$Bt4G1TAxgx@ zUn`HOl?A`>%jW3ER}Ar0{*Mmw0)|`=5Kc%d3q^&m#mZ~VWlcDanYgi3dGN3^fq%VSrlW}ie6X{D=GszN3s_E{o$oIZoLRN3Q=&pcWDBCyrRI)a4gvm~YqK_O0`rZg;!11b7 zNeg$wyYk`H+v1gm$aEk%(R@x)ok(g1bz{Lk_H>mpfOv|`4DS}^%<1@1SJbVI>SX~( z)_xqG-D{ZRK|{dVez=m`aak&Q`Ciq2xnCXEpu*hyP;lVHQJS$`@=&n200xRllriZj z<%zA@Euq&qr(K1+yC#fYi6r88f%nVzZ>zf?)LHf&p${cla_TU!cyIb$T6lpKm^}Ktmp+7?6U8DYDc;q23M+(eIb)m14GQnPxHu6N9xGA zBmQ2J2H(wK2?7WVX#-(_$DgAGC#+XSoc`dX^#gqQ_iydJoWJwFSEM zSE?A6Va9ZPuskw=l2m9S;}{>%u;az)2#|{68=@T#$z-39ipt$F(*|qJL}ahW#~VpX z!bm3$F=iAb!+0H@i)BjpAZH{S2}`ORnXW}wM-v#+*Gfgg_r`GJR}scyU?ub2&79V- zO9USDi&_Tj2jq%UtHTbg=8<8-3Ki+gqe~@kmrQielwZ^G1`L5>VvAL}qAhWb4THtw zg!w^Bo6r?v^UFXbDtc4QMmU%mabK!cCe5eL$O4ikeGGL=5TJPBcanlvvYt;?awp${ zpJ*^D!!t)Vjtk`islKa}689;skZ~pD&F6M+e%HKO*>Kmen_Uc?LwvbrgO%Y{%D5}s ztw-6p!{v-SWr7SgxmGo>Ou#O%iD=9lC@p~_;FD5$1 zv9TkF^cegH{0UyX`Ag|+`Ab%rdRt_E10*SukntnB_85}X0V9boA6nSfvyNkkEy8({ z$y?s%e?;Ea6kEo3!++Z0C_1E>)aH`l?ESj1w;%Frt78rrHJ;>2w-|`EX9cV|gb17Wj5BYNx+ku#8DTK0XW1t4; zFL5HV5;C|$?kPw-R$H$E{1!VZl_wg>LhHVj?w#;yT4#IAKvDBI(-dHV>*2@V)%6t1o|Iz_q_p{AKH6;uKGNE#gzgEK}6 zCX`b-^mPL?0Ew=T%Onu`@0e}_TO~8Nvn7M>5cYBnRrJ#J82(e-(bk#b6Y+Q2dnivy zc60~=X;PfZ_^{vKXDhyWd&R+c6A1pw<8jzzB2?D5?V8eE4hRrjR`{zRF3NyfC>oC> z;wv@-C3$_@>%kKEn5O@y8GZi`;1fmC94dlJ(IpKW$oT;vrxlJ+|p3jQrb6R^?;9=%9IeFj8Bf{p2B?n zc?B95Zk`N>T@d@sf$*X0vzb5ycBYXyF-* zm32T>7E2&IM6)QUh`3NR*x_JC5NqJi*AGd2%1R}j#cwh*@5oyzOtVHw88r~ou>3H?L+si~FXWQY_r}e$`Vl7Uk`;V1jHgl>`kRbZ(OrUGi$7BLen?h^GwM$+!`T0j?I09*_rt#GruqyH zD51zi9$Pzcdzjh7m^~8vMGB2y5>A8~OJiBI5~q3vS~nwP+lsXfvD@us@9>cll``MG zx8{z@TQvEV!hCm$0tUuki=Z>_qn`F$p;>h2l%k}~h-x*XL+tstI^VBJYYM0N0gK(^ zmfsoB?ZhqGHhG)lfxNm7vwAP*n8wE~(A<17<9@Tu89;Gg1in9p684j-M)4U8V2mh@ zR)CYzj`IW&t)tLwvH37(;x4wqGR>!%?jv2(n|P{gPFzT> z)Ek_caduOnEWqtUMtOxZMlO4R5U!VDnikFv|F z>lrO-#>%s3c7Xu zCMpS{2j6lXkDo!YDGq7*Q3KJJh+$^UVi9n@cS6k3cwI>9qpSp4j<=zjz#ocji%W0rjuZbtB)D&9ADO&cV=gDfMN&IljVFw@`bJoL?+d==5 zS?|1Wu|@8Yj8nG!J1#w$klS_X;*puYX8k4xjvF?R>$~n|iBu}G!&<}^)kIV>H_o7m ztpzEXY$uSsJ-tClbpg+wU_`5X2(M}%90rU6?5GAk8Q|IWlPHpVA28lZw)`4BMH3F!X z&iHtrC17UkOzHGE&|GU;U7wZ^(;#K|10gcA5jVJ~Tba3}bYXFx_)CgBX5Cm#Y69;dFGFSFz1?#-ew#O*CFozH78y{j*D8SNlsc8 z^g!a`hC#&UnPAnHHYbTsQ-ihw&AlJlz#htC-M?lPwS8mUb!E$4NS!`OoJK=bPxeg> ztcK_fi_SL2rXgiahn}bmROUBDXr6v}sx$Q_osD>0#0lM^x+O(%2L;8f7c zed!@4CYbfRps(I7js|6AU3wee-=e42LOEmHyhXN9$1B%8y9j|!%#m}1(R04ctwCHO zy_K!)QTpF3WDJP>Kj|%KcDt1}3?)*+^G9dA#V80@{Abx%moU4B)gd9wJa^zL6J(T~ z;^B5Z0lQ~0m&`>tXLt!sw`abRJ@rpwGvSzY6ep9kHF@U#r$0?XBsupfZ^IV6`dY69 zj~r??nE@MF!cG{VqSngEDXzBcZnXqS87sF*p80Tu-pPcx9wiJJ>1&UaW3>?Pa7+`G zD`wx|?P@4zdIuTNVHShp@6j{$u3>*x_Ib*M&HGz<=Q4CF7AnLlPJ82*6J)mh(kiMx2RUH}>GMQJd4DU#nwzYRVC84Wqo;q7wKepB! z&yy24i&|9EfzA4NvD|@V=;B&qT9eu*TKH65uCF_HO?F&?|T6R3J z`N-Q9zhm2bO}v{P@zm5lA{zmYN=-w=Lz16;Ml0lt>yq+%i;GN0r>?wUMXs=L)2H*z zELoL97?@sC66)Z6lvo40-uks%@|gb!>=V8`zeW4Vn}=n%7_jL~EC03Fhnsr+xkwiK z%17x^oploTOsALHs{h02Ve4;>-CYLw>Iw{b1W{?dd8FLR9_UA13H)8AvhBfj%9^+TlXRV6?bQhMv z7QvZs%g-K@lkl1x%jW|(@uVfyCSEETT6PbA*)+;n+jh>v>l|z!;1ADqN-y7kJi2Pp zISx!yn3#6xABE>q1W)!%yUfADeMk+={1mB5%a01rr--g(ZROQ;=hZm+gV3sfxKLN8 zR^u=*`IF;Us2wZ_Q#;;6%RgPUnFKU`3aG$-eIsaa^aQr@S9CXup}x2J&!-=880vW= zI(wrW4f=;vx`*cbS_6~W92_F6E#Em*C@%j@Zo*GiDsL)%oe=bf4{bfw;E4*W-b!@n){O39egD-fg=RZ?8`-;KogQK(Zp8mvIbXcB)bupoW zbE7n8b)Yt8RSe|8xAtNNw1v&yE~*XP){-_0pXq;JN1RG8bhyW$5_2ek|DIwo)4`kV zjM6Hv_~?Ix+4M&zqf)I`RNt-D_}3!Sop8O3>Kd-SRm2+34Ay(}DORN>c-!?#6~p4U zsF4SufkdV3O42`#26I`a4WmyAcY;c}HY!=B%sfEA6Dm4;@B@eO@53URm32Yw0fRK-aXF%b$_~jJ$hB7d=nT89RSKEF)K-0O zagV`osTFyr6Yi9)@;j6kHwzwZh2mNZu9qk#`>M7cEsPS#uMJaKG;QJvKO`eb)C-}_ z7M&(JxI*`*e3Efk0qE^PT|4&+85P;a!Xm~E+W;E%ZaeZcXM^Xc(I_pi83^^y#_NORv88z;RF&3xpG zY{-0YBO*GOy(Tz`@f7y*n@w!-#5H9%X;<-&!Z`Vx*nl}wN9(d!$@x}uvTJu0bIfWU z@vB0O@zp+68Swy@{mVzt>GzMTgNO2^!AHkV!!SIYImvjO0UGaOd*}S= zCXA|msmexzGce#5kL;ua>k<0bgZIMpD^ug->x8+(KBx+=?LXDkgIl*tKA~+=&GtZ= z!7|DXKWw-xfZYCkRy`ybu}`>aP^vs6r^QHJjZF)tYiuSQ_?SNaY^etO3O~Q^k`^(T z<^}6S-hrk^$_$M5z9K5EfKK%RR~aZilroUNh-2wFFj_f@q)+<#!$Qs!1F;jWyp;bu zW13VZoP@q#qb5OVbItCqiTOUniHpfOWSi{qU8?!!l66Xv9~pCmH$Q84cGw=al#gNA zE_=8jHOtc?RGBdBxk4vIu@l~eDOEBLXyoqq8kvW$1^z=LK*w zB`zwa);Io8X~=7`QGj=m(q1`BsuD#Dn^{>~e&2nG*A`end~5A8Oq$TvAb#;MlR~%c z7%)Agf8nODmM(Kd2e)waN@oN;ER>psINQcO?hR~530UsV-{jGbCmA#4O=Y$xi7qzK-gw zn>PP@uk>D?wuSWge!`MNj#Z!D$+O@x)UjG!dq#cBYg0q*HI=C#9>iTCCRd@@X^PeN zF4!se#e1Vjiyx7%yOB3bbr00g*I5Q>WqoS4EAB5o%4P8nGliN}Tikq=p0h7}>Nnsr1MlBXvrfD6 ztGz--e(v!(W@+vVr}5L(7cKuovG{r z)0^7^&Wi7qxh-JlG*S!29t}PV!9{XO1gq-;V{!@*7IFEk&4p6@h@-5jX?J}%H1Om!oiMp2X#v~bYoKG&w!IjoUx$?oU#Se)s3z&!GJdHB zW81W-sF&caOjO|fb+(Ax5(7$N@q2bYwWmy(CHGBZ?ZzzLc3V%OUzy%xZ;GaUqqXR= zKFrvZ%!=Va@%)%t)m};v&GL!DG#^UO@Yr7^~Q_8ezNUzPwhf$pwf0*-yDVlfu%v&o3$S*5;@y-{x2H zsLRaXzMo?S@Nr-F$)1_UN7q$Cjk83Tra1IsP0l#TEh5V@X7`ltTRraUy2;+zo(~o$ z9X6;BmNM8Y8?s&DO=RbZdl68EuOGLD5e&i#m@}@7jYDo7-#X+Ce{J8zN@Xrw zC3DX+IC{>(#kuGpO?O;fa474sHi?9`X2G$XvYQP5al&?Ys_Po~iw(WDaw+#BBK!ub zu)ckX=Tv^Pj_~iNEpKu_TuO#hn&b5ah9>Vd01%n3@#Nrb@tojFDW@inf-KsWf){A? zk^q8(&B5eMC9+>SsE|3^vCDjb#q6P&`2iz2SxxX4Xq_paF+-<#dDJYhbocr?#3*<|K|QlW9kaBJ1YPd4 zYyfTiPLbjgP!uA?TQ5umgs8RTCH+Vpf5-g>J%e~QLmOg6)VQSsBhGkr9bpQ>D=usB zSv?f^icGX&(HR(hZ;6MH0WW&!-Y(Pb`zNa8SKFSE=1T`IpW~ivXFju5k%Q=u=?gum z^*)Kl?Ct9g+|9AaY&`z->^jNL>{gv9Ai^%-tn{)OD|F&k3fZN6zjgP@ayaOwZq%rk zDDST%{@?w_R;d>*&K|{(TdDCy_FTi|LihzPGx_u71|B{zbGVm&W`U$fLs9gBp##>( zsRLnBy)$BO%Z}?ynx54SKM%bN+=esugN~!Uj-%3}Ee~8APYE0E0#^rB@k04O(e)^H z!(nmWlUx+HT`{kuH>24kzf*Z;cp@VT01+bd*lva*-gzE6pnrNgkFs;5UZ)35yXJu_ z@NBt`vH3hbGhMwR2p$9jw1T^sGcGr5i#V(=N0TMGFnZQNmg(aIF4pr+@G3 zi~cr(sy)c7J@iv^d5zh5;|&pWFT7*_ZIxb<-JX#SJYHLJVJ|0b$n&8O#}X*>KS#?Ogc< z3l6l$=T2Me3KN%3C9Peod;SjyDDItLqF@#Ni86itESp^UpnyWDW=k7ETDIzaE4xC*M>GKkeCx zU49>iv@?Wp1+&omi{yjE~ z4_$;n4V7$W%Z5K+U><};R*kypJ=Hu<&Lb_R;WMg@Stp@!$o#jtq6wn2KMJdhT_@%kwBpHEE%6s1L*%idV0hUY zF%ld-jdabUJaxJEbQ{Uh$8oiT`=r)4Kj4; zFT3f(MXl0@fT04hS%Ah+wq!6%Z#R6G+Jj-(V?o7v#_jKhUa|SH`0ov_$8Ev<9(#-18q1;W zcZmy#_Ok;0LI??fGWnczX9iVkY6ItJ+q(ri9Q-Y3b3Sw}~)i$5kN$6&%fY*68f zhp=IwU@%Pwd#hJ0?Vw;#;+mH1YfTz>5$A7t*@x3IFlaoIivfouKFEFJXgG18{R2+i zKCjS%zMe8TX<)imRmB)0R7W^#I&o$Y7XHVUE9srG=6(|!9ddf^vMyO%1e!oiBUw14 zKx}CWs_>i43q(~e^)&SEKeQmd)h`z4>oSGzg$ers@&5cH^c7_DS zt=p7+L&0QZuzS7Z2O+qC7b}Q8ViV)|!??;6ywd?o&By`1=1-3JwmFIX?;41e@)E&i zwQ}y1LZt$JT+HVmbq-u@5uUif*LW(vOmL$Gs zk3K~cGM?i8y{-P+R$8Mu$dYKnG4gtqRwvOri@JDn-%pAi_L2j}DMJA%_F># z7ZWSU#EA?a=A{@M^wg*gP@NmpjuU}?_zvM88av-Z3)Cc7Yc1Ck>s&8M&i9p5}`pyZ=QMt1bR^)gEtq3#4+H z z3j-Mi>yH5y}D6(Xxd!_TvrJBH)kLZu|H&JOK~V0WjD;=J#U|Qin`IojD)Ap6w$&-!y1Ox?09j5zgHztA&00%p#HREQXI* zd=>Vf;{m&p+%W-+El2)Bxsb53Cf%i}pTapxGU6jW-;1lN-yhAK2Bet1SU;+s9mGc5 zZf0i=)eO48xOwH}&TIVw80kMr1cEnbuA-GwKgSpWdEfKoK|tCCmT zZ0{|wqRF6U#n?_4?t?1n#2``NPrl$!{#EKDJxayeR#jJ;^;{E9Y`o(P>za%pL@IJp!T4B-Ln3_JhUv}x z^jl+~kJUg5n{S9~{J`BFWFE~EA|UYPj6Qah#s z#o~l}?Fbj(Th6y1LnLCx2r7c<}ol4U587f}c~+w`Q4DK?|(tLp(nL z4E~PM;;h|(N(6{3VS37m^y$a`#!P~6*sCy2K^{+qnYp$^%%5TOI0okub|`XBGxEv? zLko&})~{&-E34M^BvX>%+}s~(J9GU`tai%IvglNl3tCW+17&HdT3**h#A%|DTo5$b zU|k(bGs5IRlnMJ&cvk*2?q|xdIzhaq--uGI@HmDCpS0PO!`dn+;xSV^ zH2`44^Uv%oNtozPe+UJfDq=PexKdco=wR>2pzMx2zNafWPx4Lhz@SGfIhH-;b$26W zu$*vw*E|fUKjlPdCC#1a$kD2G*GM5WMdfiF^)KanT4 zTpk&rmN94AyAA0D1J-1K8GYoS4~7-BH2*>Y7%(^mK`u%eCspXLzTjiCbrt81W4R~Z zEv$#8w*LRjzuz9ZOX(`SJ9cP7;c=TVkR|Qd^)yOv{xl{?#f4M3DxjK6IEWD-C?3-N z&J0;3Ag12l30rI8{w4i8MQk=)Dsg2bmHzJW-MY%bq0I?Ee>x~)SI+-dgTShf@)woI zA zS8M4Da<>5)5t;G<3UAL8t{22vWmMic4SU4?3RmelwAx`OgS`&38zCTzm)7bfg z;AA}M4n;`eLL5q)WKfXu20%}jKs57C!D{#4WqFdqr%yBNyGc9jsfF4rHN;s$j|g8R zOGsGeb%f$(fBXJ#B4vsfn5C!L7@N3G9lBNY+5jc%&n|pq3GV>-~@as_r*OqR?K4Wr1PPzb&WLB7X7WsEa zpU`Sv%W|1*@FEP}cRP#+zH3u9yQtLW!Ng|qPzzFQRQ)Nvweo>$Pe)PP!fG#d7QXHq0-I`@@Hw(WFkt=5X2GjA7`F< z5>GSH5Tcvlq6dGrH(rna+hT{wQhO*e(T+-&9P7e`iXtl1>Jd@*flt?kyN=hKz3taIzW(R}~uy)S2!{@~??1w$e*` zZ(fASOF1LOi~^%^rNo*{!vr&gDBbQIvD#eQ-_vk17ci~4+!kr`>j)(%Ehhe`XUi%6 zv>+ONm@E4TzTcr+ir`W=5Uc~#>l*X8>Wy%2Oq;*3stvF2UC#d-d+&J0|;49NN!sxvlZ z=Y@SMREp3`8JM!r|DNMwxAP}-ET5foa^Ne_QhCeQ=LwI8w=zhUm!xgE53CTN$gquh z>N^3esX38eG|QVN44R7Nw1h$8(iezHa5JQ{EJKM+9@>e4rhTEn{m>!Y6WGIp2d=SN zNQzf()}+{|h1Z5UkZK-U1pZA^FYlk7s<;NWb^AgokEuL5JT45LCOHu>feR33Rs z*^%@11M@HLCN@Z-AyaOhPWiTk)>Yggv@{Csqdar^tZEjOX2D$BtZ+h2W1C4lnE2TG z^}xq6j2~VM;8mPW(SIqeaqAL}3xqT!Bt3X7Omp9Tg64bWoLYo7q-H*gwnmJLIB*%p zU&$Tj0o`8xEisQ|zntP}(K6keSO>8kr9lqk58lkwq01XeXE67*`Ucq{%&NFkMY~vM zJ$KUof@7aZY&|DI&(8{zVRXpiBiXrfLN41$GU2pe9->FJf{5d)P&@zoBAee2V?8yp z`mIng7j>;7ar!tE+O?Ep)Mt9E`1#ZnMa5MV0QM9hLSOw$Qhwhh$Mempv2_Pp6&LY! zveW<9Yp6=Ih)VMo5}De&L19o_jS8xS#GpB~ZqpD@2vQx43s}g_QgvrK!;%Gt<@feAysd6Ku$G_ zwDSL5aba8+=q4-wq0vN!`r(Fu27f1Js27ay8cuXEADGnWo?Q3`M|oVFqw5`zoc~Lr zs*n9&5mmQ;*{a=xv#s+NsQyFB5v~h5lV79dHu>fLL$P1U;Ltfj+2?)??O|}^#Lzk3 zp$@2oX`UKc{CcX$MZZezzCSChzPF@8bDI9y-u7QNRtzL-rA}6@y1=8U8^;Oia*LxV z)9mU-uDw;XCXPIo7n~)QNDSYNr?VxDQ~yE3*SwvShq-&f0cXoqlnP}GpYW`bKX>P& zC=&;VuSnTo0z=c2oRgloGQc)mO znXA&OK%$uh#)gL=*So*F?2h@U!dn5OZ6{h(0;H%ttj~-`M%Z-qot0MTIYt@!^a=-hL)H`TbRo{zV_FL7JG*EWU z<=STFFOaMJW(niS=xvwk4 zp(AFyO8n?sZ2;&Y7p7fFKlPvCdC@3g%gSHWHHbG;5FfcA%c>FDh7D{Z_$s!z=)rGo zD(lD{{=lk4xgwS<%G+R%_YrxYVCHo4HckQ#&&*qC1pVh*pb^w0A-v6Ym8%z%$!T$F zewA7UKz!fE>mjpNo#&NW|8#drrt!m0_*93Yr>(fo^MVqhKd$zLDUh`O1_bgB*SwNi7(eAZ;rcy& zB!O+V)^qyNV{+p$do^wCW#}8%0Rl4!1=BXhj$41Turppu8{bSc^olcGTs`Imb+BNq zOEeL&&dh?G1Tbi~$dLvMMrBIDT%pyTx)wp!eyG--4S#Z$hcLv8{4LH)yS~ zTTRV<6clxC*Jj%kPMb*SFvyGyG1gLBOI-?t!Hi->8-L@99qM8!^a2jLSS4WovM_{E zs^s8NxUh>n(=BC-0i;sn-ij+*z_>~&d&=fANC>QqUkDL(k zB3|MXWD4vfm9zl2_xR9C5AL8Au8>u4qR-HiJG8TKSxKMHU5TdtAzcCF#P9x)aC z&oWizzR6pSE{xLkLg&uix_!$D$<*(iA{0U9L$>s_10{xDJ0ek@Sy!WPT==qhRs7mcGKW$CJyPqc{)GtFWS5t$nZtdmI|& zo<9YTI-4|ewr43{vG=@IKZ-cjAPsu104#0O7*_4JfR}lRd1p%0wd3_BG6Mx7vXGaI zeafRm-w8&jLLn%r?NEq};|m)>kRr;%hlaZTf!3YAKNWvy&743h&4s3>im)qdvsADC zC9Tjh?^b6YcJE?UzfHe;AL!7J8COvw=8ykdT3yHgEv*!0{~@g=>5~5;t)58#AJV#O z@&Az4^?sG%ItRw>&YLoSjvXL19}K1^0v5gR&p}nD+P>y4D?v}+NWB@@+0f?hd^X~! zyxODZDgl^UjShD>S6H{@jtKG3uzEx75#aV|0kxUbbDQm@!Hub1DPSaR*A#52x3=cn zT5=4o#ypGp<3M9&ALN)^S2kyB%wyTCIVa#Z9OaLV9BoyZXRV&qDXH!3jgtwn9i;hN zoO2#M6X!msc_TGiHcYxkS4R4(^RmNUywK}B%=WyJ<)7ytI>9Dfr!Vw_Oy36I?S!9C z)jW~|n@_wNzd~oTC{t-{578o?gQ%gpt8a#ESI>jo-1aHUVVHqwSyR9t__Tu z2lHc5ckk&Q>o98)TPz2h;S4)gey)uqRc5JO@NxOS78(IgJ;4U2gE9j+=DT<%$@b;$ z>SF~uWlTfB`u5{2an*x1z7=v4WVcFlD+d%<5jd=+bUi(k`yqKK(n3DU|S$uRa zvZ>tYvrFj7(#&Y<^n$w-`(#an9aa4vvM;j3JfKlD-8&=5v9R+n5O^>`B@8u zJf3THG*E6^>KPLHQqHUyVhz3)x)sTzW%r%=;|J?>NnB{U4uS@_%VxIc6pOV_#Bg#) z=pL>fPvOteqZLSttbj~NFU(R+l)0TIpcwyQe@|N&T46_=XW3|-OY>AbRWzV@ugin+ z6d)J*pYoZdyi0Myc=5=*1v`knR40@BJ^K2^Q)+XXmDyiRf%`49f)82z7uyR8_B(}VKjxd3yBEqG2`rKom{~+aIHUHm0Nsf zGC=+XoNj!FW#-n-lQhOY5O?edW(dnY03li*{eB zza1XL!-Ll7PALMbrnoEr`Aua;A%wqb zMZ#^OIMoC)Gk%6@-DsJ8$goNx%)!yStfy-Gx#GkH3TKoaz1jhalC2Mda!Bp-pwWY| zAMAV+B;Fe^dY>+<@?;l@@#-ob^l%sQ#k0n z&=;2_aBjsRqH0-nu4@pj!~RruI9(O6jikM&JIt?%lV^!~ikC2(1ov4k;Wiar^3eyX z1uw2|zi2i)&}L!_I>5tCZ*9Fj^<1&9uu~&wBqk+_f3EM1m~^ZFmf&(WzubVGSd1y- zB>ot^mSVGfCg=1*L(A3|T;kp4B<{9K6=&DBP>`QV@_vH7vHs91aM}1Ue-}dP^ z45*iW`qkwnjQA=tB-@d{3$HbId5NdcT~gI-xZ~`Xg8&fKeec7Spgq+A6=}rJ5%Zfx zOL2@R@iAu%TmQB*L$(~CcY#KAR4$N8y|)^G>Y%FzQvkvA;6cOJ-*F&uBscYN&0Gvl zV$~T{SXiCR-d;reLwZ-{O&c)HrtfYyUsT{AT5I!IeR;kuv^I71q>W(Bz=>t==&j7g z3X7>^G+ZK^Z5{88AVa&b?TJ#?AWeqhELPV>D)kqq?SXU{qJl5G^*v^v3jk&C6(=GbU`Z!oA(@lWB-y#3Em*h|H4zj>~C_qNa} z>;KoDnqBx`_f+k{|F@@hRR80t8HNA$RF@~5w1@Tsl*M6sl&%dNdzW>w%u@uMYAH-~ z;+tXk|B|P|{O>&#B4qpOKR{IWsv3&Hl~)nO7SGdb)VK_ zeE|-sBjc0ChMAa@nA~|XIJQx^_Y7J>MReaE6`>1f0AS`f%K5tw^s52&^;8|cIZy$% zM{q?>ZcW#Ppdz*jHilS`mb!QkhBS>v+#}9E*p`u}wBna0hxss3Yu_+7N>i98?wYRF zS5~q_5>fgB6g{x$9z>yE<-#ogM|WVHZDphb*M2LXWlh{!~5b07P zLd+s1^bkT4U3&u&5fMmq5mA9edM7GksEL3OdX!=)A%PGQQs<3syPx}d-hFSb_j&$f zuDQsN`JUgL`JC^ZnFDjqd3SWX<;{TBOYwx9fx3jy4&RO252adee7X!)p$U6-Fu(7x zqKh0F~Y z-0G4v5imJ9ITe2$8}m#^wbRA!Pt&ey8%!72NC~bQO=eXoi9SLsBq^YcP*GsOQlA@@ z69c{gY%46m?)cm*w)^BZs&nH>p^40SK$pyCKG#41q^aXtTAiYx)!LfyVcB*r_-5xE z^h(P^Zh&S{hP;3$NW=6L{_8Zo<<<6z_DiOgJ1~;BiuW=RrXr) zT|n!zm@n%9g!TKaJe?MwnxDbrQu4QwA~>b=uv#LyaFD3;{)BeQD_^WyOHH6y_z_kt zKo>)EEj~VK2x^JF(*|IJbtyA8OsQ?)mfSS{OyG_ti*Xz2kWnS&{=jJL1IDgAN6I!% zP)mU87s^dpL7zky)@jX47rZqorxAj;3VkdPKc-+Tw!K0XZma6GY)YND5tT4|uJe(& z;?xzPWoNaWey|&rzv-@E(KKfr?<)y0UJ2P1}lfw0h{mw z`Q7Nj^y*FVb0$E8y9(}nwKzQmvt%yTIh((~)WL_&YkbuTQ4tnGBQnqljd7%sDLm6g zk9hK#HOQ#b2xt7DmB6@4JOz5Y(QGbZky?ArnuuT_UMnIzf%gy);dN$q%u4!!LStf% zpPXKmRYiV--iyKWfkqkWXzcecO)cV!Us-o z3Em#uNa(HyGL4BdhV?klPCcv*lU8vyq1F9xpBjHzny(xARTo#5KHCD^T*10{ zFwH?SaUzK3bZ_gie_OR$!_-+dTnCm#Q!z?Nsc;Y?y%`be)QCsVsC4#5*)HM7>ZlD( z1=c>)-K+35*CM$Ee#zKE-{`y+3Ah;FVMdYP;lN5JFs>F8#F5wxtmJ(+rNXCO))Z00 zg#io}8rK75umLQ*Hiv)QG*fqSJuSBucI~n zXUp6^=ntaYGO+Ut#ZXSG8Gqv?#{6di{#m7{<1_9gs_U(~qH$d%Ondda8|G`21`ny% z`;3i47?p@)`{;DJJNf*1L}kF5pT7TaiI9@==FIMmtCnB-^Wzg#DxRGfVbO{BxIs(d zOV(gSR_-GBobV0cI3~I&H;Dt}vXS;|hi|(TqcHJCm~_J$tVCeNePss!jxVt}@=;Pm zNG3SUQkIyu%1Hv)tfM$O`B@TUk}1mYcIM>{f9Jiq+9#`tgqW?h`L~*v7~a9 zBcb>EG%E8$>a!c%+n!z#;KQ3UX3UGQhapdGRO?@iIdkS7uk0yk)L=Mky+jxgr?CvV zxOsi96z_Vj6Bxs1R9$SAk0YNuxuV$Wv98!w!@*0S50K}$i`K5;(G~k3ja#6?^)4AQ z(Or5Ia}&TdQZ1{Kk6Md$XF0=J@?;rSWAJrJ7g&D8Q(JM0xU0E*$DUgUW}eum8cvlO z>Q5$?`@r%3A6eB>JEXp+SDk?%TO+WNJN%=+zzu(m+alh9?UyBtn{vYXWm{`>3!$+e zOdpILXn)ZXpjyurP@z0K>By!QGt!nJ6Sf1)5&MC-IBJ$Vj@-mlf25f@<|zgfvXmnO zv+8e2=1#l@65n&TY{!C@&c46qU-2Q_SkwTc9u!{#xCPIpmMJgzV-*^I+Qiaan9`4L z;UV9E%P#Ai8_ev%*7XwJ`i;EHa;N);UBsMqk&kz;Cw7KCOnCc(Mj9;}4CZdhd0&!) z*hzcI({k~OEtPtTifs0ebb?iUvh~BGrlpwT2|L%34yk+v%Owk{2e?A=+LC6j()7fd z%UCgYuf(M<^%%=-yZC;6B|*#REpvkJd#P%D>$>fF{Zo=uf(G9fuK z?q8g<&z*9FM>DimBzP-5C%-SP5RegmN6F$Q0&5c|pgKc%6T?a2H}8LDLKx#8!_Sys zrT$XLL-88@o7;*(DeCW5QuKR@nVZQ|5V`4hHL3CPXPiKP7W|-w6A@a%d`F z-La1Nd8XdRPkuV(r8I-*_5Nk^xH!J*otn#gm0UJ6qx3uP<~9*%&TQ80dspoU!Y}|0kDI5lBQ;u=Ygca{xto43sLL&1fg3X4F;aNdzMdIuxRrCj0 zJiSd$s2eQgfyo(#mt9gjW;M_zV1l+9knG7fg-uGzdjJz}7a6RH!{(~oI@ibDo(d}#97Z8*`w=n;|`n2 z1I53eiSQM4medKvf&$a}7?5G>M|1#uhn9%7C$@}KZoEjlA|fo_B!!Gbjkuxn{Bs*~ zN#r*FSoJ?Io}Ht8UQjpl*EX5=niyJIi|K<)JSruc%^jA%%G&6M)+2HRel-Z|TPvFd z#`*kr*rY5k@V(ACF+hFnnQ2WY2$fzcbe!>w_ZCMH{ygCB%@nYmcp z4jyOjz|^&7gl44HM8o?=mY_b2c}jA{G31HWHqW`m&0t3Vkt`?EnP>4rpq?`X6;mBt$Ev`hveVtL8QOb5Lz1bh7-mlj{X807PP2!( zy$3Q1_$u`9}NUA4Hp)a#{D0LEQY$C2V z`SV^_g@4T&?cq9>-7Br}9Ma_0FuGo^h=$`Gyw(N!m@n$QpYVKXwDsbN(`REV$OXMh zUW?;DuiNav4$X=uFFU)TCS)sa!xT2RW;GQ#8k=mgak};4kHNM>D@SvNqFvt81ZW{XqmAXa3nD~kNIf?<U6%uw-sts`Jq)kEnXzYaF_EeTj!&D$xQ&;I`;FQplJ7FC z!2P8oiWf6_CR9QtR>*x2aoYX*hhxi~y*|dv^feFWCoKo>EwsTBLz`o=)UYE0$>V@`o@6gO`x}YV|%R4Sz;5tck#-F)lUA=Zr@_YFGt0ZzH01 zGHEFBwbIli%jF-JS7cME2u5B>l9Sxw$&gY+Gkf5CLjQOL_vh1hJaMolvj12osr2b| zfqq7M=QO{(8Rgv#nEg@6bCr)LHhREvW}W1?4y|8Ih$3cM z7g4pXx8+DF<0iH_XqAI6$~J;jMSC7(s)tkc0>H)couD{cyw6?lCdYF74;e;b@iyh( z-^oHjYRfSr$)8tRA9*=DGjoUb##K%XFEFkS=^h9|0$q2yvZ-i+!kY&-^nh`+{tml z-r?k(BK%vNb^GGG!A+*bg z@@taMOEKZhULcbYdb>htWP>EIoO2CX47Gf!-qA#oH5{a_E-vSwx*L@6Bq_0J0s?Kf zji!%S?k|zgU=LfC*3)1)A?o^=R(TXq3ucpNDC2$Mj|u<5x-vWXX?C{&s5FwwzdoXJt^~@=2n2gyj_yqhh+`W(MJm8$-Cu zq{(3jnp2&)*gK_WnU{s+rsJx3XJ} zK;Uf{-L{o}eCrwm~`2i*MWL_AlmIHW8GF8C|W5!DS zpo9bMa8p+-Tp%pJPRPsQos#V+?Ha3X`aH10RnCMr3Qjw#Kbg3^r(dGQ^~L-DmIpUh z8n`FZ51IB5(7)j44$P)hBrG91hg~K)N%0I|X{CHM^ZuB1S}k`DpXa%a4wmB20r8%3 zR{^2&tT5tLEaxOkZ1WLYN*6*hC)M~8;KNf%MMde zbP>~&TFtP!B)pOjoC$*dS!@1zll$79m=bTlYA!v=LEZrj`Qk`qhL=3cX9#qF7ktsG2(e70b1FkRuG6E zXgVaGV>0uf2=KvGS7kDk(nkotb~Rg<^cef1w`l`ow)C)NF598^iG!6jkPAd)NYN>2 zoF^Z#X^VT(Y65RyapK#Y{EoQd6Zfuix0`P|=&ULyJ;_DtKlgMah<|Ms+^(Lx>XJJA zg3EPGot1@e7YM64UMzgxcGkVfbZ3t;I*Na@Io9ydOcwZgpIcme11<~%MfN=~tOJpkCIq7WV2p6lWtAT z1KuMuxs7tEgH?_3_I)+%p?$bW8#S8LvLZ+Nl7$)RkGj|>A2(YZY;-Qw)giYZm;>OBToP2&zMI;?xvw;I4h4fc#t)E zV@wDgk|I=@a?utbb8?XBQwZt-2O0<`Gk$UL)JxEbfbcScZX;??TFa0aKnff>7KRFX`Hs{Igg zLZr*M=i)s9ISLa#e2t#-xJQky*wTAr_Vx%Y_+7v%;8fJ0JfcX;yTv^{SSuTbkmz}M z4CpC~gTHt)(7RF!a)W!cW3L)Dj6eE=Wz6A?RA!RRD@Hajg9LFsc>Q8cukpYN(^*o? z<)+ce^GftDh@%-(4+J%p4}>!rQhL{PLflm}K^Rr@&agUJ3S5=XZ19rv3w`Oe&*D#P zlcC0XYgLB>$8`kGz6ph;#=Rtm3FYtXTGUn3J|sucm% z!``6nD%VyEs$sLz`765+?%KNS=w&QNR*`a&08y&Zvr2e?kh+1#VRxtcG%}<3&xbCO zfTfBz`ajhKL*ua+@7o^?arM{_(J&!!+-f3*g=|I~7fkRyyzdPZPT=>Ij4d@}=jf$= zMuy=JK9Ew9pjakkA64Q`d`)9M|P20~ty%Ia2&FxV0luhPAe- z_ZzXjoJ=voUa!iI8kvsyVi!B`OjFZEUb}nM*Y%{ICp%4Q4WS^sz=8{#{3u!_*y+L( zHMdq)ugBL)YY_-U`?&N|=ZewQ)XS{C*L;HmJ`*#6Fa)%j_n3SHnx0vhqYWhYGop9L z#dXDhQ0u~jso6&5~STro5G&SbiVTPgKxa?lV+6em;Xj1c< zbrn){Z)k1_QLVS{z3N=-@aCL`xh1em8B(h1oYc!nKD08l`EAO6j@((|_ZY3U#Qp}7l z;-p!pUS+JQC13loCeX|q00p)Csd=7}IrIobA40Iz3>NRv;uQGP6z4HefsOh|O#2xDV}w=w10x~N+u zSK#HbhVmEEN?C@qUG-y~PAgf_bing}EbljxuIro`elu-@U;vXC{kDXqrT6d*%^(TN zda(?Qwdt>B_Z)fu{H~8!Pir>gVhKC;m`3QZ4JBE+l00O~UX&}DW@x&uQ~ z%V(0)oZ2R{`^A;}$Zv~Bn6pINVXlSNC`a1)n9@*|goh|w5<}jYjIC$=0P_T3JMk02 zk+1xM*|i5D!(mSWCwb8wyQH}_)bFhepsb&)Lw-wWC5LPQ-a^1aR=+i`%9M(?5)Dyv z_z_a!sJ!p5+ZET$C22kH(L#yL?wV2RTrK5J_4&h%Lv^f*AdE`+srvjw^Eg(|^~ zO2?IYS*`1ZM$1FuQzk0~tb3;L=I|_LtBf+^a@GkKEy|W>`jiV@RymkC>==u)(ib_= z$87l0J#o;kbCX&{zz1Q;PAD}zK1Jw;qZ`wab@uWH-NZ&kZQvxg02ny`c;6h>Ll%$I zd9AO38NBZ3EmHwI__aIW3G^SAUyW~S9MA`7_HDR<%1$)3)BVE`|2H_4~FW* zScB+EONqXW3S`a{oWKmYrD)0&Z=rjXlTQot8pn%&JM(kt?H?!Go(5SYT&enrl{`r( zNmcJ4({&x_F0uNfis8$lRvej%C?cNK=l=+&89n{UzBKLHp$wMUPD2k?j+%b7Wjh*f z#h){8!al=|0sV1TN(=OB69=x?%PsJkpZZ^8lQ9S06Lce&lgA!1VmNn)I`SeC+#4u? zE->vopL;GNm4~o< zIPqO<28gWRRqct1Pi1OUgirM!?RF5#cyJr@x>@o4wy?r(>y&)xGTt2E>>lZU!p_nU8EokLb(RyunBdTFsjZl=t1}|0Y_uQ;ZCvk&VI3heir^ZP{-8kO<2du+F!*mWu7KUxReCzeyqk!8*TBtKxp*r1n8%I9z+Lp^1UtZX+XY zQ&U5uJ$p?}jdp1p?^aBwvRkp58W`=d+-quSY^MFkq67Isv1J|Zf6nsMp(B4BPVpO9 z=aNH;g+T;5B+@s=AS6Qf_aJ}9IRuaJ3l9kW zHkZrSxV~qRQITLBov#P_Yx(26f=YD1F}|T7Bg5UGzalFVS}T}Xew!r* z8Sack20{Lt31zf*obdGv2#G{S{Gw+K`u7BX=iu*H@I$_laL89e28wmJsk5<>9`^ba_P{_m6TG5;g^zli=wZVmbhagYhDEAC0=mX0Q|LXW@Ox9%Ec#fq?1m!me6>@DutrkXEY zSN^Nn!%Ob&9=*%*cVAI@wbJ~%sCv_+%Lu=>IvW%NTMSSVm_yU+E)?kA*P){#$GQS%u2y#=aWD9gLfmOr}n ez0O;+yOX}m11*~rrbnLMGA<}^}zy2S2oP1FL literal 0 HcmV?d00001 diff --git a/lms/static/images/resolvedflag.png b/lms/static/images/resolvedflag.png new file mode 100644 index 0000000000000000000000000000000000000000..8e318f786ce8bcdefc4b7a1b686ec2afd42bd6c0 GIT binary patch literal 362 zcmeAS@N?(olHy`uVBq!ia0vp^AT}Qd6Of#FQ0fnmVlH;_4B_D5xc$)o0g%gC;1OBO zz`!j8!i<;h)`8T>l(C#5R5WfrBD=NDxcD>w(6z1Xv<2dGXI zq|Ui0HL)Z!KTjbfGdGpN&`94z-_U5*^#7ND3XXWXIEF|}P5$x!|Nq72{(X0?etx`t z{J%tlVPV>{ce8)byfO1%-S5+fPc2qnoVhYnTRdI=zxBuIowINH-05iToH>z^O=70x%*se0Qj8Ic3sYm;(_oY-#xZs6^fLzz z6uf$R^Y#1pH9x-X(TD^-^6R9$$9r~cnz y-Y2P#O@+-DwO!`$Ft+*k<;|1s%`+x4GBR{uPcxDDXOIE(4}+(xpUXO@geCxqpqFs~ literal 0 HcmV?d00001 diff --git a/lms/static/sass/_discussion.scss b/lms/static/sass/_discussion.scss index 9583a8d30f..c03785859e 100644 --- a/lms/static/sass/_discussion.scss +++ b/lms/static/sass/_discussion.scss @@ -95,6 +95,7 @@ body.discussion { + .new-post-form-errors { display: none; background: $error-red; @@ -1280,8 +1281,8 @@ body.discussion { .discussion-article { position: relative; padding: 40px; - min-height: 468px; - + min-height: 468px; + a { word-wrap: break-word; } @@ -1334,6 +1335,9 @@ body.discussion { background-position: 0 0; } } + + + } .discussion-post { @@ -2436,7 +2440,6 @@ body.discussion { @extend .discussion-module } - .group-visibility-label { font-size: 12px; color:#000; @@ -2491,4 +2494,39 @@ body.discussion { .pinned-false { display:none; +} + +.discussion-flag-abuse { + font-size: 12px; + float:right; + padding-right: 5px; + font-style: italic; + } + +.notflagged .icon +{ + display: inline-block; + width: 10px; + height: 14px; + padding-right: 3px; + background: transparent url('../images/notflagged.png') no-repeat 0 0; +} + +.flagged .icon +{ + display: inline-block; + width: 10px; + height: 14px; + padding-right: 3px; + background: transparent url('../images/flagged.png') no-repeat 0 0; +} + +.flagged span { + color: #B82066; + font-style: italic; +} + +.notflagged span { + color: #888; + font-style: italic; } \ No newline at end of file diff --git a/lms/templates/discussion/_filter_dropdown.html b/lms/templates/discussion/_filter_dropdown.html index fef4abb11f..dd5b94f910 100644 --- a/lms/templates/discussion/_filter_dropdown.html +++ b/lms/templates/discussion/_filter_dropdown.html @@ -33,6 +33,14 @@ Show All Discussions + %if flag_moderator: +
  • + + Show Flagged Discussions + +
    • + + %endif
  • Following diff --git a/lms/templates/discussion/_underscore_templates.html b/lms/templates/discussion/_underscore_templates.html index 24e3b467be..110e6ffc19 100644 --- a/lms/templates/discussion/_underscore_templates.html +++ b/lms/templates/discussion/_underscore_templates.html @@ -3,6 +3,7 @@ + diff --git a/rakefiles/jasmine.rake b/rakefiles/jasmine.rake index d9b3bee427..36e4e57a27 100644 --- a/rakefiles/jasmine.rake +++ b/rakefiles/jasmine.rake @@ -35,11 +35,19 @@ def django_for_jasmine(system, django_reload) end def template_jasmine_runner(lib) - coffee_files = Dir["#{lib}/**/js/**/*.coffee", "common/static/coffee/src/**/*.coffee"] + case lib + when /common\/lib\/.+/ + coffee_files = Dir["#{lib}/**/js/**/*.coffee", "common/static/coffee/src/**/*.coffee"] + when /common\/static\/coffee/ + coffee_files = Dir["#{lib}/**/*.coffee"] + else + puts('I do not know how to run jasmine tests for #{lib}') + exit + end if !coffee_files.empty? sh("node_modules/.bin/coffee -c #{coffee_files.join(' ')}") end - phantom_jasmine_path = File.expand_path("node_modules/phantom-jasmine") + phantom_jasmine_path = File.expand_path("common/test/phantom-jasmine") common_js_root = File.expand_path("common/static/js") common_coffee_root = File.expand_path("common/static/coffee/src") @@ -50,7 +58,7 @@ def template_jasmine_runner(lib) js_specs = Dir[spec_glob].sort_by {|p| [p.split('/').length, p]} .map {|f| File.expand_path(f)} js_source = Dir[src_glob].sort_by {|p| [p.split('/').length, p]} .map {|f| File.expand_path(f)} - template = ERB.new(File.read("#{lib}/jasmine_test_runner.html.erb")) + template = ERB.new(File.read("common/templates/jasmine/jasmine_test_runner.html.erb")) template_output = "#{lib}/jasmine_test_runner.html" File.open(template_output, 'w') do |f| f.write(template.result(binding)) @@ -95,3 +103,20 @@ Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib| end end end + +desc "Open jasmine tests for discussion in your default browser" +task "browse_jasmine_discussion" do + template_jasmine_runner("common/static/coffee") do |f| + sh("python -m webbrowser -t 'file://#{f}'") + puts "Press ENTER to terminate".red + $stdin.gets + end +end + +desc "Use phantomjs to run jasmine tests for discussion from the console" +task "phantomjs_jasmine_discussion" do + phantomjs = ENV['PHANTOMJS_PATH'] || 'phantomjs' + template_jasmine_runner("common/static/coffee") do |f| + sh("#{phantomjs} node_modules/phantom-jasmine/lib/run_jasmine_test.coffee #{f}") + end +end From cbdf93473b15744e4c7130bc72159319c90bf194 Mon Sep 17 00:00:00 2001 From: Jay Zoldak Date: Tue, 7 May 2013 16:21:37 -0400 Subject: [PATCH 130/162] start to get the jasmine tests working for comment service --- .../spec/discussion/content_spec.coffee | 43 +++++++++++++++++++ .../response_comment_show_view_spec.coffee | 31 +++---------- common/templates/jasmine/base.html | 5 --- .../jasmine/jasmine_test_runner.html.erb | 11 +++-- 4 files changed, 57 insertions(+), 33 deletions(-) create mode 100644 common/static/coffee/spec/discussion/content_spec.coffee diff --git a/common/static/coffee/spec/discussion/content_spec.coffee b/common/static/coffee/spec/discussion/content_spec.coffee new file mode 100644 index 0000000000..6b6188ad19 --- /dev/null +++ b/common/static/coffee/spec/discussion/content_spec.coffee @@ -0,0 +1,43 @@ +describe 'Content', -> + beforeEach -> + # TODO: figure out a better way of handling this + # It is set up in main.coffee DiscussionApp.start + window.$$course_id = 'mitX/999/test' + window.user = new DiscussionUser {id: '567'} + + @content = new Content { + id: '01234567', + user_id: '567', + course_id: 'mitX/999/test', + body: 'this is some content', + abuse_flaggers: ['123'] + } + + it 'should exist', -> + expect(Content).toBeDefined() + + it 'is initialized correctly', -> + @content.initialize + expect(Content.contents['01234567']).toEqual @content + expect(@content.get 'id').toEqual '01234567' + expect(@content.get 'user_url').toEqual '/courses/mitX/999/test/discussion/forum/users/567' + expect(@content.get 'children').toEqual [] + expect(@content.get 'comments').toEqual(jasmine.any(Comments)) + + it 'can update info', -> + @content.updateInfo { + ability: 'can_endorse', + voted: true, + subscribed: true + } + expect(@content.get 'ability').toEqual 'can_endorse' + expect(@content.get 'voted').toEqual true + expect(@content.get 'subscribed').toEqual true + + describe 'can be flagged and unflagged', -> + beforeEach -> + spyOn @content, 'trigger' + + it 'can be flagged for abuse', -> + @content.flagAbuse + expect(@content.get 'abuse_flaggers').toEqual ['123', '567'] diff --git a/common/static/coffee/spec/discussion/view/response_comment_show_view_spec.coffee b/common/static/coffee/spec/discussion/view/response_comment_show_view_spec.coffee index a4168edb3c..814a428c70 100644 --- a/common/static/coffee/spec/discussion/view/response_comment_show_view_spec.coffee +++ b/common/static/coffee/spec/discussion/view/response_comment_show_view_spec.coffee @@ -1,26 +1,7 @@ -describe "ResponseCommentShowView", -> - beforeEach -> - setFixtures """ -
  • - -
    • - """ - # spyOn($.fn, 'load').andReturn(@moduleData) +xdescribe "ResponseCommentShowView", -> - @showView = new ResponseCommentShowView( - el: $("li") - ) - - describe "class definition", -> - it "sets the correct tagName", -> - expect(@showView.tagName).toEqual("li") + it "defines the class", -> + spyOn myComment, 'initialize' + myComment = new Comment() + myView = new ResponseCommentShowView(myComment) + expect(myView.tagName).toBeDefined() diff --git a/common/templates/jasmine/base.html b/common/templates/jasmine/base.html index c9a32f4005..3b8013a282 100644 --- a/common/templates/jasmine/base.html +++ b/common/templates/jasmine/base.html @@ -37,22 +37,17 @@ - + + + + + + + + - - + """ - it "defines the class", -> - spyOn myComment, 'initialize' - myComment = new Comment() - myView = new ResponseCommentShowView(myComment) - expect(myView.tagName).toBeDefined() + # set up a model for a new Comment + @response = new Comment { + id: '01234567', + user_id: '567', + course_id: 'mitX/999/test', + body: 'this is a response', + created_at: '2013-04-03T20:08:39Z', + abuse_flaggers: ['123'] + roles: [] + } + @view = new ResponseCommentShowView({ model: @response }) + + # spyOn(DiscussionUtil, 'loadRoles').andReturn [] + + it 'defines the tag', -> + expect($('#jasmine-fixtures')).toExist + expect(@view.tagName).toBeDefined + expect(@view.el.tagName.toLowerCase()).toBe 'li' + + it 'is tied to the model', -> + expect(@view.model).toBeDefined(); + + describe 'rendering', -> + + beforeEach -> + spyOn(@view, 'renderAttrs') + spyOn(@view, 'markAsStaff') + spyOn(@view, 'convertMath') + + it 'produces the correct HTML', -> + @view.render() + expect(@view.el.innerHTML).toContainHtml """ +
    this is a response
    +
    +
    + """ diff --git a/common/static/coffee/src/discussion/views/response_comment_show_view.coffee b/common/static/coffee/src/discussion/views/response_comment_show_view.coffee index 18d405fdb4..6023964c75 100644 --- a/common/static/coffee/src/discussion/views/response_comment_show_view.coffee +++ b/common/static/coffee/src/discussion/views/response_comment_show_view.coffee @@ -1,11 +1,11 @@ if Backbone? class @ResponseCommentShowView extends DiscussionContentView - + events: - "click .discussion-flag-abuse": "toggleFlagAbuse" + "click .discussion-flag-abuse": "toggleFlagAbuse" tagName: "li" - + initialize: -> super() @model.on "change", @updateModelDetails @@ -42,17 +42,17 @@ if Backbone? @$el.find("a.profile-link").after('staff') else if DiscussionUtil.isTA(@model.get("user_id")) @$el.find("a.profile-link").after('Community  TA') - - + + renderFlagged: => if window.user.id in @model.get("abuse_flaggers") or (DiscussionUtil.isFlagModerator and @model.get("abuse_flaggers").length > 0) - @$("[data-role=thread-flag]").addClass("flagged") + @$("[data-role=thread-flag]").addClass("flagged") @$("[data-role=thread-flag]").removeClass("notflagged") else - @$("[data-role=thread-flag]").removeClass("flagged") - @$("[data-role=thread-flag]").addClass("notflagged") - + @$("[data-role=thread-flag]").removeClass("flagged") + @$("[data-role=thread-flag]").addClass("notflagged") + updateModelDetails: => @renderFlagged() - - + + diff --git a/common/static/js/vendor/flot/jquery.timeago.js b/common/static/js/vendor/flot/jquery.timeago.js new file mode 100644 index 0000000000..2e8d29f536 --- /dev/null +++ b/common/static/js/vendor/flot/jquery.timeago.js @@ -0,0 +1,152 @@ +/** + * Timeago is a jQuery plugin that makes it easy to support automatically + * updating fuzzy timestamps (e.g. "4 minutes ago" or "about 1 day ago"). + * + * @name timeago + * @version 0.11.4 + * @requires jQuery v1.2.3+ + * @author Ryan McGeary + * @license MIT License - http://www.opensource.org/licenses/mit-license.php + * + * For usage and examples, visit: + * http://timeago.yarp.com/ + * + * Copyright (c) 2008-2012, Ryan McGeary (ryan -[at]- mcgeary [*dot*] org) + */ +(function($) { + $.timeago = function(timestamp) { + if (timestamp instanceof Date) { + return inWords(timestamp); + } else if (typeof timestamp === "string") { + return inWords($.timeago.parse(timestamp)); + } else if (typeof timestamp === "number") { + return inWords(new Date(timestamp)); + } else { + return inWords($.timeago.datetime(timestamp)); + } + }; + var $t = $.timeago; + + $.extend($.timeago, { + settings: { + refreshMillis: 60000, + allowFuture: false, + strings: { + prefixAgo: null, + prefixFromNow: null, + suffixAgo: "ago", + suffixFromNow: "from now", + seconds: "less than a minute", + minute: "about a minute", + minutes: "%d minutes", + hour: "about an hour", + hours: "about %d hours", + day: "a day", + days: "%d days", + month: "about a month", + months: "%d months", + year: "about a year", + years: "%d years", + wordSeparator: " ", + numbers: [] + } + }, + inWords: function(distanceMillis) { + var $l = this.settings.strings; + var prefix = $l.prefixAgo; + var suffix = $l.suffixAgo; + if (this.settings.allowFuture) { + if (distanceMillis < 0) { + prefix = $l.prefixFromNow; + suffix = $l.suffixFromNow; + } + } + + var seconds = Math.abs(distanceMillis) / 1000; + var minutes = seconds / 60; + var hours = minutes / 60; + var days = hours / 24; + var years = days / 365; + + function substitute(stringOrFunction, number) { + var string = $.isFunction(stringOrFunction) ? stringOrFunction(number, distanceMillis) : stringOrFunction; + var value = ($l.numbers && $l.numbers[number]) || number; + return string.replace(/%d/i, value); + } + + var words = seconds < 45 && substitute($l.seconds, Math.round(seconds)) || + seconds < 90 && substitute($l.minute, 1) || + minutes < 45 && substitute($l.minutes, Math.round(minutes)) || + minutes < 90 && substitute($l.hour, 1) || + hours < 24 && substitute($l.hours, Math.round(hours)) || + hours < 42 && substitute($l.day, 1) || + days < 30 && substitute($l.days, Math.round(days)) || + days < 45 && substitute($l.month, 1) || + days < 365 && substitute($l.months, Math.round(days / 30)) || + years < 1.5 && substitute($l.year, 1) || + substitute($l.years, Math.round(years)); + + var separator = $l.wordSeparator === undefined ? " " : $l.wordSeparator; + return $.trim([prefix, words, suffix].join(separator)); + }, + parse: function(iso8601) { + var s = $.trim(iso8601); + s = s.replace(/\.\d+/,""); // remove milliseconds + s = s.replace(/-/,"/").replace(/-/,"/"); + s = s.replace(/T/," ").replace(/Z/," UTC"); + s = s.replace(/([\+\-]\d\d)\:?(\d\d)/," $1$2"); // -04:00 -> -0400 + return new Date(s); + }, + datetime: function(elem) { + var iso8601 = $t.isTime(elem) ? $(elem).attr("datetime") : $(elem).attr("title"); + return $t.parse(iso8601); + }, + isTime: function(elem) { + // jQuery's `is()` doesn't play well with HTML5 in IE + return $(elem).get(0).tagName.toLowerCase() === "time"; // $(elem).is("time"); + } + }); + + $.fn.timeago = function() { + var self = this; + self.each(refresh); + + var $s = $t.settings; + if ($s.refreshMillis > 0) { + setInterval(function() { self.each(refresh); }, $s.refreshMillis); + } + return self; + }; + + function refresh() { + var data = prepareData(this); + if (!isNaN(data.datetime)) { + $(this).text(inWords(data.datetime)); + } + return this; + } + + function prepareData(element) { + element = $(element); + if (!element.data("timeago")) { + element.data("timeago", { datetime: $t.datetime(element) }); + var text = $.trim(element.text()); + if (text.length > 0 && !($t.isTime(element) && element.attr("title"))) { + element.attr("title", text); + } + } + return element.data("timeago"); + } + + function inWords(date) { + return $t.inWords(distance(date)); + } + + function distance(date) { + return (new Date().getTime() - date.getTime()); + } + + // fix for IE6 suckage + document.createElement("abbr"); + document.createElement("time"); +}(jQuery)); diff --git a/common/static/js/vendor/jquery.timeago.js b/common/static/js/vendor/jquery.timeago.js new file mode 100644 index 0000000000..2e8d29f536 --- /dev/null +++ b/common/static/js/vendor/jquery.timeago.js @@ -0,0 +1,152 @@ +/** + * Timeago is a jQuery plugin that makes it easy to support automatically + * updating fuzzy timestamps (e.g. "4 minutes ago" or "about 1 day ago"). + * + * @name timeago + * @version 0.11.4 + * @requires jQuery v1.2.3+ + * @author Ryan McGeary + * @license MIT License - http://www.opensource.org/licenses/mit-license.php + * + * For usage and examples, visit: + * http://timeago.yarp.com/ + * + * Copyright (c) 2008-2012, Ryan McGeary (ryan -[at]- mcgeary [*dot*] org) + */ +(function($) { + $.timeago = function(timestamp) { + if (timestamp instanceof Date) { + return inWords(timestamp); + } else if (typeof timestamp === "string") { + return inWords($.timeago.parse(timestamp)); + } else if (typeof timestamp === "number") { + return inWords(new Date(timestamp)); + } else { + return inWords($.timeago.datetime(timestamp)); + } + }; + var $t = $.timeago; + + $.extend($.timeago, { + settings: { + refreshMillis: 60000, + allowFuture: false, + strings: { + prefixAgo: null, + prefixFromNow: null, + suffixAgo: "ago", + suffixFromNow: "from now", + seconds: "less than a minute", + minute: "about a minute", + minutes: "%d minutes", + hour: "about an hour", + hours: "about %d hours", + day: "a day", + days: "%d days", + month: "about a month", + months: "%d months", + year: "about a year", + years: "%d years", + wordSeparator: " ", + numbers: [] + } + }, + inWords: function(distanceMillis) { + var $l = this.settings.strings; + var prefix = $l.prefixAgo; + var suffix = $l.suffixAgo; + if (this.settings.allowFuture) { + if (distanceMillis < 0) { + prefix = $l.prefixFromNow; + suffix = $l.suffixFromNow; + } + } + + var seconds = Math.abs(distanceMillis) / 1000; + var minutes = seconds / 60; + var hours = minutes / 60; + var days = hours / 24; + var years = days / 365; + + function substitute(stringOrFunction, number) { + var string = $.isFunction(stringOrFunction) ? stringOrFunction(number, distanceMillis) : stringOrFunction; + var value = ($l.numbers && $l.numbers[number]) || number; + return string.replace(/%d/i, value); + } + + var words = seconds < 45 && substitute($l.seconds, Math.round(seconds)) || + seconds < 90 && substitute($l.minute, 1) || + minutes < 45 && substitute($l.minutes, Math.round(minutes)) || + minutes < 90 && substitute($l.hour, 1) || + hours < 24 && substitute($l.hours, Math.round(hours)) || + hours < 42 && substitute($l.day, 1) || + days < 30 && substitute($l.days, Math.round(days)) || + days < 45 && substitute($l.month, 1) || + days < 365 && substitute($l.months, Math.round(days / 30)) || + years < 1.5 && substitute($l.year, 1) || + substitute($l.years, Math.round(years)); + + var separator = $l.wordSeparator === undefined ? " " : $l.wordSeparator; + return $.trim([prefix, words, suffix].join(separator)); + }, + parse: function(iso8601) { + var s = $.trim(iso8601); + s = s.replace(/\.\d+/,""); // remove milliseconds + s = s.replace(/-/,"/").replace(/-/,"/"); + s = s.replace(/T/," ").replace(/Z/," UTC"); + s = s.replace(/([\+\-]\d\d)\:?(\d\d)/," $1$2"); // -04:00 -> -0400 + return new Date(s); + }, + datetime: function(elem) { + var iso8601 = $t.isTime(elem) ? $(elem).attr("datetime") : $(elem).attr("title"); + return $t.parse(iso8601); + }, + isTime: function(elem) { + // jQuery's `is()` doesn't play well with HTML5 in IE + return $(elem).get(0).tagName.toLowerCase() === "time"; // $(elem).is("time"); + } + }); + + $.fn.timeago = function() { + var self = this; + self.each(refresh); + + var $s = $t.settings; + if ($s.refreshMillis > 0) { + setInterval(function() { self.each(refresh); }, $s.refreshMillis); + } + return self; + }; + + function refresh() { + var data = prepareData(this); + if (!isNaN(data.datetime)) { + $(this).text(inWords(data.datetime)); + } + return this; + } + + function prepareData(element) { + element = $(element); + if (!element.data("timeago")) { + element.data("timeago", { datetime: $t.datetime(element) }); + var text = $.trim(element.text()); + if (text.length > 0 && !($t.isTime(element) && element.attr("title"))) { + element.attr("title", text); + } + } + return element.data("timeago"); + } + + function inWords(date) { + return $t.inWords(distance(date)); + } + + function distance(date) { + return (new Date().getTime() - date.getTime()); + } + + // fix for IE6 suckage + document.createElement("abbr"); + document.createElement("time"); +}(jQuery)); diff --git a/common/templates/jasmine/base.html b/common/templates/jasmine/base.html index 3b8013a282..9a1b3bed92 100644 --- a/common/templates/jasmine/base.html +++ b/common/templates/jasmine/base.html @@ -37,18 +37,18 @@ +