Importable python_lib.zip assets

Lots of plumbing to allow an asset named python_lib.zip to be imported by jailed Python code. This function can find the "python_lib.zip" asset, and is passed down through ModuleSystem and LoncapaSystem so that capa problems have access to the zipfile.
2014-09-08 12:32:23 -04:00
parent 43ce6421cf
commit 616bbbab4e
18 changed files with 208 additions and 14 deletions
--- a/common/lib/capa/capa/capa_problem.py
+++ b/common/lib/capa/capa/capa_problem.py
@@ -13,14 +13,15 @@ Main module which shows problems (of "capa" type).
 This is used by capa_module.
 """

+from copy import deepcopy
 from datetime import datetime
 import logging
 import os.path
 import re

 from lxml import etree
+from pytz import UTC
 from xml.sax.saxutils import unescape
-from copy import deepcopy

 from capa.correctmap import CorrectMap
 import capa.inputtypes as inputtypes
@@ -28,10 +29,8 @@ import capa.customrender as customrender
 import capa.responsetypes as responsetypes
 from capa.util import contextualize_text, convert_files_to_filenames
 import capa.xqueue_interface as xqueue_interface
-
 from capa.safe_exec import safe_exec

-from pytz import UTC

 # extra things displayed after "show answers" is pressed
 solution_tags = ['solution']
@@ -84,6 +83,7 @@ class LoncapaSystem(object):
        anonymous_student_id,
        cache,
        can_execute_unsafe_code,
+        get_python_lib_zip,
        DEBUG,                                          # pylint: disable=invalid-name
        filestore,
        i18n,
@@ -98,6 +98,7 @@ class LoncapaSystem(object):
        self.anonymous_student_id = anonymous_student_id
        self.cache = cache
        self.can_execute_unsafe_code = can_execute_unsafe_code
+        self.get_python_lib_zip = get_python_lib_zip
        self.DEBUG = DEBUG                              # pylint: disable=invalid-name
        self.filestore = filestore
        self.i18n = i18n
@@ -645,6 +646,13 @@ class LoncapaProblem(object):
            code = unescape(script.text, XMLESC)
            all_code += code

+        # An asset named python_lib.zip can be imported by Python code.
+        extra_files = []
+        zip_lib = self.capa_system.get_python_lib_zip()
+        if zip_lib is not None:
+            extra_files.append(("python_lib.zip", zip_lib))
+            python_path.append("python_lib.zip")
+
        if all_code:
            try:
                safe_exec(
@@ -652,6 +660,7 @@ class LoncapaProblem(object):
                    context,
                    random_seed=self.seed,
                    python_path=python_path,
+                    extra_files=extra_files,
                    cache=self.capa_system.cache,
                    slug=self.problem_id,
                    unsafely=self.capa_system.can_execute_unsafe_code(),
@@ -664,6 +673,7 @@ class LoncapaProblem(object):
        # Store code source in context, along with the Python path needed to run it correctly.
        context['script_code'] = all_code
        context['python_path'] = python_path
+        context['extra_files'] = extra_files or None
        return context

    def _extract_html(self, problemtree):  # private
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -305,6 +305,7 @@ class LoncapaResponse(object):
                    code,
                    globals_dict,
                    python_path=self.context['python_path'],
+                    extra_files=self.context['extra_files'],
                    slug=self.id,
                    random_seed=self.context['seed'],
                    unsafely=self.capa_system.can_execute_unsafe_code(),
@@ -1480,6 +1481,7 @@ class CustomResponse(LoncapaResponse):
                            code,
                            globals_dict,
                            python_path=self.context['python_path'],
+                            extra_files=self.context['extra_files'],
                            slug=self.id,
                            random_seed=self.context['seed'],
                            unsafely=self.capa_system.can_execute_unsafe_code(),
@@ -1613,6 +1615,8 @@ class CustomResponse(LoncapaResponse):
                    self.code,
                    self.context,
                    cache=self.capa_system.cache,
+                    python_path=self.context['python_path'],
+                    extra_files=self.context['extra_files'],
                    slug=self.id,
                    random_seed=self.context['seed'],
                    unsafely=self.capa_system.can_execute_unsafe_code(),
@@ -2496,6 +2500,8 @@ class SchematicResponse(LoncapaResponse):
                self.code,
                self.context,
                cache=self.capa_system.cache,
+                python_path=self.context['python_path'],
+                extra_files=self.context['extra_files'],
                slug=self.id,
                random_seed=self.context['seed'],
                unsafely=self.capa_system.can_execute_unsafe_code(),
--- a/common/lib/capa/capa/safe_exec/safe_exec.py
+++ b/common/lib/capa/capa/safe_exec/safe_exec.py
@@ -71,7 +71,16 @@ def update_hash(hasher, obj):


@dog_stats_api.timed('capa.safe_exec.time')
-def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None, slug=None, unsafely=False):
+def safe_exec(
+    code,
+    globals_dict,
+    random_seed=None,
+    python_path=None,
+    extra_files=None,
+    cache=None,
+    slug=None,
+    unsafely=False,
+):
    """
    Execute python code safely.

@@ -81,7 +90,12 @@ def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None

    `random_seed` will be used to see the `random` module available to the code.

-    `python_path` is a list of directories to add to the Python path before execution.
+    `python_path` is a list of filenames or directories to add to the Python
+    path before execution.  If the name is not in `extra_files`, then it will
+    also be copied into the sandbox.
+
+    `extra_files` is a list of (filename, contents) pairs.  These files are
+    created in the sandbox.

    `cache` is an object with .get(key) and .set(key, value) methods.  It will be used
    to cache the execution, taking into account the code, the values of the globals,
@@ -123,7 +137,7 @@ def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None
    try:
        exec_fn(
            code_prolog + LAZY_IMPORTS + code, globals_dict,
-            python_path=python_path, slug=slug,
+            python_path=python_path, extra_files=extra_files, slug=slug,
        )
    except SafeExecException as e:
        emsg = e.message
--- a/common/lib/capa/capa/tests/init.py
+++ b/common/lib/capa/capa/tests/init.py
@@ -41,6 +41,7 @@ def test_capa_system():
        anonymous_student_id='student',
        cache=None,
        can_execute_unsafe_code=lambda: False,
+        get_python_lib_zip=lambda: None,
        DEBUG=True,
        filestore=fs.osfs.OSFS(os.path.join(TEST_DIR, "test_files")),
        i18n=gettext.NullTranslations(),
--- a/common/lib/capa/capa/tests/test_responsetypes.py
+++ b/common/lib/capa/capa/tests/test_responsetypes.py
@@ -3,15 +3,19 @@
 Tests of responsetypes
 """

+from cStringIO import StringIO
 from datetime import datetime
 import json
 import os
 import pyparsing
 import random
-import unittest
 import textwrap
-import requests
+import unittest
+import zipfile
+
 import mock
+from pytz import UTC
+import requests

 from . import new_loncapa_problem, test_capa_system, load_fixture
 import calc
@@ -23,8 +27,6 @@ from capa.util import convert_files_to_filenames
 from capa.util import compare_with_tolerance
 from capa.xqueue_interface import dateformat

-from pytz import UTC
-

 class ResponseTest(unittest.TestCase):
    """Base class for tests of capa responses."""
@@ -1712,6 +1714,28 @@ class CustomResponseTest(ResponseTest):
            except ResponseError:
                self.fail("Could not use name '{0}s' in custom response".format(module_name))

+    def test_python_lib_zip_is_available(self):
+        # Prove that we can import code from a zipfile passed down to us.
+
+        # Make a zipfile with one module in it with one function.
+        zipstring = StringIO()
+        zipf = zipfile.ZipFile(zipstring, "w")
+        zipf.writestr("my_helper.py", textwrap.dedent("""\
+            def seventeen():
+                return 17
+            """))
+        zipf.close()
+
+        # Use that module in our Python script.
+        script = textwrap.dedent("""
+            import my_helper
+            num = my_helper.seventeen()
+            """)
+        capa_system = test_capa_system()
+        capa_system.get_python_lib_zip = lambda: zipstring.getvalue()
+        problem = self.build_problem(script=script, capa_system=capa_system)
+        self.assertEqual(problem.context['num'], 17)
+

 class SchematicResponseTest(ResponseTest):
    from capa.tests.response_xml_factory import SchematicResponseXMLFactory
--- a/common/lib/xmodule/xmodule/capa_base.py
+++ b/common/lib/xmodule/xmodule/capa_base.py
@@ -298,6 +298,7 @@ class CapaMixin(CapaFields):
            anonymous_student_id=self.runtime.anonymous_student_id,
            cache=self.runtime.cache,
            can_execute_unsafe_code=self.runtime.can_execute_unsafe_code,
+            get_python_lib_zip=self.runtime.get_python_lib_zip,
            DEBUG=self.runtime.DEBUG,
            filestore=self.runtime.filestore,
            i18n=self.runtime.service(self, "i18n"),
--- a/common/lib/xmodule/xmodule/x_module.py
+++ b/common/lib/xmodule/xmodule/x_module.py
@@ -1244,7 +1244,7 @@ class ModuleSystem(MetricsMixin, ConfigurableFragmentWrapper, Runtime):  # pylin
            cache=None, can_execute_unsafe_code=None, replace_course_urls=None,
            replace_jump_to_id_urls=None, error_descriptor_class=None, get_real_user=None,
            field_data=None, get_user_role=None, rebind_noauth_module_to_user=None,
-            user_location=None, **kwargs):
+            user_location=None, get_python_lib_zip=None, **kwargs):
        """
        Create a closure around the system environment.

@@ -1293,6 +1293,10 @@ class ModuleSystem(MetricsMixin, ConfigurableFragmentWrapper, Runtime):  # pylin
        can_execute_unsafe_code - A function returning a boolean, whether or
            not to allow the execution of unsafe, unsandboxed code.

+        get_python_lib_zip - A function returning a bytestring or None.  The
+            bytestring is the contents of a zip file that should be importable
+            by other Python code running in the module.
+
        error_descriptor_class - The class to use to render XModules with errors

        get_real_user - function that takes `anonymous_student_id` and returns real user_id,
@@ -1334,6 +1338,7 @@ class ModuleSystem(MetricsMixin, ConfigurableFragmentWrapper, Runtime):  # pylin

        self.cache = cache or DoNothingCache()
        self.can_execute_unsafe_code = can_execute_unsafe_code or (lambda: False)
+        self.get_python_lib_zip = get_python_lib_zip or (lambda: None)
        self.replace_course_urls = replace_course_urls
        self.replace_jump_to_id_urls = replace_jump_to_id_urls
        self.error_descriptor_class = error_descriptor_class