Merge pull request #27795 from eduNEXT/eric/codejail_rest_service

Optionally run Codejail in a external service using REST API.

cms/envs/common.py

@@ -1000,6 +1000,28 @@ CODE_JAIL = {
 
 COURSES_WITH_UNSAFE_CODE = []
 
+# Cojail REST service
+ENABLE_CODEJAIL_REST_SERVICE = False
+# .. setting_name: CODE_JAIL_REST_SERVICE_REMOTE_EXEC
+# .. setting_default: 'common.lib.capa.capa.safe_exec.remote_exec.send_safe_exec_request_v0'
+# .. setting_description: Set the python package.module.function that is reponsible of
+#   calling the remote service in charge of jailed code execution
+CODE_JAIL_REST_SERVICE_REMOTE_EXEC = 'common.lib.capa.capa.safe_exec.remote_exec.send_safe_exec_request_v0'
+# .. setting_name: CODE_JAIL_REST_SERVICE_HOST
+# .. setting_default: 'http://127.0.0.1:8550'
+# .. setting_description: Set the codejail remote service host
+CODE_JAIL_REST_SERVICE_HOST = 'http://127.0.0.1:8550'
+# .. setting_name: CODE_JAIL_REST_SERVICE_CONNECT_TIMEOUT
+# .. setting_default: 0.5
+# .. setting_description: Set the number of seconds CMS will wait to establish an internal
+#   connection to the codejail remote service.
+CODE_JAIL_REST_SERVICE_CONNECT_TIMEOUT = 0.5  # time in seconds
+# .. setting_name: CODE_JAIL_REST_SERVICE_READ_TIMEOUT
+# .. setting_default: 3.5
+# .. setting_description: Set the number of seconds CMS will wait for a response from the
+#   codejail remote service endpoint.
+CODE_JAIL_REST_SERVICE_READ_TIMEOUT = 3.5  # time in seconds
+
 ############################ DJANGO_BUILTINS ################################
 # Change DEBUG in your environment settings files, not here
 DEBUG = False

common/lib/capa/capa/safe_exec/exceptions.py

@@ -0,0 +1,21 @@
+"""
+Exceptions related to safe exec.
+"""
+
+
+class CodejailServiceParseError(Exception):
+    """
+    An exception that is raised whenever we have issues with data parsing.
+    """
+
+
+class CodejailServiceStatusError(Exception):
+    """
+    An exception that is raised whenever Codejail service response status is different to 200.
+    """
+
+
+class CodejailServiceUnavailable(Exception):
+    """
+    An exception that is raised whenever Codejail service is unavailable.
+    """

common/lib/capa/capa/safe_exec/remote_exec.py

@@ -0,0 +1,107 @@
+"""
+Helper methods related to safe exec.
+"""
+
+import requests
+import json
+import logging
+
+from codejail.safe_exec import SafeExecException
+from django.conf import settings
+from django.utils.translation import ugettext as _
+from edx_toggles.toggles import SettingToggle
+from importlib import import_module
+from requests.exceptions import RequestException, HTTPError
+from simplejson import JSONDecodeError
+
+from .exceptions import CodejailServiceParseError, CodejailServiceStatusError, CodejailServiceUnavailable
+
+log = logging.getLogger(__name__)
+
+# .. toggle_name: ENABLE_CODEJAIL_REST_SERVICE
+# .. toggle_implementation: SettingToggle
+# .. toggle_default: False
+# .. toggle_description: Set this to True if you want to run Codejail code using
+#   a separate VM or container and communicate with edx-platform using REST API.
+# .. toggle_use_cases: open_edx
+# .. toggle_creation_date: 2021-08-19
+ENABLE_CODEJAIL_REST_SERVICE = SettingToggle(
+    "ENABLE_CODEJAIL_REST_SERVICE", default=False, module_name=__name__
+)
+
+
+def is_codejail_rest_service_enabled():
+    return ENABLE_CODEJAIL_REST_SERVICE.is_enabled()
+
+
+def get_remote_exec(*args, **kwargs):
+    """Get remote exec function based on setting and executes it."""
+    remote_exec_function_name = settings.CODE_JAIL_REST_SERVICE_REMOTE_EXEC
+    try:
+        mod_name, func_name = remote_exec_function_name.rsplit('.', 1)
+        remote_exec_module = import_module(mod_name)
+        remote_exec_function = getattr(remote_exec_module, func_name)
+        if not remote_exec_function:
+            remote_exec_function = send_safe_exec_request_v0
+    except ModuleNotFoundError:
+        return send_safe_exec_request_v0(*args, **kwargs)
+    return remote_exec_function(*args, **kwargs)
+
+
+def get_codejail_rest_service_endpoint():
+    return f"{settings.CODE_JAIL_REST_SERVICE_HOST}/api/v0/code-exec"
+
+
+def send_safe_exec_request_v0(data):
+    """
+    Sends a request to a codejail api service forwarding required code and files.
+    Arguments:
+        data: Dict containing code and other parameters
+            required for jailed code execution.
+            It also includes extra_files (python_lib.zip) required by the codejail execution.
+    Returns:
+        Response received from codejail api service
+    """
+    globals_dict = data["globals_dict"]
+    extra_files = data.pop("extra_files")
+
+    codejail_service_endpoint = get_codejail_rest_service_endpoint()
+    payload = json.dumps(data)
+
+    try:
+        response = requests.post(
+            codejail_service_endpoint,
+            files=extra_files,
+            data={'payload': payload},
+            timeout=(settings.CODE_JAIL_REST_SERVICE_CONNECT_TIMEOUT, settings.CODE_JAIL_REST_SERVICE_READ_TIMEOUT)
+        )
+
+    except RequestException as err:
+        log.error("Failed to connect to codejail api service: url=%s, params=%s",
+                  codejail_service_endpoint, str(payload))
+        raise CodejailServiceUnavailable(_(
+            "Codejail API Service is unavailable. "
+            "Please try again in a few minutes."
+        )) from err
+
+    try:
+        response.raise_for_status()
+    except HTTPError as err:
+        raise CodejailServiceStatusError(_("Codejail API Service invalid response.")) from err
+
+    try:
+        response_json = response.json()
+    except JSONDecodeError as err:
+        log.error("Invalid JSON response received from codejail api service: Response_Content=%s", response.content)
+        raise CodejailServiceParseError(_("Invalid JSON response received from codejail api service.")) from err
+
+    emsg = response_json.get("emsg")
+    exception = None
+
+    if emsg:
+        exception_msg = f"{emsg}. For more information check Codejail Service logs."
+        exception = SafeExecException(exception_msg)
+
+    globals_dict.update(response_json.get("globals_dict"))
+
+    return emsg, exception

common/lib/capa/capa/safe_exec/safe_exec.py

@@ -11,6 +11,7 @@ import six
 from six import text_type
 
 from . import lazymod
+from .remote_exec import is_codejail_rest_service_enabled, get_remote_exec
 
 # Establish the Python environment for Capa.
 # Capa assumes float-friendly division always.
@@ -143,28 +144,42 @@ def safe_exec(
     # Create the complete code we'll run.
     code_prolog = CODE_PROLOG % random_seed
 
-    # Decide which code executor to use.
-    if unsafely:
-        exec_fn = codejail_not_safe_exec
-    else:
-        exec_fn = codejail_safe_exec
+    if is_codejail_rest_service_enabled():
+        data = {
+            "code": code_prolog + LAZY_IMPORTS + code,
+            "globals_dict": globals_dict,
+            "python_path": python_path,
+            "limit_overrides_context": limit_overrides_context,
+            "slug": slug,
+            "unsafely": unsafely,
+            "extra_files": extra_files,
+        }
+
+        emsg, exception = get_remote_exec(data)
 
-    # Run the code!  Results are side effects in globals_dict.
-    try:
-        exec_fn(
-            code_prolog + LAZY_IMPORTS + code,
-            globals_dict,
-            python_path=python_path,
-            extra_files=extra_files,
-            limit_overrides_context=limit_overrides_context,
-            slug=slug,
-        )
-    except SafeExecException as e:
-        # Saving SafeExecException e in exception to be used later.
-        exception = e
-        emsg = text_type(e)
     else:
-        emsg = None
+        # Decide which code executor to use.
+        if unsafely:
+            exec_fn = codejail_not_safe_exec
+        else:
+            exec_fn = codejail_safe_exec
+
+        # Run the code!  Results are side effects in globals_dict.
+        try:
+            exec_fn(
+                code_prolog + LAZY_IMPORTS + code,
+                globals_dict,
+                python_path=python_path,
+                extra_files=extra_files,
+                limit_overrides_context=limit_overrides_context,
+                slug=slug,
+            )
+        except SafeExecException as e:
+            # Saving SafeExecException e in exception to be used later.
+            exception = e
+            emsg = text_type(e)
+        else:
+            emsg = None
 
     # Put the result back in the cache.  This is complicated by the fact that
     # the globals dict might not be entirely serializable.

lms/envs/common.py

@@ -1629,6 +1629,29 @@ CODE_JAIL = {
 #   ]
 COURSES_WITH_UNSAFE_CODE = []
 
+# Cojail REST service
+ENABLE_CODEJAIL_REST_SERVICE = False
+# .. setting_name: CODE_JAIL_REST_SERVICE_REMOTE_EXEC
+# .. setting_default: 'common.lib.capa.capa.safe_exec.remote_exec.send_safe_exec_request_v0'
+# .. setting_description: Set the python package.module.function that is reponsible of
+#   calling the remote service in charge of jailed code execution
+CODE_JAIL_REST_SERVICE_REMOTE_EXEC = 'common.lib.capa.capa.safe_exec.remote_exec.send_safe_exec_request_v0'
+# .. setting_name: CODE_JAIL_REST_SERVICE_HOST
+# .. setting_default: 'http://127.0.0.1:8550'
+# .. setting_description: Set the codejail remote service host
+CODE_JAIL_REST_SERVICE_HOST = 'http://127.0.0.1:8550'
+# .. setting_name: CODE_JAIL_REST_SERVICE_CONNECT_TIMEOUT
+# .. setting_default: 0.5
+# .. setting_description: Set the number of seconds LMS will wait to establish an internal
+#   connection to the codejail remote service.
+CODE_JAIL_REST_SERVICE_CONNECT_TIMEOUT = 0.5  # time in seconds
+# .. setting_name: CODE_JAIL_REST_SERVICE_READ_TIMEOUT
+# .. setting_default: 3.5
+# .. setting_description: Set the number of seconds LMS will wait for a response from the
+#   codejail remote service endpoint.
+CODE_JAIL_REST_SERVICE_READ_TIMEOUT = 3.5  # time in seconds
+
+
 ############################### DJANGO BUILT-INS ###############################
 # Change DEBUG in your environment settings files, not here
 DEBUG = False

requirements/edx-sandbox/py35-constraints.txt

@@ -22,6 +22,8 @@ sympy<1.7.0             # sympy 1.7.0 drops support for Python 3.5
 
 markupsafe<2.0.0        # markupsafe 2.0.0 requires Python >= 3.6
 
+nltk<3.6.3        		# nltk 3.6.3 drops support for Python 3.5
+
 cryptography<3.3        # cryptography 3.3 has dropped python3.5 support.
 PyJWT[crypto]<2.0.0     # PYJWT[crypto]==2.0.1 requires cryptography>=3.3.1
 social-auth-core<4.0.0  # social-auth-core>=4.0.0 requires PYJWT>=2.0.0

requirements/edx-sandbox/py35.in

@@ -21,7 +21,7 @@ cryptography                        # Implementations of assorted cryptography a
 lxml                                # XML parser
 matplotlib==2.2.4                   # 2D plotting library
 networkx==2.2                       # Utilities for creating, manipulating, and studying network graphs
-nltk                                # Natural language processing; used by the chem package
+nltk==3.6.2                         # Natural language processing; used by the chem package
 numpy==1.16.5                       # Numeric array processing utilities; used by scipy
 openedx-calc<2.0.0
 pyparsing==2.2.0                    # Python Parsing module

requirements/edx-sandbox/py35.txt

@@ -51,7 +51,7 @@ networkx==2.2
     # via
     #   -c requirements/edx-sandbox/../constraints.txt
     #   -r requirements/edx-sandbox/py35.in
-nltk==3.6.5
+nltk==3.6.2
     # via
     #   -r requirements/edx-sandbox/py35.in
     #   chem