feat: Add observability to codejail darklaunch.

Sets up tracing in code jail calls to track remote
and local execution, and adds custom attributes
to track error messages from the calls.

https://github.com/edx/edx-arch-experiments/issues/895

xmodule/capa/safe_exec/safe_exec.py

@@ -6,7 +6,7 @@ import logging
 from codejail.safe_exec import SafeExecException, json_safe
 from codejail.safe_exec import not_safe_exec as codejail_not_safe_exec
 from codejail.safe_exec import safe_exec as codejail_safe_exec
-from edx_django_utils.monitoring import function_trace, record_exception
+from edx_django_utils.monitoring import function_trace, record_exception, set_custom_attribute
 
 from . import lazymod
 from .remote_exec import is_codejail_rest_service_enabled, is_codejail_in_darklaunch, get_remote_exec
@@ -94,7 +94,7 @@ def safe_exec(
     limit_overrides_context=None,
     slug=None,
     unsafely=False,
-):
+):  # pylint: disable=too-many-statements
     """
     Execute python code safely.
 
@@ -157,7 +157,8 @@ def safe_exec(
             "extra_files": extra_files,
         }
 
-        emsg, exception = get_remote_exec(data)
+        with function_trace('safe_exec.remote_exec'):
+            emsg, exception = get_remote_exec(data)
 
     else:
 
@@ -169,14 +170,16 @@ def safe_exec(
 
         # Run the code!  Results are side effects in globals_dict.
         try:
-            exec_fn(
-                code_prolog + LAZY_IMPORTS + code,
-                globals_dict,
-                python_path=python_path,
-                extra_files=extra_files,
-                limit_overrides_context=limit_overrides_context,
-                slug=slug,
-            )
+            trace_name = 'safe_exec.local_exec_darklaunch' if is_codejail_in_darklaunch() else 'safe_exec.local_exec'
+            with function_trace(trace_name):
+                exec_fn(
+                    code_prolog + LAZY_IMPORTS + code,
+                    globals_dict,
+                    python_path=python_path,
+                    extra_files=extra_files,
+                    limit_overrides_context=limit_overrides_context,
+                    slug=slug,
+                )
         except SafeExecException as e:
             # Saving SafeExecException e in exception to be used later.
             exception = e
@@ -199,11 +202,15 @@ def safe_exec(
                     "unsafely": unsafely,
                     "extra_files": extra_files,
                 }
-                remote_emsg, _remote_exception = get_remote_exec(data)
+                with function_trace('safe_exec.remote_exec_darklaunch'):
+                    remote_emsg, _remote_exception = get_remote_exec(data)
                 log.info(
                     f"Remote execution in darklaunch mode produces: {darklaunch_globals} or exception: {remote_emsg}"
                 )
                 log.info(f"Local execution in darklaunch mode produces: {globals_dict} or exception: {emsg}")
+                set_custom_attribute('dark_launch_emsg_match', remote_emsg == emsg)
+                set_custom_attribute('remote_emsg_exists', remote_emsg is not None)
+                set_custom_attribute('local_emsg_exists', emsg is not None)
             except Exception as e:  # pragma: no cover  # pylint: disable=broad-except
                 # Swallows all exceptions and logs it in monitoring so that dark launch doesn't cause issues during
                 # deploy.