diff --git a/xblocks_contrib/problem/capa/safe_exec/remote_exec.py b/xblocks_contrib/problem/capa/safe_exec/remote_exec.py index bbcafdef..92496fb5 100644 --- a/xblocks_contrib/problem/capa/safe_exec/remote_exec.py +++ b/xblocks_contrib/problem/capa/safe_exec/remote_exec.py @@ -27,34 +27,12 @@ # .. toggle_creation_date: 2021-08-19 ENABLE_CODEJAIL_REST_SERVICE = SettingToggle("ENABLE_CODEJAIL_REST_SERVICE", default=False, module_name=__name__) -# .. toggle_name: ENABLE_CODEJAIL_DARKLAUNCH -# .. toggle_implementation: SettingToggle -# .. toggle_default: False -# .. toggle_description: Turn on to send requests to both the codejail service and the installed codejail library for -# testing and evaluation purposes. The results from the installed codejail library will be the ones used. -# .. toggle_warning: This toggle will only behave as expected when ENABLE_CODEJAIL_REST_SERVICE is not enabled and when -# CODE_JAIL_REST_SERVICE_REMOTE_EXEC, CODE_JAIL_REST_SERVICE_HOST, CODE_JAIL_REST_SERVICE_READ_TIMEOUT, -# and CODE_JAIL_REST_SERVICE_CONNECT_TIMEOUT are configured. -# .. toggle_use_cases: temporary -# .. toggle_creation_date: 2025-04-03 -# .. toggle_target_removal_date: 2025-05-01 -ENABLE_CODEJAIL_DARKLAUNCH = SettingToggle("ENABLE_CODEJAIL_DARKLAUNCH", default=False, module_name=__name__) - def is_codejail_rest_service_enabled(): """Return whether the codejail REST service is enabled.""" return ENABLE_CODEJAIL_REST_SERVICE.is_enabled() -def is_codejail_in_darklaunch(): - """ - Returns whether codejail dark launch is enabled. - - Codejail dark launch can only be enabled if ENABLE_CODEJAIL_REST_SERVICE is not enabled. - """ - return not is_codejail_rest_service_enabled() and ENABLE_CODEJAIL_DARKLAUNCH.is_enabled() - - def get_remote_exec(*args, **kwargs): """Get remote exec function based on setting and executes it.""" remote_exec_function_name = settings.CODE_JAIL_REST_SERVICE_REMOTE_EXEC diff --git a/xblocks_contrib/problem/capa/safe_exec/safe_exec.py b/xblocks_contrib/problem/capa/safe_exec/safe_exec.py index a6f67b0c..ba24acbd 100644 --- a/xblocks_contrib/problem/capa/safe_exec/safe_exec.py +++ b/xblocks_contrib/problem/capa/safe_exec/safe_exec.py @@ -1,25 +1,13 @@ """Capa's specialized use of codejail.safe_exec.""" -import copy import hashlib -import logging -import re -from functools import lru_cache -from typing import assert_type from codejail.safe_exec import SafeExecException, json_safe -from codejail.safe_exec import not_safe_exec as codejail_not_safe_exec -from codejail.safe_exec import safe_exec as codejail_safe_exec -from django.conf import settings -from django.dispatch import receiver -from django.test.signals import setting_changed -from edx_django_utils.monitoring import function_trace, record_exception, set_custom_attribute +from django.core.exceptions import ImproperlyConfigured +from edx_django_utils.monitoring import function_trace from . import lazymod -from .remote_exec import get_remote_exec, is_codejail_in_darklaunch, is_codejail_rest_service_enabled - -log = logging.getLogger(__name__) - +from .remote_exec import get_remote_exec, is_codejail_rest_service_enabled # Establish the Python environment for Capa. # Capa assumes float-friendly division always. @@ -172,120 +160,20 @@ def safe_exec( # Create the complete code we'll run. code_prolog = CODE_PROLOG % random_seed - if is_codejail_rest_service_enabled(): - data = { - "code": code_prolog + LAZY_IMPORTS + code, - "globals_dict": globals_dict, - "python_path": python_path, - "limit_overrides_context": limit_overrides_context, - "slug": slug, - "unsafely": unsafely, - "extra_files": extra_files, - } + if not is_codejail_rest_service_enabled(): + raise ImproperlyConfigured("To make use of this feature configure a remote Codejail service.") - with function_trace("safe_exec.remote_exec"): - emsg, exception = get_remote_exec(data) - - else: + data = { + "code": code_prolog + LAZY_IMPORTS + code, + "globals_dict": globals_dict, + "python_path": python_path, + "limit_overrides_context": limit_overrides_context, + "slug": slug, + "unsafely": unsafely, + "extra_files": extra_files, + } - # Create a copy so the originals are not modified as part of this call. - # This has to happen before local exec is run, since globals are modified - # as a side effect. - darklaunch_globals = copy.deepcopy(globals_dict) - - # Decide which code executor to use. - if unsafely: - exec_fn = codejail_not_safe_exec - else: - exec_fn = codejail_safe_exec - - # Run the code! Results are side effects in globals_dict. - try: - trace_name = "safe_exec.local_exec_darklaunch" if is_codejail_in_darklaunch() else "safe_exec.local_exec" - with function_trace(trace_name): - exec_fn( - code_prolog + LAZY_IMPORTS + code, - globals_dict, - python_path=python_path, - extra_files=extra_files, - limit_overrides_context=limit_overrides_context, - slug=slug, - ) - except BaseException as e: - # Saving SafeExecException e in exception to be used later. - exception = e - emsg = str(e) - if not isinstance(exception, SafeExecException): - # Something unexpected happened, so don't cache this evaluation. - # (We may decide to cache these in the future as well; this is just - # preserving existing behavior during a refactor of error handling.) - cacheable = False - else: - exception = None - emsg = None - - # Run the code in both the remote codejail service as well as the local codejail - # when in darklaunch mode. - if is_codejail_in_darklaunch(): - # Start adding attributes only once we're in a darklaunch - # comparison, even though these particular ones aren't specific to - # darklaunch. There can be multiple codejail calls per trace, and - # these attrs will overwrite previous values in the same trace. When - # that happens, we need to ensure we overwrite *all* of them, - # otherwise we could end up with inconsistent combinations of values. - - # .. custom_attribute_name: codejail.slug - # .. custom_attribute_description: Value of the slug parameter. This - # might be a problem ID, if present. - set_custom_attribute("codejail.slug", slug) - # .. custom_attribute_name: codejail.limit_overrides_context - # .. custom_attribute_description: Value of the limit_overrides_context - # parameter to this code execution. Generally this will be the - # course name, if present at all. - set_custom_attribute("codejail.limit_overrides_context", limit_overrides_context) - # .. custom_attribute_name: codejail.extra_files_count - # .. custom_attribute_description: Number of extra_files included - # in request. This should be 0 or 1, the latter indicating a - # python_lib.zip was present. - set_custom_attribute("codejail.extra_files_count", len(extra_files) if extra_files else 0) - - try: - data = { - "code": code_prolog + LAZY_IMPORTS + code, - "globals_dict": darklaunch_globals, - "python_path": python_path, - "limit_overrides_context": limit_overrides_context, - "slug": slug, - "unsafely": unsafely, - "extra_files": extra_files, - } - with function_trace("safe_exec.remote_exec_darklaunch"): - # Ignore the returned exception, because it's just a - # SafeExecException wrapped around emsg (if present). - remote_emsg, _ = get_remote_exec(data) - remote_exception = None - except BaseException as e: - # Swallow all exceptions and log it in monitoring so that dark launch doesn't cause issues during - # deploy. - remote_emsg = None - remote_exception = e - - try: - local_exc_unexpected = None if isinstance(exception, SafeExecException) else exception - - report_darklaunch_results( - limit_overrides_context=limit_overrides_context, - slug=slug, - globals_local=globals_dict, - emsg_local=emsg, - unexpected_exc_local=local_exc_unexpected, - globals_remote=darklaunch_globals, - emsg_remote=remote_emsg, - unexpected_exc_remote=remote_exception, - ) - except BaseException: - log.exception("Error occurred while trying to report codejail darklaunch data.") - record_exception() + emsg, exception = get_remote_exec(data) # Put the result back in the cache. This is complicated by the fact that # the globals dict might not be entirely serializable. @@ -296,212 +184,3 @@ def safe_exec( # If an exception happened, raise it now. if exception: raise exception - - -def _compile_normalizers(normalizer_setting): - """ - Compile emsg normalizer search/replace pairs into regex. - - Raises exception on bad settings. - """ - compiled = [] - for pair in normalizer_setting: - search = re.compile(assert_type(pair["search"], str)) - replace = assert_type(pair["replace"], str) - - # Test the replacement string (might contain errors) - re.sub(search, replace, "example") - - compiled.append({"search": search, "replace": replace}) - return compiled - - -@lru_cache(maxsize=1) -def emsg_normalizers(): - """ - Load emsg normalization settings. - - The output is like the setting value, except the 'search' patterns have - been compiled. - """ - default_setting = [ - { - # Character range should be at least as broad as what Python's `tempfile` uses. - "search": r"/tmp/codejail-[0-9a-zA-Z_]+", - "replace": r"/tmp/codejail-", - }, - # These are useful for eliding differences in environments due to Python version: - { - # Python 3.8 doesn't include the dir here, but Python 3.12 - # does. Normalize to the 3.8 version. - "search": r'File "/tmp/codejail-/jailed_code"', - "replace": r'File "jailed_code"', - }, - { - # Python version shows up in stack traces in the virtualenv paths - "search": r"python3\.[0-9]+", - "replace": r"python3.XX", - }, - { - # Line numbers in stack traces differ between Python versions - "search": r", line [0-9]+, in ", - "replace": r", line XXX, in ", - }, - { - # Some time after 3.8, Python started adding '^^^' indicators to stack traces - "search": r"\\n\s*\^+\s*\\n", - "replace": r"\\n", - }, - { - # Python3.8 had these stack trace elements but 3.12 does not - "search": r'\\n File "[^"]+", line [0-9]+, in \\n', - "replace": r"\\n", - }, - ] - default_normalizers = _compile_normalizers(default_setting) - - # .. setting_name: CODEJAIL_DARKLAUNCH_EMSG_NORMALIZERS - # .. setting_default: [] - # .. setting_description: A list of patterns to search and replace in codejail error - # messages during comparison in codejail-service darklaunch. Each entry is a dict - # of 'search' (a regular expression string) and 'replace' (the replacement string). - # Deployers may also need to add a search/replace pair for the location of the sandbox - # virtualenv, or any other paths that show up in stack traces. - # .. setting_warning: Note that `replace' is a pattern, allowing for - # backreferences. Any backslashes in the replacement pattern that are not - # intended as backreferences should be escaped as `\\`. - # The default list suppresses differences due to the randomly-named sandboxes - # or to differences due to Python version. See setting - # ``CODEJAIL_DARKLAUNCH_EMSG_NORMALIZERS_COMBINE`` for information on how - # this setting interacts with the defaults. - custom_setting = getattr(settings, "CODEJAIL_DARKLAUNCH_EMSG_NORMALIZERS", []) - try: - custom_normalizers = _compile_normalizers(custom_setting) - except BaseException: - log.error("Could not load custom codejail darklaunch emsg normalizers") - record_exception() - return default_normalizers - - # .. setting_name: CODEJAIL_DARKLAUNCH_EMSG_NORMALIZERS_COMBINE - # .. setting_default: 'append' - # .. setting_description: How to combine ``CODEJAIL_DARKLAUNCH_EMSG_NORMALIZERS`` - # with the defaults. If the value is 'replace', the defaults will be replaced - # with the specified patterns. If the value is 'append' (the default), the - # specified replacements will be run after the defaults. - combine = getattr(settings, "CODEJAIL_DARKLAUNCH_EMSG_NORMALIZERS_COMBINE", "append") - if combine == "replace": - return custom_normalizers - - # 'append', or unknown - return default_normalizers + custom_normalizers - - -def normalize_error_message(emsg): - """ - Remove any uninteresting sources of discrepancy from an emsg. - """ - if emsg is None: - return None - - for replacer in emsg_normalizers(): - emsg = re.sub(replacer["search"], replacer["replace"], emsg, count=0) - - return emsg - - -def report_darklaunch_results( # pylint: disable=too-many-arguments - *, - limit_overrides_context, - slug, - globals_local, - emsg_local, - unexpected_exc_local, - globals_remote, - emsg_remote, - unexpected_exc_remote, -): - """Send telemetry for results of darklaunch.""" - can_compare_output = True - - def report_arm(arm, emsg, unexpected_exception): - """ - Set custom attributes for each arm of the darklaunch experiment. - - `arm` should be 'local' or 'remote'. - """ - nonlocal can_compare_output - if unexpected_exception: - # .. custom_attribute_name: codejail.darklaunch.status.{local,remote} - # .. custom_attribute_description: Outcome of this arm of the - # darklaunch comparison. Values can be 'ok' (normal execution), - # 'safe_error' (submitted code raised an exception), or - # 'unexpected_error' (uncaught error in submitting or evaluating code). - set_custom_attribute(f"codejail.darklaunch.status.{arm}", "unexpected_error") - # .. custom_attribute_name: codejail.darklaunch.exception.{local,remote} - # .. custom_attribute_description: When the status attribute indicates - # an unexpected error, this is a string representation of the error, - # otherwise None. - set_custom_attribute(f"codejail.darklaunch.exception.{arm}", repr(unexpected_exception)) - can_compare_output = False - else: - set_custom_attribute(f"codejail.darklaunch.status.{arm}", "ok" if emsg is None else "safe_error") - set_custom_attribute(f"codejail.darklaunch.exception.{arm}", None) - - report_arm("local", emsg_local, unexpected_exc_local) - report_arm("remote", emsg_remote, unexpected_exc_remote) - - # If the arms can't be compared (unexpected errors), stop early -- the rest - # is about output comparison. - if not can_compare_output: - set_custom_attribute("codejail.darklaunch.globals_match", "N/A") - set_custom_attribute("codejail.darklaunch.emsg_match", "N/A") - log.info( - "Codejail darklaunch had unexpected exception for course=%r, slug=%r:\n" - "Local exception: %r\nRemote exception: %r", - limit_overrides_context, - slug, - unexpected_exc_local, - unexpected_exc_remote, - ) - return None - - globals_match = globals_local == globals_remote - emsg_match = normalize_error_message(emsg_local) == normalize_error_message(emsg_remote) - - if not globals_match or not emsg_match: - log.info( - "Codejail darklaunch had mismatch for course=%r, slug=%r:\n" - "emsg_match=%r, globals_match=%r\n" - "Local: globals=%r, emsg=%r\n" - "Remote: globals=%r, emsg=%r", - limit_overrides_context, - slug, - emsg_match, - globals_match, - globals_local, - emsg_local, - globals_remote, - emsg_remote, - ) - - # .. custom_attribute_name: codejail.darklaunch.globals_match - # .. custom_attribute_description: True if local and remote globals_dict - # values match, False otherwise. 'N/A' when either arm raised an - # uncaught error. - set_custom_attribute("codejail.darklaunch.globals_match", globals_match) - # .. custom_attribute_name: codejail.darklaunch.emsg_match - # .. custom_attribute_description: True if the local and remote emsg values - # (errors returned from sandbox) match, False otherwise. Differences due - # to known irrelevant factors are suppressed in this comparison, such as - # the randomized directory names used for sandboxes. 'N/A' when either - # arm raised an uncaught error. - set_custom_attribute("codejail.darklaunch.emsg_match", emsg_match) - return None - - -@receiver(setting_changed) -def reset_caches(sender, **kwargs): # pylint: disable=unused-argument - """ - Reset cached settings during unit tests. - """ - emsg_normalizers.cache_clear() diff --git a/xblocks_contrib/problem/capa/safe_exec/tests/test_safe_exec.py b/xblocks_contrib/problem/capa/safe_exec/tests/test_safe_exec.py index 560c6abc..c937ffc4 100644 --- a/xblocks_contrib/problem/capa/safe_exec/tests/test_safe_exec.py +++ b/xblocks_contrib/problem/capa/safe_exec/tests/test_safe_exec.py @@ -1,12 +1,10 @@ """Test safe_exec.py""" -import copy import hashlib -import os -import os.path +import io import textwrap import unittest -from unittest.mock import call, patch +import zipfile import pytest import random2 as random @@ -20,15 +18,32 @@ from six.moves import range from xblocks_contrib.problem.capa.safe_exec import safe_exec, update_hash -from xblocks_contrib.problem.capa.safe_exec.remote_exec import ( - is_codejail_in_darklaunch, - is_codejail_rest_service_enabled, -) -from xblocks_contrib.problem.capa.safe_exec.safe_exec import emsg_normalizers, normalize_error_message +from xblocks_contrib.problem.capa.safe_exec.remote_exec import is_codejail_rest_service_enabled from xblocks_contrib.problem.capa.tests.test_util import UseUnsafeCodejail -@UseUnsafeCodejail() +def make_python_library_zip_bytes(): + """ + Make a simple course library ZIP file, returning the bytes. + + Contains one module, `constant.py` with a single constant `THE_CONST`. + """ + memfile = io.BytesIO() + with zipfile.ZipFile(memfile, "w") as z: + z.writestr( + "constant.py", + textwrap.dedent(""" + THE_CONST = 23 + """), + ) + + memfile.seek(0) + return memfile.read() + + +PYTHON_LIB_BYTES = make_python_library_zip_bytes() + + class TestSafeExec(unittest.TestCase): """Unit tests for verifying functionality and restrictions of safe_exec.""" @@ -78,9 +93,13 @@ def test_random_is_still_importable(self): def test_python_lib(self): """Test importing Python library from custom path in safe_exec.""" - pylib = os.path.dirname(__file__) + "/test_files/pylib" g = {} - safe_exec("import constant; a = constant.THE_CONST", g, python_path=[pylib]) + safe_exec( + "import constant; a = constant.THE_CONST", + g, + python_path=["python_lib.zip"], + extra_files={"python_lib.zip": PYTHON_LIB_BYTES}, + ) def test_raising_exceptions(self): """Ensure exceptions are raised correctly in safe_exec.""" @@ -116,10 +135,8 @@ def test_cant_do_something_forbidden(self): """ # If in-platform codejail isn't configured... if not jail_code.is_configured("python"): - # ...AND if remote codejail isn't configured... if not is_codejail_rest_service_enabled(): - # ...then skip this test. pytest.skip(reason="Local or remote codejail has to be configured and enabled to run this test.") @@ -129,365 +146,6 @@ def test_cant_do_something_forbidden(self): assert "SystemExit" not in str(cm) assert "Couldn't execute jailed code" in str(cm) - def test_can_do_something_forbidden_if_run_unsafely(self): - """ - Demonstrates that running unsafe code outside the code jail - can cause issues directly in the calling process. - """ - g = {} - with pytest.raises(SystemExit) as cm: - safe_exec("import sys; sys.exit(1)", g, unsafely=True) - assert "SystemExit" in str(cm) - - -class TestCodeJailDarkLaunch(unittest.TestCase): - """ - Test that the behavior of the dark launched code behaves as expected. - """ - - @patch("xblocks_contrib.problem.capa.safe_exec.safe_exec.get_remote_exec") - @patch("xblocks_contrib.problem.capa.safe_exec.safe_exec.codejail_safe_exec") - def test_default_code_execution(self, mock_local_exec, mock_remote_exec): - """Check that default code execution uses local exec only.""" - # Test default only runs local exec. - g = {} - safe_exec("a=1", g) - assert mock_local_exec.called - assert not mock_remote_exec.called - - @override_settings(ENABLE_CODEJAIL_REST_SERVICE=True) - @patch("xblocks_contrib.problem.capa.safe_exec.safe_exec.get_remote_exec") - @patch("xblocks_contrib.problem.capa.safe_exec.safe_exec.codejail_safe_exec") - def test_code_execution_only_codejail_service(self, mock_local_exec, mock_remote_exec): - """Check execution via codejail service only.""" - # Set return values to empty values to indicate no error. - mock_remote_exec.return_value = (None, None) - # Test with only the service enabled. - g = {} - safe_exec("a=1", g) - assert not mock_local_exec.called - assert mock_remote_exec.called - - @override_settings(ENABLE_CODEJAIL_DARKLAUNCH=True) - @patch("xblocks_contrib.problem.capa.safe_exec.safe_exec.get_remote_exec") - @patch("xblocks_contrib.problem.capa.safe_exec.safe_exec.codejail_safe_exec") - def test_code_execution_darklaunch_misconfig(self, mock_local_exec, mock_remote_exec): - """Test that darklaunch doesn't run when remote service is generally enabled.""" - mock_remote_exec.return_value = (None, None) - - with override_settings(ENABLE_CODEJAIL_REST_SERVICE=True): - safe_exec("a=1", {}) - - assert not mock_local_exec.called - assert mock_remote_exec.called - - @override_settings(ENABLE_CODEJAIL_DARKLAUNCH=True) - def run_dark_launch( # pylint: disable=too-many-positional-arguments,too-many-arguments - self, - globals_dict, - local, - remote, - expect_attr_calls, - expect_log_info_calls, - expect_globals_contains, - ): - """ - Run a darklaunch scenario with mocked out local and remote execution. - - Asserts set_custom_attribute and log.info calls and (partial) contents - of globals dict. - - Return value is a dictionary of: - - - 'raised': Exception that safe_exec raised, or None. - """ - - assert is_codejail_in_darklaunch() - - with ( - patch("xblocks_contrib.problem.capa.safe_exec.safe_exec.codejail_safe_exec") as mock_local_exec, - patch("xblocks_contrib.problem.capa.safe_exec.safe_exec.get_remote_exec") as mock_remote_exec, - patch("xblocks_contrib.problem.capa.safe_exec.safe_exec.set_custom_attribute") as mock_set_custom_attribute, - patch("xblocks_contrib.problem.capa.safe_exec.safe_exec.log.info") as mock_log_info, - ): - mock_local_exec.side_effect = local - mock_remote_exec.side_effect = remote - - try: - safe_exec( - "", - globals_dict, - limit_overrides_context="course-v1:org+course+run", - slug="hw1", - ) - except BaseException as e: - safe_exec_e = e - else: - safe_exec_e = None - - # Always want both sides to be called - assert mock_local_exec.called - assert mock_remote_exec.called - - mock_set_custom_attribute.assert_has_calls(expect_attr_calls, any_order=True) - mock_log_info.assert_has_calls(expect_log_info_calls, any_order=True) - - for k, v in expect_globals_contains.items(): - assert globals_dict[k] == v - - return {"raised": safe_exec_e} - - # These don't change between the tests - standard_codejail_attr_calls = [ - call("codejail.slug", "hw1"), - call("codejail.limit_overrides_context", "course-v1:org+course+run"), - call("codejail.extra_files_count", 0), - ] - - def test_separate_globals(self): - """Test that local and remote globals are isolated from each other's side effects.""" - # Both will attempt to read and write the 'overwrite' key. - globals_dict = {"overwrite": "original"} - - local_globals = None - remote_globals = None - - def local_exec(code, globals_dict, **kwargs): # pylint: disable=unused-argument - # Preserve what local exec saw - nonlocal local_globals - local_globals = copy.deepcopy(globals_dict) - - globals_dict["overwrite"] = "mock local" - - def remote_exec(data): - # Preserve what remote exec saw - nonlocal remote_globals - remote_globals = copy.deepcopy(data["globals_dict"]) - - data["globals_dict"]["overwrite"] = "mock remote" - return (None, None) - - results = self.run_dark_launch( - globals_dict=globals_dict, - local=local_exec, - remote=remote_exec, - expect_attr_calls=[ - *self.standard_codejail_attr_calls, - call("codejail.darklaunch.status.local", "ok"), - call("codejail.darklaunch.status.remote", "ok"), - call("codejail.darklaunch.exception.local", None), - call("codejail.darklaunch.exception.remote", None), - call("codejail.darklaunch.globals_match", False), # mismatch revealed here - call("codejail.darklaunch.emsg_match", True), - ], - expect_log_info_calls=[ - call( - "Codejail darklaunch had mismatch for course=%r, slug=%r:\n" - "emsg_match=%r, globals_match=%r\n" - "Local: globals=%r, emsg=%r\n" - "Remote: globals=%r, emsg=%r", - "course-v1:org+course+run", - "hw1", - True, - False, - {"overwrite": "mock local"}, - None, - {"overwrite": "mock remote"}, - None, - ), - ], - # Should only see behavior of local exec - expect_globals_contains={"overwrite": "mock local"}, - ) - assert results["raised"] is None - - # Both arms should have only seen the original globals object, untouched - # by the other arm. - assert local_globals == {"overwrite": "original"} - assert remote_globals == {"overwrite": "original"} - - def test_remote_runs_even_if_local_raises(self): - """Test that remote exec runs even if local raises.""" - expected_error = BaseException("unexpected") - - def local_exec(code, globals_dict, **kwargs): - # Raise something other than a SafeExecException. - raise expected_error - - def remote_exec(data): # pylint: disable=unused-argument - return (None, None) - - results = self.run_dark_launch( - globals_dict={}, - local=local_exec, - remote=remote_exec, - expect_attr_calls=[ - *self.standard_codejail_attr_calls, - call("codejail.darklaunch.status.local", "unexpected_error"), - call("codejail.darklaunch.status.remote", "ok"), - call("codejail.darklaunch.exception.local", "BaseException('unexpected')"), - call("codejail.darklaunch.exception.remote", None), - call("codejail.darklaunch.globals_match", "N/A"), - call("codejail.darklaunch.emsg_match", "N/A"), - ], - expect_log_info_calls=[ - call( - "Codejail darklaunch had unexpected exception for course=%r, slug=%r:\n" - "Local exception: %r\nRemote exception: %r", - "course-v1:org+course+run", - "hw1", - expected_error, - None, - ), - ], - expect_globals_contains={}, - ) - - # Unexpected errors from local safe_exec propagate up. - assert isinstance(results["raised"], BaseException) - assert "unexpected" in repr(results["raised"]) - - def test_emsg_mismatch(self): - """Test that local and remote error messages are compared.""" - - def local_exec(code, globals_dict, **kwargs): - raise SafeExecException("oops") - - def remote_exec(data): # pylint: disable=unused-argument - return ("OH NO", SafeExecException("OH NO")) - - results = self.run_dark_launch( - globals_dict={}, - local=local_exec, - remote=remote_exec, - expect_attr_calls=[ - *self.standard_codejail_attr_calls, - call("codejail.darklaunch.status.local", "safe_error"), - call("codejail.darklaunch.status.remote", "safe_error"), - call("codejail.darklaunch.exception.local", None), - call("codejail.darklaunch.exception.remote", None), - call("codejail.darklaunch.globals_match", True), - call("codejail.darklaunch.emsg_match", False), # mismatch revealed here - ], - expect_log_info_calls=[ - call( - "Codejail darklaunch had mismatch for course=%r, slug=%r:\n" - "emsg_match=%r, globals_match=%r\n" - "Local: globals=%r, emsg=%r\n" - "Remote: globals=%r, emsg=%r", - "course-v1:org+course+run", - "hw1", - False, - True, - {}, - "oops", - {}, - "OH NO", - ), - ], - expect_globals_contains={}, - ) - assert isinstance(results["raised"], SafeExecException) - assert "oops" in repr(results["raised"]) - - def test_ignore_sandbox_dir_mismatch(self): - """Mismatch due only to differences in sandbox directory should be ignored.""" - - def local_exec(code, globals_dict, **kwargs): - raise SafeExecException("stack trace involving /tmp/codejail-1234567/whatever.py") - - def remote_exec(data): # pylint: disable=unused-argument - emsg = "stack trace involving /tmp/codejail-abcd_EFG/whatever.py" - return (emsg, SafeExecException(emsg)) - - results = self.run_dark_launch( - globals_dict={}, - local=local_exec, - remote=remote_exec, - expect_attr_calls=[ - *self.standard_codejail_attr_calls, - call("codejail.darklaunch.status.local", "safe_error"), - call("codejail.darklaunch.status.remote", "safe_error"), - call("codejail.darklaunch.exception.local", None), - call("codejail.darklaunch.exception.remote", None), - call("codejail.darklaunch.globals_match", True), - call("codejail.darklaunch.emsg_match", True), # even though not exact match - ], - expect_log_info_calls=[], - expect_globals_contains={}, - ) - assert isinstance(results["raised"], SafeExecException) - assert "whatever.py" in repr(results["raised"]) - - def test_default_normalizers(self): - """ - Default normalizers handle false mismatches we've observed. - - This just provides coverage for some of the more complicated patterns. - """ - side_1 = ( - "Couldn't execute jailed code: stdout: b'', stderr: b'Traceback" - ' (most recent call last):\\n File "/tmp/codejail-9g9715g_/jailed_code"' - ', line 19, in \\n exec(code, g_dict)\\n File ""' - ', line 1, in \\n File "", line 89, in test_add\\n' - ' File "", line 1\\n import random random.choice(range(10))' - "\\n ^\\nSyntaxError: invalid syntax\\n' with status code: 1" - ) - side_2 = ( - "Couldn't execute jailed code: stdout: b'', stderr: b'Traceback" - ' (most recent call last):\\n File "jailed_code"' - ', line 19, in \\n exec(code, g_dict)\\n File ""' - ', line 203, in \\n File "", line 89, in test_add\\n' - ' File "", line 1\\n import random random.choice(range(10))' - "\\n ^^^^^^\\nSyntaxError: invalid syntax\\n' with status code: 1" - ) - assert normalize_error_message(side_1) == normalize_error_message(side_2) - - @override_settings( - CODEJAIL_DARKLAUNCH_EMSG_NORMALIZERS=[ - { - "search": r"[0-9]+", - "replace": r"", - }, - ] - ) - def test_configurable_normalizers(self): - """We can augment the normalizers, and they run in order.""" - emsg_in = "Error in /tmp/codejail-1234abcd/whatever.py: something 12 34 other" - expect_out = "Error in /tmp/codejail-/whatever.py: something other" - assert expect_out == normalize_error_message(emsg_in) - - @override_settings( - CODEJAIL_DARKLAUNCH_EMSG_NORMALIZERS=[ - { - "search": r"[0-9]+", - "replace": r"", - }, - ], - CODEJAIL_DARKLAUNCH_EMSG_NORMALIZERS_COMBINE="replace", - ) - def test_can_replace_normalizers(self): - """We can replace the normalizers.""" - emsg_in = "Error in /tmp/codejail-1234abcd/whatever.py: something 12 34 other" - expect_out = "Error in /tmp/codejail-abcd/whatever.py: something other" - assert expect_out == normalize_error_message(emsg_in) - - @override_settings( - CODEJAIL_DARKLAUNCH_EMSG_NORMALIZERS=[ - { - "search": r"broken", - "replace": r"replace \g<>", # invalid replacement pattern - }, - ] - ) - @patch("xblocks_contrib.problem.capa.safe_exec.safe_exec.record_exception") - @patch("xblocks_contrib.problem.capa.safe_exec.safe_exec.log.error") - def test_normalizers_validate(self, mock_log_error, mock_record_exception): - """Normalizers are validated, and fall back to default list on error.""" - assert len(emsg_normalizers()) > 0 - mock_log_error.assert_called_once_with("Could not load custom codejail darklaunch emsg normalizers") - mock_record_exception.assert_called_once() - class TestLimitConfiguration(unittest.TestCase): """