From 8a4a56c3d72f894e827b7f829f09289aecca20c6 Mon Sep 17 00:00:00 2001 From: Kmad <3433032+kmad@users.noreply.github.com> Date: Wed, 31 May 2023 01:30:31 +0000 Subject: [PATCH 1/3] Added optional guardrails against destructive system interactions. --- gptfile.py | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/gptfile.py b/gptfile.py index 9b36370..7e910d3 100644 --- a/gptfile.py +++ b/gptfile.py @@ -15,6 +15,7 @@ import json import openai import pkg_resources +import argparse # Configure @@ -185,6 +186,98 @@ def print_user(text): def print_status(text): print(f"\033[1m\033[96m{text}\033[0m\n") +def print_warn(text): + print(f"\033[1m\033[93m{text}\033[0m\n") + +## Reliability guard code from OpenAI Human Eval +# https://github.com/openai/human-eval/blob/master/human_eval/execution.py#L48-L58 +# Add this by default to limit destrucive behavior. +# This feature can be disabled by invoking with --unsafe +def reliability_guard(maximum_memory_bytes: int = None): + import platform + """ + This disables various destructive functions and prevents the generated code + from interfering with the test (e.g. fork bomb, killing other processes, + removing filesystem files, etc.) + + WARNING + This function is NOT a security sandbox. Untrusted code, including, model- + generated code, should not be blindly executed outside of one. See the + Codex paper for more information about OpenAI's code sandbox, and proceed + with caution. + """ + + if maximum_memory_bytes is not None: + import resource + resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes)) + resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes)) + if not platform.uname().system == 'Darwin': + resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes)) + + + import builtins + builtins.exit = None + builtins.quit = None + + import os + os.environ['OMP_NUM_THREADS'] = '1' + + os.kill = None + os.system = None + os.putenv = None + os.remove = None + os.removedirs = None + os.rmdir = None + os.fchdir = None + os.setuid = None + os.fork = None + os.forkpty = None + os.killpg = None + os.rename = None + os.renames = None + os.truncate = None + os.replace = None + os.unlink = None + os.fchmod = None + os.fchown = None + os.chmod = None + os.chown = None + os.chroot = None + os.fchdir = None + os.lchflags = None + os.lchmod = None + os.lchown = None + + # Removing these restrictions for basic functionality + #os.getcwd = None + #os.chdir = None + + import shutil + shutil.rmtree = None + shutil.move = None + shutil.chown = None + + import subprocess + subprocess.Popen = None # type: ignore + + import sys + sys.modules['ipdb'] = None + sys.modules['joblib'] = None + sys.modules['resource'] = None + sys.modules['psutil'] = None + sys.modules['tkinter'] = None if __name__ == "__main__": + + parser = argparse.ArgumentParser(description='Interact with files using an LLM interface.') + + parser.add_argument('--unsafe', dest='RELIABILITY_GUARD_DISABLED', action='store_true') + + args = parser.parse_args() + + if args.RELIABILITY_GUARD_DISABLED: + print_warn("WARNING: Reliability guard disabled. Use at your own risk.") + else: + reliability_guard() + process_files(os.getcwd()) From 13ebd395687d1eaa8a57626082ca31cad47d6789 Mon Sep 17 00:00:00 2001 From: Kmad <3433032+kmad@users.noreply.github.com> Date: Wed, 31 May 2023 01:32:40 +0000 Subject: [PATCH 2/3] Added ability to write out generated code which fails to execute, for debugging purposes. --- gptfile.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/gptfile.py b/gptfile.py index 7e910d3..93909bc 100644 --- a/gptfile.py +++ b/gptfile.py @@ -173,6 +173,13 @@ def execute_python_code(code): exec(code, globals()) except Exception as e: print("Error executing code: ", e) + + if args.WRITE_ERROR_CODE: + import datetime + now = str(int(datetime.datetime.now().timestamp())) + with open(f"failed_execution_{now}.py", "w") as f: + f.write(code) + def print_system(text): @@ -271,7 +278,10 @@ def reliability_guard(maximum_memory_bytes: int = None): parser = argparse.ArgumentParser(description='Interact with files using an LLM interface.') - parser.add_argument('--unsafe', dest='RELIABILITY_GUARD_DISABLED', action='store_true') + parser.add_argument('--unsafe', dest='RELIABILITY_GUARD_DISABLED', action='store_true', help='Disable reliability guard, such as os.* functions. Use at your own risk.') + + # Specify file to save failed code to. + parser.add_argument('--save-failed', dest='SAVE_FAILED_TO', action='store', help='Filepath to save failed code to.') args = parser.parse_args() From 56a339c490267e0570b82deb124d3cdd7b2d11df Mon Sep 17 00:00:00 2001 From: Kmad <3433032+kmad@users.noreply.github.com> Date: Thu, 1 Jun 2023 02:02:00 +0000 Subject: [PATCH 3/3] Fixed issue with guard; moved to execution phase. --- gptfile.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/gptfile.py b/gptfile.py index 93909bc..e5abd38 100644 --- a/gptfile.py +++ b/gptfile.py @@ -169,15 +169,15 @@ def get_packages_list(): def execute_python_code(code): + if not args.RELIABILITY_GUARD_DISABLED: + reliability_guard() try: exec(code, globals()) except Exception as e: print("Error executing code: ", e) - if args.WRITE_ERROR_CODE: - import datetime - now = str(int(datetime.datetime.now().timestamp())) - with open(f"failed_execution_{now}.py", "w") as f: + if args.SAVE_FAILED_TO: + with open(f"{args.SAVE_FAILED_TO}", "w") as f: f.write(code) @@ -197,7 +197,7 @@ def print_warn(text): print(f"\033[1m\033[93m{text}\033[0m\n") ## Reliability guard code from OpenAI Human Eval -# https://github.com/openai/human-eval/blob/master/human_eval/execution.py#L48-L58 +# https://github.com/openai/human-eval/blob/master/human_eval/execution.py # Add this by default to limit destrucive behavior. # This feature can be disabled by invoking with --unsafe def reliability_guard(maximum_memory_bytes: int = None): @@ -287,7 +287,5 @@ def reliability_guard(maximum_memory_bytes: int = None): if args.RELIABILITY_GUARD_DISABLED: print_warn("WARNING: Reliability guard disabled. Use at your own risk.") - else: - reliability_guard() - + process_files(os.getcwd())