Refactor argument formatting and add quote removal to enable quoted

commands to be parallelizable

Signed-off-by: kwakubiney <[email protected]>

Author: kwakubiney

compiler/annotations_utils/util_parsing.py

@@ -66,9 +66,6 @@ def parse_arg_list_to_command_invocation(
     command, flags_options_operands
 ) -> CommandInvocationInitial:
     cmd_name = format_arg_chars(command)
-    #strip quotes from command where necessary
-    #quoted commands are interpreted as non-parallelizable
-    cmd_name = cmd_name.strip().strip('"').strip("'")
     json_data = get_json_data(cmd_name)
     set_of_all_flags: Set[str] = get_set_of_all_flags(json_data)
     dict_flag_to_primary_repr: dict[str, str] = get_dict_flag_to_primary_repr(json_data)

compiler/ast_to_ir.py

@@ -100,19 +100,23 @@ def compile_asts(ast_objects: "list[AstNode]", fileIdGen, config):
             if isinstance(compiled_ast, IR):
                 acc_ir.background_union(compiled_ast)
             else:
-                ## shouldn't happen since compile_node should have already
-                ## raised this error
-                raise UnparallelizableError(f"Node: {compiled_ast} is not pure")
+                ## TODO: Make this union the compiled_ast with the
+                ## accumulated IR, since the user wanted to run these
+                ## commands in parallel (Is that correct?)
+                # acc_ir.background_union(IR([compiled_ast]))
+                compiled_asts.append(acc_ir)
+                acc_it = None
+                compiled_asts.append(compiled_ast)
 
             ## If the current compiled ast not in background (and so
             ## the union isn't in background too), stop accumulating
-            if not acc_ir.is_in_background():
+            if not acc_ir is None and not acc_ir.is_in_background():
                 compiled_asts.append(acc_ir)
                 acc_ir = None
         else:
             ## If the compiled ast is in background, start
             ## accumulating it
-            if compiled_ast.is_in_background():
+            if isinstance(compiled_ast, IR) and compiled_ast.is_in_background():
                 acc_ir = compiled_ast
             else:
                 compiled_asts.append(compiled_ast)
@@ -124,7 +128,7 @@ def compile_asts(ast_objects: "list[AstNode]", fileIdGen, config):
     return compiled_asts
 
 
-def compile_node(ast_object, fileIdGen, config) -> IR:
+def compile_node(ast_object, fileIdGen, config):
     global compile_cases
     return ast_match(ast_object, compile_cases, fileIdGen, config)
 

compiler/orchestrator_runtime/pash_prepare_call_compiler.sh

@@ -2,13 +2,6 @@
 
 ## OUTPUT: When it completes it sets "$pash_script_to_execute"
 
-## Let daemon know that this region is done
-function inform_daemon_exit () {
-    ## Send to daemon
-    msg="Exit:${process_id}"
-    daemon_response=$(pash_communicate_daemon_just_send "$msg")
-}
-
 ## Only needed for expansion
 export pash_input_args=( "$@" )
 
@@ -40,8 +33,7 @@ pash_redir_output echo "$$: (2) Before asking the daemon for compilation..."
 msg="Compile:${pash_compiled_script_file}| Variable File:${pash_runtime_shell_variables_file}| Input IR File:${pash_input_ir_file}"
 daemon_response=$(pash_communicate_daemon "$msg") # Blocking step, daemon will not send response until it's safe to continue
 
-# WARNING: finicky!
-if [[ "$daemon_response" == *"not"*"parallelizable"* ]]; then
+if [[ "$daemon_response" == *"not all regions are parallelizable"* ]]; then
     pash_all_region_parallelizable=1 
 else 
     pash_all_region_parallelizable=0 
@@ -74,20 +66,17 @@ pash_redir_output echo "$$: (2) Compiler exited with code: $pash_runtime_return_
 ## only when --assert_all_regions_parallellizable is used do we care about all regions being parallelizable
 if [ "$pash_all_region_parallelizable" -ne 0 ] && [ "$pash_assert_all_regions_parallelizable_flag" -eq 1 ]; then
     pash_redir_output echo "$$: ERROR: (2) Compiler failed with error code because some regions were not parallelizable: $pash_all_region_parallelizable while assert_all_regions_parallelizable_flag was enabled! Exiting PaSh..."
-    inform_daemon_exit
     exit 1
 fi
 
 if [ "$pash_runtime_return_code" -ne 0 ] && [ "$pash_assert_all_regions_parallelizable_flag" -eq 1 ]; then
     pash_redir_output echo "$$: ERROR: (2) Compiler failed with error code: $pash_runtime_return_code while assert_all_regions_parallelizable_flag was enabled! Exiting PaSh..."
-    inform_daemon_exit
     exit 1
 fi
 
-## for pash_assert_compiler_success_flag, exit when return code is not 0 (general exception caught) and when all regions are parallelizable
+## for pash_assert_compiler_success_flag, exit when return code is 0 (general exception caught) and not when all regions are parallelizable
 if [ "$pash_runtime_return_code" -ne 0 ] && [ "$pash_all_region_parallelizable" -eq 0 ] && [ "$pash_assert_compiler_success_flag" -eq 1 ]; then
     pash_redir_output echo "$$: ERROR: (2) Compiler failed with error code: $pash_runtime_return_code while assert_compiler_success was enabled! Exiting PaSh..."
-    inform_daemon_exit
     exit 1
 fi
 
@@ -104,3 +93,11 @@ if [ "$pash_runtime_return_code" -ne 0 ] || [ "$pash_dry_run_compiler_flag" -eq
 else
     export pash_script_to_execute="${pash_compiled_script_file}"
 fi
+
+## Let daemon know that this region is done
+function inform_daemon_exit () {
+    ## Send to daemon
+    msg="Exit:${process_id}"
+    daemon_response=$(pash_communicate_daemon_just_send "$msg")
+} 
+

compiler/pash_compilation_server.py

@@ -379,7 +379,6 @@ def remove_process(self, process_id):
                 ]
             )
 
-        assert self.running_procs > 0
         self.running_procs -= 1
         if self.running_procs == 0:
             self.unsafe_running = False

compiler/pash_runtime.sh

@@ -141,9 +141,7 @@ else
         pash_redir_output echo "$$: (5) BaSh script exited with ec: $pash_runtime_final_status"
     else 
         function run_parallel() {
-            # ideally we'd call inform_daemon_exit with the EXIT handler,
-            # but it isn't called on Ubuntu 20.04 (for no clear reason)
-            trap inform_daemon_exit SIGTERM SIGINT
+            trap inform_daemon_exit SIGTERM SIGINT EXIT
             export SCRIPT_TO_EXECUTE="$pash_script_to_execute"
             source "$RUNTIME_DIR/pash_restore_state_and_execute.sh"
             inform_daemon_exit

compiler/shell_ast/ast_util.py

@@ -44,15 +44,23 @@ def __init__(self, text):
 def check_if_ast_is_supported(construct, arguments, **kwargs):
     return
 
-
-def format_args(args):
-    formatted_args = [format_arg_chars(arg_chars) for arg_chars in args]
-    return formatted_args
-
-
 def format_arg_chars(arg_chars):
-    chars = [format_arg_char(arg_char) for arg_char in arg_chars]
-    return "".join(chars)
+    return traverse_and_format(arg_chars)
+
+def traverse_and_format(arg_char):
+    if isinstance(arg_char, list):
+        return "".join(traverse_and_format(c) for c in arg_char)
+    elif isinstance(arg_char, QArgChar):
+        return remove_quotes_from_quoted_content(arg_char.format())
+    elif isinstance(arg_char, (CArgChar, EArgChar)):
+        return format_arg_char(arg_char)
+    else:
+        # Fallback: not sure what the fallback for non recognized characters should be?
+        return format_arg_char(arg_char)
+
+def remove_quotes_from_quoted_content(arg_char):
+    formatted_content = traverse_and_format(arg_char)
+    return formatted_content.strip('"').strip("'")
 
 
 def format_arg_char(arg_char: ArgChar) -> str: