Merge pull request #638 from python-cmd2/common_split

Added a shlex.split() wrapper to have a common way of calling it.

Author: kmvanbrunt

cmd2/cmd2.py

@@ -35,7 +35,6 @@
 import inspect
 import os
 import re
-import shlex
 import sys
 import threading
 from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, Type, Union, IO
@@ -48,8 +47,8 @@
 from . import utils
 from .argparse_completer import AutoCompleter, ACArgumentParser, ACTION_ARG_CHOICES
 from .clipboard import can_clip, get_paste_buffer, write_to_paste_buffer
-from .parsing import StatementParser, Statement, Macro, MacroArg
 from .history import History, HistoryItem
+from .parsing import StatementParser, Statement, Macro, MacroArg, shlex_split, get_command_arg_list
 
 # Set up readline
 from .rl_utils import rl_type, RlType, rl_get_point, rl_set_prompt, vt100_support, rl_make_safe_prompt
@@ -150,24 +149,6 @@ def categorize(func: Union[Callable, Iterable], category: str) -> None:
         setattr(func, HELP_CATEGORY, category)
 
 
-def parse_quoted_string(string: str, preserve_quotes: bool) -> List[str]:
-    """
-    Parse a quoted string into a list of arguments
-    :param string: the string being parsed
-    :param preserve_quotes: if True, then quotes will not be stripped
-    """
-    if isinstance(string, list):
-        # arguments are already a list, return the list we were passed
-        lexed_arglist = string
-    else:
-        # Use shlex to split the command line into a list of arguments based on shell rules
-        lexed_arglist = shlex.split(string, comments=False, posix=False)
-
-        if not preserve_quotes:
-            lexed_arglist = [utils.strip_quotes(arg) for arg in lexed_arglist]
-    return lexed_arglist
-
-
 def with_category(category: str) -> Callable:
     """A decorator to apply a category to a command function."""
     def cat_decorator(func):
@@ -178,8 +159,7 @@ def cat_decorator(func):
 
 def with_argument_list(*args: List[Callable], preserve_quotes: bool = False) -> Callable[[List], Optional[bool]]:
     """A decorator to alter the arguments passed to a do_* cmd2 method. Default passes a string of whatever the user
-    typed. With this decorator, the decorated method will receive a list of arguments parsed from user input using
-    shlex.split().
+    typed. With this decorator, the decorated method will receive a list of arguments parsed from user input.
 
     :param args: Single-element positional argument list containing do_* method this decorator is wrapping
     :param preserve_quotes: if True, then argument quotes will not be stripped
@@ -189,9 +169,9 @@ def with_argument_list(*args: List[Callable], preserve_quotes: bool = False) ->
 
     def arg_decorator(func: Callable):
         @functools.wraps(func)
-        def cmd_wrapper(self, cmdline):
-            lexed_arglist = parse_quoted_string(cmdline, preserve_quotes)
-            return func(self, lexed_arglist)
+        def cmd_wrapper(cmd2_instance, statement: Union[Statement, str]):
+            parsed_arglist = get_command_arg_list(statement, preserve_quotes)
+            return func(cmd2_instance, parsed_arglist)
 
         cmd_wrapper.__doc__ = func.__doc__
         return cmd_wrapper
@@ -214,16 +194,17 @@ def with_argparser_and_unknown_args(argparser: argparse.ArgumentParser, preserve
     import functools
 
     # noinspection PyProtectedMember
-    def arg_decorator(func: Callable[[Statement], Optional[bool]]):
+    def arg_decorator(func: Callable):
         @functools.wraps(func)
-        def cmd_wrapper(instance, cmdline):
-            lexed_arglist = parse_quoted_string(cmdline, preserve_quotes)
+        def cmd_wrapper(cmd2_instance, statement: Union[Statement, str]):
+            parsed_arglist = get_command_arg_list(statement, preserve_quotes)
+
             try:
-                args, unknown = argparser.parse_known_args(lexed_arglist)
+                args, unknown = argparser.parse_known_args(parsed_arglist)
             except SystemExit:
                 return
             else:
-                return func(instance, args, unknown)
+                return func(cmd2_instance, args, unknown)
 
         # argparser defaults the program name to sys.argv[0]
         # we want it to be the name of our command
@@ -256,16 +237,18 @@ def with_argparser(argparser: argparse.ArgumentParser,
     import functools
 
     # noinspection PyProtectedMember
-    def arg_decorator(func: Callable[[Statement], Optional[bool]]):
+    def arg_decorator(func: Callable):
         @functools.wraps(func)
-        def cmd_wrapper(instance, cmdline):
-            lexed_arglist = parse_quoted_string(cmdline, preserve_quotes)
+        def cmd_wrapper(cmd2_instance, statement: Union[Statement, str]):
+
+            parsed_arglist = get_command_arg_list(statement, preserve_quotes)
+
             try:
-                args = argparser.parse_args(lexed_arglist)
+                args = argparser.parse_args(parsed_arglist)
             except SystemExit:
                 return
             else:
-                return func(instance, args)
+                return func(cmd2_instance, args)
 
         # argparser defaults the program name to sys.argv[0]
         # we want it to be the name of our command
@@ -742,8 +725,7 @@ def tokens_for_completion(self, line: str, begidx: int, endidx: int) -> Tuple[Li
         # Parse the line into tokens
         while True:
             try:
-                # Use non-POSIX parsing to keep the quotes around the tokens
-                initial_tokens = shlex.split(tmp_line[:tmp_endidx], comments=False, posix=False)
+                initial_tokens = shlex_split(tmp_line[:tmp_endidx])
 
                 # If the cursor is at an empty token outside of a quoted string,
                 # then that is the token being completed. Add it to the list.
@@ -1735,7 +1717,7 @@ def _run_cmdfinalization_hooks(self, stop: bool, statement: Optional[Statement])
             # Fix those annoying problems that occur with terminal programs like "less" when you pipe to them
             if self.stdin.isatty():
                 import subprocess
-                proc = subprocess.Popen(shlex.split('stty sane'))
+                proc = subprocess.Popen(['stty', 'sane'])
                 proc.communicate()
 
         try:

cmd2/parsing.py

@@ -5,14 +5,24 @@
 import os
 import re
 import shlex
-from typing import List, Tuple, Dict
+from typing import Dict, List, Tuple, Union
 
 import attr
 
 from . import constants
 from . import utils
 
 
+def shlex_split(str_to_split: str) -> List[str]:
+    """A wrapper around shlex.split() that uses cmd2's preferred arguments.
+
+    This allows other classes to easily call split() the same way StatementParser does
+    :param str_to_split: the string being split
+    :return: A list of tokens
+    """
+    return shlex.split(str_to_split, comments=False, posix=False)
+
+
 @attr.s(frozen=True)
 class MacroArg:
     """
@@ -226,6 +236,34 @@ def argv(self) -> List[str]:
         return rtn
 
 
+def get_command_arg_list(to_parse: Union[Statement, str], preserve_quotes: bool) -> List[str]:
+    """
+    Called by the argument_list and argparse wrappers to retrieve just the arguments being
+    passed to their do_* methods as a list.
+
+    :param to_parse: what is being passed to the do_* method. It can be one of two types:
+                     1. An already parsed Statement
+                     2. An argument string in cases where a do_* method is explicitly called
+                        e.g.: Calling do_help('alias create') would cause to_parse to be 'alias create'
+
+    :param preserve_quotes: if True, then quotes will not be stripped from the arguments
+    :return: the arguments in a list
+    """
+    if isinstance(to_parse, Statement):
+        # In the case of a Statement, we already have what we need
+        if preserve_quotes:
+            return to_parse.arg_list
+        else:
+            return to_parse.argv[1:]
+    else:
+        # We have the arguments in a string. Use shlex to split it.
+        parsed_arglist = shlex_split(to_parse)
+        if not preserve_quotes:
+            parsed_arglist = [utils.strip_quotes(arg) for arg in parsed_arglist]
+
+        return parsed_arglist
+
+
 class StatementParser:
     """Parse raw text into command components.
 
@@ -349,7 +387,7 @@ def tokenize(self, line: str) -> List[str]:
             return []
 
         # split on whitespace
-        tokens = shlex.split(line, comments=False, posix=False)
+        tokens = shlex_split(line)
 
         # custom lexing
         tokens = self._split_on_punctuation(tokens)

tests/test_argparse.py

@@ -72,14 +72,6 @@ def do_arglist(self, arglist):
     def do_preservelist(self, arglist):
         self.stdout.write('{}'.format(arglist))
 
-    @cmd2.with_argument_list
-    @cmd2.with_argument_list
-    def do_arglisttwice(self, arglist):
-        if isinstance(arglist, list):
-            self.stdout.write(' '.join(arglist))
-        else:
-            self.stdout.write('False')
-
     known_parser = argparse.ArgumentParser()
     known_parser.add_argument('-p', '--piglatin', action='store_true', help='atinLay')
     known_parser.add_argument('-s', '--shout', action='store_true', help='N00B EMULATION MODE')
@@ -178,10 +170,6 @@ def test_preservelist(argparse_app):
     out = run_cmd(argparse_app, 'preservelist foo "bar baz"')
     assert out[0] == "['foo', '\"bar baz\"']"
 
-def test_arglist_decorator_twice(argparse_app):
-    out = run_cmd(argparse_app, 'arglisttwice "we  should" get these')
-    assert out[0] == 'we  should get these'
-
 
 class SubcommandApp(cmd2.Cmd):
     """ Example cmd2 application where we a base command which has a couple sub-commands."""