Merge pull request #339 from mindsdb/parser_fixes_1

Parser fixes 2024.01

.github/workflows/test.yml

@@ -32,6 +32,8 @@ jobs:
   coverage:
     needs: test
     runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python 3.8

mindsdb_sql/parser/ast/__init__.py

@@ -14,6 +14,6 @@
 from .delete import *
 from .drop import *
 from .create import *
+from .variable import *
 
-from mindsdb_sql.parser.dialects.mysql.variable import Variable
 from mindsdb_sql.parser.dialects.mindsdb.latest import Latest

mindsdb_sql/parser/ast/select/__init__.py

@@ -1,7 +1,7 @@
 from .select import Select
 from .common_table_expression import CommonTableExpression
 from .union import Union
-from .constant import Constant, NullConstant, SpecialConstant, Last
+from .constant import Constant, NullConstant, Last
 from .star import Star
 from .identifier import Identifier
 from .join import Join

mindsdb_sql/parser/ast/select/constant.py

@@ -4,16 +4,17 @@
 
 
 class Constant(ASTNode):
-    def __init__(self, value, *args, **kwargs):
+    def __init__(self, value, with_quotes=True, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.value = value
+        self.with_quotes = with_quotes
 
     def to_tree(self, *args, level=0, **kwargs):
         alias_str = f', alias={self.alias.to_tree()}' if self.alias else ''
         return indent(level) + f'Constant(value={repr(self.value)}{alias_str})'
 
     def get_string(self, *args, **kwargs):
-        if isinstance(self.value, str):
+        if isinstance(self.value, str) and self.with_quotes:
             out_str = f"\'{self.value}\'"
         elif isinstance(self.value, bool):
             out_str = 'TRUE' if self.value else 'FALSE'
@@ -29,26 +30,12 @@ def __init__(self, *args, **kwargs):
         super().__init__(value=None, *args, **kwargs)
 
     def to_tree(self, *args, level=0, **kwargs):
-        return '\t'*level +  'NullConstant()'
+        return '\t'*level + 'NullConstant()'
 
     def get_string(self, *args, **kwargs):
         return 'NULL'
 
 
-# TODO replace it to just Constant?
-# DEFAULT
-class SpecialConstant(ASTNode):
-    def __init__(self, name, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.name = name
-
-    def to_tree(self, *args, level=0, **kwargs):
-        return indent(level) + f'SpecialConstant(name={self.name})'
-
-    def get_string(self, *args, **kwargs):
-        return self.name
-
-
 class Last(Constant):
     def __init__(self, *args, **kwargs):
         self.value = 'last'

mindsdb_sql/parser/ast/set.py

@@ -6,95 +6,120 @@
 class Set(ASTNode):
     def __init__(self,
                  category=None,
-                 arg=None,
+                 name=None,
+                 value=None,
+                 scope=None,
                  params=None,
+                 set_list=None,
                  *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.category = category
-        self.arg = arg
-        self.params = params or {}
 
-    def to_tree(self, *args, level=0, **kwargs):
-        ind = indent(level)
-        ind1 = indent(level+1)
-        category_str = f'category={self.category}, '
-        arg_str = f'arg={self.arg.to_tree()},' if self.arg else ''
-        if self.params:
-            param_str = 'param=' + ', '.join([f'{k}:{v}' for k,v in self.params.items()])
-        else:
-            param_str = ''
-        out_str = f'{ind}Set(' \
-                  f'{category_str}' \
-                  f'{arg_str} ' \
-                  f'{param_str}' \
-                  f')'
-        return out_str
+        # names / charset / transactions
+        self.category = category
 
-    def get_string(self, *args, **kwargs):
-        if self.params:
-            param_str = ' ' + ' '.join([f'{k} {v}' for k, v in self.params.items()])
-        else:
-            param_str = ''
-
-        if isinstance(self.arg, Tuple):
-            arg_str = ', '.join([str(i) for i in self.arg.items])
-        else:
-            arg_str = f' {str(self.arg)}' if self.arg else ''
-        return f'SET {self.category if self.category else ""}{arg_str}{param_str}'
+        # name for variable assigment. category is None it this case
+        self.name = name
 
+        self.value = value
+        self.params = params or {}
 
-class SetTransaction(ASTNode):
-    def __init__(self,
-                 isolation_level=None,
-                 access_mode=None,
-                 scope=None,
-                 *args, **kwargs):
-        super().__init__(*args, **kwargs)
+        # global / session / ...
+        self.scope = scope
 
-        if isolation_level is not None:
-            isolation_level = isolation_level.upper()
-        if access_mode is not None:
-            access_mode = access_mode.upper()
-        if scope is not None:
-            scope = scope.upper()
+        # contents all set subcommands
+        self.set_list = set_list
 
-        self.scope = scope
-        self.access_mode = access_mode
-        self.isolation_level = isolation_level
 
     def to_tree(self, *args, level=0, **kwargs):
-        ind = indent(level)
-        if self.scope is None:
-            scope_str = ''
+        if self.set_list is not None:
+            items = [set.render() for set in self.set_list]
         else:
-            scope_str = f'scope={self.scope}, '
+            items = self.render()
 
-        properties = []
-        if self.isolation_level is not None:
-            properties.append('ISOLATION LEVEL ' + self.isolation_level)
-        if self.access_mode is not None:
-            properties.append(self.access_mode)
-        prop_str = ', '.join(properties)
+        ind = indent(level)
 
-        out_str = f'{ind}SetTransaction(' \
-                  f'{scope_str}' \
-                  f'properties=[{prop_str}]' \
-                  f'\n{ind})'
-        return out_str
+        return f'{ind}Set(items={items})'
 
     def get_string(self, *args, **kwargs):
-        properties = []
-        if self.isolation_level is not None:
-            properties.append('ISOLATION LEVEL ' + self.isolation_level)
-        if self.access_mode is not None:
-            properties.append(self.access_mode)
+        return 'SET ' + self.render()
 
-        prop_str = ', '.join(properties)
+    def render(self):
+        if self.set_list is not None:
+            render_list = [set.render() for set in self.set_list]
+            return ', '.join(render_list)
 
-        if self.scope is None:
-            scope_str = ''
+        if self.params:
+            param_str = ' ' + ' '.join([f'{k} {v}' for k, v in self.params.items()])
         else:
-            scope_str = self.scope + ' '
+            param_str = ''
 
-        return f'SET {scope_str}TRANSACTION {prop_str}'
+        if self.name is not None:
+            # category should be empty
+            content = f'{self.name.to_string()}={self.value.to_string()}'
+        elif self.value is not None:
+            content = f'{self.category} {self.value.to_string()}'
+        else:
+            content = f'{self.category}'
+
+        scope = ''
+        if self.scope is not None:
+            scope = f'{self.scope} '
+
+        return f'{scope}{content}{param_str}'
+
+
+# class SetTransaction(ASTNode):
+#     def __init__(self,
+#                  isolation_level=None,
+#                  access_mode=None,
+#                  scope=None,
+#                  *args, **kwargs):
+#         super().__init__(*args, **kwargs)
+#
+#         if isolation_level is not None:
+#             isolation_level = isolation_level.upper()
+#         if access_mode is not None:
+#             access_mode = access_mode.upper()
+#         if scope is not None:
+#             scope = scope.upper()
+#
+#         self.scope = scope
+#         self.access_mode = access_mode
+#         self.isolation_level = isolation_level
+#
+#     def to_tree(self, *args, level=0, **kwargs):
+#         ind = indent(level)
+#         if self.scope is None:
+#             scope_str = ''
+#         else:
+#             scope_str = f'scope={self.scope}, '
+#
+#         properties = []
+#         if self.isolation_level is not None:
+#             properties.append('ISOLATION LEVEL ' + self.isolation_level)
+#         if self.access_mode is not None:
+#             properties.append(self.access_mode)
+#         prop_str = ', '.join(properties)
+#
+#         out_str = f'{ind}SetTransaction(' \
+#                   f'{scope_str}' \
+#                   f'properties=[{prop_str}]' \
+#                   f'\n{ind})'
+#         return out_str
+#
+#     def get_string(self, *args, **kwargs):
+#         properties = []
+#         if self.isolation_level is not None:
+#             properties.append('ISOLATION LEVEL ' + self.isolation_level)
+#         if self.access_mode is not None:
+#             properties.append(self.access_mode)
+#
+#         prop_str = ', '.join(properties)
+#
+#         if self.scope is None:
+#             scope_str = ''
+#         else:
+#             scope_str = self.scope + ' '
+#
+#         return f'SET {scope_str}TRANSACTION {prop_str}'
 

mindsdb_sql/parser/dialects/mindsdb/lexer.py

@@ -42,8 +42,10 @@ class MindsDBLexer(Lexer):
         VARIABLES, SESSION, STATUS,
         GLOBAL, PROCEDURE, FUNCTION, INDEX, WARNINGS,
         ENGINES, CHARSET, COLLATION, PLUGINS, CHARACTER,
-        PERSIST, PERSIST_ONLY, DEFAULT,
+        PERSIST, PERSIST_ONLY,
         IF_EXISTS, IF_NOT_EXISTS, COLUMNS, FIELDS, COLLATE, SEARCH_PATH,
+        VARIABLE, SYSTEM_VARIABLE,
+
         # SELECT Keywords
         WITH, SELECT, DISTINCT, FROM, WHERE, AS,
         LIMIT, OFFSET, ASC, DESC, NULLS_FIRST, NULLS_LAST,
@@ -170,7 +172,6 @@ class MindsDBLexer(Lexer):
     PLUGINS = r'\bPLUGINS\b'
     PERSIST = r'\bPERSIST\b'
     PERSIST_ONLY = r'\bPERSIST_ONLY\b'
-    DEFAULT = r'\bDEFAULT\b'
     IF_EXISTS = r'\bIF[\s]+EXISTS\b'
     IF_NOT_EXISTS = r'\bIF[\s]+NOT[\s]+EXISTS\b'
     COLUMNS = r'\bCOLUMNS\b'
@@ -295,22 +296,57 @@ class MindsDBLexer(Lexer):
     def ID(self, t):
         return t
 
-    @_(r'\d+\.\d*')
+    @_(r'\d+\.\d+')
     def FLOAT(self, t):
         return t
 
     @_(r'\d+')
     def INTEGER(self, t):
         return t
 
-    @_(r"'[^']*'")
+    @_(r"'(?:[^\'\\]|\\.)*'")
     def QUOTE_STRING(self, t):
+        t.value = t.value.replace('\\"', '"').replace("\\'", "'")
         return t
 
-    @_(r'"[^"]*"')
+    @_(r'"(?:[^\"\\]|\\.)*"')
     def DQUOTE_STRING(self, t):
+        t.value = t.value.replace('\\"', '"').replace("\\'", "'")
         return t
 
     @_(r'\n+')
     def ignore_newline(self, t):
         self.lineno += len(t.value)
+
+    @_(r'@[a-zA-Z_.$]+',
+       r"@'[a-zA-Z_.$][^']*'",
+       r"@`[a-zA-Z_.$][^`]*`",
+       r'@"[a-zA-Z_.$][^"]*"'
+       )
+    def VARIABLE(self, t):
+        t.value = t.value.lstrip('@')
+
+        if t.value[0] == '"':
+            t.value = t.value.strip('\"')
+        elif t.value[0] == "'":
+            t.value = t.value.strip('\'')
+        elif t.value[0] == "`":
+            t.value = t.value.strip('`')
+        return t
+
+    @_(r'@@[a-zA-Z_.$]+',
+       r"@@'[a-zA-Z_.$][^']*'",
+       r"@@`[a-zA-Z_.$][^`]*`",
+       r'@@"[a-zA-Z_.$][^"]*"'
+       )
+    def SYSTEM_VARIABLE(self, t):
+        t.value = t.value.lstrip('@')
+
+        if t.value[0] == '"':
+            t.value = t.value.strip('\"')
+        elif t.value[0] == "'":
+            t.value = t.value.strip('\'')
+        elif t.value[0] == "`":
+            t.value = t.value.strip('`')
+        return t
+