Merge pull request #354 from mindsdb/parser_fixes_1

Parser fixes 02.2024

mindsdb_sql/parser/ast/select/constant.py

@@ -15,7 +15,8 @@ def to_tree(self, *args, level=0, **kwargs):
 
     def get_string(self, *args, **kwargs):
         if isinstance(self.value, str) and self.with_quotes:
-            out_str = f"\'{self.value}\'"
+            val = self.value.replace("'", "\\'")
+            out_str = f"\'{val}\'"
         elif isinstance(self.value, bool):
             out_str = 'TRUE' if self.value else 'FALSE'
         elif isinstance(self.value, (dt.date, dt.datetime, dt.timedelta)):

mindsdb_sql/parser/ast/select/operation.py

@@ -44,9 +44,9 @@ def get_string(self, *args, **kwargs):
         arg_strs = []
         for arg in self.args:
             arg_str = arg.to_string()
-            if isinstance(arg, BinaryOperation) or isinstance(arg, BetweenOperation):
-                # to parens
-                arg_str = f'({arg_str})'
+            # if isinstance(arg, BinaryOperation) or isinstance(arg, BetweenOperation):
+            #     # to parens
+            #     arg_str = f'({arg_str})'
             arg_strs.append(arg_str)
 
         return f'{arg_strs[0]} {self.op.upper()} {arg_strs[1]}'

mindsdb_sql/parser/ast/set.py

@@ -49,7 +49,13 @@ def render(self):
             return ', '.join(render_list)
 
         if self.params:
-            param_str = ' ' + ' '.join([f'{k} {v}' for k, v in self.params.items()])
+            params = []
+            for k, v in self.params.items():
+                if k.lower() == 'access_mode':
+                    params.append(v)
+                else:
+                    params.append(f'{k} {v}')
+            param_str = ' ' + ', '.join(params)
         else:
             param_str = ''
 

mindsdb_sql/parser/ast/show.py

@@ -69,11 +69,11 @@ def get_string(self, *args, **kwargs):
             in_str = ' ' + ' '.join(ar)
 
         modes_str = f' {" ".join(self.modes)}' if self.modes else ''
-        like_str = f' LIKE {self.like}' if self.like else ''
+        like_str = f" LIKE '{self.like}'" if self.like else ""
         where_str = f' WHERE {str(self.where)}' if self.where else ''
 
         # custom commands
-        if self.category in ('FUNCTION CODE', 'PROCEDURE CODE', 'ENGINE'):
+        if self.category in ('FUNCTION CODE', 'PROCEDURE CODE', 'ENGINE') or self.category.startswith('ENGINE '):
             return f'SHOW {self.category} {self.name}'
         elif self.category == 'REPLICA STATUS':
             channel = ''

mindsdb_sql/parser/dialects/mindsdb/chatbot.py

@@ -39,7 +39,8 @@ def get_string(self, *args, **kwargs):
         params = self.params.copy()
         params['model'] = self.model.to_string() if self.model else 'NULL'
         params['database'] = self.database.to_string()
-        params['agent'] = self.agent.to_string() if self.agent else 'NULL'
+        if self.agent:
+            params['agent'] = self.agent.to_string()
 
         using_ar = [f'{k}={repr(v)}' for k, v in params.items()]
 

mindsdb_sql/parser/dialects/mindsdb/create_database.py

@@ -43,8 +43,12 @@ def get_string(self, *args, **kwargs):
         if self.is_replace:
             replace_str = f' OR REPLACE'
 
+        engine_str = ''
+        if self.engine:
+            engine_str = f'ENGINE = {repr(self.engine)} '
+
         parameters_str = ''
         if self.parameters:
             parameters_str = f', PARAMETERS = {json.dumps(self.parameters)}'
-        out_str = f'CREATE{replace_str} DATABASE {"IF NOT EXISTS " if self.if_not_exists else ""}{self.name.to_string()} WITH ENGINE = {repr(self.engine)}{parameters_str}'
+        out_str = f'CREATE{replace_str} DATABASE {"IF NOT EXISTS " if self.if_not_exists else ""}{self.name.to_string()} {engine_str}{parameters_str}'
         return out_str

mindsdb_sql/parser/dialects/mindsdb/create_job.py

@@ -79,7 +79,7 @@ def get_string(self, *args, **kwargs):
 
         if_query_str = ''
         if self.if_query_str is not None:
-            if_query_str = f" IF '{self.if_query_str}'"
+            if_query_str = f" IF ({self.if_query_str})"
 
         out_str = f'CREATE JOB {"IF NOT EXISTS" if self.if_not_exists else ""} {self.name.to_string()} ({self.query_str}){start_str}{end_str}{repeat_str}{if_query_str}'
         return out_str

mindsdb_sql/parser/dialects/mindsdb/lexer.py

@@ -72,6 +72,7 @@ class MindsDBLexer(Lexer):
         EQUALS, NEQUALS, GREATER, GEQ, LESS, LEQ,
         AND, OR, NOT, IS, IS_NOT,
         IN, LIKE, NOT_LIKE, CONCAT, BETWEEN, WINDOW, OVER, PARTITION_BY,
+        JSON_GET, JSON_GET_STR,
 
         # Data types
         CAST, ID, INTEGER, FLOAT, QUOTE_STRING, DQUOTE_STRING, NULL, TRUE, FALSE,
@@ -263,6 +264,8 @@ class MindsDBLexer(Lexer):
     SEMICOLON = r'\;'
 
     # Operators
+    JSON_GET = r'->'
+    JSON_GET_STR = r'->>'
     PLUS = r'\+'
     MINUS = r'-'
     DIVIDE = r'/'
@@ -288,7 +291,6 @@ class MindsDBLexer(Lexer):
     OVER = r'\bOVER\b'
     PARTITION_BY = r'\bPARTITION BY\b'
 
-
     # Data types
     NULL = r'\bNULL\b'
     TRUE = r'\bTRUE\b'

mindsdb_sql/parser/dialects/mindsdb/parser.py

@@ -44,10 +44,12 @@ class MindsDBParser(Parser):
         ('left', AND),
         ('right', UNOT),
         ('left', EQUALS, NEQUALS),
+        ('nonassoc', LESS, LEQ, GREATER, GEQ, IN, BETWEEN, IS, IS_NOT, NOT_LIKE, LIKE),
+        ('left', JSON_GET),
         ('left', PLUS, MINUS),
         ('left', STAR, DIVIDE),
         ('right', UMINUS),  # Unary minus operator, unary not
-        ('nonassoc', LESS, LEQ, GREATER, GEQ, IN, BETWEEN, IS, IS_NOT, NOT_LIKE, LIKE),
+
     )
 
     # Top-level statements
@@ -421,7 +423,7 @@ def set_item(self, p):
 
         params = {}
         if isolation_level is not None:
-            params['isolation_level'] = isolation_level
+            params['isolation level'] = isolation_level
         if access_mode is not None:
             params['access_mode'] = access_mode
 
@@ -535,21 +537,18 @@ def show_category(self, p):
         return f"{p.id0} {p.id1}"
 
     # custom show commands
-    @_('SHOW ENGINE identifier STATUS',
-       'SHOW ENGINE identifier MUTEX')
-    def show(self, p):
-        return Show(
-            category=p[1],
-            name=p.identifier.to_string(),
-            modes=[p[3]]
-        )
 
     @_('SHOW id id identifier')
     def show(self, p):
         category = p[1] + ' ' + p[2]
+
+        if p[1].lower() == 'engine':
+            name = p.identifier.parts[0]
+        else:
+            name = p.identifier.to_string()
         return Show(
             category=category,
-            name=p.identifier.to_string()
+            name=name
         )
 
     @_('SHOW REPLICA STATUS FOR CHANNEL id',
@@ -777,6 +776,7 @@ def create_predictor(self, p):
         'CREATE ANOMALY DETECTION MODEL identifier FROM identifier LPAREN raw_query RPAREN',
         'CREATE ANOMALY DETECTION MODEL identifier PREDICT result_columns',
         'CREATE ANOMALY DETECTION MODEL identifier PREDICT result_columns FROM identifier LPAREN raw_query RPAREN',
+        'CREATE ANOMALY DETECTION MODEL identifier FROM identifier LPAREN raw_query RPAREN PREDICT result_columns',
         # TODO add IF_NOT_EXISTS elegantly (should be low level BNF expansion)
     )
     def create_anomaly_detection_model(self, p):
@@ -1387,17 +1387,15 @@ def expr(self, p):
        'expr LIKE expr',
        'expr NOT_LIKE expr',
        'expr CONCAT expr',
+       'expr JSON_GET constant',
+       'expr JSON_GET_STR constant',
        'expr IN expr')
     def expr(self, p):
         if hasattr(p, 'LAST'):
             arg1 = Last()
         else:
-            arg1 = p.expr1
-        if len(p) > 3:
-            op = ' '.join([p[i] for i in range(1, len(p)-1)])
-        else:
-            op = p[1]
-        return BinaryOperation(op=op, args=(p[0], arg1))
+            arg1 = p[2]
+        return BinaryOperation(op=p[1], args=(p[0], arg1))
 
     @_('MINUS expr %prec UMINUS',
        'NOT expr %prec UNOT', )

mindsdb_sql/parser/dialects/mysql/parser.py

@@ -176,7 +176,7 @@ def set_item(self, p):
 
         params = {}
         if isolation_level is not None:
-            params['isolation_level'] = isolation_level
+            params['isolation level'] = isolation_level
         if access_mode is not None:
             params['access_mode'] = access_mode
 

tests/test_parser/test_base_sql/test_misc_sql_queries.py

@@ -148,10 +148,10 @@ def test_set_transaction(self, dialect):
         ast = parse_sql(sql, dialect=dialect)
         expected_ast = Set(
             category='TRANSACTION',
-            params=dict(
-                isolation_level='REPEATABLE READ',
-                access_mode='READ WRITE',
-            ),
+            params={
+                'isolation level': 'REPEATABLE READ',
+                'access_mode': 'READ WRITE',
+            },
             scope='GLOBAL'
         )
 
@@ -164,10 +164,10 @@ def test_set_transaction(self, dialect):
 
         expected_ast = Set(
             category='TRANSACTION',
-            params=dict(
-                isolation_level='SERIALIZABLE',
-                access_mode='READ ONLY',
-            ),
+            params={
+                'isolation level': 'SERIALIZABLE',
+                'access_mode': 'READ ONLY',
+            },
             scope='SESSION'
         )
 
@@ -180,9 +180,9 @@ def test_set_transaction(self, dialect):
 
         expected_ast = Set(
             category='TRANSACTION',
-            params=dict(
-                isolation_level='READ UNCOMMITTED',
-            )
+            params={
+                'isolation level': 'READ UNCOMMITTED'
+            },
         )
 
         assert ast.to_tree() == expected_ast.to_tree()

tests/test_parser/test_base_sql/test_show.py

@@ -243,18 +243,6 @@ def test_common_like_double_where_from_in_modes(self, dialect):
 
     def test_custom(self, dialect):
 
-        for arg in ['STATUS', 'MUTEX']:
-            sql = f"SHOW ENGINE engine_name {arg}"
-            ast = parse_sql(sql, dialect=dialect)
-            expected_ast = Show(
-                category='ENGINE',
-                name='engine_name',
-                modes=[arg],
-            )
-
-            assert str(ast) == str(expected_ast)
-            assert ast.to_tree() == expected_ast.to_tree()
-
         for arg in ['FUNCTION', 'PROCEDURE']:
             sql = f"SHOW {arg} CODE obj_name"
             ast = parse_sql(sql, dialect=dialect)
@@ -304,6 +292,18 @@ def test_show_database_adapted(self):
 
 class TestMindsdb:
 
+    def test_show_engine(self):
+        for arg in ['STATUS', 'MUTEX']:
+            sql = f"SHOW ENGINE engine_name {arg}"
+            ast = parse_sql(sql)
+            expected_ast = Show(
+                category='ENGINE engine_name',
+                name=arg,
+            )
+
+            assert str(ast) == str(expected_ast)
+            assert ast.to_tree() == expected_ast.to_tree()
+
     def test_show(self):
         sql = '''
            show full databases

tests/test_parser/test_mindsdb/test_create_view.py

@@ -15,7 +15,7 @@ def test_create_view_lexer(self):
         assert tokens[1].value == 'VIEW'
         assert tokens[1].type == 'VIEW'
 
-    def test_create_view_raises_wrong_dialect(self):
+    def test_create_view_raises_wrong_dialect_error(self):
         sql = "CREATE VIEW my_view FROM integr AS ( SELECT * FROM pred )"
         for dialect in ['sqlite', 'mysql']:
             with pytest.raises(ParsingException):

tests/test_parser/test_mindsdb/test_selects.py

@@ -157,5 +157,29 @@ def test_last(self):
         assert str(ast) == str(expected_ast)
 
 
+    def test_json(self):
+        sql = """SELECT col->1->'c' from TAB1"""
+
+        ast = parse_sql(sql, dialect='mindsdb')
+        expected_ast = Select(
+            targets=[BinaryOperation(
+                op='->',
+                args=[
+                    BinaryOperation(
+                        op='->',
+                        args=[
+                            Identifier('col'),
+                            Constant(1)
+                        ]
+                    ),
+                    Constant('c')
+                ]
+            )],
+            from_table=Identifier(parts=['TAB1']),
+        )
+
+        assert ast.to_tree() == expected_ast.to_tree()
+        assert str(ast) == str(expected_ast)
+