Merge pull request #359 from mindsdb/parser_fixes_1

Parser updates 03.24

mindsdb_sql/parser/ast/create.py

@@ -9,11 +9,12 @@
 
 
 class TableColumn():
-    def __init__(self, name, type='integer'):
+    def __init__(self, name, type='integer', length=None):
         self.name = name
         self.type = type
         self.is_primary_key = False
         self.default = None
+        self.length = length
 
 
 class CreateTable(ASTNode):
@@ -72,14 +73,18 @@ def get_string(self, *args, **kwargs):
         if self.columns is not None:
             columns = []
             for col in self.columns:
-                type = str(col.type)
-                if sa_types is not None:
+
+                if not isinstance(col.type, str) and sa_types is not None:
                     if issubclass(col.type, sa_types.Integer):
                         type = 'int'
                     elif issubclass(col.type, sa_types.Float):
                         type = 'float'
                     elif issubclass(col.type, sa_types.Text):
                         type = 'text'
+                else:
+                    type = str(col.type)
+                if col.length is not None:
+                    type = f'{type}({col.length})'
                 columns.append( f'{col.name} {type}')
 
             columns_str = '({})'.format(', '.join(columns))

mindsdb_sql/parser/dialects/mindsdb/__init__.py

@@ -9,7 +9,6 @@
 from .drop_dataset import DropDataset
 from .evaluate import Evaluate
 from .latest import Latest
-from .create_file import CreateFile
 from .create_ml_engine import CreateMLEngine
 from .drop_ml_engine import DropMLEngine
 from .create_job import CreateJob

mindsdb_sql/parser/dialects/mindsdb/lexer.py

@@ -14,7 +14,7 @@ class MindsDBLexer(Lexer):
     ignore_line_comment = r'--[^\n]*'
 
     tokens = {
-        USE, DROP, CREATE, DESCRIBE, RETRAIN,REPLACE,
+        USE, DROP, CREATE, DESCRIBE, RETRAIN, REPLACE,
 
         # Misc
         SET, START, TRANSACTION, COMMIT, ROLLBACK, ALTER, EXPLAIN,
@@ -308,12 +308,12 @@ def FLOAT(self, t):
     def INTEGER(self, t):
         return t
 
-    @_(r"'(?:[^\'\\]|\\.)*'")
+    @_(r"'(?:\\.|[^'])*'")
     def QUOTE_STRING(self, t):
         t.value = t.value.replace('\\"', '"').replace("\\'", "'")
         return t
 
-    @_(r'"(?:[^\"\\]|\\.)*"')
+    @_(r'"(?:\\.|[^"])*"')
     def DQUOTE_STRING(self, t):
         t.value = t.value.replace('\\"', '"').replace("\\'", "'")
         return t

mindsdb_sql/parser/dialects/mindsdb/parser.py

@@ -16,7 +16,6 @@
 from mindsdb_sql.parser.dialects.mindsdb.trigger import CreateTrigger, DropTrigger
 from mindsdb_sql.parser.dialects.mindsdb.latest import Latest
 from mindsdb_sql.parser.dialects.mindsdb.evaluate import Evaluate
-from mindsdb_sql.parser.dialects.mindsdb.create_file import CreateFile
 from mindsdb_sql.parser.dialects.mindsdb.knowledge_base import CreateKnowledgeBase, DropKnowledgeBase
 from mindsdb_sql.parser.dialects.mindsdb.skills import CreateSkill, DropSkill, UpdateSkill
 from mindsdb_sql.exceptions import ParsingException
@@ -691,27 +690,44 @@ def drop_table(self, p):
         return DropTables(tables=[p.identifier], if_exists=p.if_exists_or_empty)
 
     # create table
-    @_('CREATE TABLE identifier select',  # TODO tests failing without it
-       'CREATE TABLE if_not_exists_or_empty identifier select',
-       'CREATE TABLE if_not_exists_or_empty identifier LPAREN select RPAREN',
-       'CREATE OR REPLACE TABLE identifier select',
-       'CREATE OR REPLACE TABLE identifier LPAREN select RPAREN')
+    @_('id id',
+       'id id LPAREN INTEGER RPAREN')
+    def table_column(self, p):
+        return TableColumn(
+            name=p[0],
+            type=p[1],
+            length=getattr(p, 'INTEGER', None)
+        )
+
+    @_('table_column',
+       'table_column_list COMMA table_column')
+    def table_column_list(self, p):
+        items = getattr(p, 'table_column_list', [])
+        items.append(p.table_column)
+        return items
+
+    @_('CREATE replace_or_empty TABLE if_not_exists_or_empty identifier LPAREN table_column_list RPAREN')
     def create_table(self, p):
-        # TODO create table with columns
-        is_replace = False
-        if hasattr(p, 'REPLACE'):
-            is_replace = True
         return CreateTable(
             name=p.identifier,
-            is_replace=is_replace,
-            from_select=p.select,
+            columns=p.table_column_list,
+            is_replace=getattr(p, 'replace_or_empty', False),
             if_not_exists=getattr(p, 'if_not_exists_or_empty', False)
         )
 
-    @_('CREATE TABLE identifier USING kw_parameter_list')
+    @_(
+       'CREATE replace_or_empty TABLE if_not_exists_or_empty identifier select',
+       'CREATE replace_or_empty TABLE if_not_exists_or_empty identifier LPAREN select RPAREN',
+    )
     def create_table(self, p):
-        params = p.kw_parameter_list
-        return CreateFile(name=p.identifier, **params)
+        is_replace = getattr(p, 'replace_or_empty', False)
+
+        return CreateTable(
+            name=p.identifier,
+            is_replace=is_replace,
+            from_select=p.select,
+            if_not_exists=getattr(p, 'if_not_exists_or_empty', False)
+        )
 
     # create predictor
 
@@ -889,19 +905,14 @@ def create_integration(self, p):
         return DropMLEngine(name=p.identifier, if_exists=p.if_exists_or_empty)
 
     # CREATE INTEGRATION
-    @_('CREATE database_engine',
-       'CREATE database_engine COMMA PARAMETERS EQUALS json',
-       'CREATE database_engine COMMA PARAMETERS json',
-       'CREATE database_engine PARAMETERS EQUALS json',
-       'CREATE database_engine PARAMETERS json',
-       'CREATE OR REPLACE database_engine COMMA PARAMETERS EQUALS json',
-       'CREATE OR REPLACE database_engine COMMA PARAMETERS json',
-       'CREATE OR REPLACE database_engine PARAMETERS EQUALS json',
-       'CREATE OR REPLACE database_engine PARAMETERS json')
+    @_('CREATE replace_or_empty database_engine',
+       'CREATE replace_or_empty database_engine COMMA PARAMETERS EQUALS json',
+       'CREATE replace_or_empty database_engine COMMA PARAMETERS json',
+       'CREATE replace_or_empty database_engine PARAMETERS EQUALS json',
+       'CREATE replace_or_empty database_engine PARAMETERS json',
+       )
     def create_integration(self, p):
-        is_replace = False
-        if hasattr(p, 'REPLACE'):
-            is_replace = True
+        is_replace = getattr(p, 'replace_or_empty', False)
 
         parameters = None
         if hasattr(p, 'json'):
@@ -1295,28 +1306,28 @@ def expr(self, p):
             p.expr.parentheses = True
         return p.expr
 
-    @_('id LPAREN expr FROM expr RPAREN')
+    @_('function_name LPAREN expr FROM expr RPAREN')
     def function(self, p):
-        return Function(op=p.id, args=[p.expr0], from_arg=p.expr1)
+        return Function(op=p[0], args=[p.expr0], from_arg=p.expr1)
 
     @_('DATABASE LPAREN RPAREN')
     def function(self, p):
         return Function(op=p.DATABASE, args=[])
 
-    @_('id LPAREN DISTINCT expr_list RPAREN')
+    @_('function_name LPAREN DISTINCT expr_list RPAREN')
     def function(self, p):
-        return Function(op=p.id, distinct=True, args=p.expr_list)
+        return Function(op=p[0], distinct=True, args=p.expr_list)
 
-    @_('id LPAREN expr_list_or_nothing RPAREN',
-       'id LPAREN star RPAREN')
+    @_('function_name LPAREN expr_list_or_nothing RPAREN',
+       'function_name LPAREN star RPAREN')
     def function(self, p):
         if hasattr(p, 'star'):
             args = [p.star]
         else:
             args = p.expr_list_or_nothing
         if not args:
             args = []
-        return Function(op=p.id, args=args)
+        return Function(op=p[0], args=args)
 
     # arguments are optional in functions, so that things like `select database()` are possible
     @_('expr BETWEEN expr AND expr')
@@ -1566,6 +1577,13 @@ def string(self, p):
     def parameter(self, p):
         return Parameter(value=p.PARAMETER)
 
+    @_('id',
+       'FULL',
+       'RIGHT',
+       'LEFT')
+    def function_name(self, p):
+        return p[0]
+
     # convert to types
     @_('ID',
        'BEGIN',
@@ -1588,7 +1606,6 @@ def parameter(self, p):
        'ENGINES',
        'EXTENDED',
        'FIELDS',
-       # 'FULL', # fixme: is parsed as alias
        'GLOBAL',
        'HORIZON',
        'HOSTS',

mindsdb_sql/planner/utils.py

@@ -299,6 +299,9 @@ def query_traversal(node, callback, is_table=False, is_target=False, parent_quer
             array.append(node_out)
         return array
 
+    # keep original node
+    return None
+
 
 def convert_join_to_list(join):
     # join tree to table list

mindsdb_sql/render/sqlalchemy_render.py

@@ -156,9 +156,11 @@ def to_expression(self, t):
                 sa_op = getattr(arg0, method)
 
                 col = sa_op(arg1)
-            else:
+            elif t.op.lower() in functions:
                 func = functions[t.op.lower()]
                 col = func(arg0, arg1)
+            else:
+                col = arg0.op(t.op)(arg1)
 
             if t.alias:
                 alias = self.get_alias(t.alias)

tests/test_parser/test_base_sql/test_base_sql.py

@@ -1,4 +1,4 @@
-import pytest
+from textwrap import dedent
 from mindsdb_sql import parse_sql
 
 from mindsdb_sql.parser.ast import *
@@ -34,22 +34,26 @@ def test_not_equal(self):
 
     def test_escaping(self):
         expected_ast = Select(
-            targets=[Constant(value="a ' \" b")]
+            targets=[
+                Constant(value="a ' \" b"),
+                Constant(value="a ' \" b"),
+                Constant(value="a \\n b"),
+                Constant(value="a \\\n b"),
+                Constant(value="a \\\n b"),
+                Constant(value="a\nb"),
+            ]
         )
 
-        sql = """
-          select 'a \\' \\" b'
-        """
-
-        ast = parse_sql(sql)
-
-        assert str(ast).lower() == str(expected_ast).lower()
-        assert ast.to_tree() == expected_ast.to_tree()
-
-        # in double quotes
-        sql = """
-          select "a \\' \\" b"
-        """
+        sql = dedent('''
+select
+'a \\' \\" b',  -- double quote
+"a \\' \\" b",  -- single quote
+"a \\n b",
+"a \\\n b", -- double quote
+'a \\\n b', -- single quote
+"a
+b"
+        ''')
 
         ast = parse_sql(sql)
 

tests/test_parser/test_base_sql/test_create.py

@@ -3,9 +3,10 @@
 from mindsdb_sql import parse_sql
 from mindsdb_sql.parser.ast import *
 
+
 @pytest.mark.parametrize('dialect', ['mysql', 'mindsdb'])
 class TestCreate:
-    def test_create(self, dialect):
+    def test_create_from_select(self, dialect):
         expected_ast = CreateTable(
             name=Identifier('int1.model_name'),
             is_replace=True,
@@ -49,3 +50,40 @@ def test_create(self, dialect):
         assert ast.to_tree() == expected_ast.to_tree()
 
 
+class TestCreateMindsdb:
+
+    def test_create(self):
+
+        for is_replace in [True, False]:
+            for if_not_exists in [True, False]:
+
+                expected_ast = CreateTable(
+                    name=Identifier('mydb.Persons'),
+                    is_replace=is_replace,
+                    if_not_exists=if_not_exists,
+                    columns=[
+                        TableColumn(name='PersonID', type='int'),
+                        TableColumn(name='LastName', type='varchar', length=255),
+                        TableColumn(name='FirstName', type='char', length=10),
+                        TableColumn(name='Info', type='json'),
+                        TableColumn(name='City', type='varchar'),
+                    ]
+                )
+                replace_str = 'OR REPLACE' if is_replace else ''
+                exist_str = 'IF NOT EXISTS' if if_not_exists else ''
+
+                sql = f'''
+                 CREATE {replace_str} TABLE {exist_str} mydb.Persons(
+                    PersonID int,
+                    LastName varchar(255),
+                    FirstName char(10),
+                    Info json,
+                    City varchar
+                 )
+                '''
+                print(sql)
+                ast = parse_sql(sql)
+
+                assert str(ast).lower() == str(expected_ast).lower()
+                assert ast.to_tree() == expected_ast.to_tree()
+

tests/test_parser/test_base_sql/test_select_structure.py

@@ -1019,3 +1019,20 @@ def test_case(self):
 
         assert ast.to_tree() == expected_ast.to_tree()
         assert str(ast) == str(expected_ast)
+
+    def test_select_left(self):
+        sql = f'select left(a, 1) from tab1'
+        ast = parse_sql(sql)
+
+        expected_ast = Select(
+            targets=[
+                Function(op='left', args=[
+                    Identifier('a'),
+                    Constant(1)
+                ])
+            ],
+            from_table=Identifier('tab1')
+        )
+
+        assert ast.to_tree() == expected_ast.to_tree()
+        assert str(ast) == str(expected_ast)