Merge pull request #356 from mindsdb/staging

Release 0.10.5

mindsdb_sql/__about__.py

@@ -1,6 +1,6 @@
 __title__ = 'mindsdb_sql'
 __package_name__ = 'mindsdb_sql'
-__version__ = '0.10.4'
+__version__ = '0.10.5'
 __description__ = "Pure python SQL parser"
 __email__ = "[email protected]"
 __author__ = 'MindsDB Inc'

mindsdb_sql/parser/ast/select/constant.py

@@ -15,7 +15,8 @@ def to_tree(self, *args, level=0, **kwargs):
 
     def get_string(self, *args, **kwargs):
         if isinstance(self.value, str) and self.with_quotes:
-            out_str = f"\'{self.value}\'"
+            val = self.value.replace("'", "\\'")
+            out_str = f"\'{val}\'"
         elif isinstance(self.value, bool):
             out_str = 'TRUE' if self.value else 'FALSE'
         elif isinstance(self.value, (dt.date, dt.datetime, dt.timedelta)):

mindsdb_sql/parser/ast/select/operation.py

@@ -44,9 +44,9 @@ def get_string(self, *args, **kwargs):
         arg_strs = []
         for arg in self.args:
             arg_str = arg.to_string()
-            if isinstance(arg, BinaryOperation) or isinstance(arg, BetweenOperation):
-                # to parens
-                arg_str = f'({arg_str})'
+            # if isinstance(arg, BinaryOperation) or isinstance(arg, BetweenOperation):
+            #     # to parens
+            #     arg_str = f'({arg_str})'
             arg_strs.append(arg_str)
 
         return f'{arg_strs[0]} {self.op.upper()} {arg_strs[1]}'

mindsdb_sql/parser/ast/set.py

@@ -49,7 +49,13 @@ def render(self):
             return ', '.join(render_list)
 
         if self.params:
-            param_str = ' ' + ' '.join([f'{k} {v}' for k, v in self.params.items()])
+            params = []
+            for k, v in self.params.items():
+                if k.lower() == 'access_mode':
+                    params.append(v)
+                else:
+                    params.append(f'{k} {v}')
+            param_str = ' ' + ', '.join(params)
         else:
             param_str = ''
 

mindsdb_sql/parser/ast/show.py

@@ -69,11 +69,11 @@ def get_string(self, *args, **kwargs):
             in_str = ' ' + ' '.join(ar)
 
         modes_str = f' {" ".join(self.modes)}' if self.modes else ''
-        like_str = f' LIKE {self.like}' if self.like else ''
+        like_str = f" LIKE '{self.like}'" if self.like else ""
         where_str = f' WHERE {str(self.where)}' if self.where else ''
 
         # custom commands
-        if self.category in ('FUNCTION CODE', 'PROCEDURE CODE', 'ENGINE'):
+        if self.category in ('FUNCTION CODE', 'PROCEDURE CODE', 'ENGINE') or self.category.startswith('ENGINE '):
             return f'SHOW {self.category} {self.name}'
         elif self.category == 'REPLICA STATUS':
             channel = ''

mindsdb_sql/parser/dialects/mindsdb/chatbot.py

@@ -39,7 +39,8 @@ def get_string(self, *args, **kwargs):
         params = self.params.copy()
         params['model'] = self.model.to_string() if self.model else 'NULL'
         params['database'] = self.database.to_string()
-        params['agent'] = self.agent.to_string() if self.agent else 'NULL'
+        if self.agent:
+            params['agent'] = self.agent.to_string()
 
         using_ar = [f'{k}={repr(v)}' for k, v in params.items()]
 

mindsdb_sql/parser/dialects/mindsdb/create_database.py

@@ -43,8 +43,12 @@ def get_string(self, *args, **kwargs):
         if self.is_replace:
             replace_str = f' OR REPLACE'
 
+        engine_str = ''
+        if self.engine:
+            engine_str = f'ENGINE = {repr(self.engine)} '
+
         parameters_str = ''
         if self.parameters:
             parameters_str = f', PARAMETERS = {json.dumps(self.parameters)}'
-        out_str = f'CREATE{replace_str} DATABASE {"IF NOT EXISTS " if self.if_not_exists else ""}{self.name.to_string()} WITH ENGINE = {repr(self.engine)}{parameters_str}'
+        out_str = f'CREATE{replace_str} DATABASE {"IF NOT EXISTS " if self.if_not_exists else ""}{self.name.to_string()} {engine_str}{parameters_str}'
         return out_str

mindsdb_sql/parser/dialects/mindsdb/create_job.py

@@ -79,7 +79,7 @@ def get_string(self, *args, **kwargs):
 
         if_query_str = ''
         if self.if_query_str is not None:
-            if_query_str = f" IF '{self.if_query_str}'"
+            if_query_str = f" IF ({self.if_query_str})"
 
         out_str = f'CREATE JOB {"IF NOT EXISTS" if self.if_not_exists else ""} {self.name.to_string()} ({self.query_str}){start_str}{end_str}{repeat_str}{if_query_str}'
         return out_str

mindsdb_sql/parser/dialects/mindsdb/lexer.py

@@ -72,6 +72,7 @@ class MindsDBLexer(Lexer):
         EQUALS, NEQUALS, GREATER, GEQ, LESS, LEQ,
         AND, OR, NOT, IS, IS_NOT,
         IN, LIKE, NOT_LIKE, CONCAT, BETWEEN, WINDOW, OVER, PARTITION_BY,
+        JSON_GET, JSON_GET_STR,
 
         # Data types
         CAST, ID, INTEGER, FLOAT, QUOTE_STRING, DQUOTE_STRING, NULL, TRUE, FALSE,
@@ -263,6 +264,8 @@ class MindsDBLexer(Lexer):
     SEMICOLON = r'\;'
 
     # Operators
+    JSON_GET = r'->'
+    JSON_GET_STR = r'->>'
     PLUS = r'\+'
     MINUS = r'-'
     DIVIDE = r'/'
@@ -288,7 +291,6 @@ class MindsDBLexer(Lexer):
     OVER = r'\bOVER\b'
     PARTITION_BY = r'\bPARTITION BY\b'
 
-
     # Data types
     NULL = r'\bNULL\b'
     TRUE = r'\bTRUE\b'

mindsdb_sql/parser/dialects/mindsdb/parser.py

@@ -44,10 +44,12 @@ class MindsDBParser(Parser):
         ('left', AND),
         ('right', UNOT),
         ('left', EQUALS, NEQUALS),
+        ('nonassoc', LESS, LEQ, GREATER, GEQ, IN, BETWEEN, IS, IS_NOT, NOT_LIKE, LIKE),
+        ('left', JSON_GET),
         ('left', PLUS, MINUS),
         ('left', STAR, DIVIDE),
         ('right', UMINUS),  # Unary minus operator, unary not
-        ('nonassoc', LESS, LEQ, GREATER, GEQ, IN, BETWEEN, IS, IS_NOT, NOT_LIKE, LIKE),
+
     )
 
     # Top-level statements
@@ -402,6 +404,8 @@ def set_item(self, p):
        'CHARSET',
        )
     def charset(self, p):
+        if hasattr(p, 'SET'):
+            return f'{p[0]} {p[1]}'
         return p[0]
 
     # set transaction
@@ -419,7 +423,7 @@ def set_item(self, p):
 
         params = {}
         if isolation_level is not None:
-            params['isolation_level'] = isolation_level
+            params['isolation level'] = isolation_level
         if access_mode is not None:
             params['access_mode'] = access_mode
 
@@ -523,75 +527,28 @@ def show(self, p):
             modes=modes
         )
 
-    @_('SCHEMAS',
-       'DATABASES',
-       'TABLES',
-       'OPEN TABLES',
-       'TRIGGERS',
-       'COLUMNS',
-       'FIELDS',
-       'PLUGINS',
-       'VARIABLES',
-       'INDEXES',
-       'KEYS',
-       'SESSION VARIABLES',
-       'GLOBAL VARIABLES',
-       'GLOBAL STATUS',
-       'SESSION STATUS',
-       'PROCEDURE STATUS',
-       'FUNCTION STATUS',
-       'TABLE STATUS',
-       'MASTER STATUS',
-       'STATUS',
-       'STORAGE ENGINES',
-       'PROCESSLIST',
-       'INDEX',
-       'CREATE TABLE',
-       'WARNINGS',
-       'ENGINES',
-       'CHARSET',
-       'CHARACTER SET',
-       'COLLATION',
-       'BINARY LOGS',
-       'MASTER LOGS',
-       'PRIVILEGES',
-       'PROFILES',
-       'REPLICAS',
-       'SLAVE HOSTS',
-       # Mindsdb specific
-       'VIEWS',
-       'STREAMS',
-       'PREDICTORS',
-       'INTEGRATIONS',
-       'DATASOURCES',
-       'PUBLICATIONS',
-       'DATASETS',
-       'MODELS',
-       'ML_ENGINES',
-       'HANDLERS',
-       'SEARCH_PATH',
-       'KNOWLEDGE_BASES',
-       'ALL')
+    @_(
+       'id',
+       'id id',
+    )
     def show_category(self, p):
-        return ' '.join([x for x in p])
+        if hasattr(p, 'id'):
+            return p.id
+        return f"{p.id0} {p.id1}"
 
     # custom show commands
-    @_('SHOW ENGINE identifier STATUS',
-       'SHOW ENGINE identifier MUTEX')
-    def show(self, p):
-        return Show(
-            category=p[1],
-            name=p.identifier.to_string(),
-            modes=[p[3]]
-        )
 
-    @_('SHOW FUNCTION CODE identifier',
-       'SHOW PROCEDURE CODE identifier')
+    @_('SHOW id id identifier')
     def show(self, p):
         category = p[1] + ' ' + p[2]
+
+        if p[1].lower() == 'engine':
+            name = p.identifier.parts[0]
+        else:
+            name = p.identifier.to_string()
         return Show(
             category=category,
-            name=p.identifier.to_string()
+            name=name
         )
 
     @_('SHOW REPLICA STATUS FOR CHANNEL id',
@@ -819,6 +776,7 @@ def create_predictor(self, p):
         'CREATE ANOMALY DETECTION MODEL identifier FROM identifier LPAREN raw_query RPAREN',
         'CREATE ANOMALY DETECTION MODEL identifier PREDICT result_columns',
         'CREATE ANOMALY DETECTION MODEL identifier PREDICT result_columns FROM identifier LPAREN raw_query RPAREN',
+        'CREATE ANOMALY DETECTION MODEL identifier FROM identifier LPAREN raw_query RPAREN PREDICT result_columns',
         # TODO add IF_NOT_EXISTS elegantly (should be low level BNF expansion)
     )
     def create_anomaly_detection_model(self, p):
@@ -1135,14 +1093,6 @@ def select(self, p):
         select.where = where_expr
         return select
 
-    # Special cases for keyword-like identifiers
-    @_('select FROM TABLES')
-    def select(self, p):
-        select = p.select
-        ensure_select_keyword_order(select, 'FROM')
-        select.from_table = Identifier(p.TABLES)
-        return select
-
     @_('select FROM from_table_aliased',
        'select FROM join_tables_implicit',
        'select FROM join_tables')
@@ -1437,17 +1387,15 @@ def expr(self, p):
        'expr LIKE expr',
        'expr NOT_LIKE expr',
        'expr CONCAT expr',
+       'expr JSON_GET constant',
+       'expr JSON_GET_STR constant',
        'expr IN expr')
     def expr(self, p):
         if hasattr(p, 'LAST'):
             arg1 = Last()
         else:
-            arg1 = p.expr1
-        if len(p) > 3:
-            op = ' '.join([p[i] for i in range(1, len(p)-1)])
-        else:
-            op = p[1]
-        return BinaryOperation(op=op, args=(p[0], arg1))
+            arg1 = p[2]
+        return BinaryOperation(op=p[1], args=(p[0], arg1))
 
     @_('MINUS expr %prec UMINUS',
        'NOT expr %prec UNOT', )
@@ -1645,6 +1593,7 @@ def parameter(self, p):
        'HORIZON',
        'HOSTS',
        'INDEXES',
+       'INDEX',
        'INTEGRATION',
        'INTEGRATIONS',
        'ISOLATION',
@@ -1686,6 +1635,7 @@ def parameter(self, p):
        'STREAM',
        'STREAMS',
        'TABLES',
+       'TABLE',
        'TRAIN',
        'TRANSACTION',
        'TRIGGERS',
@@ -1696,7 +1646,17 @@ def parameter(self, p):
        'WARNINGS',
        'MODEL',
        'MODELS',
-       'AGENT'
+       'AGENT',
+       'SCHEMAS',
+       'FUNCTION',
+       'charset',
+       'PROCEDURE',
+       'ML_ENGINES',
+       'HANDLERS',
+       'BINARY',
+       'KNOWLEDGE_BASES',
+       'ALL',
+       'CREATE',
        )
     def id(self, p):
         return p[0]

mindsdb_sql/parser/dialects/mysql/parser.py

@@ -176,7 +176,7 @@ def set_item(self, p):
 
         params = {}
         if isolation_level is not None:
-            params['isolation_level'] = isolation_level
+            params['isolation level'] = isolation_level
         if access_mode is not None:
             params['access_mode'] = access_mode
 
@@ -594,14 +594,6 @@ def select(self, p):
         select.where = where_expr
         return select
 
-    # Special cases for keyword-like identifiers
-    @_('select FROM TABLES')
-    def select(self, p):
-        select = p.select
-        ensure_select_keyword_order(select, 'FROM')
-        select.from_table = Identifier(p.TABLES)
-        return select
-
     @_('select FROM from_table_aliased',
        'select FROM join_tables_implicit',
        'select FROM join_tables')

mindsdb_sql/planner/plan_join.py

@@ -104,6 +104,7 @@ def plan(self, query):
         ):
             query2 = copy.deepcopy(query)
             query2.from_table = None
+            query2.using = None
             sup_select = QueryStep(query2, from_table=join_step.result)
             self.planner.plan.add_step(sup_select)
             return sup_select
@@ -429,11 +430,25 @@ def process_predictor(self, item, query_in):
                     # exclude condition
                     el._orig_node.args = [Constant(0), Constant(0)]
 
+        # params for model
+        model_params = None
+
+        if query_in.using is not None:
+            model_params = {}
+            for param, value in query_in.using.items():
+                if '.' in param:
+                    alias = param.split('.')[0]
+                    if (alias,) in item.aliases:
+                        new_param = '.'.join(param.split('.')[1:])
+                        model_params[new_param] = value
+                else:
+                    model_params[param] = value
+
         predictor_step = self.planner.plan.add_step(ApplyPredictorStep(
             namespace=item.integration,
             dataframe=data_step.result,
             predictor=item.table,
-            params=query_in.using,
+            params=model_params,
             row_dict=row_dict,
         ))
         self.step_stack.append(predictor_step)