Skip to content

Commit

Permalink
Merge pull request #349 from mindsdb/staging
Browse files Browse the repository at this point in the history
Release 0.10.4
  • Loading branch information
ea-rus authored Feb 2, 2024
2 parents 99d8b34 + ae53487 commit c2f8a7e
Show file tree
Hide file tree
Showing 16 changed files with 135 additions and 30 deletions.
2 changes: 1 addition & 1 deletion mindsdb_sql/__about__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
__title__ = 'mindsdb_sql'
__package_name__ = 'mindsdb_sql'
__version__ = '0.10.3'
__version__ = '0.10.4'
__description__ = "Pure python SQL parser"
__email__ = "[email protected]"
__author__ = 'MindsDB Inc'
Expand Down
13 changes: 12 additions & 1 deletion mindsdb_sql/parser/dialects/mindsdb/create_job.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ def __init__(self,
start_str=None,
end_str=None,
repeat_str=None,
if_query_str=None,
if_not_exists=False,
*args, **kwargs):
super().__init__(*args, **kwargs)
Expand All @@ -22,6 +23,7 @@ def __init__(self,
self.repeat_str = repeat_str
self.date_format = '%Y-%m-%d %H:%M:%S'
self.if_not_exists = if_not_exists
self.if_query_str = if_query_str

def to_tree(self, *args, level=0, **kwargs):
ind = indent(level)
Expand All @@ -46,13 +48,18 @@ def to_tree(self, *args, level=0, **kwargs):
if self.if_not_exists:
if_not_exists_str = f'\n{ind1}if_not_exists=True,'

if_query_str = ''
if self.if_query_str is not None:
if_query_str = f"\n{ind1}if_query='{self.if_query_str}'"

out_str = f'{ind}CreateJob(' \
f'{if_not_exists_str}' \
f'{name_str}' \
f'{query_str}' \
f'{start_str}' \
f'{end_str}' \
f'{repeat_str}' \
f'{if_query_str}' \
f'\n{ind})'
return out_str

Expand All @@ -70,5 +77,9 @@ def get_string(self, *args, **kwargs):
if self.repeat_str is not None:
repeat_str = f" EVERY '{self.repeat_str}'"

out_str = f'CREATE JOB {"IF NOT EXISTS" if self.if_not_exists else ""} {self.name.to_string()} ({self.query_str}){start_str}{end_str}{repeat_str}'
if_query_str = ''
if self.if_query_str is not None:
if_query_str = f" IF '{self.if_query_str}'"

out_str = f'CREATE JOB {"IF NOT EXISTS" if self.if_not_exists else ""} {self.name.to_string()} ({self.query_str}){start_str}{end_str}{repeat_str}{if_query_str}'
return out_str
11 changes: 8 additions & 3 deletions mindsdb_sql/parser/dialects/mindsdb/create_predictor.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ def __init__(self,
self.is_replace = is_replace
self.if_not_exists = if_not_exists
self.task = task
self._action = 'CREATE'

def to_tree(self, *args, level=0, **kwargs):
ind = indent(level)
Expand Down Expand Up @@ -74,8 +75,10 @@ def to_tree(self, *args, level=0, **kwargs):
using_str = f'\n{ind1}using={repr(self.using)},'

if_not_exists_str = f'\n{ind1}if_not_exists={self.if_not_exists},' if self.if_not_exists else ''
or_replace_str = f'\n{ind1}is_replace={self.is_replace},' if self.is_replace else ''

out_str = f'{ind}{self.__class__.__name__}(' \
f'{or_replace_str}' \
f'{if_not_exists_str}' \
f'{name_str}' \
f'{integration_name_str}' \
Expand Down Expand Up @@ -126,9 +129,11 @@ def get_string(self, *args, **kwargs):
if self.integration_name is not None:
integration_name_str = f'FROM {self.integration_name.to_string()} '

or_replace_str = ' OR REPLACE' if self.is_replace else ''
if_not_exists_str = 'IF NOT EXISTS ' if self.if_not_exists else ''
object_str = self._object + ' ' if self._object else ''

out_str = f'{self._command} {if_not_exists_str}{self.name.to_string()} {integration_name_str}{query_str}' \
out_str = f'{self._action}{or_replace_str} {object_str}{if_not_exists_str}{self.name.to_string()} {integration_name_str}{query_str}' \
f'{datasource_name_str}' \
f'{targets_str} ' \
f'{order_by_str}' \
Expand All @@ -143,12 +148,12 @@ def get_string(self, *args, **kwargs):
class CreatePredictor(CreatePredictorBase):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._command = 'CREATE PREDICTOR'
self._object = 'PREDICTOR'


# Models by task type
class CreateAnomalyDetectionModel(CreatePredictorBase):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._command = 'CREATE ANOMALY DETECTION MODEL'
self._object = 'ANOMALY DETECTION MODEL'
self.task = Identifier('AnomalyDetection')
3 changes: 2 additions & 1 deletion mindsdb_sql/parser/dialects/mindsdb/finetune_predictor.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,5 @@
class FinetunePredictor(CreatePredictorBase):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._command = 'FINETUNE'
self._action = 'FINETUNE'
self._object = ''
6 changes: 4 additions & 2 deletions mindsdb_sql/parser/dialects/mindsdb/lexer.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ class MindsDBLexer(Lexer):
GLOBAL, PROCEDURE, FUNCTION, INDEX, WARNINGS,
ENGINES, CHARSET, COLLATION, PLUGINS, CHARACTER,
PERSIST, PERSIST_ONLY,
IF_EXISTS, IF_NOT_EXISTS, COLUMNS, FIELDS, COLLATE, SEARCH_PATH,
IF_EXISTS, IF_NOT_EXISTS, IF, COLUMNS, FIELDS, COLLATE, SEARCH_PATH,
VARIABLE, SYSTEM_VARIABLE,

# SELECT Keywords
Expand Down Expand Up @@ -71,7 +71,7 @@ class MindsDBLexer(Lexer):
PLUS, MINUS, DIVIDE, MODULO,
EQUALS, NEQUALS, GREATER, GEQ, LESS, LEQ,
AND, OR, NOT, IS, IS_NOT,
IN, LIKE, CONCAT, BETWEEN, WINDOW, OVER, PARTITION_BY,
IN, LIKE, NOT_LIKE, CONCAT, BETWEEN, WINDOW, OVER, PARTITION_BY,

# Data types
CAST, ID, INTEGER, FLOAT, QUOTE_STRING, DQUOTE_STRING, NULL, TRUE, FALSE,
Expand Down Expand Up @@ -174,6 +174,7 @@ class MindsDBLexer(Lexer):
PERSIST_ONLY = r'\bPERSIST_ONLY\b'
IF_EXISTS = r'\bIF[\s]+EXISTS\b'
IF_NOT_EXISTS = r'\bIF[\s]+NOT[\s]+EXISTS\b'
IF = r'\bIF\b'
COLUMNS = r'\bCOLUMNS\b'
FIELDS = r'\bFIELDS\b'
EXTENDED = r'\bEXTENDED\b'
Expand Down Expand Up @@ -275,6 +276,7 @@ class MindsDBLexer(Lexer):
AND = r'\bAND\b'
OR = r'\bOR\b'
IS_NOT = r'\bIS[\s]+NOT\b'
NOT_LIKE = r'\bNOT[\s]+LIKE\b'
NOT = r'\bNOT\b'
IS = r'\bIS\b'
LIKE = r'\bLIKE\b'
Expand Down
45 changes: 36 additions & 9 deletions mindsdb_sql/parser/dialects/mindsdb/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ class MindsDBParser(Parser):
('left', PLUS, MINUS),
('left', STAR, DIVIDE),
('right', UMINUS), # Unary minus operator, unary not
('nonassoc', LESS, LEQ, GREATER, GEQ, IN, BETWEEN, IS, IS_NOT, LIKE),
('nonassoc', LESS, LEQ, GREATER, GEQ, IN, BETWEEN, IS, IS_NOT, NOT_LIKE, LIKE),
)

# Top-level statements
Expand Down Expand Up @@ -229,10 +229,20 @@ def drop_trigger(self, p):
# -- Jobs --
@_('CREATE JOB if_not_exists_or_empty identifier LPAREN raw_query RPAREN job_schedule',
'CREATE JOB if_not_exists_or_empty identifier AS LPAREN raw_query RPAREN job_schedule',
'CREATE JOB if_not_exists_or_empty identifier LPAREN raw_query RPAREN job_schedule IF LPAREN raw_query RPAREN',
'CREATE JOB if_not_exists_or_empty identifier AS LPAREN raw_query RPAREN job_schedule IF LPAREN raw_query RPAREN',
'CREATE JOB if_not_exists_or_empty identifier LPAREN raw_query RPAREN',
'CREATE JOB if_not_exists_or_empty identifier AS LPAREN raw_query RPAREN')
'CREATE JOB if_not_exists_or_empty identifier AS LPAREN raw_query RPAREN',
'CREATE JOB if_not_exists_or_empty identifier LPAREN raw_query RPAREN IF LPAREN raw_query RPAREN',
'CREATE JOB if_not_exists_or_empty identifier AS LPAREN raw_query RPAREN IF LPAREN raw_query RPAREN'
)
def create_job(self, p):
query_str = tokens_to_string(p.raw_query)
if hasattr(p, 'raw_query0'):
query_str = tokens_to_string(p.raw_query0)
if_query_str = tokens_to_string(p.raw_query1)
else:
query_str = tokens_to_string(p.raw_query)
if_query_str = None

job_schedule = getattr(p, 'job_schedule', {})

Expand All @@ -254,6 +264,7 @@ def create_job(self, p):
return CreateJob(
name=p.identifier,
query_str=query_str,
if_query_str=if_query_str,
start_str=start_str,
end_str=end_str,
repeat_str=repeat_str,
Expand Down Expand Up @@ -776,10 +787,11 @@ def create_predictor(self, p):
p.create_predictor.order_by = p.ordering_terms
return p.create_predictor

@_('CREATE PREDICTOR if_not_exists_or_empty identifier FROM identifier LPAREN raw_query RPAREN PREDICT result_columns',
'CREATE PREDICTOR if_not_exists_or_empty identifier PREDICT result_columns',
'CREATE MODEL if_not_exists_or_empty identifier FROM identifier LPAREN raw_query RPAREN PREDICT result_columns',
'CREATE MODEL if_not_exists_or_empty identifier PREDICT result_columns')
@_('CREATE replace_or_empty PREDICTOR if_not_exists_or_empty identifier FROM identifier LPAREN raw_query RPAREN PREDICT result_columns',
'CREATE replace_or_empty PREDICTOR if_not_exists_or_empty identifier PREDICT result_columns',
'CREATE replace_or_empty MODEL if_not_exists_or_empty identifier FROM identifier LPAREN raw_query RPAREN PREDICT result_columns',
'CREATE replace_or_empty MODEL if_not_exists_or_empty identifier PREDICT result_columns'
)
def create_predictor(self, p):
query_str = None
if hasattr(p, 'raw_query'):
Expand All @@ -796,7 +808,8 @@ def create_predictor(self, p):
integration_name=getattr(p, 'identifier1', None),
query_str=query_str,
targets=p.result_columns,
if_not_exists=p.if_not_exists_or_empty
if_not_exists=p.if_not_exists_or_empty,
is_replace=p.replace_or_empty
)

# Typed models
Expand Down Expand Up @@ -1422,14 +1435,19 @@ def expr(self, p):
'expr NOT expr',
'expr IS expr',
'expr LIKE expr',
'expr NOT_LIKE expr',
'expr CONCAT expr',
'expr IN expr')
def expr(self, p):
if hasattr(p, 'LAST'):
arg1 = Last()
else:
arg1 = p.expr1
return BinaryOperation(op=p[1], args=(p[0], arg1))
if len(p) > 3:
op = ' '.join([p[i] for i in range(1, len(p)-1)])
else:
op = p[1]
return BinaryOperation(op=op, args=(p[0], arg1))

@_('MINUS expr %prec UMINUS',
'NOT expr %prec UNOT', )
Expand Down Expand Up @@ -1729,6 +1747,15 @@ def variable(self, p):
def variable(self, p):
return Variable(value=p.VARIABLE)

@_(
'OR REPLACE',
'empty'
)
def replace_or_empty(self, p):
if hasattr(p, 'REPLACE'):
return True
return False

@_(
'IF_NOT_EXISTS',
'empty'
Expand Down
6 changes: 3 additions & 3 deletions mindsdb_sql/parser/dialects/mindsdb/retrain_predictor.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
from mindsdb_sql.parser.ast.base import ASTNode
from mindsdb_sql.parser.utils import indent
from .create_predictor import CreatePredictorBase


class RetrainPredictor(CreatePredictorBase):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._command = 'RETRAIN'
self._action = 'RETRAIN'
self._object = ''
15 changes: 13 additions & 2 deletions mindsdb_sql/planner/plan_join.py
Original file line number Diff line number Diff line change
Expand Up @@ -407,16 +407,27 @@ def process_predictor(self, item, query_in):
raise NotImplementedError("TS predictor is not supported here yet")
data_step = self.step_stack[-1]
row_dict = None

predict_target = item.predictor_info.get('to_predict')
if isinstance(predict_target, list) and len(predict_target) > 0:
predict_target = predict_target[0]
if predict_target is not None:
predict_target = predict_target.lower()

if item.conditions:
row_dict = {}
for el in item.conditions:
for i, el in enumerate(item.conditions):
if isinstance(el.args[0], Identifier) and el.op == '=':
col_name = el.args[0].parts[-1]
if col_name.lower() == predict_target:
# don't add predict target to parameters
continue

if isinstance(el.args[1], (Constant, Parameter)):
row_dict[el.args[0].parts[-1]] = el.args[1].value

# exclude condition
item.conditions[0]._orig_node.args = [Constant(0), Constant(0)]
el._orig_node.args = [Constant(0), Constant(0)]

predictor_step = self.planner.plan.add_step(ApplyPredictorStep(
namespace=item.integration,
Expand Down
3 changes: 2 additions & 1 deletion mindsdb_sql/planner/query_planner.py
Original file line number Diff line number Diff line change
Expand Up @@ -289,9 +289,10 @@ def plan_api_db_select(self, query):
# keep only limit and where
# the rest goes to outer select
query2 = Select(
targets=[Star()],
targets=query.targets,
from_table=query.from_table,
where=query.where,
order_by=query.order_by,
limit=query.limit,
)
prev_step = self.plan_integration_select(query2)
Expand Down
1 change: 1 addition & 0 deletions mindsdb_sql/render/sqlalchemy_render.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ def to_expression(self, t):
"is": "is_",
"is not": "is_not",
"like": "like",
"not like": "notlike",
"in": "in_",
"not in": "notin_",
"||": "concat",
Expand Down
18 changes: 18 additions & 0 deletions tests/test_parser/test_base_sql/test_select_operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -564,3 +564,21 @@ def test_function_with_from(self, dialect):
assert str(ast) == str(expected_ast)
assert ast.to_tree() == expected_ast.to_tree()

class TestOperationsMindsdb:
def test_select_binary_operations(self):
for op in ['not like',]:
sql = f'SELECT column1 {op.upper()} column2 FROM tab'
ast = parse_sql(sql)

expected_ast = Select(
targets=[BinaryOperation(op=op,
args=(
Identifier.from_path_str('column1'), Identifier.from_path_str('column2')
)),
],
from_table=Identifier.from_path_str('tab')
)

assert str(ast).lower() == sql.lower()
assert str(ast) == str(expected_ast)
assert ast.to_tree() == expected_ast.to_tree()
11 changes: 11 additions & 0 deletions tests/test_parser/test_mindsdb/test_create_predictor.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,17 @@ def test_create_predictor_no_with(self):

assert ast.to_tree() == expected_ast.to_tree()

# test or replace
sql = """CREATE or replace model pred
FROM integration_name
(select * FROM table_name)
PREDICT f1 as f1_alias, f2
"""
ast = parse_sql(sql, dialect='mindsdb')
expected_ast.is_replace = True

assert ast.to_tree() == expected_ast.to_tree()

def test_create_predictor_quotes(self):
sql = """CREATE PREDICTOR xxx
FROM `yyy`
Expand Down
17 changes: 17 additions & 0 deletions tests/test_parser/test_mindsdb/test_jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,23 @@ def test_create_job(self):
assert str(ast) == str(expected_ast)
assert ast.to_tree() == expected_ast.to_tree()

# 4

sql = '''
create job j1 ( retrain p1 )
every 2 hours
if (select a from table)
'''
ast = parse_sql(sql, dialect='mindsdb')
expected_ast = CreateJob(
name=Identifier('j1'),
query_str="retrain p1",
if_query_str="select a from table",
repeat_str="2 hours"
)
assert str(ast) == str(expected_ast)
assert ast.to_tree() == expected_ast.to_tree()

def test_create_job_minimal_with_if_not_exists(self):
sql = '''
create job if not exists proj2.j1 (
Expand Down
6 changes: 3 additions & 3 deletions tests/test_planner/test_integration_select.py
Original file line number Diff line number Diff line change
Expand Up @@ -524,7 +524,7 @@ def test_select_from_table_subselect_api_integration(self):
steps=[
FetchDataframeStep(
integration='int1',
query=parse_sql('select * from tab1'),
query=parse_sql('select tab1.`id` AS `id` from tab1'),
),
SubSelectStep(
dataframe=Result(0),
Expand All @@ -534,7 +534,7 @@ def test_select_from_table_subselect_api_integration(self):
FetchDataframeStep(
integration='int1',
query=Select(
targets=[Star()],
targets=[Identifier('tab2.x', alias=Identifier('x'))],
from_table=Identifier('tab2'),
where=BinaryOperation(
op='in',
Expand Down Expand Up @@ -597,7 +597,7 @@ def test_delete_from_table_subselect_api_integration(self):
steps=[
FetchDataframeStep(
integration='int1',
query=parse_sql('select * from tab1'),
query=parse_sql('select tab1.`id` AS `id` from tab1'),
),
SubSelectStep(
dataframe=Result(0),
Expand Down
Loading

0 comments on commit c2f8a7e

Please sign in to comment.