diff --git a/.github/workflows/add_to_pr_review.yml b/.github/workflows/add_to_pr_review.yml new file mode 100644 index 0000000..384f2be --- /dev/null +++ b/.github/workflows/add_to_pr_review.yml @@ -0,0 +1,16 @@ +name: Add Pull Requests to PR review project + +on: + pull_request: + types: + - opened + +jobs: + add-to-project: + name: Add issue to project + runs-on: ubuntu-latest + steps: + - uses: actions/add-to-project@v0.5.0 + with: + project-url: https://github.com/orgs/mindsdb/projects/65 + github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} diff --git a/.github/workflows/add_to_roadmap_project_v2.yml b/.github/workflows/add_to_roadmap_project_v2.yml new file mode 100644 index 0000000..240c700 --- /dev/null +++ b/.github/workflows/add_to_roadmap_project_v2.yml @@ -0,0 +1,14 @@ +name: Add issue to roadmap project +on: + issues: + types: + - opened +jobs: + add-to-project: + name: Add issue to roadmap project + runs-on: ubuntu-latest + steps: + - uses: actions/add-to-project@v0.4.0 + with: + project-url: https://github.com/orgs/mindsdb/projects/53 + github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} \ No newline at end of file diff --git a/.github/workflows/mindsdb_python_sdk.yml b/.github/workflows/mindsdb_python_sdk.yml index 79a59f1..24bfae5 100644 --- a/.github/workflows/mindsdb_python_sdk.yml +++ b/.github/workflows/mindsdb_python_sdk.yml @@ -1,7 +1,6 @@ -name: MindsDB Native workflow +name: PR workflow on: - push: pull_request: branches: - stable @@ -43,6 +42,8 @@ jobs: needs: test if: github.ref != 'refs/heads/stable' runs-on: ubuntu-latest + permissions: + pull-requests: write steps: - uses: actions/checkout@v3 - name: Set up Python 3.8 @@ -65,25 +66,3 @@ jobs: with: pytest-coverage-path: ./pytest-coverage.txt junitxml-path: ./pytest.xml - - deploy: - runs-on: ubuntu-latest - needs: test - if: github.ref == 'refs/heads/stable' - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - name: Install dependencies - run: | - python -m pip install --upgrade pip==20.2.4 - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py sdist - twine upload dist/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..e3c8481 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,61 @@ +name: Release + +on: + push: + branches: + - stable + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + python-version: [3.7,3.8,3.9] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip==22.0.4 + pip install -r requirements.txt + pip install --no-cache-dir . + - name: Run tests + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + + env PYTHONPATH=./ pytest tests/ + + fi + shell: bash + env: + CHECK_FOR_UPDATES: False + DATABASE_CREDENTIALS_STRINGIFIED_JSON: ${{ secrets.DATABASE_CREDENTIALS }} + CLOUD_TEST_EMAIL: ${{ secrets.CLOUD_TEST_EMAIL }} + CLOUD_TEST_PASSWORD: ${{ secrets.CLOUD_TEST_PASSWORD }} + + + deploy: + runs-on: ubuntu-latest + needs: test + if: github.ref == 'refs/heads/stable' + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Install dependencies + run: | + python -m pip install --upgrade pip==20.2.4 + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist + twine upload dist/* diff --git a/README.md b/README.md index aea6708..d217ec1 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ https://github.com/mindsdb/mindsdb_python_sdk/tree/staging/examples ## API Documentation -The API documentation for the MindsDB SDK can be found at https://mindsdb.github.io/mindsdb_python_sdk/. You can generate the API documentation locally by following these steps: +The API documentation for the MindsDB SDK can be found at https://mindsdb.github.io/mindsdb_python_sdk/. ### Generating API docs locally: diff --git a/docs/source/conf.py b/docs/source/conf.py index a0df610..03b66bf 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -16,7 +16,11 @@ # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon'] +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', + 'sphinx.ext.autosectionlabel' +] templates_path = ['_templates'] exclude_patterns = [] diff --git a/docs/source/index.rst b/docs/source/index.rst index c7705ac..3457c21 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -90,7 +90,7 @@ Base usage More More examples ------------ +------------- ``_ @@ -107,12 +107,13 @@ API documentation :maxdepth: 1 :caption: Modules: - server database + + project handlers + ml_engines - project model tables views diff --git a/docs/source/jobs.rst b/docs/source/jobs.rst index aa47c3e..7fb2858 100644 --- a/docs/source/jobs.rst +++ b/docs/source/jobs.rst @@ -1,6 +1,9 @@ Jobs ------------------------- +.. _my-reference-label: + + .. automodule:: mindsdb_sdk.jobs :members: :undoc-members: diff --git a/docs/source/server.rst b/docs/source/server.rst index 3c7284c..9d8a942 100644 --- a/docs/source/server.rst +++ b/docs/source/server.rst @@ -4,4 +4,7 @@ Server .. automodule:: mindsdb_sdk.server :members: :undoc-members: - :show-inheritance: \ No newline at end of file + :show-inheritance: + + + :ref:`jobs` \ No newline at end of file diff --git a/examples/working_with_tables.py b/examples/working_with_tables.py index 73ca881..ee2f2fa 100644 --- a/examples/working_with_tables.py +++ b/examples/working_with_tables.py @@ -3,21 +3,44 @@ con = mindsdb_sdk.connect() -# get user's database (connected to mindsdb as rental_db) -db = con.databases.rental_db +# connect to mindsdb example database +example_db = con.databases.create( + 'example_db', + engine='postgres', + connection_args={ + "user": "demo_user", + "password": "demo_password", + "host": "3.220.66.106", + "port": "5432", + "database": "demo" + } +) -# get table -table1 = db.tables.house_sales +# connect to the empty user database +my_db = con.databases.create( + 'my_db', + engine='postgres', + connection_args={ + "user": "postgres", + "host": "localhost", + "port": "5432", + "database": "my_database" + } +) +# get home_rentals table +table1 = example_db.tables.get('demo_data.home_rentals') # ---- create new table ---- -# copy create table house_sales and fill it with rows with type=house -table2 = db.tables.create('house_sales2', table1.filter(type='house')) +# create table home_rentals in user db and fill it with rows with location=great +table2 = my_db.tables.create('home_rentals', table1.filter(location='great')) + # create table from csv file + df = pd.read_csv('my_data.csv') -table3 = db.tables.create('my_table', df) +table3 = my_db.tables.create('my_table', df) # ---- insert into table ---- @@ -28,16 +51,16 @@ # ---- update data in table ---- -# get all rows with type=house from table1 and update values in table2 using key ('saledate', 'type', 'bedrooms') +# get all rows with number_of_rooms=1 from table1 and update values in table2 using key ('location', 'neighborhood') table2.update( - table1.filter(type='house'), - on=['saledate', 'type', 'bedrooms'] + table1.filter(number_of_rooms=1), + on=['location', 'neighborhood'] ) # ---- delete rows from table ---- # delete all rows where bedrooms=2 -table2.delete(bedrooms=2) +table2.delete(number_of_rooms=1) diff --git a/mindsdb_sdk/__about__.py b/mindsdb_sdk/__about__.py index 186d972..39ac92b 100755 --- a/mindsdb_sdk/__about__.py +++ b/mindsdb_sdk/__about__.py @@ -1,6 +1,6 @@ __title__ = 'mindsdb_sdk' __package_name__ = 'mindsdb_sdk' -__version__ = '2.0.0' +__version__ = '2.1.0' __description__ = "MindsDB Python SDK, provides an SDK to use a remote mindsdb instance" __email__ = "jorge@mindsdb.com" __author__ = 'MindsDB Inc' diff --git a/mindsdb_sdk/connect.py b/mindsdb_sdk/connect.py index 0a5c8ed..af998b4 100644 --- a/mindsdb_sdk/connect.py +++ b/mindsdb_sdk/connect.py @@ -20,17 +20,17 @@ def connect(url: str = None, login: str = None, password: str = None, is_managed Connect to local server - >>> server = mindsdb_sdk.connect() - >>> server = mindsdb_sdk.connect('http://127.0.0.1:47334') + >>> con = mindsdb_sdk.connect() + >>> con = mindsdb_sdk.connect('http://127.0.0.1:47334') Connect to cloud server - >>> server = mindsdb_sdk.connect(login='a@b.com', password='-') - >>> server = mindsdb_sdk.connect('https://cloud.mindsdb.com', login='a@b.com', password='-') + >>> con = mindsdb_sdk.connect(login='a@b.com', password='-') + >>> con = mindsdb_sdk.connect('https://cloud.mindsdb.com', login='a@b.com', password='-') Connect to MindsDB pro - >>> server = mindsdb_sdk.connect('http://', login='a@b.com', password='-', is_managed=True) + >>> con = mindsdb_sdk.connect('http://', login='a@b.com', password='-', is_managed=True) """ if url is None: diff --git a/mindsdb_sdk/connectors/rest_api.py b/mindsdb_sdk/connectors/rest_api.py index f27f36c..cbbba7f 100644 --- a/mindsdb_sdk/connectors/rest_api.py +++ b/mindsdb_sdk/connectors/rest_api.py @@ -124,7 +124,7 @@ def upload_file(self, name: str, df: pd.DataFrame): # convert to file fd = io.BytesIO() - df.to_csv(fd) + df.to_csv(fd, index=False) fd.seek(0) url = self.url + f'/api/files/{name}' @@ -139,4 +139,24 @@ def upload_file(self, name: str, df: pd.DataFrame): 'file': fd, } ) - _raise_for_status(r) \ No newline at end of file + _raise_for_status(r) + + @_try_relogin + def upload_byom(self, name: str, code: str, requirements: str): + + url = self.url + f'/api/handlers/byom/{name}' + r = self.session.put( + url, + files={ + 'code': code, + 'modules': requirements, + } + ) + _raise_for_status(r) + + def status(self) -> dict: + + r = self.session.get(self.url + f'/api/status') + _raise_for_status(r) + + return r.json() diff --git a/mindsdb_sdk/databases.py b/mindsdb_sdk/databases.py index 2e69b7b..f5e0257 100644 --- a/mindsdb_sdk/databases.py +++ b/mindsdb_sdk/databases.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Union from mindsdb_sql.parser.dialects.mindsdb import CreateDatabase from mindsdb_sql.parser.ast import DropDatabase, Identifier @@ -7,6 +7,7 @@ from .query import Query from .tables import Tables +from .handlers import Handler class Database: @@ -93,7 +94,7 @@ def list(self) -> List[Database]: """ return [Database(self, name) for name in self._list_databases()] - def create(self, name: str, engine: str, connection_args: dict) -> Database: + def create(self, name: str, engine: Union[str, Handler], connection_args: dict) -> Database: """ Create new integration and return it @@ -102,6 +103,9 @@ def create(self, name: str, engine: str, connection_args: dict) -> Database: :param connection_args: {"key": "value"} object with the connection parameters specific for each engine :return: created Database object """ + if isinstance(engine, Handler): + engine = engine.name + ast_query = CreateDatabase( name=Identifier(name), engine=engine, diff --git a/mindsdb_sdk/handlers.py b/mindsdb_sdk/handlers.py index 31b3472..e2dd361 100644 --- a/mindsdb_sdk/handlers.py +++ b/mindsdb_sdk/handlers.py @@ -9,6 +9,9 @@ @dataclass(init=False) class Handler: + """ + :meta private: + """ name: str title: str version: str @@ -26,26 +29,7 @@ def __init__(self, **kwargs): class Handlers(CollectionBase): """ - **Handlers colection** - - Examples of usage: - - ML handlers: - - Get list - >>> con.ml_handlers.list() - - Get - >>> openai_handler = con.ml_handlers.openai - - DATA handlers: - - Get list - >>> con.data_handlers.list() - - Get - >>> pg_handler = con.data_handlers.postgres - + :meta private: """ def __init__(self, api, type): @@ -91,3 +75,43 @@ def get(self, name: str) -> Handler: if item.name == name: return item raise AttributeError(f"Handler doesn't exist: {name}") + + +class MLHandlers(Handlers): + """ + **ML handlers colection** + + Examples of usage: + + Get list + + >>> con.ml_handlers.list() + + Get + + >>> openai_handler = con.ml_handlers.openai + >>> openai_handler = con.ml_handlers.get('openai') + + """ + + ... + + +class DataHandlers(Handlers): + """ + **DATA handlers colection** + + Examples of usage: + + Get list + + >>> con.data_handlers.list() + + Get + + >>> pg_handler = con.data_handlers.postgres + >>> pg_handler = con.data_handlers.get('postgres') + + """ + + ... \ No newline at end of file diff --git a/mindsdb_sdk/jobs.py b/mindsdb_sdk/jobs.py index 222efc0..369a0da 100644 --- a/mindsdb_sdk/jobs.py +++ b/mindsdb_sdk/jobs.py @@ -99,7 +99,10 @@ def create(self, name: str, query_str: str, repeat_str: str = None) -> Union[Job, None]: """ Create new job in project and return it. - If it is not possible (job executed and not accessible anymore): return None + + If it is not possible (job executed and not accessible anymore): + return None + More info: https://docs.mindsdb.com/sql/create/jobs :param name: name of the job diff --git a/mindsdb_sdk/ml_engines.py b/mindsdb_sdk/ml_engines.py index eeb15f2..7cb67b3 100644 --- a/mindsdb_sdk/ml_engines.py +++ b/mindsdb_sdk/ml_engines.py @@ -10,6 +10,9 @@ @dataclass class MLEngine: + """ + :meta private: + """ name: str handler: str connection_data: dict @@ -23,12 +26,15 @@ class MLEngines(CollectionBase): Examples of usage: Get list + >>> ml_engines = con.ml_engines.list() Get + >>> openai_engine = con.ml_engines.openai1 Create + >>> con.ml_engines.create( ... 'openai1', ... 'openai', @@ -36,8 +42,19 @@ class MLEngines(CollectionBase): ...) Drop + >>> con.ml_engines.drop('openai1') + Upload BYOM model. After uploading a new ml engin will be availbe to create new model from it. + + >>> model_code = open('/path/to/model/code').read() + >>> model_requirements = open('/path/to/model/requirements').read() + >>> ml_engine = con.ml_engines.create_byom( + ... 'my_byom_engine', + ... code=model_code, + ... requirements=model_requirements + ...) + """ def __init__(self, api): @@ -46,6 +63,7 @@ def __init__(self, api): def list(self) -> List[MLEngine]: """ Returns list of ml engines on server + :return: list of ml engines """ @@ -77,6 +95,7 @@ def get(self, name: str) -> MLEngine: def create(self, name: str, handler: Union[str, Handler], connection_data: dict = None) -> MLEngine: """ Create new ml engine and return it + :param name: ml engine name, string :param handler: handler name, string or Handler :param connection_data: parameters for ml engine, dict, optional @@ -92,9 +111,29 @@ def create(self, name: str, handler: Union[str, Handler], connection_data: dict return MLEngine(name, handler, connection_data) + def create_byom(self, name: str, code: str, requirements: Union[str, List[str]] = None): + """ + Create new BYOM ML engine and return it + + :param code: model python code in string + :param requirements: requirements for model. Optional if there is no special requirements. + It can be content of 'requirement.txt' file or list of strings (item for every requirement). + :return: created BYOM ml engine object + """ + + if requirements is None: + requirements = '' + elif isinstance(requirements, list): + requirements = '\n'.join(requirements) + + self.api.upload_byom(name, code, requirements) + + return MLEngine(name, 'byom', {}) + def drop(self, name: str): """ Drop ml engine by name + :param name: name """ ast_query = DropMLEngine(Identifier(name)) diff --git a/mindsdb_sdk/models.py b/mindsdb_sdk/models.py index ee7cb8b..0ff6f02 100644 --- a/mindsdb_sdk/models.py +++ b/mindsdb_sdk/models.py @@ -117,9 +117,11 @@ def predict(self, data: Union[pd.DataFrame, Query, dict], params: dict = None) - """ Make prediction using model - if data is dataframe it uses /model/predict http method and sends dataframe over it - if data is select query with one table it replaces table to jon table and predictor - and sends query over sql/query http method + if data is dataframe + it uses /model/predict http method and sends dataframe over it + + if data is select query with one table + it replaces table to jon table and predictor and sends query over sql/query http method if data is select from join other complex query it modifies query to: 'select from (input query) join model' and sends it over sql/query http method @@ -337,6 +339,8 @@ def drop_version(self, num: int) -> ModelVersion: """ Drop version of the model + >>> models.rentals_model.drop_version(version=10) + :param num: version to drop """ @@ -390,40 +394,6 @@ class Models(CollectionBase): >>> model = models.get('model1') >>> model = models.get('model1', version=2) - Create - - Create, using params and qeury as string - - >>> model = models.create( - ... 'rentals_model', - ... predict='price', - ... engine='lightwood', - ... database='example_db', - ... query='select * from table', - ... options={ - ... 'module': 'LightGBM' - ... }, - ... timeseries_options={ - ... 'order': 'date', - ... 'group': ['a', 'b'] - ... } - ...) - - Create, using deferred query. 'query' will be executed and converted to dataframe on mindsdb backend. - - >>> query = databases.db.query('select * from table') - >>> model = models.create( - ... 'rentals_model', - ... predict='price', - ... query=query, - ...) - - - Drop - - >>> models.drop('rentals_model') - >>> models.rentals_model.drop_version(version=10) - """ def __init__(self, project, api): @@ -445,6 +415,32 @@ def create( If query/database is passed, it will be executed on mindsdb side + Create, using params and qeury as string + + >>> model = models.create( + ... 'rentals_model', + ... predict='price', + ... engine='lightwood', + ... database='example_db', + ... query='select * from table', + ... options={ + ... 'module': 'LightGBM' + ... }, + ... timeseries_options={ + ... 'order': 'date', + ... 'group': ['a', 'b'] + ... } + ...) + + Create, using deferred query. 'query' will be executed and converted to dataframe on mindsdb backend. + + >>> query = databases.db.query('select * from table') + >>> model = models.create( + ... 'rentals_model', + ... predict='price', + ... query=query, + ...) + :param name: name of the model :param predict: prediction target :param engine: ml engine for new model, default is mindsdb @@ -467,6 +463,9 @@ def create( targets = [Identifier(predict)] else: targets = None + if database is None: + raise RuntimeError('Database is not defined') + ast_query = CreatePredictor( name=Identifier(name), query_str=query, @@ -537,6 +536,8 @@ def drop(self, name: str): """ Drop model from project with all versions + >>> models.drop('rentals_model') + :param name: name of the model """ ast_query = DropPredictor(name=Identifier(name)) @@ -549,8 +550,10 @@ def list(self, with_versions: bool = False, """ List models (or model versions) in project - If with_versions = True it shows all models with version (executes 'select * from models_versions') - Otherwise it shows only models (executes 'select * from models') + If with_versions = True + it shows all models with version (executes 'select * from models_versions') + + Otherwise it shows only models (executes 'select * from models') :param with_versions: show model versions :param name: to show models or versions only with selected name, optional diff --git a/mindsdb_sdk/projects.py b/mindsdb_sdk/projects.py index d234ab5..d79deb9 100644 --- a/mindsdb_sdk/projects.py +++ b/mindsdb_sdk/projects.py @@ -21,9 +21,9 @@ class Project: Server instance allows to manipulate project and databases (integration) on mindsdb server Attributes for accessing to different objects: - - models - - views - - jobs + - models, see :func:`~mindsdb_sdk.models.Models` + - views, see :func:`~mindsdb_sdk.views.Views` + - jobs, see :func:`~mindsdb_sdk.jobs.Jobs` It is possible to cal queries from project context: @@ -109,23 +109,23 @@ class Projects(CollectionBase): Projects ---------- - # list of projects + list of projects >>> projects.list() - # create + create >>> project = projects.create('proj') - # drop + drop >>> projects.drop('proj') - # get existing + get existing >>> projects.get('proj') - # by attribute + by attribute >>> projects.proj """ diff --git a/mindsdb_sdk/server.py b/mindsdb_sdk/server.py index d6e6319..ccb8671 100644 --- a/mindsdb_sdk/server.py +++ b/mindsdb_sdk/server.py @@ -9,14 +9,17 @@ class Server(Project): Server instance allows to manipulate project and databases (integration) on mindsdb server Attributes for accessing to different objects: - - projects - - databases - - ml_engines - Server is also root(mindsdb) project and has its attributes - - models - - views - - jobs + - projects, see :func:`~mindsdb_sdk.projects.Projects` + - databases, see :func:`~mindsdb_sdk.databases.Databases` + - ml_engines, see :func:`~mindsdb_sdk.ml_engines.MLEngines` + - ml_handlers, see :func:`~mindsdb_sdk.handlers.MLHandlers` + - data_handlers, see :func:`~mindsdb_sdk.handlers.DataHandlers` + + Server is also root(mindsdb) project and has attributes of project: + - models, see :func:`~mindsdb_sdk.models.Models` + - views, see :func:`~mindsdb_sdk.views.Views` + - jobs, see :func:`~mindsdb_sdk.jobs.Jobs` """ @@ -45,6 +48,16 @@ def __init__(self, api): self.ml_handlers = Handlers(self.api, 'ml') self.data_handlers = Handlers(self.api, 'data') + def status(self) -> dict: + """ + Get server information. It could content version + Example of getting version for local: + >>> print(server.status()['mindsdb_version']) + + :return: server status info + """ + return self.api.status() + def __repr__(self): return f'{self.__class__.__name__}({self.api.url})' diff --git a/mindsdb_sdk/tables.py b/mindsdb_sdk/tables.py index 78d7f3e..b19fd49 100644 --- a/mindsdb_sdk/tables.py +++ b/mindsdb_sdk/tables.py @@ -102,7 +102,9 @@ def insert(self, query: Union[pd.DataFrame, Query]): def delete(self, **kwargs): """ - Deletes record from table using filters table.delete(a=1, b=2) + Deletes record from table using filters + + >>> table.delete(a=1, b=2) :param kwargs: filter """ @@ -121,15 +123,16 @@ def delete(self, **kwargs): def update(self, values: Union[dict, Query], on: list = None, filters: dict = None): ''' Update table by condition of from other table. + If 'values' is a dict: - - it will be an update by condition - - 'filters' is required - - used command: update table set a=1 where x=1 + it will be an update by condition + 'filters' is required + used command: update table set a=1 where x=1 If 'values' is a Query: - - it will be an update from select - - 'on' is required - - used command: update table on a,b from (query) + it will be an update from select + 'on' is required + used command: update table on a,b from (query) :param values: input for update, can be dict or query :param on: list of column to map subselect to table ['a', 'b', ...] @@ -233,10 +236,6 @@ def get(self, name: str) -> Table: :return: Table object """ - if name not in self._list_tables(): - if '.' not in name: - # fixme: schemas not visible in 'show tables' - raise AttributeError("Table doesn't exist") return Table(self.database, name) def create(self, name: str, query: Union[pd.DataFrame, Query], replace: bool = False) -> Table: diff --git a/mindsdb_sdk/utils/objects_collection.py b/mindsdb_sdk/utils/objects_collection.py index 054c026..8405824 100644 --- a/mindsdb_sdk/utils/objects_collection.py +++ b/mindsdb_sdk/utils/objects_collection.py @@ -19,39 +19,39 @@ def __getattr__(self, name): return self.get(name) -class MethodCollection(CollectionBase): - - def __init__(self, name, methods): - self.name = name - self.methods = methods - - def __repr__(self): - return f'{self.__class__.__name__}({self.name})' - - def get(self, *args, **kwargs): - method = self.methods.get('get') - if method is None: - raise NotImplementedError() - - return method(*args, **kwargs) - - def list(self, *args, **kwargs): - method = self.methods.get('list') - if method is None: - raise NotImplementedError() - - return method(*args, **kwargs) - - def create(self, *args, **kwargs): - method = self.methods.get('create') - if method is None: - raise NotImplementedError() - - return method(*args, **kwargs) - - def drop(self, name): - method = self.methods.get('drop') - if method is None: - raise NotImplementedError() - - return method(name) +# class MethodCollection(CollectionBase): +# +# def __init__(self, name, methods): +# self.name = name +# self.methods = methods +# +# def __repr__(self): +# return f'{self.__class__.__name__}({self.name})' +# +# def get(self, *args, **kwargs): +# method = self.methods.get('get') +# if method is None: +# raise NotImplementedError() +# +# return method(*args, **kwargs) +# +# def list(self, *args, **kwargs): +# method = self.methods.get('list') +# if method is None: +# raise NotImplementedError() +# +# return method(*args, **kwargs) +# +# def create(self, *args, **kwargs): +# method = self.methods.get('create') +# if method is None: +# raise NotImplementedError() +# +# return method(*args, **kwargs) +# +# def drop(self, name): +# method = self.methods.get('drop') +# if method is None: +# raise NotImplementedError() +# +# return method(name) diff --git a/requirements.txt b/requirements.txt index eb8d5b6..f6f24fb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ requests pandas >= 1.3.5 -mindsdb-sql >= 0.7.0, < 0.8.0 +mindsdb-sql >= 0.7.0, < 0.11.0 diff --git a/tests/test_sdk.py b/tests/test_sdk.py index f1013f1..702242b 100644 --- a/tests/test_sdk.py +++ b/tests/test_sdk.py @@ -126,16 +126,16 @@ def check_model(self, model, database, mock_post): f'Finetune {model.project.name}.{model_name} FROM d1 (select a from t1)' ) - model.retrain(query, options={ 'x': 2 }) + model.retrain(query, options={'x': 2}) check_sql_call( mock_post, f'RETRAIN {model.project.name}.{model_name} FROM {query.database} ({query.sql}) USING x=2' ) - model.retrain('select a from t1', database='d1') + model.retrain('select a from t1', database='d1', engine='openai') check_sql_call( mock_post, - f'RETRAIN {model.project.name}.{model_name} FROM d1 (select a from t1)' + f'RETRAIN {model.project.name}.{model_name} FROM d1 (select a from t1) USING engine=\'openai\'' ) # describe @@ -178,17 +178,29 @@ def check_table(self, table, mock_post): class Test(BaseFlow): + @patch('requests.Session.get') @patch('requests.Session.put') @patch('requests.Session.post') - def test_flow(self, mock_post, mock_put): + def test_flow(self, mock_post, mock_put, mock_get): + + # check local + server = mindsdb_sdk.connect() + str(server) + + assert server.api.url == 'http://127.0.0.1:47334' + # check cloud login server = mindsdb_sdk.connect(login='a@b.com') - # check login call_args = mock_post.call_args assert call_args[0][0] == 'https://cloud.mindsdb.com/cloud/login' assert call_args[1]['json']['email'] == 'a@b.com' + # server status + server.status() + call_args = mock_get.call_args + assert call_args[0][0] == 'https://cloud.mindsdb.com/api/status' + # --------- databases ------------- response_mock(mock_post, pd.DataFrame([{'NAME': 'db1'}])) @@ -197,6 +209,7 @@ def test_flow(self, mock_post, mock_put): check_sql_call(mock_post, "select NAME from information_schema.databases where TYPE='data'") database = databases[0] + str(database) assert database.name == 'db1' self.check_database(database) @@ -364,6 +377,7 @@ def check_project_models(self, project, database, mock_post): f'CREATE PREDICTOR m2 FROM example_db (select * from t1) PREDICT price ORDER BY date GROUP BY a, b WINDOW 10 HORIZON 2 USING module="LightGBM", `engine`="lightwood"' ) assert model.name == 'm2' + model.wait_complete() self.check_model(model, database) # create, using deferred query. @@ -373,6 +387,7 @@ def check_project_models(self, project, database, mock_post): predict='price', query=query, ) + str(query) check_sql_call( mock_post, @@ -599,6 +614,7 @@ def test_flow(self, mock_post, mock_put): self.check_project(project, database) project = con.projects.create('proj1') + str(project) check_sql_call( mock_post, 'CREATE DATABASE proj1 WITH ENGINE = "mindsdb", PARAMETERS = {}') self.check_project(project, database) @@ -670,9 +686,9 @@ def test_flow(self, mock_post, mock_put): con.ml_engines.create( 'openai1', openai_handler, - connection_data={'api_key': '111'} + connection_data={'api_key': 'sk-11'} ) - check_sql_call(mock_post, 'CREATE ML_ENGINE openai1 FROM openai USING api_key = "111"') + check_sql_call(mock_post, 'CREATE ML_ENGINE openai1 FROM openai USING api_key = \'sk-11\'') con.ml_engines.create( 'openai1', @@ -684,6 +700,27 @@ def test_flow(self, mock_post, mock_put): con.ml_engines.drop('openai1') check_sql_call(mock_post, 'DROP ML_ENGINE openai1') + # byom + model = ''' +import pandas as pd + +class CustomPredictor(): + + def train(self, df, target_col, args=None): + + self.target_col=target_col + + def predict(self, df): + return pd.Dataframe([{'predict': self.target_col}]) +''' + requirements = '''pandas''' + + con.ml_engines.create_byom('b1', model, requirements) + call_args = mock_put.call_args + assert call_args[0][0] == 'https://cloud.mindsdb.com/api/handlers/byom/b1' + assert call_args[1]['files']['code'] == model + assert call_args[1]['files']['modules'] == requirements + def check_project(self, project, database): self.check_project_views( project, database) @@ -755,6 +792,7 @@ def check_project_models(self, project, database, mock_post): models = project.models.list() model = models[0] # Model object + str(model) assert model.name == 'm1' assert model.get_status() == 'complete' @@ -784,7 +822,7 @@ def check_project_models(self, project, database, mock_post): 'window': 10, 'horizon': 2 }, - module = 'LightGBM', # has to be in options + module='LightGBM', # has to be in options ) check_sql_call( mock_post, @@ -904,6 +942,11 @@ def check_database(self, database, mock_post): table2 = database.tables.create('t2', query) check_sql_call(mock_post, f'create table {database.name}.t2 (select * from tbl1)') + # create with replace + database.tables.create('t2', query, replace=True) + check_sql_call(mock_post, f'create or replace table {database.name}.t2 (select * from tbl1)') + + assert table2.name == 't2' self.check_table(table2) @@ -962,7 +1005,9 @@ def check_project_jobs(self, project, mock_post): assert job.name == 'job1' assert job.query_str == 'select 1' + dir(project.jobs) job = project.jobs.job1 + str(job) assert job.name == 'job1' assert job.query_str == 'select 1' @@ -972,6 +1017,13 @@ def check_project_jobs(self, project, mock_post): f"select * from jobs where name = 'job1'" ) + job.get_history() + + check_sql_call( + mock_post, + f"select * from jobs_history where name = 'job1'" + ) + project.jobs.create( name='job2', query_str='retrain m1', @@ -986,6 +1038,17 @@ def check_project_jobs(self, project, mock_post): call_stack_num=-2 ) + project.jobs.create( + name='job2', + query_str='retrain m1' + ) + + check_sql_call( + mock_post, + f"CREATE JOB job2 (retrain m1)", + call_stack_num=-2 + ) + project.jobs.drop('job2') check_sql_call(