Merge pull request #137 from mindsdb/staging

Release 3.0.1

mindsdb_sdk/__about__.py

@@ -1,6 +1,6 @@
 __title__ = 'mindsdb_sdk'
 __package_name__ = 'mindsdb_sdk'
-__version__ = '3.0.0'
+__version__ = '3.0.1'
 __description__ = "MindsDB Python SDK, provides an SDK to use a remote mindsdb instance"
 __email__ = "[email protected]"
 __author__ = 'MindsDB Inc'

mindsdb_sdk/agents.py

@@ -245,7 +245,7 @@ def add_files(self, name: str, file_paths: List[str], description: str, knowledg
         Add a list of files to the agent for retrieval.
 
         :param name: Name of the agent
-        :param file_paths: List of paths to the files to be added.
+        :param file_paths: List of paths or URLs to the files to be added.
         :param description: Description of the file. Used by agent to know when to do retrieval
         :param knowledge_base: Name of an existing knowledge base to be used. Will create a default knowledge base if not given.
         """

mindsdb_sdk/connectors/rest_api.py

@@ -5,6 +5,7 @@
 
 import requests
 import pandas as pd
+import validators
 
 from mindsdb_sdk import __about__
 from sseclient import SSEClient
@@ -149,6 +150,7 @@ def read_file_as_bytes(file_path: str):
             raise Exception(f'Permission denied when reading file {file_path}.')
         except Exception as e:
             raise Exception(f'Unknown error occurred when reading file {file_path} - {str(e)}')
+
     @staticmethod
     def read_dataframe_as_csv(data: pd.DataFrame):
         """
@@ -161,6 +163,16 @@ def read_dataframe_as_csv(data: pd.DataFrame):
         fd.seek(0)
         return fd.read()
 
+    @staticmethod
+    def read_file_as_webpage(url: str):
+        """
+        Read and return content of a file in bytes, given its URL.
+        :param file_path: URL of the file to read.
+        :return: File content in bytes.
+        """
+        data = requests.get(url)
+        return data.content
+
     def upload_data(self, file_name: str, data: bytes):
         """
         Upload binary data to MindsDB.
@@ -194,6 +206,8 @@ def upload_file(self, name: str, data: Union[pd.DataFrame, str]):
         """
         if isinstance(data, pd.DataFrame):
             data_in_bytes = self.read_dataframe_as_csv(data)
+        elif validators.url(data):
+            data_in_bytes = self.read_file_as_webpage(data)
         else:
             data_in_bytes = self.read_file_as_bytes(data)
 

mindsdb_sdk/utils/mind.py

@@ -43,6 +43,17 @@ class DatabaseConfig(DataSourceConfig):
     tables: List[str] = []
 
 
+class FileConfig(DataSourceConfig):
+    """
+    Represents a colection of files that can be made available to a Mind.
+    """
+
+    # Local file paths and/or URLs.
+    paths: List[str] = []
+
+    # TODO: Configure Vector storage. Use defaults for now.
+
+
 # Create mind entity util function
 def create_mind(
         base_url: str,

requirements.txt

@@ -5,3 +5,4 @@ docstring-parser >= 0.7.3
 tenacity >= 8.0.1
 openai >= 1.15.0
 sseclient-py >= 1.8.0
+validators == 0.20.0