Merge pull request #102 from md-abid-hussain/kb

Implemented set of functions for Knowledge Bases

src/index.ts

@@ -12,6 +12,7 @@ import {
   retryUnauthenticatedRequest,
 } from './util/http';
 import TablesModule from './tables/tablesModule';
+import knowledgeBaseModule from './knowledge_bases/knowledge_baseModule';
 import AgentsModule from './agents/agentsModule';
 import skillsModule from './skills/skillsModule';
 import HttpAuthenticator from './httpAuthenticator';
@@ -61,8 +62,13 @@ const MLEngines = new MLEnginesModule.MLEnginesRestApiClient(
   defaultAxiosInstance,
   httpAuthenticator
 );
+const KnowledgeBases = new knowledgeBaseModule.KnowledgeBaseRestApiClient(
+  SQL,
+  defaultAxiosInstance
+);
 const Agents = new AgentsModule.AgentsRestApiClient(defaultAxiosInstance);
 const Skills = new skillsModule.SkillsRestApiClient(defaultAxiosInstance);
+
 const Callbacks = new CallbacksModule.CallbacksRestApiClient(
   defaultAxiosInstance,
   httpAuthenticator
@@ -92,6 +98,7 @@ const connect = async function (options: ConnectionOptions): Promise<void> {
   Projects.client = httpClient;
   MLEngines.client = httpClient;
   Callbacks.client = httpClient;
+  KnowledgeBases.client = httpClient;
 
   if (options.logging) {
     const logger = new Logger(options.logging.logger, options.logging.logLevel);
@@ -127,6 +134,7 @@ export default {
   MLEngines,
   Agents,
   Callbacks,
+  KnowledgeBases,
   Skills,
 };
 export {

src/knowledge_bases/knowledge_base.ts

@@ -0,0 +1,202 @@
+import SqlQueryResult from '../sql/sqlQueryResult';
+import KnowledgeBaseApiClient from './knowledge_baseApiClient';
+
+/**
+ * Parameters for configuring a Knowledge Base.
+ * 
+ * @property {Array<string>} [metadata_columns] - Optional array of metadata column names.
+ * @property {Array<string>} [content_columns] - Optional array of content column names.
+ * @property {string | null} [id_column] - Optional ID column name, can be null.
+ */
+export type KnowledgeBaseParams = {
+  metadata_columns?: Array<string>;
+  content_columns?: Array<string>;
+  id_column?: string | null;
+}
+
+/**
+ * Represent MindsDB Knowledge Base and all supported operations.
+ */
+export default class KnowledgeBase {
+  /** API client for executing Knowledge Base operations   */
+  knowledgeBaseApiClient: KnowledgeBaseApiClient;
+  /** Name of Knowledge Base */
+  name: string;
+  /** Project name in which knowledge base belongs */
+  project: string;
+  /** Table name of knowledge base */
+  tableName: string;
+  /** Storage name of knowledge base */
+  storage: string | null;
+  /** Name of Embedding model used */
+  model: string | null;
+  /** Params for knowledge base in JSON Object */
+  params: KnowledgeBaseParams;
+  /** Metadata columns name */
+  metadataColumns: Array<string>;
+  /** Content column names (default content) */
+  contentColumns: Array<string>;
+  /** ID column name */
+  idColumn: string | null;
+  /** Query string for knowledge base */
+  query: string | null;
+  /** Value to limit the knowledge base output result */
+  limit: number | null | undefined;
+  /** SQL query for knowledge base */
+  sql: string;
+
+  /**
+   * Constructor for Knowledge Base
+   *
+   * @param {KnowledgeBaseApiClient} knowledgeBaseApiClient - API client for executing Knowledge Base operations
+   * @param {string} project - Project name in which knowledge base belongs
+   * @param {unknown} data - Knowledge Base data in JSON Object
+   */
+  constructor(
+    knowledgeBaseApiClient: KnowledgeBaseApiClient,
+    project: string,
+    data: {
+      name: string;
+      storage: string | null;
+      model: string | null;
+      params: KnowledgeBaseParams | string;
+    }
+  ) {
+    this.knowledgeBaseApiClient = knowledgeBaseApiClient;
+    this.project = project;
+    this.name = data.name;
+    this.tableName = `${project}.${this.name}`;
+    this.storage = data.storage || null;
+    this.model = data.model || null;
+    let params = data.params || {};
+    let kbParams: KnowledgeBaseParams = {}
+    if (typeof params === 'string') {
+      try {
+        kbParams = JSON.parse(params);
+      } catch (error) {
+        kbParams = {};
+      }
+    }
+
+    this.metadataColumns = kbParams['metadata_columns'] || [];
+    this.contentColumns = kbParams['content_columns'] || [];
+    this.idColumn = kbParams['id_column'] || null;
+
+    this.params = kbParams;
+
+    this.query = null;
+    this.limit = null;
+    this.sql = `SELECT * FROM ${this.tableName};`;
+    this.updateQuery();
+  }
+
+  // Private method to generate SQL query for knowledge base
+  private updateQuery() {
+    let astQuery: string = `SELECT * FROM ${this.tableName}`;
+    if (this.query) {
+      astQuery += ` WHERE CONTENT = '${this.query}'`;
+    }
+
+    if (this.limit) {
+      astQuery += ` LIMIT ${this.limit}`;
+    }
+
+    astQuery += ';';
+    this.sql = astQuery;
+  }
+
+  /**
+   * Insert Web Pages into Knowledge Base
+   *
+   * @param {Array<string>} urls - Array of URLs
+   * @param {number} crawlDepth - Depth of crawl
+   * @param {Array<string>} filters - Array of filters
+   */
+  async insertWebPages(
+    urls: Array<string>,
+    crawlDepth: number = 1,
+    filters: Array<string> | null = null
+  ) {
+    await this.knowledgeBaseApiClient.insertWebpagesIntoKnowledgeBase(
+      this.project,
+      this.name,
+      urls,
+      crawlDepth,
+      filters
+    );
+  }
+
+  /**
+   * Query Knowledge Base for given query and limit
+   *
+   * @param {string} query - Query string (e.g. 'apple')
+   * @param {number} limit - Limit the output result
+   * @returns {KnowledgeBase} - New Knowledge Base object
+   */
+  find(query: string, limit: number | undefined | null): KnowledgeBase {
+    const clonedKnowledgeBase = new KnowledgeBase(
+      this.knowledgeBaseApiClient,
+      this.project,
+      JSON.parse(
+        JSON.stringify({
+          name: this.name,
+          storage: this.storage,
+          model: this.model,
+          params: this.params,
+        })
+      )
+    );
+    clonedKnowledgeBase.query = query;
+    clonedKnowledgeBase.limit = limit;
+    clonedKnowledgeBase.updateQuery();
+    return clonedKnowledgeBase;
+  }
+
+  /**
+   * Insert data into Knowledge Base
+   * kb.insert([{column1: value1, column2: value2}]);
+   *
+   * @param {Array<unknown>} records - Array of JSON object
+   */
+  async insert(records: Array<{
+    [key: string]: unknown;
+  }>) {
+    let valueString = '';
+    records.forEach((row) => {
+      valueString += '(';
+      Object.values(row).forEach((cellValue, index, array) => {
+        if (typeof cellValue === 'string') {
+          // Escape single quotes
+          cellValue = cellValue.replace(/'/g, "\\'");
+          // Escape double quotes
+          cellValue = (cellValue as string).replace(/"/g, '\\"');
+          valueString += `'${cellValue}'`;
+        } else {
+          valueString += cellValue;
+        }
+        if (index < array.length - 1) {
+          valueString += ', ';
+        }
+      });
+      valueString += '), ';
+    });
+
+    // Remove the trailing comma and space
+    valueString = valueString.slice(0, -2);
+
+    const columnNames = Object.keys(records[0]).join(', ');
+
+    const sqlQuery = `INSERT INTO ${this.tableName} (${columnNames}) VALUES ${valueString};`;
+    console.log(sqlQuery);
+    await this.knowledgeBaseApiClient.insertDataIntoKnowledgeBase(sqlQuery);
+  }
+
+  /**
+   * Fetch data from Knowledge Base
+   *
+   * @returns {SqlQueryResult} - Result of SQL query
+   */
+  async fetch(): Promise<SqlQueryResult> {
+    return await this.knowledgeBaseApiClient.fetch(this.sql);
+  }
+}

src/knowledge_bases/knowledge_baseApiClient.ts

@@ -0,0 +1,85 @@
+import SqlQueryResult from '../sql/sqlQueryResult';
+import KnowledgeBase from './knowledge_base';
+
+/** Abstract class outlining Knowledge Base API operations supported by the SDK. */
+export default abstract class KnowledgeBaseApiClient {
+  /**
+   * Gets all knowledge bases for a project.
+   *
+   * @param {string} project  - Project name in which knowledge base belongs
+   */
+  abstract getAllKnowledgeBase(project: string): Promise<Array<KnowledgeBase>>;
+
+  /**
+   * Gets a knowledge base by name for a project.
+   *
+   * @param {string} name  - Name of the knowledge base
+   * @param {string} project - Project name in which knowledge base belongs
+   */
+  abstract getKnowledgeBase(
+    name: string,
+    project: string
+  ): Promise<KnowledgeBase>;
+
+  /**
+   * Creates a knowledge base with the given name, project, model, storage, metadataColumns, contentColumns, idColumn, and params.
+   *
+   * @param {string} name - Name of the knowledge base to be created
+   * @param {string} project - Project name in which knowledge base belongs
+   * @param {string} model - Name of the embedding model used (optional)
+   * @param {string} storage - Name of the storage used (optional)
+   * @param {Array<string>} metadataColumns - Metadata columns name (optional)
+   * @param {Array<string>} contentColumns - Content column names (default content) (optional)
+   * @param {string} idColumn - ID column name (optional)
+   * @param {unknown} params - Params for knowledge base in JSON Object (optional)
+   */
+  abstract createKnowledgeBase(
+    name: string,
+    project: string,
+    model?: string,
+    storage?: string,
+    metadataColumns?: Array<string>,
+    contentColumns?: Array<string>,
+    idColumn?: string,
+    params?: unknown
+  ): Promise<KnowledgeBase>;
+
+  /**
+   * Deletes a knowledge base by name for a project.
+   *
+   * @param {string} name - Name of the knowledge base to be deleted
+   * @param {string} project - Project name in which knowledge base belongs
+   */
+  abstract deleteKnowledgeBase(name: string, project: string): Promise<void>;
+
+  /**
+   * Inserts webpages into a knowledge base.
+   *
+   * @param {string} project - Project name in which knowledge base belongs
+   * @param {string} knowledgeBaseName - Name of the knowledge base
+   * @param {Array<string>} urls - List of URLs to be inserted
+   * @param {number} crawlDepth - Depth of the crawl
+   * @param {Array<string> | null} filters - Filters to be applied during the crawl (optional)
+   */
+  abstract insertWebpagesIntoKnowledgeBase(
+    project: string,
+    knowledgeBaseName: string,
+    urls: Array<string>,
+    crawlDepth: number,
+    filters: Array<string> | null
+  ): Promise<void>;
+
+  /**
+   * Inserts data into a knowledge base using an SQL query.
+   *
+   * @param {string} sqlQuery - The SQL query to insert data into the knowledge base
+   */
+  abstract insertDataIntoKnowledgeBase(sqlQuery: string): Promise<void>;
+
+  /**
+   * Fetches data from a knowledge base using an SQL query.
+   *
+   * @param {string} sqlQuery - The SQL query to fetch data from the knowledge base
+   */
+  abstract fetch(sqlQuery: string): Promise<SqlQueryResult>;
+}

src/knowledge_bases/knowledge_baseModule.ts

@@ -0,0 +1,3 @@
+import KnowledgeBaseRestApiClient from './knowledge_baseRestApiClient';
+
+export default { KnowledgeBaseRestApiClient: KnowledgeBaseRestApiClient };