chatbot-old.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "89b99453-0ca8-42d7-b45c-2946850c575d",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ed12740a-3931-479e-8176-3ad16b03e35f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "couldn't load  icon.png\n",
      "couldn't load  logo.png\n",
      "couldn't load  login_via_keycloak.png\n",
      "couldn't load  login_with_openshift.png\n",
      "couldn't load  Keycloak_Manageaccount.png\n",
      "couldn't load  Keycloak_ChangePassword.png\n"
     ]
    }
   ],
   "source": [
    "from langchain.docstore.document import Document\n",
    "import os\n",
    "\n",
    "folder_name = \"data\"\n",
    "\n",
    "documents = []\n",
    "for root, dirs, files in os.walk(folder_name):\n",
    "    for file in files:\n",
    "        try:\n",
    "            with open(os.path.join(root, file), \"r\", encoding=\"utf-8\") as o:\n",
    "                code = o.readlines()\n",
    "                d = Document(page_content=\"\\n\".join(code), metadata={\"source\": os.path.join(root, file)})\n",
    "                documents.append(d)\n",
    "        except UnicodeDecodeError:\n",
    "            print(\"couldn't load \", file)\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d7ea1ed4-b1d5-458b-8144-ad3947a80abd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "783"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(documents)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6a17ad8-dc6d-4468-9584-8d6ee91bf380",
   "metadata": {},
   "source": [
    "## Store data in vector db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4135c75a-e4ce-48d2-ba99-a3e48e222786",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.embeddings import HuggingFaceEmbeddings\n",
    "from langchain.vectorstores import Chroma\n",
    "\n",
    "hfemb = HuggingFaceEmbeddings(model_name=\"krlvi/sentence-t5-base-nlpl-code-x-glue\")\n",
    "persist_directory = \"db\"\n",
    "db = Chroma.from_documents(documents, hfemb, persist_directory=persist_directory)\n",
    "db.persist()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df80c9cd-c745-4fce-849c-8d9cd3b91d3d",
   "metadata": {},
   "source": [
    "## Load vector db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "af10c68d-1b02-43c8-acac-af5b52eb9928",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.embeddings import HuggingFaceEmbeddings\n",
    "from langchain.vectorstores import Chroma\n",
    "\n",
    "hfemb = HuggingFaceEmbeddings(model_name=\"krlvi/sentence-t5-base-nlpl-code-x-glue\")\n",
    "\n",
    "persist_directory = \"db\"\n",
    "db = Chroma(persist_directory=persist_directory, embedding_function=hfemb)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3392bf00-9725-4455-b351-c676947f8e0e",
   "metadata": {},
   "source": [
    "## Vector Data retriever"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5b8d8254-917d-4609-a2d4-cbe476b00094",
   "metadata": {},
   "outputs": [],
   "source": [
    "#retriever = db.as_retriever(search_type=\"similarity_score_threshold\", search_kwargs={\"score_threshold\": .2, \"k\": 5})\n",
    "\n",
    "retriever = db.as_retriever()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "468c19aa-fb94-4bdd-b5dd-8af2b063c2dd",
   "metadata": {},
   "source": [
    "## QA/Chat LLM model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b3b967e8-b8fd-4bd5-a775-5f30135fb84a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.chains import ConversationalRetrievalChain\n",
    "from langchain.memory import ConversationSummaryMemory\n",
    "\n",
    "from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler\n",
    "from langchain.llms import GPT4All\n",
    "\n",
    "# Callbacks support token-wise streaming\n",
    "callbacks = [StreamingStdOutCallbackHandler()]\n",
    "\n",
    "# Verbose is required to pass to the callback manager\n",
    "llm = GPT4All(model=\"models/mistral-7b-openorca.Q4_0.gguf\", callbacks=callbacks, verbose=False)\n",
    "\n",
    "#memory = ConversationSummaryMemory(\n",
    "#    llm=llm, memory_key=\"chat_history\", return_messages=True\n",
    "#)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1aa1f77a-b98e-4139-adfd-b742a67b54c1",
   "metadata": {},
   "source": [
    "## Setup chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7f946427-ed0e-4d2f-b63c-9917ccd423ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.chains import ConversationalRetrievalChain\n",
    "\n",
    "qa_chain = ConversationalRetrievalChain.from_llm(llm=llm, retriever=retriever, return_source_documents=True, max_tokens_limit=2046)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "699c6210-8dd2-406b-9a5b-503261e2318a",
   "metadata": {},
   "source": [
    "## Query helper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0b545f94-d030-4224-b094-f6abf34e92a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def query(q):\n",
    "    print(f\"\\nQuestion: {q}\\n\")\n",
    "    result = qa_chain({\"question\":q, \"chat_history\":[]})\n",
    "    #print(f\"Answer: {result['answer']}\")\n",
    "    print(f\"\\n\\nSources\\n{[x.metadata['source'] for x in result['source_documents']]}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3edfbd94-e7c6-47d2-b60d-9aa90017ebf1",
   "metadata": {},
   "source": [
    "## Chat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e133e433-235e-4dc4-90c1-94cf452a0659",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Question: What do you know about argocd-operator?\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (1802 > 1024). Running this sequence through the model will result in indexing errors\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " The Argo CD Operator is a Kubernetes operator that manages the full lifecycle of Argo CD clusters, including installation, upgrading, backing up and restoring, as well as providing insights through Prometheus and Grafana. It aims to automate tasks required when operating an Argo CD cluster and provide easy configuration, seamless upgrades, backups, metrics exposure, and autoscaling capabilities.\n",
      "\n",
      "Sources\n",
      "['data/code/docs/index.md']\n"
     ]
    }
   ],
   "source": [
    "query(\"What do you know about argocd-operator?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "daeb4d88-a611-4874-8d26-81fea9656f95",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Question: What is the source code langauge of argocd-operator?\n",
      "\n",
      " The source code language for Argo CD Operator is Go.\n",
      "\n",
      "Sources\n",
      "[]\n"
     ]
    }
   ],
   "source": [
    "query(\"What is the source code langauge of argocd-operator?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1c455c5b-c606-477b-bc00-0f95709c8ee1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Question: How to use this operator?\n",
      "\n",
      " To use the Argo CD Operator, follow these steps: 1) Install the operator using one of the methods described in the installation guide. This can be done through OpenShift console or manually if desired. 2) Once installed, create new `ArgoCD` resources to manage your applications and deployments\n",
      "\n",
      "Sources\n",
      "['data/code/docs/install/start.md', 'data/code/docs/install/openshift.md', 'data/code/build/bin/entrypoint']\n"
     ]
    }
   ],
   "source": [
    "query(\"How to use this operator?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "972100fc-cbd5-4d6f-b090-55437ce27393",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Question: How to install it on kubernetes?\n",
      "\n",
      " To manually install ArgoCD Operator, follow these steps: 1) Install a Kubernetes cluster using minikube or any other tool; 2) Download the source code from release section; 3) Modify the namespace in `config/default/kustomization.yaml` if needed; 4) Enable conversion webhook support (optional); 5) Deploy the operator using 'make deploy' command.\n",
      "\n",
      "Sources\n",
      "['data/code/docs/install/manual.md']\n"
     ]
    }
   ],
   "source": [
    "query(\"How to install it on kubernetes?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "21116859-f1cc-4598-960e-bb1d751402a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Question: Can you read go.mod file of this project?\n",
      "\n",
      " Yes, you can read a Go.mod file in a project. It is a configuration file used by the Go programming language for managing dependencies and version control within a project.\n",
      "\n",
      "Sources\n",
      "[]\n"
     ]
    }
   ],
   "source": [
    "query(\"Can you read go.mod file of this project?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b72d73a1-9323-468f-83ac-287616c4791a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Question: does argocd-operator source as go.mod file?\n",
      "\n",
      " Yes, it seems that argocd-operator uses a go.mod file to manage its dependencies.\n",
      "\n",
      "Sources\n",
      "['data/issues/462.txt']\n"
     ]
    }
   ],
   "source": [
    "query(\"does argocd-operator source as go.mod file?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e8d1fb1d-4805-42e7-a4a2-c46c4f516f1e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Question: can you read it?\n",
      "\n",
      " Yes, I can read the provided context.\n",
      "\n",
      "Sources\n",
      "['data/code/.readthedocs.yml', 'data/code/config/prometheus/kustomization.yaml', 'data/code/CONTRIBUTING.md', 'data/code/OWNERS']\n"
     ]
    }
   ],
   "source": [
    "query(\"can you read it?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0c86d969-c10b-40b4-9306-16b53e20112b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Question: Great. Tell me the go version in go.mod of argocd-opertator\n",
      "\n",
      " The current go version used by ArgoCD Operator is not mentioned explicitly, but it can be found in the go.mod file under the \"go\" directive.\n",
      "\n",
      "Sources\n",
      "['data/issues/462.txt']\n"
     ]
    }
   ],
   "source": [
    "query(\"Great. Tell me the go version in go.mod of argocd-opertator\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1c9309a2-7e5a-477d-ad0b-5346c175cbe3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Question: Can you read it from 'go' directive and tell me?\n",
      "\n",
      " The 'go' directive is used in programming languages like Python or Ruby to indicate a function should be executed immediately without waiting for any return value. In this case, the text provided does not contain a 'go' directive, so I cannot answer your question based on that information alone.\n",
      "\n",
      "Sources\n",
      "[]\n"
     ]
    }
   ],
   "source": [
    "query(\"Can you read it from 'go' directive and tell me?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "0f3d7f13-49aa-4bb4-b707-dabc75ec9909",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Question: Can you read go version from 'go' directive in argocd-operator's go.mod file?\n",
      "\n",
      " Yes, the 'go' directive in a Go module specifies the minimum required Go version to build and run the code. In this case, it is possible to read the go version from the 'go' directive in argocd-operator's go.mod file.\n",
      "\n",
      "Sources\n",
      "['data/issues/462.txt', 'data/code/Dockerfile']\n"
     ]
    }
   ],
   "source": [
    "query(\"Can you read go version from 'go' directive in argocd-operator's go.mod file?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d2a4905a-dc91-4ad4-92d1-a681c062dec2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Question: Then read and tell me?\n",
      "\n",
      " I'm sorry, but as an AI model, I cannot provide PRs or make changes directly on GitHub. However, if you would like assistance with the content of your proposal, feel free to ask!\n",
      "\n",
      "Sources\n",
      "['data/code/docs/proposals/001-proposal-template.md', 'data/code/.readthedocs.yml', 'data/issues/679.txt']\n"
     ]
    }
   ],
   "source": [
    "query(\"Then read and tell me?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d6287840-9325-46c2-96a2-11641e469953",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Question: Which sso does argocd-operator support?\n",
      "\n",
      " The Argo CD Operator currently supports Red Hat Single Sign-On (RH-SSO) version 7.5 by default, but it can be overridden in the operator group spec to use a different version or provider if needed.\n",
      "\n",
      "Sources\n",
      "['data/issues/274.txt', 'data/issues/153.txt', 'data/code/deploy/operator_group.yaml', 'data/issues/976.txt']\n"
     ]
    }
   ],
   "source": [
    "query(\"Which sso does argocd-operator support?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a9e90b65-11b9-4dd6-b093-000974670b71",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Question: Does it support dex?\n",
      "\n",
      " Yes, the current operator supports Dex authentication with an external Dex server.\n",
      "\n",
      "Sources\n",
      "['data/issues/278.txt']\n"
     ]
    }
   ],
   "source": [
    "query(\"Does it support dex?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "39252a76-a657-4e69-85df-21444b034960",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Question: Give a example of dex with argocd-operator\n",
      "\n",
      " \n",
      "\n",
      "The DEX deployment should be configured with the correct command line arguments. For instance, in your ArgoCD configuration file (specified above), you have set `dex:` to use an image from quay.io/redhat-cop/dex and version v2.22.0-openshift. This is a valid example of DEX with argocd-operator.\n",
      "\n",
      "Sources\n",
      "['data/issues/306.txt']\n"
     ]
    }
   ],
   "source": [
    "query(\"Give a example of dex with argocd-operator\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5bb31629-f4ee-4a23-bc07-a981ea581b9c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Question: What is the latest version of ArgoCD CRD?\n",
      "\n"
     ]
    }
   ],
   "source": [
    "query(\"What is the latest version of ArgoCD CRD?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca4abecc-407a-428c-8ee4-ae60f69e1f14",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}