Chatbot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import openai\n",
    "import sys\n",
    "sys.path.append('../..')\n",
    "\n",
    "from dotenv import load_dotenv, find_dotenv\n",
    "_=load_dotenv(find_dotenv())\n",
    "openai.api_key  = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.document_loaders import PyPDFLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pypdf in ./.venv/lib/python3.9/site-packages (4.2.0)\n",
      "Requirement already satisfied: typing_extensions>=4.0 in ./.venv/lib/python3.9/site-packages (from pypdf) (4.11.0)\n",
      "\u001b[33mWARNING: You are using pip version 22.0.4; however, version 24.0 is available.\n",
      "You should consider upgrading via the '/home/rj23/Desktop/LangChain/.venv/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# The course will show the pip installs you would need to install packages on your own machine.\n",
    "# These packages are already installed on this platform and should not be run again.\n",
    "! pip install pypdf "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.document_loaders import PyPDFLoader\n",
    "\n",
    "# List of file paths\n",
    "file_paths = [\n",
    "    \"Insurance_Documents/about-our-insurance-services.pdf\",\n",
    "    \"Insurance_Documents/critical-illness-cover-policy-document-and-summary-of-cover.pdf\",\n",
    "    \"Insurance_Documents/hsbc-life-privacy-notice-overview.pdf\",\n",
    "    \"Insurance_Documents/life-cover-policy-document-and-summary-of-cover.pdf\"\n",
    "]\n",
    "\n",
    "# Initialize a loader for each file and load pages\n",
    "all_pages = []\n",
    "for file_path in file_paths:\n",
    "    loader = PyPDFLoader(file_path)\n",
    "    pages = loader.load()\n",
    "    all_pages.extend(pages)  # Collect all pages in a single list\n",
    "\n",
    "# all_pages now contains pages from all documents\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "page=pages[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 Contents \n",
      "Policy Summary 3 \n",
      "What is HSBC Life Cover?  3 \n",
      "Can I apply for cover?  3 \n",
      "How lo ng can the policy run for?  4 \n",
      "How much cover can I choose?  4 \n",
      "What type of cover can I apply for?  4 \n",
      "How do I pay for the cover?  4 \n",
      "What does the policy cover?  4 \n",
      "Are there any other features included?  5 \n",
      "Why might my policy not pay out?  5 \n",
      "How do I make a claim?  6 \n",
      "How do I cancel my policy?  6 \n",
      "How do I make a complaint?  7 \n",
      "Compensation 7 \n",
      "Important Information about your policy  8 \n",
      "Tax 8 \n",
      "HSBC Life (UK) Limited  8 \n",
      "Registered Office  9 \n",
      "Your Informa tion 9 \n",
      "Policy Document 10 \n",
      "Introduction  10 \n",
      "Policy Conditions  11 \n",
      "Definitions  20 \n",
      "An overview of h ow we collect and use your information  24 \n"
     ]
    }
   ],
   "source": [
    "print(page.page_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'source': 'Insurance_Documents/life-cover-policy-document-and-summary-of-cover.pdf',\n",
       " 'page': 1}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "page.metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size = 1000,\n",
    "    chunk_overlap = 150\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "splits = text_splitter.split_documents(all_pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "194"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(splits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rj23/Desktop/LangChain/.venv/lib/python3.9/site-packages/langchain_core/_api/deprecation.py:119: LangChainDeprecationWarning: The class `OpenAIEmbeddings` was deprecated in LangChain 0.0.9 and will be removed in 0.2.0. An updated version of the class exists in the langchain-openai package and should be used instead. To use it run `pip install -U langchain-openai` and import as `from langchain_openai import OpenAIEmbeddings`.\n",
      "  warn_deprecated(\n"
     ]
    }
   ],
   "source": [
    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
    "embedding = OpenAIEmbeddings(openai_api_key='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence1 = \"i like dogs\"\n",
    "sentence2 = \"i like canines\"\n",
    "sentence3 = \"the weather is ugly outside\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding1 = embedding.embed_query(sentence1)\n",
    "embedding2 = embedding.embed_query(sentence2)\n",
    "embedding3= embedding.embed_query(sentence3)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9631227500523621"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.dot(embedding1, embedding2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7703257495981697"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.dot(embedding1, embedding3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.759162740110803"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.dot(embedding2, embedding3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vectorstores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.vectorstores import Chroma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "persist_directory = 'Insurance_Documents/chroma/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf ./Insurance_Documents/chroma "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectordb = Chroma.from_documents(\n",
    "    documents=splits,\n",
    "    embedding=embedding,\n",
    "    persist_directory=persist_directory\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "194\n"
     ]
    }
   ],
   "source": [
    "print(vectordb._collection.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "question = \"What did they say about Policy Document?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = vectordb.similarity_search(question,k=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "which you can claim are given in the Policy Document under the Critical Illness Definitions heading  \n",
      "Why might my policy not pay out?  \n",
      "There are certain reasons your policy may not pay out in full or at all. The main reasons you need to be aware of are \n",
      "listed below.  \n",
      "These are fully detailed in Conditions 4 and 5 of the Policy Conditions section of this booklet which you should read \n",
      "caref ully. \n",
      "General Exclusions  \n",
      "There are some exclusions that you need to be aware of where we won’t pay a claim  \n",
      " You are first diagnosed with one of the specified critical illnesses or undergoes one of the specified surgical \n",
      "procedures within 90 days of the sta rt date of the Policy  \n",
      " You do not survive 10 days from either the date of diagnosis of one of the specified critical illnesses or the date of \n",
      "one of the specified surgical procedures (see Condition 1 of the Policy Conditions)\n"
     ]
    }
   ],
   "source": [
    "print(docs[13].page_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'page': 9, 'source': 'Insurance_Documents/life-cover-policy-document-and-summary-of-cover.pdf'}\n",
      "{'page': 10, 'source': 'Insurance_Documents/critical-illness-cover-policy-document-and-summary-of-cover.pdf'}\n",
      "{'page': 10, 'source': 'Insurance_Documents/critical-illness-cover-policy-document-and-summary-of-cover.pdf'}\n",
      "{'page': 9, 'source': 'Insurance_Documents/life-cover-policy-document-and-summary-of-cover.pdf'}\n",
      "{'page': 0, 'source': 'Insurance_Documents/life-cover-policy-document-and-summary-of-cover.pdf'}\n",
      "{'page': 0, 'source': 'Insurance_Documents/hsbc-life-privacy-notice-overview.pdf'}\n",
      "{'page': 13, 'source': 'Insurance_Documents/critical-illness-cover-policy-document-and-summary-of-cover.pdf'}\n",
      "{'page': 17, 'source': 'Insurance_Documents/critical-illness-cover-policy-document-and-summary-of-cover.pdf'}\n",
      "{'page': 2, 'source': 'Insurance_Documents/critical-illness-cover-policy-document-and-summary-of-cover.pdf'}\n",
      "{'page': 13, 'source': 'Insurance_Documents/life-cover-policy-document-and-summary-of-cover.pdf'}\n",
      "{'page': 21, 'source': 'Insurance_Documents/life-cover-policy-document-and-summary-of-cover.pdf'}\n",
      "{'page': 21, 'source': 'Insurance_Documents/life-cover-policy-document-and-summary-of-cover.pdf'}\n",
      "{'page': 15, 'source': 'Insurance_Documents/life-cover-policy-document-and-summary-of-cover.pdf'}\n",
      "{'page': 5, 'source': 'Insurance_Documents/critical-illness-cover-policy-document-and-summary-of-cover.pdf'}\n",
      "{'page': 18, 'source': 'Insurance_Documents/critical-illness-cover-policy-document-and-summary-of-cover.pdf'}\n"
     ]
    }
   ],
   "source": [
    "for doc in docs:\n",
    "    print(doc.metadata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectordb.persist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "194\n"
     ]
    }
   ],
   "source": [
    "print(vectordb._collection.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs_ss= vectordb.similarity_search(\"What is the policy document about?\",k=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'10 Policy Document  \\nIntroduction  \\nImportant  \\nThe terms and conditions of the Policy are set out i'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docs_ss[0].page_content[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\uf075 This Policy Document  \\n\\uf075 Policy Schedule(s)  \\n\\uf075 Any amendments, endorsements or specially agreed t'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docs_ss[1].page_content[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "question=\"What does the poicy cover for critical illness?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = vectordb.similarity_search(\n",
    "    question,\n",
    "    k=3,\n",
    "    filter={\"source\":\"Insurance_Documents/critical-illness-cover-policy-document-and-summary-of-cover.pdf\"})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'page': 11, 'source': 'Insurance_Documents/critical-illness-cover-policy-document-and-summary-of-cover.pdf'}\n",
      "{'page': 2, 'source': 'Insurance_Documents/critical-illness-cover-policy-document-and-summary-of-cover.pdf'}\n",
      "{'page': 2, 'source': 'Insurance_Documents/critical-illness-cover-policy-document-and-summary-of-cover.pdf'}\n"
     ]
    }
   ],
   "source": [
    "for d in docs:\n",
    "    print(d.metadata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.llms import OpenAI\n",
    "from langchain.retrievers.self_query.base import SelfQueryRetriever\n",
    "from langchain.chains.query_constructor.base import AttributeInfo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata_field_info = [\n",
    "    AttributeInfo(\n",
    "        name=\"source\",\n",
    "        description=\"The policy the chunk is from, should be one of `Insurance_Documents/critical-illness-cover-policy-document-and-summary-of-cover.pdf`, `Insurance_Docuuments/life-cover-policy-document-and-summary-of-cover.pdf`, or `about-our-insurance-services.pdf`\",\n",
    "        type=\"string\",\n",
    "    ),\n",
    "    AttributeInfo(\n",
    "        name=\"page\",\n",
    "        description=\"The page from the policy the chunk is from\",\n",
    "        type=\"integer\",\n",
    "    ),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import lark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rj23/Desktop/LangChain/.venv/lib/python3.9/site-packages/langchain_core/_api/deprecation.py:119: LangChainDeprecationWarning: The class `OpenAI` was deprecated in LangChain 0.0.10 and will be removed in 0.2.0. An updated version of the class exists in the langchain-openai package and should be used instead. To use it run `pip install -U langchain-openai` and import as `from langchain_openai import OpenAI`.\n",
      "  warn_deprecated(\n"
     ]
    }
   ],
   "source": [
    "\n",
    "document_content_description = \"A chunk of text from an insurance policy document.\"\n",
    "llm = OpenAI(model='gpt-3.5-turbo-instruct', temperature=0, openai_api_key='')\n",
    "retriever = SelfQueryRetriever.from_llm(\n",
    "    llm,\n",
    "    vectordb,\n",
    "    document_content_description,\n",
    "    metadata_field_info,\n",
    "    verbose=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = retriever.get_relevant_documents(question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'page': 11, 'source': 'Insurance_Documents/critical-illness-cover-policy-document-and-summary-of-cover.pdf'}\n",
      "{'page': 26, 'source': 'Insurance_Documents/critical-illness-cover-policy-document-and-summary-of-cover.pdf'}\n",
      "{'page': 23, 'source': 'Insurance_Documents/critical-illness-cover-policy-document-and-summary-of-cover.pdf'}\n",
      "{'page': 5, 'source': 'Insurance_Documents/critical-illness-cover-policy-document-and-summary-of-cover.pdf'}\n"
     ]
    }
   ],
   "source": [
    "for d in docs:\n",
    "    print(d.metadata)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "COMPRESSION\n",
    "\n",
    "Another approach for improving the quality of retrieved docs is compression.\n",
    "\n",
    "Information most relevant to a query may be buried in a document with a lot of irrelevant text.\n",
    "\n",
    "Passing that full document through your application can lead to more expensive LLM calls and poorer responses.\n",
    "\n",
    "Contextual compression is meant to fix this."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.retrievers import ContextualCompressionRetriever\n",
    "from langchain.retrievers.document_compressors import LLMChainExtractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pretty_print_docs(docs):\n",
    "    print(f\"\\n{'-' * 100}\\n\".join([f\"Document {i+1}:\\n\\n\" + d.page_content for i, d in enumerate(docs)]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Wrap our vectorstore\n",
    "llm = OpenAI(temperature=0, model=\"gpt-3.5-turbo-instruct\", openai_api_key='')\n",
    "compressor = LLMChainExtractor.from_llm(llm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "compression_retriever = ContextualCompressionRetriever(\n",
    "    base_compressor=compressor,\n",
    "    base_retriever=vectordb.as_retriever()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rj23/Desktop/LangChain/.venv/lib/python3.9/site-packages/langchain/chains/llm.py:316: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
      "  warnings.warn(\n",
      "/home/rj23/Desktop/LangChain/.venv/lib/python3.9/site-packages/langchain/chains/llm.py:316: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
      "  warnings.warn(\n",
      "/home/rj23/Desktop/LangChain/.venv/lib/python3.9/site-packages/langchain/chains/llm.py:316: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
      "  warnings.warn(\n",
      "/home/rj23/Desktop/LangChain/.venv/lib/python3.9/site-packages/langchain/chains/llm.py:316: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Document 1:\n",
      "\n",
      "- Critical illness cover (Life/Lives Insured)\n",
      "- the benefit will become payable\n",
      "- if after the start date of the Policy, the Life/Lives Insured is Diagnosed with one or more of the specified critical illnesses detailed in the Critical Illness Definitions section or has undergone one of the specified surgical procedures detailed in the Critical Illness Definitions and survives for a period of 10 days from either the date of Diagnosis or from the date of the specified surgical procedure.\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Document 2:\n",
      "\n",
      "- Your online Life Cover Policy can continue if you cancel your Critical Illness Policy.\n",
      "- Your Critical Illness Policy must not exceed the Sum Insured or term on your online Life Policy.\n",
      "- What is HSBC Critical Illness Cover?\n",
      "- HSBC Critical Illness pays a lump sum is paid if you:\n",
      "-  Suffer a critical Illness, or\n",
      "-  Undergo a surgical procedure covered\n",
      "- before the policy end date\n",
      "- You can take out a policy to cover one person ( single life ) or two people ( joint life ).\n",
      "- This p olicy has no cash in value at any time\n",
      "-  When your policy reaches the chosen end date, your cover stops and no further premiums will be taken\n",
      "-  If no claim is made before the policy ends, no benefit will be payable and we will not refund any premiums you\n",
      "- have paid\n",
      "- Who can apply for cover?\n",
      "- You can apply if you:\n",
      "-  Are aged between 17 and 69 , and\n",
      "-  Resident within the UK. (The UK does not include the Channel Islands and Isle of Man)\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Document 3:\n",
      "\n",
      "Critical Illness  – payment of the sum assured  \n",
      " If you are diagnosed with a Critical Illness or undergo a surgical procedure (see full list below)  \n",
      "Children’s Critical illness  – payment of £25,000 or 50% of the Sum Insured (whichever is lower)  \n",
      " If any of your children (including natural , step or legally adopted) are diagnosed with one of the critical illnesses or \n",
      "undergo a surgical procedure (see full list below)  \n",
      "What illnesses and surgical procedures are covered under HSBC Critical Illness? \n",
      "The complete list of Critical Illness conditio ns and surgical procedures we cover are set out below:  \n",
      " Aorta graft surgery  – for disease  \n",
      " Bacterial meningitis  – resulting in permanent symptoms  \n",
      " Benign Brain Tumour  – resulting in permanent symptoms or surgical removal  \n",
      " Benign spinal cord tumour – resulting in permanent symptoms\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Document 4:\n",
      "\n",
      "This is a summary of the Critical Illness Cover policy for Personal Protection (“Policy”) and does not contain the full \n",
      "policy conditions. The full policy conditions, including exclusions and defined terms, can be found in the Policy \n",
      "Document section of th is booklet.  \n",
      "Only for purchase alongside Life Cover as an add -on online  \n",
      "You can only apply online for this Critical Illness cover, when purchasing Life Cover online .\n"
     ]
    }
   ],
   "source": [
    "question = \"what does the policy cover for critical illness?\"\n",
    "compressed_docs = compression_retriever.get_relevant_documents(question)\n",
    "pretty_print_docs(compressed_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "compression_retriever = ContextualCompressionRetriever(\n",
    "    base_compressor=compressor,\n",
    "    base_retriever=vectordb.as_retriever(search_type = \"mmr\")\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rj23/Desktop/LangChain/.venv/lib/python3.9/site-packages/langchain/chains/llm.py:316: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
      "  warnings.warn(\n",
      "/home/rj23/Desktop/LangChain/.venv/lib/python3.9/site-packages/langchain/chains/llm.py:316: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
      "  warnings.warn(\n",
      "/home/rj23/Desktop/LangChain/.venv/lib/python3.9/site-packages/langchain/chains/llm.py:316: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
      "  warnings.warn(\n",
      "/home/rj23/Desktop/LangChain/.venv/lib/python3.9/site-packages/langchain/chains/llm.py:316: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Document 1:\n",
      "\n",
      "- Critical illness cover (Life/Lives Insured)\n",
      "- the benefit will become payable\n",
      "- the Life/Lives Insured is Diagnosed with one or more of the specified critical illnesses detailed in the Critical Illness Definitions section\n",
      "- has undergone one of the specified surgical procedures detailed in the Critical Illness Definitions\n",
      "- survives for a period of 10 days from either the date of Diagnosis or from the date of the specified surgical procedure\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Document 2:\n",
      "\n",
      "Critical Illness  – payment of the sum assured  \n",
      " If you are diagnosed with a Critical Illness or undergo a surgical procedure (see full list below)  \n",
      "Children’s Critical illness  – payment of £25,000 or 50% of the Sum Insured (whichever is lower)  \n",
      " If any of your children (including natural , step or legally adopted) are diagnosed with one of the critical illnesses or \n",
      "undergo a surgical procedure (see full list below)  \n",
      "What illnesses and surgical procedures are covered under HSBC Critical Illness? \n",
      "The complete list of Critical Illness conditio ns and surgical procedures we cover are set out below:  \n",
      " Aorta graft surgery  – for disease  \n",
      " Bacterial meningitis  – resulting in permanent symptoms  \n",
      " Benign Brain Tumour  – resulting in permanent symptoms or surgical removal  \n",
      " Benign spinal cord tumour – resulting in permanent symptoms\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Document 3:\n",
      "\n",
      "HSBC Critical Illness Cover\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Document 4:\n",
      "\n",
      "The first section lists the Critical Illnesses covered under the Policy and any exclusions.\n"
     ]
    }
   ],
   "source": [
    "question = \"what does the policy cover for critical illness??\"\n",
    "compressed_docs = compression_retriever.get_relevant_documents(question)\n",
    "pretty_print_docs(compressed_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "194\n"
     ]
    }
   ],
   "source": [
    "print(vectordb._collection.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "question = \"What are the types of illnesses covered?\"\n",
    "docs = vectordb.similarity_search(question,k=3)\n",
    "len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gpt-3.5-turbo\n"
     ]
    }
   ],
   "source": [
    "import datetime\n",
    "current_date = datetime.datetime.now().date()\n",
    "if current_date < datetime.date(2023, 9, 2):\n",
    "    llm_name = \"gpt-3.5-turbo-0301\"\n",
    "else:\n",
    "    llm_name = \"gpt-3.5-turbo\"\n",
    "print(llm_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.chat_models import ChatOpenAI\n",
    "llm = ChatOpenAI(model_name=llm_name, temperature=0, openai_api_key='')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retrieval QA Chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.chains import RetrievalQA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "qa_chain = RetrievalQA.from_chain_type(\n",
    "    llm,\n",
    "    retriever=vectordb.as_retriever() # Use the vectorstore as the retriever\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rj23/Desktop/LangChain/.venv/lib/python3.9/site-packages/langchain_core/_api/deprecation.py:119: LangChainDeprecationWarning: The method `Chain.__call__` was deprecated in langchain 0.1.0 and will be removed in 0.2.0. Use invoke instead.\n",
      "  warn_deprecated(\n"
     ]
    }
   ],
   "source": [
    "result = qa_chain({\"query\": question})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The types of illnesses covered under the HSBC Critical Illness policy include Aorta graft surgery for disease, Bacterial meningitis resulting in permanent symptoms, Benign Brain Tumour resulting in permanent symptoms or surgical removal, and Benign spinal cord tumour resulting in permanent symptoms.'"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result[\"result\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.prompts import PromptTemplate\n",
    "\n",
    "# Build prompt\n",
    "template = \"\"\"Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer. Use three sentences maximum. Keep the answer as concise as possible. Always say \"thanks for asking!\" at the end of the answer. \n",
    "{context}\n",
    "Question: {question}\n",
    "Helpful Answer:\"\"\"\n",
    "QA_CHAIN_PROMPT = PromptTemplate.from_template(template)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run chain\n",
    "qa_chain = RetrievalQA.from_chain_type(\n",
    "    llm,\n",
    "    retriever=vectordb.as_retriever(),\n",
    "    return_source_documents=True,\n",
    "    chain_type_kwargs={\"prompt\": QA_CHAIN_PROMPT}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "question = \"What comes under serious illness?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = qa_chain({\"query\": question})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Serious illnesses covered under the policy include heart valve replacement or repair, kidney failure requiring dialysis, loss of a hand or foot, loss of speech, aorta graft surgery for disease, and bacterial meningitis resulting in permanent symptoms. Thanks for asking!'"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result[\"result\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Document(page_content='6 \\uf075 Pulmonary artery surgery  – with surgery to divide the breastbone  \\n\\uf075 Removal of an eyeball  \\n\\uf075 Spinal stroke  – resulting in permanent symptoms  \\n\\uf075 Stroke  – resulting in permanent symptoms  \\n\\uf075 Terminal illness  \\n\\uf075 Third degree burns  – covering 20% of the body’s surface area  \\n\\uf075 Traumatic Brain injury  – resulting in permanent symptoms  \\nThese headings are only a guide to what is covered by the Policy.  \\nThese typically use medical terms to describe the illnesses but in some cases the cover may be limited. For example:  \\n\\uf075 Some types of cancer are not covered; and  \\nTo make a claim for some critical illnesses you have to have permanent or persisting symptoms  \\nFull details of these benefits are set out in the Policy Document section of this booklet.  \\nThe full definitions of the specified critical illnesses and surgical procedures  covered and the circumstances in \\nwhich you can claim are given in the Policy Document under the Critical Illness Definitions heading', metadata={'page': 5, 'source': 'Insurance_Documents/critical-illness-cover-policy-document-and-summary-of-cover.pdf'})"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result[\"source_documents\"][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrieval QA Chain Types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "qa_chain_mr = RetrievalQA.from_chain_type(\n",
    "    llm,\n",
    "    retriever=vectordb.as_retriever(),\n",
    "    chain_type=\"map_reduce\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = qa_chain_mr({\"query\": question})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Serious illnesses that may be covered under a policy typically include conditions such as heart attack, stroke, cancer, organ transplant, and other severe medical conditions. The specific list of serious illnesses covered can vary depending on the insurance policy. For full details on what is considered a serious illness under a specific policy, it is recommended to refer to the Policy Document section of the insurance booklet.'"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result[\"result\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
    "os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.langchain.plus\"\n",
    "os.environ[\"LANGCHAIN_API_KEY\"] = \"ls__2de7c8d7ec664712a6123125c100f9c5\" # replace dots with your api key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Serious illnesses typically include conditions such as cancer, heart attack, stroke, organ failure, and other life-threatening diseases or injuries. It may also encompass chronic conditions that significantly impact a person's quality of life or require ongoing medical treatment. The specific list of serious illnesses covered will be outlined in the Policy Document under the Critical Illness Definitions heading. It is important to refer to the policy details for a comprehensive understanding of what qualifies as a serious illness for the purposes of making a claim.\""
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qa_chain_mr = RetrievalQA.from_chain_type(\n",
    "    llm,\n",
    "    retriever=vectordb.as_retriever(),\n",
    "    chain_type=\"map_reduce\"\n",
    ")\n",
    "result = qa_chain_mr({\"query\": question})\n",
    "result[\"result\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://smith.langchain.com/o/65d7e76c-a9dd-5c5b-b6af-2222bbd5a7ee/projects/p/ad5254ac-6c02-4131-8bab-ea947449ce3e/r/edbfe093-14cd-47e7-bb5c-9b0ed4f685ac?trace_id=92e144fa-b970-486c-96bc-1a3482a0728c&start_time=2024-04-21T22:29:20.973000\n"
     ]
    }
   ],
   "source": [
    "from langsmith import Client\n",
    "\n",
    "client = Client()\n",
    "\n",
    "url = next(client.list_runs(project_name=\"default\")).url\n",
    "print(url)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'panel'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m/home/rj23/Desktop/LangChain/loaders.ipynb Cell 75\u001b[0m line \u001b[0;36m1\n\u001b[0;32m----> <a href='vscode-notebook-cell:/home/rj23/Desktop/LangChain/loaders.ipynb#Y221sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mpanel\u001b[39;00m \u001b[39mas\u001b[39;00m \u001b[39mpn\u001b[39;00m  \u001b[39m# GUI\u001b[39;00m\n\u001b[1;32m      <a href='vscode-notebook-cell:/home/rj23/Desktop/LangChain/loaders.ipynb#Y221sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m pn\u001b[39m.\u001b[39mextension()\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'panel'"
     ]
    }
   ],
   "source": [
    "import panel as pn  # GUI\n",
    "pn.extension()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}