04_02_RAG-Evaluation-DataSetPrep.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare Evaluation Dataset\n",
    "\n",
    "Source: https://docs.ragas.io/en/latest/getstarted/rag_testset_generation/#choose-your-llm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Install Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install ragas\n",
    "%pip install unstructured\n",
    "%pip install unstructured[pdf]\n",
    "%pip install langchain-openai"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Azure configurations\n",
    "\n",
    "You always need to run this!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dotenv import load_dotenv\n",
    "import os\n",
    "\n",
    "load_dotenv() # take environment variables from .env.\n",
    "\n",
    "azure_openai_endpoint = os.getenv(\"AZURE_OPENAI_ENDPOINT\")\n",
    "azure_openai_key = os.getenv(\"AZURE_OPENAI_KEY\")\n",
    "azure_openai_deployment = os.getenv(\"AZURE_OPENAI_DEPLOYMENT\")\n",
    "azure_openai_embeddings_deployment = os.getenv(\"AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT\")\n",
    "azure_openai_api_version = \"2024-10-01-preview\"\n",
    "azure_search_service_endpoint = os.getenv(\"AZURE_SEARCH_SERVICE_ENDPOINT\")\n",
    "azure_search_service_admin_key = os.getenv(\"AZURE_SEARCH_ADMIN_KEY\")\n",
    "azure_search_service_index_name = \"az-search-index-001\"\n",
    "azure_storage_connection_string = os.getenv(\"AZURE_STORAGE_CONNECTION_STRING\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Documents\n",
    "\n",
    "Note: I had an error loading to files first time so I had to run this in the github codespaces terminal:\n",
    "- sudo apt-get update\n",
    "- sudo apt-get install -y libgl1-mesa-glx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import DirectoryLoader\n",
    "\n",
    "path = \"data/nasa-book/\"\n",
    "loader = DirectoryLoader(path, glob=\"**/*.pdf\")\n",
    "docs = loader.load()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare Language Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_openai import AzureChatOpenAI\n",
    "from langchain_openai import AzureOpenAIEmbeddings\n",
    "from ragas.llms import LangchainLLMWrapper\n",
    "from ragas.embeddings import LangchainEmbeddingsWrapper\n",
    "\n",
    "generator_llm = LangchainLLMWrapper(AzureChatOpenAI(\n",
    "    openai_api_version=azure_openai_api_version,\n",
    "    azure_endpoint=azure_openai_endpoint,\n",
    "    azure_deployment=azure_openai_deployment,\n",
    "    model=azure_openai_deployment,\n",
    "    validate_base_url=False,\n",
    "    api_key=azure_openai_key\n",
    "))\n",
    "\n",
    "# init the embeddings for answer_relevancy, answer_correctness and answer_similarity\n",
    "generator_embeddings = LangchainEmbeddingsWrapper(AzureOpenAIEmbeddings(\n",
    "    openai_api_version=azure_openai_api_version,\n",
    "    azure_endpoint=azure_openai_endpoint,\n",
    "    azure_deployment=azure_openai_embeddings_deployment,\n",
    "    model=azure_openai_embeddings_deployment,\n",
    "    api_key=azure_openai_key\n",
    "))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate the Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ragas.testset import TestsetGenerator\n",
    "\n",
    "generator = TestsetGenerator(llm=generator_llm, embedding_model=generator_embeddings)\n",
    "dataset = generator.generate_with_langchain_docs(docs, testset_size=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save just the query and ground truth to a JSONL file for evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = dataset.to_pandas()\n",
    "\n",
    "# Create DataFrame\n",
    "df = dataset.to_pandas()\n",
    "\n",
    "# Function to clean text\n",
    "def clean_text(text_list):\n",
    "    cleaned_text_list = []\n",
    "    for text in text_list:\n",
    "        # Remove the UUID (assuming it's always at the start and followed by two newlines)\n",
    "        cleaned_text = text.split('\\n\\n', 1)[-1]\n",
    "        cleaned_text_list.append(cleaned_text)\n",
    "    return cleaned_text_list\n",
    "\n",
    "# Apply the function to the 'reference_contexts' column to remove UUID at the start\n",
    "df['reference_contexts'] = df['reference_contexts'].apply(clean_text)\n",
    "\n",
    "# Save to CSV file\n",
    "df.to_csv('data/output/evalset.csv', index=False)\n",
    "\n",
    "# Create a new DataFrame for EvalCollection\n",
    "eval_collection = pd.DataFrame(columns=['query', 'response', 'context', 'ground_truth'])\n",
    "\n",
    "# Populate the new DataFrame\n",
    "eval_collection['query'] = df['user_input']\n",
    "eval_collection['ground_truth'] = df['reference']\n",
    "eval_collection['response'] = ''\n",
    "eval_collection['context'] = ''\n",
    "\n",
    "# Save the DataFrame as a JSONL file\n",
    "eval_collection.to_json('data/output/eval.jsonl', orient='records', lines=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate the Response and Context from the Language Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azure.search.documents import SearchClient\n",
    "from azure.search.documents.models import VectorizableTextQuery\n",
    "from azure.core.credentials import AzureKeyCredential\n",
    "from openai import AzureOpenAI\n",
    "\n",
    "# Get credential from Azure AI Search Admin key\n",
    "credential = AzureKeyCredential(azure_search_service_admin_key)\n",
    "search_client = SearchClient(endpoint=azure_search_service_endpoint, \n",
    "                             credential=credential, \n",
    "                             index_name=azure_search_service_index_name)\n",
    "\n",
    "# Azure OpenAI client\n",
    "openai_client = AzureOpenAI(\n",
    "    # to get version: https://learn.microsoft.com/en-us/azure/ai-services/openai/api-version-deprecation\n",
    "    api_version=azure_openai_api_version,\n",
    "    azure_endpoint=azure_openai_endpoint,\n",
    "    api_key=azure_openai_key)\n",
    "\n",
    "# Provide instructions to the model\n",
    "SYSTEM_PROMPT=\"\"\"\n",
    "You are an AI assistant that helps users learn from the information found in the source material.\n",
    "Answer the query using only the sources provided below.\n",
    "Use bullets if the answer has multiple points.\n",
    "If the answer is longer than 3 sentences, provide a summary.\n",
    "Answer ONLY with the facts listed in the list of sources below. Cite your source when you answer the question\n",
    "If there isn't enough information below, say you don't know.\n",
    "Do not generate answers that don't use the sources below.\n",
    "Query: {query}\n",
    "Sources:\\n{sources}\n",
    "\"\"\"\n",
    "# Iterate over each row in eval_collection\n",
    "for index, row in eval_collection.iterrows():\n",
    "    # User Query\n",
    "    query = row['query']  \n",
    "\n",
    "    # Convert query into vector form\n",
    "    vector_query = VectorizableTextQuery(text=query, \n",
    "                                        k_nearest_neighbors=50, \n",
    "                                        fields=\"text_vector\",\n",
    "                                        weight=1)\n",
    "\n",
    "    results = search_client.search(\n",
    "        query_type=\"semantic\", \n",
    "        semantic_configuration_name='my-semantic-config',\n",
    "        search_text=query,\n",
    "        vector_queries= [vector_query],\n",
    "        select=[\"title\",\"chunk\"],\n",
    "        top=3,\n",
    "    )\n",
    "\n",
    "    # Use a unique separator to make the sources distinct. \n",
    "    # We chose repeated equal signs (=) followed by a newline because it's unlikely the source documents contain this sequence.\n",
    "    sources_formatted = \"=================\\n\".join([f'TITLE: {document[\"title\"]}, CONTENT: {document[\"chunk\"]}' for document in results])\n",
    "\n",
    "    # Update the context in the DataFrame\n",
    "    eval_collection.at[index, 'context'] = sources_formatted\n",
    "\n",
    "    response = openai_client.chat.completions.create(\n",
    "        messages=[\n",
    "            {\n",
    "                \"role\": \"user\",\n",
    "                \"content\": SYSTEM_PROMPT.format(query=query, sources=sources_formatted)\n",
    "            }\n",
    "        ],\n",
    "        model=azure_openai_deployment\n",
    "    )\n",
    "\n",
    "    # Update the response in the DataFrame\n",
    "    eval_collection.at[index, 'response'] = response.choices[0].message.content\n",
    "\n",
    "# Save the updated DataFrame as a JSONL file\n",
    "eval_collection.to_json('data/output/eval.jsonl', orient='records', lines=True)\n",
    "\n",
    "# Print success message\n",
    "print(\"EvalCollection has been saved to eval.jsonl\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}