Personal Finance RAG AI.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9c8c77bd-5383-4944-81cd-295d6cd20f8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import fitz\n",
    "import spacy\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from prefect import task, flow\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a61b4ae3-a561-452c-908f-3c77035467b1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No Data\n"
     ]
    }
   ],
   "source": [
    "def remove_timestamp(text):\n",
    "    return re.sub(r'\\[\\d{2}:\\d{2}:\\d{2}\\]', '', text).strip()\n",
    "\n",
    "text = '''\n",
    "No Data\n",
    "'''\n",
    "\n",
    "cleaned_text = remove_timestamp(text)\n",
    "print(cleaned_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7d180bc7-5f73-4ca2-9158-e0406ce4b97e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('stopwords', quiet=True)\n",
    "nltk.download('punkt', quiet=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1eb499b3-6e10-4b0b-9ab7-359a5310e751",
   "metadata": {},
   "outputs": [],
   "source": [
    "@task\n",
    "def extract_pdf_text(pdf_path):\n",
    "    doc = fitz.open(pdf_path)\n",
    "    text = \"\"\n",
    "    for page_num in tqdm(range(doc.page_count), desc=\"Extracting pages\"):\n",
    "        page = doc.load_page(page_num)\n",
    "        text += page.get_text()\n",
    "    return text\n",
    "\n",
    "@task\n",
    "def clean_text(text):\n",
    "    paragraphs = text.split('\\n\\n')\n",
    "    cleaned_paragraphs = []\n",
    "    for paragraph in paragraphs:\n",
    "        if re.match(r'^[A-Z][a-z]+:', paragraph):\n",
    "            cleaned_paragraphs.append(paragraph.strip())\n",
    "        else:\n",
    "            paragraph = re.sub(r'\\s+', ' ', paragraph)\n",
    "            paragraph = re.sub(r'([.,?!:;])(\\S)', r'\\1 \\2', paragraph)\n",
    "            paragraph = re.sub(r'\\b(ca) (nt)\\b', r\"can't\", paragraph)\n",
    "            paragraph = re.sub(r'\\b(do) (nt)\\b', r\"don't\", paragraph)\n",
    "            paragraph = re.sub(r'\\b(wo) (nt)\\b', r\"won't\", paragraph)\n",
    "            paragraph = re.sub(r'[^\\w\\s.,?!:;\\'-]', '', paragraph)\n",
    "            cleaned_paragraphs.append(paragraph.strip())\n",
    "    return '\\n\\n'.join(cleaned_paragraphs)\n",
    "\n",
    "@task\n",
    "def process_text(text, nlp, stop_words):\n",
    "    processed_text = []\n",
    "    for paragraph in text.split('\\n\\n'):\n",
    "        if paragraph.startswith(\"Podcast Episode\") or paragraph.startswith(\"Synopsis:\") or \"Transcript:\" in paragraph:\n",
    "            processed_text.append(paragraph)\n",
    "        else:\n",
    "            doc = nlp(paragraph)\n",
    "            processed_words = []\n",
    "            for token in doc:\n",
    "                if token.text.lower() not in stop_words and len(token.text) > 1:\n",
    "                    if token.ent_type_ in ['PERSON', 'ORG', 'GPE']:\n",
    "                        processed_words.append(token.text)\n",
    "                    else:\n",
    "                        processed_words.append(token.lemma_.lower())\n",
    "            processed_text.append(' '.join(processed_words))\n",
    "    return '\\n\\n'.join(processed_text)\n",
    "\n",
    "@task\n",
    "def extract_metadata(text):\n",
    "    episodes = re.split(r'Podcast Episode \\d+:', text)[1:]\n",
    "    all_episodes_data = []\n",
    "    episode_num = 0\n",
    "    \n",
    "    for episode_text in episodes:\n",
    "        episode_num += 1\n",
    "        lines = episode_text.strip().split('\\n')\n",
    "        episode_info = \"\"\n",
    "        transcript = \"\"\n",
    "        episode_info = lines[0].strip()\n",
    "        transcript_start = -1\n",
    "        for i, line in enumerate(lines):\n",
    "            if line.startswith(\"Podcast Transcript:\"):\n",
    "                transcript_start = i\n",
    "        if transcript_start != -1:\n",
    "            transcript = '\\n'.join(lines[transcript_start+1:]).strip()\n",
    "        all_episodes_data.append({\n",
    "            \"episode_num\" : episode_num,\n",
    "            \"episode_info\" : episode_info,\n",
    "            \"transcript\" : transcript\n",
    "        })\n",
    "    return all_episodes_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d3d2423b-7049-4173-ab6b-e77f72c9f965",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\sambi\\AppData\\Roaming\\Python\\Python311\\site-packages\\sentence_transformers\\cross_encoder\\CrossEncoder.py:11: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import tqdm, trange\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sentence_transformers import SentenceTransformer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
    "import torch\n",
    "import traceback\n",
    "from openai import OpenAI\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4c707147-27f6-4551-8a0b-b31003ebb3af",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['OPENAI_API_KEY'] = 'OPENAI_API_KEY_HERE'\n",
    "\n",
    "@task\n",
    "def load_models():\n",
    "    api_key = os.environ.get('OPENAI_API_KEY')\n",
    "    sentence_model = SentenceTransformer('all-MiniLM-L6-v2')\n",
    "    client = OpenAI(api_key=api_key)\n",
    "    return sentence_model, client\n",
    "\n",
    "@task\n",
    "def create_chunks(all_episodes_data, chunk_size=500, overlap=100):\n",
    "    chunks = []\n",
    "    for episode in all_episodes_data:\n",
    "        episode_text = f\"Episode {episode['episode_num']}: \\n{episode['transcript']}\"\n",
    "        words = episode_text.split()\n",
    "        for i in range(0, len(words), chunk_size - overlap):\n",
    "            chunk = ' '.join(words[i:i+chunk_size])\n",
    "            chunks.append(chunk)\n",
    "    return chunks\n",
    "\n",
    "@task\n",
    "def preprocess_corpus(all_episodes_data, sentence_model):\n",
    "    chunks = create_chunks(all_episodes_data)\n",
    "    chunk_embeddings = sentence_model.encode(chunks)\n",
    "    return chunks, chunk_embeddings\n",
    "\n",
    "@task\n",
    "def retrieve_relevant_chunks(query, chunks, chunk_embeddings, sentence_model, top_k=4):\n",
    "    query_embedding = sentence_model.encode([query])\n",
    "    similarities = cosine_similarity(query_embedding, chunk_embeddings)[0]\n",
    "    top_indices = np.argsort(similarities)[-top_k:][::-1]\n",
    "    relevant_chunks = [chunks[i] for i in top_indices]\n",
    "    return relevant_chunks\n",
    "\n",
    "@task\n",
    "def sliding_window_generation(prompt, context, client, max_length=2048, stride=1000):\n",
    "    generated_text = \"\"\n",
    "    input_text = f\"{prompt}\\n\\n{context}\"\n",
    "    \n",
    "    for i in range(0, len(input_text), stride):\n",
    "        window = input_text[i:i+max_length]\n",
    "        response = client.chat.completions.create(\n",
    "            model='gpt-4o',\n",
    "            messages = [\n",
    "                {'role':'system', 'content': \"You are a personal finance assistant designed to provide concise, accurate information based on Ramit Sethi's teachings. Your primary task is to directly answer the user's question using the provided context as general knowledge. Do not offer personalized advice or refer to specific individuals mentioned in the context. Treat the context as background information only, not as a scenario to be addressed. Focus solely on the user's explicit query. Keep your responses brief and to the point, ideally under 150 words. If the user's question cannot be fully answered using the given context, provide a clear, factual response based on general personal finance principles without speculation or elaboration. If you have multiple answers you would like to tell the user, condense them into a single, concise, logical answer for the user.\"},\n",
    "                {'role':'user', 'content':window}\n",
    "            ],\n",
    "            max_tokens = 512\n",
    "        )\n",
    "        generated_text += response.choices[0].message.content.strip()\n",
    "    return generated_text\n",
    "\n",
    "@flow(name=\"Integrated Text Preprocessing and RAG Flow\")\n",
    "def integrated_preprocessing_rag_flow(pdf_path, query):\n",
    "    nlp = spacy.load('en_core_web_lg')\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    stop_words.update(['actually', 'basically', 'literally', 'you know', 'um', 'uh'])\n",
    "\n",
    "    extracted_text = extract_pdf_text(pdf_path)\n",
    "    \n",
    "    all_episodes_data = extract_metadata(extracted_text)\n",
    "    sentence_model, client = load_models()\n",
    "    chunks, chunk_embeddings = preprocess_corpus(all_episodes_data, sentence_model)\n",
    "    relevant_chunks = retrieve_relevant_chunks(query, chunks, chunk_embeddings, sentence_model)\n",
    "    context = ' '.join(relevant_chunks)\n",
    "    print(f\"Context given:\\n{context}\")\n",
    "    answer = sliding_window_generation(query, context, client)\n",
    "    return all_episodes_data, answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "03d1c097-6a6b-4ed5-af58-e838c40a6108",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:19:56.803 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | prefect.engine - Created flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'determined-slug'</span> for flow<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> 'Integrated Text Preprocessing and RAG Flow'</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:19:56.803 | \u001b[36mINFO\u001b[0m    | prefect.engine - Created flow run\u001b[35m 'determined-slug'\u001b[0m for flow\u001b[1;35m 'Integrated Text Preprocessing and RAG Flow'\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:19:58.686 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'determined-slug'</span> - Created task run 'extract_pdf_text-0' for task 'extract_pdf_text'\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:19:58.686 | \u001b[36mINFO\u001b[0m    | Flow run\u001b[35m 'determined-slug'\u001b[0m - Created task run 'extract_pdf_text-0' for task 'extract_pdf_text'\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:19:58.689 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'determined-slug'</span> - Executing 'extract_pdf_text-0' immediately...\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:19:58.689 | \u001b[36mINFO\u001b[0m    | Flow run\u001b[35m 'determined-slug'\u001b[0m - Executing 'extract_pdf_text-0' immediately...\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting pages: 100%|█████████████████████████████████████████████████████████████| 323/323 [00:00<00:00, 364.66it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:19:59.678 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Task run 'extract_pdf_text-0' - Finished in state <span style=\"color: #008000; text-decoration-color: #008000\">Completed</span>()\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:19:59.678 | \u001b[36mINFO\u001b[0m    | Task run 'extract_pdf_text-0' - Finished in state \u001b[32mCompleted\u001b[0m()\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:19:59.713 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'determined-slug'</span> - Created task run 'extract_metadata-0' for task 'extract_metadata'\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:19:59.713 | \u001b[36mINFO\u001b[0m    | Flow run\u001b[35m 'determined-slug'\u001b[0m - Created task run 'extract_metadata-0' for task 'extract_metadata'\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:19:59.714 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'determined-slug'</span> - Executing 'extract_metadata-0' immediately...\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:19:59.714 | \u001b[36mINFO\u001b[0m    | Flow run\u001b[35m 'determined-slug'\u001b[0m - Executing 'extract_metadata-0' immediately...\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:19:59.804 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Task run 'extract_metadata-0' - Finished in state <span style=\"color: #008000; text-decoration-color: #008000\">Completed</span>()\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:19:59.804 | \u001b[36mINFO\u001b[0m    | Task run 'extract_metadata-0' - Finished in state \u001b[32mCompleted\u001b[0m()\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:19:59.837 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'determined-slug'</span> - Created task run 'load_models-0' for task 'load_models'\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:19:59.837 | \u001b[36mINFO\u001b[0m    | Flow run\u001b[35m 'determined-slug'\u001b[0m - Created task run 'load_models-0' for task 'load_models'\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:19:59.839 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'determined-slug'</span> - Executing 'load_models-0' immediately...\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:19:59.839 | \u001b[36mINFO\u001b[0m    | Flow run\u001b[35m 'determined-slug'\u001b[0m - Executing 'load_models-0' immediately...\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\sambi\\anaconda3\\Lib\\site-packages\\huggingface_hub\\file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:20:01.341 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Task run 'load_models-0' - Finished in state <span style=\"color: #008000; text-decoration-color: #008000\">Completed</span>()\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:20:01.341 | \u001b[36mINFO\u001b[0m    | Task run 'load_models-0' - Finished in state \u001b[32mCompleted\u001b[0m()\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:20:01.377 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'determined-slug'</span> - Created task run 'preprocess_corpus-0' for task 'preprocess_corpus'\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:20:01.377 | \u001b[36mINFO\u001b[0m    | Flow run\u001b[35m 'determined-slug'\u001b[0m - Created task run 'preprocess_corpus-0' for task 'preprocess_corpus'\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:20:01.378 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'determined-slug'</span> - Executing 'preprocess_corpus-0' immediately...\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:20:01.378 | \u001b[36mINFO\u001b[0m    | Flow run\u001b[35m 'determined-slug'\u001b[0m - Executing 'preprocess_corpus-0' immediately...\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:20:01.464 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Task run 'preprocess_corpus-0' - Created task run 'create_chunks-0' for task 'create_chunks'\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:20:01.464 | \u001b[36mINFO\u001b[0m    | Task run 'preprocess_corpus-0' - Created task run 'create_chunks-0' for task 'create_chunks'\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:20:01.466 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Task run 'preprocess_corpus-0' - Executing 'create_chunks-0' immediately...\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:20:01.466 | \u001b[36mINFO\u001b[0m    | Task run 'preprocess_corpus-0' - Executing 'create_chunks-0' immediately...\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:20:01.566 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Task run 'create_chunks-0' - Finished in state <span style=\"color: #008000; text-decoration-color: #008000\">Completed</span>()\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:20:01.566 | \u001b[36mINFO\u001b[0m    | Task run 'create_chunks-0' - Finished in state \u001b[32mCompleted\u001b[0m()\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\sambi\\anaconda3\\Lib\\site-packages\\paramiko\\transport.py:219: CryptographyDeprecationWarning: Blowfish has been deprecated\n",
      "  \"class\": algorithms.Blowfish,\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:20:28.604 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Task run 'preprocess_corpus-0' - Finished in state <span style=\"color: #008000; text-decoration-color: #008000\">Completed</span>()\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:20:28.604 | \u001b[36mINFO\u001b[0m    | Task run 'preprocess_corpus-0' - Finished in state \u001b[32mCompleted\u001b[0m()\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:20:28.693 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'determined-slug'</span> - Created task run 'retrieve_relevant_chunks-0' for task 'retrieve_relevant_chunks'\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:20:28.693 | \u001b[36mINFO\u001b[0m    | Flow run\u001b[35m 'determined-slug'\u001b[0m - Created task run 'retrieve_relevant_chunks-0' for task 'retrieve_relevant_chunks'\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:20:28.695 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'determined-slug'</span> - Executing 'retrieve_relevant_chunks-0' immediately...\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:20:28.695 | \u001b[36mINFO\u001b[0m    | Flow run\u001b[35m 'determined-slug'\u001b[0m - Executing 'retrieve_relevant_chunks-0' immediately...\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:20:28.859 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Task run 'retrieve_relevant_chunks-0' - Finished in state <span style=\"color: #008000; text-decoration-color: #008000\">Completed</span>()\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:20:28.859 | \u001b[36mINFO\u001b[0m    | Task run 'retrieve_relevant_chunks-0' - Finished in state \u001b[32mCompleted\u001b[0m()\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context given:\n",
      "parents. Ramit Sethi: I think it’s pretty cool that Chantha understands Calvin’s childhood affects him. And that understanding, that’s not going to make their money problems disappear, but at least that understanding is a great sign that the two of them can take this journey together. I will say, though, that I’m not sure Calvin has connected his childhood with his financial behaviors of today, the anxiety, the constant comparing cost of everything, and the stress that is putting on their relationship. When you think about your relationship with money today, what role do you think that your family upbringing had in that relationship? Calvin: It makes me just never want to be in debt and be in that position. Ramit Sethi: Because what? Calvin: I don’t know. That is a wonder because it doesn’t help me if I was at home, but I don’t like—and that’s why I don’t feel good at the end of the month when I’m in the red. Ramit Sethi: What happens if you get into debt? Calvin: It’s hard to get out. Ramit Sethi: Yeah, what else? Calvin: I would have to work more to make the money. Ramit Sethi: What happened to your dad as he got worse and worse with his money? Calvin: He and my mom would get into arguments financially. That’s it. And then, my dad, I don’t think he cares. It seems like he cares, but I don’t think he cares. Ramit Sethi: So, how come you care? Calvin: Because I think it’s going to fall onto me if anything happens because I’m the one that’s the most successful financially—on paper, the most successful in our family. Ramit Sethi: And you’ve been doing it for so long, too. You’ve been helping in so many ways. Calvin: Yeah. Ramit Sethi: Yeah. Did your parents look at the prices of stuff when you were a kid? Calvin: They do, but they don’t—what do you call it? They don’t weigh it out, and much work, and stuff. Ramit Sethi: Like they don’t create Excel models to compare it? Calvin: No. Nothing. None of that stuff. And they just buy it and they don’t think too much about it. Up until this day, they just buy unnecessary things they don’t need. They buy too much of things that they should be buying. Ramit Sethi: And when they do that, how does it make you feel? Calvin: It frustrates me and I get into arguments with them, too. Ramit Sethi: And when Chantha buys valet parking, how does it make you feel? Calvin: I don’t like it either. Ramit Sethi: Any connection between the two? Calvin: No. The only connection is I don’t want to spend the money on that because I don’t think we need it. Ramit Sethi: I’m wearing a pretty nice shirt right now. I don’t need it. I don’t need it at all. I could have bought it at Target. Target has very nice clothes, but I decided I like Episode 1: Ramit Sethi: This is the I Will Teach You To Be Rich podcast. And I’m your host, Ramit Sethi. Now, I’ve always been fascinated with money and psychology. I remember writing about money while I was a student at Stanford, studying technology and psychology there, and loving talking to my friends about money. So many of us claim that certain things are important to us, and yet our behavior doesn’t match up to it. Think about it. There’s money. Yeah, I know, I should save more. There’s flossing. Yeah, I know I should floss more. There’s calling our parents or going to the gym. There are so many things in our lives that are genuinely important to us; and yet, we find it difficult to do. And I know what it’s like. Growing up, I would joke that I was just a skinny Indian guy. I would look around and I would see a bunch of people who looked fit. They seemed to stay fit and put on muscle effortlessly; I just felt like I was on the outside. Like everybody had read a book about getting fit, and I was absent that day at school. And it took me many years of getting help, hiring trainers, and reading books to finally figure out what other people seem to effortlessly know. And I think a lot of us feel like that about money like we were absent the day that everyone else seemed to learn it. So, I’m fascinated with money psychology, and I’m also fascinated with how it plays out in relationships. Part of this is inspired by my relationship. When I was dating my now wife and we got married, we went through a lot of difficult conversations to be able to get on the same page about money. And so, I started talking to other couples. I talked to friends, I talked to readers, and you will not believe some of the stories that I heard. We have couples that have vastly different spending styles – couples where one person earns a hundred times what the other does, and in almost every couple, they grew up with totally different philosophies on money. And I thought to myself, when was the last time any of us had the chance to listen to a real couple sharing real stories about their money problems and a conversation where they shared real numbers also? And the answer is never. Most of us have these conversations – sometimes, fights with our partner – but we never get the chance to sit in a room like a fly on the wall and listen to other couples talk about their money problems. Well, that’s what I wanted to bring you on this podcast in the hopes that it can inspire you and maybe help you with your money psychology as well. Alyssa and Ilan are like so many of the couples that I’ll speak to on this podcast. They come to me Episode 2: Ramit Sethi: My guests today are Ashley and Greg, and they have a household income of over $250,000, and a substantial net worth, but what happens when only one person in a relationship wants to spend money and the other is an aggressive or even obsessive saver? Would your family say that you are cheap? Ashley: Yes. Greg: Yeah. Ramit Sethi: What about friends? Ashley: Yes. Greg: Some. I don’t think all of them. Ashley: Yeah. Ramit Sethi: You want to know something, you’re cheap. If your friends and family would say it, you are cheap. It’s one thing to be careful with your money, but being cheap is different. It puts a strain on relationships. It shapes and changes your identity. And you end up hoarding money with no purpose, whatsoever. Ashley: Why do you return all the gifts that I buy you, because you don’t want me to spend money? Greg: I don’t like the gifts. Ramit Sethi: And the real issue that we’re going to discuss today is not about money, it’s much deeper, and we’re going to figure out what’s going on here. Do you two want to ask each other anything or share how you’re feeling? Greg: No. Ramit Sethi: Today’s couple is Ashley and Greg. Ashley reached out to me because there’s a serious problem in their relationship, the way that they view their finances is creating a rift between them and it’s threatening to tear them apart. Ashley: There have been times where our friends want to go out for dinner, and we don’t go, and then we stop getting invited, or like we’re particular about where we want to go, they’ll pick expensive places, and we say, okay, pick something cheaper. And then, I feel like it’s just a problem for them. Ramit Sethi: You think you are a cheapskate? Ashley: There are some things that I’m willing to spend money on, but not Greg. He’s a cheapskate. Ramit Sethi: Okay. First of all, when you ask people if they’re cheap, they will never, ever admit it. But usually, in the next sentence, they’ll tell you everything you need to know. Oh, no, no, no, I’m not cheap, I just prefer to find old food outside the back of a dumpster, because it’s a great value, but no, no, I’m not cheap. In this case, noticing the deflection that Ashley employs, she immediately pawns off the cheapness onto Greg. Let’s listen to what he says. Greg, is that true? Greg: I don’t put a lot of value on material things. Ramit Sethi: So, neither of you will admit that you’re cheapskates, except one of you just threw the other under the bus. I just want to get their income and savings out of the way, because a lot of people believe that, oh, they’re cheap, because they don’t have a lot of money. And it turns out that cheap and net worth are not correlated, whatsoever. Listen to their numbers. Episode 3: Ramit Sethi: Hi. My name is Ramit Sethi. And today, I’m talking to Calvin and Chantha. Now, they’ve been together for three years, and until recently, Calvin has insisted on paying for just about everything. The problem is he can’t afford to, and he obsesses and agonizes over money. He feels anxious. He’s always looking for the cheapest price. Now, he’s in his mid-30s, and together, Calvin and Chantha make over $250,000 a year, and they have over half a million dollars in savings. So, why does he feel so bad about money? Let’s find out the clues. We’ll see what we can do together. Chantha: He’s great. He wants to pay for everything, but the struggle is he obsesses and stresses out over everything also when it comes to money. And so, I’ve always talked to him about contributing a percentage of our paychecks into a household expense and combining the household expenses. But Calvin has always wanted to keep a lot of things separate, and he’s also wanting to, I guess, be the man of the house and take care of everything, until he comes to me, and says, he keeps going negative after paying for everything. Our expenses are pretty high. Calvin: I pay for too much stuff, and therefore, that’s in debt. In turn, it makes me worry about money, worry about the cost of everything, always looking for the cheapest route or the cheapest alternative. And I told her, I think that you may not feel that way, because you’re doing fine, because obviously, I’m in a negative, you must have extra money, and you must go okay and I don’t. Chantha: Yeah. I think Calvin two years late on that when we first moved in together, I had discussed this with him in the beginning, because I didn’t expect him to pay for everything, and I’m more than happy and more than willing to pay for my share. I think him struggling and realizing that he cannot support both of us, especially with the way I spend according to him. And so, I think that I’m glad he finally realized it. I think if he had sat down with me, looked at the numbers, and gone through everything together from the beginning when we first moved in, I don’t think he would feel this much anxiety. Calvin: I realized that a little bit late. I take in about 120,000 a year. Ramit Sethi: Okay. I get that he wants to pay for everything, but Chantha also has money of her own. Let’s find out what’s going on with her. Tell me how much you make in total per year. Chantha: About 130,000 a year. Ramit Sethi: Did you catch that? Chantha makes more than Calvin does. Now, there’s an obvious solution to this problem, and that is for them to reallocate the way that they do their spending, for them to spend proportionately, so Calvin’s spending will go down towards their joint\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:20:28.907 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'determined-slug'</span> - Created task run 'sliding_window_generation-0' for task 'sliding_window_generation'\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:20:28.907 | \u001b[36mINFO\u001b[0m    | Flow run\u001b[35m 'determined-slug'\u001b[0m - Created task run 'sliding_window_generation-0' for task 'sliding_window_generation'\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:20:28.910 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'determined-slug'</span> - Executing 'sliding_window_generation-0' immediately...\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:20:28.910 | \u001b[36mINFO\u001b[0m    | Flow run\u001b[35m 'determined-slug'\u001b[0m - Executing 'sliding_window_generation-0' immediately...\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:20:55.337 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Task run 'sliding_window_generation-0' - Finished in state <span style=\"color: #008000; text-decoration-color: #008000\">Completed</span>()\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:20:55.337 | \u001b[36mINFO\u001b[0m    | Task run 'sliding_window_generation-0' - Finished in state \u001b[32mCompleted\u001b[0m()\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:20:55.371 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span>    | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'determined-slug'</span> - Finished in state <span style=\"color: #008000; text-decoration-color: #008000\">Completed</span>()\n",
       "</pre>\n"
      ],
      "text/plain": [
       "00:20:55.371 | \u001b[36mINFO\u001b[0m    | Flow run\u001b[35m 'determined-slug'\u001b[0m - Finished in state \u001b[32mCompleted\u001b[0m()\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Query: \n",
      "        What is Ramit Sethi's most critical lesson for families struggling with money or struggling to spend money?\n",
      "    \n",
      "Answer: Ramit Sethi's most critical lesson for families struggling with money is understanding the psychological and emotional factors that drive their financial behaviors. Recognizing how childhood experiences impact their relationship with money can help individuals understand and change their spending and saving habits. This mutual understanding can be pivotal in addressing financial challenges together.Getting into debt can lead to several challenges, including increased stress and difficulty escaping the debt cycle. Ramit Sethi suggests that effective money management can prevent such issues, emphasizing the importance of having a plan and being conscious of spending. Without careful management, arguments about finances can strain relationships, and continuous financial problems can lead to long-term negative impacts, such as strained family dynamics and a constant need to work harder just to cover debts.Ramit Sethi emphasizes the importance of aligning your spending with what truly matters to you. He acknowledges that many people buy things they don't need and this behavior can cause frustration, especially when one partner sees such spending as unnecessary. Sethi suggests focusing on understanding the psychological aspects behind spending choices and encourages people to spend money on things they love while cutting costs mercilessly on things they don’t.It sounds like you're exploring the intersection of money management and psychology, particularly in relationships. Ramit Sethi often emphasizes the importance of aligning financial behavior with values, which can be challenging. Reflecting on his teachings, it's crucial for individuals and couples to have open conversations about money, align their financial behavior with their priorities, and seek knowledge and help when needed. This understanding can foster better financial habits and enhance relationship harmony through effective communication.Ramit Sethi often discusses the importance of open communication about money in relationships. He emphasizes understanding each partner’s money psychology and spending habits, as couples frequently have different financial backgrounds and philosophies. This understanding can help navigate potential conflicts and find common ground in managing finances together. It's crucial for couples to have honest conversations about their financial goals, values, and habits to build a strong financial partnership.If one partner in a relationship is a big spender and the other is an aggressive saver, it can cause significant tension. Ramit Sethi suggests that the issue often goes beyond money and involves deeper psychological factors. To address this tension, couples should openly discuss their financial habits and values, and work towards understanding each other's perspectives. It may also be helpful to find a compromise that allows both partners to feel comfortable and valued in their approach to shared finances.To address financial disagreements in relationships, it's crucial to have open and honest conversations about values and priorities. According to Ramit Sethi, the focus should be on understanding the deeper issues beyond just money. Establishing a system where both partners feel heard and agreed-upon spending rules can help alleviate tension. Regularly discussing financial goals and how to achieve them together can foster a healthier dynamic.Identifying whether you are a cheapskate involves evaluating your spending habits and attitudes towards money. A key indicator is if you consistently avoid spending money, even when it's reasonable, or if you prioritize saving over any form of enjoyment or convenience. According to Ramit Sethi, cheapskates often don't admit to being cheap but display behaviors like finding extreme ways to save money and experiencing stress or anxiety related to spending. To have a healthier perception of money, consider balancing savings with spending on things that bring joy and value.Based on Ramit Sethi's teachings, the issue Calvin faces is not directly related to the amount of money he or they have, but rather his mindset and approach towards money. Despite having a high combined income and significant savings, Calvin's anxiety and need to handle all expenses alone create financial stress. Addressing this may involve re-evaluating their financial roles and expenses, promoting better communication, and potentially combining finances to share the burden more equitably, rather than Calvin trying to shoulder everything himself.Allocating household expenses proportionately based on income is a useful approach. Both partners contribute a percentage of their income towards expenses. For instance, if one partner earns $120,000 and the other $130,000 per year, they could each contribute a proportionate share of their household expenses. This ensures that financial burdens are shared more equitably, reducing stress on one partner.To address this financial situation, Ramit Sethi would suggest reassessing and reallocating the budgeting strategy so each person contributes proportionately to their combined expenses based on their income. This approach can help reduce Calvin's financial burden and balance the contributions more equitably.\n"
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    pdf_path = \"podcast_transcript_small_version.pdf\"\n",
    "    query = \"\"\"\n",
    "        What is Ramit Sethi's most critical lesson for families struggling with money or struggling to spend money?\n",
    "    \"\"\"\n",
    "    try:\n",
    "        all_episodes_data, answer = integrated_preprocessing_rag_flow(pdf_path, query)\n",
    "        print(\"\\nQuery:\", query)\n",
    "        print(\"Answer:\", answer)\n",
    "    except Exception as e:\n",
    "        print(f\"An error occurred: {str(e)}\")\n",
    "        print(traceback.format_exc())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8959f369-c0a5-40fc-86a1-faf32435ec03",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}