datafiller.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c3d4461e-c254-4a6c-8d8c-fe44ecdd495a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/satz/miniconda3/envs/tempadb/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from geopy.geocoders import Nominatim\n",
    "import time\n",
    "import random\n",
    "import math\n",
    "import requests\n",
    "import json\n",
    "import pandas as pd\n",
    "import os\n",
    "from collections import defaultdict\n",
    "import google.generativeai as genai\n",
    "from PIL import Image\n",
    "import re\n",
    "import string"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "424cced8-1cea-4c61-ae35-296fdc894f6f",
   "metadata": {},
   "source": [
    "# Load Data\n",
    "\n",
    "source: https://github.com/emanhamed/Houses-dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3483b09c-7a12-45de-ac24-b9b1f2dd052a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nob        int64\n",
      "noba     float64\n",
      "area       int64\n",
      "zip        int64\n",
      "price      int64\n",
      "dtype: object\n",
      "535\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nob</th>\n",
       "      <th>noba</th>\n",
       "      <th>area</th>\n",
       "      <th>zip</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4053</td>\n",
       "      <td>85255</td>\n",
       "      <td>869500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3343</td>\n",
       "      <td>36372</td>\n",
       "      <td>865200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3923</td>\n",
       "      <td>85266</td>\n",
       "      <td>889000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4022</td>\n",
       "      <td>85262</td>\n",
       "      <td>910000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4116</td>\n",
       "      <td>85266</td>\n",
       "      <td>971226</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   nob  noba  area    zip   price\n",
       "0    4   4.0  4053  85255  869500\n",
       "1    4   3.0  3343  36372  865200\n",
       "2    3   4.0  3923  85266  889000\n",
       "3    5   5.0  4022  85262  910000\n",
       "4    3   4.0  4116  85266  971226"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DATAPATH = \"hdataset\"\n",
    "\n",
    "with open(\"secret.json\", 'r') as f:\n",
    "    data = json.load(f)\n",
    "\n",
    "metadata = pd.read_csv(f\"{DATAPATH}/HousesInfo.txt\", sep=\" \")\n",
    "print(metadata.dtypes)\n",
    "print(len(metadata))\n",
    "metadata.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c2ff6e1f-1f14-4ab1-b2e3-d82ef9df87f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "535"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def group_images_by_id(folder_path):\n",
    "    grouped_images = defaultdict(list)\n",
    "\n",
    "    for filename in os.listdir(folder_path):\n",
    "        if not filename.endswith(\".txt\"):\n",
    "            file_id = filename.split(\"_\")[0]\n",
    "            grouped_images[int(file_id)].append(f\"{folder_path}/{filename}\")\n",
    "    \n",
    "    return grouped_images\n",
    "\n",
    "imagepaths = group_images_by_id(DATAPATH)\n",
    "len(imagepaths)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43c8d791-6dc9-493d-adf6-f1696d970564",
   "metadata": {},
   "source": [
    "# Generate Random Addresses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5a28b542-3e0a-4ab0-a28e-8d4d19d90a2f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded saved dump!\n"
     ]
    }
   ],
   "source": [
    "locgcode = {\n",
    "    \"chennai\": \"13.0836939,80.270186\", \n",
    "    \"kerala\": \"10.3528744,76.5120396\", \n",
    "    \"madurai\" : \"9.9261153,78.1140983\", \n",
    "    \"hyderabad\": \"24.8296995,67.1279058\", \n",
    "    \"kanchipuram\" : \"12.9647163,79.9839686\", \n",
    "    \"delhi\": \"28.6273928,77.1716954\",\n",
    "    \"bangalore\": \"12.98815675,77.62260003796\"\n",
    "}\n",
    "\n",
    "locrange = {\n",
    "    \"chennai\" :(1,13), \n",
    "    \"kerala\" :(1,29), \n",
    "    \"madurai\": (2, 9), \n",
    "    \"hyderabad\": (1, 20), \n",
    "    \"kanchipuram\": (1,8), \n",
    "    \"delhi\": (3,50),\n",
    "    \"bangalore\": (1,10),\n",
    "}\n",
    "\n",
    "\n",
    "def km_to_lat_lon_offset(km, latitude):\n",
    "    lat_offset = km / 111.0\n",
    "\n",
    "    lon_offset = km / (111.0 * math.cos(math.radians(latitude)))\n",
    "\n",
    "    return lat_offset, lon_offset\n",
    "\n",
    "\n",
    "def get_nearby_locations(lat, lon, radius_range=(2, 10), n=10):\n",
    "    geolocator = Nominatim(user_agent=\"geo_locator\")\n",
    "\n",
    "    nearby_locations = []\n",
    "\n",
    "    for i in range(n):\n",
    "        distance_km = random.uniform(*radius_range)\n",
    "\n",
    "        lat_offset_max, lon_offset_max = km_to_lat_lon_offset(distance_km, lat)\n",
    "\n",
    "        lat_offset = random.uniform(-lat_offset_max, lat_offset_max)\n",
    "        lon_offset = random.uniform(-lon_offset_max, lon_offset_max)\n",
    "\n",
    "        offset_point = (lat + lat_offset, lon + lon_offset)\n",
    "        try:\n",
    "            location = geolocator.reverse(offset_point)\n",
    "            if location and location.address:\n",
    "                nearby_locations.append(\n",
    "                    {\n",
    "                        \"address\": location.address,\n",
    "                        \"latitude\": offset_point[0],\n",
    "                        \"longitude\": offset_point[1],\n",
    "                    }\n",
    "                )\n",
    "            time.sleep(1)\n",
    "        except Exception as e:\n",
    "            print(f\"Error with location {offset_point}: {e}\")\n",
    "\n",
    "    return nearby_locations\n",
    "\n",
    "fpath = \"random_address_dump.json\"\n",
    "if not os.path.exists(fpath):\n",
    "    print(\"Generating Addresses...\")\n",
    "    locale_to_address = {}\n",
    "    for k, v in locgcode.items():\n",
    "        lat, lon = v.split(',')\n",
    "        locale_to_address[k] = get_nearby_locations(float(lat), float(lon), locrange[k], n=50)\n",
    "        \n",
    "    with open(fpath, \"w\") as f:\n",
    "        json.dump(locale_to_address, f, indent=4)\n",
    "    print(\"Completed!\")\n",
    "else: \n",
    "    with open(fpath, \"r\") as f:\n",
    "        locale_to_address = json.load(f)\n",
    "    print(\"Loaded saved dump!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36c1d261-0678-4ec2-a6e4-760f47e8c17a",
   "metadata": {},
   "source": [
    "# Get Property Details with LVLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "07e3e3f2-e464-40f1-adc6-581ac1d0238c",
   "metadata": {},
   "outputs": [],
   "source": [
    "genai.configure(api_key=data[\"GEMINI_API_KEY\"])\n",
    "\n",
    "generation_config = {\n",
    "  \"temperature\": 1.42,\n",
    "  \"top_p\": 0.95,\n",
    "  \"top_k\": 40,\n",
    "  \"max_output_tokens\": 8192,\n",
    "  \"response_mime_type\": \"text/plain\",\n",
    "}\n",
    "\n",
    "model = genai.GenerativeModel(\n",
    "  model_name=\"gemini-1.5-flash\",\n",
    "  generation_config=generation_config,\n",
    ")\n",
    "\n",
    "proptypes = {\n",
    "  \"apartment\",\n",
    "  \"house\",\n",
    "  \"room\",\n",
    "  \"condo\",\n",
    "  \"studio\",\n",
    "  \"townhouse\",\n",
    "  \"villa\",\n",
    "};\n",
    "\n",
    "all_amenities = {\n",
    "  \"gym\",\n",
    "  \"wifi\",\n",
    "  \"swimming pool\",\n",
    "  \"parking\",\n",
    "  \"ac\",\n",
    "  \"pet friendly\",\n",
    "  \"security system\",\n",
    "  \"laundry service\",\n",
    "  \"children's play area\",\n",
    "  \"room service\",\n",
    "  \"bbq area\",\n",
    "  \"rooftop terrace\",\n",
    "  \"elevator\",\n",
    "  \"balcony\",\n",
    "  \"library\",\n",
    "  \"game room\",\n",
    "  \"tennis court\",\n",
    "  \"basketball court\",\n",
    "  \"sauna\",\n",
    "  \"hot tub\",\n",
    "  \"breakfast included\",\n",
    "  \"daily housekeeping\",\n",
    "  \"in-room safe\",\n",
    "  \"mini-bar\",\n",
    "  \"24-hour reception\",\n",
    "  \"water purifier\",\n",
    "  \"grocery delivery\",\n",
    "  # \"kitchen\",\n",
    "}\n",
    "\n",
    "proplist =''.join(f'- {t}\\n' for t in proptypes)\n",
    "aminlist = ', '.join(all_amenities)\n",
    "\n",
    "allchars = string.ascii_lowercase + string.digits \n",
    "\n",
    "def getrands(n=3):\n",
    "    random_string = ''.join(random.choice(allchars) for _ in range(n))\n",
    "    return random_string\n",
    "    \n",
    "allgeneratedids= {'init', }\n",
    "def get_legal_documentid():\n",
    "    legalid = 'init' \n",
    "    while legalid in allgeneratedids:\n",
    "        legalid = f\"{getrands(2)}axfhgsd8{getrands(3)}\"\n",
    "    return legalid\n",
    "\n",
    "    \n",
    "def sort_and_check_kitchen(data):\n",
    "    sorted_data = sorted(data, key=lambda x: (not 'frontal' in x, x))\n",
    "    kitchen_available = any('kitchen' in item for item in data)\n",
    "    return sorted_data, kitchen_available\n",
    "\n",
    "def get_property_type(image):\n",
    "    prompt = f\"\"\"\n",
    "Analyze the given image of the property and tell what type of property it is from the below list:\n",
    "{proplist}\n",
    "\n",
    "Just give the property type back don't tell anything except your selected property type.\n",
    "\"\"\".strip()\n",
    "    response = model.generate_content([prompt, image]).text\n",
    "\n",
    "    ptype = re.sub(r'[^a-zA-Z]', '', response).lower().strip() \n",
    "    return ptype if ptype in proptypes else \"house\"\n",
    "\n",
    "\n",
    "def get_amenities(image,ptype, addKitchen):\n",
    "    amenities = set()\n",
    "    \n",
    "    if addKitchen:\n",
    "        amenities.add(\"kitchen\")\n",
    "\n",
    "    prompt = f\"\"\"\n",
    "You are given an image of a {ptype}. Based on the characteristics of the property and your assessment, select which amenities it would likely have from the following list: [ {aminlist}]. \n",
    "Consider both the visual cues in the image and the common features of similar properties to make your selection. \n",
    "Return only the amenities in Python list format like this: ['a1', 'a2', ...].\"\"\".strip()\n",
    "    \n",
    "    response = model.generate_content([prompt, image]).text\n",
    "    a = response.find('[')\n",
    "    b = response.rfind(']')\n",
    "    retamn = [re.sub(r'[^a-zA-Z]', '', amn).lower().strip() for amn in response[a+1:b].split(',')]\n",
    "    for amn in retamn:\n",
    "        if amn in all_amenities:\n",
    "            amenities.add(amn)\n",
    "\n",
    "    # default ameinites \n",
    "    if len(amenities) <=1:\n",
    "        for amn in random.choices(list(all_amenities),k=4):\n",
    "            amenities.add(amn)\n",
    "    else:\n",
    "        amenities.add(\"wifi\")\n",
    "    \n",
    "    return list(amenities)\n",
    "\n",
    "\n",
    "def get_description(metad, address, ptype, image):\n",
    "    prompt = f\"\"\"\n",
    "Examine the provided image of a {ptype.capitalize()}, it has {int(metad['nob'])} bedrooms and {int(metad['noba'])} bathrooms and it is located in {address}. Generate a well-rounded description (up to 200 words) contains distinctive elements visible in the image, aiming to convey a comprehensive sense of the property.\n",
    "\n",
    "Additionally, create a short, compelling title for this property in no more than 20 characters and two words.\n",
    "\n",
    "Format your response in JSON as follows: \n",
    "{{\"description\": \"<description>\", \"title\": \"<title>\"}}\n",
    "\"\"\".strip()\n",
    "    response = model.generate_content([prompt, image]).text.strip()\n",
    "    a = response.find('{')\n",
    "    b = response.rfind('}')\n",
    "    data = json.loads(response[a:b+1])\n",
    "    desc = data[\"description\"].strip()\n",
    "    if len(desc) > 500:\n",
    "        desc = desc[0:500]\n",
    "    return desc, data[\"title\"]\n",
    "    \n",
    "def get_price_inr(ogprice):\n",
    "    return int(ogprice*.69 / 12)\n",
    "\n",
    "def get_property_details(metaentry, images, address):\n",
    "    images, kitchenaval = sort_and_check_kitchen(images)\n",
    "    \n",
    "    frontpic = Image.open(images[0])\n",
    "\n",
    "    ptype = get_property_type(frontpic)\n",
    "    time.sleep(.3)\n",
    "    amenities = get_amenities(frontpic, ptype, kitchenaval) \n",
    "    time.sleep(.3)\n",
    "    description, title = get_description(metaentry,  address, ptype, frontpic)\n",
    "    time.sleep(.3)\n",
    "    \n",
    "    rent = get_price_inr(metaentry[\"price\"])\n",
    "    time.sleep(1)\n",
    "    return  amenities, ptype, rent, description, title, images"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "634b7ba3-df9c-424b-a0df-08b204a12fed",
   "metadata": {},
   "source": [
    "# Fill the database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "182dfa3e-8ea6-4459-8b21-41f8cf471fcc",
   "metadata": {},
   "outputs": [],
   "source": [
    "SERVER_URL = \"http://localhost:6969\"\n",
    "\n",
    "def send_property_data(form, image_files, token):\n",
    "    files = []\n",
    "    for image_path in image_files:\n",
    "        filename = os.path.basename(image_path)\n",
    "        mime_type = 'image/jpeg'\n",
    "        if filename.lower().endswith('.png'):\n",
    "            mime_type = 'image/png'\n",
    "            \n",
    "        \n",
    "        files.append(\n",
    "            ('images', (filename, open(image_path, 'rb'), mime_type))\n",
    "        )\n",
    "        \n",
    "    headers = {\n",
    "        # \"Content-Type\": \"application/json\",\n",
    "        \"authorization\": f\"Bearer {token}\"\n",
    "    }\n",
    "    \n",
    "    form = {key: str(value) for key, value in form.items()}\n",
    "\n",
    "    try:\n",
    "        response = requests.post(\n",
    "            f\"{SERVER_URL}/owner/property\",\n",
    "            data=form,\n",
    "            files=files,\n",
    "            headers=headers\n",
    "            \n",
    "        )\n",
    "        # \n",
    "        response.raise_for_status()\n",
    "        return\n",
    "        \n",
    "    except requests.exceptions.RequestException as e:\n",
    "        print(f\"Error sending request: {e}\")\n",
    "        print(form)\n",
    "        print(\"message\", response.json())\n",
    "        raise\n",
    "    \n",
    "    finally:\n",
    "        for _, file_tuple in files:\n",
    "            # if idx != \"images\":\n",
    "                # continue\n",
    "                \n",
    "            try:\n",
    "                file_tuple[1].close()\n",
    "            except:\n",
    "                pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "cf175d2b-eade-49ce-ac74-47a53c653f00",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Users: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 36381.82user/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import logging\n",
    "logging.basicConfig(filename=\"datafiller_errors.log\", level=logging.ERROR, \n",
    "                    format=\"%(asctime)s - %(levelname)s - %(message)s\")\n",
    "\n",
    "prop_range = (7, 20)\n",
    "picked_props = {0, }\n",
    "\n",
    "def get_random_property():\n",
    "    propidx = 0\n",
    "    while propidx in picked_props:\n",
    "        propidx = random.randint(1, len(metadata))\n",
    "    picked_props.add(propidx) \n",
    "    return propidx \n",
    "\n",
    "for uname, info in tqdm(data['USER_DATA'].items(), desc=\"Users\", unit=\"user\"):\n",
    "    propcount = random.randint(*prop_range)\n",
    "\n",
    "    print(f\"Adding {propcount} properties for @{uname}\")\n",
    "    addresses = random.sample(locale_to_address[info[\"locale\"]], propcount)\n",
    "\n",
    "    for i in tqdm(range(propcount), desc=\"Properties\", unit=\"property\"):\n",
    "        try:\n",
    "            prop = get_random_property() \n",
    "            address = addresses[i][\"address\"]\n",
    "            metaentry = metadata.loc[prop]\n",
    "            amenities, ptype, rent, description, title, images = get_property_details(address=address, images=imagepaths[prop], metaentry=metaentry)\n",
    "            form = {\n",
    "                \"address\": address, \n",
    "                \"title\": title, \n",
    "                \"description\": description, \n",
    "                \"rent\": rent, \n",
    "                \"propertyType\": ptype, \n",
    "                \"amenities\": json.dumps(amenities), \n",
    "                \"numberOfBedrooms\": int(metaentry[\"nob\"]), \n",
    "                \"legalDocumentId\": get_legal_documentid()\n",
    "            }\n",
    "            send_property_data(form, images, info['token'])\n",
    "            time.sleep(1)\n",
    "        except Exception as e:\n",
    "            logging.error(f\"Error adding property {prop} for user {uname}:\\n\"\n",
    "                          f\"Form: {form}\\n\"\n",
    "                          f\"Error: {str(e)}\")\n",
    "            raise e\n",
    "        \n",
    "\n",
    "    print(\"Completed!!\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "f79f67fb-2672-47b7-b50a-2fae516699d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Added: 115\n"
     ]
    }
   ],
   "source": [
    "print(\"Added:\", len(picked_props))\n",
    "with open(\"pickpr.json\", 'w') as f:\n",
    "    json.dump(list(picked_props), f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tmpadb (venv)",
   "language": "python",
   "name": "tempadb"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}