Scraping

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1 Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-07T20:07:15.182622Z",
     "start_time": "2019-10-07T20:07:14.053423Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import requests, bs4, urllib.parse,re\n",
    "from bs4 import BeautifulSoup\n",
    "from geopy.geocoders import Nominatim\n",
    "from multiprocessing import  Pool"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "# All property listings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import time\n",
    "from selenium import webdriver\n",
    "from bs4 import BeautifulSoup as bs\n",
    "\n",
    "browser = webdriver.Chrome(\"C:\\chromedriver_win32\\chromedriver.exe\")\n",
    "\n",
    "#GO TO MAGICBRICKS\n",
    "browser.get(\"https://www.magicbricks.com/property-for-sale/residential-real-estate?proptype=Multistorey-Apartment,Builder-Floor-Apartment,Penthouse,Studio-Apartment&cityName=Hyderabad\")\n",
    "\n",
    "#CODE TO SCROLL THROUGH DATA\n",
    "lenOfPage = browser.execute_script(\"window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;\")\n",
    "match=False\n",
    "while(match==False):\n",
    "        lastCount = lenOfPage\n",
    "        time.sleep(10)\n",
    "        lenOfPage = browser.execute_script(\"window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;\")\n",
    "        if lastCount==lenOfPage:\n",
    "            match=True\n",
    "\n",
    "\n",
    "source_data = browser.page_source\n",
    "\n",
    "#WRITE TO TEXT FILE\n",
    "text_file = open(\"Output.txt\", \"w\")\n",
    "text_file.write(str(source_data.encode(\"utf-8\")))\n",
    "text_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-07T20:25:08.611630Z",
     "start_time": "2019-10-07T20:25:05.068932Z"
    }
   },
   "outputs": [],
   "source": [
    "#READ FROM TEXT FILE\n",
    "text_file = open(\"Output.txt\", \"r\")\n",
    "source_data = text_file.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-07T20:27:55.735643Z",
     "start_time": "2019-10-07T20:25:31.994395Z"
    }
   },
   "outputs": [],
   "source": [
    "soup2 = BeautifulSoup(source_data, 'html.parser')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Put data together"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-07T21:11:10.390531Z",
     "start_time": "2019-10-07T21:11:10.371582Z"
    }
   },
   "outputs": [],
   "source": [
    "#FUNCTION TO GET THE DATA FROM HTML\n",
    "def get_data(soup):\n",
    "\n",
    "#TO GET UNIQUE HOUSE ID\n",
    "    ids = []\n",
    "    for x in soup.find_all(\"span\", attrs = {\"data-idtag\" : \"idTag\"}):\n",
    "        ids.append(x[\"data-idvalue\"])\n",
    "\n",
    "#TO GET THE LOCATION\n",
    "    loc = []\n",
    "    for x in soup.find_all(\"div\", {\"itemprop\": \"address\"}):\n",
    "        try:\n",
    "            loc.append(x.find('meta', {\"itemprop\": \"addressLocality\"})['content'])\n",
    "        except:\n",
    "            loc.append('location_not_available')\n",
    "\n",
    "#TO GET THE PRICE\n",
    "    price = []\n",
    "    for x in soup.find_all('div', attrs = {\"class\":\"m-srp-card__info flex__item\"}):\n",
    "        price.append(x.find('span').contents[0])\n",
    "\n",
    "#CONVERT TO PANDAS DATAFRAME\n",
    "    d = {'house_id' : ids, 'location' : loc, 'price' : price}\n",
    "    data = pd.DataFrame(data= d)\n",
    "    \n",
    "#SPLIT THE LOCATION COLUMN AFTER COMMA \n",
    "    loc_col = data.location.str.split(\",\",expand=True).iloc[:,0]\n",
    "    loc_col.columns = ['location_only']\n",
    "    data[\"location_only\"] = loc_col\n",
    "    \n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-07T21:11:23.363336Z",
     "start_time": "2019-10-07T21:11:15.462801Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#CALL FUNCTION\n",
    "data = get_data(soup2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-07T14:17:29.717836Z",
     "start_time": "2019-10-07T14:17:29.665951Z"
    }
   },
   "source": [
    "# Zip codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-07T21:14:07.143119Z",
     "start_time": "2019-10-07T21:14:07.133119Z"
    }
   },
   "outputs": [],
   "source": [
    "# Function to extract zip codes from locality using geopy lilbrary\n",
    "\n",
    "def zip_extract(local):\n",
    "    try:\n",
    "        add = geolocator.geocode(local + \", rangareddy\").address\n",
    "        zip = int(re.search('5(\\d{5})', add, re.IGNORECASE).group())\n",
    "    except:\n",
    "        try:\n",
    "            add = geolocator.geocode(local + \", hyderabad\").address\n",
    "            zip = int(re.search('5(\\d{5})', add, re.IGNORECASE).group())\n",
    "        except:\n",
    "            try:\n",
    "                add = geolocator.geocode(local + \", telangana\").address\n",
    "                zip = int(re.search('5(\\d{5})', add, re.IGNORECASE).group())\n",
    "            except:\n",
    "                zip = 0\n",
    "    print(local)\n",
    "    return zip\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-07T21:31:53.900940Z",
     "start_time": "2019-10-07T21:17:13.240286Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Narsingi\n",
      "Gachibowli\n",
      "Kondapur\n",
      "Madhura Nagar-Nizampet\n",
      "Yapral\n",
      "Puppalaguda\n",
      "Shaikpet\n",
      "Banjara Hills\n",
      "Suchitra Circle\n",
      "Tellapur\n",
      "Mokila\n",
      "Bachupally\n",
      "Adibatla\n",
      "Rajendra Nagar\n",
      "Bandlaguda Jagir\n",
      "Miyapur\n",
      "Annojiguda\n",
      "Pragathi Nagar\n",
      "Kompally\n",
      "Nallagandla\n",
      "Kokapet\n",
      "Manikonda\n",
      "Nanakram Guda\n",
      "Chandanagar\n",
      "Beeramguda\n",
      "Financial District\n",
      "Pasumamula\n",
      "Alkapur Township\n",
      "ECIL\n",
      "Hitech City\n",
      "Manchirevula\n",
      "Sanath Nagar\n",
      "Kothaguda\n",
      "Kukatpally\n",
      "Medipally\n",
      "Toli Chowki\n",
      "Kollur\n",
      "Patancheru\n",
      "Jagadgiri Gutta\n",
      "Isnapur\n",
      "Bagh Amberpet\n",
      "Aminpur\n",
      "Nizampet\n",
      "Saidabad\n",
      "Ram nagar\n",
      "DD Colony\n",
      "Old Bowenpally\n",
      "Ravi Colony\n",
      "Khaja Guda\n",
      "Krishna Reddy Pet\n",
      "Begumpet\n",
      "Attapur\n",
      "Ramachandra Puram\n",
      "Appa junction\n",
      "Erragadda\n",
      "Mithila Nagar-Banjara Hills\n",
      "Gajularamaram\n",
      "Hafeezpet\n",
      "Turkayamjal\n",
      "Kukatpally Housing Board Colony\n",
      "location_not_available\n",
      "Kavadiguda\n",
      "East Marredpally\n",
      "Durga Nagar-Nacharam\n",
      "Madhapur\n",
      "Kothapet\n",
      "Kapra\n",
      "Habeeb Nagar\n",
      "Green Valley\n",
      "Sai Aishwarya Layout-Manikonda\n",
      "Lalaguda\n",
      "Alwal\n",
      "Alakapur\n",
      "Bandari Layout\n",
      "Raj Bhawan Road\n",
      "Ameerpet\n",
      "Sainikpuri\n",
      "Kailash Hills\n",
      "Jayaprakash Narayan Nagar\n",
      "Turkapalli\n",
      "Jubilee Hills\n",
      "KTR Colony\n",
      "Bowrampet\n",
      "Ashok Nagar\n",
      "Somajiguda\n",
      "West Marredpally\n",
      "Kalyan Nagar-Moti Nagar\n",
      "Old Alwal\n",
      "Nagaram\n",
      "Mayuri Nagar\n",
      "Santosh Nagar\n",
      "New Santoshnagar\n",
      "Bolarum\n",
      "Quthbullapur\n",
      "APGOs Coop HSG Society\n",
      "Vikas Nagar-Bandlaguda Jagir\n",
      "Koti\n",
      "Chintal\n",
      "Gandhi Nagar\n",
      "Nagole\n",
      "Patighanpur\n",
      "Domalguda\n",
      "Mansoorabad\n",
      "Mallapur\n",
      "Sayeedabad\n",
      "Hyder Nagar\n",
      "Ambedkar Nagar-Hafeezpet\n",
      "Old Malakpet\n",
      "Silpa Gram Craft Village\n",
      "Nacharam\n",
      "Dr A.S. Rao Nagar\n",
      "Devender Colony\n",
      "Income Tax Colony\n",
      "ALIND Employees Colony\n",
      "Sri Krishna Devaraya Nagar\n",
      "Prakash Nagar\n",
      "Gachibowli CUC\n",
      "Uppal\n",
      "KPHB Road\n",
      "Bikshapathi Nagar\n",
      "Gopalnagar Society\n",
      "Sri Ramnagar - Block C\n",
      "Vikrampuri\n",
      "Whitefields\n",
      "K P H B Phase 5\n",
      "Prajay City\n",
      "Telecom Nagar\n",
      "Shamirpet\n",
      "Madinaguda\n",
      "Shanti Nagar\n",
      "Goshamahal\n",
      "Shirdi Sai Nagar\n",
      "Kavuri Hills\n",
      "Masab Tank\n",
      "Dammaiguda\n",
      "Tarnaka\n",
      "Gopanapalli\n",
      "Venkatraya Nagar\n",
      "Mehdipatnam\n",
      "Mallampet\n",
      "Rajarajeshwari Nagar\n",
      "Gandipet\n",
      "Madhava Nagar Colony\n",
      "New Nallakunta\n",
      "Cherukupally Pratap Reddy Colony\n",
      "Noorkhan Bazaar\n",
      "Gopal Nagar\n",
      "Malakpet\n",
      "Secunderabad\n",
      "Barkatpura\n",
      "Kakatiya Nagar\n",
      "Saleem Nagar\n",
      "Suraram\n",
      "Langar Houz\n",
      "Srinagar Colony\n",
      "West Venkatapuram\n",
      "Boduppal\n",
      "Sai Pruthvi Enclave\n",
      "Bandam Kommu\n",
      "Jawahar Nagar\n",
      "Nandagiri Hills\n",
      "Peerzadiguda\n",
      "Rai Durg\n",
      "Rajiv Gandhi Nagar Colony\n",
      "Hill Ridge Springs\n",
      "Hyder Shah Kote\n",
      "Sai Baba Nagar\n",
      "Sun City\n",
      "IDA Mallapur\n",
      "Madhavi Nagar\n",
      "Meerpet\n",
      "New Malakpet\n",
      "Vijaya Nagar Colony\n",
      "Balaji Colony\n",
      "Hydershakote\n",
      "Irram Manzil \n",
      "Peeramcheru\n",
      "Saraswati Nagar\n",
      "Gachibowli Phase 3\n",
      "Panchavati Colony\n",
      "Anandbagh\n",
      "Ghatkesar\n",
      "Neknampur\n",
      "Mega Hills\n",
      "Nalagantla\n",
      "Raghavendra Nagar Colony\n",
      "Moti Nagar\n",
      "Dullapally\n",
      "Deepthisri Nagar\n",
      "Kakatiya Hills\n",
      "Alkapuri\n",
      "Camelot Layout\n",
      "Chandrayanguttabalguda\n",
      "Bandlaguda\n",
      "Shapur Nagar\n",
      "Adda Gutta\n",
      "Saroornagar\n",
      "Red Hills\n",
      "Musheerabad\n",
      "Allwyn Colony\n",
      "Maheshwaram\n",
      "Devender Nagar\n",
      "Neredmet\n",
      "Vanasthalipuram\n",
      "Sai Nagar-Peerzadiguda\n",
      "Shameerpet\n",
      "Janachaitanya colony\n",
      "Happy Homes Colony\n",
      "Shanti Nagar Colony\n",
      "Jeedimetla\n",
      "Muppas Panchavati Colony\n",
      "Injapur\n",
      "Masjid Banda\n",
      "Shanakar Nagar\n",
      "Patancheruvu\n",
      "Himayath Nagar\n",
      "Moosapet\n",
      "Kushaiguda\n",
      "Asif Nagar\n",
      "Film Nagar\n",
      "Kondakal\n",
      "Aziz Bagh\n",
      "Indresham\n",
      "Setwinabad\n",
      "Tulasi Nagar\n",
      "Mallapur-Balapur\n",
      "Boinpalli\n",
      "Upparpally\n",
      "Raghavendra Colony\n",
      "Yakubpura\n",
      "Yousufguda\n",
      "Anjaneyanagar Colony\n",
      "Kalyan Nagar-Erragadda\n",
      "Kachiguda\n",
      "LB Nagar\n",
      "Jaya Puri Colony\n",
      "Sanjeeva Reddy Nagar\n",
      "Vani Nagar\n",
      "Sri Laxmi Nagar Colony-Manikonda\n",
      "Rampally\n",
      "Pathancheru\n",
      "Almasguda\n",
      "Vikas Nagar-Dilsukh Nagar\n",
      "Pochhama Basti\n",
      "Hyderaguda-Attapur\n",
      "Osman Nagar\n",
      "Dattatreya Colony\n",
      "Hari Hara Puram\n",
      "Kala Dera\n",
      "Amberpet\n",
      "Talab Katta\n",
      "Sangareddy District\n",
      "Vijaya Puri Colony\n",
      "Parvathi Nagar-Madhapur\n",
      "Arvind Nagar\n",
      "Sri Krishna Nagar Colony\n",
      "Bahadurpura West\n",
      "Lakdikapul\n",
      "Bollaram\n",
      "Kanchan Bagh\n",
      "Osmania University\n",
      "Warangal Highway\n",
      "Gagilapur\n",
      "Brundavan Colony\n",
      "Mytri Nagar\n",
      "Golden Heights Colony\n",
      "Sai Nagar-Bairamalguda\n",
      "Jagathgiri Gutta\n",
      "Ruby Block\n",
      "Chandra Nagar\n",
      "Gandhinagar Colony\n",
      "Jaihind Nagar Colony\n",
      "Huda Colony-Mehdipatnam\n",
      "Bongloor\n",
      "Surya Nagar Colony\n",
      "Murad Nagar\n",
      "Sri Krishna Nagar-Quthbullapur\n",
      "Borabanda\n",
      "Kistareddypet\n",
      "Ashok Nagar-Ramachandra Puram\n",
      "Jyothi Nagar Colony\n",
      "B N Reddy Nagar\n",
      "Rama Krishna Puram\n",
      "Sri Sai Nagar-Canara Nagar\n",
      "Industrial Development Area Bollaram\n",
      "Bairamalguda\n",
      "Canara Nagar\n",
      "Yakutpura\n",
      "Chadarghat\n",
      "Sri Sai Homes\n",
      "Jai Jawahar Nagar\n",
      "Sree Prabhupada Township\n",
      "Humayun Nagar\n",
      "Santosh Nagar-Mehdipatnam\n",
      "AWHO Colony\n",
      "Tilak Nagar\n",
      "Narayanguda\n",
      "Nagolu Enclave\n",
      "Chirag Ali Lane\n",
      "Prashanth Nagar\n",
      "Akbarbagh\n",
      "Narapally\n",
      "Balanagar\n",
      "Dilsukh Nagar\n",
      "Pocharam\n",
      "Hanuman Nagar Colony\n",
      "Kismatpur\n",
      "Vanasthalipuram PHASE I\n",
      "Serilingampally\n",
      "Shirdi Enclave-Yapral\n",
      "Aditya Nagar\n",
      "Charminar\n",
      "Jawahar Nagar-Chikkadpally\n",
      "Gandi Maisamma\n",
      "Navodaya Colony\n",
      "Punjagutta\n",
      "New Allenby\n",
      "Vasanth Nagar Colony\n",
      "Medchal\n",
      "Habsiguda\n",
      "Phase 2\n",
      "Jaya Nagar\n",
      "Moula Ali\n",
      "Gundlapochampally\n",
      "Sri Nagar Colony\n",
      "Laxmi Nagar-Kondapur\n",
      "Vittal Rao Nagar\n",
      "Chintalakunta\n",
      "Badangpet\n",
      "Venkatapuram\n",
      "Jubilee Enclave\n",
      "RTC Colony\n",
      "Matrusri Nagar\n",
      "Mudfort\n",
      "Balapur\n",
      "Maytas Hill County\n",
      "Padmarao Nagar\n",
      "Chikkadpally\n",
      "R K Nagar\n",
      "Sindhi Colony\n",
      "Safilguda\n",
      "Hema Nagar\n",
      "Srinivasa Nagar-Padamrao Nagar\n",
      "Kalyan Puri\n",
      "Mamatha Nagar Colony\n",
      "IDPL Colony\n",
      "Raghavendra Nagar-Nacharam\n",
      "Moti Nagar-Erragadda\n",
      "Anupama Homes\n",
      "Shankar Nagar\n",
      "Gudi Malkapur\n",
      "Nagarjuna Nagar\n",
      "Mayur Marg\n",
      "Khairatabad\n",
      "Hyderguda\n",
      "Sai Nagar-Quthbullapur\n",
      "Nanal nagar\n",
      "Hanuman Nagar-Tarnaka\n",
      "Samata Colony\n",
      "Gowlidoddy\n",
      "Sai Nagar-Hafeezpet\n",
      "APHB Colony-Moula Ali\n",
      "Kalyan Nagar\n",
      "Bowenpally\n",
      "Saidabad Colony\n",
      "Hasmathpet\n",
      "Vidyanagar\n",
      "Friends Colony\n",
      "Panjagutta Colony\n",
      "New Maruthi Nagar\n",
      "Moosarambagh\n",
      "Mythri Nagar\n",
      "Gudimalkapur\n",
      "Ahmadguda\n",
      "Muthangi\n",
      "Lumbini Avenue\n",
      "Marredpally\n",
      "Bahadurpura\n",
      "Quli Qutub Shah Layout\n",
      "Jalapalli\n",
      "Dwaraka Nagar\n",
      "Durga Nagar Colony\n",
      "Chanakyapuri\n",
      "Hyderabad Expressway\n",
      "Sangareddy\n",
      "Moneykonda\n",
      "Shivarampally\n",
      "Mohammed Nagar\n",
      "NH  9\n",
      "Huda\n",
      "Mahadevpur Colony\n",
      "Chengincherla\n",
      "Maruthi Nagar-Langar Houz\n",
      "Salar Jung Colony\n",
      "Adarsh Nagar Colony-Nagole\n",
      "Balaji Hills Colony\n",
      "P S Rao Nagar\n",
      "Naidu Colony\n",
      "Kalasiguda\n",
      "Lal Darwaza\n",
      "Sai Nagar-Kukatpally\n",
      "Shivanagar Colony\n",
      "Satyam Enclave\n",
      "Officers Colony\n",
      "Gayathri Nagar\n",
      "Qutub Shahi Tombs\n",
      "Karmanghat\n",
      "P Janardhan Reddy Nagar\n",
      "Dabeerpura\n",
      "Konda Nagar\n",
      "Hakimpet\n",
      "Tukkuguda\n",
      "Jamal Colony\n",
      "Cherlapally IDA\n",
      "Deen Dayal Nagar\n",
      "Gowlipura\n",
      "Kishan Bagh\n",
      "Tanasha Nagar\n",
      "Bairagiguda\n",
      "Lingampally-Serilingampally\n",
      "Shivam Rd\n",
      "Gayatri Nagar\n",
      "Thimmapuram\n",
      "King Koti\n",
      "Sai Nagar-Indresham\n",
      "Saket\n",
      "Sarojini Naidu Nagar\n",
      "Gokhale Nagar\n",
      "Visakhapatnam\n",
      "Yadagirigutta\n",
      "Sri Ram Nagar Colony\n",
      "Nekanampur\n",
      "Bahadurpally\n"
     ]
    }
   ],
   "source": [
    "geolocator = Nominatim(user_agent=\"geoapiExercises\")\n",
    "\n",
    "data_locs = data[['location_only']].drop_duplicates() # Take unique locations\n",
    "data_locs['zip_code'] = data_locs['location_only'].apply(lambda x: zip_extract(x)) # Get zip codes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Join zip codes back with data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-07T21:32:34.972559Z",
     "start_time": "2019-10-07T21:32:34.946627Z"
    }
   },
   "outputs": [],
   "source": [
    "data_final = pd.merge(data, data_locs, on = 'location_only')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}