Scrapper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "904b7992",
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "from datetime import datetime\n",
    "import requests\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "ce86dce6",
   "metadata": {},
   "outputs": [],
   "source": [
    "template = \"https://www.indeed.com/jobs?q={}&l={}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "7f273192",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_url(position, location):\n",
    "    template = \"https://www.indeed.com/jobs?q={}&l={}\"\n",
    "    url = template.format(position, location)\n",
    "    return url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "47ef090e",
   "metadata": {},
   "outputs": [],
   "source": [
    "url = get_url('senior accountant', 'charlotte nc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "23965ea4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://www.indeed.com/jobs?q=senior accountant&l=charlotte nc'"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "73040256",
   "metadata": {},
   "source": [
    "## Extract raw html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "46b6adf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "response = requests.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "4b303806",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "eec8feb7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response.reason"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "61a2576b",
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(response.text, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "c07ea8b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "cards = soup.find_all('div', 'job_seen_beacon')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "2e47ca6b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cards)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8275665f",
   "metadata": {},
   "source": [
    "## Prototype the model with a single record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "31080e54",
   "metadata": {},
   "outputs": [],
   "source": [
    "card = cards[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "40beff51",
   "metadata": {},
   "outputs": [],
   "source": [
    "atag = card.h2.a.span"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "075a957f",
   "metadata": {},
   "outputs": [],
   "source": [
    "job_title = atag.get('title')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "31e3ec34",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Senior Accountant'"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "job_title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "395fe6ea",
   "metadata": {},
   "outputs": [],
   "source": [
    " link = card.h2.a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "ff11c904",
   "metadata": {},
   "outputs": [],
   "source": [
    "job_url = 'https://www.indeed.com' + link.get('href')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "45e748fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "company=card.find('span', 'companyName').text.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "bc67c144",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Next Glass, Inc.'"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "company"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "c8456d27",
   "metadata": {},
   "outputs": [],
   "source": [
    "job_location = card.find('div', 'companyLocation').text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "20102405",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Remote in Charlotte, NC 28208'"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "job_location"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "56764004",
   "metadata": {},
   "outputs": [],
   "source": [
    "job_summary = card.find('div', 'job-snippet').text.strip().replace('\\n',' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "1cfc0914",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Prepare and/or review workpapers and schedules to ensure audit readiness. With recent Private Equity investment and five successful acquisitions, Next Glass is…'"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "job_summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "b79153e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "post_date = card.find('span', 'date').text[6:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "f85588cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "today = datetime.today().strftime('%Y-%m-%d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "3deb2512",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2022-06-25'"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "today"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "0bf04417",
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    job_salary = card.find('span', 'estimated-salary').text[10:]\n",
    "except AttributeError:\n",
    "    job_salary = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "f76ab54e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'$78.7K - $99.7K a year'"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "job_salary"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "313c2d2a",
   "metadata": {},
   "source": [
    "## Generalize the model with a function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "bf82f3ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_record(card):\n",
    "    \"\"\"Extract job data from a single record\"\"\"\n",
    "    atag = card.h2.a.span\n",
    "    job_title = atag.get('title')\n",
    "    job_url = 'https://www.indeed.com' + link.get('href')\n",
    "    company=card.find('span', 'companyName').text.strip()\n",
    "    job_location = card.find('div', 'companyLocation').text\n",
    "    job_summary = card.find('div', 'job-snippet').text.strip().replace('\\n',' ')\n",
    "    post_date = card.find('span', 'date').text[6:]\n",
    "    today = datetime.today().strftime('%Y-%m-%d')\n",
    "    try:\n",
    "        job_salary = card.find('span', 'estimated-salary').text[10:]\n",
    "    except AttributeError:\n",
    "        job_salary = ''\n",
    "    record = (job_title, company, job_location, post_date, today, job_summary, job_salary, job_url)\n",
    "    return record  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "488ba443",
   "metadata": {},
   "outputs": [],
   "source": [
    "records = []\n",
    "for card in cards:\n",
    "    record = get_record(card)\n",
    "    records.append(record)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "1125002d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Senior Accountant',\n",
       " 'Next Glass, Inc.',\n",
       " 'Remote in Charlotte, NC 28208',\n",
       " '18 days ago',\n",
       " '2022-06-25',\n",
       " 'Prepare and/or review workpapers and schedules to ensure audit readiness. With recent Private Equity investment and five successful acquisitions, Next Glass is…',\n",
       " '$78.7K - $99.7K a year',\n",
       " 'https://www.indeed.com/rc/clk?jk=566afbc2856bde69&fccid=300be14ffee74507&vjs=3')"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "records[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad4ca8f3",
   "metadata": {},
   "source": [
    "## Getting the next page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "07bbce46",
   "metadata": {},
   "outputs": [],
   "source": [
    "while True:\n",
    "    try:\n",
    "       url = 'http://www.indeed.com' + soup.find('a', {'aria-label': 'Next'}).get('href')\n",
    "    except AttributeError:\n",
    "        break\n",
    "    response = requests.get(url)\n",
    "    soup = BeautifulSoup(response.text, 'html.parser')\n",
    "    cards = soup.find_all('div', 'job_seen_beacon')\n",
    "    \n",
    "    for card in cards:\n",
    "        record = get_record(card)\n",
    "        records.append(record)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "04811df2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "75"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(records)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e61ed194",
   "metadata": {},
   "source": [
    "## Putting it all together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "77863812",
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "from datetime import datetime\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "def get_url(position, location):\n",
    "    template = \"https://www.indeed.com/jobs?q={}&l={}\"\n",
    "    url = template.format(position, location)\n",
    "    return url\n",
    "\n",
    "def get_record(card):\n",
    "    \"\"\"Extract job data from a single record\"\"\"\n",
    "    atag = card.h2.a.span\n",
    "    job_title = atag.get('title')\n",
    "    job_url = 'https://www.indeed.com' + link.get('href')\n",
    "    company=card.find('span', 'companyName').text.strip()\n",
    "    job_location = card.find('div', 'companyLocation').text\n",
    "    job_summary = card.find('div', 'job-snippet').text.strip().replace('\\n',' ')\n",
    "    post_date = card.find('span', 'date').text[6:]\n",
    "    today = datetime.today().strftime('%Y-%m-%d')\n",
    "    print(today)\n",
    "    try:\n",
    "        job_salary = card.find('span', 'estimated-salary').text[10:]\n",
    "    except AttributeError:\n",
    "        job_salary = ''\n",
    "    record = (job_title, company, job_location, post_date, today, job_summary, job_salary, job_url)\n",
    "    return record  \n",
    "\n",
    "def main(position, location):\n",
    "    \"\"\"Run the main program routine\"\"\"\n",
    "    records = []\n",
    "    url = get_url(position, location)\n",
    "    \n",
    "    # Extract the job data\n",
    "    while True:\n",
    "        response = requests.get(url)\n",
    "        soup = BeautifulSoup(response.text, 'html.parser')\n",
    "        cards = soup.find_all('div', 'job_seen_beacon')\n",
    "\n",
    "        for card in cards:\n",
    "            record = get_record(card)\n",
    "            records.append(record)\n",
    "        try:\n",
    "           url = 'http://www.indeed.com' + soup.find('a', {'aria-label': 'Next'}).get('href')\n",
    "        except AttributeError:\n",
    "            break\n",
    "            \n",
    "    # Save the job data\n",
    "    with open('results.csv', 'w', newline='', encoding='utf-8') as f:\n",
    "        writer = csv.writer(f)\n",
    "        writer.writerow(['JobTitle', 'Company', 'Location', 'PostDate', 'ExtractDate', 'Summary', 'Salary', 'JobUrl'])\n",
    "        writer.writerows(records)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "f8020f7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n",
      "2022-06-25\n"
     ]
    }
   ],
   "source": [
    "# run the main program\n",
    "main('senior accountant', 'charlotte, nc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8565e048",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}