12-6-2022_Scraping Song Lyrics For Top 100 Songs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2340f40e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing required libraries\n",
    "import pandas as pd\n",
    "import time\n",
    "\n",
    "from webdriver_manager.microsoft import EdgeChromiumDriverManager\n",
    "\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support import expected_conditions as EC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f6d8ebba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initializing the driver\n",
    "#driver = webdriver.Edge(EdgeChromiumDriverManager().install())\n",
    "driver = webdriver.Firefox(executable_path=r\"C:\\Users\\Amisha\\OneDrive\\Aegis\\NLP\\Project\\geckodriver.exe\")\n",
    "#driver = webdriver.Chrome(r\"C:\\Users\\Amisha\\OneDrive\\Aegis\\NLP\\Project\\chromedriver.exe\")\n",
    "# Browsing to billboard hot-100 chart on the driver\n",
    "#driver.get(\"https://www.billboard.com/charts/hot-100/\")\n",
    "dates = [str(i) for i in range(2012,2023)]\n",
    "#driver.get(\"https://www.google.com\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "db37fa9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "artists_data = {} # List to store name of artists\n",
    "songs_data = {} # List to store top 100 songs\n",
    "\n",
    "for j in dates:\n",
    "    link = \"https://www.billboard.com/charts/hot-100/{}-07-01/\".format(j)\n",
    "    driver.implicitly_wait(10)\n",
    "    driver.get(link)\n",
    "\n",
    "    # Container for referencing the division with top 100 songs\n",
    "    div = driver.find_elements_by_class_name('o-chart-results-list-row-container')\n",
    "\n",
    "    songs, artists = [], []\n",
    "     \n",
    "    # Fetching the top 100 songs\n",
    "    for i in div:\n",
    "        # Getting the song name\n",
    "        song = i.find_element_by_css_selector('li ul li h3').text\n",
    "        # Getting the artist name\n",
    "        name = i.find_element_by_css_selector('li ul li span').text\n",
    "        # Appending the song and artist name to the lists\n",
    "        songs.append(song)\n",
    "        artists.append(name)\n",
    "        \n",
    "    artists_data[j] = artists\n",
    "    songs_data[j] = songs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9d1490db",
   "metadata": {},
   "outputs": [],
   "source": [
    "#artists_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "13d7ae9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#songs_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "45303ac4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Terminating a session and closing the active browser\n",
    "driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "5e29594b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initializing the driver\n",
    "#driver = webdriver.Edge(EdgeChromiumDriverManager().install())\n",
    "driver = webdriver.Firefox(executable_path=r\"C:\\Users\\Amisha\\OneDrive\\Aegis\\NLP\\Project\\geckodriver.exe\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "476069b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Browsing to Google\n",
    "driver.get(\"https://www.google.com\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "a161faad",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dataframe to store the song, artist and lyrics\n",
    "df = pd.DataFrame(columns = ['Top100Year','SongTitle','Artist','LyricsStatus','Lyrics','ReleaseYear','Genre'])\n",
    "\n",
    "# Fetching song lyrics for top 100 songs\n",
    "for year in songs_data.keys():\n",
    "    songs_list = songs_data[year]\n",
    "    artists_list = artists_data[year]\n",
    "    # Index Variable\n",
    "    a = 0\n",
    "    for song in songs_list:\n",
    "        # Selenium WebDriverWait is one of the Explicit waits.\n",
    "        # Explicit waits are confined to a particular web element. \n",
    "        # Explicit Wait is code you define to wait for a certain condition to occur before proceeding further in the code.\n",
    "        # It is applied on certain element with defined expected condition and time. \n",
    "        # This wait is only applied to the specified element. This wait can also throw exception when element is not found.\n",
    "        # Here we will wait for the google search button element to be clickable.\n",
    "        # https://www.softwaretestingmaterial.com/webdriverwait-selenium-webdriver/\n",
    "        element = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.NAME, \"q\")))\n",
    "        # Entering the search text\n",
    "        element.send_keys(\"{} {} lyrics\".format(artists_list[a],song))\n",
    "        # Submiting the form for google search\n",
    "        element.submit()\n",
    "        # To prevent captcha being initialized due to bot activity\n",
    "        time.sleep(10)\n",
    "        try:\n",
    "            # Fetching the default lyrics fetched by google using 'lyricfind'\n",
    "            lyric = driver.find_element_by_class_name('xaAUmb').text\n",
    "            # If default lyrics is available, set lyricstatus to True\n",
    "            # Lyric Status True indicates lyrics are available\n",
    "            status = True\n",
    "        except:\n",
    "            # If default lyrics is not available, set lyricstatus to False\n",
    "            # Lyric Status False indicates lyrics are not available\n",
    "            status = False\n",
    "            lyric = None\n",
    "        try:\n",
    "            # Fetching genre and release year of the song\n",
    "            info = driver.find_elements_by_css_selector('div.rVusze')\n",
    "            for i in info:\n",
    "                if i.find_element_by_class_name('w8qArf').text == 'Released:':\n",
    "                    release_year = i.find_element_by_class_name('LrzXr').text\n",
    "                if i.find_element_by_class_name('w8qArf').text == 'Genres:':\n",
    "                    genre = i.find_element_by_class_name('LrzXr').text\n",
    "            if len(release_year) == 0:\n",
    "                release_year = None\n",
    "            if len(genre) == 0:\n",
    "                genre = None\n",
    "        except:\n",
    "            # If genre and year information is not available\n",
    "            release_year = None\n",
    "            genre = None\n",
    "        # Adding the song, artist and lyrics to dataframe\n",
    "        df.loc[len(df.index)] = [year, song, artists_list[a], status, lyric, release_year, genre] \n",
    "        a+=1\n",
    "        # Browsing back to Google search home page\n",
    "        driver.back()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "05f0622b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Terminating a session and closing the active browser\n",
    "driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "bd2a0d64",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Top100Year</th>\n",
       "      <th>SongTitle</th>\n",
       "      <th>Artist</th>\n",
       "      <th>LyricsStatus</th>\n",
       "      <th>Lyrics</th>\n",
       "      <th>ReleaseYear</th>\n",
       "      <th>Genre</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2012</td>\n",
       "      <td>Call Me Maybe</td>\n",
       "      <td>Carly Rae Jepsen</td>\n",
       "      <td>True</td>\n",
       "      <td>I threw a wish in the well\\nDon't ask me, I'll...</td>\n",
       "      <td>2012</td>\n",
       "      <td>R&amp;B/Soul, Pop</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2012</td>\n",
       "      <td>Payphone</td>\n",
       "      <td>Maroon 5 Featuring Wiz Khalifa</td>\n",
       "      <td>True</td>\n",
       "      <td>I'm at a payphone trying to call home\\nAll of ...</td>\n",
       "      <td>2012</td>\n",
       "      <td>R&amp;B/Soul, Pop</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2012</td>\n",
       "      <td>Somebody That I Used To Know</td>\n",
       "      <td>Gotye Featuring Kimbra</td>\n",
       "      <td>True</td>\n",
       "      <td>Now and then I think of when we were together\\...</td>\n",
       "      <td>2011</td>\n",
       "      <td>R&amp;B/Soul, Pop</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2012</td>\n",
       "      <td>Wide Awake</td>\n",
       "      <td>Katy Perry</td>\n",
       "      <td>True</td>\n",
       "      <td>I'm wide awake\\nI'm wide awake\\nI'm wide awake...</td>\n",
       "      <td>2012</td>\n",
       "      <td>R&amp;B/Soul, Pop, UK R&amp;B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2012</td>\n",
       "      <td>Where Have You Been</td>\n",
       "      <td>Rihanna</td>\n",
       "      <td>True</td>\n",
       "      <td>I've been everywhere, man\\nLooking for someone...</td>\n",
       "      <td>2011</td>\n",
       "      <td>Dance-pop, R&amp;B/Soul, Electro house, Pop, UK R&amp;B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1095</th>\n",
       "      <td>2022</td>\n",
       "      <td>N95</td>\n",
       "      <td>Kendrick Lamar</td>\n",
       "      <td>True</td>\n",
       "      <td>Hello, new world, all the boys and girls\\nI go...</td>\n",
       "      <td>2022</td>\n",
       "      <td>Pop, Country</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1096</th>\n",
       "      <td>2022</td>\n",
       "      <td>Love Me More</td>\n",
       "      <td>Sam Smith</td>\n",
       "      <td>True</td>\n",
       "      <td>Have you ever felt like being somebody else?\\n...</td>\n",
       "      <td>2022</td>\n",
       "      <td>R&amp;B/Soul, Pop</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1097</th>\n",
       "      <td>2022</td>\n",
       "      <td>New Truck</td>\n",
       "      <td>Dylan Scott</td>\n",
       "      <td>True</td>\n",
       "      <td>I need a new truck\\nThere ain't nothing wrong ...</td>\n",
       "      <td>2022</td>\n",
       "      <td>R&amp;B/Soul, Pop</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1098</th>\n",
       "      <td>2022</td>\n",
       "      <td>She's All I Wanna Be</td>\n",
       "      <td>Tate McRae</td>\n",
       "      <td>True</td>\n",
       "      <td>You want the girl with the small waist\\nAnd th...</td>\n",
       "      <td>2022</td>\n",
       "      <td>R&amp;B/Soul, Pop</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099</th>\n",
       "      <td>2022</td>\n",
       "      <td>Desesperados</td>\n",
       "      <td>Rauw Alejandro &amp; Chencho Corleone</td>\n",
       "      <td>True</td>\n",
       "      <td>¿Qué me hizo usted?\\nQue la quiero volver a ve...</td>\n",
       "      <td>2021</td>\n",
       "      <td>R&amp;B/Soul, Pop</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1100 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Top100Year                     SongTitle  \\\n",
       "0          2012                 Call Me Maybe   \n",
       "1          2012                      Payphone   \n",
       "2          2012  Somebody That I Used To Know   \n",
       "3          2012                    Wide Awake   \n",
       "4          2012           Where Have You Been   \n",
       "...         ...                           ...   \n",
       "1095       2022                           N95   \n",
       "1096       2022                  Love Me More   \n",
       "1097       2022                     New Truck   \n",
       "1098       2022          She's All I Wanna Be   \n",
       "1099       2022                  Desesperados   \n",
       "\n",
       "                                 Artist LyricsStatus  \\\n",
       "0                      Carly Rae Jepsen         True   \n",
       "1        Maroon 5 Featuring Wiz Khalifa         True   \n",
       "2                Gotye Featuring Kimbra         True   \n",
       "3                            Katy Perry         True   \n",
       "4                               Rihanna         True   \n",
       "...                                 ...          ...   \n",
       "1095                     Kendrick Lamar         True   \n",
       "1096                          Sam Smith         True   \n",
       "1097                        Dylan Scott         True   \n",
       "1098                         Tate McRae         True   \n",
       "1099  Rauw Alejandro & Chencho Corleone         True   \n",
       "\n",
       "                                                 Lyrics ReleaseYear  \\\n",
       "0     I threw a wish in the well\\nDon't ask me, I'll...        2012   \n",
       "1     I'm at a payphone trying to call home\\nAll of ...        2012   \n",
       "2     Now and then I think of when we were together\\...        2011   \n",
       "3     I'm wide awake\\nI'm wide awake\\nI'm wide awake...        2012   \n",
       "4     I've been everywhere, man\\nLooking for someone...        2011   \n",
       "...                                                 ...         ...   \n",
       "1095  Hello, new world, all the boys and girls\\nI go...        2022   \n",
       "1096  Have you ever felt like being somebody else?\\n...        2022   \n",
       "1097  I need a new truck\\nThere ain't nothing wrong ...        2022   \n",
       "1098  You want the girl with the small waist\\nAnd th...        2022   \n",
       "1099  ¿Qué me hizo usted?\\nQue la quiero volver a ve...        2021   \n",
       "\n",
       "                                                Genre  \n",
       "0                                       R&B/Soul, Pop  \n",
       "1                                       R&B/Soul, Pop  \n",
       "2                                       R&B/Soul, Pop  \n",
       "3                               R&B/Soul, Pop, UK R&B  \n",
       "4     Dance-pop, R&B/Soul, Electro house, Pop, UK R&B  \n",
       "...                                               ...  \n",
       "1095                                     Pop, Country  \n",
       "1096                                    R&B/Soul, Pop  \n",
       "1097                                    R&B/Soul, Pop  \n",
       "1098                                    R&B/Soul, Pop  \n",
       "1099                                    R&B/Soul, Pop  \n",
       "\n",
       "[1100 rows x 7 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "8b89e38f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Converting DataFrame to CSV\n",
    "df.to_csv('billboard_2012_to_2022_top_100_song_lyrics.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c19257dd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}