Another_way_of_scraping_Companies.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "dietary-arizona",
   "metadata": {},
   "source": [
    "# Web Scraping of Companies in \"Welcome to the Jungle\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "naked-flash",
   "metadata": {},
   "source": [
    "Web scraping of all the companies at Welcome to the Jungle website, their characteristics and number of positions open."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "alert-recorder",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "floppy-regression",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initializing the dataframe to store the companies database:\n",
    "companies = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "uniform-consistency",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>languages</th>\n",
       "      <th>name</th>\n",
       "      <th>alternative_names</th>\n",
       "      <th>reference</th>\n",
       "      <th>slug</th>\n",
       "      <th>jobs_count</th>\n",
       "      <th>sectors</th>\n",
       "      <th>labels</th>\n",
       "      <th>nb_employees</th>\n",
       "      <th>offices</th>\n",
       "      <th>...</th>\n",
       "      <th>_highlightResult.sectors_name.cs</th>\n",
       "      <th>_highlightResult.sectors_name.sk</th>\n",
       "      <th>_highlightResult.offices</th>\n",
       "      <th>_highlightResult.tools</th>\n",
       "      <th>_highlightResult.tools_name</th>\n",
       "      <th>cover_image.fr</th>\n",
       "      <th>cover_image.en</th>\n",
       "      <th>cover_image.es</th>\n",
       "      <th>cover_image.cs</th>\n",
       "      <th>cover_image.sk</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[fr]</td>\n",
       "      <td>Tomcat Factory</td>\n",
       "      <td></td>\n",
       "      <td>aV9p0GJ</td>\n",
       "      <td>tomcat-factory</td>\n",
       "      <td>2</td>\n",
       "      <td>[{'name': {'fr': 'Incubateur / Accélérateur', ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>7.0</td>\n",
       "      <td>[{'city': 'Levallois-Perret', 'country': 'Fran...</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'Sektor podnikových služeb': {'value': 'Incu...</td>\n",
       "      <td>[{'Sektor podnikových služieb': {'value': 'Ink...</td>\n",
       "      <td>[{'city': {'value': 'Levallois-Perret', 'match...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[fr]</td>\n",
       "      <td>MyUnisoft</td>\n",
       "      <td></td>\n",
       "      <td>ZJ7ozLL</td>\n",
       "      <td>myunisoft</td>\n",
       "      <td>2</td>\n",
       "      <td>[{'name': {'fr': 'Expertise comptable', 'en': ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>36.0</td>\n",
       "      <td>[{'city': 'Massy', 'country': 'France', 'count...</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'Poradenství a audit': {'value': 'Účetnictví...</td>\n",
       "      <td>[{'Poradenstvo a audit': {'value': 'Účtovníctv...</td>\n",
       "      <td>[{'city': {'value': 'Massy', 'matchLevel': 'no...</td>\n",
       "      <td>[{'backend': {'value': 'Node.js', 'matchLevel'...</td>\n",
       "      <td>[{'backend': {'value': 'Node.js', 'matchLevel'...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[en, fr]</td>\n",
       "      <td>Quable</td>\n",
       "      <td></td>\n",
       "      <td>NZ8pzZ3</td>\n",
       "      <td>quable</td>\n",
       "      <td>5</td>\n",
       "      <td>[{'name': {'fr': 'Digital Marketing / Data Mar...</td>\n",
       "      <td>[]</td>\n",
       "      <td>25.0</td>\n",
       "      <td>[{'city': 'Aix-En-Provence', 'country': 'Franc...</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'Poradenství a audit': {'value': 'Digital Ma...</td>\n",
       "      <td>[{'Poradenstvo a audit': {'value': 'Digitálny ...</td>\n",
       "      <td>[{'city': {'value': 'Aix-En-Provence', 'matchL...</td>\n",
       "      <td>[{'backend': {'value': 'PHP', 'matchLevel': 'n...</td>\n",
       "      <td>[{'backend': {'value': 'PHP', 'matchLevel': 'n...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[en, fr]</td>\n",
       "      <td>Innovafeed</td>\n",
       "      <td></td>\n",
       "      <td>46V9oyQ</td>\n",
       "      <td>innovafeed-1</td>\n",
       "      <td>33</td>\n",
       "      <td>[{'name': {'fr': 'Agroalimentaire / Nutrition ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>150.0</td>\n",
       "      <td>[{'city': 'Decatur', 'country': 'United States...</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'Průmysl / Zemědělství': {'value': 'Krmiva /...</td>\n",
       "      <td>[{'Priemysel': {'value': 'Krmivá / Výživa zvie...</td>\n",
       "      <td>[{'city': {'value': 'Decatur', 'matchLevel': '...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[fr]</td>\n",
       "      <td>Ubitransport</td>\n",
       "      <td></td>\n",
       "      <td>j8K4Yez</td>\n",
       "      <td>ubitransport</td>\n",
       "      <td>15</td>\n",
       "      <td>[{'name': {'fr': 'Logiciels', 'en': 'Software'...</td>\n",
       "      <td>[]</td>\n",
       "      <td>110.0</td>\n",
       "      <td>[{'city': 'Besançon', 'country': 'France', 'co...</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'Tech': {'value': 'Software', 'matchLevel': ...</td>\n",
       "      <td>[{'Technológie': {'value': 'Softvér', 'matchLe...</td>\n",
       "      <td>[{'city': {'value': 'Besançon', 'matchLevel': ...</td>\n",
       "      <td>[{'backend': {'value': 'PHP', 'matchLevel': 'n...</td>\n",
       "      <td>[{'backend': {'value': 'PHP', 'matchLevel': 'n...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[fr]</td>\n",
       "      <td>Isifid</td>\n",
       "      <td></td>\n",
       "      <td>MbOxJeQ</td>\n",
       "      <td>isifid</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'name': {'fr': 'FinTech / InsurTech', 'en': ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>13.0</td>\n",
       "      <td>[{'city': 'Paris', 'country': 'France', 'count...</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'Bankovnictví / Pojišťovnictví / Finance': {...</td>\n",
       "      <td>[{'Bankovníctvo / Poisťovníctvo / Financie': {...</td>\n",
       "      <td>[{'city': {'value': 'Paris', 'matchLevel': 'no...</td>\n",
       "      <td>[{'backend': {'value': 'PostgreSQL', 'matchLev...</td>\n",
       "      <td>[{'backend': {'value': 'PostgreSQL', 'matchLev...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>[fr]</td>\n",
       "      <td>SEKOIA</td>\n",
       "      <td></td>\n",
       "      <td>D9z7WGl</td>\n",
       "      <td>sekoia</td>\n",
       "      <td>6</td>\n",
       "      <td>[{'name': {'fr': 'Intelligence artificielle / ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>85.0</td>\n",
       "      <td>[{'city': 'Paris', 'country': 'France', 'count...</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'Tech': {'value': 'Artificial Intelligence /...</td>\n",
       "      <td>[{'Technológie': {'value': 'Umelá inteligencia...</td>\n",
       "      <td>[{'city': {'value': 'Paris', 'matchLevel': 'no...</td>\n",
       "      <td>[{'backend': {'value': 'Node.js', 'matchLevel'...</td>\n",
       "      <td>[{'backend': {'value': 'Node.js', 'matchLevel'...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>[fr]</td>\n",
       "      <td>DriiveMe</td>\n",
       "      <td></td>\n",
       "      <td>LqzY4Q</td>\n",
       "      <td>driiveme</td>\n",
       "      <td>4</td>\n",
       "      <td>[{'name': {'fr': 'Logistique', 'en': 'Logistic...</td>\n",
       "      <td>[]</td>\n",
       "      <td>30.0</td>\n",
       "      <td>[{'city': 'Paris', 'country': 'France', 'count...</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'Mobilita / Transport': {'value': 'Logistika...</td>\n",
       "      <td>[{'Mobilita / Doprava': {'value': 'Logistika',...</td>\n",
       "      <td>[{'city': {'value': 'Paris', 'matchLevel': 'no...</td>\n",
       "      <td>[{'frontend': {'value': 'Bootstrap', 'matchLev...</td>\n",
       "      <td>[{'frontend': {'value': 'Bootstrap', 'matchLev...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>[en, fr]</td>\n",
       "      <td>VO2 GROUP</td>\n",
       "      <td></td>\n",
       "      <td>OPmkMmj</td>\n",
       "      <td>vo2-group</td>\n",
       "      <td>7</td>\n",
       "      <td>[{'name': {'fr': 'IT / Digital', 'en': 'IT / D...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>300.0</td>\n",
       "      <td>[{'city': 'Casablanca', 'country': 'Morocco', ...</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'Poradenství a audit': {'value': 'IT / Digit...</td>\n",
       "      <td>[{'Poradenstvo a audit': {'value': 'IT / Digit...</td>\n",
       "      <td>[{'city': {'value': 'Casablanca', 'matchLevel'...</td>\n",
       "      <td>[{'backend': {'value': 'Node.js', 'matchLevel'...</td>\n",
       "      <td>[{'backend': {'value': 'Node.js', 'matchLevel'...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>[fr]</td>\n",
       "      <td>LANCASTER</td>\n",
       "      <td></td>\n",
       "      <td>xMKjgwA</td>\n",
       "      <td>lancaster</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'name': {'fr': 'Luxe', 'en': 'Luxury', 'es':...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100.0</td>\n",
       "      <td>[{'city': 'Paris', 'country': 'France', 'count...</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'Fashion / Luxury / Beauty / Lifestyle': {'v...</td>\n",
       "      <td>[{'Móda / Luxus / Krása / Životný štýl': {'val...</td>\n",
       "      <td>[{'city': {'value': 'Paris', 'matchLevel': 'no...</td>\n",
       "      <td>[{'misc': {'value': 'Excel', 'matchLevel': 'no...</td>\n",
       "      <td>[{'misc': {'value': 'Excel', 'matchLevel': 'no...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 72 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   languages             name alternative_names reference            slug  \\\n",
       "0       [fr]  Tomcat Factory                      aV9p0GJ  tomcat-factory   \n",
       "1       [fr]        MyUnisoft                     ZJ7ozLL       myunisoft   \n",
       "2   [en, fr]           Quable                     NZ8pzZ3          quable   \n",
       "3   [en, fr]      Innovafeed                      46V9oyQ    innovafeed-1   \n",
       "4       [fr]     Ubitransport                     j8K4Yez    ubitransport   \n",
       "..       ...              ...               ...       ...             ...   \n",
       "5       [fr]           Isifid                     MbOxJeQ          isifid   \n",
       "6       [fr]           SEKOIA                     D9z7WGl          sekoia   \n",
       "7       [fr]         DriiveMe                      LqzY4Q        driiveme   \n",
       "8   [en, fr]        VO2 GROUP                     OPmkMmj       vo2-group   \n",
       "9       [fr]        LANCASTER                     xMKjgwA       lancaster   \n",
       "\n",
       "    jobs_count                                            sectors labels  \\\n",
       "0            2  [{'name': {'fr': 'Incubateur / Accélérateur', ...     []   \n",
       "1            2  [{'name': {'fr': 'Expertise comptable', 'en': ...     []   \n",
       "2            5  [{'name': {'fr': 'Digital Marketing / Data Mar...     []   \n",
       "3           33  [{'name': {'fr': 'Agroalimentaire / Nutrition ...     []   \n",
       "4           15  [{'name': {'fr': 'Logiciels', 'en': 'Software'...     []   \n",
       "..         ...                                                ...    ...   \n",
       "5            0  [{'name': {'fr': 'FinTech / InsurTech', 'en': ...     []   \n",
       "6            6  [{'name': {'fr': 'Intelligence artificielle / ...     []   \n",
       "7            4  [{'name': {'fr': 'Logistique', 'en': 'Logistic...     []   \n",
       "8            7  [{'name': {'fr': 'IT / Digital', 'en': 'IT / D...    NaN   \n",
       "9            0  [{'name': {'fr': 'Luxe', 'en': 'Luxury', 'es':...    NaN   \n",
       "\n",
       "    nb_employees                                            offices  ...  \\\n",
       "0            7.0  [{'city': 'Levallois-Perret', 'country': 'Fran...  ...   \n",
       "1           36.0  [{'city': 'Massy', 'country': 'France', 'count...  ...   \n",
       "2           25.0  [{'city': 'Aix-En-Provence', 'country': 'Franc...  ...   \n",
       "3          150.0  [{'city': 'Decatur', 'country': 'United States...  ...   \n",
       "4          110.0  [{'city': 'Besançon', 'country': 'France', 'co...  ...   \n",
       "..           ...                                                ...  ...   \n",
       "5           13.0  [{'city': 'Paris', 'country': 'France', 'count...  ...   \n",
       "6           85.0  [{'city': 'Paris', 'country': 'France', 'count...  ...   \n",
       "7           30.0  [{'city': 'Paris', 'country': 'France', 'count...  ...   \n",
       "8          300.0  [{'city': 'Casablanca', 'country': 'Morocco', ...  ...   \n",
       "9          100.0  [{'city': 'Paris', 'country': 'France', 'count...  ...   \n",
       "\n",
       "                     _highlightResult.sectors_name.cs  \\\n",
       "0   [{'Sektor podnikových služeb': {'value': 'Incu...   \n",
       "1   [{'Poradenství a audit': {'value': 'Účetnictví...   \n",
       "2   [{'Poradenství a audit': {'value': 'Digital Ma...   \n",
       "3   [{'Průmysl / Zemědělství': {'value': 'Krmiva /...   \n",
       "4   [{'Tech': {'value': 'Software', 'matchLevel': ...   \n",
       "..                                                ...   \n",
       "5   [{'Bankovnictví / Pojišťovnictví / Finance': {...   \n",
       "6   [{'Tech': {'value': 'Artificial Intelligence /...   \n",
       "7   [{'Mobilita / Transport': {'value': 'Logistika...   \n",
       "8   [{'Poradenství a audit': {'value': 'IT / Digit...   \n",
       "9   [{'Fashion / Luxury / Beauty / Lifestyle': {'v...   \n",
       "\n",
       "                     _highlightResult.sectors_name.sk  \\\n",
       "0   [{'Sektor podnikových služieb': {'value': 'Ink...   \n",
       "1   [{'Poradenstvo a audit': {'value': 'Účtovníctv...   \n",
       "2   [{'Poradenstvo a audit': {'value': 'Digitálny ...   \n",
       "3   [{'Priemysel': {'value': 'Krmivá / Výživa zvie...   \n",
       "4   [{'Technológie': {'value': 'Softvér', 'matchLe...   \n",
       "..                                                ...   \n",
       "5   [{'Bankovníctvo / Poisťovníctvo / Financie': {...   \n",
       "6   [{'Technológie': {'value': 'Umelá inteligencia...   \n",
       "7   [{'Mobilita / Doprava': {'value': 'Logistika',...   \n",
       "8   [{'Poradenstvo a audit': {'value': 'IT / Digit...   \n",
       "9   [{'Móda / Luxus / Krása / Životný štýl': {'val...   \n",
       "\n",
       "                             _highlightResult.offices  \\\n",
       "0   [{'city': {'value': 'Levallois-Perret', 'match...   \n",
       "1   [{'city': {'value': 'Massy', 'matchLevel': 'no...   \n",
       "2   [{'city': {'value': 'Aix-En-Provence', 'matchL...   \n",
       "3   [{'city': {'value': 'Decatur', 'matchLevel': '...   \n",
       "4   [{'city': {'value': 'Besançon', 'matchLevel': ...   \n",
       "..                                                ...   \n",
       "5   [{'city': {'value': 'Paris', 'matchLevel': 'no...   \n",
       "6   [{'city': {'value': 'Paris', 'matchLevel': 'no...   \n",
       "7   [{'city': {'value': 'Paris', 'matchLevel': 'no...   \n",
       "8   [{'city': {'value': 'Casablanca', 'matchLevel'...   \n",
       "9   [{'city': {'value': 'Paris', 'matchLevel': 'no...   \n",
       "\n",
       "                               _highlightResult.tools  \\\n",
       "0                                                 NaN   \n",
       "1   [{'backend': {'value': 'Node.js', 'matchLevel'...   \n",
       "2   [{'backend': {'value': 'PHP', 'matchLevel': 'n...   \n",
       "3                                                 NaN   \n",
       "4   [{'backend': {'value': 'PHP', 'matchLevel': 'n...   \n",
       "..                                                ...   \n",
       "5   [{'backend': {'value': 'PostgreSQL', 'matchLev...   \n",
       "6   [{'backend': {'value': 'Node.js', 'matchLevel'...   \n",
       "7   [{'frontend': {'value': 'Bootstrap', 'matchLev...   \n",
       "8   [{'backend': {'value': 'Node.js', 'matchLevel'...   \n",
       "9   [{'misc': {'value': 'Excel', 'matchLevel': 'no...   \n",
       "\n",
       "                          _highlightResult.tools_name cover_image.fr  \\\n",
       "0                                                 NaN            NaN   \n",
       "1   [{'backend': {'value': 'Node.js', 'matchLevel'...            NaN   \n",
       "2   [{'backend': {'value': 'PHP', 'matchLevel': 'n...            NaN   \n",
       "3                                                 NaN            NaN   \n",
       "4   [{'backend': {'value': 'PHP', 'matchLevel': 'n...            NaN   \n",
       "..                                                ...            ...   \n",
       "5   [{'backend': {'value': 'PostgreSQL', 'matchLev...            NaN   \n",
       "6   [{'backend': {'value': 'Node.js', 'matchLevel'...            NaN   \n",
       "7   [{'frontend': {'value': 'Bootstrap', 'matchLev...            NaN   \n",
       "8   [{'backend': {'value': 'Node.js', 'matchLevel'...            NaN   \n",
       "9   [{'misc': {'value': 'Excel', 'matchLevel': 'no...            NaN   \n",
       "\n",
       "   cover_image.en cover_image.es cover_image.cs cover_image.sk  \n",
       "0             NaN            NaN            NaN            NaN  \n",
       "1             NaN            NaN            NaN            NaN  \n",
       "2             NaN            NaN            NaN            NaN  \n",
       "3             NaN            NaN            NaN            NaN  \n",
       "4             NaN            NaN            NaN            NaN  \n",
       "..            ...            ...            ...            ...  \n",
       "5             NaN            NaN            NaN            NaN  \n",
       "6             NaN            NaN            NaN            NaN  \n",
       "7             NaN            NaN            NaN            NaN  \n",
       "8             NaN            NaN            NaN            NaN  \n",
       "9             NaN            NaN            NaN            NaN  \n",
       "\n",
       "[1000 rows x 72 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def get_data():\n",
    "    # Initializing the dataframe to store the companies database:\n",
    "    companies = pd.DataFrame()\n",
    "    \n",
    "    # Basic fixed variables for the response\n",
    "    url = \"https://csekhvms53-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=Algolia%20for%20vanilla%20JavaScript%20(lite)%203.24.12%3Breact-instantsearch%205.3.2%3BJS%20Helper%20(2.28.1)&x-algolia-application-id=CSEKHVMS53&x-algolia-api-key=02f0d440abc99cae37e126886438b266\"\n",
    "    text = \"\"\"accept: application/json\n",
    "    Accept-Encoding: gzip, deflate, br\n",
    "    Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\n",
    "    Cache-Control: no-cache\n",
    "    Connection: keep-alive\n",
    "    Content-Length: 665\n",
    "    content-type: application/x-www-form-urlencoded\n",
    "    Host: csekhvms53-dsn.algolia.net\n",
    "    Origin: https://www.welcometothejungle.com\n",
    "    Pragma: no-cache\n",
    "    Referer: https://www.welcometothejungle.com/en/companies?page=1\n",
    "    Sec-Fetch-Dest: empty\n",
    "    Sec-Fetch-Mode: cors\n",
    "    Sec-Fetch-Site: cross-site\n",
    "    User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36\"\"\"\n",
    "    headers = {h[0]:h[1] for x in text.split('\\n') if (h := x.split(': '))}\n",
    "    \n",
    "    # Variable necessary because of the POST request and also the variable that rules over the pagination:\n",
    "    payload = \"\"\"{\"requests\":[{\"indexName\":\"wk_cms_organizations_production\",\"params\":\"query=&hitsPerPage=30&maxValuesPerFacet=100&page=0&restrictSearchableAttributes=%5B%22name%22%2C%22alternative_names%22%2C%22offices.city%22%2C%22offices.state%22%2C%22offices.country%22%2C%22sectors.name.en%22%2C%22sectors.parent.en%22%2C%22tools_name%22%5D&highlightPreTag=%3Cais-highlight-0000000000%3E&highlightPostTag=%3C%2Fais-highlight-0000000000%3E&aroundLatLng=48.85718%2C2.34141&aroundRadius=20000&aroundPrecision=20000&filters=website.reference%3Awttj_fr&facets=%5B%22offices.country_code%22%2C%22offices.state%22%2C%22offices.district%22%2C%22offices.location%22%2C%22online%22%2C%22sectors.parent.en%22%2C%22size.en%22%2C%22languages%22%5D&tagFilters=&facetFilters=%5B%22offices.country_code%3AFR%22%5D\"}]}\"\"\"\n",
    "    \n",
    "    # Connecting to the website and gettign the first results (on the first page)\n",
    "    response = requests.post(url, headers=headers, data=payload)\n",
    "    data = response.json()[\"results\"][0][\"hits\"]\n",
    "\n",
    "    # Agregating the first rows of data in the dataframe:\n",
    "    companies = pd.concat([companies,pd.json_normalize(data)])\n",
    "\n",
    "    # Calculating the maximum number of pages for scrapping\n",
    "    totalpages = int(np.ceil(response.json()[\"results\"][0][\"nbPages\"]))\n",
    "    totalpages\n",
    "\n",
    "    # Looping through pages 2 until the end of the results requests:\n",
    "    for i in range(totalpages-1):\n",
    "        \n",
    "        # Changing the payload variable to adjust for pagination and updating the \"response\" and \"data\" variables to get the next page:\n",
    "        payload = \"\"\"{\"requests\":[{\"indexName\":\"wk_cms_organizations_production\",\"params\":\"query=&hitsPerPage=30&maxValuesPerFacet=100&page=\"\"\"+str(i+1)+\"\"\"&restrictSearchableAttributes=%5B%22name%22%2C%22alternative_names%22%2C%22offices.city%22%2C%22offices.state%22%2C%22offices.country%22%2C%22sectors.name.en%22%2C%22sectors.parent.en%22%2C%22tools_name%22%5D&highlightPreTag=%3Cais-highlight-0000000000%3E&highlightPostTag=%3C%2Fais-highlight-0000000000%3E&aroundLatLng=48.85718%2C2.34141&aroundRadius=20000&aroundPrecision=20000&filters=website.reference%3Awttj_fr&facets=%5B%22offices.country_code%22%2C%22offices.state%22%2C%22offices.district%22%2C%22offices.location%22%2C%22online%22%2C%22sectors.parent.en%22%2C%22size.en%22%2C%22languages%22%5D&tagFilters=&facetFilters=%5B%22offices.country_code%3AFR%22%5D\"}]}\"\"\"\n",
    "        response = requests.post(url, headers=headers, data=payload)\n",
    "        data = response.json()[\"results\"][0][\"hits\"]\n",
    "\n",
    "        # Agregating data in the dataframe created for this finality\n",
    "        companies = pd.concat([companies,pd.json_normalize(data)])\n",
    "    \n",
    "    # Resetting index so it would be representative of the Company ID\n",
    "    companies.set_index(\"reference\", inplace=True)\n",
    "    \n",
    "    return companies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "illegal-contrast",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['languages', 'name', 'alternative_names', 'reference', 'slug', 'jobs_count', 'sectors', 'labels', 'nb_employees', 'offices', 'tools', 'tools_name', 'published_at', 'has_tech_page', '_geoloc', 'objectID', 'website.reference', 'size.fr', 'size.en', 'size.es', 'size.cs', 'size.sk', 'sectors_name.fr', 'sectors_name.en', 'sectors_name.es', 'sectors_name.cs', 'sectors_name.sk', 'cover_image.fr.url', 'cover_image.fr.large.url', 'cover_image.fr.thumb.url', 'cover_image.fr.small.url', 'cover_image.en.url', 'cover_image.en.large.url', 'cover_image.en.thumb.url', 'cover_image.en.small.url', 'cover_image.es.url', 'cover_image.es.large.url', 'cover_image.es.thumb.url', 'cover_image.es.small.url', 'cover_image.cs.url', 'cover_image.cs.large.url', 'cover_image.cs.thumb.url', 'cover_image.cs.small.url', 'cover_image.sk.url', 'cover_image.sk.large.url', 'cover_image.sk.thumb.url', 'cover_image.sk.small.url', 'logo.url', 'logo.thumb.url', '_highlightResult.name.value', '_highlightResult.name.matchLevel', '_highlightResult.name.matchedWords', '_highlightResult.alternative_names.value', '_highlightResult.alternative_names.matchLevel', '_highlightResult.alternative_names.matchedWords', '_highlightResult.slug.value', '_highlightResult.slug.matchLevel', '_highlightResult.slug.matchedWords', '_highlightResult.sectors', '_highlightResult.sectors_name.fr', '_highlightResult.sectors_name.en', '_highlightResult.sectors_name.es', '_highlightResult.sectors_name.cs', '_highlightResult.sectors_name.sk', '_highlightResult.offices', '_highlightResult.tools', '_highlightResult.tools_name', 'cover_image.fr', 'cover_image.en', 'cover_image.es', 'cover_image.cs', 'cover_image.sk']\n"
     ]
    }
   ],
   "source": [
    "col = [c for c in companies.columns]\n",
    "print(col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "legitimate-harrison",
   "metadata": {},
   "outputs": [],
   "source": [
    "def drop_columns(df):\n",
    "    \n",
    "    # Droping columns that present all NaN values:\n",
    "    df.dropna(axis=1, how=\"all\", inplace=True)\n",
    "    \n",
    "    # Initializing the variable that will store the columns' names to be droped\n",
    "    columns_to_drop=[]\n",
    "    \n",
    "    # Droping all columns that present at least 80% of NaN values:\n",
    "    limit = 0.3 * df.shape[0]\n",
    "    \n",
    "    for col in df.columns:\n",
    "        if df[col].isna().sum()>=limit:\n",
    "            columns_to_drop += col\n",
    "    \n",
    "    # Droping columns that are not in English language (\".en\")\n",
    "    for language in [\".cs\", \".es\", \".fr\", \".sk\"]:\n",
    "        columns_to_drop += [c for c in df.columns if language in c]\n",
    "        \n",
    "    # Droping from all useless columns that were manually identified as useless\n",
    "    for col_name in [\"_highlightResult.\", \"_image.\", \"logo.\", 'alternative_names', 'labels']:\n",
    "        columns_to_drop += [c for c in df.columns if col_name in c]\n",
    "    \n",
    "    # Cleaning all duplicates from columns list:\n",
    "    columns_to_drop = list(set(columns_to_drop))\n",
    "    \n",
    "    # Finally, drop all columns previously selected\n",
    "    df.drop(columns=columns_to_drop, inplace=True)\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "personalized-minutes",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1000, 72)\n",
      "(1000, 17)\n",
      "languages\n"
     ]
    }
   ],
   "source": [
    "# Testing the effect of droping columns functions\n",
    "companies_test = companies.copy()\n",
    "print(companies_test.shape)\n",
    "companies_test = clean_columns(companies_test)\n",
    "print(companies_test.shape)\n",
    "print(companies_test.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "honey-maryland",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['languages', 'name', 'reference', 'slug', 'jobs_count', 'sectors',\n",
      "       'nb_employees', 'offices', 'tools', 'tools_name', 'published_at',\n",
      "       'has_tech_page', '_geoloc', 'objectID', 'website.reference', 'size.en',\n",
      "       'sectors_name.en'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(companies_test.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "forty-compensation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>languages</th>\n",
       "      <th>name</th>\n",
       "      <th>reference</th>\n",
       "      <th>slug</th>\n",
       "      <th>jobs_count</th>\n",
       "      <th>sectors</th>\n",
       "      <th>nb_employees</th>\n",
       "      <th>offices</th>\n",
       "      <th>tools</th>\n",
       "      <th>tools_name</th>\n",
       "      <th>published_at</th>\n",
       "      <th>has_tech_page</th>\n",
       "      <th>_geoloc</th>\n",
       "      <th>objectID</th>\n",
       "      <th>website.reference</th>\n",
       "      <th>size.en</th>\n",
       "      <th>sectors_name.en</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[fr]</td>\n",
       "      <td>Tomcat Factory</td>\n",
       "      <td>aV9p0GJ</td>\n",
       "      <td>tomcat-factory</td>\n",
       "      <td>2</td>\n",
       "      <td>[{'name': {'fr': 'Incubateur / Accélérateur', ...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>[{'city': 'Levallois-Perret', 'country': 'Fran...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>2021-02-19T15:02:44.511+01:00</td>\n",
       "      <td>False</td>\n",
       "      <td>[{'lat': 48.88737, 'lng': 2.28544}]</td>\n",
       "      <td>10944</td>\n",
       "      <td>wttj_fr</td>\n",
       "      <td>&lt; 15 employees</td>\n",
       "      <td>[{'Corporate Services': 'Incubator / Accelerat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[fr]</td>\n",
       "      <td>MyUnisoft</td>\n",
       "      <td>ZJ7ozLL</td>\n",
       "      <td>myunisoft</td>\n",
       "      <td>2</td>\n",
       "      <td>[{'name': {'fr': 'Expertise comptable', 'en': ...</td>\n",
       "      <td>36.0</td>\n",
       "      <td>[{'city': 'Massy', 'country': 'France', 'count...</td>\n",
       "      <td>[{'parent': 'backend', 'backend': 'Node.js', '...</td>\n",
       "      <td>[{'backend': 'Node.js'}, {'frontend': 'React J...</td>\n",
       "      <td>2021-02-19T11:19:39.294+01:00</td>\n",
       "      <td>True</td>\n",
       "      <td>[{'lat': 48.72308, 'lng': 2.27037}]</td>\n",
       "      <td>10980</td>\n",
       "      <td>wttj_fr</td>\n",
       "      <td>Between 15 and 50 employees</td>\n",
       "      <td>[{'Consulting / Audit': 'Accounting'}, {'Banki...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[en, fr]</td>\n",
       "      <td>Quable</td>\n",
       "      <td>NZ8pzZ3</td>\n",
       "      <td>quable</td>\n",
       "      <td>5</td>\n",
       "      <td>[{'name': {'fr': 'Digital Marketing / Data Mar...</td>\n",
       "      <td>25.0</td>\n",
       "      <td>[{'city': 'Aix-En-Provence', 'country': 'Franc...</td>\n",
       "      <td>[{'parent': 'backend', 'backend': 'PHP', 'imag...</td>\n",
       "      <td>[{'backend': 'PHP'}, {'frontend': 'JavaScript'...</td>\n",
       "      <td>2021-02-19T10:01:10.270+01:00</td>\n",
       "      <td>True</td>\n",
       "      <td>[{'lat': 43.52909, 'lng': 5.41105}, {'lat': 44...</td>\n",
       "      <td>944</td>\n",
       "      <td>wttj_fr</td>\n",
       "      <td>Between 15 and 50 employees</td>\n",
       "      <td>[{'Consulting / Audit': 'Digital Marketing / D...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[en, fr]</td>\n",
       "      <td>Innovafeed</td>\n",
       "      <td>46V9oyQ</td>\n",
       "      <td>innovafeed-1</td>\n",
       "      <td>33</td>\n",
       "      <td>[{'name': {'fr': 'Agroalimentaire / Nutrition ...</td>\n",
       "      <td>150.0</td>\n",
       "      <td>[{'city': 'Decatur', 'country': 'United States...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>2021-02-18T19:52:20.567+01:00</td>\n",
       "      <td>False</td>\n",
       "      <td>[{'lat': 33.77357, 'lng': -84.29745}, {'lat': ...</td>\n",
       "      <td>9461</td>\n",
       "      <td>wttj_fr</td>\n",
       "      <td>Between 50 and 250 employees</td>\n",
       "      <td>[{'Industry': 'Agri-food / Animal Nutrition'}]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[fr]</td>\n",
       "      <td>Ubitransport</td>\n",
       "      <td>j8K4Yez</td>\n",
       "      <td>ubitransport</td>\n",
       "      <td>15</td>\n",
       "      <td>[{'name': {'fr': 'Logiciels', 'en': 'Software'...</td>\n",
       "      <td>110.0</td>\n",
       "      <td>[{'city': 'Besançon', 'country': 'France', 'co...</td>\n",
       "      <td>[{'parent': 'backend', 'backend': 'PHP', 'imag...</td>\n",
       "      <td>[{'backend': 'PHP'}, {'frontend': 'JavaScript'...</td>\n",
       "      <td>2021-02-18T16:40:13.817+01:00</td>\n",
       "      <td>True</td>\n",
       "      <td>[{'lat': 47.2389, 'lng': 6.01377}, {'lat': 43....</td>\n",
       "      <td>6059</td>\n",
       "      <td>wttj_fr</td>\n",
       "      <td>Between 50 and 250 employees</td>\n",
       "      <td>[{'Tech': 'Software'}, {'Mobility / Transport'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[fr]</td>\n",
       "      <td>Isifid</td>\n",
       "      <td>MbOxJeQ</td>\n",
       "      <td>isifid</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'name': {'fr': 'FinTech / InsurTech', 'en': ...</td>\n",
       "      <td>13.0</td>\n",
       "      <td>[{'city': 'Paris', 'country': 'France', 'count...</td>\n",
       "      <td>[{'parent': 'backend', 'backend': 'PostgreSQL'...</td>\n",
       "      <td>[{'backend': 'PostgreSQL'}, {'backend': 'Sprin...</td>\n",
       "      <td>2020-02-13T10:58:26.761+01:00</td>\n",
       "      <td>True</td>\n",
       "      <td>[{'lat': 48.87194, 'lng': 2.34853}, {'lat': 48...</td>\n",
       "      <td>7538</td>\n",
       "      <td>wttj_fr</td>\n",
       "      <td>&lt; 15 employees</td>\n",
       "      <td>[{'Banking / Insurance / Finance': 'FinTech / ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>[fr]</td>\n",
       "      <td>SEKOIA</td>\n",
       "      <td>D9z7WGl</td>\n",
       "      <td>sekoia</td>\n",
       "      <td>6</td>\n",
       "      <td>[{'name': {'fr': 'Intelligence artificielle / ...</td>\n",
       "      <td>85.0</td>\n",
       "      <td>[{'city': 'Paris', 'country': 'France', 'count...</td>\n",
       "      <td>[{'parent': 'backend', 'backend': 'Node.js', '...</td>\n",
       "      <td>[{'backend': 'Node.js'}, {'frontend': 'Angular...</td>\n",
       "      <td>2020-02-13T10:18:15.126+01:00</td>\n",
       "      <td>True</td>\n",
       "      <td>[{'lat': 48.8700955, 'lng': 2.3256528}, {'lat'...</td>\n",
       "      <td>2536</td>\n",
       "      <td>wttj_fr</td>\n",
       "      <td>Between 50 and 250 employees</td>\n",
       "      <td>[{'Tech': 'Artificial Intelligence / Machine L...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>[fr]</td>\n",
       "      <td>DriiveMe</td>\n",
       "      <td>LqzY4Q</td>\n",
       "      <td>driiveme</td>\n",
       "      <td>4</td>\n",
       "      <td>[{'name': {'fr': 'Logistique', 'en': 'Logistic...</td>\n",
       "      <td>30.0</td>\n",
       "      <td>[{'city': 'Paris', 'country': 'France', 'count...</td>\n",
       "      <td>[{'parent': 'frontend', 'frontend': 'Bootstrap...</td>\n",
       "      <td>[{'frontend': 'Bootstrap'}, {'backend': 'PHP'}...</td>\n",
       "      <td>2020-02-13T09:43:16.404+01:00</td>\n",
       "      <td>True</td>\n",
       "      <td>[{'lat': 48.88473, 'lng': 2.31925}, {'lat': 48...</td>\n",
       "      <td>7565</td>\n",
       "      <td>wttj_fr</td>\n",
       "      <td>Between 15 and 50 employees</td>\n",
       "      <td>[{'Mobility / Transport': 'Logistics'}]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>[en, fr]</td>\n",
       "      <td>VO2 GROUP</td>\n",
       "      <td>OPmkMmj</td>\n",
       "      <td>vo2-group</td>\n",
       "      <td>7</td>\n",
       "      <td>[{'name': {'fr': 'IT / Digital', 'en': 'IT / D...</td>\n",
       "      <td>300.0</td>\n",
       "      <td>[{'city': 'Casablanca', 'country': 'Morocco', ...</td>\n",
       "      <td>[{'parent': 'backend', 'backend': 'Node.js', '...</td>\n",
       "      <td>[{'backend': 'Node.js'}, {'frontend': 'Angular...</td>\n",
       "      <td>2020-02-12T17:45:45.558+01:00</td>\n",
       "      <td>True</td>\n",
       "      <td>[{'lat': 33.59662, 'lng': -7.61889}, {'lat': 4...</td>\n",
       "      <td>1981</td>\n",
       "      <td>wttj_fr</td>\n",
       "      <td>Between 250 and 2,000 employees</td>\n",
       "      <td>[{'Consulting / Audit': 'IT / Digital'}, {'Con...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>[fr]</td>\n",
       "      <td>LANCASTER</td>\n",
       "      <td>xMKjgwA</td>\n",
       "      <td>lancaster</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'name': {'fr': 'Luxe', 'en': 'Luxury', 'es':...</td>\n",
       "      <td>100.0</td>\n",
       "      <td>[{'city': 'Paris', 'country': 'France', 'count...</td>\n",
       "      <td>[{'parent': 'misc', 'misc': 'Excel', 'image': ...</td>\n",
       "      <td>[{'misc': 'Excel'}, {'data': 'Google Analytics...</td>\n",
       "      <td>2020-02-12T16:19:49.047+01:00</td>\n",
       "      <td>False</td>\n",
       "      <td>[{'lat': 48.87265, 'lng': 2.32893}]</td>\n",
       "      <td>7561</td>\n",
       "      <td>wttj_fr</td>\n",
       "      <td>Between 50 and 250 employees</td>\n",
       "      <td>[{'Fashion / Luxury / Beauty / Lifestyle': 'Lu...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   languages             name reference            slug  jobs_count  \\\n",
       "0       [fr]  Tomcat Factory    aV9p0GJ  tomcat-factory           2   \n",
       "1       [fr]        MyUnisoft   ZJ7ozLL       myunisoft           2   \n",
       "2   [en, fr]           Quable   NZ8pzZ3          quable           5   \n",
       "3   [en, fr]      Innovafeed    46V9oyQ    innovafeed-1          33   \n",
       "4       [fr]     Ubitransport   j8K4Yez    ubitransport          15   \n",
       "..       ...              ...       ...             ...         ...   \n",
       "5       [fr]           Isifid   MbOxJeQ          isifid           0   \n",
       "6       [fr]           SEKOIA   D9z7WGl          sekoia           6   \n",
       "7       [fr]         DriiveMe    LqzY4Q        driiveme           4   \n",
       "8   [en, fr]        VO2 GROUP   OPmkMmj       vo2-group           7   \n",
       "9       [fr]        LANCASTER   xMKjgwA       lancaster           0   \n",
       "\n",
       "                                              sectors  nb_employees  \\\n",
       "0   [{'name': {'fr': 'Incubateur / Accélérateur', ...           7.0   \n",
       "1   [{'name': {'fr': 'Expertise comptable', 'en': ...          36.0   \n",
       "2   [{'name': {'fr': 'Digital Marketing / Data Mar...          25.0   \n",
       "3   [{'name': {'fr': 'Agroalimentaire / Nutrition ...         150.0   \n",
       "4   [{'name': {'fr': 'Logiciels', 'en': 'Software'...         110.0   \n",
       "..                                                ...           ...   \n",
       "5   [{'name': {'fr': 'FinTech / InsurTech', 'en': ...          13.0   \n",
       "6   [{'name': {'fr': 'Intelligence artificielle / ...          85.0   \n",
       "7   [{'name': {'fr': 'Logistique', 'en': 'Logistic...          30.0   \n",
       "8   [{'name': {'fr': 'IT / Digital', 'en': 'IT / D...         300.0   \n",
       "9   [{'name': {'fr': 'Luxe', 'en': 'Luxury', 'es':...         100.0   \n",
       "\n",
       "                                              offices  \\\n",
       "0   [{'city': 'Levallois-Perret', 'country': 'Fran...   \n",
       "1   [{'city': 'Massy', 'country': 'France', 'count...   \n",
       "2   [{'city': 'Aix-En-Provence', 'country': 'Franc...   \n",
       "3   [{'city': 'Decatur', 'country': 'United States...   \n",
       "4   [{'city': 'Besançon', 'country': 'France', 'co...   \n",
       "..                                                ...   \n",
       "5   [{'city': 'Paris', 'country': 'France', 'count...   \n",
       "6   [{'city': 'Paris', 'country': 'France', 'count...   \n",
       "7   [{'city': 'Paris', 'country': 'France', 'count...   \n",
       "8   [{'city': 'Casablanca', 'country': 'Morocco', ...   \n",
       "9   [{'city': 'Paris', 'country': 'France', 'count...   \n",
       "\n",
       "                                                tools  \\\n",
       "0                                                  []   \n",
       "1   [{'parent': 'backend', 'backend': 'Node.js', '...   \n",
       "2   [{'parent': 'backend', 'backend': 'PHP', 'imag...   \n",
       "3                                                  []   \n",
       "4   [{'parent': 'backend', 'backend': 'PHP', 'imag...   \n",
       "..                                                ...   \n",
       "5   [{'parent': 'backend', 'backend': 'PostgreSQL'...   \n",
       "6   [{'parent': 'backend', 'backend': 'Node.js', '...   \n",
       "7   [{'parent': 'frontend', 'frontend': 'Bootstrap...   \n",
       "8   [{'parent': 'backend', 'backend': 'Node.js', '...   \n",
       "9   [{'parent': 'misc', 'misc': 'Excel', 'image': ...   \n",
       "\n",
       "                                           tools_name  \\\n",
       "0                                                  []   \n",
       "1   [{'backend': 'Node.js'}, {'frontend': 'React J...   \n",
       "2   [{'backend': 'PHP'}, {'frontend': 'JavaScript'...   \n",
       "3                                                  []   \n",
       "4   [{'backend': 'PHP'}, {'frontend': 'JavaScript'...   \n",
       "..                                                ...   \n",
       "5   [{'backend': 'PostgreSQL'}, {'backend': 'Sprin...   \n",
       "6   [{'backend': 'Node.js'}, {'frontend': 'Angular...   \n",
       "7   [{'frontend': 'Bootstrap'}, {'backend': 'PHP'}...   \n",
       "8   [{'backend': 'Node.js'}, {'frontend': 'Angular...   \n",
       "9   [{'misc': 'Excel'}, {'data': 'Google Analytics...   \n",
       "\n",
       "                     published_at has_tech_page  \\\n",
       "0   2021-02-19T15:02:44.511+01:00         False   \n",
       "1   2021-02-19T11:19:39.294+01:00          True   \n",
       "2   2021-02-19T10:01:10.270+01:00          True   \n",
       "3   2021-02-18T19:52:20.567+01:00         False   \n",
       "4   2021-02-18T16:40:13.817+01:00          True   \n",
       "..                            ...           ...   \n",
       "5   2020-02-13T10:58:26.761+01:00          True   \n",
       "6   2020-02-13T10:18:15.126+01:00          True   \n",
       "7   2020-02-13T09:43:16.404+01:00          True   \n",
       "8   2020-02-12T17:45:45.558+01:00          True   \n",
       "9   2020-02-12T16:19:49.047+01:00         False   \n",
       "\n",
       "                                              _geoloc objectID  \\\n",
       "0                 [{'lat': 48.88737, 'lng': 2.28544}]    10944   \n",
       "1                 [{'lat': 48.72308, 'lng': 2.27037}]    10980   \n",
       "2   [{'lat': 43.52909, 'lng': 5.41105}, {'lat': 44...      944   \n",
       "3   [{'lat': 33.77357, 'lng': -84.29745}, {'lat': ...     9461   \n",
       "4   [{'lat': 47.2389, 'lng': 6.01377}, {'lat': 43....     6059   \n",
       "..                                                ...      ...   \n",
       "5   [{'lat': 48.87194, 'lng': 2.34853}, {'lat': 48...     7538   \n",
       "6   [{'lat': 48.8700955, 'lng': 2.3256528}, {'lat'...     2536   \n",
       "7   [{'lat': 48.88473, 'lng': 2.31925}, {'lat': 48...     7565   \n",
       "8   [{'lat': 33.59662, 'lng': -7.61889}, {'lat': 4...     1981   \n",
       "9                 [{'lat': 48.87265, 'lng': 2.32893}]     7561   \n",
       "\n",
       "   website.reference                          size.en  \\\n",
       "0            wttj_fr                   < 15 employees   \n",
       "1            wttj_fr      Between 15 and 50 employees   \n",
       "2            wttj_fr      Between 15 and 50 employees   \n",
       "3            wttj_fr     Between 50 and 250 employees   \n",
       "4            wttj_fr     Between 50 and 250 employees   \n",
       "..               ...                              ...   \n",
       "5            wttj_fr                   < 15 employees   \n",
       "6            wttj_fr     Between 50 and 250 employees   \n",
       "7            wttj_fr      Between 15 and 50 employees   \n",
       "8            wttj_fr  Between 250 and 2,000 employees   \n",
       "9            wttj_fr     Between 50 and 250 employees   \n",
       "\n",
       "                                      sectors_name.en  \n",
       "0   [{'Corporate Services': 'Incubator / Accelerat...  \n",
       "1   [{'Consulting / Audit': 'Accounting'}, {'Banki...  \n",
       "2   [{'Consulting / Audit': 'Digital Marketing / D...  \n",
       "3      [{'Industry': 'Agri-food / Animal Nutrition'}]  \n",
       "4   [{'Tech': 'Software'}, {'Mobility / Transport'...  \n",
       "..                                                ...  \n",
       "5   [{'Banking / Insurance / Finance': 'FinTech / ...  \n",
       "6   [{'Tech': 'Artificial Intelligence / Machine L...  \n",
       "7             [{'Mobility / Transport': 'Logistics'}]  \n",
       "8   [{'Consulting / Audit': 'IT / Digital'}, {'Con...  \n",
       "9   [{'Fashion / Luxury / Beauty / Lifestyle': 'Lu...  \n",
       "\n",
       "[1000 rows x 17 columns]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "companies_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "published-madness",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>languages</th>\n",
       "      <th>name</th>\n",
       "      <th>alternative_names</th>\n",
       "      <th>reference</th>\n",
       "      <th>slug</th>\n",
       "      <th>jobs_count</th>\n",
       "      <th>sectors</th>\n",
       "      <th>labels</th>\n",
       "      <th>nb_employees</th>\n",
       "      <th>offices</th>\n",
       "      <th>tools</th>\n",
       "      <th>tools_name</th>\n",
       "      <th>published_at</th>\n",
       "      <th>has_tech_page</th>\n",
       "      <th>_geoloc</th>\n",
       "      <th>objectID</th>\n",
       "      <th>website.reference</th>\n",
       "      <th>size.en</th>\n",
       "      <th>sectors_name.en</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[fr]</td>\n",
       "      <td>Renault Digital</td>\n",
       "      <td></td>\n",
       "      <td>aW0mYX7</td>\n",
       "      <td>renault-digital</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'name': {'fr': 'Application mobile', 'en': '...</td>\n",
       "      <td>[]</td>\n",
       "      <td>200.0</td>\n",
       "      <td>[{'city': 'Boulogne-Billancourt', 'country': '...</td>\n",
       "      <td>[{'parent': 'backend', 'backend': 'Node.js', '...</td>\n",
       "      <td>[{'backend': 'Node.js'}, {'frontend': 'Angular...</td>\n",
       "      <td>2020-02-13T15:10:21.838+01:00</td>\n",
       "      <td>True</td>\n",
       "      <td>[{'lat': 48.8306909, 'lng': 2.2320178}]</td>\n",
       "      <td>4876</td>\n",
       "      <td>wttj_fr</td>\n",
       "      <td>Between 50 and 250 employees</td>\n",
       "      <td>[{'Tech': 'Mobile Apps'}, {'Tech': 'Artificial...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  languages             name alternative_names reference             slug  \\\n",
       "3      [fr]  Renault Digital                     aW0mYX7  renault-digital   \n",
       "\n",
       "   jobs_count                                            sectors labels  \\\n",
       "3           0  [{'name': {'fr': 'Application mobile', 'en': '...     []   \n",
       "\n",
       "   nb_employees                                            offices  \\\n",
       "3         200.0  [{'city': 'Boulogne-Billancourt', 'country': '...   \n",
       "\n",
       "                                               tools  \\\n",
       "3  [{'parent': 'backend', 'backend': 'Node.js', '...   \n",
       "\n",
       "                                          tools_name  \\\n",
       "3  [{'backend': 'Node.js'}, {'frontend': 'Angular...   \n",
       "\n",
       "                    published_at has_tech_page  \\\n",
       "3  2020-02-13T15:10:21.838+01:00          True   \n",
       "\n",
       "                                   _geoloc objectID website.reference  \\\n",
       "3  [{'lat': 48.8306909, 'lng': 2.2320178}]     4876           wttj_fr   \n",
       "\n",
       "                        size.en  \\\n",
       "3  Between 50 and 250 employees   \n",
       "\n",
       "                                     sectors_name.en  \n",
       "3  [{'Tech': 'Mobile Apps'}, {'Tech': 'Artificial...  "
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "companies_test[companies_test[\"slug\"]==\"renault-digital\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "immediate-isolation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>languages</th>\n",
       "      <th>name</th>\n",
       "      <th>alternative_names</th>\n",
       "      <th>reference</th>\n",
       "      <th>slug</th>\n",
       "      <th>jobs_count</th>\n",
       "      <th>sectors</th>\n",
       "      <th>labels</th>\n",
       "      <th>nb_employees</th>\n",
       "      <th>offices</th>\n",
       "      <th>tools</th>\n",
       "      <th>tools_name</th>\n",
       "      <th>published_at</th>\n",
       "      <th>has_tech_page</th>\n",
       "      <th>_geoloc</th>\n",
       "      <th>objectID</th>\n",
       "      <th>website.reference</th>\n",
       "      <th>size.en</th>\n",
       "      <th>sectors_name.en</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [languages, name, alternative_names, reference, slug, jobs_count, sectors, labels, nb_employees, offices, tools, tools_name, published_at, has_tech_page, _geoloc, objectID, website.reference, size.en, sectors_name.en]\n",
       "Index: []"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "companies_test[companies_test[\"reference\"]==\"aqRkp77\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "negative-pursuit",
   "metadata": {},
   "outputs": [],
   "source": [
    "companies_test[companies_test[\"reference\"]==\"aqRkp77\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "competitive-specification",
   "metadata": {},
   "outputs": [],
   "source": [
    "def unpack_lists():\n",
    "    \n",
    "    # Unpacking languages column \n",
    "    for i in range(len(companies.columns)):\n",
    "        if type(companies_test.iloc[1,0])==\"list\":\n",
    "            for element in \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "universal-compiler",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['fr']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(companies_test.iloc[1,0])\n",
    "type(companies_test.iloc[1,0])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}