35-debugging-code.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/matt/envs/menv/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3418: DtypeWarning: Columns (7) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "url = 'https://github.com/mattharrison/datasets/raw/master'\\\n",
    "      '/data/dirtydevil.txt'\n",
    "df = pd.read_csv(url, skiprows=lambda num: num <34 or num == 35,\n",
    "                 sep='\\t')\n",
    "def to_denver_time(df_, time_col, tz_col):\n",
    "    return (df_\n",
    "       .assign(**{tz_col: df_[tz_col].replace('MDT', 'MST7MDT')})\n",
    "       .groupby(tz_col)\n",
    "       [time_col]\n",
    "       .transform(lambda s: pd.to_datetime(s)\n",
    "           .dt.tz_localize(s.name, ambiguous=True)\n",
    "           .dt.tz_convert('America/Denver'))\n",
    "    )\n",
    "def tweak_river(df_):\n",
    "    return (df_\n",
    "     .assign(datetime=to_denver_time(df_, 'datetime', 'tz_cd'))\n",
    "     .rename(columns={'144166_00060': 'cfs',\n",
    "                      '144167_00065': 'gage_height'})\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>agency_cd</th>\n",
       "      <th>site_no</th>\n",
       "      <th>datetime</th>\n",
       "      <th>tz_cd</th>\n",
       "      <th>cfs</th>\n",
       "      <th>144166_00060_cd</th>\n",
       "      <th>gage_height</th>\n",
       "      <th>144167_00065_cd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>2001-05-07 01:00:00-06:00</td>\n",
       "      <td>MDT</td>\n",
       "      <td>71.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>2001-05-07 01:15:00-06:00</td>\n",
       "      <td>MDT</td>\n",
       "      <td>71.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>2001-05-07 01:30:00-06:00</td>\n",
       "      <td>MDT</td>\n",
       "      <td>71.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>2001-05-07 01:45:00-06:00</td>\n",
       "      <td>MDT</td>\n",
       "      <td>70.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>2001-05-07 02:00:00-06:00</td>\n",
       "      <td>MDT</td>\n",
       "      <td>70.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539300</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>2020-09-28 08:30:00-06:00</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.53</td>\n",
       "      <td>P</td>\n",
       "      <td>6.16</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539301</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>2020-09-28 08:45:00-06:00</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.20</td>\n",
       "      <td>P</td>\n",
       "      <td>6.15</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539302</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>2020-09-28 09:00:00-06:00</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.20</td>\n",
       "      <td>P</td>\n",
       "      <td>6.15</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539303</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>2020-09-28 09:15:00-06:00</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.20</td>\n",
       "      <td>P</td>\n",
       "      <td>6.15</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539304</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>2020-09-28 09:30:00-06:00</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.20</td>\n",
       "      <td>P</td>\n",
       "      <td>6.15</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>539305 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       agency_cd  site_no                  datetime tz_cd    cfs  \\\n",
       "0           USGS  9333500 2001-05-07 01:00:00-06:00   MDT  71.00   \n",
       "1           USGS  9333500 2001-05-07 01:15:00-06:00   MDT  71.00   \n",
       "2           USGS  9333500 2001-05-07 01:30:00-06:00   MDT  71.00   \n",
       "3           USGS  9333500 2001-05-07 01:45:00-06:00   MDT  70.00   \n",
       "4           USGS  9333500 2001-05-07 02:00:00-06:00   MDT  70.00   \n",
       "...          ...      ...                       ...   ...    ...   \n",
       "539300      USGS  9333500 2020-09-28 08:30:00-06:00   MDT   9.53   \n",
       "539301      USGS  9333500 2020-09-28 08:45:00-06:00   MDT   9.20   \n",
       "539302      USGS  9333500 2020-09-28 09:00:00-06:00   MDT   9.20   \n",
       "539303      USGS  9333500 2020-09-28 09:15:00-06:00   MDT   9.20   \n",
       "539304      USGS  9333500 2020-09-28 09:30:00-06:00   MDT   9.20   \n",
       "\n",
       "       144166_00060_cd  gage_height 144167_00065_cd  \n",
       "0               A:[91]          NaN             NaN  \n",
       "1               A:[91]          NaN             NaN  \n",
       "2               A:[91]          NaN             NaN  \n",
       "3               A:[91]          NaN             NaN  \n",
       "4               A:[91]          NaN             NaN  \n",
       "...                ...          ...             ...  \n",
       "539300               P         6.16               P  \n",
       "539301               P         6.15               P  \n",
       "539302               P         6.15               P  \n",
       "539303               P         6.15               P  \n",
       "539304               P         6.15               P  \n",
       "\n",
       "[539305 rows x 8 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd = tweak_river(df)\n",
    "dd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd2 = pd.read_json(dd.to_json())\n",
    "dd.equals(dd2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "agency_cd               0\n",
       "site_no                 0\n",
       "datetime           539305\n",
       "tz_cd                   0\n",
       "cfs                 48048\n",
       "144166_00060_cd     46181\n",
       "gage_height        125656\n",
       "144167_00065_cd    105928\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(dd\n",
    "  .ne(dd2)\n",
    "  .sum()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "agency_cd            0.000000\n",
       "site_no              0.000000\n",
       "datetime           100.000000\n",
       "tz_cd                0.000000\n",
       "cfs                  8.909244\n",
       "144166_00060_cd      8.563058\n",
       "gage_height         23.299617\n",
       "144167_00065_cd     19.641576\n",
       "dtype: float64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(dd\n",
    "  .ne(dd2)\n",
    "  .mean()\n",
    "  .mul(100)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "ename": "AssertionError",
     "evalue": "Attributes of DataFrame.iloc[:, 2] (column name=\"datetime\") are different\n\nAttribute \"dtype\" are different\n[left]:  datetime64[ns, America/Denver]\n[right]: datetime64[ns]",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-b146eb3ff951>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtesting\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0massert_frame_equal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdd2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "    \u001b[0;31m[... skipping hidden 3 frame]\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/_testing/asserters.py\u001b[0m in \u001b[0;36mraise_assert_detail\u001b[0;34m(obj, message, left, right, diff, index_values)\u001b[0m\n\u001b[1;32m    663\u001b[0m         \u001b[0mmsg\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34mf\"\\n[diff]: {diff}\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    664\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 665\u001b[0;31m     \u001b[0;32mraise\u001b[0m \u001b[0mAssertionError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    666\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    667\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAssertionError\u001b[0m: Attributes of DataFrame.iloc[:, 2] (column name=\"datetime\") are different\n\nAttribute \"dtype\" are different\n[left]:  datetime64[ns, America/Denver]\n[right]: datetime64[ns]"
     ]
    }
   ],
   "source": [
    "pd.testing.assert_frame_equal(dd, dd2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "pd.testing.assert_frame_equal(dd,\n",
    " (dd2\n",
    "    .assign(datetime=dd2.datetime\n",
    "        .dt.tz_localize('UTC')\n",
    "        .dt.tz_convert('America/Denver')))\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd.equals(dd2\n",
    " .assign(datetime=dd2.datetime\n",
    "     .dt.tz_localize('UTC')\n",
    "     .dt.tz_convert('America/Denver'))\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "ename": "AssertionError",
     "evalue": "DataFrame.iloc[:, 4] (column name=\"cfs\") are different\n\nDataFrame.iloc[:, 4] (column name=\"cfs\") values are different (0.34619 %)\n[index]: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, ...]\n[left]:  [71.0, 71.0, 71.0, 70.0, 70.0, 69.0, 70.0, 70.0, 70.0, 70.0, 69.0, 67.0, 67.0, 66.0, 66.0, 65.0, 66.0, 66.0, 66.0, 65.0, 64.0, 64.0, 64.0, 64.0, 64.0, 65.0, 64.0, 64.0, 64.0, 64.0, 63.0, 64.0, 63.0, 63.0, 65.0, 65.0, 66.0, 66.0, 66.0, 67.0, 68.0, 68.0, 69.0, 69.0, 70.0, 71.0, 72.0, 75.0, 76.0, 75.0, 75.0, 75.0, 76.0, 77.0, 78.0, 78.0, 78.0, 78.0, 78.0, 77.0, 78.0, 78.0, 77.0, 75.0, 73.0, 73.0, 73.0, 75.0, 76.0, 76.0, 76.0, 75.0, 75.0, 75.0, 75.0, 75.0, 75.0, 76.0, 76.0, 75.0, 75.0, 76.0, 76.0, 76.0, 75.0, 75.0, 73.0, 73.0, 73.0, 73.0, 75.0, 75.0, 75.0, 75.0, 75.0, 76.0, 75.0, 73.0, 75.0, 75.0, ...]\n[right]: [71.0, 71.0, 71.0, 70.0, 70.0, 69.0, 70.0, 70.0, 70.0, 70.0, 69.0, 67.0, 67.0, 66.0, 66.0, 65.0, 66.0, 66.0, 66.0, 65.0, 64.0, 64.0, 64.0, 64.0, 64.0, 65.0, 64.0, 64.0, 64.0, 64.0, 63.0, 64.0, 63.0, 63.0, 65.0, 65.0, 66.0, 66.0, 66.0, 67.0, 68.0, 68.0, 69.0, 69.0, 70.0, 71.0, 72.0, 75.0, 76.0, 75.0, 75.0, 75.0, 76.0, 77.0, 78.0, 78.0, 78.0, 78.0, 78.0, 77.0, 78.0, 78.0, 77.0, 75.0, 73.0, 73.0, 73.0, 75.0, 76.0, 76.0, 76.0, 75.0, 75.0, 75.0, 75.0, 75.0, 75.0, 76.0, 76.0, 75.0, 75.0, 76.0, 76.0, 76.0, 75.0, 75.0, 73.0, 73.0, 73.0, 73.0, 75.0, 75.0, 75.0, 75.0, 75.0, 76.0, 75.0, 73.0, 75.0, 75.0, ...]",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-637c96f0edac>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m pd.testing.assert_frame_equal(dd,\n\u001b[0m\u001b[1;32m      2\u001b[0m  (dd2\n\u001b[1;32m      3\u001b[0m     .assign(datetime=dd2.datetime\n\u001b[1;32m      4\u001b[0m         \u001b[0;34m.\u001b[0m\u001b[0mdt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtz_localize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'UTC'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         .dt.tz_convert('America/Denver'))),\n",
      "    \u001b[0;31m[... skipping hidden 3 frame]\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/_testing/asserters.py\u001b[0m in \u001b[0;36m_raise\u001b[0;34m(left, right, err_msg)\u001b[0m\n\u001b[1;32m    733\u001b[0m             \u001b[0mdiff\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdiff\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;36m100.0\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mleft\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    734\u001b[0m             \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34mf\"{obj} values are different ({np.round(diff, 5)} %)\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 735\u001b[0;31m             \u001b[0mraise_assert_detail\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mleft\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mright\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex_values\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindex_values\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    736\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    737\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mAssertionError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merr_msg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/_testing/asserters.py\u001b[0m in \u001b[0;36mraise_assert_detail\u001b[0;34m(obj, message, left, right, diff, index_values)\u001b[0m\n\u001b[1;32m    663\u001b[0m         \u001b[0mmsg\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34mf\"\\n[diff]: {diff}\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    664\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 665\u001b[0;31m     \u001b[0;32mraise\u001b[0m \u001b[0mAssertionError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    666\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    667\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAssertionError\u001b[0m: DataFrame.iloc[:, 4] (column name=\"cfs\") are different\n\nDataFrame.iloc[:, 4] (column name=\"cfs\") values are different (0.34619 %)\n[index]: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, ...]\n[left]:  [71.0, 71.0, 71.0, 70.0, 70.0, 69.0, 70.0, 70.0, 70.0, 70.0, 69.0, 67.0, 67.0, 66.0, 66.0, 65.0, 66.0, 66.0, 66.0, 65.0, 64.0, 64.0, 64.0, 64.0, 64.0, 65.0, 64.0, 64.0, 64.0, 64.0, 63.0, 64.0, 63.0, 63.0, 65.0, 65.0, 66.0, 66.0, 66.0, 67.0, 68.0, 68.0, 69.0, 69.0, 70.0, 71.0, 72.0, 75.0, 76.0, 75.0, 75.0, 75.0, 76.0, 77.0, 78.0, 78.0, 78.0, 78.0, 78.0, 77.0, 78.0, 78.0, 77.0, 75.0, 73.0, 73.0, 73.0, 75.0, 76.0, 76.0, 76.0, 75.0, 75.0, 75.0, 75.0, 75.0, 75.0, 76.0, 76.0, 75.0, 75.0, 76.0, 76.0, 76.0, 75.0, 75.0, 73.0, 73.0, 73.0, 73.0, 75.0, 75.0, 75.0, 75.0, 75.0, 76.0, 75.0, 73.0, 75.0, 75.0, ...]\n[right]: [71.0, 71.0, 71.0, 70.0, 70.0, 69.0, 70.0, 70.0, 70.0, 70.0, 69.0, 67.0, 67.0, 66.0, 66.0, 65.0, 66.0, 66.0, 66.0, 65.0, 64.0, 64.0, 64.0, 64.0, 64.0, 65.0, 64.0, 64.0, 64.0, 64.0, 63.0, 64.0, 63.0, 63.0, 65.0, 65.0, 66.0, 66.0, 66.0, 67.0, 68.0, 68.0, 69.0, 69.0, 70.0, 71.0, 72.0, 75.0, 76.0, 75.0, 75.0, 75.0, 76.0, 77.0, 78.0, 78.0, 78.0, 78.0, 78.0, 77.0, 78.0, 78.0, 77.0, 75.0, 73.0, 73.0, 73.0, 75.0, 76.0, 76.0, 76.0, 75.0, 75.0, 75.0, 75.0, 75.0, 75.0, 76.0, 76.0, 75.0, 75.0, 76.0, 76.0, 76.0, 75.0, 75.0, 73.0, 73.0, 73.0, 73.0, 75.0, 75.0, 75.0, 75.0, 75.0, 76.0, 75.0, 73.0, 75.0, 75.0, ...]"
     ]
    }
   ],
   "source": [
    "pd.testing.assert_frame_equal(dd,\n",
    " (dd2\n",
    "    .assign(datetime=dd2.datetime\n",
    "        .dt.tz_localize('UTC')\n",
    "        .dt.tz_convert('America/Denver'))),\n",
    " check_exact=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>agency_cd</th>\n",
       "      <th>site_no</th>\n",
       "      <th>datetime</th>\n",
       "      <th>tz_cd</th>\n",
       "      <th>cfs</th>\n",
       "      <th>144166_00060_cd</th>\n",
       "      <th>gage_height</th>\n",
       "      <th>144167_00065_cd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>96246</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>2007-07-03 19:45:00-06:00</td>\n",
       "      <td>MDT</td>\n",
       "      <td>1.70</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96247</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>2007-07-03 20:00:00-06:00</td>\n",
       "      <td>MDT</td>\n",
       "      <td>1.70</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96248</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>2007-07-03 20:15:00-06:00</td>\n",
       "      <td>MDT</td>\n",
       "      <td>1.70</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96249</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>2007-07-03 20:30:00-06:00</td>\n",
       "      <td>MDT</td>\n",
       "      <td>1.70</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96250</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>2007-07-03 20:45:00-06:00</td>\n",
       "      <td>MDT</td>\n",
       "      <td>1.70</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>538678</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>2020-09-21 21:00:00-06:00</td>\n",
       "      <td>MDT</td>\n",
       "      <td>6.56</td>\n",
       "      <td>P</td>\n",
       "      <td>6.06</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>538728</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>2020-09-22 09:30:00-06:00</td>\n",
       "      <td>MDT</td>\n",
       "      <td>6.56</td>\n",
       "      <td>P</td>\n",
       "      <td>6.06</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>538735</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>2020-09-22 11:15:00-06:00</td>\n",
       "      <td>MDT</td>\n",
       "      <td>6.56</td>\n",
       "      <td>P</td>\n",
       "      <td>6.06</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>538739</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>2020-09-22 12:15:00-06:00</td>\n",
       "      <td>MDT</td>\n",
       "      <td>6.56</td>\n",
       "      <td>P</td>\n",
       "      <td>6.06</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>538753</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>2020-09-22 15:45:00-06:00</td>\n",
       "      <td>MDT</td>\n",
       "      <td>6.56</td>\n",
       "      <td>P</td>\n",
       "      <td>6.06</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>48048 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       agency_cd  site_no                  datetime tz_cd   cfs  \\\n",
       "96246       USGS  9333500 2007-07-03 19:45:00-06:00   MDT  1.70   \n",
       "96247       USGS  9333500 2007-07-03 20:00:00-06:00   MDT  1.70   \n",
       "96248       USGS  9333500 2007-07-03 20:15:00-06:00   MDT  1.70   \n",
       "96249       USGS  9333500 2007-07-03 20:30:00-06:00   MDT  1.70   \n",
       "96250       USGS  9333500 2007-07-03 20:45:00-06:00   MDT  1.70   \n",
       "...          ...      ...                       ...   ...   ...   \n",
       "538678      USGS  9333500 2020-09-21 21:00:00-06:00   MDT  6.56   \n",
       "538728      USGS  9333500 2020-09-22 09:30:00-06:00   MDT  6.56   \n",
       "538735      USGS  9333500 2020-09-22 11:15:00-06:00   MDT  6.56   \n",
       "538739      USGS  9333500 2020-09-22 12:15:00-06:00   MDT  6.56   \n",
       "538753      USGS  9333500 2020-09-22 15:45:00-06:00   MDT  6.56   \n",
       "\n",
       "       144166_00060_cd  gage_height 144167_00065_cd  \n",
       "96246           A:[91]          NaN             NaN  \n",
       "96247           A:[91]          NaN             NaN  \n",
       "96248           A:[91]          NaN             NaN  \n",
       "96249           A:[91]          NaN             NaN  \n",
       "96250           A:[91]          NaN             NaN  \n",
       "...                ...          ...             ...  \n",
       "538678               P         6.06               P  \n",
       "538728               P         6.06               P  \n",
       "538735               P         6.06               P  \n",
       "538739               P         6.06               P  \n",
       "538753               P         6.06               P  \n",
       "\n",
       "[48048 rows x 8 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd[dd.cfs.ne(dd2.cfs)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.7, 1.7000000000000002)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd.loc[96246].cfs, dd2.loc[96246].cfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd.round(2).equals(\n",
    "  dd2\n",
    "    .assign(datetime=dd2.datetime\n",
    "            .dt.tz_localize('UTC').\n",
    "            dt.tz_convert('America/Denver'))\n",
    "    .round(2)\n",
    " )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cmp_dfs(df1, df2, round_amt=3):\n",
    "    diff_cols = set(df1.columns) ^ set(df2.columns)\n",
    "    if diff_cols:\n",
    "        print(f'Different columns {diff_cols}')\n",
    "    if df1.shape != df2.shape:\n",
    "        print(f'Different shapes {df1.shape} {df2.shape}')\n",
    "    bad = False\n",
    "    for col in df1.columns:\n",
    "        s1 = df1[col]\n",
    "        s2 = df2[col]\n",
    "        if s1.equals(s2):\n",
    "            continue\n",
    "        bad = True            \n",
    "        if s1.dtype != s2.dtype:\n",
    "            print(f'{col} types differ {s1.dtype} vs {s2.dtype}')\n",
    "        if s1.dtype == float:\n",
    "            if s1.round(round_amt).equals(s2.round(round_amt)):\n",
    "                print(f'{col} has rounding differences'\n",
    "                      f'{df1[s1.ne(s2)][col].dropna().iloc[0]} '\n",
    "                      f'vs {df2[s1.ne(s2)][col].dropna().iloc[0]}')\n",
    "        else:\n",
    "            print(f'{col} differs {df1[s1.ne(s2)][col].dropna()}')\n",
    "    if not bad:\n",
    "        print('Same')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "datetime types differ datetime64[ns, America/Denver] vs datetime64[ns]\n",
      "datetime differs 0        2001-05-07 01:00:00-06:00\n",
      "1        2001-05-07 01:15:00-06:00\n",
      "2        2001-05-07 01:30:00-06:00\n",
      "3        2001-05-07 01:45:00-06:00\n",
      "4        2001-05-07 02:00:00-06:00\n",
      "                    ...           \n",
      "539300   2020-09-28 08:30:00-06:00\n",
      "539301   2020-09-28 08:45:00-06:00\n",
      "539302   2020-09-28 09:00:00-06:00\n",
      "539303   2020-09-28 09:15:00-06:00\n",
      "539304   2020-09-28 09:30:00-06:00\n",
      "Name: datetime, Length: 539305, dtype: datetime64[ns, America/Denver]\n",
      "cfs has rounding differences1.7 vs 1.7000000000000002\n",
      "gage_height has rounding differences3.28 vs 3.2800000000000002\n"
     ]
    }
   ],
   "source": [
    "cmp_dfs(dd, dd2)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/matt/envs/menv/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3418: DtypeWarning: Columns (68,70,71,72,73,74,76,79) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "autos = pd.read_csv('https://github.com/mattharrison/datasets/raw/'\n",
    "    'master/data/vehicles.csv.zip')\n",
    "def to_tz(df_, time_col, tz_offset, tz_name):\n",
    "    return (df_\n",
    "             .groupby(tz_offset)\n",
    "             [time_col]\n",
    "             .transform(lambda s: pd.to_datetime(s)\n",
    "                 .dt.tz_localize(s.name, ambiguous=True)\n",
    "                 .dt.tz_convert(tz_name))\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tweak_autos(autos):\n",
    "    cols = ['city08', 'comb08', 'highway08', 'cylinders',\n",
    "            'displ', 'drive', 'eng_dscr', 'fuelCost08',\n",
    "            'make', 'model', 'trany', 'range', 'createdOn',\n",
    "            'year']\n",
    "    return (autos\n",
    "     [cols]\n",
    "     .assign(cylinders=autos.cylinders.fillna(0).astype('int8'),\n",
    "             displ=autos.displ.fillna(0).astype('float16'),\n",
    "             drive=autos.drive.fillna('Other').astype('category'),\n",
    "             automatic=autos.trany.str.contains('Auto'),\n",
    "             speeds=autos.trany.str.extract(r'(\\d)+').fillna('20')\n",
    "                    .astype('int8'),\n",
    "             offset=autos.createdOn\n",
    "                    .str.extract(r'\\d\\d:\\d\\d ([A-Z]{3}?)')\n",
    "                    .replace('EDT', 'EST5EDT'),\n",
    "             str_date=(autos.createdOn.str.slice(4,19) + ' ' +\n",
    "                       autos.createdOn.str.slice(-4)),\n",
    "             createdOn=lambda df_: to_tz(df_, 'str_date',\n",
    "                       'offset', 'America/New_York'),\n",
    "             ffs=autos.eng_dscr.str.contains('FFS')\n",
    "            )\n",
    "     .astype({'highway08': 'int8', 'city08': 'int16',\n",
    "              'comb08': 'int16', 'fuelCost08': 'int16',\n",
    "              'range': 'int16',  'year': 'int16',\n",
    "              'make': 'category'})\n",
    "     .drop(columns=['trany', 'eng_dscr'])\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city08</th>\n",
       "      <th>comb08</th>\n",
       "      <th>highway08</th>\n",
       "      <th>cylinders</th>\n",
       "      <th>displ</th>\n",
       "      <th>drive</th>\n",
       "      <th>fuelCost08</th>\n",
       "      <th>make</th>\n",
       "      <th>model</th>\n",
       "      <th>range</th>\n",
       "      <th>createdOn</th>\n",
       "      <th>year</th>\n",
       "      <th>automatic</th>\n",
       "      <th>speeds</th>\n",
       "      <th>offset</th>\n",
       "      <th>str_date</th>\n",
       "      <th>ffs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>19</td>\n",
       "      <td>21</td>\n",
       "      <td>25</td>\n",
       "      <td>4</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>Rear-Wheel Drive</td>\n",
       "      <td>2000</td>\n",
       "      <td>Alfa Romeo</td>\n",
       "      <td>Spider Veloce 2000</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1985</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>12</td>\n",
       "      <td>4.898438</td>\n",
       "      <td>Rear-Wheel Drive</td>\n",
       "      <td>3850</td>\n",
       "      <td>Ferrari</td>\n",
       "      <td>Testarossa</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1985</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>23</td>\n",
       "      <td>27</td>\n",
       "      <td>33</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>Front-Wheel Drive</td>\n",
       "      <td>1550</td>\n",
       "      <td>Dodge</td>\n",
       "      <td>Charger</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1985</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>8</td>\n",
       "      <td>5.199219</td>\n",
       "      <td>Rear-Wheel Drive</td>\n",
       "      <td>3850</td>\n",
       "      <td>Dodge</td>\n",
       "      <td>B150/B250 Wagon 2WD</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1985</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17</td>\n",
       "      <td>19</td>\n",
       "      <td>23</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>2700</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD Turbo</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41139</th>\n",
       "      <td>19</td>\n",
       "      <td>22</td>\n",
       "      <td>26</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>Front-Wheel Drive</td>\n",
       "      <td>1900</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41140</th>\n",
       "      <td>20</td>\n",
       "      <td>23</td>\n",
       "      <td>28</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>Front-Wheel Drive</td>\n",
       "      <td>1850</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41141</th>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>24</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>2000</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41142</th>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>24</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>2000</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41143</th>\n",
       "      <td>16</td>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>2900</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD Turbo</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>41144 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       city08  comb08  highway08  cylinders     displ  \\\n",
       "0          19      21         25          4  2.000000   \n",
       "1           9      11         14         12  4.898438   \n",
       "2          23      27         33          4  2.199219   \n",
       "3          10      11         12          8  5.199219   \n",
       "4          17      19         23          4  2.199219   \n",
       "...       ...     ...        ...        ...       ...   \n",
       "41139      19      22         26          4  2.199219   \n",
       "41140      20      23         28          4  2.199219   \n",
       "41141      18      21         24          4  2.199219   \n",
       "41142      18      21         24          4  2.199219   \n",
       "41143      16      18         21          4  2.199219   \n",
       "\n",
       "                            drive  fuelCost08        make  \\\n",
       "0                Rear-Wheel Drive        2000  Alfa Romeo   \n",
       "1                Rear-Wheel Drive        3850     Ferrari   \n",
       "2               Front-Wheel Drive        1550       Dodge   \n",
       "3                Rear-Wheel Drive        3850       Dodge   \n",
       "4      4-Wheel or All-Wheel Drive        2700      Subaru   \n",
       "...                           ...         ...         ...   \n",
       "41139           Front-Wheel Drive        1900      Subaru   \n",
       "41140           Front-Wheel Drive        1850      Subaru   \n",
       "41141  4-Wheel or All-Wheel Drive        2000      Subaru   \n",
       "41142  4-Wheel or All-Wheel Drive        2000      Subaru   \n",
       "41143  4-Wheel or All-Wheel Drive        2900      Subaru   \n",
       "\n",
       "                     model  range                 createdOn  year automatic  \\\n",
       "0       Spider Veloce 2000      0 2013-01-01 00:00:00-05:00  1985     False   \n",
       "1               Testarossa      0 2013-01-01 00:00:00-05:00  1985     False   \n",
       "2                  Charger      0 2013-01-01 00:00:00-05:00  1985     False   \n",
       "3      B150/B250 Wagon 2WD      0 2013-01-01 00:00:00-05:00  1985      True   \n",
       "4         Legacy AWD Turbo      0 2013-01-01 00:00:00-05:00  1993     False   \n",
       "...                    ...    ...                       ...   ...       ...   \n",
       "41139               Legacy      0 2013-01-01 00:00:00-05:00  1993      True   \n",
       "41140               Legacy      0 2013-01-01 00:00:00-05:00  1993     False   \n",
       "41141           Legacy AWD      0 2013-01-01 00:00:00-05:00  1993      True   \n",
       "41142           Legacy AWD      0 2013-01-01 00:00:00-05:00  1993     False   \n",
       "41143     Legacy AWD Turbo      0 2013-01-01 00:00:00-05:00  1993      True   \n",
       "\n",
       "       speeds offset              str_date    ffs  \n",
       "0           5    EST  Jan 01 00:00:00 2013   True  \n",
       "1           5    EST  Jan 01 00:00:00 2013  False  \n",
       "2           5    EST  Jan 01 00:00:00 2013   True  \n",
       "3           3    EST  Jan 01 00:00:00 2013    NaN  \n",
       "4           5    EST  Jan 01 00:00:00 2013   True  \n",
       "...       ...    ...                   ...    ...  \n",
       "41139       4    EST  Jan 01 00:00:00 2013   True  \n",
       "41140       5    EST  Jan 01 00:00:00 2013   True  \n",
       "41141       4    EST  Jan 01 00:00:00 2013   True  \n",
       "41142       5    EST  Jan 01 00:00:00 2013   True  \n",
       "41143       4    EST  Jan 01 00:00:00 2013   True  \n",
       "\n",
       "[41144 rows x 17 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweak_autos(autos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tweak_autos(autos):\n",
    "    cols = ['city08', 'comb08', 'highway08', 'cylinders',\n",
    "            'displ', 'drive', 'eng_dscr', 'fuelCost08',\n",
    "            'make', 'model', 'trany', 'range', 'createdOn',\n",
    "            'year']\n",
    "    return (autos\n",
    "     [cols]\n",
    "    # .assign(cylinders=autos.cylinders.fillna(0).astype('int8'),\n",
    "    #         displ=autos.displ.fillna(0).astype('float16'),\n",
    "    #         drive=autos.drive.fillna('Other').astype('category'),\n",
    "    #         automatic=autos.trany.str.contains('Auto'),\n",
    "    #         speeds=autos.trany.str.extract(r'(\\d)+').fillna('20')\n",
    "    #                .astype('int8'),\n",
    "    #         offset=autos.createdOn\n",
    "    #                .str.extract(r'\\d\\d:\\d\\d ([A-Z]{3}?)')\n",
    "    #                .replace('EDT', 'EST5EDT'),\n",
    "    #         str_date=(autos.createdOn.str.slice(4,19) + ' ' +\n",
    "    #                   autos.createdOn.str.slice(-4)),\n",
    "    #         createdOn=lambda df_: to_tz(df_, 'str_date',\n",
    "    #                   'offset', 'America/New_York'),\n",
    "    #         ffs=autos.eng_dscr.str.contains('FFS')\n",
    "    #        )\n",
    "    # .astype({'highway08': 'int8', 'city08': 'int16',\n",
    "    #          'comb08': 'int16', 'fuelCost08': 'int16',\n",
    "    #          'range': 'int16',  'year': 'int16',\n",
    "    #          'make': 'category'})\n",
    "    # .drop(columns=['trany', 'eng_dscr'])\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city08</th>\n",
       "      <th>comb08</th>\n",
       "      <th>highway08</th>\n",
       "      <th>cylinders</th>\n",
       "      <th>displ</th>\n",
       "      <th>drive</th>\n",
       "      <th>eng_dscr</th>\n",
       "      <th>fuelCost08</th>\n",
       "      <th>make</th>\n",
       "      <th>model</th>\n",
       "      <th>trany</th>\n",
       "      <th>range</th>\n",
       "      <th>createdOn</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>19</td>\n",
       "      <td>21</td>\n",
       "      <td>25</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Rear-Wheel Drive</td>\n",
       "      <td>(FFS)</td>\n",
       "      <td>2000</td>\n",
       "      <td>Alfa Romeo</td>\n",
       "      <td>Spider Veloce 2000</td>\n",
       "      <td>Manual 5-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>Tue Jan 01 00:00:00 EST 2013</td>\n",
       "      <td>1985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>12.0</td>\n",
       "      <td>4.9</td>\n",
       "      <td>Rear-Wheel Drive</td>\n",
       "      <td>(GUZZLER)</td>\n",
       "      <td>3850</td>\n",
       "      <td>Ferrari</td>\n",
       "      <td>Testarossa</td>\n",
       "      <td>Manual 5-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>Tue Jan 01 00:00:00 EST 2013</td>\n",
       "      <td>1985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>23</td>\n",
       "      <td>27</td>\n",
       "      <td>33</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.2</td>\n",
       "      <td>Front-Wheel Drive</td>\n",
       "      <td>(FFS)</td>\n",
       "      <td>1550</td>\n",
       "      <td>Dodge</td>\n",
       "      <td>Charger</td>\n",
       "      <td>Manual 5-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>Tue Jan 01 00:00:00 EST 2013</td>\n",
       "      <td>1985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.2</td>\n",
       "      <td>Rear-Wheel Drive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3850</td>\n",
       "      <td>Dodge</td>\n",
       "      <td>B150/B250 Wagon 2WD</td>\n",
       "      <td>Automatic 3-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>Tue Jan 01 00:00:00 EST 2013</td>\n",
       "      <td>1985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17</td>\n",
       "      <td>19</td>\n",
       "      <td>23</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.2</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>(FFS,TRBO)</td>\n",
       "      <td>2700</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD Turbo</td>\n",
       "      <td>Manual 5-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>Tue Jan 01 00:00:00 EST 2013</td>\n",
       "      <td>1993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41139</th>\n",
       "      <td>19</td>\n",
       "      <td>22</td>\n",
       "      <td>26</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.2</td>\n",
       "      <td>Front-Wheel Drive</td>\n",
       "      <td>(FFS)</td>\n",
       "      <td>1900</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy</td>\n",
       "      <td>Automatic 4-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>Tue Jan 01 00:00:00 EST 2013</td>\n",
       "      <td>1993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41140</th>\n",
       "      <td>20</td>\n",
       "      <td>23</td>\n",
       "      <td>28</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.2</td>\n",
       "      <td>Front-Wheel Drive</td>\n",
       "      <td>(FFS)</td>\n",
       "      <td>1850</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy</td>\n",
       "      <td>Manual 5-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>Tue Jan 01 00:00:00 EST 2013</td>\n",
       "      <td>1993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41141</th>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>24</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.2</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>(FFS)</td>\n",
       "      <td>2000</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD</td>\n",
       "      <td>Automatic 4-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>Tue Jan 01 00:00:00 EST 2013</td>\n",
       "      <td>1993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41142</th>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>24</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.2</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>(FFS)</td>\n",
       "      <td>2000</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD</td>\n",
       "      <td>Manual 5-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>Tue Jan 01 00:00:00 EST 2013</td>\n",
       "      <td>1993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41143</th>\n",
       "      <td>16</td>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.2</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>(FFS,TRBO)</td>\n",
       "      <td>2900</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD Turbo</td>\n",
       "      <td>Automatic 4-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>Tue Jan 01 00:00:00 EST 2013</td>\n",
       "      <td>1993</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>41144 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       city08  comb08  highway08  cylinders  displ  \\\n",
       "0          19      21         25        4.0    2.0   \n",
       "1           9      11         14       12.0    4.9   \n",
       "2          23      27         33        4.0    2.2   \n",
       "3          10      11         12        8.0    5.2   \n",
       "4          17      19         23        4.0    2.2   \n",
       "...       ...     ...        ...        ...    ...   \n",
       "41139      19      22         26        4.0    2.2   \n",
       "41140      20      23         28        4.0    2.2   \n",
       "41141      18      21         24        4.0    2.2   \n",
       "41142      18      21         24        4.0    2.2   \n",
       "41143      16      18         21        4.0    2.2   \n",
       "\n",
       "                            drive    eng_dscr  fuelCost08        make  \\\n",
       "0                Rear-Wheel Drive       (FFS)        2000  Alfa Romeo   \n",
       "1                Rear-Wheel Drive   (GUZZLER)        3850     Ferrari   \n",
       "2               Front-Wheel Drive       (FFS)        1550       Dodge   \n",
       "3                Rear-Wheel Drive         NaN        3850       Dodge   \n",
       "4      4-Wheel or All-Wheel Drive  (FFS,TRBO)        2700      Subaru   \n",
       "...                           ...         ...         ...         ...   \n",
       "41139           Front-Wheel Drive       (FFS)        1900      Subaru   \n",
       "41140           Front-Wheel Drive       (FFS)        1850      Subaru   \n",
       "41141  4-Wheel or All-Wheel Drive       (FFS)        2000      Subaru   \n",
       "41142  4-Wheel or All-Wheel Drive       (FFS)        2000      Subaru   \n",
       "41143  4-Wheel or All-Wheel Drive  (FFS,TRBO)        2900      Subaru   \n",
       "\n",
       "                     model            trany  range  \\\n",
       "0       Spider Veloce 2000     Manual 5-spd      0   \n",
       "1               Testarossa     Manual 5-spd      0   \n",
       "2                  Charger     Manual 5-spd      0   \n",
       "3      B150/B250 Wagon 2WD  Automatic 3-spd      0   \n",
       "4         Legacy AWD Turbo     Manual 5-spd      0   \n",
       "...                    ...              ...    ...   \n",
       "41139               Legacy  Automatic 4-spd      0   \n",
       "41140               Legacy     Manual 5-spd      0   \n",
       "41141           Legacy AWD  Automatic 4-spd      0   \n",
       "41142           Legacy AWD     Manual 5-spd      0   \n",
       "41143     Legacy AWD Turbo  Automatic 4-spd      0   \n",
       "\n",
       "                          createdOn  year  \n",
       "0      Tue Jan 01 00:00:00 EST 2013  1985  \n",
       "1      Tue Jan 01 00:00:00 EST 2013  1985  \n",
       "2      Tue Jan 01 00:00:00 EST 2013  1985  \n",
       "3      Tue Jan 01 00:00:00 EST 2013  1985  \n",
       "4      Tue Jan 01 00:00:00 EST 2013  1993  \n",
       "...                             ...   ...  \n",
       "41139  Tue Jan 01 00:00:00 EST 2013  1993  \n",
       "41140  Tue Jan 01 00:00:00 EST 2013  1993  \n",
       "41141  Tue Jan 01 00:00:00 EST 2013  1993  \n",
       "41142  Tue Jan 01 00:00:00 EST 2013  1993  \n",
       "41143  Tue Jan 01 00:00:00 EST 2013  1993  \n",
       "\n",
       "[41144 rows x 14 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweak_autos(autos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tweak_autos(autos):\n",
    "    cols = ['city08', 'comb08', 'highway08', 'cylinders',\n",
    "            'displ', 'drive', 'eng_dscr', 'fuelCost08',\n",
    "            'make', 'model', 'trany', 'range', 'createdOn',\n",
    "            'year']\n",
    "    return (autos\n",
    "     [cols]\n",
    "     .assign(cylinders=autos.cylinders.fillna(0).astype('int8'),\n",
    "             displ=autos.displ.fillna(0).astype('float16'),\n",
    "             drive=autos.drive.fillna('Other').astype('category'),\n",
    "             automatic=autos.trany.str.contains('Auto'),\n",
    "             speeds=autos.trany.str.extract(r'(\\d)+').fillna('20')\n",
    "                    .astype('int8'),\n",
    "             offset=autos.createdOn\n",
    "                    .str.extract(r'\\d\\d:\\d\\d ([A-Z]{3}?)')\n",
    "                    .replace('EDT', 'EST5EDT'),\n",
    "             str_date=(autos.createdOn.str.slice(4,19) + ' ' +\n",
    "                       autos.createdOn.str.slice(-4)),\n",
    "             createdOn=lambda df_: to_tz(df_, 'str_date',\n",
    "                       'offset', 'America/New_York'),\n",
    "             ffs=autos.eng_dscr.str.contains('FFS')\n",
    "            )\n",
    "    # .astype({'highway08': 'int8', 'city08': 'int16',\n",
    "    #          'comb08': 'int16', 'fuelCost08': 'int16',\n",
    "    #          'range': 'int16',  'year': 'int16',\n",
    "    #          'make': 'category'})\n",
    "    # .drop(columns=['trany', 'eng_dscr'])\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city08</th>\n",
       "      <th>comb08</th>\n",
       "      <th>highway08</th>\n",
       "      <th>cylinders</th>\n",
       "      <th>displ</th>\n",
       "      <th>drive</th>\n",
       "      <th>eng_dscr</th>\n",
       "      <th>fuelCost08</th>\n",
       "      <th>make</th>\n",
       "      <th>model</th>\n",
       "      <th>trany</th>\n",
       "      <th>range</th>\n",
       "      <th>createdOn</th>\n",
       "      <th>year</th>\n",
       "      <th>automatic</th>\n",
       "      <th>speeds</th>\n",
       "      <th>offset</th>\n",
       "      <th>str_date</th>\n",
       "      <th>ffs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>19</td>\n",
       "      <td>21</td>\n",
       "      <td>25</td>\n",
       "      <td>4</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>Rear-Wheel Drive</td>\n",
       "      <td>(FFS)</td>\n",
       "      <td>2000</td>\n",
       "      <td>Alfa Romeo</td>\n",
       "      <td>Spider Veloce 2000</td>\n",
       "      <td>Manual 5-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1985</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>12</td>\n",
       "      <td>4.898438</td>\n",
       "      <td>Rear-Wheel Drive</td>\n",
       "      <td>(GUZZLER)</td>\n",
       "      <td>3850</td>\n",
       "      <td>Ferrari</td>\n",
       "      <td>Testarossa</td>\n",
       "      <td>Manual 5-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1985</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>23</td>\n",
       "      <td>27</td>\n",
       "      <td>33</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>Front-Wheel Drive</td>\n",
       "      <td>(FFS)</td>\n",
       "      <td>1550</td>\n",
       "      <td>Dodge</td>\n",
       "      <td>Charger</td>\n",
       "      <td>Manual 5-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1985</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>8</td>\n",
       "      <td>5.199219</td>\n",
       "      <td>Rear-Wheel Drive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3850</td>\n",
       "      <td>Dodge</td>\n",
       "      <td>B150/B250 Wagon 2WD</td>\n",
       "      <td>Automatic 3-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1985</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17</td>\n",
       "      <td>19</td>\n",
       "      <td>23</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>(FFS,TRBO)</td>\n",
       "      <td>2700</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD Turbo</td>\n",
       "      <td>Manual 5-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41139</th>\n",
       "      <td>19</td>\n",
       "      <td>22</td>\n",
       "      <td>26</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>Front-Wheel Drive</td>\n",
       "      <td>(FFS)</td>\n",
       "      <td>1900</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy</td>\n",
       "      <td>Automatic 4-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41140</th>\n",
       "      <td>20</td>\n",
       "      <td>23</td>\n",
       "      <td>28</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>Front-Wheel Drive</td>\n",
       "      <td>(FFS)</td>\n",
       "      <td>1850</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy</td>\n",
       "      <td>Manual 5-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41141</th>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>24</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>(FFS)</td>\n",
       "      <td>2000</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD</td>\n",
       "      <td>Automatic 4-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41142</th>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>24</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>(FFS)</td>\n",
       "      <td>2000</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD</td>\n",
       "      <td>Manual 5-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41143</th>\n",
       "      <td>16</td>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>(FFS,TRBO)</td>\n",
       "      <td>2900</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD Turbo</td>\n",
       "      <td>Automatic 4-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>41144 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       city08  comb08  highway08  cylinders     displ  \\\n",
       "0          19      21         25          4  2.000000   \n",
       "1           9      11         14         12  4.898438   \n",
       "2          23      27         33          4  2.199219   \n",
       "3          10      11         12          8  5.199219   \n",
       "4          17      19         23          4  2.199219   \n",
       "...       ...     ...        ...        ...       ...   \n",
       "41139      19      22         26          4  2.199219   \n",
       "41140      20      23         28          4  2.199219   \n",
       "41141      18      21         24          4  2.199219   \n",
       "41142      18      21         24          4  2.199219   \n",
       "41143      16      18         21          4  2.199219   \n",
       "\n",
       "                            drive    eng_dscr  fuelCost08        make  \\\n",
       "0                Rear-Wheel Drive       (FFS)        2000  Alfa Romeo   \n",
       "1                Rear-Wheel Drive   (GUZZLER)        3850     Ferrari   \n",
       "2               Front-Wheel Drive       (FFS)        1550       Dodge   \n",
       "3                Rear-Wheel Drive         NaN        3850       Dodge   \n",
       "4      4-Wheel or All-Wheel Drive  (FFS,TRBO)        2700      Subaru   \n",
       "...                           ...         ...         ...         ...   \n",
       "41139           Front-Wheel Drive       (FFS)        1900      Subaru   \n",
       "41140           Front-Wheel Drive       (FFS)        1850      Subaru   \n",
       "41141  4-Wheel or All-Wheel Drive       (FFS)        2000      Subaru   \n",
       "41142  4-Wheel or All-Wheel Drive       (FFS)        2000      Subaru   \n",
       "41143  4-Wheel or All-Wheel Drive  (FFS,TRBO)        2900      Subaru   \n",
       "\n",
       "                     model            trany  range                 createdOn  \\\n",
       "0       Spider Veloce 2000     Manual 5-spd      0 2013-01-01 00:00:00-05:00   \n",
       "1               Testarossa     Manual 5-spd      0 2013-01-01 00:00:00-05:00   \n",
       "2                  Charger     Manual 5-spd      0 2013-01-01 00:00:00-05:00   \n",
       "3      B150/B250 Wagon 2WD  Automatic 3-spd      0 2013-01-01 00:00:00-05:00   \n",
       "4         Legacy AWD Turbo     Manual 5-spd      0 2013-01-01 00:00:00-05:00   \n",
       "...                    ...              ...    ...                       ...   \n",
       "41139               Legacy  Automatic 4-spd      0 2013-01-01 00:00:00-05:00   \n",
       "41140               Legacy     Manual 5-spd      0 2013-01-01 00:00:00-05:00   \n",
       "41141           Legacy AWD  Automatic 4-spd      0 2013-01-01 00:00:00-05:00   \n",
       "41142           Legacy AWD     Manual 5-spd      0 2013-01-01 00:00:00-05:00   \n",
       "41143     Legacy AWD Turbo  Automatic 4-spd      0 2013-01-01 00:00:00-05:00   \n",
       "\n",
       "       year automatic  speeds offset              str_date    ffs  \n",
       "0      1985     False       5    EST  Jan 01 00:00:00 2013   True  \n",
       "1      1985     False       5    EST  Jan 01 00:00:00 2013  False  \n",
       "2      1985     False       5    EST  Jan 01 00:00:00 2013   True  \n",
       "3      1985      True       3    EST  Jan 01 00:00:00 2013    NaN  \n",
       "4      1993     False       5    EST  Jan 01 00:00:00 2013   True  \n",
       "...     ...       ...     ...    ...                   ...    ...  \n",
       "41139  1993      True       4    EST  Jan 01 00:00:00 2013   True  \n",
       "41140  1993     False       5    EST  Jan 01 00:00:00 2013   True  \n",
       "41141  1993      True       4    EST  Jan 01 00:00:00 2013   True  \n",
       "41142  1993     False       5    EST  Jan 01 00:00:00 2013   True  \n",
       "41143  1993      True       4    EST  Jan 01 00:00:00 2013   True  \n",
       "\n",
       "[41144 rows x 19 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweak_autos(autos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tweak_autos(autos):\n",
    "    cols = ['city08', 'comb08', 'highway08', 'cylinders',\n",
    "            'displ', 'drive', 'eng_dscr', 'fuelCost08',\n",
    "            'make', 'model', 'trany', 'range', 'createdOn',\n",
    "            'year']\n",
    "    return (autos\n",
    "     [cols]\n",
    "     .assign(cylinders=autos.cylinders.fillna(0).astype('int8'),\n",
    "             displ=autos.displ.fillna(0).astype('float16'),\n",
    "             drive=autos.drive.fillna('Other').astype('category'),\n",
    "             automatic=autos.trany.str.contains('Auto'),\n",
    "             speeds=autos.trany.str.extract(r'(\\d)+').fillna('20')\n",
    "                    .astype('int8'),\n",
    "             offset=autos.createdOn\n",
    "                    .str.extract(r'\\d\\d:\\d\\d ([A-Z]{3}?)')\n",
    "                    .replace('EDT', 'EST5EDT'),\n",
    "             str_date=(autos.createdOn.str.slice(4,19) + ' ' +\n",
    "                       autos.createdOn.str.slice(-4)),\n",
    "             createdOn=lambda df_: to_tz(df_, 'str_date',\n",
    "                       'offset', 'America/New_York'),\n",
    "             ffs=autos.eng_dscr.str.contains('FFS')\n",
    "            )\n",
    "     .astype({'highway08': 'int8', 'city08': 'int16',\n",
    "              'comb08': 'int16', 'fuelCost08': 'int16',\n",
    "              'range': 'int16',  'year': 'int16',\n",
    "              'make': 'category'})\n",
    "     .drop(columns=['trany', 'eng_dscr'])\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city08</th>\n",
       "      <th>comb08</th>\n",
       "      <th>highway08</th>\n",
       "      <th>cylinders</th>\n",
       "      <th>displ</th>\n",
       "      <th>drive</th>\n",
       "      <th>fuelCost08</th>\n",
       "      <th>make</th>\n",
       "      <th>model</th>\n",
       "      <th>range</th>\n",
       "      <th>createdOn</th>\n",
       "      <th>year</th>\n",
       "      <th>automatic</th>\n",
       "      <th>speeds</th>\n",
       "      <th>offset</th>\n",
       "      <th>str_date</th>\n",
       "      <th>ffs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>19</td>\n",
       "      <td>21</td>\n",
       "      <td>25</td>\n",
       "      <td>4</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>Rear-Wheel Drive</td>\n",
       "      <td>2000</td>\n",
       "      <td>Alfa Romeo</td>\n",
       "      <td>Spider Veloce 2000</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1985</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>12</td>\n",
       "      <td>4.898438</td>\n",
       "      <td>Rear-Wheel Drive</td>\n",
       "      <td>3850</td>\n",
       "      <td>Ferrari</td>\n",
       "      <td>Testarossa</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1985</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>23</td>\n",
       "      <td>27</td>\n",
       "      <td>33</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>Front-Wheel Drive</td>\n",
       "      <td>1550</td>\n",
       "      <td>Dodge</td>\n",
       "      <td>Charger</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1985</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>8</td>\n",
       "      <td>5.199219</td>\n",
       "      <td>Rear-Wheel Drive</td>\n",
       "      <td>3850</td>\n",
       "      <td>Dodge</td>\n",
       "      <td>B150/B250 Wagon 2WD</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1985</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17</td>\n",
       "      <td>19</td>\n",
       "      <td>23</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>2700</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD Turbo</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41139</th>\n",
       "      <td>19</td>\n",
       "      <td>22</td>\n",
       "      <td>26</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>Front-Wheel Drive</td>\n",
       "      <td>1900</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41140</th>\n",
       "      <td>20</td>\n",
       "      <td>23</td>\n",
       "      <td>28</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>Front-Wheel Drive</td>\n",
       "      <td>1850</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41141</th>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>24</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>2000</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41142</th>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>24</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>2000</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41143</th>\n",
       "      <td>16</td>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>2900</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD Turbo</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>41144 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       city08  comb08  highway08  cylinders     displ  \\\n",
       "0          19      21         25          4  2.000000   \n",
       "1           9      11         14         12  4.898438   \n",
       "2          23      27         33          4  2.199219   \n",
       "3          10      11         12          8  5.199219   \n",
       "4          17      19         23          4  2.199219   \n",
       "...       ...     ...        ...        ...       ...   \n",
       "41139      19      22         26          4  2.199219   \n",
       "41140      20      23         28          4  2.199219   \n",
       "41141      18      21         24          4  2.199219   \n",
       "41142      18      21         24          4  2.199219   \n",
       "41143      16      18         21          4  2.199219   \n",
       "\n",
       "                            drive  fuelCost08        make  \\\n",
       "0                Rear-Wheel Drive        2000  Alfa Romeo   \n",
       "1                Rear-Wheel Drive        3850     Ferrari   \n",
       "2               Front-Wheel Drive        1550       Dodge   \n",
       "3                Rear-Wheel Drive        3850       Dodge   \n",
       "4      4-Wheel or All-Wheel Drive        2700      Subaru   \n",
       "...                           ...         ...         ...   \n",
       "41139           Front-Wheel Drive        1900      Subaru   \n",
       "41140           Front-Wheel Drive        1850      Subaru   \n",
       "41141  4-Wheel or All-Wheel Drive        2000      Subaru   \n",
       "41142  4-Wheel or All-Wheel Drive        2000      Subaru   \n",
       "41143  4-Wheel or All-Wheel Drive        2900      Subaru   \n",
       "\n",
       "                     model  range                 createdOn  year automatic  \\\n",
       "0       Spider Veloce 2000      0 2013-01-01 00:00:00-05:00  1985     False   \n",
       "1               Testarossa      0 2013-01-01 00:00:00-05:00  1985     False   \n",
       "2                  Charger      0 2013-01-01 00:00:00-05:00  1985     False   \n",
       "3      B150/B250 Wagon 2WD      0 2013-01-01 00:00:00-05:00  1985      True   \n",
       "4         Legacy AWD Turbo      0 2013-01-01 00:00:00-05:00  1993     False   \n",
       "...                    ...    ...                       ...   ...       ...   \n",
       "41139               Legacy      0 2013-01-01 00:00:00-05:00  1993      True   \n",
       "41140               Legacy      0 2013-01-01 00:00:00-05:00  1993     False   \n",
       "41141           Legacy AWD      0 2013-01-01 00:00:00-05:00  1993      True   \n",
       "41142           Legacy AWD      0 2013-01-01 00:00:00-05:00  1993     False   \n",
       "41143     Legacy AWD Turbo      0 2013-01-01 00:00:00-05:00  1993      True   \n",
       "\n",
       "       speeds offset              str_date    ffs  \n",
       "0           5    EST  Jan 01 00:00:00 2013   True  \n",
       "1           5    EST  Jan 01 00:00:00 2013  False  \n",
       "2           5    EST  Jan 01 00:00:00 2013   True  \n",
       "3           3    EST  Jan 01 00:00:00 2013    NaN  \n",
       "4           5    EST  Jan 01 00:00:00 2013   True  \n",
       "...       ...    ...                   ...    ...  \n",
       "41139       4    EST  Jan 01 00:00:00 2013   True  \n",
       "41140       5    EST  Jan 01 00:00:00 2013   True  \n",
       "41141       4    EST  Jan 01 00:00:00 2013   True  \n",
       "41142       5    EST  Jan 01 00:00:00 2013   True  \n",
       "41143       4    EST  Jan 01 00:00:00 2013   True  \n",
       "\n",
       "[41144 rows x 17 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweak_autos(autos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import display, HTML\n",
    "def show(df_, rows=20, cols=30, title=None):\n",
    "    if title:\n",
    "        display(HTML(f'<h2>{title}</h2>'))\n",
    "    with pd.option_context('display.min_rows', rows,\n",
    "                           'display.max_columns', cols):\n",
    "        display(df_)\n",
    "    return df_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shape(df_):\n",
    "    print(df_.shape)\n",
    "    return df_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_var(df, var_name):\n",
    "    globals()[var_name] = df\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tweak_autos(autos):\n",
    "    cols = ['city08', 'comb08', 'highway08', 'cylinders',\n",
    "            'displ', 'drive', 'eng_dscr', 'fuelCost08',\n",
    "            'make', 'model', 'trany', 'range', 'createdOn',\n",
    "            'year']\n",
    "    return (autos\n",
    "     [cols]\n",
    "     .assign(cylinders=autos.cylinders.fillna(0).astype('int8'),\n",
    "             displ=autos.displ.fillna(0).astype('float16'),\n",
    "             drive=autos.drive.fillna('Other').astype('category'),\n",
    "             automatic=autos.trany.str.contains('Auto'),\n",
    "             speeds=autos.trany.str.extract(r'(\\d)+').fillna('20')\n",
    "                    .astype('int8'),\n",
    "             offset=autos.createdOn\n",
    "                    .str.extract(r'\\d\\d:\\d\\d ([A-Z]{3}?)')\n",
    "                    .replace('EDT', 'EST5EDT'),\n",
    "             str_date=(autos.createdOn.str.slice(4,19) + ' ' +\n",
    "                       autos.createdOn.str.slice(-4)),\n",
    "             createdOn=lambda df_: to_tz(df_, 'str_date',\n",
    "                       'offset', 'America/New_York'),\n",
    "             ffs=autos.eng_dscr.str.contains('FFS')\n",
    "            )\n",
    "     .pipe(get_var, 'new_cols')\n",
    "     .astype({'highway08': 'int8', 'city08': 'int16',\n",
    "              'comb08': 'int16', 'fuelCost08': 'int16',\n",
    "              'range': 'int16',  'year': 'int16',\n",
    "              'make': 'category'})\n",
    "     .drop(columns=['trany', 'eng_dscr'])\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "res = tweak_autos(autos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city08</th>\n",
       "      <th>comb08</th>\n",
       "      <th>highway08</th>\n",
       "      <th>cylinders</th>\n",
       "      <th>displ</th>\n",
       "      <th>drive</th>\n",
       "      <th>eng_dscr</th>\n",
       "      <th>fuelCost08</th>\n",
       "      <th>make</th>\n",
       "      <th>model</th>\n",
       "      <th>trany</th>\n",
       "      <th>range</th>\n",
       "      <th>createdOn</th>\n",
       "      <th>year</th>\n",
       "      <th>automatic</th>\n",
       "      <th>speeds</th>\n",
       "      <th>offset</th>\n",
       "      <th>str_date</th>\n",
       "      <th>ffs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>19</td>\n",
       "      <td>21</td>\n",
       "      <td>25</td>\n",
       "      <td>4</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>Rear-Wheel Drive</td>\n",
       "      <td>(FFS)</td>\n",
       "      <td>2000</td>\n",
       "      <td>Alfa Romeo</td>\n",
       "      <td>Spider Veloce 2000</td>\n",
       "      <td>Manual 5-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1985</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>12</td>\n",
       "      <td>4.898438</td>\n",
       "      <td>Rear-Wheel Drive</td>\n",
       "      <td>(GUZZLER)</td>\n",
       "      <td>3850</td>\n",
       "      <td>Ferrari</td>\n",
       "      <td>Testarossa</td>\n",
       "      <td>Manual 5-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1985</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>23</td>\n",
       "      <td>27</td>\n",
       "      <td>33</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>Front-Wheel Drive</td>\n",
       "      <td>(FFS)</td>\n",
       "      <td>1550</td>\n",
       "      <td>Dodge</td>\n",
       "      <td>Charger</td>\n",
       "      <td>Manual 5-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1985</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>8</td>\n",
       "      <td>5.199219</td>\n",
       "      <td>Rear-Wheel Drive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3850</td>\n",
       "      <td>Dodge</td>\n",
       "      <td>B150/B250 Wagon 2WD</td>\n",
       "      <td>Automatic 3-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1985</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17</td>\n",
       "      <td>19</td>\n",
       "      <td>23</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>(FFS,TRBO)</td>\n",
       "      <td>2700</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD Turbo</td>\n",
       "      <td>Manual 5-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41139</th>\n",
       "      <td>19</td>\n",
       "      <td>22</td>\n",
       "      <td>26</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>Front-Wheel Drive</td>\n",
       "      <td>(FFS)</td>\n",
       "      <td>1900</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy</td>\n",
       "      <td>Automatic 4-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41140</th>\n",
       "      <td>20</td>\n",
       "      <td>23</td>\n",
       "      <td>28</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>Front-Wheel Drive</td>\n",
       "      <td>(FFS)</td>\n",
       "      <td>1850</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy</td>\n",
       "      <td>Manual 5-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41141</th>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>24</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>(FFS)</td>\n",
       "      <td>2000</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD</td>\n",
       "      <td>Automatic 4-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41142</th>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>24</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>(FFS)</td>\n",
       "      <td>2000</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD</td>\n",
       "      <td>Manual 5-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41143</th>\n",
       "      <td>16</td>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>(FFS,TRBO)</td>\n",
       "      <td>2900</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD Turbo</td>\n",
       "      <td>Automatic 4-spd</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>41144 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       city08  comb08  highway08  cylinders     displ  \\\n",
       "0          19      21         25          4  2.000000   \n",
       "1           9      11         14         12  4.898438   \n",
       "2          23      27         33          4  2.199219   \n",
       "3          10      11         12          8  5.199219   \n",
       "4          17      19         23          4  2.199219   \n",
       "...       ...     ...        ...        ...       ...   \n",
       "41139      19      22         26          4  2.199219   \n",
       "41140      20      23         28          4  2.199219   \n",
       "41141      18      21         24          4  2.199219   \n",
       "41142      18      21         24          4  2.199219   \n",
       "41143      16      18         21          4  2.199219   \n",
       "\n",
       "                            drive    eng_dscr  fuelCost08        make  \\\n",
       "0                Rear-Wheel Drive       (FFS)        2000  Alfa Romeo   \n",
       "1                Rear-Wheel Drive   (GUZZLER)        3850     Ferrari   \n",
       "2               Front-Wheel Drive       (FFS)        1550       Dodge   \n",
       "3                Rear-Wheel Drive         NaN        3850       Dodge   \n",
       "4      4-Wheel or All-Wheel Drive  (FFS,TRBO)        2700      Subaru   \n",
       "...                           ...         ...         ...         ...   \n",
       "41139           Front-Wheel Drive       (FFS)        1900      Subaru   \n",
       "41140           Front-Wheel Drive       (FFS)        1850      Subaru   \n",
       "41141  4-Wheel or All-Wheel Drive       (FFS)        2000      Subaru   \n",
       "41142  4-Wheel or All-Wheel Drive       (FFS)        2000      Subaru   \n",
       "41143  4-Wheel or All-Wheel Drive  (FFS,TRBO)        2900      Subaru   \n",
       "\n",
       "                     model            trany  range                 createdOn  \\\n",
       "0       Spider Veloce 2000     Manual 5-spd      0 2013-01-01 00:00:00-05:00   \n",
       "1               Testarossa     Manual 5-spd      0 2013-01-01 00:00:00-05:00   \n",
       "2                  Charger     Manual 5-spd      0 2013-01-01 00:00:00-05:00   \n",
       "3      B150/B250 Wagon 2WD  Automatic 3-spd      0 2013-01-01 00:00:00-05:00   \n",
       "4         Legacy AWD Turbo     Manual 5-spd      0 2013-01-01 00:00:00-05:00   \n",
       "...                    ...              ...    ...                       ...   \n",
       "41139               Legacy  Automatic 4-spd      0 2013-01-01 00:00:00-05:00   \n",
       "41140               Legacy     Manual 5-spd      0 2013-01-01 00:00:00-05:00   \n",
       "41141           Legacy AWD  Automatic 4-spd      0 2013-01-01 00:00:00-05:00   \n",
       "41142           Legacy AWD     Manual 5-spd      0 2013-01-01 00:00:00-05:00   \n",
       "41143     Legacy AWD Turbo  Automatic 4-spd      0 2013-01-01 00:00:00-05:00   \n",
       "\n",
       "       year automatic  speeds offset              str_date    ffs  \n",
       "0      1985     False       5    EST  Jan 01 00:00:00 2013   True  \n",
       "1      1985     False       5    EST  Jan 01 00:00:00 2013  False  \n",
       "2      1985     False       5    EST  Jan 01 00:00:00 2013   True  \n",
       "3      1985      True       3    EST  Jan 01 00:00:00 2013    NaN  \n",
       "4      1993     False       5    EST  Jan 01 00:00:00 2013   True  \n",
       "...     ...       ...     ...    ...                   ...    ...  \n",
       "41139  1993      True       4    EST  Jan 01 00:00:00 2013   True  \n",
       "41140  1993     False       5    EST  Jan 01 00:00:00 2013   True  \n",
       "41141  1993      True       4    EST  Jan 01 00:00:00 2013   True  \n",
       "41142  1993     False       5    EST  Jan 01 00:00:00 2013   True  \n",
       "41143  1993      True       4    EST  Jan 01 00:00:00 2013   True  \n",
       "\n",
       "[41144 rows x 19 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def err(*args):\n",
    "    1/0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tweak_autos(autos):\n",
    "    cols = ['city08', 'comb08', 'highway08', 'cylinders',\n",
    "            'displ', 'drive', 'eng_dscr', 'fuelCost08',\n",
    "            'make', 'model', 'trany', 'range', 'createdOn',\n",
    "            'year']\n",
    "    return (autos\n",
    "     [cols]\n",
    "     .assign(cylinders=autos.cylinders.fillna(0).astype('int8'),\n",
    "             displ=autos.displ.fillna(0).astype('float16'),\n",
    "             drive=autos.drive.fillna('Other').astype('category'),\n",
    "             automatic=autos.trany.str.contains('Auto'),\n",
    "             speeds=autos.trany.str.extract(r'(\\d)+').fillna('20')\n",
    "                    .astype('int8'),\n",
    "             offset=autos.createdOn\n",
    "                    .str.extract(r'\\d\\d:\\d\\d ([A-Z]{3}?)')\n",
    "                    .replace('EDT', 'EST5EDT'),\n",
    "             str_date=(autos.createdOn.str.slice(4,19) + ' ' +\n",
    "                       autos.createdOn.str.slice(-4)),\n",
    "             createdOn=lambda df_: to_tz(df_, 'str_date',\n",
    "                       'offset', 'America/New_York'),\n",
    "             ffs=autos.eng_dscr.str.contains('FFS')\n",
    "            )\n",
    "     .pipe(err)\n",
    "     .astype({'highway08': 'int8', 'city08': 'int16',\n",
    "              'comb08': 'int16', 'fuelCost08': 'int16',\n",
    "              'range': 'int16',  'year': 'int16',\n",
    "              'make': 'category'})\n",
    "     .drop(columns=['trany', 'eng_dscr'])\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "ename": "ZeroDivisionError",
     "evalue": "division by zero",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mZeroDivisionError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-32-72295a272a22>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtweak_autos\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mautos\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-31-f7f27d16fbba>\u001b[0m in \u001b[0;36mtweak_autos\u001b[0;34m(autos)\u001b[0m\n\u001b[1;32m      4\u001b[0m             \u001b[0;34m'make'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'model'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'trany'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'range'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'createdOn'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m             'year']\n\u001b[0;32m----> 6\u001b[0;31m     return (autos\n\u001b[0m\u001b[1;32m      7\u001b[0m      \u001b[0;34m[\u001b[0m\u001b[0mcols\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m      .assign(cylinders=autos.cylinders.fillna(0).astype('int8'),\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36mpipe\u001b[0;34m(self, func, *args, **kwargs)\u001b[0m\n\u001b[1;32m   5428\u001b[0m         ...  )  # doctest: +SKIP\n\u001b[1;32m   5429\u001b[0m         \"\"\"\n\u001b[0;32m-> 5430\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mcom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpipe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5431\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5432\u001b[0m     \u001b[0;31m# ----------------------------------------------------------------------\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/common.py\u001b[0m in \u001b[0;36mpipe\u001b[0;34m(obj, func, *args, **kwargs)\u001b[0m\n\u001b[1;32m    469\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    470\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 471\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    472\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    473\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-30-1fd410846b52>\u001b[0m in \u001b[0;36merr\u001b[0;34m(*args)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0;36m1\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mZeroDivisionError\u001b[0m: division by zero"
     ]
    }
   ],
   "source": [
    "res = tweak_autos(autos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.debugger import set_trace\n",
    "def err(*args):\n",
    "    set_trace()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DebugException(Exception):\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def debug_var(thing, *, name='debug_item', raise_ex=True):\n",
    "    globals()[name] = thing\n",
    "    if raise_ex:\n",
    "        raise DebugException\n",
    "    return thing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tweak_autos(autos):\n",
    "    cols = ['city08', 'comb08', 'highway08', 'cylinders',\n",
    "            'displ', 'drive', 'eng_dscr', 'fuelCost08',\n",
    "            'make', 'model', 'trany', 'range', 'createdOn',\n",
    "            'year']\n",
    "    return (autos\n",
    "     [cols]\n",
    "     .assign(cylinders=autos.cylinders.fillna(0).astype('int8'),\n",
    "             displ=autos.displ.fillna(0).astype('float16'),\n",
    "             drive=autos.drive.fillna('Other').astype('category'),\n",
    "             automatic=autos.trany.str.contains('Auto'),\n",
    "             speeds=autos.trany.str.extract(r'(\\d)+').fillna('20')\n",
    "                    .astype('int8'),\n",
    "             offset=autos.createdOn\n",
    "                    .str.extract(r'\\d\\d:\\d\\d ([A-Z]{3}?)')\n",
    "                    .replace('EDT', 'EST5EDT'),\n",
    "             str_date=(autos.createdOn.str.slice(4,19) + ' ' +\n",
    "                       autos.createdOn.str.slice(-4)),\n",
    "             createdOn=lambda df_: to_tz(df_, 'str_date',\n",
    "                       'offset', 'America/New_York'),\n",
    "             ffs=autos.eng_dscr.str.contains('FFS')\n",
    "            )\n",
    "     .astype({'highway08': 'int8', 'city08': 'int16',\n",
    "              'comb08': 'int16', 'fuelCost08': 'int16',\n",
    "              'range': 'int16',  'year': 'int16',\n",
    "              'make': 'category'})\n",
    "     .drop(columns=['trany', 'eng_dscr'])\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "ename": "DebugException",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mDebugException\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-37-a1d816b10f80>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mautos2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtweak_autos\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mautos\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mautos2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdebug_var\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'this'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, func, axis, raw, result_type, args, **kwargs)\u001b[0m\n\u001b[1;32m   8734\u001b[0m             \u001b[0mkwargs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   8735\u001b[0m         )\n\u001b[0;32m-> 8736\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   8737\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   8738\u001b[0m     def applymap(\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/apply.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    686\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_raw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    687\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 688\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_standard\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    689\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    690\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0magg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/apply.py\u001b[0m in \u001b[0;36mapply_standard\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    810\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    811\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mapply_standard\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 812\u001b[0;31m         \u001b[0mresults\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mres_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_series_generator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    813\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    814\u001b[0m         \u001b[0;31m# wrap results\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/apply.py\u001b[0m in \u001b[0;36mapply_series_generator\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    826\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseries_gen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    827\u001b[0m                 \u001b[0;31m# ignore SettingWithCopy here in case the user mutates\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 828\u001b[0;31m                 \u001b[0mresults\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    829\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresults\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mABCSeries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    830\u001b[0m                     \u001b[0;31m# If we have a view on v, we need to make a copy because\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/apply.py\u001b[0m in \u001b[0;36mf\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m    129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    130\u001b[0m             \u001b[0;32mdef\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 131\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    133\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-35-5ad52e3ab221>\u001b[0m in \u001b[0;36mdebug_var\u001b[0;34m(thing, name, raise_ex)\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mglobals\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mthing\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mraise_ex\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mDebugException\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mthing\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mDebugException\u001b[0m: "
     ]
    }
   ],
   "source": [
    "autos2 = tweak_autos(autos)\n",
    "autos2.apply(debug_var, name='this')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        19\n",
       "1         9\n",
       "2        23\n",
       "3        10\n",
       "4        17\n",
       "         ..\n",
       "41139    19\n",
       "41140    20\n",
       "41141    18\n",
       "41142    18\n",
       "41143    16\n",
       "Name: city08, Length: 41144, dtype: int16"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "this"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "ename": "DebugException",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mDebugException\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-39-70e837e89998>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mautos2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdebug_var\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, func, axis, raw, result_type, args, **kwargs)\u001b[0m\n\u001b[1;32m   8734\u001b[0m             \u001b[0mkwargs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   8735\u001b[0m         )\n\u001b[0;32m-> 8736\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   8737\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   8738\u001b[0m     def applymap(\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/apply.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    686\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_raw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    687\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 688\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_standard\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    689\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    690\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0magg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/apply.py\u001b[0m in \u001b[0;36mapply_standard\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    810\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    811\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mapply_standard\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 812\u001b[0;31m         \u001b[0mresults\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mres_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_series_generator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    813\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    814\u001b[0m         \u001b[0;31m# wrap results\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/apply.py\u001b[0m in \u001b[0;36mapply_series_generator\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    826\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseries_gen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    827\u001b[0m                 \u001b[0;31m# ignore SettingWithCopy here in case the user mutates\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 828\u001b[0;31m                 \u001b[0mresults\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    829\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresults\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mABCSeries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    830\u001b[0m                     \u001b[0;31m# If we have a view on v, we need to make a copy because\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-35-5ad52e3ab221>\u001b[0m in \u001b[0;36mdebug_var\u001b[0;34m(thing, name, raise_ex)\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mglobals\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mthing\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mraise_ex\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mDebugException\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mthing\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mDebugException\u001b[0m: "
     ]
    }
   ],
   "source": [
    "autos2.apply(debug_var, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "city08                               19\n",
       "comb08                               21\n",
       "highway08                            25\n",
       "cylinders                             4\n",
       "displ                               2.0\n",
       "drive                  Rear-Wheel Drive\n",
       "fuelCost08                         2000\n",
       "make                         Alfa Romeo\n",
       "model                Spider Veloce 2000\n",
       "range                                 0\n",
       "createdOn     2013-01-01 00:00:00-05:00\n",
       "year                               1985\n",
       "automatic                         False\n",
       "speeds                                5\n",
       "offset                              EST\n",
       "str_date           Jan 01 00:00:00 2013\n",
       "ffs                                True\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "debug_item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "ename": "DebugException",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mDebugException\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-41-324b4e781702>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m (autos2\n\u001b[0m\u001b[1;32m      2\u001b[0m  \u001b[0;34m.\u001b[0m\u001b[0massign\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_col\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdebug_var\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m )\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36massign\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m   4479\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4480\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4481\u001b[0;31m             \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_if_callable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   4482\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4483\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/common.py\u001b[0m in \u001b[0;36mapply_if_callable\u001b[0;34m(maybe_callable, obj, **kwargs)\u001b[0m\n\u001b[1;32m    356\u001b[0m     \"\"\"\n\u001b[1;32m    357\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmaybe_callable\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 358\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mmaybe_callable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    359\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    360\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mmaybe_callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-35-5ad52e3ab221>\u001b[0m in \u001b[0;36mdebug_var\u001b[0;34m(thing, name, raise_ex)\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mglobals\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mthing\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mraise_ex\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mDebugException\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mthing\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mDebugException\u001b[0m: "
     ]
    }
   ],
   "source": [
    "(autos2\n",
    " .assign(new_col=debug_var)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city08</th>\n",
       "      <th>comb08</th>\n",
       "      <th>highway08</th>\n",
       "      <th>cylinders</th>\n",
       "      <th>displ</th>\n",
       "      <th>drive</th>\n",
       "      <th>fuelCost08</th>\n",
       "      <th>make</th>\n",
       "      <th>model</th>\n",
       "      <th>range</th>\n",
       "      <th>createdOn</th>\n",
       "      <th>year</th>\n",
       "      <th>automatic</th>\n",
       "      <th>speeds</th>\n",
       "      <th>offset</th>\n",
       "      <th>str_date</th>\n",
       "      <th>ffs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>19</td>\n",
       "      <td>21</td>\n",
       "      <td>25</td>\n",
       "      <td>4</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>Rear-Wheel Drive</td>\n",
       "      <td>2000</td>\n",
       "      <td>Alfa Romeo</td>\n",
       "      <td>Spider Veloce 2000</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1985</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>12</td>\n",
       "      <td>4.898438</td>\n",
       "      <td>Rear-Wheel Drive</td>\n",
       "      <td>3850</td>\n",
       "      <td>Ferrari</td>\n",
       "      <td>Testarossa</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1985</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>23</td>\n",
       "      <td>27</td>\n",
       "      <td>33</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>Front-Wheel Drive</td>\n",
       "      <td>1550</td>\n",
       "      <td>Dodge</td>\n",
       "      <td>Charger</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1985</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>8</td>\n",
       "      <td>5.199219</td>\n",
       "      <td>Rear-Wheel Drive</td>\n",
       "      <td>3850</td>\n",
       "      <td>Dodge</td>\n",
       "      <td>B150/B250 Wagon 2WD</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1985</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17</td>\n",
       "      <td>19</td>\n",
       "      <td>23</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>2700</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD Turbo</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41139</th>\n",
       "      <td>19</td>\n",
       "      <td>22</td>\n",
       "      <td>26</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>Front-Wheel Drive</td>\n",
       "      <td>1900</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41140</th>\n",
       "      <td>20</td>\n",
       "      <td>23</td>\n",
       "      <td>28</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>Front-Wheel Drive</td>\n",
       "      <td>1850</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41141</th>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>24</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>2000</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41142</th>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>24</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>2000</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41143</th>\n",
       "      <td>16</td>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>4</td>\n",
       "      <td>2.199219</td>\n",
       "      <td>4-Wheel or All-Wheel Drive</td>\n",
       "      <td>2900</td>\n",
       "      <td>Subaru</td>\n",
       "      <td>Legacy AWD Turbo</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-01-01 00:00:00-05:00</td>\n",
       "      <td>1993</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "      <td>EST</td>\n",
       "      <td>Jan 01 00:00:00 2013</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>41144 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       city08  comb08  highway08  cylinders     displ  \\\n",
       "0          19      21         25          4  2.000000   \n",
       "1           9      11         14         12  4.898438   \n",
       "2          23      27         33          4  2.199219   \n",
       "3          10      11         12          8  5.199219   \n",
       "4          17      19         23          4  2.199219   \n",
       "...       ...     ...        ...        ...       ...   \n",
       "41139      19      22         26          4  2.199219   \n",
       "41140      20      23         28          4  2.199219   \n",
       "41141      18      21         24          4  2.199219   \n",
       "41142      18      21         24          4  2.199219   \n",
       "41143      16      18         21          4  2.199219   \n",
       "\n",
       "                            drive  fuelCost08        make  \\\n",
       "0                Rear-Wheel Drive        2000  Alfa Romeo   \n",
       "1                Rear-Wheel Drive        3850     Ferrari   \n",
       "2               Front-Wheel Drive        1550       Dodge   \n",
       "3                Rear-Wheel Drive        3850       Dodge   \n",
       "4      4-Wheel or All-Wheel Drive        2700      Subaru   \n",
       "...                           ...         ...         ...   \n",
       "41139           Front-Wheel Drive        1900      Subaru   \n",
       "41140           Front-Wheel Drive        1850      Subaru   \n",
       "41141  4-Wheel or All-Wheel Drive        2000      Subaru   \n",
       "41142  4-Wheel or All-Wheel Drive        2000      Subaru   \n",
       "41143  4-Wheel or All-Wheel Drive        2900      Subaru   \n",
       "\n",
       "                     model  range                 createdOn  year automatic  \\\n",
       "0       Spider Veloce 2000      0 2013-01-01 00:00:00-05:00  1985     False   \n",
       "1               Testarossa      0 2013-01-01 00:00:00-05:00  1985     False   \n",
       "2                  Charger      0 2013-01-01 00:00:00-05:00  1985     False   \n",
       "3      B150/B250 Wagon 2WD      0 2013-01-01 00:00:00-05:00  1985      True   \n",
       "4         Legacy AWD Turbo      0 2013-01-01 00:00:00-05:00  1993     False   \n",
       "...                    ...    ...                       ...   ...       ...   \n",
       "41139               Legacy      0 2013-01-01 00:00:00-05:00  1993      True   \n",
       "41140               Legacy      0 2013-01-01 00:00:00-05:00  1993     False   \n",
       "41141           Legacy AWD      0 2013-01-01 00:00:00-05:00  1993      True   \n",
       "41142           Legacy AWD      0 2013-01-01 00:00:00-05:00  1993     False   \n",
       "41143     Legacy AWD Turbo      0 2013-01-01 00:00:00-05:00  1993      True   \n",
       "\n",
       "       speeds offset              str_date    ffs  \n",
       "0           5    EST  Jan 01 00:00:00 2013   True  \n",
       "1           5    EST  Jan 01 00:00:00 2013  False  \n",
       "2           5    EST  Jan 01 00:00:00 2013   True  \n",
       "3           3    EST  Jan 01 00:00:00 2013    NaN  \n",
       "4           5    EST  Jan 01 00:00:00 2013   True  \n",
       "...       ...    ...                   ...    ...  \n",
       "41139       4    EST  Jan 01 00:00:00 2013   True  \n",
       "41140       5    EST  Jan 01 00:00:00 2013   True  \n",
       "41141       4    EST  Jan 01 00:00:00 2013   True  \n",
       "41142       5    EST  Jan 01 00:00:00 2013   True  \n",
       "41143       4    EST  Jan 01 00:00:00 2013   True  \n",
       "\n",
       "[41144 rows x 17 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "debug_item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "ename": "DebugException",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mDebugException\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-43-dc99ee69aa8c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;34m(\u001b[0m\u001b[0mautos2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroupby\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'make'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'city08'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mdebug_var\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/groupby/generic.py\u001b[0m in \u001b[0;36maggregate\u001b[0;34m(self, func, engine, engine_kwargs, *args, **kwargs)\u001b[0m\n\u001b[1;32m    977\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    978\u001b[0m         \u001b[0mop\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mGroupByApply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 979\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    980\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_dict_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    981\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/apply.py\u001b[0m in \u001b[0;36magg\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    160\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mis_dict_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 161\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg_dict_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    162\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mis_list_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    163\u001b[0m             \u001b[0;31m# we require a list, but not a 'str'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/apply.py\u001b[0m in \u001b[0;36magg_dict_like\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    433\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    434\u001b[0m             \u001b[0;31m# key used for column selection and output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 435\u001b[0;31m             results = {\n\u001b[0m\u001b[1;32m    436\u001b[0m                 \u001b[0mkey\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_gotitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mndim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhow\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhow\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    437\u001b[0m             }\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/apply.py\u001b[0m in \u001b[0;36m<dictcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    434\u001b[0m             \u001b[0;31m# key used for column selection and output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    435\u001b[0m             results = {\n\u001b[0;32m--> 436\u001b[0;31m                 \u001b[0mkey\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_gotitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mndim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhow\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhow\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    437\u001b[0m             }\n\u001b[1;32m    438\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/groupby/generic.py\u001b[0m in \u001b[0;36maggregate\u001b[0;34m(self, func, engine, engine_kwargs, *args, **kwargs)\u001b[0m\n\u001b[1;32m    263\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    264\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 265\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_python_agg_general\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    266\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    267\u001b[0m                 \u001b[0;31m# TODO: KeyError is raised in _python_agg_general,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/groupby/groupby.py\u001b[0m in \u001b[0;36m_python_agg_general\u001b[0;34m(self, func, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1308\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1309\u001b[0m                 \u001b[0;31m# if this function is invalid for this dtype, we will ignore it.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1310\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrouper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg_series\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1311\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1312\u001b[0m                 warnings.warn(\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/groupby/ops.py\u001b[0m in \u001b[0;36magg_series\u001b[0;34m(self, obj, func, preserve_dtype)\u001b[0m\n\u001b[1;32m   1026\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1027\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1028\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_aggregate_series_fast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1029\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1030\u001b[0m         \u001b[0mnpvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmaybe_convert_objects\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtry_float\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/groupby/ops.py\u001b[0m in \u001b[0;36m_aggregate_series_fast\u001b[0;34m(self, obj, func)\u001b[0m\n\u001b[1;32m   1051\u001b[0m         \u001b[0mids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mids\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1052\u001b[0m         \u001b[0msgrouper\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlibreduction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSeriesGrouper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mngroups\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1053\u001b[0;31m         \u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msgrouper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1054\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1055\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/_libs/reduction.pyx\u001b[0m in \u001b[0;36mpandas._libs.reduction.SeriesGrouper.get_result\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/_libs/reduction.pyx\u001b[0m in \u001b[0;36mpandas._libs.reduction._BaseGrouper._apply_to_group\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/groupby/groupby.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m   1294\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_python_agg_general\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1295\u001b[0m         \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_builtin_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1296\u001b[0;31m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1297\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1298\u001b[0m         \u001b[0;31m# iterate through \"columns\" ex exclusions to populate output dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-35-5ad52e3ab221>\u001b[0m in \u001b[0;36mdebug_var\u001b[0;34m(thing, name, raise_ex)\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mglobals\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mthing\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mraise_ex\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mDebugException\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mthing\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mDebugException\u001b[0m: "
     ]
    }
   ],
   "source": [
    "(autos2.groupby('make').agg({'city08': debug_var}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Series([], Name: city08, dtype: int16)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "debug_item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 539305 entries, 0 to 539304\n",
      "Data columns (total 8 columns):\n",
      " #   Column           Non-Null Count   Dtype                         \n",
      "---  ------           --------------   -----                         \n",
      " 0   agency_cd        539305 non-null  object                        \n",
      " 1   site_no          539305 non-null  int64                         \n",
      " 2   datetime         539305 non-null  datetime64[ns, America/Denver]\n",
      " 3   tz_cd            539305 non-null  object                        \n",
      " 4   cfs              493124 non-null  float64                       \n",
      " 5   144166_00060_cd  493124 non-null  object                        \n",
      " 6   gage_height      433377 non-null  float64                       \n",
      " 7   144167_00065_cd  433377 non-null  object                        \n",
      "dtypes: datetime64[ns, America/Denver](1), float64(2), int64(1), object(4)\n",
      "memory usage: 135.1 MB\n"
     ]
    }
   ],
   "source": [
    "dd.info(memory_usage='deep')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "UsageError: Cell magic `%%memit` not found.\n"
     ]
    }
   ],
   "source": [
    "%%memit     # doctest: +SKIP\n",
    "dd = tweak_river(df)     # doctest: +SKIP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "UsageError: Can't use statement directly after '%%time'!\n"
     ]
    }
   ],
   "source": [
    "%%time     # doctest: +SKIP\n",
    "dd = tweak_river(df)     # doctest: +SKIP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "205 ms ± 2.27 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit     # doctest: +SKIP\n",
    "dd = tweak_river(df)     # doctest: +SKIP"
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "cell_metadata_filter": "-all",
   "main_language": "python",
   "notebook_metadata_filter": "-all"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}