Add requirements and tests

Author: V1NAY8

eland/etl.py

@@ -22,9 +22,9 @@
 import pandas as pd  # type: ignore
 from elasticsearch import Elasticsearch
 from elasticsearch.helpers import parallel_bulk
+from pandas.core.config_init import is_terminal  # type: ignore
 from pandas.io.parsers import _c_parser_defaults  # type: ignore
-from tqdm.notebook import tqdm
-from pandas.core.config_init import is_terminal
+from tqdm.notebook import tqdm  # type: ignore
 
 from eland import DataFrame
 from eland.common import DEFAULT_CHUNK_SIZE, ensure_es_client
@@ -110,7 +110,7 @@ def pandas_to_eland(
     show_progressbar: Optional[bool], default 'None'
         * True : show a progress bar only if we detect Jupyter Notebook (for now)
         * False : don't show a progress bar
-        * None : show a progress bar only if we detect Jupyter Notebook 
+        * None : show a progress bar only if we detect Jupyter Notebook
 
     Returns
     -------

requirements-dev.txt

@@ -10,3 +10,4 @@ xgboost>=1
 nox
 lightgbm>=2.3.0
 pytest-cov
+tqdm

requirements.txt

@@ -1,3 +1,4 @@
 elasticsearch>=7.7
 pandas>=1
 matplotlib
+tqdm

tests/notebook/test_etl.ipynb

@@ -42,8 +42,8 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "2020-10-28 22:01:46.397163: read 10000 rows\n",
-      "2020-10-28 22:01:47.100938: read 13059 rows\n"
+      "2020-10-28 16:20:43.614617: read 10000 rows\n",
+      "2020-10-28 16:20:44.276337: read 13059 rows\n"
      ]
     }
    ],
@@ -80,7 +80,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "3e3e6e7371be43aabd4f9a2bb62ed737",
+       "model_id": "2743ed058a8b44919ad4908f9cac0e95",
        "version_major": 2,
        "version_minor": 0
       },
@@ -299,6 +299,140 @@
    "source": [
     "es.indices.delete(index='churn', ignore=[400, 404])"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "97224e2219bf4d828739103324748fb2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HBox(children=(HTML(value='Progress'), FloatProgress(value=0.0, max=13059.0), HTML(value='')))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "# NBVAL_IGNORE_OUTPUT\n",
+    "ed_df = ed.pandas_to_eland(pd_df, es_client='localhost', es_dest_index=\"pandas_flights\", show_progressbar=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'acknowledged': True}"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "es.indices.delete(index='pandas_flights', ignore=[400, 404])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# NBVAL_IGNORE_OUTPUT\n",
+    "ed_df = ed.pandas_to_eland(pd_df, es_client='localhost', es_dest_index=\"pandas_flights\", show_progressbar=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'acknowledged': True}"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "es.indices.delete(index='pandas_flights', ignore=[400, 404])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "37529288403e4db7b3ae8d97c8666d2e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HBox(children=(HTML(value='Progress'), FloatProgress(value=0.0, max=13059.0), HTML(value='')))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "# NBVAL_IGNORE_OUTPUT\n",
+    "ed_df = ed.pandas_to_eland(pd_df, es_client='localhost', es_dest_index=\"pandas_flights\", show_progressbar=None)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'acknowledged': True}"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "es.indices.delete(index='pandas_flights', ignore=[400, 404])"
+   ]
   }
  ],
  "metadata": {