Skip to content

Commit 9a88bb9

Browse files
committed
Changes Requested
1 parent 84a07c7 commit 9a88bb9

File tree

4 files changed

+30
-15
lines changed

4 files changed

+30
-15
lines changed

eland/etl.py

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import pandas as pd # type: ignore
2323
from elasticsearch import Elasticsearch
2424
from elasticsearch.helpers import parallel_bulk
25-
from pandas.core.config_init import is_terminal # type: ignore
2625
from pandas.io.parsers import _c_parser_defaults # type: ignore
2726
from tqdm.notebook import tqdm # type: ignore
2827

@@ -218,7 +217,14 @@ def pandas_to_eland(
218217

219218
if show_progressbar is None or show_progressbar is True:
220219
# Detect jupyter notebook
221-
show_progressbar = False if is_terminal() else True
220+
try:
221+
from IPython import get_ipython
222+
223+
ip = get_ipython()
224+
if hasattr(ip, "kernel"):
225+
show_progressbar = True
226+
except ImportError:
227+
show_progressbar = False
222228

223229
def action_generator(
224230
pd_df: pd.DataFrame,
@@ -260,11 +266,11 @@ def action_generator(
260266
parallel_bulk(
261267
client=es_client,
262268
actions=action_generator(
263-
pd_df,
264-
es_dropna,
265-
use_pandas_index_for_es_ids,
266-
es_dest_index,
267-
show_progressbar,
269+
pd_df=pd_df,
270+
es_dropna=es_dropna,
271+
use_pandas_index_for_es_ids=use_pandas_index_for_es_ids,
272+
es_dest_index=es_dest_index,
273+
show_progressbar=show_progressbar,
268274
),
269275
thread_count=thread_count,
270276
chunk_size=int(chunksize / thread_count),

setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@
7070
classifiers=CLASSIFIERS,
7171
keywords="elastic eland pandas python",
7272
packages=find_packages(include=["eland", "eland.*"]),
73-
install_requires=["elasticsearch>=7.7", "pandas>=1", "matplotlib", "numpy"],
73+
install_requires=["elasticsearch>=7.7", "pandas>=1", "matplotlib", "numpy", "tqdm"],
7474
python_requires=">=3.6",
7575
package_data={"eland": ["py.typed"]},
7676
include_package_data=True,

tests/etl/test_pandas_to_eland.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,14 +70,16 @@ def test_es_if_exists_fail(self):
7070
"to 'append' or 'replace' data."
7171
)
7272

73-
def test_es_if_exists_replace(self):
73+
@pytest.mark.parametrize("show_progressbar", [True, False, None])
74+
def test_es_if_exists_replace(self, show_progressbar):
7475
# Assert that 'replace' allows for creation
7576
df1 = pandas_to_eland(
7677
pd_df2,
7778
es_client=ES_TEST_CLIENT,
7879
es_dest_index="test-index",
7980
es_if_exists="replace",
8081
es_refresh=True,
82+
show_progressbar=show_progressbar,
8183
).to_pandas()
8284
assert_frame_equal(pd_df2, df1)
8385

tests/notebook/test_etl.ipynb

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@
4242
"name": "stdout",
4343
"output_type": "stream",
4444
"text": [
45-
"2020-10-28 16:20:43.614617: read 10000 rows\n",
46-
"2020-10-28 16:20:44.276337: read 13059 rows\n"
45+
"2020-11-17 00:02:17.869106: read 10000 rows\n",
46+
"2020-11-17 00:02:18.873135: read 13059 rows\n"
4747
]
4848
}
4949
],
@@ -80,7 +80,7 @@
8080
{
8181
"data": {
8282
"application/vnd.jupyter.widget-view+json": {
83-
"model_id": "2743ed058a8b44919ad4908f9cac0e95",
83+
"model_id": "a1871aae58ca4aada70ab4df4e1b9673",
8484
"version_major": 2,
8585
"version_minor": 0
8686
},
@@ -238,7 +238,7 @@
238238
{
239239
"data": {
240240
"text/plain": [
241-
"{'took': 0,\n",
241+
"{'took': 4,\n",
242242
" 'timed_out': False,\n",
243243
" '_shards': {'total': 1, 'successful': 1, 'skipped': 0, 'failed': 0},\n",
244244
" 'hits': {'total': {'value': 2, 'relation': 'eq'},\n",
@@ -308,7 +308,7 @@
308308
{
309309
"data": {
310310
"application/vnd.jupyter.widget-view+json": {
311-
"model_id": "97224e2219bf4d828739103324748fb2",
311+
"model_id": "b70a238f828c41cd9ebbb1e74164b057",
312312
"version_major": 2,
313313
"version_minor": 0
314314
},
@@ -390,7 +390,7 @@
390390
{
391391
"data": {
392392
"application/vnd.jupyter.widget-view+json": {
393-
"model_id": "37529288403e4db7b3ae8d97c8666d2e",
393+
"model_id": "877234408b3d40f7af37b0bd9dec9c05",
394394
"version_major": 2,
395395
"version_minor": 0
396396
},
@@ -433,6 +433,13 @@
433433
"source": [
434434
"es.indices.delete(index='pandas_flights', ignore=[400, 404])"
435435
]
436+
},
437+
{
438+
"cell_type": "code",
439+
"execution_count": null,
440+
"metadata": {},
441+
"outputs": [],
442+
"source": []
436443
}
437444
],
438445
"metadata": {

0 commit comments

Comments
 (0)