Fixed names and updated utils

miodeqqq · Dec 3, 2016 · 19f2820 · 19f2820
1 parent 583af10
commit 19f2820
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 14 deletions.
diff --git a/pdf_to_grobid.py b/pdf_to_grobid.py
@@ -9,7 +9,7 @@
 
 import requests
 
-from .utils import Colors, urls_dict
+from .utils import Colors, urls_dict, headers
 
 PDFS = sys.argv[1]
 XMLS_PATH_DOWNLOAD = sys.argv[2]
@@ -18,7 +18,7 @@
 class Grobid(object):
     """
     Usage: ./pdf_to_grobid.py pdfs_dir grobid_output_dir grobid_url
-    For example: ./pdf_to_grobid.py pdfs_data grobid_output_data local
+    For example: ./pdf_to_grobid.py pdfs_data grobid_output_data grobid
     """
 
     def __init__(self, *args, **kwargs):
@@ -102,10 +102,6 @@ def process_pdfs_to_grobid(self):
                     'input': pdf
                 }
 
-                headers = {
-                    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36',
-                }
-
                 try:
 
                     grobid = session.post(

diff --git a/pdf_to_tika.py b/pdf_to_tika.py
@@ -9,16 +9,16 @@
 
 import requests
 
-from .utils import Colors, urls_dict
+from .utils import Colors, urls_dict, headers
 
 PDFS = sys.argv[1]
 TXT_PATH_DOWNLOAD = sys.argv[2]
 
 
 class Tika(object):
     """
-    Usage: ./pdf_file_to_tika.py pdfs_dir tika_output_dir tika_url
-    For example: ./pdf_file_to_grobid.py pdfs_data tika_output_data rs
+    Usage: ./pdf_to_tika.py pdfs_dir tika_output_dir tika_url
+    For example: ./pdf_to_tika.py pdfs_data tika_output_data tika
     """
 
     def __init__(self, *args, **kwargs):
@@ -96,10 +96,6 @@ def process_pdfs_to_tika(self):
 
                 pdf_file = open(file, 'rb')
 
-                headers = {
-                    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36',
-                }
-
                 try:
 
                     tika = session.put(
@@ -148,7 +144,7 @@ def process_pdfs_to_tika(self):
                                 file_ext='.txt'), 'w') as tika_output:
                             tika_output.write(txt_file)
 
-                            print('Saved grobid TXT output to file!')
+                            print('Saved TIKA TXT output to file!')
 
                             print(Colors.BOLD + '________________________________________________' + Colors.ENDC)
 

diff --git a/utils.py b/utils.py
@@ -20,4 +20,8 @@ class Colors():
 urls_dict = {
     'grobid': 'http://localhost:1234/processFulltextDocument',
     'tika': 'http://localhost:9876/tika',
+}
+
+headers = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36',
 }