Compatibility Issue Detected with Python 3.11 and aws lambda (#73)

* Fixed compatibility issue with python3.11 and aws lambda

* updated pytest and pytest-cov versions in requirements.txt file

* Addressed review comments

Author: amarnathcls

filestack/uploads/intelligent_ingestion.py

@@ -2,12 +2,13 @@
 import os
 import sys
 import mimetypes
+import multiprocessing
 import hashlib
 import logging
 import functools
 import threading
 from urllib3.util.retry import Retry
-from multiprocessing.pool import ThreadPool
+from concurrent.futures import ThreadPoolExecutor
 
 from base64 import b64encode
 
@@ -27,7 +28,7 @@
 CHUNK_SIZE = 8 * MB
 MIN_CHUNK_SIZE = 32 * 1024
 MAX_DELAY = 4
-NUM_THREADS = 4
+NUM_THREADS = multiprocessing.cpu_count()
 
 lock = threading.Lock()
 
@@ -130,8 +131,8 @@ def upload(apikey, filepath, file_obj, storage, params=None, security=None):
         upload_part, apikey, filename, filepath, filesize, storage, start_response
     )
 
-    with ThreadPool(NUM_THREADS) as pool:
-        pool.map(fii_upload, parts)
+    with ThreadPoolExecutor(max_workers=NUM_THREADS) as executor:
+        list(executor.map(fii_upload, parts))
 
     payload.update({
         'uri': start_response['uri'],
@@ -147,7 +148,7 @@ def upload(apikey, filepath, file_obj, storage, params=None, security=None):
 
     complete_url = 'https://{}/multipart/complete'.format(start_response['location_url'])
     session = requests.Session()
-    retries = Retry(total=7, backoff_factor=0.2, status_forcelist=[202], method_whitelist=frozenset(['POST']))
+    retries = Retry(total=7, backoff_factor=0.2, status_forcelist=[202], allowed_methods=frozenset(['POST']))
     session.mount('http://', requests.adapters.HTTPAdapter(max_retries=retries))
     response = session.post(complete_url, json=payload, headers=config.HEADERS)
     if response.status_code != 200:

filestack/uploads/multipart.py

@@ -4,7 +4,7 @@
 import multiprocessing
 from base64 import b64encode
 from functools import partial
-from multiprocessing.pool import ThreadPool
+from concurrent.futures import ThreadPoolExecutor
 
 from filestack import config
 from filestack.utils import requests
@@ -113,10 +113,10 @@ def multipart_upload(apikey, filepath, file_obj, storage, params=None, security=
     chunks = make_chunks(filepath, file_obj, filesize)
     start_response = multipart_request(config.MULTIPART_START_URL, payload, params, security)
     upload_func = partial(upload_chunk, apikey, filename, storage, start_response)
-
-    with ThreadPool(upload_processes) as pool:
-        uploaded_parts = pool.map(upload_func, chunks)
-
+    
+    with ThreadPoolExecutor(max_workers=upload_processes) as executor:
+        uploaded_parts = list(executor.map(upload_func, chunks))
+    
     location_url = start_response.pop('location_url')
     payload.update(start_response)
     payload['parts'] = uploaded_parts

requirements.txt

@@ -1,5 +1,5 @@
-pytest==4.6.3
-pytest-cov==2.5.0
+pytest>=4.6.3
+pytest-cov>=2.5.0
 requests>=2.31.0
 responses==0.14.0
 trafaret==2.0.2