feat(s3): s3client(), file_exists() methods

Author: gadicc

utils/storage/S3Storage.py

@@ -1,4 +1,5 @@
 import boto3
+import botocore
 import re
 import os
 import time
@@ -45,13 +46,14 @@ def __init__(self, url, path=""):
         self.bucket_name = s3_dest["bucket"]
         self.path = s3_dest["path"]
 
-        self._s3 = None
+        self._s3resource = None
+        self._s3client = None
         self._bucket = None
         print("self.endpoint_url", self.endpoint_url)
 
-    def s3(self):
-        if self._s3:
-            return self._s3
+    def s3resource(self):
+        if self._s3resource:
+            return self._s3resource
 
         self._s3 = boto3.resource(
             "s3",
@@ -60,11 +62,22 @@ def s3(self):
         )
         return self._s3
 
+    def s3client(self):
+        if self._s3client:
+            return self._s3client
+
+        self._s3client = boto3.client(
+            "s3",
+            endpoint_url=self.endpoint_url,
+            config=Config(signature_version="s3v4"),
+        )
+        return self._s3client
+
     def bucket(self):
         if self._bucket:
             return self._bucket
 
-        self._bucket = self.s3().Bucket(self.bucket_name)
+        self._bucket = self.s3resource().Bucket(self.bucket_name)
         return self._bucket
 
     def upload_file(self, source, dest):
@@ -88,7 +101,7 @@ def download_file(self, dest):
         if not dest:
             dest = self.path.split("/").pop()
         print(f"Downloading {self.url} to {dest}...")
-        object = self.s3().Object(self.bucket_name, self.path)
+        object = self.s3resource().Object(self.bucket_name, self.path)
         object.load()
 
         with tqdm(
@@ -98,3 +111,18 @@ def download_file(self, dest):
                 Filename=dest,
                 Callback=lambda bytes_transffered: bar.update(bytes_transffered),
             )
+
+    def file_exists(self):
+        # res = self.s3client().list_objects_v2(
+        #    Bucket=self.bucket_name, Prefix=self.path, MaxKeys=1
+        # )
+        # return "Contents" in res
+        object = self.s3resource().Object(self.bucket_name, self.path)
+        try:
+            object.load()
+        except botocore.exceptions.ClientError as error:
+            if error.response["Error"]["Code"] == "404":
+                return False
+            else:
+                raise
+        return True