Skip to content

Commit 6e7a860

Browse files
bigbitbusbigbitbus
committed
Major refactor, aimed at reducing disk usage on the test host
1 parent b0b884c commit 6e7a860

File tree

4 files changed

+221
-127
lines changed

4 files changed

+221
-127
lines changed

‎awsexercizer.py

+87-52
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,27 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
import os, sys, time
15+
import os, sys, time, pickle, boto3, botocore
1616
from os.path import join, getsize, exists
17-
import boto3, botocore
1817
from botocore.client import ClientError
19-
from exercizer import Exercizer
2018
from traceback import format_exc
2119
from pprint import pprint
22-
import pickle
20+
from exercizer import Exercizer
21+
2322
class AWSExercizer(Exercizer):
24-
def __init__(self, env_credentials= {}, region_name ='us-east-1'):
25-
Exercizer.__init__(self)
23+
def __init__(
24+
self,
25+
env_credentials= {},
26+
region_name ='us-east-1',
27+
container_name='blobtester',
28+
fileSizeskb = [],
29+
localDir= '/tmp/localDir',
30+
numIters = 1):
31+
Exercizer.__init__(
32+
self,
33+
fileSizeskb,
34+
localDir,
35+
numIters)
2636
if region_name == 'us-east-1': # aws quirk - you can't specify us-east-1 explicitly
2737
self.storage_client = boto3.client('s3',
2838
aws_access_key_id = os.environ.get(env_credentials['account']),
@@ -33,66 +43,90 @@ def __init__(self, env_credentials= {}, region_name ='us-east-1'):
3343
aws_secret_access_key = os.environ.get(env_credentials['secret']),
3444
region_name=region_name)
3545
self.region_name = region_name
46+
self.container_name = container_name
3647

3748

38-
def UploadObjectsToContainer(self, container_name='blobtester', localDir = '/tmp/smalldir'):
49+
def UploadObjectsToContainer(self):
3950
# create bucket if it does not exist
4051
try:
41-
self.storage_client.head_bucket(Bucket=container_name)
52+
self.storage_client.head_bucket(Bucket=self.container_name)
4253
except ClientError:
4354
if self.region_name != 'us-east-1':
4455
container = self.storage_client.create_bucket(
45-
Bucket=container_name,
56+
Bucket=self.container_name,
4657
CreateBucketConfiguration={
4758
'LocationConstraint': self.region_name
4859
})
4960
else:
50-
container = self.storage_client.create_bucket(Bucket=container_name)
61+
container = self.storage_client.create_bucket(
62+
Bucket=self.container_name)
5163

52-
dic_uploadData = {}
53-
for root, dirs, files in os.walk(localDir):
54-
for name in files:
55-
filePath = join(root,name)
56-
self.startTimer()
57-
try:
58-
self.storage_client.upload_file(filePath, container_name, filePath)
59-
dic_uploadData[filePath] = (self.endTimer(), getsize(filePath))
60-
except:
61-
print ('Failure uploading {}'.format(filePath))
62-
print (format_exc())
63-
self.endTimer()
64-
dic_uploadData[filePath] = -1
65-
return dic_uploadData
64+
list_uploadData = []
65+
for eachFile in self.manifest:
66+
filePath, intfilesize = eachFile
67+
print "U",
68+
print eachFile
69+
self.makeOneRandomBinFile(filePath, intfilesize)
70+
self.startTimer()
71+
try:
72+
self.storage_client.upload_file(
73+
filePath, self.container_name, filePath)
74+
list_uploadData.append(
75+
(self.endTimer(), getsize(filePath), 'aws_upload'))
76+
except:
77+
print ('Failure uploading {}'.format(filePath))
78+
print (format_exc())
79+
self.endTimer()
80+
os.remove(filePath)
81+
return list_uploadData
6682

67-
def ListObjectsInContainer(self, container_name = 'blobtester'):
83+
def ListObjectsInContainer(self):
6884
'''
6985
Return generator with the list of blobs
7086
'''
71-
return self.storage_client.list_objects_v2(Bucket=container_name)['Contents']
87+
objList = self.storage_client.list_objects_v2(
88+
Bucket=self.container_name)
89+
if 'Contents' in objList:
90+
return objList['Contents']
91+
else:
92+
return []
7293

73-
def DownloadObjectsFromContainer(self, container_name = 'blobtester', localDir = '/tmp/smalldir'):
74-
if not exists(localDir):
75-
os.makedirs(localDir)
76-
dic_downloadData = {}
77-
self.startTimer()
78-
blobList = self.ListObjectsInContainer(container_name)
79-
dic_downloadData['_listObjectsInContainer'] = (self.endTimer(), "Container objects listing time")
94+
def DownloadObjectsFromContainer(self):
95+
if not exists(self.localDir):
96+
os.makedirs(self.localDir)
97+
list_downloadData = []
98+
blobList = self.ListObjectsInContainer()
8099
for aBlob in blobList:
81100
self.startTimer()
82-
localPath = join(localDir,aBlob['Key'].split('/')[-1])
83-
self.storage_client.download_file(container_name, aBlob['Key'], localPath)
84-
dic_downloadData[localPath] = (self.endTimer(), getsize(localPath))
85-
return dic_downloadData
101+
localPath = join(self.localDir,aBlob['Key'].split('/')[-1])
102+
self.storage_client.download_file(
103+
self.container_name,
104+
aBlob['Key'], localPath)
105+
blobsize = getsize(localPath)
106+
list_downloadData.append(
107+
(self.endTimer(), blobsize, 'aws_download'))
108+
print "D",
109+
print (localPath, blobsize)
110+
os.remove(localPath)
111+
self.startTimer()
112+
self.storage_client.delete_object(
113+
Bucket = self.container_name,
114+
Key = aBlob['Key'])
115+
list_downloadData.append((self.endTimer(), blobsize, 'aws_delete'))
116+
return list_downloadData
86117

87118
def DeleteContainer(self, container_name='blobtester'):
88119
self.startTimer()
89-
blobList = self.ListObjectsInContainer(container_name)
120+
blobList = self.ListObjectsInContainer()
90121
deleteList = []
91122
for aBlob in blobList:
92123
deleteList.append({'Key': aBlob['Key']})
93-
self.storage_client.delete_objects(Bucket = container_name, Delete = { 'Objects': deleteList })
94-
self.storage_client.delete_bucket(Bucket = container_name)
95-
return {container_name: self.endTimer(), 'operation':'Deleted'}
124+
if len(deleteList) > 0:
125+
self.storage_client.delete_objects(
126+
Bucket = self.container_name,
127+
Delete = { 'Objects': deleteList })
128+
self.storage_client.delete_bucket(Bucket = self.container_name)
129+
return {self.container_name: self.endTimer(), 'operation':'Deleted'}
96130

97131

98132
if __name__=="__main__":
@@ -101,17 +135,18 @@ def DeleteContainer(self, container_name='blobtester'):
101135
'account': 'S3KEY',
102136
'secret':'S3SECRET'
103137
}
104-
awsex = AWSExercizer(env_credentials = env_credentials)
105-
# Upload
106-
# print "Upload objects"
107-
# pprint(awsex.UploadObjectsToContainer())
108-
# Download
109-
# print "Download objects"
110-
# pprint(awsex.DownloadObjectsFromContainer())
111-
# Delete
138+
awsex = AWSExercizer(
139+
env_credentials = env_credentials,
140+
localDir = sys.argv[1],
141+
numIters = sys.argv[2],
142+
fileSizeskb = sys.argv[3:])
112143

113-
pickle.dump(awsex.UploadObjectsToContainer(localDir = sys.argv[1]), open('/tmp/outputdata/objbench/aws_upload.pkl','wb'))
144+
pickle.dump(
145+
awsex.UploadObjectsToContainer(),
146+
open('/tmp/outputdata/objbench/aws_upload.pkl','wb'))
114147
# Download
115-
pickle.dump(awsex.DownloadObjectsFromContainer(localDir = sys.argv[1]), open('/tmp/outputdata/objbench/aws_download.pkl','wb'))
116-
print "Delete buckets"
148+
pickle.dump(
149+
awsex.DownloadObjectsFromContainer(),
150+
open('/tmp/outputdata/objbench/aws_download.pkl','wb'))
151+
print "Delete bucket"
117152
pprint(awsex.DeleteContainer())

‎azureexercizer.py

+78-42
Original file line numberDiff line numberDiff line change
@@ -12,61 +12,88 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
import os, sys, time
15+
import os, sys, time, pickle
1616
from os.path import join, getsize, exists
1717
from azure.storage.blob import BlockBlobService
1818
from pprint import pprint
1919
from exercizer import Exercizer
20-
import pickle
21-
class AzureExercizer(Exercizer):
22-
def __init__(self, env_credentials):
23-
Exercizer.__init__(self)
20+
from traceback import format_exc
21+
class AzureExercizer(Exercizer): #No easy way of getting auto regions
22+
def __init__(
23+
self,
24+
env_credentials,
25+
container_name='blobtester',
26+
fileSizeskb = [],
27+
localDir= '/tmp/self.localDir',
28+
numIters = 1):
29+
Exercizer.__init__(
30+
self,
31+
fileSizeskb,
32+
localDir,
33+
numIters)
34+
2435
self.storage_client = BlockBlobService(
2536
os.environ.get(env_credentials['account']),
2637
os.environ.get(env_credentials['secret']))
38+
self.container_name = container_name
2739

28-
def UploadObjectsToContainer(self, container_name='blobtester', localDir = '/tmp/smalldir'):
29-
self.storage_client.create_container(container_name)
30-
dic_uploadData = {}
31-
for root, dirs, files in os.walk(localDir):
32-
for name in files:
33-
filePath = join(root,name)
34-
self.startTimer()
35-
try:
36-
self.storage_client.create_blob_from_path(container_name,
37-
name, filePath)
38-
dic_uploadData[filePath] = (self.endTimer(), getsize(filePath))
39-
except:
40-
print ('Failure uploading {}'.format(filePath))
41-
print (format_exc())
42-
self.endTimer()
43-
dic_uploadData[filePath] = -1
44-
return dic_uploadData
40+
def UploadObjectsToContainer(self):
41+
self.storage_client.create_container(self.container_name)
42+
list_uploadData = []
43+
for eachFile in self.manifest:
44+
filePath, intfilesize = eachFile
45+
print "U",
46+
print eachFile
47+
self.makeOneRandomBinFile(filePath, intfilesize)
48+
self.startTimer()
49+
try:
50+
self.storage_client.create_blob_from_path(
51+
self.container_name,
52+
filePath.split('/')[-1],
53+
filePath)
54+
list_uploadData.append(
55+
(self.endTimer(), getsize(filePath), 'azure_upload'))
56+
except:
57+
print ('Failure uploading {}'.format(filePath))
58+
print format_exc()
59+
self.endTimer()
60+
return list_uploadData
4561

46-
def ListObjectsInContainer(self, container_name = 'blobtester'):
62+
def ListObjectsInContainer(self):
4763
'''
48-
Return generator with the list of blob names
64+
Return list with the list of blob names
4965
'''
50-
return self.storage_client.list_blobs(container_name)
66+
return list(self.storage_client.list_blobs(self.container_name))
5167

52-
def DownloadObjectsFromContainer(self, container_name = 'blobtester', localDir = '/tmp/smalldir'):
53-
if not exists(localDir):
54-
os.makedirs(localDir)
55-
dic_downloadData = {}
56-
self.startTimer()
57-
blobListGenerator = self.ListObjectsInContainer(container_name)
58-
dic_downloadData['_listObjectsInContainer'] = (self.endTimer(), "Container objects listing time")
59-
for aBlob in blobListGenerator:
68+
def DownloadObjectsFromContainer(self):
69+
if not exists(self.localDir):
70+
os.makedirs(self.localDir)
71+
list_downloadData = []
72+
blobList = self.ListObjectsInContainer()
73+
for aBlob in blobList:
74+
self.startTimer()
75+
localPath = join(self.localDir,aBlob.name.split('/')[-1])
76+
self.storage_client.get_blob_to_path(
77+
self.container_name,
78+
aBlob.name,
79+
localPath )
80+
blobsize = getsize(localPath)
81+
list_downloadData.append(
82+
(self.endTimer(), blobsize , 'azure_download'))
83+
print "D",
84+
print (localPath, blobsize)
85+
os.remove(localPath)
6086
self.startTimer()
61-
localPath = join(localDir,aBlob.name.split('/')[-1])
62-
self.storage_client.get_blob_to_path(container_name, aBlob.name, localPath )
63-
dic_downloadData[localPath] = (self.endTimer(), getsize(localPath))
64-
return dic_downloadData
87+
self.storage_client.delete_blob(
88+
self.container_name, aBlob.name)
89+
list_downloadData.append(
90+
(self.endTimer(), blobsize , 'azure_delete'))
91+
return list_downloadData
6592

6693
def DeleteContainer(self, container_name='blobtester'):
6794
self.startTimer()
68-
self.storage_client.delete_container(container_name)
69-
return {container_name: self.endTimer(), 'operation':'Deleted'}
95+
self.storage_client.delete_container(self.container_name)
96+
return {self.container_name: self.endTimer(), 'operation':'Deleted'}
7097

7198

7299

@@ -77,15 +104,24 @@ def DeleteContainer(self, container_name='blobtester'):
77104
'secret':'AZBLOBKEY'
78105
}
79106

80-
azex = AzureExercizer(env_credentials)
107+
azex = AzureExercizer(
108+
env_credentials,
109+
localDir = sys.argv[1],
110+
numIters = sys.argv[2],
111+
fileSizeskb = sys.argv[3:])
81112
# Upload
82113
# pprint(azex.UploadObjectsToContainer())
83114
# Download
84115
# pprint(azex.DownloadObjectsFromContainer())
85116

86-
pickle.dump(azex.UploadObjectsToContainer(localDir = sys.argv[1]), open('/tmp/outputdata/objbench/az_upload.pkl','wb'))
117+
pickle.dump(
118+
azex.UploadObjectsToContainer(),
119+
open('/tmp/outputdata/objbench/az_upload.pkl','wb'))
87120
# Download
88-
pickle.dump(azex.DownloadObjectsFromContainer(localDir = sys.argv[1]), open('/tmp/outputdata/objbench/az_download.pkl','wb'))
121+
pickle.dump(
122+
azex.DownloadObjectsFromContainer(),
123+
open('/tmp/outputdata/objbench/az_download.pkl','wb'))
89124

90125
# Delete
126+
print "Delete bucket"
91127
pprint(azex.DeleteContainer())

‎exercizer.py

+18-6
Original file line numberDiff line numberDiff line change
@@ -12,22 +12,26 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
import os, time
15+
import os, time, uuid
1616
from os.path import join, getsize ,exists
1717
from traceback import format_exc
1818
class Exercizer(object):
19-
def __init__(self):
19+
def __init__(self, filesSizeskb, localDir, numIters):
2020
self.resetTimer()
21-
21+
self.fileSizeskb = filesSizeskb
22+
self.localDir = localDir
23+
self.numIters = int(numIters)
24+
self.uploadManifest()
25+
2226
def makeOneRandomBinFile (self,
2327
filePath ='/tmp/file',
24-
sizeinkb = 100,
28+
sizeinbytes = 100,
2529
):
2630
try:
2731
with open(filePath, 'wb') as fout:
28-
fout.write(os.urandom(sizeinkb*1000))
32+
fout.write(os.urandom(sizeinbytes))
2933
except:
30-
print "Error creating {} of size {}".format(filePath,sizeinkb)
34+
print "Error creating {} of size {}".format(filePath, sizeinbytes)
3135
print format_exc()
3236

3337
def makeRandomBinFiles (self,
@@ -62,3 +66,11 @@ def endTimer(self):
6266
self.startTime = -1
6367
return (timeElapsed)
6468

69+
def uploadManifest(self):
70+
self.manifest = []
71+
for ii in range(0, self.numIters):
72+
for fSizekb in self.fileSizeskb:
73+
intfilesize = int(fSizekb)*1000
74+
filePath = join(self.localDir,str(uuid.uuid4()))
75+
self.manifest.append((filePath, intfilesize))
76+

0 commit comments

Comments
 (0)