Skip to content

Commit b51f1f2

Browse files
JayRaj21JayRaj21
authored
Add files via upload
1 parent cdd529a commit b51f1f2

File tree

1 file changed

+331
-0
lines changed

1 file changed

+331
-0
lines changed

‎AdvCompSys-Project-4.py‎

Lines changed: 331 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,331 @@
1+
from time import perf_counter
2+
from threading import Thread, Lock
3+
import zlib
4+
5+
"""
6+
Your implementation should support the following operations:
7+
(1) Put(key, value): Inserts or updates a key-value pair
8+
(2) Get(key): Retrieves the value associated with a given key
9+
(3) Delete(key): Removes a key-value pair from the store
10+
(4) Support for multiple concurrent read and write operations
11+
(5) Optional (as a bonus): Support lossless in-memory data compression at small speed performance loss
12+
13+
You should also write a testbench to test your in-memory key-value store under
14+
(1) different operational concurrency (one user, two users, 4 users, …),
15+
(2) different number of internal working threads (4, 8, 16, …),
16+
(3) Different read vs. write ratio, and...
17+
(4) Different value size (8B, 64B, 256B) (you may keep the key size as 8B).
18+
19+
Number of iterations: numUsers*numThreadsPerUser*readWriteRatio*valueSize*8B
20+
"""
21+
22+
encodeMap = dict()
23+
myLock = Lock()
24+
encList = list()
25+
26+
test = [b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12"]
27+
28+
smallData = [b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes"]
29+
30+
mediumData = [b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
31+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
32+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
33+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
34+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
35+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
36+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
37+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes"]
38+
39+
largeData = [b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
40+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
41+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
42+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
43+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
44+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
45+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
46+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
47+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
48+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
49+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
50+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
51+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
52+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
53+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
54+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
55+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
56+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
57+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
58+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
59+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
60+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
61+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
62+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
63+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
64+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
65+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
66+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
67+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
68+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
69+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes",
70+
b"honda123", b"chevy123", b"toyota12", b"honda123", b"toyota12", b"ford1234", b"chevy123", b"mercedes"]
71+
72+
def dictEncode(data, compressFlag):
73+
global encodeMap
74+
global myLock
75+
global encList
76+
myLock.acquire()
77+
counter = len(encodeMap.keys()) + 1
78+
for i in range(len(data)):
79+
if (compressFlag):
80+
if zlib.compress(data[i]) in encodeMap.values():
81+
tempIndex = (list(encodeMap.values())).index(zlib.compress(data[i]))
82+
encList.append(list(encodeMap.keys())[tempIndex])
83+
else:
84+
encodeMap[counter] = zlib.compress(data[i])
85+
encList.append(counter)
86+
counter += 1
87+
else:
88+
if data[i] in encodeMap.values():
89+
tempIndex = (list(encodeMap.values())).index(data[i])
90+
encList.append(list(encodeMap.keys())[tempIndex])
91+
else:
92+
encodeMap[counter] = data[i]
93+
encList.append(counter)
94+
counter += 1
95+
myLock.release()
96+
return
97+
98+
def dictDecode(data, compressFlag):
99+
global encodeMap
100+
global myLock
101+
decodeList = list()
102+
myLock.acquire()
103+
for i in range(len(data)):
104+
if (compressFlag):
105+
decodeList.append(zlib.decompress(encodeMap[data[i]]))
106+
else:
107+
decodeList.append(encodeMap[data[i]])
108+
myLock.release()
109+
#print(f"Decoded Length: {len(decodeList)}")
110+
#print(f"Decoded Value: {decodeList}\n")
111+
return
112+
113+
def SingleUsers(data, maxThreads):
114+
threadList = list()
115+
for i in range(1,maxThreads+1):
116+
thread = Thread(target=executeSingleUser, args=(data, ))
117+
threadList.append(thread)
118+
119+
startTime = perf_counter()
120+
for thrd in threadList:
121+
thrd.start()
122+
123+
for thred in threadList:
124+
thred.join()
125+
126+
dictDecode(encList, 0)
127+
#print(f"Mapping: {encodeMap}\n")
128+
129+
endTime = perf_counter()
130+
return f"{endTime - startTime}"
131+
132+
def TwoUsers(data, maxThreads):
133+
print(f"***Starting Two User Test***")
134+
userList = list()
135+
for i in range(1,3):
136+
thread = Thread(target=SingleUsers, args=(data, maxThreads))
137+
userList.append(thread)
138+
139+
startTime = perf_counter()
140+
141+
for thrd in userList:
142+
thrd.start()
143+
144+
for thred in userList:
145+
thred.join()
146+
147+
endTime = perf_counter()
148+
return f"{endTime - startTime}"
149+
150+
def FourUsers(data, maxThreads):
151+
print(f"***Four Single User Test***")
152+
userList = list()
153+
for i in range(1,5):
154+
thread = Thread(target=SingleUsers, args=(data, maxThreads))
155+
userList.append(thread)
156+
157+
startTime = perf_counter()
158+
159+
for thrd in userList:
160+
thrd.start()
161+
162+
for thred in userList:
163+
thred.join()
164+
endTime = perf_counter()
165+
return f"{endTime - startTime}"
166+
167+
def executeSingleUser(data):
168+
dictEncode(data, 0)
169+
#print(f"Encoded Length:{len(encList)}\n")
170+
#print(f"Encoded Values:{encList}\n")
171+
return
172+
173+
def EqualReadWrite(data):
174+
print("***Start equal read/write ratio test***\n")
175+
threadList = list()
176+
for i in range(1,2):
177+
thread = Thread(target=executeSingleUser, args=(data, ))
178+
threadList.append(thread)
179+
thread = Thread(target=dictDecode, args=(encList, 0))
180+
threadList.append(thread)
181+
182+
startTime = perf_counter()
183+
for thrd in threadList:
184+
thrd.start()
185+
186+
for thred in threadList:
187+
thred.join()
188+
189+
#print(f"Mapping: {encodeMap}\n")
190+
endTime = perf_counter()
191+
return f"{endTime - startTime}"
192+
193+
def TwoReadsOneWrite(data):
194+
print("***Start 2R/1W test***\n")
195+
threadList = list()
196+
for i in range(1,2):
197+
thread = Thread(target=executeSingleUser, args=(data, ))
198+
threadList.append(thread)
199+
thread = Thread(target=dictDecode, args=(encList, 0))
200+
threadList.append(thread)
201+
thread = Thread(target=dictDecode, args=(encList, 0))
202+
threadList.append(thread)
203+
204+
startTime = perf_counter()
205+
for thrd in threadList:
206+
thrd.start()
207+
208+
for thred in threadList:
209+
thred.join()
210+
211+
#print(f"Mapping: {encodeMap}\n")
212+
endTime = perf_counter()
213+
return f"{endTime - startTime}"
214+
215+
def OneReadTwoWrites(data):
216+
print("***Start 1R/2W test***\n")
217+
threadList = list()
218+
for i in range(1,2):
219+
thread = Thread(target=executeSingleUser, args=(data, ))
220+
threadList.append(thread)
221+
thread = Thread(target=executeSingleUser, args=(data, ))
222+
threadList.append(thread)
223+
thread = Thread(target=dictDecode, args=(encList, 0))
224+
threadList.append(thread)
225+
226+
startTime = perf_counter()
227+
for thrd in threadList:
228+
thrd.start()
229+
230+
for thred in threadList:
231+
thred.join()
232+
233+
#print(f"Mapping: {encodeMap}\n")
234+
endTime = perf_counter()
235+
return f"{endTime - startTime}"
236+
237+
def prefixScan(encodedData, text):
238+
print("*** Starting Prefix Scan ***")
239+
global encodeMap
240+
foundValues = list()
241+
startTime = perf_counter()
242+
for elem in encodedData:
243+
if ((encodeMap[elem].decode("utf-8")).startswith(text)):
244+
foundValues.append(encodeMap[elem].decode("utf-8"))
245+
else:
246+
continue
247+
248+
endTime = perf_counter()
249+
if (len(foundValues) > 0):
250+
print(f"Found Values: {foundValues}")
251+
else:
252+
print(f"No prefix instances of \"{text}\" are present")
253+
254+
return f"{endTime - startTime}"
255+
256+
257+
if __name__ == "__main__":
258+
#Time values will vary since they are not consistent when redoing tests
259+
print()
260+
# 1. Internal threads test
261+
# print(SingleUsers(smallData, 4))
262+
# print(SingleUsers(smallData, 8))
263+
# print(SingleUsers(smallData, 16))
264+
265+
# 2. User test
266+
# print(SingleUsers(smallData, 1))
267+
# print(TwoUsers(smallData, 1))
268+
# print(FourUsers(smallData, 1))
269+
270+
# 3. Read/Write Tests -> Only test one line at a time, otherwise it breaks
271+
# print(EqualReadWrite(smallData))
272+
# print(TwoReadsOneWrite(smallData))
273+
# print(OneReadTwoWrites(smallData))
274+
275+
# 4. Data Size Test
276+
# print(SingleUsers(smallData, 1))
277+
# print(SingleUsers(mediumData, 1))
278+
# print(SingleUsers(largeData, 1))
279+
280+
# 5. Prefix Scan Test
281+
# dictEncode(mediumData, 0)
282+
# time = prefixScan(encList, "toy")
283+
# print(time)
284+
# print()
285+
# time = prefixScan(encList, "or")
286+
# print(time)
287+
288+
# 6. Data Compression test
289+
# startTime = perf_counter()
290+
# dictEncode(smallData, 1)
291+
# print(encList)
292+
# dictDecode(encList, 1)
293+
# endTime = perf_counter()
294+
# print(endTime-startTime)
295+
296+
# startTime = perf_counter()
297+
# dictEncode(smallData, 0)
298+
# #print(encList)
299+
# dictDecode(encList, 0)
300+
# endTime = perf_counter()
301+
# print(endTime-startTime)
302+
# print(encodeMap)
303+
304+
# Internal Threads Times
305+
# 4 threads --> 0.0005763880035374314
306+
# 8 threads --> 0.000879093015100807
307+
# 16 threads --> 0.0012480900040827692
308+
309+
# User Times
310+
# 1 Users --> 0.00022179502411745489
311+
# 2 Users --> 0.00035875398316420615
312+
# 4 Users --> 0.0004780120216310024
313+
314+
# Read/Write Times
315+
# Equal r/w --> 0.00028044500504620373
316+
# 2R/1W ------> 0.00048807798884809017
317+
# 1R/2W ------> 0.0004117719945497811
318+
319+
# Data Size Times
320+
# 0.00013885300722904503
321+
# 0.00012462001177482307
322+
# 0.0001767530047800392
323+
324+
# Prefix Scan Times
325+
# Prefix Exists --------> 1.6236997907981277e-05
326+
# Prefix Doesnt exist --> 9.587995009496808e-06
327+
328+
# Data Compression Times
329+
# With compression -----> 0.0006870539800729603
330+
# Without compression --> 1.235899981111288e-05
331+

0 commit comments

Comments
 (0)