From b2f939ee8f0e45f54ab31385ad2e73bca5884ba4 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 00:21:30 +0530
Subject: [PATCH 001/129] streamlit_app.py
---
streamlit_app.py | 19 ++++++++++++-------
1 file changed, 12 insertions(+), 7 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index dc94f5a..ed4302e 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -273,12 +273,14 @@ def calculate_rouge_scores(answer,context):
ideal_answer=st.text_area(label="Give your ideal answer instead",value="")
qar=[]
qar.append([query,ans,time,score,ideal_answer,rouge_scores])
- df=pd.DataFrame(qar)
+ file_name=pd.DataFrame(qar)
bucket = 'aiex' # already created on S3
csv_buffer = StringIO()
- df.to_csv(csv_buffer)
- s3_resource = boto3.resource('s3',aws_access_key_id=os.environ["ACCESS_ID"],aws_secret_access_key= os.environ["ACCESS_KEY"])
- s3_resource.Object(bucket, 'df.csv').put(Body=csv_buffer.getvalue())
+ file_name.to_csv(csv_buffer)
+ timestr = time.strftime("%Y%m%d-%H%M%S")
+ file_name="df "+timestr+ ".csv"
+ s3_resource= boto3.resource('s3',aws_access_key_id=os.environ["ACCESS_ID"],aws_secret_access_key= os.environ["ACCESS_KEY"])
+ s3_resource.Object(bucket,file_name).put(Body=csv_buffer.getvalue())
if (ans=='I don\'t know.' or ans=='I don\'t know'):
ans = chatbot(query,db)
message = {"role": "assistant", "content": ans}
@@ -297,12 +299,14 @@ def calculate_rouge_scores(answer,context):
ideal_answer=st.text_area(label="Give your ideal answer instead",value="")
qar=[]
qar.append([query,ans,time,score,ideal_answer,rouge_scores])
- df=pd.DataFrame(qar)
+ file_name=pd.DataFrame(qar)
bucket = 'aiex' # already created on S3
csv_buffer = StringIO()
- df.to_csv(csv_buffer)
+ file_name.to_csv(csv_buffer)
+ timestr = time.strftime("%Y%m%d-%H%M%S")
+ file_name="df "+timestr+ ".csv"
s3_resource= boto3.resource('s3',aws_access_key_id=os.environ["ACCESS_ID"],aws_secret_access_key= os.environ["ACCESS_KEY"])
- s3_resource.Object(bucket, 'df.csv').put(Body=csv_buffer.getvalue())
+ s3_resource.Object(bucket,file_name).put(Body=csv_buffer.getvalue())
#Generate a slider that takes input from 0 to 5 and asks for an ideal_answer
@@ -366,3 +370,4 @@ def footer():
footer()
+
From aad86a043439562d550c705f723b68f2c9e7a9b5 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 00:28:37 +0530
Subject: [PATCH 002/129] streamlit_app.py
---
streamlit_app.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index ed4302e..be0c4f1 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -292,11 +292,11 @@ def calculate_rouge_scores(answer,context):
ans = chatbot(query,db)
message = {"role": "assistant", "content": ans}
rouge_scores=calculate_rouge_scores(ans,context)
- score = st.slider("Select the creativity level for this answer:", min_value=0.0,max_value=5.0,value=2.5,step=0.5)
+ score = st.slider("Rate the answer on scale of 5, 5:excellent,1:bad", min_value=0.0,max_value=5.0,value=2.5,step=0.5)
#key=f"slider-{st.session_state['query_counter']}")
st.write("Liker score is: ",score)
#st.write(context)
- ideal_answer=st.text_area(label="Give your ideal answer instead",value="")
+ ideal_answer=st.text_area(label="Give your ideal answer:Enter the reference source to actual answer",value="")
qar=[]
qar.append([query,ans,time,score,ideal_answer,rouge_scores])
file_name=pd.DataFrame(qar)
From a3074a31cc41f1689a84746c191ad63fdd9a41c8 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 00:35:29 +0530
Subject: [PATCH 003/129] streamlit_app.py
---
streamlit_app.py | 52 +++++++++++++++++++-----------------------------
1 file changed, 20 insertions(+), 32 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index be0c4f1..6d23fed 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -265,22 +265,7 @@ def calculate_rouge_scores(answer,context):
with st.spinner("Thinking..."):
if len(context) < 2000:
ans, context, keys = chatbot_slim(query, context, keywords)
- rouge_scores=calculate_rouge_scores(ans,context)
- score = st.slider("Select the creativity level for this answer:", min_value=0.0,max_value=5.0,value=2.5,step=0.5)
- #key=f"slider-{st.session_state['query_counter']}")
- st.write("Liker score is: ",score)
- #st.write(context)
- ideal_answer=st.text_area(label="Give your ideal answer instead",value="")
- qar=[]
- qar.append([query,ans,time,score,ideal_answer,rouge_scores])
- file_name=pd.DataFrame(qar)
- bucket = 'aiex' # already created on S3
- csv_buffer = StringIO()
- file_name.to_csv(csv_buffer)
- timestr = time.strftime("%Y%m%d-%H%M%S")
- file_name="df "+timestr+ ".csv"
- s3_resource= boto3.resource('s3',aws_access_key_id=os.environ["ACCESS_ID"],aws_secret_access_key= os.environ["ACCESS_KEY"])
- s3_resource.Object(bucket,file_name).put(Body=csv_buffer.getvalue())
+
if (ans=='I don\'t know.' or ans=='I don\'t know'):
ans = chatbot(query,db)
message = {"role": "assistant", "content": ans}
@@ -291,25 +276,28 @@ def calculate_rouge_scores(answer,context):
else:
ans = chatbot(query,db)
message = {"role": "assistant", "content": ans}
- rouge_scores=calculate_rouge_scores(ans,context)
- score = st.slider("Rate the answer on scale of 5, 5:excellent,1:bad", min_value=0.0,max_value=5.0,value=2.5,step=0.5)
- #key=f"slider-{st.session_state['query_counter']}")
- st.write("Liker score is: ",score)
- #st.write(context)
- ideal_answer=st.text_area(label="Give your ideal answer:Enter the reference source to actual answer",value="")
- qar=[]
- qar.append([query,ans,time,score,ideal_answer,rouge_scores])
- file_name=pd.DataFrame(qar)
- bucket = 'aiex' # already created on S3
- csv_buffer = StringIO()
- file_name.to_csv(csv_buffer)
- timestr = time.strftime("%Y%m%d-%H%M%S")
- file_name="df "+timestr+ ".csv"
- s3_resource= boto3.resource('s3',aws_access_key_id=os.environ["ACCESS_ID"],aws_secret_access_key= os.environ["ACCESS_KEY"])
- s3_resource.Object(bucket,file_name).put(Body=csv_buffer.getvalue())
+
#Generate a slider that takes input from 0 to 5 and asks for an ideal_answer
+ with st.chat_message("assistant"):
+ rouge_scores=calculate_rouge_scores(ans,context)
+ score = st.slider("Rate the answer on scale of 5, 5:excellent,1:bad", min_value=0.0,max_value=5.0,value=2.5,step=0.5)
+ #key=f"slider-{st.session_state['query_counter']}")
+ st.write("Liker score is: ",score)
+ #st.write(context)
+ ideal_answer=st.text_area(label="Give your ideal answer:Enter the reference source to actual answer",value="")
+ qar=[]
+ qar.append([query,ans,time,score,ideal_answer,rouge_scores])
+ file_name=pd.DataFrame(qar)
+ bucket = 'aiex' # already created on S3
+ csv_buffer = StringIO()
+ file_name.to_csv(csv_buffer)
+ timestr = time.strftime("%Y%m%d-%H%M%S")
+ file_name="df "+timestr+ ".csv"
+ s3_resource= boto3.resource('s3',aws_access_key_id=os.environ["ACCESS_ID"],aws_secret_access_key= os.environ["ACCESS_KEY"])
+ s3_resource.Object(bucket,file_name).put(Body=csv_buffer.getvalue())
+
From 6ed407acf0fcbcbb95df6760e46147aca1df4895 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 00:37:02 +0530
Subject: [PATCH 004/129] streamlit_app.py
---
streamlit_app.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 6d23fed..4113ba8 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -294,9 +294,9 @@ def calculate_rouge_scores(answer,context):
csv_buffer = StringIO()
file_name.to_csv(csv_buffer)
timestr = time.strftime("%Y%m%d-%H%M%S")
- file_name="df "+timestr+ ".csv"
- s3_resource= boto3.resource('s3',aws_access_key_id=os.environ["ACCESS_ID"],aws_secret_access_key= os.environ["ACCESS_KEY"])
- s3_resource.Object(bucket,file_name).put(Body=csv_buffer.getvalue())
+ file_name="df "+timestr+ ".csv"
+ s3_resource= boto3.resource('s3',aws_access_key_id=os.environ["ACCESS_ID"],aws_secret_access_key= os.environ["ACCESS_KEY"])
+ s3_resource.Object(bucket,file_name).put(Body=csv_buffer.getvalue())
From ae66529b1a7cbd199fca35d4171699cb3bb8057e Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 00:38:05 +0530
Subject: [PATCH 005/129] streamlit_app.py
---
streamlit_app.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 4113ba8..ecdff79 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -281,7 +281,7 @@ def calculate_rouge_scores(answer,context):
#Generate a slider that takes input from 0 to 5 and asks for an ideal_answer
with st.chat_message("assistant"):
- rouge_scores=calculate_rouge_scores(ans,context)
+ rouge_scores=calculate_rouge_scores(ans,context)
score = st.slider("Rate the answer on scale of 5, 5:excellent,1:bad", min_value=0.0,max_value=5.0,value=2.5,step=0.5)
#key=f"slider-{st.session_state['query_counter']}")
st.write("Liker score is: ",score)
@@ -298,6 +298,8 @@ def calculate_rouge_scores(answer,context):
s3_resource= boto3.resource('s3',aws_access_key_id=os.environ["ACCESS_ID"],aws_secret_access_key= os.environ["ACCESS_KEY"])
s3_resource.Object(bucket,file_name).put(Body=csv_buffer.getvalue())
+
+
From 5cbcffdbb104698e861a372df0dc43d28f12e33b Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 00:39:17 +0530
Subject: [PATCH 006/129] streamlit_app.py
---
streamlit_app.py | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index ecdff79..734d158 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -282,21 +282,21 @@ def calculate_rouge_scores(answer,context):
#Generate a slider that takes input from 0 to 5 and asks for an ideal_answer
with st.chat_message("assistant"):
rouge_scores=calculate_rouge_scores(ans,context)
- score = st.slider("Rate the answer on scale of 5, 5:excellent,1:bad", min_value=0.0,max_value=5.0,value=2.5,step=0.5)
+ score = st.slider("Rate the answer on scale of 5, 5:excellent,1:bad", min_value=0.0,max_value=5.0,value=2.5,step=0.5)
#key=f"slider-{st.session_state['query_counter']}")
- st.write("Liker score is: ",score)
+ st.write("Liker score is: ",score)
#st.write(context)
- ideal_answer=st.text_area(label="Give your ideal answer:Enter the reference source to actual answer",value="")
- qar=[]
- qar.append([query,ans,time,score,ideal_answer,rouge_scores])
- file_name=pd.DataFrame(qar)
- bucket = 'aiex' # already created on S3
- csv_buffer = StringIO()
- file_name.to_csv(csv_buffer)
- timestr = time.strftime("%Y%m%d-%H%M%S")
- file_name="df "+timestr+ ".csv"
- s3_resource= boto3.resource('s3',aws_access_key_id=os.environ["ACCESS_ID"],aws_secret_access_key= os.environ["ACCESS_KEY"])
- s3_resource.Object(bucket,file_name).put(Body=csv_buffer.getvalue())
+ ideal_answer=st.text_area(label="Give your ideal answer:Enter the reference source to actual answer",value="")
+ qar=[]
+ qar.append([query,ans,time,score,ideal_answer,rouge_scores])
+ file_name=pd.DataFrame(qar)
+ bucket = 'aiex' # already created on S3
+ csv_buffer = StringIO()
+ file_name.to_csv(csv_buffer)
+ timestr = time.strftime("%Y%m%d-%H%M%S")
+ file_name="df "+timestr+ ".csv"
+ s3_resource= boto3.resource('s3',aws_access_key_id=os.environ["ACCESS_ID"],aws_secret_access_key= os.environ["ACCESS_KEY"])
+ s3_resource.Object(bucket,file_name).put(Body=csv_buffer.getvalue())
From fc627f3cea6004bc34dd351ffb1a0859f5e1995c Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 00:41:49 +0530
Subject: [PATCH 007/129] streamlit_app.py
---
streamlit_app.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 734d158..0dfd45e 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -282,11 +282,11 @@ def calculate_rouge_scores(answer,context):
#Generate a slider that takes input from 0 to 5 and asks for an ideal_answer
with st.chat_message("assistant"):
rouge_scores=calculate_rouge_scores(ans,context)
- score = st.slider("Rate the answer on scale of 5, 5:excellent,1:bad", min_value=0.0,max_value=5.0,value=2.5,step=0.5)
+ score = st.slider("Rate the answer on scale of 5, 5=excellent,1=bad", min_value=0.0,max_value=5.0,value=2.5,step=0.5)
#key=f"slider-{st.session_state['query_counter']}")
st.write("Liker score is: ",score)
#st.write(context)
- ideal_answer=st.text_area(label="Give your ideal answer:Enter the reference source to actual answer",value="")
+ ideal_answer=st.text_area(label="Give your ideal answer --> Enter the reference source to actual answer",value="")
qar=[]
qar.append([query,ans,time,score,ideal_answer,rouge_scores])
file_name=pd.DataFrame(qar)
From 5fab3a2804d31010f150c7fa9b2c43bc4de4c544 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 00:43:48 +0530
Subject: [PATCH 008/129] streamlit_app.py
---
streamlit_app.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 0dfd45e..3b36afd 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -284,7 +284,7 @@ def calculate_rouge_scores(answer,context):
rouge_scores=calculate_rouge_scores(ans,context)
score = st.slider("Rate the answer on scale of 5, 5=excellent,1=bad", min_value=0.0,max_value=5.0,value=2.5,step=0.5)
#key=f"slider-{st.session_state['query_counter']}")
- st.write("Liker score is: ",score)
+ st.write("Rating provided by user: ",score)
#st.write(context)
ideal_answer=st.text_area(label="Give your ideal answer --> Enter the reference source to actual answer",value="")
qar=[]
From 2a4b4fb9ebde9325db79e157dda7009674fa635c Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 01:30:02 +0530
Subject: [PATCH 009/129] streamlit_app.py
---
streamlit_app.py | 32 +++++++++++++++++++++++---------
1 file changed, 23 insertions(+), 9 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 3b36afd..3e9e56a 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -238,17 +238,22 @@ def calculate_rouge_scores(answer,context):
#slider_value = 2.5 # Default value for the slider
- with st.chat_message("user"):
+ # with st.chat_message("assistant"):
#query_audio_placeholder = st.empty()
#audio = audiorecorder("Click to record", "Click to stop recording")
#query_placeholder = st.empty()
- query_text = st.text_area(label="Let me know what you have in mind!")
- st.session_state.messages.append({"role": "user", "content": (query_text)})
- # Add a slider for each question
- # score = st.slider("Select the creativity level for this answer:", 0.0, 5.0, 2.5)
+ #query_text = st.text_area(label="Let me know what you have in mind!")
+ # st.session_state.messages.append({"role": "user", "content": (query_text)})
+ # Add a slider for each question
+ # score = st.slider("Select the creativity level for this answer:", 0.0, 5.0, 2.5)
#key=f"slider-{st.session_state['query_counter']}")
- # st.write("Liker score is: ",score)
+ # st.write("Liker score is: ",score)
#query_text = st.chat_input("Let me know what you have in mind")
+ with st.chat_input("Let me know what you have in mind!"):
+ query_text = st.text_area(label="Let me know what you have in mind!")
+ st.session_state.messages.append({"role": "user", "content": (query_text)})
+ st.markdown(query_text)
+
if query_text != "":# or not audio.empty() and not os.path.exists("query.wav"):
@@ -268,14 +273,23 @@ def calculate_rouge_scores(answer,context):
if (ans=='I don\'t know.' or ans=='I don\'t know'):
ans = chatbot(query,db)
- message = {"role": "assistant", "content": ans}
+ # message = {"role": "assistant", "content": ans}
+ st.session_state.messages.append({"role": "user", "content": ans})
+ with st.chat_message("assistant"):
+ st.markdown(ans)
else:
- message = {"role": "assistant", "content": ans}
+ #message = {"role": "assistant", "content": ans}
+ st.session_state.messages.append({"role": "user", "content": ans})
+ with st.chat_message("assistant"):
+ st.markdown(ans)
else:
ans = chatbot(query,db)
- message = {"role": "assistant", "content": ans}
+ #message = {"role": "assistant", "content": ans}
+ st.session_state.messages.append({"role": "user", "content": ans})
+ with st.chat_message("assistant"):
+ st.markdown(ans)
From 2dbc6be04299be156a3da789c29fbdec54566632 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 01:34:56 +0530
Subject: [PATCH 010/129] streamlit_app.py
---
streamlit_app.py | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 3e9e56a..d1c6d89 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -249,10 +249,17 @@ def calculate_rouge_scores(answer,context):
#key=f"slider-{st.session_state['query_counter']}")
# st.write("Liker score is: ",score)
#query_text = st.chat_input("Let me know what you have in mind")
- with st.chat_input("Let me know what you have in mind!"):
- query_text = st.text_area(label="Let me know what you have in mind!")
- st.session_state.messages.append({"role": "user", "content": (query_text)})
- st.markdown(query_text)
+ if query_text := st.chat_input("Let me know what you have in mind!"):
+ # Add user message to chat history
+ st.session_state.messages.append({"role": "user", "content": query_text})
+ # Display user message in chat message container
+ with st.chat_message("user"):
+ st.markdown(query_text)
+
+ #query_text = st.chat_input("")
+ #query_text = st.text_area(label="Let me know what you have in mind!")
+ # st.session_state.messages.append({"role": "user", "content": (query_text)})
+ # st.markdown(query_text)
From 32dde65864d3b7cf719700b8c750cfc9ae2acdb7 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 01:36:39 +0530
Subject: [PATCH 011/129] streamlit_app.py
---
streamlit_app.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index d1c6d89..b72caaa 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -276,10 +276,10 @@ def calculate_rouge_scores(answer,context):
with st.chat_message("assistant"):
with st.spinner("Thinking..."):
if len(context) < 2000:
- ans, context, keys = chatbot_slim(query, context, keywords)
+ ans, context, keys = chatbot_slim(str(query), context, keywords)
if (ans=='I don\'t know.' or ans=='I don\'t know'):
- ans = chatbot(query,db)
+ ans = chatbot(str(query),db)
# message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
with st.chat_message("assistant"):
@@ -292,7 +292,7 @@ def calculate_rouge_scores(answer,context):
st.markdown(ans)
else:
- ans = chatbot(query,db)
+ ans = chatbot(str(query),db)
#message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
with st.chat_message("assistant"):
From 9a4e1b1dfd03bde41f6bea22f4f8ecee7dc6dbaf Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 01:39:31 +0530
Subject: [PATCH 012/129] Update streamlit_app.py
---
streamlit_app.py | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index b72caaa..9b25018 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -282,21 +282,18 @@ def calculate_rouge_scores(answer,context):
ans = chatbot(str(query),db)
# message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
- with st.chat_message("assistant"):
- st.markdown(ans)
+ st.markdown(ans)
else:
#message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
- with st.chat_message("assistant"):
- st.markdown(ans)
+ st.markdown(ans)
else:
ans = chatbot(str(query),db)
#message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
- with st.chat_message("assistant"):
- st.markdown(ans)
+ st.markdown(ans)
From d0d0c5c559cd942bdcb335674218eceb7ee2b6a8 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 01:40:56 +0530
Subject: [PATCH 013/129] streamlit_app.py
---
streamlit_app.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 9b25018..654b470 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -335,7 +335,7 @@ def calculate_rouge_scores(answer,context):
mymidia_placeholder.empty()
time.sleep(1)
mymidia_placeholder.markdown(md, unsafe_allow_html=True)
- st.session_state.messages.append(message)
+ st.session_state.messages.append(ans)
st.session_state["query_status"] = False
st.session_state["text_input_status"] = False
st.session_state["audio_input_status"] = False
From b50a3d25ccfc72591944fdbebf7cb925a65be1f2 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 01:42:44 +0530
Subject: [PATCH 014/129] streamlit_app.py
---
streamlit_app.py | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 654b470..16593e9 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -280,20 +280,20 @@ def calculate_rouge_scores(answer,context):
if (ans=='I don\'t know.' or ans=='I don\'t know'):
ans = chatbot(str(query),db)
- # message = {"role": "assistant", "content": ans}
+ message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(ans)
+ st.markdown(message)
else:
- #message = {"role": "assistant", "content": ans}
+ message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(ans)
+ st.markdown(message)
else:
ans = chatbot(str(query),db)
- #message = {"role": "assistant", "content": ans}
+ message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(ans)
+ st.markdown(message)
@@ -335,7 +335,7 @@ def calculate_rouge_scores(answer,context):
mymidia_placeholder.empty()
time.sleep(1)
mymidia_placeholder.markdown(md, unsafe_allow_html=True)
- st.session_state.messages.append(ans)
+ st.session_state.messages.append(message)
st.session_state["query_status"] = False
st.session_state["text_input_status"] = False
st.session_state["audio_input_status"] = False
From 166fdb498d2804324509432c415f193d9079b26c Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 01:46:21 +0530
Subject: [PATCH 015/129] streamlit_app.py
---
streamlit_app.py | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 16593e9..969b9c5 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -276,24 +276,24 @@ def calculate_rouge_scores(answer,context):
with st.chat_message("assistant"):
with st.spinner("Thinking..."):
if len(context) < 2000:
- ans, context, keys = chatbot_slim(str(query), context, keywords)
+ ans, context, keys = chatbot_slim(query, context, keywords)
if (ans=='I don\'t know.' or ans=='I don\'t know'):
- ans = chatbot(str(query),db)
+ ans = chatbot(query,db)
message = {"role": "assistant", "content": ans}
- st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(message)
+ #st.session_state.messages.append({"role": "user", "content": ans})
+ # st.markdown(message)
else:
message = {"role": "assistant", "content": ans}
- st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(message)
+ # st.session_state.messages.append({"role": "user", "content": ans})
+ # st.markdown(message)
else:
- ans = chatbot(str(query),db)
+ ans = chatbot(query,db)
message = {"role": "assistant", "content": ans}
- st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(message)
+ # st.session_state.messages.append({"role": "user", "content": ans})
+ # st.markdown(message)
@@ -345,7 +345,7 @@ def calculate_rouge_scores(answer,context):
# ------------------------------------------------------------------------------#
if st.session_state.messages != []:
- for message in st.session_state.messages[::-1]:
+ for message in st.session_state.messages[::1]:
with st.chat_message(message["role"]):
st.write(message["content"])
From ce2a92b1c7b41080631a706733d0699e97734ecf Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 01:47:15 +0530
Subject: [PATCH 016/129] streamlit_app.py
---
streamlit_app.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 969b9c5..03ad17a 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -276,10 +276,10 @@ def calculate_rouge_scores(answer,context):
with st.chat_message("assistant"):
with st.spinner("Thinking..."):
if len(context) < 2000:
- ans, context, keys = chatbot_slim(query, context, keywords)
+ ans, context, keys = chatbot_slim(str(query), context, keywords)
if (ans=='I don\'t know.' or ans=='I don\'t know'):
- ans = chatbot(query,db)
+ ans = chatbot(str(query),db)
message = {"role": "assistant", "content": ans}
#st.session_state.messages.append({"role": "user", "content": ans})
# st.markdown(message)
@@ -290,7 +290,7 @@ def calculate_rouge_scores(answer,context):
# st.markdown(message)
else:
- ans = chatbot(query,db)
+ ans = chatbot(str(query),db)
message = {"role": "assistant", "content": ans}
# st.session_state.messages.append({"role": "user", "content": ans})
# st.markdown(message)
From cab698ebddfdf53293270bd3e537bc2dcd9e40c0 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 01:54:45 +0530
Subject: [PATCH 017/129] streamlit_app.py
---
streamlit_app.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 03ad17a..4acbd55 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -346,8 +346,8 @@ def calculate_rouge_scores(answer,context):
if st.session_state.messages != []:
for message in st.session_state.messages[::1]:
- with st.chat_message(message["role"]):
- st.write(message["content"])
+ with st.chat_message(message.role):
+ st.write(message.content)
From 4c98cba8821473e505bd0135d3075efcb9bbf1ab Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 02:08:36 +0530
Subject: [PATCH 018/129] streamlit_app.py
---
streamlit_app.py | 33 ++++++++++++++++++---------------
1 file changed, 18 insertions(+), 15 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 4acbd55..cfa1cf1 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -263,7 +263,7 @@ def calculate_rouge_scores(answer,context):
- if query_text != "":# or not audio.empty() and not os.path.exists("query.wav"):
+ #if query_text != "":# or not audio.empty() and not os.path.exists("query.wav"):
if query_text != "":
st.session_state["query_status"] = True
st.session_state["text_input_status"] = True
@@ -280,20 +280,20 @@ def calculate_rouge_scores(answer,context):
if (ans=='I don\'t know.' or ans=='I don\'t know'):
ans = chatbot(str(query),db)
- message = {"role": "assistant", "content": ans}
- #st.session_state.messages.append({"role": "user", "content": ans})
- # st.markdown(message)
+ #message = {"role": "assistant", "content": ans}
+ st.session_state.messages.append({"role": "user", "content": ans})
+ st.markdown(message)
else:
- message = {"role": "assistant", "content": ans}
- # st.session_state.messages.append({"role": "user", "content": ans})
- # st.markdown(message)
+ #message = {"role": "assistant", "content": ans}
+ st.session_state.messages.append({"role": "user", "content": ans})
+ st.markdown(message)
else:
ans = chatbot(str(query),db)
- message = {"role": "assistant", "content": ans}
- # st.session_state.messages.append({"role": "user", "content": ans})
- # st.markdown(message)
+ #message = {"role": "assistant", "content": ans}
+ st.session_state.messages.append({"role": "user", "content": ans})
+ st.markdown(message)
@@ -335,19 +335,22 @@ def calculate_rouge_scores(answer,context):
mymidia_placeholder.empty()
time.sleep(1)
mymidia_placeholder.markdown(md, unsafe_allow_html=True)
- st.session_state.messages.append(message)
+ st.session_state.messages.append(ans)
+ with st.chat_message("user"):
+ st.markdown(ans)
st.session_state["query_status"] = False
st.session_state["text_input_status"] = False
st.session_state["audio_input_status"] = False
+
# ------------------------------------------------------------------------------#
# -------------------------QUERY AUDIO INPUT - RETURNING TEXT QUERY-------------#
# ------------------------------------------------------------------------------#
-if st.session_state.messages != []:
- for message in st.session_state.messages[::1]:
- with st.chat_message(message.role):
- st.write(message.content)
+#if st.session_state.messages != []:
+# for message in st.session_state.messages[::1]:
+# with st.chat_message(message.role):
+# st.write(message.content)
From f8476f7ed7681102350a4edb0400050fefba0cb1 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 02:10:08 +0530
Subject: [PATCH 019/129] streamlit_app.py
---
streamlit_app.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index cfa1cf1..1f1b819 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -293,7 +293,7 @@ def calculate_rouge_scores(answer,context):
ans = chatbot(str(query),db)
#message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(message)
+ st.markdown(ans)
From 138223be036f37b6912a8adb2784394971d03121 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 02:16:31 +0530
Subject: [PATCH 020/129] streamlit_app.py
---
streamlit_app.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 1f1b819..1de0e01 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -282,12 +282,12 @@ def calculate_rouge_scores(answer,context):
ans = chatbot(str(query),db)
#message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(message)
+ st.markdown(ans)
else:
#message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(message)
+ st.markdown(ans)
else:
ans = chatbot(str(query),db)
From 636a01fa58a22904d02aa46b685c06e77f7aade4 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 02:20:31 +0530
Subject: [PATCH 021/129] streamlit_app.py
---
streamlit_app.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 1de0e01..aadba5e 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -336,8 +336,8 @@ def calculate_rouge_scores(answer,context):
time.sleep(1)
mymidia_placeholder.markdown(md, unsafe_allow_html=True)
st.session_state.messages.append(ans)
- with st.chat_message("user"):
- st.markdown(ans)
+ #with st.chat_message("user"):
+ #st.markdown(ans)
st.session_state["query_status"] = False
st.session_state["text_input_status"] = False
st.session_state["audio_input_status"] = False
From 5202f437a5445e36c2c12214cb5a4e2f462d6653 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 02:40:20 +0530
Subject: [PATCH 022/129] streamlit_app.py
---
streamlit_app.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index aadba5e..81ef87d 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -231,7 +231,9 @@ def calculate_rouge_scores(answer,context):
if uploaded_file is not None and st.session_state["db_created"] == True:
st.title("Ask me anything about the document!")
-
+ for message in st.session_state.messages:
+ with st.chat_message(message["role"]):
+ st.markdown(message["content"])
# User-provided prompt
#if prompt := st.chat_input():
From eafc853a6481befbb03b3efab1487f391eb38b59 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 02:42:29 +0530
Subject: [PATCH 023/129] streamlit_app.py
---
streamlit_app.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 81ef87d..0effc1c 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -282,18 +282,18 @@ def calculate_rouge_scores(answer,context):
if (ans=='I don\'t know.' or ans=='I don\'t know'):
ans = chatbot(str(query),db)
- #message = {"role": "assistant", "content": ans}
+ message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
st.markdown(ans)
else:
- #message = {"role": "assistant", "content": ans}
+ message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
st.markdown(ans)
else:
ans = chatbot(str(query),db)
- #message = {"role": "assistant", "content": ans}
+ message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
st.markdown(ans)
@@ -337,7 +337,7 @@ def calculate_rouge_scores(answer,context):
mymidia_placeholder.empty()
time.sleep(1)
mymidia_placeholder.markdown(md, unsafe_allow_html=True)
- st.session_state.messages.append(ans)
+ st.session_state.messages.append(message)
#with st.chat_message("user"):
#st.markdown(ans)
st.session_state["query_status"] = False
From 03f6fd5ff0812e7631c25294b0f0a98b8ae5a205 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 02:48:33 +0530
Subject: [PATCH 024/129] streamlit_app.py
---
streamlit_app.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 0effc1c..668e7cb 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -284,18 +284,18 @@ def calculate_rouge_scores(answer,context):
ans = chatbot(str(query),db)
message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(ans)
+ st.markdown(message)
else:
message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(ans)
+ st.markdown(message)
else:
ans = chatbot(str(query),db)
message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(ans)
+ st.markdown(message)
From 61561e765592b93f5157a76c9305e11c274eb45d Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 02:55:09 +0530
Subject: [PATCH 025/129] streamlit_app.py
---
streamlit_app.py | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 668e7cb..a8bba02 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -282,18 +282,18 @@ def calculate_rouge_scores(answer,context):
if (ans=='I don\'t know.' or ans=='I don\'t know'):
ans = chatbot(str(query),db)
- message = {"role": "assistant", "content": ans}
+ #message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
st.markdown(message)
else:
- message = {"role": "assistant", "content": ans}
+ #message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
st.markdown(message)
else:
ans = chatbot(str(query),db)
- message = {"role": "assistant", "content": ans}
+ #message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
st.markdown(message)
@@ -337,13 +337,18 @@ def calculate_rouge_scores(answer,context):
mymidia_placeholder.empty()
time.sleep(1)
mymidia_placeholder.markdown(md, unsafe_allow_html=True)
- st.session_state.messages.append(message)
+
+ st.session_state.messages.append(ans)
#with st.chat_message("user"):
#st.markdown(ans)
st.session_state["query_status"] = False
st.session_state["text_input_status"] = False
st.session_state["audio_input_status"] = False
+ messages=[
+ {"role": m["role"], "content": m["content"]}
+ for m in st.session_state.messages
+ ]
# ------------------------------------------------------------------------------#
# -------------------------QUERY AUDIO INPUT - RETURNING TEXT QUERY-------------#
From 61dcafeab443fee40763bf7a6fba40eb5b862310 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 03:02:46 +0530
Subject: [PATCH 026/129] streamlit_app.py
---
streamlit_app.py | 17 ++++++-----------
1 file changed, 6 insertions(+), 11 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index a8bba02..57232ab 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -231,9 +231,9 @@ def calculate_rouge_scores(answer,context):
if uploaded_file is not None and st.session_state["db_created"] == True:
st.title("Ask me anything about the document!")
- for message in st.session_state.messages:
- with st.chat_message(message["role"]):
- st.markdown(message["content"])
+ #for message in st.session_state.messages:
+ # with st.chat_message(message["role"]):
+ # st.markdown(message["content"])
# User-provided prompt
#if prompt := st.chat_input():
@@ -284,18 +284,18 @@ def calculate_rouge_scores(answer,context):
ans = chatbot(str(query),db)
#message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(message)
+ st.markdown(ans)
else:
#message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(message)
+ st.markdown(ans)
else:
ans = chatbot(str(query),db)
#message = {"role": "assistant", "content": ans}
st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(message)
+ st.markdown(ans)
@@ -344,11 +344,6 @@ def calculate_rouge_scores(answer,context):
st.session_state["query_status"] = False
st.session_state["text_input_status"] = False
st.session_state["audio_input_status"] = False
-
- messages=[
- {"role": m["role"], "content": m["content"]}
- for m in st.session_state.messages
- ]
# ------------------------------------------------------------------------------#
# -------------------------QUERY AUDIO INPUT - RETURNING TEXT QUERY-------------#
From b7dea14b8ab4f679f21bd0a8544b786a698b1b6d Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 03:21:22 +0530
Subject: [PATCH 027/129] streamlit_app.py
---
streamlit_app.py | 202 ++++++++---------------------------------------
1 file changed, 32 insertions(+), 170 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 57232ab..4043ae1 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -64,7 +64,7 @@ def image(src_as_string, **style):
def link(link, text, **style):
- return a(_href=link, _target="_blank", style=styles(**style))(text)
+ return a(_href=link, _target="_blank", style=styles(**style))
def layout(*args):
@@ -166,8 +166,14 @@ def calculate_rouge_scores(answer,context):
# -----------------UPLOAD THE SRC DOCUMENT-----------------#
# ---------------------------------------------------------#
st.title("Please let me know what you want to talk about by choosing a file below!")
+
+# Initialize session state variables
if "uploaded_status" not in st.session_state:
st.session_state["uploaded_status"] = False
+if "query_counter" not in st.session_state:
+ st.session_state["query_counter"] = 0
+if "messages" not in st.session_state:
+ st.session_state.messages = []
uploaded_file = st.file_uploader(label = "")
if st.session_state["uploaded_status"] == False and uploaded_file is not None:
@@ -175,36 +181,16 @@ def calculate_rouge_scores(answer,context):
readdoc_splittext.clear()
readdoc_splittext_pdf.clear()
-if "query_counter" not in st.session_state:
- st.session_state["query_counter"] = 0
-if "query_status" not in st.session_state:
- st.session_state["query_status"] = False
-if "audio_input_status" not in st.session_state:
- st.session_state["audio_input_status"] = False
-if "text_input_status" not in st.session_state:
- st.session_state["text_input_status"] = False
-
-if "db_created" not in st.session_state:
- st.session_state["db_created"] = False
-
if (uploaded_file is not None):
st.session_state["uploaded_status"] = True
elif uploaded_file is None:
st.session_state["uploaded_status"] = False
st.session_state["query_counter"] = 0
- st.session_state["db_created"] = False
- st.session_state["text_input_status"] = False
- st.session_state["query_status"] = False
- st.session_state["audio_input_status"] = False
- st.write("Dear user, clearing unnecesary data fo you to start afresh!!")
create_db.clear()
readdoc_splittext.clear()
readdoc_splittext_pdf.clear()
- st.write("You can upload your document now!")
-
+ st.write("You can upload your document now.")
-if "messages" not in st.session_state.keys():
- st.session_state.messages = []
if (uploaded_file is not None):
file_path = os.path.join( os.getcwd(), uploaded_file.name)
@@ -231,156 +217,32 @@ def calculate_rouge_scores(answer,context):
if uploaded_file is not None and st.session_state["db_created"] == True:
st.title("Ask me anything about the document!")
- #for message in st.session_state.messages:
- # with st.chat_message(message["role"]):
- # st.markdown(message["content"])
-
- # User-provided prompt
- #if prompt := st.chat_input():
-
- #slider_value = 2.5 # Default value for the slider
-
- # with st.chat_message("assistant"):
- #query_audio_placeholder = st.empty()
- #audio = audiorecorder("Click to record", "Click to stop recording")
- #query_placeholder = st.empty()
- #query_text = st.text_area(label="Let me know what you have in mind!")
- # st.session_state.messages.append({"role": "user", "content": (query_text)})
- # Add a slider for each question
- # score = st.slider("Select the creativity level for this answer:", 0.0, 5.0, 2.5)
- #key=f"slider-{st.session_state['query_counter']}")
- # st.write("Liker score is: ",score)
- #query_text = st.chat_input("Let me know what you have in mind")
- if query_text := st.chat_input("Let me know what you have in mind!"):
- # Add user message to chat history
+
+ # Display the chat input box
+ query_text = st.chat_input("Let me know what you have in mind!")
+
+ # Check if the user has entered a query
+ if query_text != "":
+ # Add the user message to the messages list
st.session_state.messages.append({"role": "user", "content": query_text})
- # Display user message in chat message container
+
+ # Display the user message in the chat message container
with st.chat_message("user"):
st.markdown(query_text)
-
- #query_text = st.chat_input("")
- #query_text = st.text_area(label="Let me know what you have in mind!")
- # st.session_state.messages.append({"role": "user", "content": (query_text)})
- # st.markdown(query_text)
-
-
-
- #if query_text != "":# or not audio.empty() and not os.path.exists("query.wav"):
- if query_text != "":
- st.session_state["query_status"] = True
- st.session_state["text_input_status"] = True
- st.session_state["query_counter"] += 1
- query = query_text
- context, keywords = create_context(query, text_split, headings, para_texts)
-
-
- # Generate a new response if the last message is not from the assistant
- with st.chat_message("assistant"):
- with st.spinner("Thinking..."):
- if len(context) < 2000:
- ans, context, keys = chatbot_slim(str(query), context, keywords)
-
- if (ans=='I don\'t know.' or ans=='I don\'t know'):
- ans = chatbot(str(query),db)
- #message = {"role": "assistant", "content": ans}
- st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(ans)
-
- else:
- #message = {"role": "assistant", "content": ans}
- st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(ans)
-
- else:
- ans = chatbot(str(query),db)
- #message = {"role": "assistant", "content": ans}
- st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(ans)
-
-
-
- #Generate a slider that takes input from 0 to 5 and asks for an ideal_answer
+
+ # Generate a response from the chatbot
with st.chat_message("assistant"):
- rouge_scores=calculate_rouge_scores(ans,context)
- score = st.slider("Rate the answer on scale of 5, 5=excellent,1=bad", min_value=0.0,max_value=5.0,value=2.5,step=0.5)
- #key=f"slider-{st.session_state['query_counter']}")
- st.write("Rating provided by user: ",score)
- #st.write(context)
- ideal_answer=st.text_area(label="Give your ideal answer --> Enter the reference source to actual answer",value="")
- qar=[]
- qar.append([query,ans,time,score,ideal_answer,rouge_scores])
- file_name=pd.DataFrame(qar)
- bucket = 'aiex' # already created on S3
- csv_buffer = StringIO()
- file_name.to_csv(csv_buffer)
- timestr = time.strftime("%Y%m%d-%H%M%S")
- file_name="df "+timestr+ ".csv"
- s3_resource= boto3.resource('s3',aws_access_key_id=os.environ["ACCESS_ID"],aws_secret_access_key= os.environ["ACCESS_KEY"])
- s3_resource.Object(bucket,file_name).put(Body=csv_buffer.getvalue())
-
-
-
-
-
-
- # -----------text to speech--------------------------#
- texttospeech_raw(ans, language="en")
- mymidia_placeholder = st.empty()
- with open("answer.wav", "rb") as audio_file:
- audio_bytes = audio_file.read()
- b64 = base64.b64encode(audio_bytes).decode()
- md = f"""
-
- """
- mymidia_placeholder.empty()
- time.sleep(1)
- mymidia_placeholder.markdown(md, unsafe_allow_html=True)
-
- st.session_state.messages.append(ans)
- #with st.chat_message("user"):
- #st.markdown(ans)
- st.session_state["query_status"] = False
- st.session_state["text_input_status"] = False
- st.session_state["audio_input_status"] = False
-
-# ------------------------------------------------------------------------------#
-# -------------------------QUERY AUDIO INPUT - RETURNING TEXT QUERY-------------#
-# ------------------------------------------------------------------------------#
-
-#if st.session_state.messages != []:
-# for message in st.session_state.messages[::1]:
-# with st.chat_message(message.role):
-# st.write(message.content)
-
-
-
-myargs = [
- "Made in India",
- "" " with ❤️ by ",
- link("https://www.linkedin.com/in/anupamisb/", "@Anupam"),
- br(),
- link("https://anupam-purwar.github.io/page/", "SpeeKAR ChatBoT"),
- br(),
- link("https://www.linkedin.com/in/rahul-sundar-311a6977/", "@Rahul"),
- br(),
- link("https://github.com/RahulSundar", "SpeeKAR ChatBoT"),
-]
-
-
-def footer():
- myargs = [
- "Made in India",
- "" " with ❤️ by ",
- link("https://www.linkedin.com/in/anupamisb/", " Anupam for "),
- link("https://anupam-purwar.github.io/page/", "SpeeKAR ChatBoT"),
- ", and",
- link("https://www.linkedin.com/in/rahul-sundar-311a6977/", "@Rahul"),
- link("https://github.com/RahulSundar", "SpeeKAR ChatBoT"),
- ]
- layout(*myargs)
-
-
-footer()
+ # Your existing code to generate a response from the chatbot
+ # ...
+
+ # Add the assistant's response to the messages list
+ st.session_state.messages.append({"role": "assistant", "content": ans})
+
+ # Display the assistant's response in the chat message container
+ st.markdown(ans)
+ # At the end of the script
+ # Loop through all messages and display them
+ for message in st.session_state.messages:
+ with st.chat_message(message["role"]):
+ st.markdown(message["content"])
From 400249a03a8ea05d5bb7080acca88aa3c9da4d0e Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Wed, 7 Feb 2024 03:24:32 +0530
Subject: [PATCH 028/129] streamlit_app.py
---
streamlit_app.py | 203 +++++++++++++++++++++++++++++++++++++++--------
1 file changed, 170 insertions(+), 33 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 4043ae1..8a45eee 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -64,7 +64,7 @@ def image(src_as_string, **style):
def link(link, text, **style):
- return a(_href=link, _target="_blank", style=styles(**style))
+ return a(_href=link, _target="_blank", style=styles(**style))(text)
def layout(*args):
@@ -166,14 +166,8 @@ def calculate_rouge_scores(answer,context):
# -----------------UPLOAD THE SRC DOCUMENT-----------------#
# ---------------------------------------------------------#
st.title("Please let me know what you want to talk about by choosing a file below!")
-
-# Initialize session state variables
if "uploaded_status" not in st.session_state:
st.session_state["uploaded_status"] = False
-if "query_counter" not in st.session_state:
- st.session_state["query_counter"] = 0
-if "messages" not in st.session_state:
- st.session_state.messages = []
uploaded_file = st.file_uploader(label = "")
if st.session_state["uploaded_status"] == False and uploaded_file is not None:
@@ -181,16 +175,36 @@ def calculate_rouge_scores(answer,context):
readdoc_splittext.clear()
readdoc_splittext_pdf.clear()
+if "query_counter" not in st.session_state:
+ st.session_state["query_counter"] = 0
+if "query_status" not in st.session_state:
+ st.session_state["query_status"] = False
+if "audio_input_status" not in st.session_state:
+ st.session_state["audio_input_status"] = False
+if "text_input_status" not in st.session_state:
+ st.session_state["text_input_status"] = False
+
+if "db_created" not in st.session_state:
+ st.session_state["db_created"] = False
+
if (uploaded_file is not None):
st.session_state["uploaded_status"] = True
elif uploaded_file is None:
st.session_state["uploaded_status"] = False
st.session_state["query_counter"] = 0
+ st.session_state["db_created"] = False
+ st.session_state["text_input_status"] = False
+ st.session_state["query_status"] = False
+ st.session_state["audio_input_status"] = False
+ st.write("Dear user, clearing unnecesary data fo you to start afresh!!")
create_db.clear()
readdoc_splittext.clear()
readdoc_splittext_pdf.clear()
- st.write("You can upload your document now.")
+ st.write("You can upload your document now!")
+
+if "messages" not in st.session_state.keys():
+ st.session_state.messages = []
if (uploaded_file is not None):
file_path = os.path.join( os.getcwd(), uploaded_file.name)
@@ -217,32 +231,155 @@ def calculate_rouge_scores(answer,context):
if uploaded_file is not None and st.session_state["db_created"] == True:
st.title("Ask me anything about the document!")
-
- # Display the chat input box
- query_text = st.chat_input("Let me know what you have in mind!")
-
- # Check if the user has entered a query
- if query_text != "":
- # Add the user message to the messages list
+ #for message in st.session_state.messages:
+ # with st.chat_message(message["role"]):
+ # st.markdown(message["content"])
+
+ # User-provided prompt
+ #if prompt := st.chat_input():
+
+ #slider_value = 2.5 # Default value for the slider
+
+ # with st.chat_message("assistant"):
+ #query_audio_placeholder = st.empty()
+ #audio = audiorecorder("Click to record", "Click to stop recording")
+ #query_placeholder = st.empty()
+ #query_text = st.text_area(label="Let me know what you have in mind!")
+ # st.session_state.messages.append({"role": "user", "content": (query_text)})
+ # Add a slider for each question
+ # score = st.slider("Select the creativity level for this answer:", 0.0, 5.0, 2.5)
+ #key=f"slider-{st.session_state['query_counter']}")
+ # st.write("Liker score is: ",score)
+ #query_text = st.chat_input("Let me know what you have in mind")
+ if query_text := st.chat_input("Let me know what you have in mind!"):
+ # Add user message to chat history
st.session_state.messages.append({"role": "user", "content": query_text})
-
- # Display the user message in the chat message container
+ # Display user message in chat message container
with st.chat_message("user"):
st.markdown(query_text)
-
- # Generate a response from the chatbot
+
+ #query_text = st.chat_input("")
+ #query_text = st.text_area(label="Let me know what you have in mind!")
+ # st.session_state.messages.append({"role": "user", "content": (query_text)})
+ # st.markdown(query_text)
+
+
+
+ #if query_text != "":# or not audio.empty() and not os.path.exists("query.wav"):
+ if query_text != "":
+ st.session_state["query_status"] = True
+ st.session_state["text_input_status"] = True
+ st.session_state["query_counter"] += 1
+ query = query_text
+ context, keywords = create_context(query, text_split, headings, para_texts)
+
+
+ # Generate a new response if the last message is not from the assistant
with st.chat_message("assistant"):
- # Your existing code to generate a response from the chatbot
- # ...
-
- # Add the assistant's response to the messages list
- st.session_state.messages.append({"role": "assistant", "content": ans})
-
- # Display the assistant's response in the chat message container
- st.markdown(ans)
-
- # At the end of the script
- # Loop through all messages and display them
- for message in st.session_state.messages:
- with st.chat_message(message["role"]):
- st.markdown(message["content"])
+ with st.spinner("Thinking..."):
+ if len(context) < 2000:
+ ans, context, keys = chatbot_slim(str(query), context, keywords)
+
+ if (ans=='I don\'t know.' or ans=='I don\'t know'):
+ ans = chatbot(str(query),db)
+ #message = {"role": "assistant", "content": ans}
+ st.session_state.messages.append({"role": "user", "content": ans})
+ st.markdown(ans)
+
+ else:
+ #message = {"role": "assistant", "content": ans}
+ st.session_state.messages.append({"role": "user", "content": ans})
+ st.markdown(ans)
+
+ else:
+ ans = chatbot(str(query),db)
+ #message = {"role": "assistant", "content": ans}
+ st.session_state.messages.append({"role": "user", "content": ans})
+ st.markdown(ans)
+
+
+
+ #Generate a slider that takes input from 0 to 5 and asks for an ideal_answer
+ with st.chat_message("assistant"):
+ rouge_scores=calculate_rouge_scores(ans,context)
+ score = st.slider("Rate the answer on scale of 5, 5=excellent,1=bad", min_value=0.0,max_value=5.0,value=2.5,step=0.5)
+ #key=f"slider-{st.session_state['query_counter']}")
+ st.write("Rating provided by user: ",score)
+ #st.write(context)
+ ideal_answer=st.text_area(label="Give your ideal answer --> Enter the reference source to actual answer",value="")
+ qar=[]
+ qar.append([query,ans,time,score,ideal_answer,rouge_scores])
+ file_name=pd.DataFrame(qar)
+ bucket = 'aiex' # already created on S3
+ csv_buffer = StringIO()
+ file_name.to_csv(csv_buffer)
+ timestr = time.strftime("%Y%m%d-%H%M%S")
+ file_name="df "+timestr+ ".csv"
+ s3_resource= boto3.resource('s3',aws_access_key_id=os.environ["ACCESS_ID"],aws_secret_access_key= os.environ["ACCESS_KEY"])
+ s3_resource.Object(bucket,file_name).put(Body=csv_buffer.getvalue())
+
+
+
+
+
+
+ # -----------text to speech--------------------------#
+ texttospeech_raw(ans, language="en")
+ mymidia_placeholder = st.empty()
+ with open("answer.wav", "rb") as audio_file:
+ audio_bytes = audio_file.read()
+ b64 = base64.b64encode(audio_bytes).decode()
+ md = f"""
+
+ """
+ mymidia_placeholder.empty()
+ time.sleep(1)
+ mymidia_placeholder.markdown(md, unsafe_allow_html=True)
+
+ st.session_state.messages.append(ans)
+ #with st.chat_message("user"):
+ #st.markdown(ans)
+ st.session_state["query_status"] = False
+ st.session_state["text_input_status"] = False
+ st.session_state["audio_input_status"] = False
+
+# ------------------------------------------------------------------------------#
+# -------------------------QUERY AUDIO INPUT - RETURNING TEXT QUERY-------------#
+# ------------------------------------------------------------------------------#
+
+#if st.session_state.messages != []:
+# for message in st.session_state.messages[::1]:
+# with st.chat_message(message.role):
+# st.write(message.content)
+
+
+
+myargs = [
+ "Made in India",
+ "" " with ❤️ by ",
+ link("https://www.linkedin.com/in/anupamisb/", "@Anupam"),
+ br(),
+ link("https://anupam-purwar.github.io/page/", "SpeeKAR ChatBoT"),
+ br(),
+ link("https://www.linkedin.com/in/rahul-sundar-311a6977/", "@Rahul"),
+ br(),
+ link("https://github.com/RahulSundar", "SpeeKAR ChatBoT"),
+]
+
+
+def footer():
+ myargs = [
+ "Made in India",
+ "" " with ❤️ by ",
+ link("https://www.linkedin.com/in/anupamisb/", " Anupam for "),
+ link("https://anupam-purwar.github.io/page/", "SpeeKAR ChatBoT"),
+ ", and",
+ link("https://www.linkedin.com/in/rahul-sundar-311a6977/", "@Rahul"),
+ link("https://github.com/RahulSundar", "SpeeKAR ChatBoT"),
+ ]
+ layout(*myargs)
+
+
+footer()
From 3f294e2853926c16d64fed9e38223999a1ef3ce1 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 8 Feb 2024 18:31:00 +0530
Subject: [PATCH 029/129] streamlit_app.py
---
streamlit_app.py | 187 +++++++++--------------------------------------
1 file changed, 33 insertions(+), 154 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 8a45eee..cabbd53 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -64,7 +64,7 @@ def image(src_as_string, **style):
def link(link, text, **style):
- return a(_href=link, _target="_blank", style=styles(**style))(text)
+ return a(_href=link, _target="_blank", style=styles(**style))
def layout(*args):
@@ -166,8 +166,14 @@ def calculate_rouge_scores(answer,context):
# -----------------UPLOAD THE SRC DOCUMENT-----------------#
# ---------------------------------------------------------#
st.title("Please let me know what you want to talk about by choosing a file below!")
+
+# Initialize session state variables
if "uploaded_status" not in st.session_state:
st.session_state["uploaded_status"] = False
+if "query_counter" not in st.session_state:
+ st.session_state["query_counter"] = 0
+if "messages" not in st.session_state:
+ st.session_state.messages = []
uploaded_file = st.file_uploader(label = "")
if st.session_state["uploaded_status"] == False and uploaded_file is not None:
@@ -175,36 +181,16 @@ def calculate_rouge_scores(answer,context):
readdoc_splittext.clear()
readdoc_splittext_pdf.clear()
-if "query_counter" not in st.session_state:
- st.session_state["query_counter"] = 0
-if "query_status" not in st.session_state:
- st.session_state["query_status"] = False
-if "audio_input_status" not in st.session_state:
- st.session_state["audio_input_status"] = False
-if "text_input_status" not in st.session_state:
- st.session_state["text_input_status"] = False
-
-if "db_created" not in st.session_state:
- st.session_state["db_created"] = False
-
if (uploaded_file is not None):
st.session_state["uploaded_status"] = True
elif uploaded_file is None:
st.session_state["uploaded_status"] = False
st.session_state["query_counter"] = 0
- st.session_state["db_created"] = False
- st.session_state["text_input_status"] = False
- st.session_state["query_status"] = False
- st.session_state["audio_input_status"] = False
- st.write("Dear user, clearing unnecesary data fo you to start afresh!!")
create_db.clear()
readdoc_splittext.clear()
readdoc_splittext_pdf.clear()
- st.write("You can upload your document now!")
-
+ st.write("You can upload your document now.")
-if "messages" not in st.session_state.keys():
- st.session_state.messages = []
if (uploaded_file is not None):
file_path = os.path.join( os.getcwd(), uploaded_file.name)
@@ -231,144 +217,37 @@ def calculate_rouge_scores(answer,context):
if uploaded_file is not None and st.session_state["db_created"] == True:
st.title("Ask me anything about the document!")
- #for message in st.session_state.messages:
- # with st.chat_message(message["role"]):
- # st.markdown(message["content"])
-
- # User-provided prompt
- #if prompt := st.chat_input():
-
- #slider_value = 2.5 # Default value for the slider
-
- # with st.chat_message("assistant"):
- #query_audio_placeholder = st.empty()
- #audio = audiorecorder("Click to record", "Click to stop recording")
- #query_placeholder = st.empty()
- #query_text = st.text_area(label="Let me know what you have in mind!")
- # st.session_state.messages.append({"role": "user", "content": (query_text)})
- # Add a slider for each question
- # score = st.slider("Select the creativity level for this answer:", 0.0, 5.0, 2.5)
- #key=f"slider-{st.session_state['query_counter']}")
- # st.write("Liker score is: ",score)
- #query_text = st.chat_input("Let me know what you have in mind")
- if query_text := st.chat_input("Let me know what you have in mind!"):
- # Add user message to chat history
+
+ # Display the chat input box
+ query_text = st.chat_input("Let me know what you have in mind!")
+
+ # Check if the user has entered a query
+ if query_text != "":
+ # Add the user message to the messages list
st.session_state.messages.append({"role": "user", "content": query_text})
- # Display user message in chat message container
+
+ # Display the user message in the chat message container
with st.chat_message("user"):
st.markdown(query_text)
-
- #query_text = st.chat_input("")
- #query_text = st.text_area(label="Let me know what you have in mind!")
- # st.session_state.messages.append({"role": "user", "content": (query_text)})
- # st.markdown(query_text)
-
-
-
- #if query_text != "":# or not audio.empty() and not os.path.exists("query.wav"):
- if query_text != "":
- st.session_state["query_status"] = True
- st.session_state["text_input_status"] = True
- st.session_state["query_counter"] += 1
- query = query_text
- context, keywords = create_context(query, text_split, headings, para_texts)
-
-
- # Generate a new response if the last message is not from the assistant
- with st.chat_message("assistant"):
- with st.spinner("Thinking..."):
- if len(context) < 2000:
- ans, context, keys = chatbot_slim(str(query), context, keywords)
-
- if (ans=='I don\'t know.' or ans=='I don\'t know'):
- ans = chatbot(str(query),db)
- #message = {"role": "assistant", "content": ans}
- st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(ans)
-
- else:
- #message = {"role": "assistant", "content": ans}
- st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(ans)
-
- else:
- ans = chatbot(str(query),db)
- #message = {"role": "assistant", "content": ans}
- st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(ans)
-
-
-
- #Generate a slider that takes input from 0 to 5 and asks for an ideal_answer
+
+ # Generate a response from the chatbot
with st.chat_message("assistant"):
- rouge_scores=calculate_rouge_scores(ans,context)
- score = st.slider("Rate the answer on scale of 5, 5=excellent,1=bad", min_value=0.0,max_value=5.0,value=2.5,step=0.5)
- #key=f"slider-{st.session_state['query_counter']}")
- st.write("Rating provided by user: ",score)
- #st.write(context)
- ideal_answer=st.text_area(label="Give your ideal answer --> Enter the reference source to actual answer",value="")
- qar=[]
- qar.append([query,ans,time,score,ideal_answer,rouge_scores])
- file_name=pd.DataFrame(qar)
- bucket = 'aiex' # already created on S3
- csv_buffer = StringIO()
- file_name.to_csv(csv_buffer)
- timestr = time.strftime("%Y%m%d-%H%M%S")
- file_name="df "+timestr+ ".csv"
- s3_resource= boto3.resource('s3',aws_access_key_id=os.environ["ACCESS_ID"],aws_secret_access_key= os.environ["ACCESS_KEY"])
- s3_resource.Object(bucket,file_name).put(Body=csv_buffer.getvalue())
-
-
-
-
-
-
- # -----------text to speech--------------------------#
- texttospeech_raw(ans, language="en")
- mymidia_placeholder = st.empty()
- with open("answer.wav", "rb") as audio_file:
- audio_bytes = audio_file.read()
- b64 = base64.b64encode(audio_bytes).decode()
- md = f"""
-
- """
- mymidia_placeholder.empty()
- time.sleep(1)
- mymidia_placeholder.markdown(md, unsafe_allow_html=True)
-
- st.session_state.messages.append(ans)
- #with st.chat_message("user"):
- #st.markdown(ans)
- st.session_state["query_status"] = False
- st.session_state["text_input_status"] = False
- st.session_state["audio_input_status"] = False
-
-# ------------------------------------------------------------------------------#
-# -------------------------QUERY AUDIO INPUT - RETURNING TEXT QUERY-------------#
-# ------------------------------------------------------------------------------#
-
-#if st.session_state.messages != []:
-# for message in st.session_state.messages[::1]:
-# with st.chat_message(message.role):
-# st.write(message.content)
-
-
-
-myargs = [
- "Made in India",
- "" " with ❤️ by ",
- link("https://www.linkedin.com/in/anupamisb/", "@Anupam"),
- br(),
- link("https://anupam-purwar.github.io/page/", "SpeeKAR ChatBoT"),
- br(),
- link("https://www.linkedin.com/in/rahul-sundar-311a6977/", "@Rahul"),
- br(),
- link("https://github.com/RahulSundar", "SpeeKAR ChatBoT"),
-]
+ # Your existing code to generate a response from the chatbot
+ # ...
+
+ # Add the assistant's response to the messages list
+ st.session_state.messages.append({"role": "assistant", "content": ans})
+
+ # Display the assistant's response in the chat message container
+ st.markdown(ans)
+ # At the end of the script
+ # Loop through all messages and display them
+ for message in st.session_state.messages:
+ with st.chat_message(message["role"]):
+ st.markdown(message["content"])
+# Footer function
def footer():
myargs = [
"Made in India",
From 11a3f285beba83954c9e32f79bc4a23e7ec3efdd Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 8 Feb 2024 18:49:16 +0530
Subject: [PATCH 030/129] streamlit_app.py
---
streamlit_app.py | 73 ++++++++++++++++++++++++++++++++++++++++++------
1 file changed, 65 insertions(+), 8 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index cabbd53..6b0c4af 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -232,14 +232,71 @@ def calculate_rouge_scores(answer,context):
# Generate a response from the chatbot
with st.chat_message("assistant"):
- # Your existing code to generate a response from the chatbot
- # ...
-
- # Add the assistant's response to the messages list
- st.session_state.messages.append({"role": "assistant", "content": ans})
-
- # Display the assistant's response in the chat message container
- st.markdown(ans)
+ with st.spinner("Thinking..."):
+ if len(context) < 2000:
+ ans, context, keys = chatbot_slim(str(query), context, keywords)
+
+ if (ans=='I don\'t know.' or ans=='I don\'t know'):
+ ans = chatbot(str(query),db)
+ #message = {"role": "assistant", "content": ans}
+ st.session_state.messages.append({"role": "user", "content": ans})
+ st.markdown(ans)
+
+ else:
+ #message = {"role": "assistant", "content": ans}
+ st.session_state.messages.append({"role": "user", "content": ans})
+ st.markdown(ans)
+
+ else:
+ ans = chatbot(str(query),db)
+ # message = {"role": "assistant", "content": ans}
+ st.session_state.messages.append({"role": "assistant", "content": ans})
+ st.markdown(ans)
+
+
+
+ #Generate a slider that takes input from 0 to 5 and asks for an ideal_answer
+ with st.chat_message("assistant"):
+ rouge_scores=calculate_rouge_scores(ans,context)
+ score = st.slider("Rate the answer on scale of 5, 5=excellent,1=bad", min_value=0.0,max_value=5.0,value=2.5,step=0.5)
+ #key=f"slider-{st.session_state['query_counter']}")
+ st.write("Rating provided by user: ",score)
+ #st.write(context)
+ ideal_answer=st.text_area(label="Give your ideal answer --> Enter the reference source to actual answer",value="")
+ qar=[]
+ qar.append([query,ans,time,score,ideal_answer,rouge_scores])
+ file_name=pd.DataFrame(qar)
+ bucket = 'aiex' # already created on S3
+ csv_buffer = StringIO()
+ file_name.to_csv(csv_buffer)
+ timestr = time.strftime("%Y%m%d-%H%M%S")
+ file_name="df "+timestr+ ".csv"
+ s3_resource= boto3.resource('s3',aws_access_key_id=os.environ["ACCESS_ID"],aws_secret_access_key= os.environ["ACCESS_KEY"])
+ s3_resource.Object(bucket,file_name).put(Body=csv_buffer.getvalue())
+ # -----------text to speech--------------------------#
+ texttospeech_raw(ans, language="en")
+ mymidia_placeholder = st.empty()
+ with open("answer.wav", "rb") as audio_file:
+ audio_bytes = audio_file.read()
+ b64 = base64.b64encode(audio_bytes).decode()
+ md = f"""
+
+ """
+ mymidia_placeholder.empty()
+ time.sleep(1)
+ mymidia_placeholder.markdown(md, unsafe_allow_html=True)
+
+ st.session_state.messages.append(ans)
+ #with st.chat_message("user"):
+ #st.markdown(ans)
+ st.session_state["query_status"] = False
+ st.session_state["text_input_status"] = False
+ st.session_state["audio_input_status"] = False
+
+
+
# At the end of the script
# Loop through all messages and display them
From c532d083fb49d66dd846a14e8042ed4bf593ff24 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 8 Feb 2024 19:18:46 +0530
Subject: [PATCH 031/129] streamlit_app.py
---
streamlit_app.py | 130 +++++++++++++++++++++++++++++++++++------------
1 file changed, 97 insertions(+), 33 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 6b0c4af..d7bfe30 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -64,7 +64,7 @@ def image(src_as_string, **style):
def link(link, text, **style):
- return a(_href=link, _target="_blank", style=styles(**style))
+ return a(_href=link, _target="_blank", style=styles(**style))(text)
def layout(*args):
@@ -166,14 +166,8 @@ def calculate_rouge_scores(answer,context):
# -----------------UPLOAD THE SRC DOCUMENT-----------------#
# ---------------------------------------------------------#
st.title("Please let me know what you want to talk about by choosing a file below!")
-
-# Initialize session state variables
if "uploaded_status" not in st.session_state:
st.session_state["uploaded_status"] = False
-if "query_counter" not in st.session_state:
- st.session_state["query_counter"] = 0
-if "messages" not in st.session_state:
- st.session_state.messages = []
uploaded_file = st.file_uploader(label = "")
if st.session_state["uploaded_status"] == False and uploaded_file is not None:
@@ -181,17 +175,37 @@ def calculate_rouge_scores(answer,context):
readdoc_splittext.clear()
readdoc_splittext_pdf.clear()
+if "query_counter" not in st.session_state:
+ st.session_state["query_counter"] = 0
+if "query_status" not in st.session_state:
+ st.session_state["query_status"] = False
+if "audio_input_status" not in st.session_state:
+ st.session_state["audio_input_status"] = False
+if "text_input_status" not in st.session_state:
+ st.session_state["text_input_status"] = False
+
+if "db_created" not in st.session_state:
+ st.session_state["db_created"] = False
+
if (uploaded_file is not None):
st.session_state["uploaded_status"] = True
elif uploaded_file is None:
st.session_state["uploaded_status"] = False
st.session_state["query_counter"] = 0
+ st.session_state["db_created"] = False
+ st.session_state["text_input_status"] = False
+ st.session_state["query_status"] = False
+ st.session_state["audio_input_status"] = False
+ st.write("Dear user, clearing unnecesary data fo you to start afresh!!")
create_db.clear()
readdoc_splittext.clear()
readdoc_splittext_pdf.clear()
- st.write("You can upload your document now.")
+ st.write("You can upload your document now!")
+if "messages" not in st.session_state.keys():
+ st.session_state.messages = []
+
if (uploaded_file is not None):
file_path = os.path.join( os.getcwd(), uploaded_file.name)
with open(file_path,"wb") as f:
@@ -217,20 +231,50 @@ def calculate_rouge_scores(answer,context):
if uploaded_file is not None and st.session_state["db_created"] == True:
st.title("Ask me anything about the document!")
-
- # Display the chat input box
- query_text = st.chat_input("Let me know what you have in mind!")
-
- # Check if the user has entered a query
- if query_text != "":
- # Add the user message to the messages list
+ #for message in st.session_state.messages:
+ # with st.chat_message(message["role"]):
+ # st.markdown(message["content"])
+
+ # User-provided prompt
+ #if prompt := st.chat_input():
+
+ #slider_value = 2.5 # Default value for the slider
+
+ # with st.chat_message("assistant"):
+ #query_audio_placeholder = st.empty()
+ #audio = audiorecorder("Click to record", "Click to stop recording")
+ #query_placeholder = st.empty()
+ #query_text = st.text_area(label="Let me know what you have in mind!")
+ # st.session_state.messages.append({"role": "user", "content": (query_text)})
+ # Add a slider for each question
+ # score = st.slider("Select the creativity level for this answer:", 0.0, 5.0, 2.5)
+ #key=f"slider-{st.session_state['query_counter']}")
+ # st.write("Liker score is: ",score)
+ #query_text = st.chat_input("Let me know what you have in mind")
+ if query_text := st.chat_input("Let me know what you have in mind!"):
+ # Add user message to chat history
st.session_state.messages.append({"role": "user", "content": query_text})
-
- # Display the user message in the chat message container
+ # Display user message in chat message container
with st.chat_message("user"):
st.markdown(query_text)
-
- # Generate a response from the chatbot
+
+ #query_text = st.chat_input("")
+ #query_text = st.text_area(label="Let me know what you have in mind!")
+ # st.session_state.messages.append({"role": "user", "content": (query_text)})
+ # st.markdown(query_text)
+
+
+
+ #if query_text != "":# or not audio.empty() and not os.path.exists("query.wav"):
+ if query_text != "":
+ st.session_state["query_status"] = True
+ st.session_state["text_input_status"] = True
+ st.session_state["query_counter"] += 1
+ query = query_text
+ context, keywords = create_context(query, text_split, headings, para_texts)
+
+
+ # Generate a new response if the last message is not from the assistant
with st.chat_message("assistant"):
with st.spinner("Thinking..."):
if len(context) < 2000:
@@ -249,13 +293,13 @@ def calculate_rouge_scores(answer,context):
else:
ans = chatbot(str(query),db)
- # message = {"role": "assistant", "content": ans}
- st.session_state.messages.append({"role": "assistant", "content": ans})
+ #message = {"role": "assistant", "content": ans}
+ st.session_state.messages.append({"role": "user", "content": ans})
st.markdown(ans)
-
+
+
-
- #Generate a slider that takes input from 0 to 5 and asks for an ideal_answer
+ #Generate a slider that takes input from 0 to 5 and asks for an ideal_answer
with st.chat_message("assistant"):
rouge_scores=calculate_rouge_scores(ans,context)
score = st.slider("Rate the answer on scale of 5, 5=excellent,1=bad", min_value=0.0,max_value=5.0,value=2.5,step=0.5)
@@ -273,7 +317,13 @@ def calculate_rouge_scores(answer,context):
file_name="df "+timestr+ ".csv"
s3_resource= boto3.resource('s3',aws_access_key_id=os.environ["ACCESS_ID"],aws_secret_access_key= os.environ["ACCESS_KEY"])
s3_resource.Object(bucket,file_name).put(Body=csv_buffer.getvalue())
- # -----------text to speech--------------------------#
+
+
+
+
+
+
+ # -----------text to speech--------------------------#
texttospeech_raw(ans, language="en")
mymidia_placeholder = st.empty()
with open("answer.wav", "rb") as audio_file:
@@ -295,20 +345,34 @@ def calculate_rouge_scores(answer,context):
st.session_state["text_input_status"] = False
st.session_state["audio_input_status"] = False
+# ------------------------------------------------------------------------------#
+# -------------------------QUERY AUDIO INPUT - RETURNING TEXT QUERY-------------#
+# ------------------------------------------------------------------------------#
-
+#if st.session_state.messages != []:
+# for message in st.session_state.messages[::1]:
+# with st.chat_message(message.role):
+# st.write(message.content)
+
+
+
+myargs = [
+ "Made in India",
+ "" " with ❤ by ",
+ link("https://www.linkedin.com/in/anupamisb/", "@Anupam"),
+ br(),
+ link("https://anupam-purwar.github.io/page/", "SpeeKAR ChatBoT"),
+ br(),
+ link("https://www.linkedin.com/in/rahul-sundar-311a6977/", "@Rahul"),
+ br(),
+ link("https://github.com/RahulSundar", "SpeeKAR ChatBoT"),
+]
- # At the end of the script
- # Loop through all messages and display them
- for message in st.session_state.messages:
- with st.chat_message(message["role"]):
- st.markdown(message["content"])
-# Footer function
def footer():
myargs = [
"Made in India",
- "" " with ❤️ by ",
+ "" " with ❤ by ",
link("https://www.linkedin.com/in/anupamisb/", " Anupam for "),
link("https://anupam-purwar.github.io/page/", "SpeeKAR ChatBoT"),
", and",
From e36034aacf37e7f2ff104eeaaff67ee8ac0972f0 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 8 Feb 2024 19:27:40 +0530
Subject: [PATCH 032/129] streamlit_app.py
---
streamlit_app.py | 106 ++++++++++-------------------------------------
1 file changed, 23 insertions(+), 83 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index d7bfe30..eca2778 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -7,7 +7,6 @@
import glob
import base64
from io import StringIO
-import boto3
import openai
import tiktoken
@@ -121,13 +120,6 @@ def process_query(speech_input, email, passwd):
def generate_kARanswer(query, text_split):
ans, context, keys = chatbot_slim(query, text_split)
return ans, context, keys
-from rouge import Rouge
-import time
-
-def calculate_rouge_scores(answer,context):
- rouge = Rouge()
- rouge_scores = rouge.get_scores(answer,context)
- return rouge_scores
# -------------------------------------------------------------------------#
@@ -231,97 +223,48 @@ def calculate_rouge_scores(answer,context):
if uploaded_file is not None and st.session_state["db_created"] == True:
st.title("Ask me anything about the document!")
- #for message in st.session_state.messages:
- # with st.chat_message(message["role"]):
- # st.markdown(message["content"])
+
# User-provided prompt
#if prompt := st.chat_input():
- #slider_value = 2.5 # Default value for the slider
-
- # with st.chat_message("assistant"):
+
+ with st.chat_message("user"):
#query_audio_placeholder = st.empty()
#audio = audiorecorder("Click to record", "Click to stop recording")
#query_placeholder = st.empty()
- #query_text = st.text_area(label="Let me know what you have in mind!")
- # st.session_state.messages.append({"role": "user", "content": (query_text)})
- # Add a slider for each question
- # score = st.slider("Select the creativity level for this answer:", 0.0, 5.0, 2.5)
- #key=f"slider-{st.session_state['query_counter']}")
- # st.write("Liker score is: ",score)
- #query_text = st.chat_input("Let me know what you have in mind")
- if query_text := st.chat_input("Let me know what you have in mind!"):
- # Add user message to chat history
- st.session_state.messages.append({"role": "user", "content": query_text})
- # Display user message in chat message container
- with st.chat_message("user"):
- st.markdown(query_text)
-
- #query_text = st.chat_input("")
- #query_text = st.text_area(label="Let me know what you have in mind!")
- # st.session_state.messages.append({"role": "user", "content": (query_text)})
- # st.markdown(query_text)
-
-
-
- #if query_text != "":# or not audio.empty() and not os.path.exists("query.wav"):
+ query_text = st.text_area(label = "Let me know what you have in mind!")
+ st.session_state.messages.append({"role": "user", "content": query_text})
+ if query_text != "":# or not audio.empty() and not os.path.exists("query.wav"):
if query_text != "":
st.session_state["query_status"] = True
st.session_state["text_input_status"] = True
st.session_state["query_counter"] += 1
+
+
query = query_text
+
context, keywords = create_context(query, text_split, headings, para_texts)
- # Generate a new response if the last message is not from the assistant
+ # Generate a new response if last message is not from assistant
with st.chat_message("assistant"):
with st.spinner("Thinking..."):
if len(context) < 2000:
- ans, context, keys = chatbot_slim(str(query), context, keywords)
-
- if (ans=='I don\'t know.' or ans=='I don\'t know'):
- ans = chatbot(str(query),db)
- #message = {"role": "assistant", "content": ans}
- st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(ans)
-
+ ans, context, keys = chatbot_slim(query, context, keywords)
+
+ if (ans=='I don\'t know.' or ans=='I don\'t know' ):
+ ans = chatbot(query,db)
+ message = {"role": "assistant", "content": ans}
else:
- #message = {"role": "assistant", "content": ans}
- st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(ans)
+ message = {"role": "assistant", "content": ans}
else:
- ans = chatbot(str(query),db)
- #message = {"role": "assistant", "content": ans}
- st.session_state.messages.append({"role": "user", "content": ans})
- st.markdown(ans)
+ ans = chatbot(query,db)
+ message = {"role": "assistant", "content": ans}
-
- #Generate a slider that takes input from 0 to 5 and asks for an ideal_answer
- with st.chat_message("assistant"):
- rouge_scores=calculate_rouge_scores(ans,context)
- score = st.slider("Rate the answer on scale of 5, 5=excellent,1=bad", min_value=0.0,max_value=5.0,value=2.5,step=0.5)
- #key=f"slider-{st.session_state['query_counter']}")
- st.write("Rating provided by user: ",score)
- #st.write(context)
- ideal_answer=st.text_area(label="Give your ideal answer --> Enter the reference source to actual answer",value="")
- qar=[]
- qar.append([query,ans,time,score,ideal_answer,rouge_scores])
- file_name=pd.DataFrame(qar)
- bucket = 'aiex' # already created on S3
- csv_buffer = StringIO()
- file_name.to_csv(csv_buffer)
- timestr = time.strftime("%Y%m%d-%H%M%S")
- file_name="df "+timestr+ ".csv"
- s3_resource= boto3.resource('s3',aws_access_key_id=os.environ["ACCESS_ID"],aws_secret_access_key= os.environ["ACCESS_KEY"])
- s3_resource.Object(bucket,file_name).put(Body=csv_buffer.getvalue())
-
-
-
-
# -----------text to speech--------------------------#
texttospeech_raw(ans, language="en")
@@ -337,10 +280,7 @@ def calculate_rouge_scores(answer,context):
mymidia_placeholder.empty()
time.sleep(1)
mymidia_placeholder.markdown(md, unsafe_allow_html=True)
-
- st.session_state.messages.append(ans)
- #with st.chat_message("user"):
- #st.markdown(ans)
+ st.session_state.messages.append(message)
st.session_state["query_status"] = False
st.session_state["text_input_status"] = False
st.session_state["audio_input_status"] = False
@@ -349,10 +289,10 @@ def calculate_rouge_scores(answer,context):
# -------------------------QUERY AUDIO INPUT - RETURNING TEXT QUERY-------------#
# ------------------------------------------------------------------------------#
-#if st.session_state.messages != []:
-# for message in st.session_state.messages[::1]:
-# with st.chat_message(message.role):
-# st.write(message.content)
+if st.session_state.messages != []:
+ for message in st.session_state.messages[::-1]:
+ with st.chat_message(message["role"]):
+ st.write(message["content"])
From fd2f138c03e65d0a62a92d3b1e87db9396c9d86a Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 8 Feb 2024 20:41:41 +0530
Subject: [PATCH 033/129] streamlit_app.py
---
streamlit_app.py | 105 ++++++++++++++++++++++++++++++-----------------
1 file changed, 68 insertions(+), 37 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index eca2778..2762120 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -7,6 +7,7 @@
import glob
import base64
from io import StringIO
+import boto3
import openai
import tiktoken
@@ -63,7 +64,7 @@ def image(src_as_string, **style):
def link(link, text, **style):
- return a(_href=link, _target="_blank", style=styles(**style))(text)
+ return a(_href=link, _target="_blank", style=styles(**style))
def layout(*args):
@@ -120,6 +121,13 @@ def process_query(speech_input, email, passwd):
def generate_kARanswer(query, text_split):
ans, context, keys = chatbot_slim(query, text_split)
return ans, context, keys
+from rouge import Rouge
+import time
+
+def calculate_rouge_scores(answer,context):
+ rouge = Rouge()
+ rouge_scores = rouge.get_scores(answer,context)
+ return rouge_scores
# -------------------------------------------------------------------------#
@@ -192,9 +200,18 @@ def generate_kARanswer(query, text_split):
create_db.clear()
readdoc_splittext.clear()
readdoc_splittext_pdf.clear()
- st.write("You can upload your document now!")
+ st.write("You can upload your document now.")
+
+import streamlit as st
+if "uploaded_status" not in st.session_state:
+ st.session_state["uploaded_status"] = False
+if "query_counter" not in st.session_state:
+ st.session_state["query_counter"] = 0
+if "messages" not in st.session_state:
+ st.session_state.messages = []
+
if "messages" not in st.session_state.keys():
st.session_state.messages = []
@@ -224,49 +241,65 @@ def generate_kARanswer(query, text_split):
if uploaded_file is not None and st.session_state["db_created"] == True:
st.title("Ask me anything about the document!")
-
- # User-provided prompt
- #if prompt := st.chat_input():
-
-
- with st.chat_message("user"):
- #query_audio_placeholder = st.empty()
- #audio = audiorecorder("Click to record", "Click to stop recording")
- #query_placeholder = st.empty()
- query_text = st.text_area(label = "Let me know what you have in mind!")
- st.session_state.messages.append({"role": "user", "content": query_text})
- if query_text != "":# or not audio.empty() and not os.path.exists("query.wav"):
+ # Display the chat input box
+ query_text = st.chat_input("Let me know what you have in mind!")
if query_text != "":
st.session_state["query_status"] = True
st.session_state["text_input_status"] = True
st.session_state["query_counter"] += 1
+ st.session_state.messages.append({"role": "user", "content": query_text})
+
+ # Display the user message in the chat message container
+ with st.chat_message("user"):
+ st.markdown(query_text)
query = query_text
context, keywords = create_context(query, text_split, headings, para_texts)
-
- # Generate a new response if last message is not from assistant
+
+ # Generate a response from the chatbot
with st.chat_message("assistant"):
with st.spinner("Thinking..."):
if len(context) < 2000:
- ans, context, keys = chatbot_slim(query, context, keywords)
-
- if (ans=='I don\'t know.' or ans=='I don\'t know' ):
- ans = chatbot(query,db)
+ ans, context, keys = chatbot_slim(str(query), context, keywords)
+
+ if (ans=='I don\'t know.' or ans=='I don\'t know'):
+ ans = chatbot(str(query),db)
message = {"role": "assistant", "content": ans}
- else:
+ #st.session_state.messages.append({"role": "user", "content": ans})
+ st.markdown(ans)
+ else:
message = {"role": "assistant", "content": ans}
+ #st.session_state.messages.append({"role": "user", "content": ans})
+ st.markdown(ans)
+
else:
- ans = chatbot(query,db)
-
+ ans = chatbot(str(query),db)
message = {"role": "assistant", "content": ans}
-
-
-
- # -----------text to speech--------------------------#
+ #st.session_state.messages.append({"role": "user", "content": ans})
+ st.markdown(ans)
+ #Generate a slider that takes input from 0 to 5 and asks for an ideal_answer
+ with st.chat_message("assistant"):
+ rouge_scores=calculate_rouge_scores(ans,context)
+ score = st.slider("Rate the answer on scale of 5, 5=excellent,1=bad", min_value=0.0,max_value=5.0,value=2.5,step=0.5)
+ #key=f"slider-{st.session_state['query_counter']}")
+ st.write("Rating provided by user: ",score)
+ #st.write(context)
+ ideal_answer=st.text_area(label="Give your ideal answer --> Enter the reference source to actual answer",value="")
+ qar=[]
+ qar.append([query,ans,time,score,ideal_answer,rouge_scores])
+ file_name=pd.DataFrame(qar)
+ bucket = 'aiex' # already created on S3
+ csv_buffer = StringIO()
+ file_name.to_csv(csv_buffer)
+ timestr = time.strftime("%Y%m%d-%H%M%S")
+ file_name="df "+timestr+ ".csv"
+ s3_resource= boto3.resource('s3',aws_access_key_id=os.environ["ACCESS_ID"],aws_secret_access_key= os.environ["ACCESS_KEY"])
+ s3_resource.Object(bucket,file_name).put(Body=csv_buffer.getvalue())
+# -----------text to speech--------------------------#
texttospeech_raw(ans, language="en")
mymidia_placeholder = st.empty()
with open("answer.wav", "rb") as audio_file:
@@ -280,21 +313,19 @@ def generate_kARanswer(query, text_split):
mymidia_placeholder.empty()
time.sleep(1)
mymidia_placeholder.markdown(md, unsafe_allow_html=True)
- st.session_state.messages.append(message)
+
+ st.session_state.messages.append(ans)
+ #with st.chat_message("user"):
+ #st.markdown(ans)
st.session_state["query_status"] = False
st.session_state["text_input_status"] = False
st.session_state["audio_input_status"] = False
-# ------------------------------------------------------------------------------#
-# -------------------------QUERY AUDIO INPUT - RETURNING TEXT QUERY-------------#
-# ------------------------------------------------------------------------------#
-
-if st.session_state.messages != []:
- for message in st.session_state.messages[::-1]:
+ # At the end of the script
+ # Loop through all messages and display them
+ for message in st.session_state.messages:
with st.chat_message(message["role"]):
- st.write(message["content"])
-
-
+ st.markdown(message["content"])
myargs = [
"Made in India",
From 4b447f0be7c751742c67e38807985b82233dce92 Mon Sep 17 00:00:00 2001
From: "deepsource-autofix[bot]"
<62050782+deepsource-autofix[bot]@users.noreply.github.com>
Date: Thu, 8 Feb 2024 15:16:15 +0000
Subject: [PATCH 034/129] refactor: use identity check for comparison to a
singleton
Comparisons to the singleton objects, like `True`, `False`, and `None`, should be done with identity, not equality. Use `is` or `is not`.
---
streamlit_app.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 2762120..7d1e341 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -170,7 +170,7 @@ def calculate_rouge_scores(answer,context):
st.session_state["uploaded_status"] = False
uploaded_file = st.file_uploader(label = "")
-if st.session_state["uploaded_status"] == False and uploaded_file is not None:
+if st.session_state["uploaded_status"] is False and uploaded_file is not None:
create_db.clear()
readdoc_splittext.clear()
readdoc_splittext_pdf.clear()
@@ -238,7 +238,7 @@ def calculate_rouge_scores(answer,context):
st.session_state["db_created"] = True
- if uploaded_file is not None and st.session_state["db_created"] == True:
+ if uploaded_file is not None and st.session_state["db_created"] is True:
st.title("Ask me anything about the document!")
# Display the chat input box
From b4e1de51ecb267ef51eb183e3047a7283ab87449 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Fri, 9 Feb 2024 02:29:53 +0530
Subject: [PATCH 035/129] streamlit_app.py
---
streamlit_app.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 7d1e341..263a131 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -268,18 +268,18 @@ def calculate_rouge_scores(answer,context):
if (ans=='I don\'t know.' or ans=='I don\'t know'):
ans = chatbot(str(query),db)
message = {"role": "assistant", "content": ans}
- #st.session_state.messages.append({"role": "user", "content": ans})
+ st.session_state.messages.append({"role": "user", "content": ans})
st.markdown(ans)
else:
message = {"role": "assistant", "content": ans}
- #st.session_state.messages.append({"role": "user", "content": ans})
+ st.session_state.messages.append({"role": "user", "content": ans})
st.markdown(ans)
else:
ans = chatbot(str(query),db)
message = {"role": "assistant", "content": ans}
- #st.session_state.messages.append({"role": "user", "content": ans})
+ st.session_state.messages.append({"role": "user", "content": ans})
st.markdown(ans)
#Generate a slider that takes input from 0 to 5 and asks for an ideal_answer
with st.chat_message("assistant"):
From 3ac3a3c764ac96a06a96015a8c0a46e8562f5fad Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Fri, 9 Feb 2024 23:49:00 +0530
Subject: [PATCH 036/129] streamlit_app.py
---
streamlit_app.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 263a131..b97a87d 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -314,7 +314,7 @@ def calculate_rouge_scores(answer,context):
time.sleep(1)
mymidia_placeholder.markdown(md, unsafe_allow_html=True)
- st.session_state.messages.append(ans)
+ #st.session_state.messages.append(ans)
#with st.chat_message("user"):
#st.markdown(ans)
st.session_state["query_status"] = False
From d943e9aefdfa65515720ea1a722b368cb961373a Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 17 Feb 2024 14:19:18 +0530
Subject: [PATCH 037/129] qa.py
---
qa.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/qa.py b/qa.py
index 7e3faaa..3c54e0b 100644
--- a/qa.py
+++ b/qa.py
@@ -100,6 +100,7 @@
UnstructuredODTLoader,
UnstructuredPowerPointLoader,
UnstructuredWordDocumentLoader,
+ UnstructuredFileLoader
)
import os
import glob
From e776403b503950ef1b505322732f2ed7cab8b8c5 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Tue, 20 Feb 2024 16:48:40 +0530
Subject: [PATCH 038/129] qa.py
---
qa.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 50 insertions(+)
diff --git a/qa.py b/qa.py
index 3c54e0b..7f0c874 100644
--- a/qa.py
+++ b/qa.py
@@ -316,6 +316,56 @@ def get_paragraphs(headings, paragraph_sentences):
return paragraph_list
+#---------------READ THE .TXT FILE AND GENERATE THE SPLIT--------------------#
+@st.cache_resource(show_spinner=True)
+def readdoc_splittext(filename):
+ if ".txt" in filename:
+ loader = UnstructuredFileLoader(filename)
+ docs = loader.load()
+ block_dict = get_block_dict_fromDoc(docs)
+ span_df = get_docfeature_dataframe(block_dict)
+ docs_clean = span_df[span_df["font_size"]>=span_df["font_size"].mode()[0]]
+ #doc_clean.head()
+ paragraphs = docs_clean.text[docs_clean.font_size == span_df.font_size.mode()[0]]
+ #print(paragraphs.values,paragraphs.index)
+ headings = docs_clean.text[docs_clean.font_size > span_df.font_size.mode()[0]]
+ #print(headings.values, headings.index)
+ paragraph_list = get_paragraphs(headings, paragraphs)
+ headings_list = headings.values.tolist()
+ n = 1500 #Number of characters to be included in a single chunk of text
+ all_text=''
+ for text in paragraph_list:
+ all_text+=text
+ a=glob.glob(filename)
+ #print(a)
+ chunk_size = 1024
+ chunk_overlap=10
+ text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+ texts_isb=[]
+ texts_raw = []
+ documents=[]
+ for i in range(len(a)):
+ documents.extend(UnstructuredFileLoader(a[i]).load())
+ for j in range(
+ len(
+ text_splitter.split_documents(
+ UnstructuredFileLoader(a[i]).load()
+ )
+ )
+ ):
+ text_chunk = text_splitter.split_documents(
+ UnstructuredFileLoader(a[i]).load()
+ )[j]
+ text_chunk.page_content = text_chunk.page_content.replace("\n", " ")
+ text_chunk.page_content = text_chunk.page_content.replace("\\n", " ")
+ text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
+ text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
+ texts_isb.append(text_chunk.page_content)
+ texts_raw.append(text_chunk)
+
+ text_split = texts_isb
+ return all_text, text_split, texts_raw, headings_list, paragraph_list
+
@st.cache_resource(show_spinner=True)
def readdoc_splittext_pdf(filename):
"""
From 43d775b4873b5f799381a4f464f3456c75485c3a Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Tue, 20 Feb 2024 16:49:48 +0530
Subject: [PATCH 039/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 7f0c874..59fd5b1 100644
--- a/qa.py
+++ b/qa.py
@@ -318,7 +318,7 @@ def get_paragraphs(headings, paragraph_sentences):
#---------------READ THE .TXT FILE AND GENERATE THE SPLIT--------------------#
@st.cache_resource(show_spinner=True)
-def readdoc_splittext(filename):
+def readdoc_splittext_txt(filename):
if ".txt" in filename:
loader = UnstructuredFileLoader(filename)
docs = loader.load()
From bc172c35f8cdc5310681ce072d0c5c6e385415b3 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Tue, 20 Feb 2024 16:51:11 +0530
Subject: [PATCH 040/129] streamlit_app.py
---
streamlit_app.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/streamlit_app.py b/streamlit_app.py
index b97a87d..61d36d5 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -17,6 +17,7 @@
from qa import (
speechtotext,
readdoc_splittext,
+ readdoc_splittext_txt,
readdoc_splittext_pdf,
create_context,
create_db,
From c945cabd62035c96d74c64b37205873b5bef1d49 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Tue, 20 Feb 2024 16:54:38 +0530
Subject: [PATCH 041/129] streamlit_app.py
---
streamlit_app.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/streamlit_app.py b/streamlit_app.py
index 61d36d5..5b30416 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -174,6 +174,7 @@ def calculate_rouge_scores(answer,context):
if st.session_state["uploaded_status"] is False and uploaded_file is not None:
create_db.clear()
readdoc_splittext.clear()
+ readdoc_splittext_txt.clear()
readdoc_splittext_pdf.clear()
if "query_counter" not in st.session_state:
@@ -200,6 +201,7 @@ def calculate_rouge_scores(answer,context):
st.write("Dear user, clearing unnecesary data fo you to start afresh!!")
create_db.clear()
readdoc_splittext.clear()
+ readdoc_splittext_txt.clear()
readdoc_splittext_pdf.clear()
st.write("You can upload your document now.")
@@ -229,6 +231,8 @@ def calculate_rouge_scores(answer,context):
all_text, text_split, text_chunk, headings, para_texts = readdoc_splittext(filename)#uploaded_file.name)
elif (".doc" in filename) and (".docx" not in filename): #uploaded_file.name:
all_text, text_split, text_chunk, headings, para_texts = readdoc_splittext(filename)#uploaded_file.name)
+ elif ".txt" in filename:
+ all_text, text_split, text_chunk, headings, para_texts = readdoc_splittext_txt(filename)#uploaded_file.name)
elif ".pdf" in filename: #uploaded_file.name:
all_text, text_split, text_chunk, headings, para_texts = readdoc_splittext_pdf(filename)#uploaded_file.name)
From 9ed319f85491fedff40f943b32ddd043d88fe813 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 22 Feb 2024 14:04:37 +0530
Subject: [PATCH 042/129] qa.py
---
qa.py | 18 +++++++++++-------
1 file changed, 11 insertions(+), 7 deletions(-)
diff --git a/qa.py b/qa.py
index 59fd5b1..324b72e 100644
--- a/qa.py
+++ b/qa.py
@@ -322,17 +322,21 @@ def readdoc_splittext_txt(filename):
if ".txt" in filename:
loader = UnstructuredFileLoader(filename)
docs = loader.load()
- block_dict = get_block_dict_fromDoc(docs)
- span_df = get_docfeature_dataframe(block_dict)
- docs_clean = span_df[span_df["font_size"]>=span_df["font_size"].mode()[0]]
+ #block_dict = get_block_dict_fromDoc(docs)
+ #span_df = get_docfeature_dataframe(block_dict)
+ #docs_clean = span_df[span_df["font_size"]>=span_df["font_size"].mode()[0]]
#doc_clean.head()
- paragraphs = docs_clean.text[docs_clean.font_size == span_df.font_size.mode()[0]]
+ #paragraphs = docs_clean.text[docs_clean.font_size == span_df.font_size.mode()[0]]
#print(paragraphs.values,paragraphs.index)
- headings = docs_clean.text[docs_clean.font_size > span_df.font_size.mode()[0]]
+ #headings = docs_clean.text[docs_clean.font_size > span_df.font_size.mode()[0]]
#print(headings.values, headings.index)
+ pat1= re.compile(r".+\:")
+ pat2=re.compile(r".+\.\n")
+ headings=pat1.search(docs)
+ paragraphs=pat2.search(docs)
paragraph_list = get_paragraphs(headings, paragraphs)
headings_list = headings.values.tolist()
- n = 1500 #Number of characters to be included in a single chunk of text
+ n = 1500 #Number of characters to be included in a single chunk of text
all_text=''
for text in paragraph_list:
all_text+=text
@@ -362,7 +366,7 @@ def readdoc_splittext_txt(filename):
text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
texts_isb.append(text_chunk.page_content)
texts_raw.append(text_chunk)
-
+
text_split = texts_isb
return all_text, text_split, texts_raw, headings_list, paragraph_list
From 16843f0965e35f72fd2af5fc6649e5c90587c791 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 22 Feb 2024 15:18:31 +0530
Subject: [PATCH 043/129] streamlit_app.py
---
streamlit_app.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 5b30416..e08e868 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -232,9 +232,9 @@ def calculate_rouge_scores(answer,context):
elif (".doc" in filename) and (".docx" not in filename): #uploaded_file.name:
all_text, text_split, text_chunk, headings, para_texts = readdoc_splittext(filename)#uploaded_file.name)
elif ".txt" in filename:
- all_text, text_split, text_chunk, headings, para_texts = readdoc_splittext_txt(filename)#uploaded_file.name)
+ all_text, text_split, text_raw, headings_list, paragraph_list = readdoc_splittext_txt(filename)#uploaded_file.name)
elif ".pdf" in filename: #uploaded_file.name:
- all_text, text_split, text_chunk, headings, para_texts = readdoc_splittext_pdf(filename)#uploaded_file.name)
+ all_text, text_split, text_raw, headings_list, paragraph_list = readdoc_splittext_pdf(filename)#uploaded_file.name)
with st.chat_message("assistant"):
st.write("Hi! Getting your contexts ready for query! Please wait!")
From 859a159fe79d4fd60dfa01db2c47a3d0c666c779 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 22 Feb 2024 15:23:40 +0530
Subject: [PATCH 044/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 324b72e..456992f 100644
--- a/qa.py
+++ b/qa.py
@@ -321,7 +321,7 @@ def get_paragraphs(headings, paragraph_sentences):
def readdoc_splittext_txt(filename):
if ".txt" in filename:
loader = UnstructuredFileLoader(filename)
- docs = loader.load()
+ docs = str(loader.load())
#block_dict = get_block_dict_fromDoc(docs)
#span_df = get_docfeature_dataframe(block_dict)
#docs_clean = span_df[span_df["font_size"]>=span_df["font_size"].mode()[0]]
From 40c4de93243753712d7fb406424dd0da14a4a62b Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 22 Feb 2024 15:27:16 +0530
Subject: [PATCH 045/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 456992f..f2a652e 100644
--- a/qa.py
+++ b/qa.py
@@ -334,7 +334,7 @@ def readdoc_splittext_txt(filename):
pat2=re.compile(r".+\.\n")
headings=pat1.search(docs)
paragraphs=pat2.search(docs)
- paragraph_list = get_paragraphs(headings, paragraphs)
+ paragraph_list = paragraph.values.tolist()
headings_list = headings.values.tolist()
n = 1500 #Number of characters to be included in a single chunk of text
all_text=''
From 6025541f1947422bd83037a495feeef774033552 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 22 Feb 2024 15:27:55 +0530
Subject: [PATCH 046/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index f2a652e..c74f004 100644
--- a/qa.py
+++ b/qa.py
@@ -334,7 +334,7 @@ def readdoc_splittext_txt(filename):
pat2=re.compile(r".+\.\n")
headings=pat1.search(docs)
paragraphs=pat2.search(docs)
- paragraph_list = paragraph.values.tolist()
+ paragraph_list = paragraphs.values.tolist()
headings_list = headings.values.tolist()
n = 1500 #Number of characters to be included in a single chunk of text
all_text=''
From e83e74e64bf1ae3a5717f7d14ea9ff97e8a72ef0 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 22 Feb 2024 15:28:54 +0530
Subject: [PATCH 047/129] qa.py
---
qa.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/qa.py b/qa.py
index c74f004..6fe5fb8 100644
--- a/qa.py
+++ b/qa.py
@@ -334,8 +334,8 @@ def readdoc_splittext_txt(filename):
pat2=re.compile(r".+\.\n")
headings=pat1.search(docs)
paragraphs=pat2.search(docs)
- paragraph_list = paragraphs.values.tolist()
- headings_list = headings.values.tolist()
+ paragraph_list = paragraphs.tolist()
+ headings_list = headings.tolist()
n = 1500 #Number of characters to be included in a single chunk of text
all_text=''
for text in paragraph_list:
From f1ada9c844c81456c115777dcc5471927e4be83c Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 22 Feb 2024 15:31:52 +0530
Subject: [PATCH 048/129] qa.py
---
qa.py | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/qa.py b/qa.py
index 6fe5fb8..961a4f8 100644
--- a/qa.py
+++ b/qa.py
@@ -332,10 +332,8 @@ def readdoc_splittext_txt(filename):
#print(headings.values, headings.index)
pat1= re.compile(r".+\:")
pat2=re.compile(r".+\.\n")
- headings=pat1.search(docs)
- paragraphs=pat2.search(docs)
- paragraph_list = paragraphs.tolist()
- headings_list = headings.tolist()
+ headings_list=pat1.findall(docs)
+ paragraphs_list=pat2.findall(docs)
n = 1500 #Number of characters to be included in a single chunk of text
all_text=''
for text in paragraph_list:
@@ -368,7 +366,7 @@ def readdoc_splittext_txt(filename):
texts_raw.append(text_chunk)
text_split = texts_isb
- return all_text, text_split, texts_raw, headings_list, paragraph_list
+ return all_text, text_split, texts_raw, headings_list, paragraphs_list
@st.cache_resource(show_spinner=True)
def readdoc_splittext_pdf(filename):
From da53d3b665c20381350fecf1bbf19d904725882c Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 22 Feb 2024 15:32:26 +0530
Subject: [PATCH 049/129] qa.py
---
qa.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/qa.py b/qa.py
index 961a4f8..ec8071e 100644
--- a/qa.py
+++ b/qa.py
@@ -333,7 +333,7 @@ def readdoc_splittext_txt(filename):
pat1= re.compile(r".+\:")
pat2=re.compile(r".+\.\n")
headings_list=pat1.findall(docs)
- paragraphs_list=pat2.findall(docs)
+ paragraph_list=pat2.findall(docs)
n = 1500 #Number of characters to be included in a single chunk of text
all_text=''
for text in paragraph_list:
@@ -366,7 +366,7 @@ def readdoc_splittext_txt(filename):
texts_raw.append(text_chunk)
text_split = texts_isb
- return all_text, text_split, texts_raw, headings_list, paragraphs_list
+ return all_text, text_split, texts_raw, headings_list, paragraph_list
@st.cache_resource(show_spinner=True)
def readdoc_splittext_pdf(filename):
From 9c3bd4c134a68957f3f913dfbae8ae13049d232b Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 22 Feb 2024 15:33:51 +0530
Subject: [PATCH 050/129] streamlit_app.py
---
streamlit_app.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index e08e868..816a39d 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -239,7 +239,7 @@ def calculate_rouge_scores(answer,context):
with st.chat_message("assistant"):
st.write("Hi! Getting your contexts ready for query! Please wait!")
- hf, db = create_db(text_chunk, uploaded_file.name)
+ hf, db = create_db(texts_raw, uploaded_file.name)
st.session_state["db_created"] = True
From e12bc6d45caf67db0dba29cb43167325f4a4ff90 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 22 Feb 2024 15:44:14 +0530
Subject: [PATCH 051/129] qa.py
---
qa.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/qa.py b/qa.py
index ec8071e..b8dec38 100644
--- a/qa.py
+++ b/qa.py
@@ -734,10 +734,10 @@ def chatbot_slim(question, context, keywords):
@st.cache_resource(show_spinner=True)
-def create_db(_text_chunk, _uploaded_file_name):
+def create_db(_texts_raw, _uploaded_file_name):
hf= OpenAIEmbeddings(model="text-embedding-ada-002", openai_api_key=openai.api_key)
- db = FAISS.from_documents(_text_chunk, hf)
+ db = FAISS.from_documents(_texts_raw, hf)
db.save_local("faiss_index_anupam" + _uploaded_file_name)
db=FAISS.load_local("faiss_index_anupam" + _uploaded_file_name, hf)
return hf, db
From 2747e2a9702998327a5a1314862a72ec21ba7045 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 22 Feb 2024 15:45:29 +0530
Subject: [PATCH 052/129] streamlit_app.py
---
streamlit_app.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 816a39d..8b01b22 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -239,7 +239,7 @@ def calculate_rouge_scores(answer,context):
with st.chat_message("assistant"):
st.write("Hi! Getting your contexts ready for query! Please wait!")
- hf, db = create_db(texts_raw, uploaded_file.name)
+ hf, db = create_db(_texts_raw, uploaded_file.name)
st.session_state["db_created"] = True
From cbd0b402b3de50efbd59806cd03490e6b08d3465 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 22 Feb 2024 15:46:29 +0530
Subject: [PATCH 053/129] streamlit_app.py
---
streamlit_app.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 8b01b22..ce74234 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -232,14 +232,14 @@ def calculate_rouge_scores(answer,context):
elif (".doc" in filename) and (".docx" not in filename): #uploaded_file.name:
all_text, text_split, text_chunk, headings, para_texts = readdoc_splittext(filename)#uploaded_file.name)
elif ".txt" in filename:
- all_text, text_split, text_raw, headings_list, paragraph_list = readdoc_splittext_txt(filename)#uploaded_file.name)
+ all_text, text_split, texts_raw, headings_list, paragraph_list = readdoc_splittext_txt(filename)#uploaded_file.name)
elif ".pdf" in filename: #uploaded_file.name:
- all_text, text_split, text_raw, headings_list, paragraph_list = readdoc_splittext_pdf(filename)#uploaded_file.name)
+ all_text, text_split, texts_raw, headings_list, paragraph_list = readdoc_splittext_pdf(filename)#uploaded_file.name)
with st.chat_message("assistant"):
st.write("Hi! Getting your contexts ready for query! Please wait!")
- hf, db = create_db(_texts_raw, uploaded_file.name)
+ hf, db = create_db(texts_raw, uploaded_file.name)
st.session_state["db_created"] = True
From 12f425c8adfd306578ab52a59afbb3d92c0af4ad Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 22 Feb 2024 15:47:29 +0530
Subject: [PATCH 054/129] streamlit_app.py
---
streamlit_app.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index ce74234..fe77b03 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -261,7 +261,7 @@ def calculate_rouge_scores(answer,context):
query = query_text
- context, keywords = create_context(query, text_split, headings, para_texts)
+ context, keywords = create_context(query, text_split, headings_list, para_texts)
# Generate a response from the chatbot
From 43d45f7bd8562246f895975dd9a6870209236c04 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 22 Feb 2024 15:48:34 +0530
Subject: [PATCH 055/129] streamlit_app.py
---
streamlit_app.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index fe77b03..4ccdd55 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -261,7 +261,7 @@ def calculate_rouge_scores(answer,context):
query = query_text
- context, keywords = create_context(query, text_split, headings_list, para_texts)
+ context, keywords = create_context(query, text_split, headings_list, paragraph_list)
# Generate a response from the chatbot
From dc52d8d6e47f3fcb268ef1defe5f04c5ce5f9ca2 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 22 Feb 2024 16:10:31 +0530
Subject: [PATCH 056/129] qa.py
---
qa.py | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 53 insertions(+)
diff --git a/qa.py b/qa.py
index b8dec38..dc85b99 100644
--- a/qa.py
+++ b/qa.py
@@ -315,6 +315,59 @@ def get_paragraphs(headings, paragraph_sentences):
return paragraph_list
+#---------------READ THE .PPTX FILE AND GENERATE THE SPLIT--------------------#
+@st.cache_resource(show_spinner=True)
+def readdoc_splittext_pptx(filename):
+ if ".pptx" in filename:
+ loader = UnstructuredPowerPointLoader(filename)
+ docs = str(loader.load())
+ block_dict = get_block_dict_fromDoc(docs)
+ span_df = get_docfeature_dataframe(block_dict)
+ docs_clean = span_df[span_df["font_size"]>=span_df["font_size"].mode()[0]]
+ docs_clean.head()
+ paragraphs = docs_clean.text[docs_clean.font_size == span_df.font_size.mode()[0]]
+ #print(paragraphs.values,paragraphs.index)
+ headings = docs_clean.text[docs_clean.font_size > span_df.font_size.mode()[0]]
+ #print(headings.values, headings.index)
+ #pat1= re.compile(r".+\:")
+ #pat2=re.compile(r".+\.\n")
+ #headings=pat1.search(docs)
+ #paragraphs=pat2.search(docs)
+ paragraph_list = get_paragraphs(headings, paragraphs)
+ headings_list = headings.values.tolist()
+ n = 1500 #Number of characters to be included in a single chunk of text
+ all_text=''
+ for text in paragraph_list:
+ all_text+=text
+ a=glob.glob(filename)
+ #print(a)
+ chunk_size = 1024
+ chunk_overlap=10
+ text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+ texts_isb=[]
+ texts_raw = []
+ documents=[]
+ for i in range(len(a)):
+ documents.extend(UnstructuredPowerPointLoader(a[i]).load())
+ for j in range(
+ len(
+ text_splitter.split_documents(
+ UnstructuredPowerPointLoader(a[i]).load()
+ )
+ )
+ ):
+ text_chunk = text_splitter.split_documents(
+ UnstructuredPowerPointLoader(a[i]).load()
+ )[j]
+ text_chunk.page_content = text_chunk.page_content.replace("\n", " ")
+ text_chunk.page_content = text_chunk.page_content.replace("\\n", " ")
+ text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
+ text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
+ texts_isb.append(text_chunk.page_content)
+ texts_raw.append(text_chunk)
+
+ text_split = texts_isb
+ return all_text, text_split, texts_raw, headings_list, paragraph_list
#---------------READ THE .TXT FILE AND GENERATE THE SPLIT--------------------#
@st.cache_resource(show_spinner=True)
From dc66b48613c29eb66abfc0274c857402a42fe1e1 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 22 Feb 2024 16:13:28 +0530
Subject: [PATCH 057/129] streamlit_app.py
---
streamlit_app.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/streamlit_app.py b/streamlit_app.py
index 4ccdd55..dc5dbf6 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -18,6 +18,7 @@
speechtotext,
readdoc_splittext,
readdoc_splittext_txt,
+ readdoc_splittext_pptx,
readdoc_splittext_pdf,
create_context,
create_db,
@@ -175,6 +176,7 @@ def calculate_rouge_scores(answer,context):
create_db.clear()
readdoc_splittext.clear()
readdoc_splittext_txt.clear()
+ readdoc_splittext_pptx.clear()
readdoc_splittext_pdf.clear()
if "query_counter" not in st.session_state:
@@ -202,6 +204,7 @@ def calculate_rouge_scores(answer,context):
create_db.clear()
readdoc_splittext.clear()
readdoc_splittext_txt.clear()
+ readdoc_splittext_pptx.clear()
readdoc_splittext_pdf.clear()
st.write("You can upload your document now.")
@@ -233,6 +236,8 @@ def calculate_rouge_scores(answer,context):
all_text, text_split, text_chunk, headings, para_texts = readdoc_splittext(filename)#uploaded_file.name)
elif ".txt" in filename:
all_text, text_split, texts_raw, headings_list, paragraph_list = readdoc_splittext_txt(filename)#uploaded_file.name)
+ elif ".pptx" in filename: #uploaded_file.name:
+ all_text, text_split, texts_raw, headings_list, paragraph_list = readdoc_splittext_pdf(filename)#uploaded_file.name)
elif ".pdf" in filename: #uploaded_file.name:
all_text, text_split, texts_raw, headings_list, paragraph_list = readdoc_splittext_pdf(filename)#uploaded_file.name)
From a3d8cf5228aaa06d316a5db2c6f24523ba00767a Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 22 Feb 2024 16:21:39 +0530
Subject: [PATCH 058/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index dc85b99..954289b 100644
--- a/qa.py
+++ b/qa.py
@@ -320,7 +320,7 @@ def get_paragraphs(headings, paragraph_sentences):
def readdoc_splittext_pptx(filename):
if ".pptx" in filename:
loader = UnstructuredPowerPointLoader(filename)
- docs = str(loader.load())
+ docs = loader.load()
block_dict = get_block_dict_fromDoc(docs)
span_df = get_docfeature_dataframe(block_dict)
docs_clean = span_df[span_df["font_size"]>=span_df["font_size"].mode()[0]]
From 3e961810417235d8020be29aafaa13c203161c98 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Thu, 22 Feb 2024 16:24:06 +0530
Subject: [PATCH 059/129] streamlit_app.py
---
streamlit_app.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index dc5dbf6..0290a0c 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -237,7 +237,7 @@ def calculate_rouge_scores(answer,context):
elif ".txt" in filename:
all_text, text_split, texts_raw, headings_list, paragraph_list = readdoc_splittext_txt(filename)#uploaded_file.name)
elif ".pptx" in filename: #uploaded_file.name:
- all_text, text_split, texts_raw, headings_list, paragraph_list = readdoc_splittext_pdf(filename)#uploaded_file.name)
+ all_text, text_split, texts_raw, headings_list, paragraph_list = readdoc_splittext_pptx(filename)#uploaded_file.name)
elif ".pdf" in filename: #uploaded_file.name:
all_text, text_split, texts_raw, headings_list, paragraph_list = readdoc_splittext_pdf(filename)#uploaded_file.name)
From 54bd5250775e566e9b5c95aa4ac56c510f4d785b Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Fri, 23 Feb 2024 16:50:47 +0530
Subject: [PATCH 060/129] qa.py
---
qa.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/qa.py b/qa.py
index 954289b..880e491 100644
--- a/qa.py
+++ b/qa.py
@@ -30,6 +30,7 @@
# -------DATA FRAME/DOCX/TEXT HANDLING----------$
import pandas as pd
import pprint as pp
+import python-pptx
from docx import Document as Docxreader
from docx.shared import Inches
import textwrap
From c12843ac166dcfc6d3da85a94de0ffe0f1635135 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Fri, 23 Feb 2024 16:51:25 +0530
Subject: [PATCH 061/129] requirements.txt
---
requirements.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/requirements.txt b/requirements.txt
index a6d398a..fa493b8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -18,6 +18,7 @@ requests
ffmpeg-python
mutagen
python-docx
+python-pptx
audio-recorder-streamlit
SpeechRecognition
streamlit-audiorecorder
From 2b320c61daadef96d356b779b378c3c779e347e7 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Fri, 23 Feb 2024 16:59:20 +0530
Subject: [PATCH 062/129] qa.py
---
qa.py | 39 +++++++++++++++++++--------------------
1 file changed, 19 insertions(+), 20 deletions(-)
diff --git a/qa.py b/qa.py
index 880e491..0fb6fb0 100644
--- a/qa.py
+++ b/qa.py
@@ -30,7 +30,7 @@
# -------DATA FRAME/DOCX/TEXT HANDLING----------$
import pandas as pd
import pprint as pp
-import python-pptx
+from pptx import Presentation
from docx import Document as Docxreader
from docx.shared import Inches
import textwrap
@@ -316,24 +316,31 @@ def get_paragraphs(headings, paragraph_sentences):
return paragraph_list
+def extract_text_from_pptx(filename):
+ text = ""
+ prs = Presentation(filename)
+ for slide in prs.slides:
+ for shape in slide.shapes:
+ if hasattr(shape, "text"):
+ text += shape.text + "\n"
+ return text
#---------------READ THE .PPTX FILE AND GENERATE THE SPLIT--------------------#
@st.cache_resource(show_spinner=True)
def readdoc_splittext_pptx(filename):
if ".pptx" in filename:
loader = UnstructuredPowerPointLoader(filename)
- docs = loader.load()
- block_dict = get_block_dict_fromDoc(docs)
+ docs = extract_text_from_pptx(filename)
+ block_dict = get_block_dict_fromDoc(doc)
span_df = get_docfeature_dataframe(block_dict)
- docs_clean = span_df[span_df["font_size"]>=span_df["font_size"].mode()[0]]
- docs_clean.head()
- paragraphs = docs_clean.text[docs_clean.font_size == span_df.font_size.mode()[0]]
- #print(paragraphs.values,paragraphs.index)
- headings = docs_clean.text[docs_clean.font_size > span_df.font_size.mode()[0]]
+ doc_clean = span_df[span_df["font_size"]>=span_df["font_size"].mode()[0]]
+ #doc_clean.head()
+ paragraphs = doc_clean.text[doc_clean.font_size == span_df.font_size.mode()[0]]
+ #print(paragraphs.values,paragraphs.index)
+ headings = doc_clean.text[doc_clean.font_size > span_df.font_size.mode()[0]]
#print(headings.values, headings.index)
- #pat1= re.compile(r".+\:")
- #pat2=re.compile(r".+\.\n")
- #headings=pat1.search(docs)
- #paragraphs=pat2.search(docs)
+ paragraph_list = get_paragraphs(headings, paragraphs)
+ headings_list = headings.values.tolist()
+
paragraph_list = get_paragraphs(headings, paragraphs)
headings_list = headings.values.tolist()
n = 1500 #Number of characters to be included in a single chunk of text
@@ -376,14 +383,6 @@ def readdoc_splittext_txt(filename):
if ".txt" in filename:
loader = UnstructuredFileLoader(filename)
docs = str(loader.load())
- #block_dict = get_block_dict_fromDoc(docs)
- #span_df = get_docfeature_dataframe(block_dict)
- #docs_clean = span_df[span_df["font_size"]>=span_df["font_size"].mode()[0]]
- #doc_clean.head()
- #paragraphs = docs_clean.text[docs_clean.font_size == span_df.font_size.mode()[0]]
- #print(paragraphs.values,paragraphs.index)
- #headings = docs_clean.text[docs_clean.font_size > span_df.font_size.mode()[0]]
- #print(headings.values, headings.index)
pat1= re.compile(r".+\:")
pat2=re.compile(r".+\.\n")
headings_list=pat1.findall(docs)
From f4486b2333c220864d1733c7f89abe236823c745 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Fri, 23 Feb 2024 17:00:24 +0530
Subject: [PATCH 063/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 0fb6fb0..b69b0d4 100644
--- a/qa.py
+++ b/qa.py
@@ -330,7 +330,7 @@ def readdoc_splittext_pptx(filename):
if ".pptx" in filename:
loader = UnstructuredPowerPointLoader(filename)
docs = extract_text_from_pptx(filename)
- block_dict = get_block_dict_fromDoc(doc)
+ block_dict = get_block_dict_fromDoc(docs)
span_df = get_docfeature_dataframe(block_dict)
doc_clean = span_df[span_df["font_size"]>=span_df["font_size"].mode()[0]]
#doc_clean.head()
From 9e6314f9f82131fdde7bad655f544fe280b9f6b5 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Fri, 23 Feb 2024 17:03:29 +0530
Subject: [PATCH 064/129] qa.py
---
qa.py | 17 ++++-------------
1 file changed, 4 insertions(+), 13 deletions(-)
diff --git a/qa.py b/qa.py
index b69b0d4..8046a4e 100644
--- a/qa.py
+++ b/qa.py
@@ -330,19 +330,10 @@ def readdoc_splittext_pptx(filename):
if ".pptx" in filename:
loader = UnstructuredPowerPointLoader(filename)
docs = extract_text_from_pptx(filename)
- block_dict = get_block_dict_fromDoc(docs)
- span_df = get_docfeature_dataframe(block_dict)
- doc_clean = span_df[span_df["font_size"]>=span_df["font_size"].mode()[0]]
- #doc_clean.head()
- paragraphs = doc_clean.text[doc_clean.font_size == span_df.font_size.mode()[0]]
- #print(paragraphs.values,paragraphs.index)
- headings = doc_clean.text[doc_clean.font_size > span_df.font_size.mode()[0]]
- #print(headings.values, headings.index)
- paragraph_list = get_paragraphs(headings, paragraphs)
- headings_list = headings.values.tolist()
-
- paragraph_list = get_paragraphs(headings, paragraphs)
- headings_list = headings.values.tolist()
+ pat1= re.compile(r".+\:")
+ pat2=re.compile(r".+\.\n")
+ headings_list=pat1.findall(docs)
+ paragraph_list=pat2.findall(docs)
n = 1500 #Number of characters to be included in a single chunk of text
all_text=''
for text in paragraph_list:
From 131d7c60441cec68e588458f81297a97d1521588 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Fri, 23 Feb 2024 17:37:46 +0530
Subject: [PATCH 065/129] qa.py
---
qa.py | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/qa.py b/qa.py
index 726d42c..e15abc1 100644
--- a/qa.py
+++ b/qa.py
@@ -324,12 +324,24 @@ def extract_text_from_pptx(filename):
if hasattr(shape, "text"):
text += shape.text + "\n"
return text
+def extract_image_addresses(filename):
+ image_addresses = []
+
+ prs = Presentation(filename)
+
+ for slide in prs.slides:
+ for shape in slide.shapes:
+ if shape.shape_type == 13: # Check if shape is an image
+ image_addresses.append(shape.image.anchor.photovalue.url)
+
+ return image_addresses
#---------------READ THE .PPTX FILE AND GENERATE THE SPLIT--------------------#
@st.cache_resource(show_spinner=True)
def readdoc_splittext_pptx(filename):
if ".pptx" in filename:
loader = UnstructuredPowerPointLoader(filename)
docs = extract_text_from_pptx(filename)
+ image_addresses = extract_image_addresses(filename)
pat1= re.compile(r".+\:")
pat2=re.compile(r".+\.\n")
headings_list=pat1.findall(docs)
From acf7bffe71d00cdba4d49f30c6fddae4f41c6e45 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Fri, 23 Feb 2024 17:38:08 +0530
Subject: [PATCH 066/129] requirements.txt
---
requirements.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/requirements.txt b/requirements.txt
index fa493b8..add2ed7 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -19,6 +19,7 @@ ffmpeg-python
mutagen
python-docx
python-pptx
+pytesseract
audio-recorder-streamlit
SpeechRecognition
streamlit-audiorecorder
From 7aaa0c255d8150d9578ee651c1906f95932a8cfd Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Fri, 23 Feb 2024 17:40:18 +0530
Subject: [PATCH 067/129] requirements.txt
---
requirements.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/requirements.txt b/requirements.txt
index add2ed7..924ad97 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -19,6 +19,7 @@ ffmpeg-python
mutagen
python-docx
python-pptx
+pillow
pytesseract
audio-recorder-streamlit
SpeechRecognition
From f6ce1837854259cdacfda82c99f388c52f54af55 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Fri, 23 Feb 2024 17:43:13 +0530
Subject: [PATCH 068/129] qa.py
---
qa.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/qa.py b/qa.py
index e15abc1..5aaff3d 100644
--- a/qa.py
+++ b/qa.py
@@ -31,6 +31,8 @@
import pandas as pd
import pprint as pp
from pptx import Presentation
+from PIL import Image
+import pytesseract
from docx import Document as Docxreader
from docx.shared import Inches
import textwrap
From 1c01a5297d033fdc32d6dc9b433998a0ca8c34ac Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Fri, 23 Feb 2024 18:11:48 +0530
Subject: [PATCH 069/129] qa.py
---
qa.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/qa.py b/qa.py
index 5aaff3d..d24bc2d 100644
--- a/qa.py
+++ b/qa.py
@@ -344,6 +344,9 @@ def readdoc_splittext_pptx(filename):
loader = UnstructuredPowerPointLoader(filename)
docs = extract_text_from_pptx(filename)
image_addresses = extract_image_addresses(filename)
+ for i in range(1,len(image_addresses)):
+ images_text=''
+ images_text=pytesseract.image_to_string(Image.open('image_addresses[i]'))
pat1= re.compile(r".+\:")
pat2=re.compile(r".+\.\n")
headings_list=pat1.findall(docs)
@@ -352,6 +355,7 @@ def readdoc_splittext_pptx(filename):
all_text=''
for text in paragraph_list:
all_text+=text
+ all_text+=images_text
a=glob.glob(filename)
#print(a)
chunk_size = 1024
From 91ed327c9de8f2de16de099edcfeed6b63e8bf11 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Fri, 23 Feb 2024 18:13:49 +0530
Subject: [PATCH 070/129] qa.py
---
qa.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/qa.py b/qa.py
index d24bc2d..7f26e98 100644
--- a/qa.py
+++ b/qa.py
@@ -344,8 +344,8 @@ def readdoc_splittext_pptx(filename):
loader = UnstructuredPowerPointLoader(filename)
docs = extract_text_from_pptx(filename)
image_addresses = extract_image_addresses(filename)
- for i in range(1,len(image_addresses)):
- images_text=''
+ images_text=''
+ for i in range(len(image_addresses)):
images_text=pytesseract.image_to_string(Image.open('image_addresses[i]'))
pat1= re.compile(r".+\:")
pat2=re.compile(r".+\.\n")
From 5cb2ef4939e9e851bf25e15aada303d0c7f0a5b1 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 06:12:23 +0530
Subject: [PATCH 071/129] requirements.txt
---
requirements.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/requirements.txt b/requirements.txt
index 924ad97..5668cf8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,6 +6,7 @@ beautifulsoup4==4.12.2
openai==0.27.7
openapi-schema-pydantic==1.2.4
streamlit
+google-cloud-vision
python-dotenv==1.0.0
accelerate==0.21.0
gradio==3.37.0
From a44660304583c34088214118feff1bc4e21cde39 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 07:00:47 +0530
Subject: [PATCH 072/129] qa.py
---
qa.py | 171 ++++++++++++++++++++++++++++++++++++++++++----------------
1 file changed, 125 insertions(+), 46 deletions(-)
diff --git a/qa.py b/qa.py
index 7f26e98..5a4eb77 100644
--- a/qa.py
+++ b/qa.py
@@ -35,6 +35,8 @@
import pytesseract
from docx import Document as Docxreader
from docx.shared import Inches
+from google.cloud import vision
+from google.oauth2 import service_account
import textwrap
import glob
from spire.doc import *
@@ -338,53 +340,130 @@ def extract_image_addresses(filename):
return image_addresses
#---------------READ THE .PPTX FILE AND GENERATE THE SPLIT--------------------#
-@st.cache_resource(show_spinner=True)
-def readdoc_splittext_pptx(filename):
- if ".pptx" in filename:
- loader = UnstructuredPowerPointLoader(filename)
- docs = extract_text_from_pptx(filename)
- image_addresses = extract_image_addresses(filename)
- images_text=''
- for i in range(len(image_addresses)):
- images_text=pytesseract.image_to_string(Image.open('image_addresses[i]'))
- pat1= re.compile(r".+\:")
- pat2=re.compile(r".+\.\n")
- headings_list=pat1.findall(docs)
- paragraph_list=pat2.findall(docs)
- n = 1500 #Number of characters to be included in a single chunk of text
- all_text=''
- for text in paragraph_list:
- all_text+=text
- all_text+=images_text
- a=glob.glob(filename)
- #print(a)
- chunk_size = 1024
- chunk_overlap=10
- text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
- texts_isb=[]
- texts_raw = []
- documents=[]
- for i in range(len(a)):
- documents.extend(UnstructuredPowerPointLoader(a[i]).load())
- for j in range(
- len(
- text_splitter.split_documents(
- UnstructuredPowerPointLoader(a[i]).load()
- )
- )
- ):
- text_chunk = text_splitter.split_documents(
- UnstructuredPowerPointLoader(a[i]).load()
- )[j]
- text_chunk.page_content = text_chunk.page_content.replace("\n", " ")
- text_chunk.page_content = text_chunk.page_content.replace("\\n", " ")
- text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
- text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
- texts_isb.append(text_chunk.page_content)
- texts_raw.append(text_chunk)
+#@st.cache_resource(show_spinner=True)
+#def readdoc_splittext_pptx(filename):
+ # if ".pptx" in filename:
+ # loader = UnstructuredPowerPointLoader(filename)
+ # docs = extract_text_from_pptx(filename)
+ #image_addresses = extract_image_addresses(filename)
+ # Authenticate using service account credentials
+ #credentials = service_account.Credentials.from_service_account_file('able-store-415222-3c73cfca4950.json') # Path to your JSON service account key file
+ #client = vision.ImageAnnotatorClient(credentials=credentials)
+ #images_text=''
+ # Perform OCR on an image
+ #for image_address in image_addresses:
+ # with io.open(image_address, 'rb') as image_file:
+ # content = image_file.read()
+ #image = vision.Image(content=content)
+ #response = client.text_detection(image=image)
+ #texts = response.text_annotations
+ #for text in texts:
+ # print(text.description)
+ #pat1= re.compile(r".+\:")
+ #pat2=re.compile(r".+\.\n")
+ #headings_list=pat1.findall(docs)
+ #paragraph_list=pat2.findall(docs)
+ #n = 1500 #Number of characters to be included in a single chunk of text
+ #all_text=''
+ #for text in paragraph_list:
+ # all_text+=text
+ #all_text+=images_text
+ #a=glob.glob(filename)
+ # #print(a)
+ #chunk_size = 1024
+ #chunk_overlap=10
+ #text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+ #texts_isb=[]
+ #texts_raw = []
+ #documents=[]
+ #for i in range(len(a)):
+ # documents.extend(UnstructuredPowerPointLoader(a[i]).load())
+ # for j in range(
+ # len(
+ # text_splitter.split_documents(
+ # UnstructuredPowerPointLoader(a[i]).load()
+ # )
+ # )
+ # ):
+ # text_chunk = text_splitter.split_documents(
+ # UnstructuredPowerPointLoader(a[i]).load()
+ # )[j]
+ # text_chunk.page_content = text_chunk.page_content.replace("\n", " ")
+ # text_chunk.page_content = text_chunk.page_content.replace("\\n", " ")
+ # text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
+ # text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
+ # texts_isb.append(text_chunk.page_content)
+ # texts_raw.append(text_chunk)
+
+# text_split = texts_isb
+# return all_text, text_split, texts_raw, headings_list, paragraph_list
- text_split = texts_isb
- return all_text, text_split, texts_raw, headings_list, paragraph_list
+def readdoc_splittext_pptx(filename):
+ if ".pptx" in filename:
+ loader = UnstructuredPowerPointLoader(filename)
+ docs = loader.extract_text()
+ image_addresses = loader.extract_image_addresses()
+
+ # Authenticate using service account credentials
+ credentials = service_account.Credentials.from_service_account_file('able-store-415222-3c73cfca4950.json')
+ client = vision.ImageAnnotatorClient(credentials=credentials)
+
+ images_text = ''
+ # Perform OCR on images
+ for image_address in image_addresses:
+ with io.open(image_address, 'rb') as image_file:
+ content = image_file.read()
+
+ image = vision.Image(content=content)
+ response = client.text_detection(image=image)
+ texts = response.text_annotations
+ for text in texts:
+ images_text += text.description + '\n'
+
+ # Extract headers and paragraphs from OCR results of images
+ pat1 = re.compile(r".+\:")
+ pat2 = re.compile(r".+\.\n")
+ headings_list_img = pat1.findall(text.description)
+ paragraph_list_img = pat2.findall(text.description)
+
+ # Append headers and paragraphs from images to existing lists
+ headings_list.extend(headings_list_img)
+ paragraph_list.extend(paragraph_list_img)
+
+ pat1 = re.compile(r".+\:")
+ pat2 = re.compile(r".+\.\n")
+ headings_list_pptx = pat1.findall(docs)
+ paragraph_list_pptx = pat2.findall(docs)
+
+ n = 1500 # Number of characters to be included in a single chunk of text
+ all_text = ''
+ for text in paragraph_list_pptx:
+ all_text += text
+ all_text += images_text
+
+ a = glob.glob(filename)
+
+ chunk_size = 1024
+ chunk_overlap = 10
+ text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+
+ texts_isb = []
+ texts_raw = []
+ documents = []
+ for i in range(len(a)):
+ documents.extend(loader.load())
+ for j in range(len(text_splitter.split_documents(loader.load()))):
+ text_chunk = text_splitter.split_documents(loader.load())[j]
+ text_chunk.page_content = text_chunk.page_content.replace("\n", " ")
+ text_chunk.page_content = text_chunk.page_content.replace("\\n", " ")
+ text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
+ text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
+ texts_isb.append(text_chunk.page_content)
+ texts_raw.append(text_chunk)
+
+ text_split = texts_isb
+
+ return all_text, text_split, texts_raw, headings_list_pptx, paragraph_list_pptx
#---------------READ THE .TXT FILE AND GENERATE THE SPLIT--------------------#
@st.cache_resource(show_spinner=True)
From 1a2def7bd5c2db696d45e81dcca8e8d048d82cc3 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 07:02:38 +0530
Subject: [PATCH 073/129] requirements.txt
---
requirements.txt | 2 ++
1 file changed, 2 insertions(+)
diff --git a/requirements.txt b/requirements.txt
index 5668cf8..5af0c45 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,6 +7,8 @@ openai==0.27.7
openapi-schema-pydantic==1.2.4
streamlit
google-cloud-vision
+google.cloud
+google.outh2
python-dotenv==1.0.0
accelerate==0.21.0
gradio==3.37.0
From 228c7aee59c610b534c8410ba0be2a336ff01045 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 07:03:44 +0530
Subject: [PATCH 074/129] requirements.txt
---
requirements.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/requirements.txt b/requirements.txt
index 5af0c45..36f6084 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,7 +8,7 @@ openapi-schema-pydantic==1.2.4
streamlit
google-cloud-vision
google.cloud
-google.outh2
+google.oauth2
python-dotenv==1.0.0
accelerate==0.21.0
gradio==3.37.0
From 0e08a31005a92aabc91c3762e079b4da0f888b48 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 07:07:34 +0530
Subject: [PATCH 075/129] requirements.txt
---
requirements.txt | 2 --
1 file changed, 2 deletions(-)
diff --git a/requirements.txt b/requirements.txt
index 36f6084..5668cf8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,8 +7,6 @@ openai==0.27.7
openapi-schema-pydantic==1.2.4
streamlit
google-cloud-vision
-google.cloud
-google.oauth2
python-dotenv==1.0.0
accelerate==0.21.0
gradio==3.37.0
From 3483e4e0ac5c273ba513ddb82381a5ef2fedc805 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 07:16:56 +0530
Subject: [PATCH 076/129] streamlit_app.py
---
streamlit_app.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 98f1b7f..ef956ef 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -241,7 +241,7 @@ def calculate_rouge_scores(answer,context):
elif ".txt" in filename:
all_text, text_split, texts_raw, headings_list, paragraph_list = readdoc_splittext_txt(filename)#uploaded_file.name)
elif ".pptx" in filename: #uploaded_file.name:
- all_text, text_split, texts_raw, headings_list, paragraph_list = readdoc_splittext_pptx(filename)#uploaded_file.name)
+ all_text, text_split, texts_raw, headings_list_pptx, paragraph_list_pptx = readdoc_splittext_pptx(filename)#uploaded_file.name)
elif ".pdf" in filename: #uploaded_file.name:
all_text, text_split, texts_raw, headings_list, paragraph_list = readdoc_splittext_pdf(filename)#uploaded_file.name)
From 169ca5984ed3bf04c7c1dcaf28f7c8f434496419 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 07:18:16 +0530
Subject: [PATCH 077/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 5a4eb77..8a73c41 100644
--- a/qa.py
+++ b/qa.py
@@ -397,7 +397,7 @@ def extract_image_addresses(filename):
# text_split = texts_isb
# return all_text, text_split, texts_raw, headings_list, paragraph_list
-
+@st.cache_resource(show_spinner=True)
def readdoc_splittext_pptx(filename):
if ".pptx" in filename:
loader = UnstructuredPowerPointLoader(filename)
From ec0b1785066ad6ecbfb290c78fe3df270b0c7482 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 07:19:58 +0530
Subject: [PATCH 078/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 8a73c41..285f691 100644
--- a/qa.py
+++ b/qa.py
@@ -401,7 +401,7 @@ def extract_image_addresses(filename):
def readdoc_splittext_pptx(filename):
if ".pptx" in filename:
loader = UnstructuredPowerPointLoader(filename)
- docs = loader.extract_text()
+ docs = loader.load()
image_addresses = loader.extract_image_addresses()
# Authenticate using service account credentials
From d5445ec73c6f18a0c0c1b5624ccd52dca3ed2f0e Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 07:21:13 +0530
Subject: [PATCH 079/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 285f691..957967c 100644
--- a/qa.py
+++ b/qa.py
@@ -402,7 +402,7 @@ def readdoc_splittext_pptx(filename):
if ".pptx" in filename:
loader = UnstructuredPowerPointLoader(filename)
docs = loader.load()
- image_addresses = loader.extract_image_addresses()
+ image_addresses = extract_image_addresses(filename)
# Authenticate using service account credentials
credentials = service_account.Credentials.from_service_account_file('able-store-415222-3c73cfca4950.json')
From bf700236e85f7b06762616bd04e9f3efef228a61 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 07:32:46 +0530
Subject: [PATCH 080/129] qa.py
---
qa.py | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/qa.py b/qa.py
index 957967c..39ed7f6 100644
--- a/qa.py
+++ b/qa.py
@@ -405,9 +405,12 @@ def readdoc_splittext_pptx(filename):
image_addresses = extract_image_addresses(filename)
# Authenticate using service account credentials
- credentials = service_account.Credentials.from_service_account_file('able-store-415222-3c73cfca4950.json')
- client = vision.ImageAnnotatorClient(credentials=credentials)
-
+ #credentials = service_account.Credentials.from_service_account_file('able-store-415222-3c73cfca4950.json')
+ #client = vision.ImageAnnotatorClient(credentials=credentials)
+ # Set the path to the service account key as an environment variable
+ os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "able-store-415222-3c73cfca4950.json"
+ # Load credentials from the environment variable
+ credentials = service_account.Credentials.from_service_account_info()
images_text = ''
# Perform OCR on images
for image_address in image_addresses:
From 74d6c786973a10167d7ee2d3c05f16d05d58499f Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 07:36:11 +0530
Subject: [PATCH 081/129] qa.py
---
qa.py | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 39ed7f6..99c3bc0 100644
--- a/qa.py
+++ b/qa.py
@@ -410,7 +410,19 @@ def readdoc_splittext_pptx(filename):
# Set the path to the service account key as an environment variable
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "able-store-415222-3c73cfca4950.json"
# Load credentials from the environment variable
- credentials = service_account.Credentials.from_service_account_info()
+ credentials = service_account.Credentials.from_service_account_info({
+ "type": "service_account",
+ "project_id": "able-store-415222",
+ "private_key_id": "3c73cfca4950749ff7d8b9411e28e7db84c89c75",
+ "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDZEH33tpJHp4XL\n3+jMloFGguZCzQuT8HwobVK7gI+x/fNfbamU2gzla+vP7GGmIP8JYn0S1cvWY0ki\nyakN7Hxp6DmkIeOpK9piojKriUFUFXu86JiaEmYRUplSeifOCexHM4KIgM4ABKc9\nogGVilJUdoLEigaMXF8/XS7O2uBRaApegUhSCZnYEp64VXVVyPoRtFAcKhmp5y4T\n+x58TjTX/7oEC6nWvfq1opJBzX++svH8f/eQ4FfpCh2WfyDyQk7GNyRxQMM9Xbnx\nwM9aWejuDN+yBufqWjoDI1wOqBNNDAuOr/Gdkpp8E5CqP/mBNwsK5LoMyWaAgrUq\nKkrE/M2fAgMBAAECggEAFXIer/m2zCOOAY2Wx5fbH+dQ7qqpiS1KZwgeE25pRBGu\nGPCRv6YnfgdCniFhqHM8H2Y+qzy6Te0zTZ+U9041IJWUFFZQ0ucFRZd24CHIdZFF\nz20/noFQ2kdJ5K8y/ZVXZdV+ARSQxcmv6FP4ShkHd1baJqbobLe3pWAEfuV7Ii+x\nlW026XwMvMZqGIn5Vw0ra4waJbexmGIHFy29pc7r8RTzgpxXEkSMi3JBPq4q6cfK\nV7ckoUw5qtvhFvhjTH+B2ipPII5+YxAo6MoxTwMrH3aZUrjTVjWzTQ3Fbb72Il77\nlKKAuzNNlNycN5T5cR3WR8rJ9lf2xfIqw9Jw44YDkQKBgQD+63n1aovkv8VLAJeJ\nbb/CRAU3ohrJpx3oRTSdwOkfVRITdJpPzxtZUun8xgFjmR50qptRNobZegbBF+FQ\nRjKu7L69UC0LEEBItn9ud+iQ64JNp+f2Wq7doTRqXHVZBANr3fSwZteQyqh2UHeF\nGQmVk7t35vIj9vXF/4sfAzhFpwKBgQDZ+/O9Y3CmyiHNX7sAJe4n//5W1VEW1nsC\nyO4+fIJPv7NPhJN8AeC//LltKKKm7C87Mc9JTkHQEtTeQoqW8/xlTwGpop068QmE\nVGWM9bfA1hLSgtwp8GoGrT92FX+12BKg5D6UxDslNDzXhrBqXGyVTqIhuPWy0zWw\nL1unWZenSQKBgQC1f2is8Dg8HMHWvcwmv+oo4Y8pZhRWgNLNXgCxVPlmIoalLX70\n1ctuFJeeLkgs5ocFn7bH4t+uFbCbaGo2YlSRsOO5HE2FtANhAfbG6z69d8Clk6eX\nAkfAapdMJxoxz6az9SrTMdXHNFMvMel27TWitrViEB9Ute+VEnW2Fe/JvwKBgAbx\nDg7+5qx5DWCD5umXS6E8drX/LwjiJaKuEWAuCNxPR3+FwkiVqrmSa7k9LQjRxqNE\n78vu0Qu2Pc8iIVWzSVtUi8ICKq2g1WPAaEd337UlXA4WGrq/LDEwPTAeeSWqTtWO\nzytfoF6L7lASuvV4IgETMviN8k0Sisgkie+nW7v5AoGBAMr4rftvEEUxOPuu+U7h\newe63BiLTU4F+OMGXhIHQIz88HdQk6w4XTj6Nrb2pivqCGPFNgTr4+WmMfe9DkAH\nxzny8Kdp9HUob6u8DrkWI2OLwKAnReVy3E3wd8MLBeZTahLHVc4vtpMH4L7cXpKv\nYlCS/pHOIfU6R5pPkSXj6ozZ\n-----END PRIVATE KEY-----\n",
+ "client_email": "speekar@able-store-415222.iam.gserviceaccount.com",
+ "client_id": "110531954629070375774",
+ "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+ "token_uri": "https://oauth2.googleapis.com/token",
+ "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+ "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/speekar%40able-store-415222.iam.gserviceaccount.com",
+ "universe_domain": "googleapis.com"
+ })
images_text = ''
# Perform OCR on images
for image_address in image_addresses:
From dda180fa6a229f99f5916f6975ef45f00c304d07 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 07:38:12 +0530
Subject: [PATCH 082/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 99c3bc0..16d807d 100644
--- a/qa.py
+++ b/qa.py
@@ -401,7 +401,7 @@ def extract_image_addresses(filename):
def readdoc_splittext_pptx(filename):
if ".pptx" in filename:
loader = UnstructuredPowerPointLoader(filename)
- docs = loader.load()
+ docs = str(loader.load())
image_addresses = extract_image_addresses(filename)
# Authenticate using service account credentials
From e7d0d721f88f98ae218dbf25465f599704013ee7 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 07:44:27 +0530
Subject: [PATCH 083/129] qa.py
---
qa.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 16d807d..fcb630f 100644
--- a/qa.py
+++ b/qa.py
@@ -424,6 +424,8 @@ def readdoc_splittext_pptx(filename):
"universe_domain": "googleapis.com"
})
images_text = ''
+ headings_list=[]
+ paragraph_list=[]
# Perform OCR on images
for image_address in image_addresses:
with io.open(image_address, 'rb') as image_file:
@@ -449,6 +451,8 @@ def readdoc_splittext_pptx(filename):
pat2 = re.compile(r".+\.\n")
headings_list_pptx = pat1.findall(docs)
paragraph_list_pptx = pat2.findall(docs)
+ headings_list.extend(headings_list_pptx)
+ paragraph_list.extend(paragraph_list_pptx)
n = 1500 # Number of characters to be included in a single chunk of text
all_text = ''
@@ -478,7 +482,7 @@ def readdoc_splittext_pptx(filename):
text_split = texts_isb
- return all_text, text_split, texts_raw, headings_list_pptx, paragraph_list_pptx
+ return all_text, text_split, texts_raw, headings_list, paragraph_list
#---------------READ THE .TXT FILE AND GENERATE THE SPLIT--------------------#
@st.cache_resource(show_spinner=True)
From 263d57fbe09f966f50a1c73997f940dc2c6ae514 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 07:45:24 +0530
Subject: [PATCH 084/129] streamlit_app.py
---
streamlit_app.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index ef956ef..98f1b7f 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -241,7 +241,7 @@ def calculate_rouge_scores(answer,context):
elif ".txt" in filename:
all_text, text_split, texts_raw, headings_list, paragraph_list = readdoc_splittext_txt(filename)#uploaded_file.name)
elif ".pptx" in filename: #uploaded_file.name:
- all_text, text_split, texts_raw, headings_list_pptx, paragraph_list_pptx = readdoc_splittext_pptx(filename)#uploaded_file.name)
+ all_text, text_split, texts_raw, headings_list, paragraph_list = readdoc_splittext_pptx(filename)#uploaded_file.name)
elif ".pdf" in filename: #uploaded_file.name:
all_text, text_split, texts_raw, headings_list, paragraph_list = readdoc_splittext_pdf(filename)#uploaded_file.name)
From f0aa9408239d0dc2fcade90c1b7af43e9c77d1d1 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 18:01:52 +0530
Subject: [PATCH 085/129] qa.py
---
qa.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index fcb630f..5acb291 100644
--- a/qa.py
+++ b/qa.py
@@ -336,7 +336,10 @@ def extract_image_addresses(filename):
for slide in prs.slides:
for shape in slide.shapes:
if shape.shape_type == 13: # Check if shape is an image
- image_addresses.append(shape.image.anchor.photovalue.url)
+ # Get the image address (file path or URL)
+ image_address = shape.image.filename
+ # Append the image address to the list
+ image_addresses.append(image_address)
return image_addresses
#---------------READ THE .PPTX FILE AND GENERATE THE SPLIT--------------------#
From cf93326de60d1c69a81c572d699f6f0f1d63df89 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 18:03:34 +0530
Subject: [PATCH 086/129] qa.py
---
qa.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/qa.py b/qa.py
index 5acb291..a617146 100644
--- a/qa.py
+++ b/qa.py
@@ -430,8 +430,8 @@ def readdoc_splittext_pptx(filename):
headings_list=[]
paragraph_list=[]
# Perform OCR on images
- for image_address in image_addresses:
- with io.open(image_address, 'rb') as image_file:
+ for i in range(image_addresses):
+ with io.open(image_addresses[i], 'rb') as image_file:
content = image_file.read()
image = vision.Image(content=content)
From 5ba90c73eeb3eb2a9166fa45b288e9dffc286e02 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 18:04:14 +0530
Subject: [PATCH 087/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index a617146..e46353c 100644
--- a/qa.py
+++ b/qa.py
@@ -430,7 +430,7 @@ def readdoc_splittext_pptx(filename):
headings_list=[]
paragraph_list=[]
# Perform OCR on images
- for i in range(image_addresses):
+ for i in range(len(image_addresses)):
with io.open(image_addresses[i], 'rb') as image_file:
content = image_file.read()
From 433a69ea1b44369f47ffbabb020363b02df0e2c7 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 18:14:54 +0530
Subject: [PATCH 088/129] qa.py
---
qa.py | 91 ++++++++++++++++++++++++++++++-----------------------------
1 file changed, 46 insertions(+), 45 deletions(-)
diff --git a/qa.py b/qa.py
index e46353c..33a7a3b 100644
--- a/qa.py
+++ b/qa.py
@@ -404,64 +404,65 @@ def extract_image_addresses(filename):
def readdoc_splittext_pptx(filename):
if ".pptx" in filename:
loader = UnstructuredPowerPointLoader(filename)
- docs = str(loader.load())
+ # Extract text from the PowerPoint presentation
+ docs = str(loader.load()) # Assuming loader.load() returns the text content of the presentation
+
+ # Extract image addresses from the PowerPoint presentation
image_addresses = extract_image_addresses(filename)
# Authenticate using service account credentials
- #credentials = service_account.Credentials.from_service_account_file('able-store-415222-3c73cfca4950.json')
- #client = vision.ImageAnnotatorClient(credentials=credentials)
- # Set the path to the service account key as an environment variable
- os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "able-store-415222-3c73cfca4950.json"
- # Load credentials from the environment variable
credentials = service_account.Credentials.from_service_account_info({
- "type": "service_account",
- "project_id": "able-store-415222",
- "private_key_id": "3c73cfca4950749ff7d8b9411e28e7db84c89c75",
- "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDZEH33tpJHp4XL\n3+jMloFGguZCzQuT8HwobVK7gI+x/fNfbamU2gzla+vP7GGmIP8JYn0S1cvWY0ki\nyakN7Hxp6DmkIeOpK9piojKriUFUFXu86JiaEmYRUplSeifOCexHM4KIgM4ABKc9\nogGVilJUdoLEigaMXF8/XS7O2uBRaApegUhSCZnYEp64VXVVyPoRtFAcKhmp5y4T\n+x58TjTX/7oEC6nWvfq1opJBzX++svH8f/eQ4FfpCh2WfyDyQk7GNyRxQMM9Xbnx\nwM9aWejuDN+yBufqWjoDI1wOqBNNDAuOr/Gdkpp8E5CqP/mBNwsK5LoMyWaAgrUq\nKkrE/M2fAgMBAAECggEAFXIer/m2zCOOAY2Wx5fbH+dQ7qqpiS1KZwgeE25pRBGu\nGPCRv6YnfgdCniFhqHM8H2Y+qzy6Te0zTZ+U9041IJWUFFZQ0ucFRZd24CHIdZFF\nz20/noFQ2kdJ5K8y/ZVXZdV+ARSQxcmv6FP4ShkHd1baJqbobLe3pWAEfuV7Ii+x\nlW026XwMvMZqGIn5Vw0ra4waJbexmGIHFy29pc7r8RTzgpxXEkSMi3JBPq4q6cfK\nV7ckoUw5qtvhFvhjTH+B2ipPII5+YxAo6MoxTwMrH3aZUrjTVjWzTQ3Fbb72Il77\nlKKAuzNNlNycN5T5cR3WR8rJ9lf2xfIqw9Jw44YDkQKBgQD+63n1aovkv8VLAJeJ\nbb/CRAU3ohrJpx3oRTSdwOkfVRITdJpPzxtZUun8xgFjmR50qptRNobZegbBF+FQ\nRjKu7L69UC0LEEBItn9ud+iQ64JNp+f2Wq7doTRqXHVZBANr3fSwZteQyqh2UHeF\nGQmVk7t35vIj9vXF/4sfAzhFpwKBgQDZ+/O9Y3CmyiHNX7sAJe4n//5W1VEW1nsC\nyO4+fIJPv7NPhJN8AeC//LltKKKm7C87Mc9JTkHQEtTeQoqW8/xlTwGpop068QmE\nVGWM9bfA1hLSgtwp8GoGrT92FX+12BKg5D6UxDslNDzXhrBqXGyVTqIhuPWy0zWw\nL1unWZenSQKBgQC1f2is8Dg8HMHWvcwmv+oo4Y8pZhRWgNLNXgCxVPlmIoalLX70\n1ctuFJeeLkgs5ocFn7bH4t+uFbCbaGo2YlSRsOO5HE2FtANhAfbG6z69d8Clk6eX\nAkfAapdMJxoxz6az9SrTMdXHNFMvMel27TWitrViEB9Ute+VEnW2Fe/JvwKBgAbx\nDg7+5qx5DWCD5umXS6E8drX/LwjiJaKuEWAuCNxPR3+FwkiVqrmSa7k9LQjRxqNE\n78vu0Qu2Pc8iIVWzSVtUi8ICKq2g1WPAaEd337UlXA4WGrq/LDEwPTAeeSWqTtWO\nzytfoF6L7lASuvV4IgETMviN8k0Sisgkie+nW7v5AoGBAMr4rftvEEUxOPuu+U7h\newe63BiLTU4F+OMGXhIHQIz88HdQk6w4XTj6Nrb2pivqCGPFNgTr4+WmMfe9DkAH\nxzny8Kdp9HUob6u8DrkWI2OLwKAnReVy3E3wd8MLBeZTahLHVc4vtpMH4L7cXpKv\nYlCS/pHOIfU6R5pPkSXj6ozZ\n-----END PRIVATE KEY-----\n",
- "client_email": "speekar@able-store-415222.iam.gserviceaccount.com",
- "client_id": "110531954629070375774",
- "auth_uri": "https://accounts.google.com/o/oauth2/auth",
- "token_uri": "https://oauth2.googleapis.com/token",
- "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
- "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/speekar%40able-store-415222.iam.gserviceaccount.com",
- "universe_domain": "googleapis.com"
+ "type": "service_account",
+ "project_id": "3c73cfca4950749ff7d8b9411e28e7db84c89c75",
+ "private_key_id": "your-private-key-id",
+ "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDZEH33tpJHp4XL\n3+jMloFGguZCzQuT8HwobVK7gI+x/fNfbamU2gzla+vP7GGmIP8JYn0S1cvWY0ki\nyakN7Hxp6DmkIeOpK9piojKriUFUFXu86JiaEmYRUplSeifOCexHM4KIgM4ABKc9\nogGVilJUdoLEigaMXF8/XS7O2uBRaApegUhSCZnYEp64VXVVyPoRtFAcKhmp5y4T\n+x58TjTX/7oEC6nWvfq1opJBzX++svH8f/eQ4FfpCh2WfyDyQk7GNyRxQMM9Xbnx\nwM9aWejuDN+yBufqWjoDI1wOqBNNDAuOr/Gdkpp8E5CqP/mBNwsK5LoMyWaAgrUq\nKkrE/M2fAgMBAAECggEAFXIer/m2zCOOAY2Wx5fbH+dQ7qqpiS1KZwgeE25pRBGu\nGPCRv6YnfgdCniFhqHM8H2Y+qzy6Te0zTZ+U9041IJWUFFZQ0ucFRZd24CHIdZFF\nz20/noFQ2kdJ5K8y/ZVXZdV+ARSQxcmv6FP4ShkHd1baJqbobLe3pWAEfuV7Ii+x\nlW026XwMvMZqGIn5Vw0ra4waJbexmGIHFy29pc7r8RTzgpxXEkSMi3JBPq4q6cfK\nV7ckoUw5qtvhFvhjTH+B2ipPII5+YxAo6MoxTwMrH3aZUrjTVjWzTQ3Fbb72Il77\nlKKAuzNNlNycN5T5cR3WR8rJ9lf2xfIqw9Jw44YDkQKBgQD+63n1aovkv8VLAJeJ\nbb/CRAU3ohrJpx3oRTSdwOkfVRITdJpPzxtZUun8xgFjmR50qptRNobZegbBF+FQ\nRjKu7L69UC0LEEBItn9ud+iQ64JNp+f2Wq7doTRqXHVZBANr3fSwZteQyqh2UHeF\nGQmVk7t35vIj9vXF/4sfAzhFpwKBgQDZ+/O9Y3CmyiHNX7sAJe4n//5W1VEW1nsC\nyO4+fIJPv7NPhJN8AeC//LltKKKm7C87Mc9JTkHQEtTeQoqW8/xlTwGpop068QmE\nVGWM9bfA1hLSgtwp8GoGrT92FX+12BKg5D6UxDslNDzXhrBqXGyVTqIhuPWy0zWw\nL1unWZenSQKBgQC1f2is8Dg8HMHWvcwmv+oo4Y8pZhRWgNLNXgCxVPlmIoalLX70\n1ctuFJeeLkgs5ocFn7bH4t+uFbCbaGo2YlSRsOO5HE2FtANhAfbG6z69d8Clk6eX\nAkfAapdMJxoxz6az9SrTMdXHNFMvMel27TWitrViEB9Ute+VEnW2Fe/JvwKBgAbx\nDg7+5qx5DWCD5umXS6E8drX/LwjiJaKuEWAuCNxPR3+FwkiVqrmSa7k9LQjRxqNE\n78vu0Qu2Pc8iIVWzSVtUi8ICKq2g1WPAaEd337UlXA4WGrq/LDEwPTAeeSWqTtWO\nzytfoF6L7lASuvV4IgETMviN8k0Sisgkie+nW7v5AoGBAMr4rftvEEUxOPuu+U7h\newe63BiLTU4F+OMGXhIHQIz88HdQk6w4XTj6Nrb2pivqCGPFNgTr4+WmMfe9DkAH\nxzny8Kdp9HUob6u8DrkWI2OLwKAnReVy3E3wd8MLBeZTahLHVc4vtpMH4L7cXpKv\nYlCS/pHOIfU6R5pPkSXj6ozZ\n-----END PRIVATE KEY-----\n",,
+ "client_email": "your-client-email",
+ "client_id": "your-client-id",
+ "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+ "token_uri": "https://oauth2.googleapis.com/token",
+ "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+ "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/your-client-email"
})
+
images_text = ''
- headings_list=[]
- paragraph_list=[]
- # Perform OCR on images
- for i in range(len(image_addresses)):
- with io.open(image_addresses[i], 'rb') as image_file:
- content = image_file.read()
-
- image = vision.Image(content=content)
- response = client.text_detection(image=image)
- texts = response.text_annotations
- for text in texts:
- images_text += text.description + '\n'
-
- # Extract headers and paragraphs from OCR results of images
- pat1 = re.compile(r".+\:")
- pat2 = re.compile(r".+\.\n")
- headings_list_img = pat1.findall(text.description)
- paragraph_list_img = pat2.findall(text.description)
-
- # Append headers and paragraphs from images to existing lists
- headings_list.extend(headings_list_img)
- paragraph_list.extend(paragraph_list_img)
+ headings_list = []
+ paragraph_list = []
+ # Initialize Google Cloud Vision API client
+ client = vision.ImageAnnotatorClient(credentials=credentials)
+
+ # Perform OCR on images
+ for image_address in image_addresses:
+ if os.path.exists(image_address): # Check if the image file exists
+ with io.open(image_address, 'rb') as image_file:
+ content = image_file.read()
+
+ image = vision.Image(content=content)
+ response = client.text_detection(image=image)
+ texts = response.text_annotations
+ for text in texts:
+ images_text += text.description + '\n'
+
+ # Extract headers and paragraphs from OCR results of images
+ pat1 = re.compile(r".+\:")
+ pat2 = re.compile(r".+\.\n")
+ headings_list_img = pat1.findall(text.description)
+ paragraph_list_img = pat2.findall(text.description)
+
+ # Append headers and paragraphs from images to existing lists
+ headings_list.extend(headings_list_img)
+ paragraph_list.extend(paragraph_list_img)
+
+ # Extract headers and paragraphs from the PowerPoint text
pat1 = re.compile(r".+\:")
pat2 = re.compile(r".+\.\n")
headings_list_pptx = pat1.findall(docs)
paragraph_list_pptx = pat2.findall(docs)
headings_list.extend(headings_list_pptx)
paragraph_list.extend(paragraph_list_pptx)
-
- n = 1500 # Number of characters to be included in a single chunk of text
- all_text = ''
- for text in paragraph_list_pptx:
- all_text += text
- all_text += images_text
+ n = 1500
+ # Combine all text from PowerPoint and OCR'd images
+ all_text = ''.join(paragraph_list_pptx) + images_text
a = glob.glob(filename)
From 9a263cb9359f2913255c9e463ff9c9494e8887a8 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 18:15:48 +0530
Subject: [PATCH 089/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 33a7a3b..ddf5d06 100644
--- a/qa.py
+++ b/qa.py
@@ -415,7 +415,7 @@ def readdoc_splittext_pptx(filename):
"type": "service_account",
"project_id": "3c73cfca4950749ff7d8b9411e28e7db84c89c75",
"private_key_id": "your-private-key-id",
- "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDZEH33tpJHp4XL\n3+jMloFGguZCzQuT8HwobVK7gI+x/fNfbamU2gzla+vP7GGmIP8JYn0S1cvWY0ki\nyakN7Hxp6DmkIeOpK9piojKriUFUFXu86JiaEmYRUplSeifOCexHM4KIgM4ABKc9\nogGVilJUdoLEigaMXF8/XS7O2uBRaApegUhSCZnYEp64VXVVyPoRtFAcKhmp5y4T\n+x58TjTX/7oEC6nWvfq1opJBzX++svH8f/eQ4FfpCh2WfyDyQk7GNyRxQMM9Xbnx\nwM9aWejuDN+yBufqWjoDI1wOqBNNDAuOr/Gdkpp8E5CqP/mBNwsK5LoMyWaAgrUq\nKkrE/M2fAgMBAAECggEAFXIer/m2zCOOAY2Wx5fbH+dQ7qqpiS1KZwgeE25pRBGu\nGPCRv6YnfgdCniFhqHM8H2Y+qzy6Te0zTZ+U9041IJWUFFZQ0ucFRZd24CHIdZFF\nz20/noFQ2kdJ5K8y/ZVXZdV+ARSQxcmv6FP4ShkHd1baJqbobLe3pWAEfuV7Ii+x\nlW026XwMvMZqGIn5Vw0ra4waJbexmGIHFy29pc7r8RTzgpxXEkSMi3JBPq4q6cfK\nV7ckoUw5qtvhFvhjTH+B2ipPII5+YxAo6MoxTwMrH3aZUrjTVjWzTQ3Fbb72Il77\nlKKAuzNNlNycN5T5cR3WR8rJ9lf2xfIqw9Jw44YDkQKBgQD+63n1aovkv8VLAJeJ\nbb/CRAU3ohrJpx3oRTSdwOkfVRITdJpPzxtZUun8xgFjmR50qptRNobZegbBF+FQ\nRjKu7L69UC0LEEBItn9ud+iQ64JNp+f2Wq7doTRqXHVZBANr3fSwZteQyqh2UHeF\nGQmVk7t35vIj9vXF/4sfAzhFpwKBgQDZ+/O9Y3CmyiHNX7sAJe4n//5W1VEW1nsC\nyO4+fIJPv7NPhJN8AeC//LltKKKm7C87Mc9JTkHQEtTeQoqW8/xlTwGpop068QmE\nVGWM9bfA1hLSgtwp8GoGrT92FX+12BKg5D6UxDslNDzXhrBqXGyVTqIhuPWy0zWw\nL1unWZenSQKBgQC1f2is8Dg8HMHWvcwmv+oo4Y8pZhRWgNLNXgCxVPlmIoalLX70\n1ctuFJeeLkgs5ocFn7bH4t+uFbCbaGo2YlSRsOO5HE2FtANhAfbG6z69d8Clk6eX\nAkfAapdMJxoxz6az9SrTMdXHNFMvMel27TWitrViEB9Ute+VEnW2Fe/JvwKBgAbx\nDg7+5qx5DWCD5umXS6E8drX/LwjiJaKuEWAuCNxPR3+FwkiVqrmSa7k9LQjRxqNE\n78vu0Qu2Pc8iIVWzSVtUi8ICKq2g1WPAaEd337UlXA4WGrq/LDEwPTAeeSWqTtWO\nzytfoF6L7lASuvV4IgETMviN8k0Sisgkie+nW7v5AoGBAMr4rftvEEUxOPuu+U7h\newe63BiLTU4F+OMGXhIHQIz88HdQk6w4XTj6Nrb2pivqCGPFNgTr4+WmMfe9DkAH\nxzny8Kdp9HUob6u8DrkWI2OLwKAnReVy3E3wd8MLBeZTahLHVc4vtpMH4L7cXpKv\nYlCS/pHOIfU6R5pPkSXj6ozZ\n-----END PRIVATE KEY-----\n",,
+ "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDZEH33tpJHp4XL\n3+jMloFGguZCzQuT8HwobVK7gI+x/fNfbamU2gzla+vP7GGmIP8JYn0S1cvWY0ki\nyakN7Hxp6DmkIeOpK9piojKriUFUFXu86JiaEmYRUplSeifOCexHM4KIgM4ABKc9\nogGVilJUdoLEigaMXF8/XS7O2uBRaApegUhSCZnYEp64VXVVyPoRtFAcKhmp5y4T\n+x58TjTX/7oEC6nWvfq1opJBzX++svH8f/eQ4FfpCh2WfyDyQk7GNyRxQMM9Xbnx\nwM9aWejuDN+yBufqWjoDI1wOqBNNDAuOr/Gdkpp8E5CqP/mBNwsK5LoMyWaAgrUq\nKkrE/M2fAgMBAAECggEAFXIer/m2zCOOAY2Wx5fbH+dQ7qqpiS1KZwgeE25pRBGu\nGPCRv6YnfgdCniFhqHM8H2Y+qzy6Te0zTZ+U9041IJWUFFZQ0ucFRZd24CHIdZFF\nz20/noFQ2kdJ5K8y/ZVXZdV+ARSQxcmv6FP4ShkHd1baJqbobLe3pWAEfuV7Ii+x\nlW026XwMvMZqGIn5Vw0ra4waJbexmGIHFy29pc7r8RTzgpxXEkSMi3JBPq4q6cfK\nV7ckoUw5qtvhFvhjTH+B2ipPII5+YxAo6MoxTwMrH3aZUrjTVjWzTQ3Fbb72Il77\nlKKAuzNNlNycN5T5cR3WR8rJ9lf2xfIqw9Jw44YDkQKBgQD+63n1aovkv8VLAJeJ\nbb/CRAU3ohrJpx3oRTSdwOkfVRITdJpPzxtZUun8xgFjmR50qptRNobZegbBF+FQ\nRjKu7L69UC0LEEBItn9ud+iQ64JNp+f2Wq7doTRqXHVZBANr3fSwZteQyqh2UHeF\nGQmVk7t35vIj9vXF/4sfAzhFpwKBgQDZ+/O9Y3CmyiHNX7sAJe4n//5W1VEW1nsC\nyO4+fIJPv7NPhJN8AeC//LltKKKm7C87Mc9JTkHQEtTeQoqW8/xlTwGpop068QmE\nVGWM9bfA1hLSgtwp8GoGrT92FX+12BKg5D6UxDslNDzXhrBqXGyVTqIhuPWy0zWw\nL1unWZenSQKBgQC1f2is8Dg8HMHWvcwmv+oo4Y8pZhRWgNLNXgCxVPlmIoalLX70\n1ctuFJeeLkgs5ocFn7bH4t+uFbCbaGo2YlSRsOO5HE2FtANhAfbG6z69d8Clk6eX\nAkfAapdMJxoxz6az9SrTMdXHNFMvMel27TWitrViEB9Ute+VEnW2Fe/JvwKBgAbx\nDg7+5qx5DWCD5umXS6E8drX/LwjiJaKuEWAuCNxPR3+FwkiVqrmSa7k9LQjRxqNE\n78vu0Qu2Pc8iIVWzSVtUi8ICKq2g1WPAaEd337UlXA4WGrq/LDEwPTAeeSWqTtWO\nzytfoF6L7lASuvV4IgETMviN8k0Sisgkie+nW7v5AoGBAMr4rftvEEUxOPuu+U7h\newe63BiLTU4F+OMGXhIHQIz88HdQk6w4XTj6Nrb2pivqCGPFNgTr4+WmMfe9DkAH\nxzny8Kdp9HUob6u8DrkWI2OLwKAnReVy3E3wd8MLBeZTahLHVc4vtpMH4L7cXpKv\nYlCS/pHOIfU6R5pPkSXj6ozZ\n-----END PRIVATE KEY-----\n",
"client_email": "your-client-email",
"client_id": "your-client-id",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
From d355193e0780088f8a5f704b3488e0faa9267b71 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 18:20:44 +0530
Subject: [PATCH 090/129] qa.py
---
qa.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/qa.py b/qa.py
index ddf5d06..915353f 100644
--- a/qa.py
+++ b/qa.py
@@ -413,8 +413,8 @@ def readdoc_splittext_pptx(filename):
# Authenticate using service account credentials
credentials = service_account.Credentials.from_service_account_info({
"type": "service_account",
- "project_id": "3c73cfca4950749ff7d8b9411e28e7db84c89c75",
- "private_key_id": "your-private-key-id",
+ "project_id": "able-store-415222",
+ "private_key_id": "3c73cfca4950749ff7d8b9411e28e7db84c89c75",
"private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDZEH33tpJHp4XL\n3+jMloFGguZCzQuT8HwobVK7gI+x/fNfbamU2gzla+vP7GGmIP8JYn0S1cvWY0ki\nyakN7Hxp6DmkIeOpK9piojKriUFUFXu86JiaEmYRUplSeifOCexHM4KIgM4ABKc9\nogGVilJUdoLEigaMXF8/XS7O2uBRaApegUhSCZnYEp64VXVVyPoRtFAcKhmp5y4T\n+x58TjTX/7oEC6nWvfq1opJBzX++svH8f/eQ4FfpCh2WfyDyQk7GNyRxQMM9Xbnx\nwM9aWejuDN+yBufqWjoDI1wOqBNNDAuOr/Gdkpp8E5CqP/mBNwsK5LoMyWaAgrUq\nKkrE/M2fAgMBAAECggEAFXIer/m2zCOOAY2Wx5fbH+dQ7qqpiS1KZwgeE25pRBGu\nGPCRv6YnfgdCniFhqHM8H2Y+qzy6Te0zTZ+U9041IJWUFFZQ0ucFRZd24CHIdZFF\nz20/noFQ2kdJ5K8y/ZVXZdV+ARSQxcmv6FP4ShkHd1baJqbobLe3pWAEfuV7Ii+x\nlW026XwMvMZqGIn5Vw0ra4waJbexmGIHFy29pc7r8RTzgpxXEkSMi3JBPq4q6cfK\nV7ckoUw5qtvhFvhjTH+B2ipPII5+YxAo6MoxTwMrH3aZUrjTVjWzTQ3Fbb72Il77\nlKKAuzNNlNycN5T5cR3WR8rJ9lf2xfIqw9Jw44YDkQKBgQD+63n1aovkv8VLAJeJ\nbb/CRAU3ohrJpx3oRTSdwOkfVRITdJpPzxtZUun8xgFjmR50qptRNobZegbBF+FQ\nRjKu7L69UC0LEEBItn9ud+iQ64JNp+f2Wq7doTRqXHVZBANr3fSwZteQyqh2UHeF\nGQmVk7t35vIj9vXF/4sfAzhFpwKBgQDZ+/O9Y3CmyiHNX7sAJe4n//5W1VEW1nsC\nyO4+fIJPv7NPhJN8AeC//LltKKKm7C87Mc9JTkHQEtTeQoqW8/xlTwGpop068QmE\nVGWM9bfA1hLSgtwp8GoGrT92FX+12BKg5D6UxDslNDzXhrBqXGyVTqIhuPWy0zWw\nL1unWZenSQKBgQC1f2is8Dg8HMHWvcwmv+oo4Y8pZhRWgNLNXgCxVPlmIoalLX70\n1ctuFJeeLkgs5ocFn7bH4t+uFbCbaGo2YlSRsOO5HE2FtANhAfbG6z69d8Clk6eX\nAkfAapdMJxoxz6az9SrTMdXHNFMvMel27TWitrViEB9Ute+VEnW2Fe/JvwKBgAbx\nDg7+5qx5DWCD5umXS6E8drX/LwjiJaKuEWAuCNxPR3+FwkiVqrmSa7k9LQjRxqNE\n78vu0Qu2Pc8iIVWzSVtUi8ICKq2g1WPAaEd337UlXA4WGrq/LDEwPTAeeSWqTtWO\nzytfoF6L7lASuvV4IgETMviN8k0Sisgkie+nW7v5AoGBAMr4rftvEEUxOPuu+U7h\newe63BiLTU4F+OMGXhIHQIz88HdQk6w4XTj6Nrb2pivqCGPFNgTr4+WmMfe9DkAH\nxzny8Kdp9HUob6u8DrkWI2OLwKAnReVy3E3wd8MLBeZTahLHVc4vtpMH4L7cXpKv\nYlCS/pHOIfU6R5pPkSXj6ozZ\n-----END PRIVATE KEY-----\n",
"client_email": "your-client-email",
"client_id": "your-client-id",
From 9c34bd03e06630a4a27eac223296d8be76c3ee21 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 18:23:57 +0530
Subject: [PATCH 091/129] qa.py
---
qa.py | 23 ++++++++++++-----------
1 file changed, 12 insertions(+), 11 deletions(-)
diff --git a/qa.py b/qa.py
index 915353f..437ba83 100644
--- a/qa.py
+++ b/qa.py
@@ -412,17 +412,18 @@ def readdoc_splittext_pptx(filename):
# Authenticate using service account credentials
credentials = service_account.Credentials.from_service_account_info({
- "type": "service_account",
- "project_id": "able-store-415222",
- "private_key_id": "3c73cfca4950749ff7d8b9411e28e7db84c89c75",
- "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDZEH33tpJHp4XL\n3+jMloFGguZCzQuT8HwobVK7gI+x/fNfbamU2gzla+vP7GGmIP8JYn0S1cvWY0ki\nyakN7Hxp6DmkIeOpK9piojKriUFUFXu86JiaEmYRUplSeifOCexHM4KIgM4ABKc9\nogGVilJUdoLEigaMXF8/XS7O2uBRaApegUhSCZnYEp64VXVVyPoRtFAcKhmp5y4T\n+x58TjTX/7oEC6nWvfq1opJBzX++svH8f/eQ4FfpCh2WfyDyQk7GNyRxQMM9Xbnx\nwM9aWejuDN+yBufqWjoDI1wOqBNNDAuOr/Gdkpp8E5CqP/mBNwsK5LoMyWaAgrUq\nKkrE/M2fAgMBAAECggEAFXIer/m2zCOOAY2Wx5fbH+dQ7qqpiS1KZwgeE25pRBGu\nGPCRv6YnfgdCniFhqHM8H2Y+qzy6Te0zTZ+U9041IJWUFFZQ0ucFRZd24CHIdZFF\nz20/noFQ2kdJ5K8y/ZVXZdV+ARSQxcmv6FP4ShkHd1baJqbobLe3pWAEfuV7Ii+x\nlW026XwMvMZqGIn5Vw0ra4waJbexmGIHFy29pc7r8RTzgpxXEkSMi3JBPq4q6cfK\nV7ckoUw5qtvhFvhjTH+B2ipPII5+YxAo6MoxTwMrH3aZUrjTVjWzTQ3Fbb72Il77\nlKKAuzNNlNycN5T5cR3WR8rJ9lf2xfIqw9Jw44YDkQKBgQD+63n1aovkv8VLAJeJ\nbb/CRAU3ohrJpx3oRTSdwOkfVRITdJpPzxtZUun8xgFjmR50qptRNobZegbBF+FQ\nRjKu7L69UC0LEEBItn9ud+iQ64JNp+f2Wq7doTRqXHVZBANr3fSwZteQyqh2UHeF\nGQmVk7t35vIj9vXF/4sfAzhFpwKBgQDZ+/O9Y3CmyiHNX7sAJe4n//5W1VEW1nsC\nyO4+fIJPv7NPhJN8AeC//LltKKKm7C87Mc9JTkHQEtTeQoqW8/xlTwGpop068QmE\nVGWM9bfA1hLSgtwp8GoGrT92FX+12BKg5D6UxDslNDzXhrBqXGyVTqIhuPWy0zWw\nL1unWZenSQKBgQC1f2is8Dg8HMHWvcwmv+oo4Y8pZhRWgNLNXgCxVPlmIoalLX70\n1ctuFJeeLkgs5ocFn7bH4t+uFbCbaGo2YlSRsOO5HE2FtANhAfbG6z69d8Clk6eX\nAkfAapdMJxoxz6az9SrTMdXHNFMvMel27TWitrViEB9Ute+VEnW2Fe/JvwKBgAbx\nDg7+5qx5DWCD5umXS6E8drX/LwjiJaKuEWAuCNxPR3+FwkiVqrmSa7k9LQjRxqNE\n78vu0Qu2Pc8iIVWzSVtUi8ICKq2g1WPAaEd337UlXA4WGrq/LDEwPTAeeSWqTtWO\nzytfoF6L7lASuvV4IgETMviN8k0Sisgkie+nW7v5AoGBAMr4rftvEEUxOPuu+U7h\newe63BiLTU4F+OMGXhIHQIz88HdQk6w4XTj6Nrb2pivqCGPFNgTr4+WmMfe9DkAH\nxzny8Kdp9HUob6u8DrkWI2OLwKAnReVy3E3wd8MLBeZTahLHVc4vtpMH4L7cXpKv\nYlCS/pHOIfU6R5pPkSXj6ozZ\n-----END PRIVATE KEY-----\n",
- "client_email": "your-client-email",
- "client_id": "your-client-id",
- "auth_uri": "https://accounts.google.com/o/oauth2/auth",
- "token_uri": "https://oauth2.googleapis.com/token",
- "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
- "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/your-client-email"
- })
+ "type": "service_account",
+ "project_id": "able-store-415222",
+ "private_key_id": "3c73cfca4950749ff7d8b9411e28e7db84c89c75",
+ "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDZEH33tpJHp4XL\n3+jMloFGguZCzQuT8HwobVK7gI+x/fNfbamU2gzla+vP7GGmIP8JYn0S1cvWY0ki\nyakN7Hxp6DmkIeOpK9piojKriUFUFXu86JiaEmYRUplSeifOCexHM4KIgM4ABKc9\nogGVilJUdoLEigaMXF8/XS7O2uBRaApegUhSCZnYEp64VXVVyPoRtFAcKhmp5y4T\n+x58TjTX/7oEC6nWvfq1opJBzX++svH8f/eQ4FfpCh2WfyDyQk7GNyRxQMM9Xbnx\nwM9aWejuDN+yBufqWjoDI1wOqBNNDAuOr/Gdkpp8E5CqP/mBNwsK5LoMyWaAgrUq\nKkrE/M2fAgMBAAECggEAFXIer/m2zCOOAY2Wx5fbH+dQ7qqpiS1KZwgeE25pRBGu\nGPCRv6YnfgdCniFhqHM8H2Y+qzy6Te0zTZ+U9041IJWUFFZQ0ucFRZd24CHIdZFF\nz20/noFQ2kdJ5K8y/ZVXZdV+ARSQxcmv6FP4ShkHd1baJqbobLe3pWAEfuV7Ii+x\nlW026XwMvMZqGIn5Vw0ra4waJbexmGIHFy29pc7r8RTzgpxXEkSMi3JBPq4q6cfK\nV7ckoUw5qtvhFvhjTH+B2ipPII5+YxAo6MoxTwMrH3aZUrjTVjWzTQ3Fbb72Il77\nlKKAuzNNlNycN5T5cR3WR8rJ9lf2xfIqw9Jw44YDkQKBgQD+63n1aovkv8VLAJeJ\nbb/CRAU3ohrJpx3oRTSdwOkfVRITdJpPzxtZUun8xgFjmR50qptRNobZegbBF+FQ\nRjKu7L69UC0LEEBItn9ud+iQ64JNp+f2Wq7doTRqXHVZBANr3fSwZteQyqh2UHeF\nGQmVk7t35vIj9vXF/4sfAzhFpwKBgQDZ+/O9Y3CmyiHNX7sAJe4n//5W1VEW1nsC\nyO4+fIJPv7NPhJN8AeC//LltKKKm7C87Mc9JTkHQEtTeQoqW8/xlTwGpop068QmE\nVGWM9bfA1hLSgtwp8GoGrT92FX+12BKg5D6UxDslNDzXhrBqXGyVTqIhuPWy0zWw\nL1unWZenSQKBgQC1f2is8Dg8HMHWvcwmv+oo4Y8pZhRWgNLNXgCxVPlmIoalLX70\n1ctuFJeeLkgs5ocFn7bH4t+uFbCbaGo2YlSRsOO5HE2FtANhAfbG6z69d8Clk6eX\nAkfAapdMJxoxz6az9SrTMdXHNFMvMel27TWitrViEB9Ute+VEnW2Fe/JvwKBgAbx\nDg7+5qx5DWCD5umXS6E8drX/LwjiJaKuEWAuCNxPR3+FwkiVqrmSa7k9LQjRxqNE\n78vu0Qu2Pc8iIVWzSVtUi8ICKq2g1WPAaEd337UlXA4WGrq/LDEwPTAeeSWqTtWO\nzytfoF6L7lASuvV4IgETMviN8k0Sisgkie+nW7v5AoGBAMr4rftvEEUxOPuu+U7h\newe63BiLTU4F+OMGXhIHQIz88HdQk6w4XTj6Nrb2pivqCGPFNgTr4+WmMfe9DkAH\nxzny8Kdp9HUob6u8DrkWI2OLwKAnReVy3E3wd8MLBeZTahLHVc4vtpMH4L7cXpKv\nYlCS/pHOIfU6R5pPkSXj6ozZ\n-----END PRIVATE KEY-----\n",
+ "client_email": "speekar@able-store-415222.iam.gserviceaccount.com",
+ "client_id": "110531954629070375774",
+ "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+ "token_uri": "https://oauth2.googleapis.com/token",
+ "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+ "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/speekar%40able-store-415222.iam.gserviceaccount.com",
+ "universe_domain": "googleapis.com"
+})
images_text = ''
headings_list = []
From d666bc26172be831483b22bbf9f1e523d676742a Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 18:25:38 +0530
Subject: [PATCH 092/129] qa.py
---
qa.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/qa.py b/qa.py
index 437ba83..e050314 100644
--- a/qa.py
+++ b/qa.py
@@ -409,6 +409,7 @@ def readdoc_splittext_pptx(filename):
# Extract image addresses from the PowerPoint presentation
image_addresses = extract_image_addresses(filename)
+ print(image_addresses)
# Authenticate using service account credentials
credentials = service_account.Credentials.from_service_account_info({
From f786f41f6e65fb5802568f116897831ee696a874 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 18:33:14 +0530
Subject: [PATCH 093/129] qa.py
---
qa.py | 24 ++++++++++++------------
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/qa.py b/qa.py
index e050314..5be8a4f 100644
--- a/qa.py
+++ b/qa.py
@@ -413,18 +413,18 @@ def readdoc_splittext_pptx(filename):
# Authenticate using service account credentials
credentials = service_account.Credentials.from_service_account_info({
- "type": "service_account",
- "project_id": "able-store-415222",
- "private_key_id": "3c73cfca4950749ff7d8b9411e28e7db84c89c75",
- "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDZEH33tpJHp4XL\n3+jMloFGguZCzQuT8HwobVK7gI+x/fNfbamU2gzla+vP7GGmIP8JYn0S1cvWY0ki\nyakN7Hxp6DmkIeOpK9piojKriUFUFXu86JiaEmYRUplSeifOCexHM4KIgM4ABKc9\nogGVilJUdoLEigaMXF8/XS7O2uBRaApegUhSCZnYEp64VXVVyPoRtFAcKhmp5y4T\n+x58TjTX/7oEC6nWvfq1opJBzX++svH8f/eQ4FfpCh2WfyDyQk7GNyRxQMM9Xbnx\nwM9aWejuDN+yBufqWjoDI1wOqBNNDAuOr/Gdkpp8E5CqP/mBNwsK5LoMyWaAgrUq\nKkrE/M2fAgMBAAECggEAFXIer/m2zCOOAY2Wx5fbH+dQ7qqpiS1KZwgeE25pRBGu\nGPCRv6YnfgdCniFhqHM8H2Y+qzy6Te0zTZ+U9041IJWUFFZQ0ucFRZd24CHIdZFF\nz20/noFQ2kdJ5K8y/ZVXZdV+ARSQxcmv6FP4ShkHd1baJqbobLe3pWAEfuV7Ii+x\nlW026XwMvMZqGIn5Vw0ra4waJbexmGIHFy29pc7r8RTzgpxXEkSMi3JBPq4q6cfK\nV7ckoUw5qtvhFvhjTH+B2ipPII5+YxAo6MoxTwMrH3aZUrjTVjWzTQ3Fbb72Il77\nlKKAuzNNlNycN5T5cR3WR8rJ9lf2xfIqw9Jw44YDkQKBgQD+63n1aovkv8VLAJeJ\nbb/CRAU3ohrJpx3oRTSdwOkfVRITdJpPzxtZUun8xgFjmR50qptRNobZegbBF+FQ\nRjKu7L69UC0LEEBItn9ud+iQ64JNp+f2Wq7doTRqXHVZBANr3fSwZteQyqh2UHeF\nGQmVk7t35vIj9vXF/4sfAzhFpwKBgQDZ+/O9Y3CmyiHNX7sAJe4n//5W1VEW1nsC\nyO4+fIJPv7NPhJN8AeC//LltKKKm7C87Mc9JTkHQEtTeQoqW8/xlTwGpop068QmE\nVGWM9bfA1hLSgtwp8GoGrT92FX+12BKg5D6UxDslNDzXhrBqXGyVTqIhuPWy0zWw\nL1unWZenSQKBgQC1f2is8Dg8HMHWvcwmv+oo4Y8pZhRWgNLNXgCxVPlmIoalLX70\n1ctuFJeeLkgs5ocFn7bH4t+uFbCbaGo2YlSRsOO5HE2FtANhAfbG6z69d8Clk6eX\nAkfAapdMJxoxz6az9SrTMdXHNFMvMel27TWitrViEB9Ute+VEnW2Fe/JvwKBgAbx\nDg7+5qx5DWCD5umXS6E8drX/LwjiJaKuEWAuCNxPR3+FwkiVqrmSa7k9LQjRxqNE\n78vu0Qu2Pc8iIVWzSVtUi8ICKq2g1WPAaEd337UlXA4WGrq/LDEwPTAeeSWqTtWO\nzytfoF6L7lASuvV4IgETMviN8k0Sisgkie+nW7v5AoGBAMr4rftvEEUxOPuu+U7h\newe63BiLTU4F+OMGXhIHQIz88HdQk6w4XTj6Nrb2pivqCGPFNgTr4+WmMfe9DkAH\nxzny8Kdp9HUob6u8DrkWI2OLwKAnReVy3E3wd8MLBeZTahLHVc4vtpMH4L7cXpKv\nYlCS/pHOIfU6R5pPkSXj6ozZ\n-----END PRIVATE KEY-----\n",
- "client_email": "speekar@able-store-415222.iam.gserviceaccount.com",
- "client_id": "110531954629070375774",
- "auth_uri": "https://accounts.google.com/o/oauth2/auth",
- "token_uri": "https://oauth2.googleapis.com/token",
- "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
- "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/speekar%40able-store-415222.iam.gserviceaccount.com",
- "universe_domain": "googleapis.com"
-})
+ "type": "service_account",
+ "project_id": "able-store-415222",
+ "private_key_id": "3c73cfca4950749ff7d8b9411e28e7db84c89c75",
+ "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDZEH33tpJHp4XL\n3+jMloFGguZCzQuT8HwobVK7gI+x/fNfbamU2gzla+vP7GGmIP8JYn0S1cvWY0ki\nyakN7Hxp6DmkIeOpK9piojKriUFUFXu86JiaEmYRUplSeifOCexHM4KIgM4ABKc9\nogGVilJUdoLEigaMXF8/XS7O2uBRaApegUhSCZnYEp64VXVVyPoRtFAcKhmp5y4T\n+x58TjTX/7oEC6nWvfq1opJBzX++svH8f/eQ4FfpCh2WfyDyQk7GNyRxQMM9Xbnx\nwM9aWejuDN+yBufqWjoDI1wOqBNNDAuOr/Gdkpp8E5CqP/mBNwsK5LoMyWaAgrUq\nKkrE/M2fAgMBAAECggEAFXIer/m2zCOOAY2Wx5fbH+dQ7qqpiS1KZwgeE25pRBGu\nGPCRv6YnfgdCniFhqHM8H2Y+qzy6Te0zTZ+U9041IJWUFFZQ0ucFRZd24CHIdZFF\nz20/noFQ2kdJ5K8y/ZVXZdV+ARSQxcmv6FP4ShkHd1baJqbobLe3pWAEfuV7Ii+x\nlW026XwMvMZqGIn5Vw0ra4waJbexmGIHFy29pc7r8RTzgpxXEkSMi3JBPq4q6cfK\nV7ckoUw5qtvhFvhjTH+B2ipPII5+YxAo6MoxTwMrH3aZUrjTVjWzTQ3Fbb72Il77\nlKKAuzNNlNycN5T5cR3WR8rJ9lf2xfIqw9Jw44YDkQKBgQD+63n1aovkv8VLAJeJ\nbb/CRAU3ohrJpx3oRTSdwOkfVRITdJpPzxtZUun8xgFjmR50qptRNobZegbBF+FQ\nRjKu7L69UC0LEEBItn9ud+iQ64JNp+f2Wq7doTRqXHVZBANr3fSwZteQyqh2UHeF\nGQmVk7t35vIj9vXF/4sfAzhFpwKBgQDZ+/O9Y3CmyiHNX7sAJe4n//5W1VEW1nsC\nyO4+fIJPv7NPhJN8AeC//LltKKKm7C87Mc9JTkHQEtTeQoqW8/xlTwGpop068QmE\nVGWM9bfA1hLSgtwp8GoGrT92FX+12BKg5D6UxDslNDzXhrBqXGyVTqIhuPWy0zWw\nL1unWZenSQKBgQC1f2is8Dg8HMHWvcwmv+oo4Y8pZhRWgNLNXgCxVPlmIoalLX70\n1ctuFJeeLkgs5ocFn7bH4t+uFbCbaGo2YlSRsOO5HE2FtANhAfbG6z69d8Clk6eX\nAkfAapdMJxoxz6az9SrTMdXHNFMvMel27TWitrViEB9Ute+VEnW2Fe/JvwKBgAbx\nDg7+5qx5DWCD5umXS6E8drX/LwjiJaKuEWAuCNxPR3+FwkiVqrmSa7k9LQjRxqNE\n78vu0Qu2Pc8iIVWzSVtUi8ICKq2g1WPAaEd337UlXA4WGrq/LDEwPTAeeSWqTtWO\nzytfoF6L7lASuvV4IgETMviN8k0Sisgkie+nW7v5AoGBAMr4rftvEEUxOPuu+U7h\newe63BiLTU4F+OMGXhIHQIz88HdQk6w4XTj6Nrb2pivqCGPFNgTr4+WmMfe9DkAH\nxzny8Kdp9HUob6u8DrkWI2OLwKAnReVy3E3wd8MLBeZTahLHVc4vtpMH4L7cXpKv\nYlCS/pHOIfU6R5pPkSXj6ozZ\n-----END PRIVATE KEY-----\n",
+ "client_email": "speekar@able-store-415222.iam.gserviceaccount.com",
+ "client_id": "110531954629070375774",
+ "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+ "token_uri": "https://oauth2.googleapis.com/token",
+ "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+ "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/speekar%40able-store-415222.iam.gserviceaccount.com",
+ "universe_domain": "googleapis.com"
+ })
images_text = ''
headings_list = []
From 981c49c31094c52bb23be9b298e48a860a2bf247 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 21:51:37 +0530
Subject: [PATCH 094/129] requirements.txt
---
requirements.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/requirements.txt b/requirements.txt
index 5668cf8..6daa008 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,6 +7,7 @@ openai==0.27.7
openapi-schema-pydantic==1.2.4
streamlit
google-cloud-vision
+google-cloud-storage
python-dotenv==1.0.0
accelerate==0.21.0
gradio==3.37.0
From 8aa314bab147681fc1da62b0c08d0f1cfa15dbd2 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 21:53:10 +0530
Subject: [PATCH 095/129] requirements.txt
From b4f5707f725905de96ff130f4c005ed31af6b23a Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 24 Feb 2024 21:54:19 +0530
Subject: [PATCH 096/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 5be8a4f..faefcac 100644
--- a/qa.py
+++ b/qa.py
@@ -2,7 +2,7 @@
import io
import scipy
import numpy as np
-
+import json
# ----------------SPEECH RECOGNITION /AUDIO/TEXT TO SPEECH DEPENDENCIES-------------#
from scipy.io.wavfile import read as wav_read
from pydub import AudioSegment
From 136edf721721c581d82a3531f5b3ec2da0483de4 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sun, 25 Feb 2024 11:59:27 +0530
Subject: [PATCH 097/129] qa.py
---
qa.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index faefcac..0de7f7d 100644
--- a/qa.py
+++ b/qa.py
@@ -37,6 +37,7 @@
from docx.shared import Inches
from google.cloud import vision
from google.oauth2 import service_account
+from google.cloud import storage
import textwrap
import glob
from spire.doc import *
@@ -410,7 +411,8 @@ def readdoc_splittext_pptx(filename):
# Extract image addresses from the PowerPoint presentation
image_addresses = extract_image_addresses(filename)
print(image_addresses)
-
+ bucket_name='BUCKET-NAME'
+ os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = '"C:\Users\saikr\Downloads\able-store-415222-3c73cfca4950.json"'
# Authenticate using service account credentials
credentials = service_account.Credentials.from_service_account_info({
"type": "service_account",
From b896636081a9aef40ec2a73008d512437bab7e6e Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sun, 25 Feb 2024 13:08:05 +0530
Subject: [PATCH 098/129] able-store-415222-3c73cfca4950.json
---
able-store-415222-3c73cfca4950.json | 13 +++++++++++++
1 file changed, 13 insertions(+)
create mode 100644 able-store-415222-3c73cfca4950.json
diff --git a/able-store-415222-3c73cfca4950.json b/able-store-415222-3c73cfca4950.json
new file mode 100644
index 0000000..6bf7494
--- /dev/null
+++ b/able-store-415222-3c73cfca4950.json
@@ -0,0 +1,13 @@
+{
+ "type": "service_account",
+ "project_id": "able-store-415222",
+ "private_key_id": "3c73cfca4950749ff7d8b9411e28e7db84c89c75",
+ "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDZEH33tpJHp4XL\n3+jMloFGguZCzQuT8HwobVK7gI+x/fNfbamU2gzla+vP7GGmIP8JYn0S1cvWY0ki\nyakN7Hxp6DmkIeOpK9piojKriUFUFXu86JiaEmYRUplSeifOCexHM4KIgM4ABKc9\nogGVilJUdoLEigaMXF8/XS7O2uBRaApegUhSCZnYEp64VXVVyPoRtFAcKhmp5y4T\n+x58TjTX/7oEC6nWvfq1opJBzX++svH8f/eQ4FfpCh2WfyDyQk7GNyRxQMM9Xbnx\nwM9aWejuDN+yBufqWjoDI1wOqBNNDAuOr/Gdkpp8E5CqP/mBNwsK5LoMyWaAgrUq\nKkrE/M2fAgMBAAECggEAFXIer/m2zCOOAY2Wx5fbH+dQ7qqpiS1KZwgeE25pRBGu\nGPCRv6YnfgdCniFhqHM8H2Y+qzy6Te0zTZ+U9041IJWUFFZQ0ucFRZd24CHIdZFF\nz20/noFQ2kdJ5K8y/ZVXZdV+ARSQxcmv6FP4ShkHd1baJqbobLe3pWAEfuV7Ii+x\nlW026XwMvMZqGIn5Vw0ra4waJbexmGIHFy29pc7r8RTzgpxXEkSMi3JBPq4q6cfK\nV7ckoUw5qtvhFvhjTH+B2ipPII5+YxAo6MoxTwMrH3aZUrjTVjWzTQ3Fbb72Il77\nlKKAuzNNlNycN5T5cR3WR8rJ9lf2xfIqw9Jw44YDkQKBgQD+63n1aovkv8VLAJeJ\nbb/CRAU3ohrJpx3oRTSdwOkfVRITdJpPzxtZUun8xgFjmR50qptRNobZegbBF+FQ\nRjKu7L69UC0LEEBItn9ud+iQ64JNp+f2Wq7doTRqXHVZBANr3fSwZteQyqh2UHeF\nGQmVk7t35vIj9vXF/4sfAzhFpwKBgQDZ+/O9Y3CmyiHNX7sAJe4n//5W1VEW1nsC\nyO4+fIJPv7NPhJN8AeC//LltKKKm7C87Mc9JTkHQEtTeQoqW8/xlTwGpop068QmE\nVGWM9bfA1hLSgtwp8GoGrT92FX+12BKg5D6UxDslNDzXhrBqXGyVTqIhuPWy0zWw\nL1unWZenSQKBgQC1f2is8Dg8HMHWvcwmv+oo4Y8pZhRWgNLNXgCxVPlmIoalLX70\n1ctuFJeeLkgs5ocFn7bH4t+uFbCbaGo2YlSRsOO5HE2FtANhAfbG6z69d8Clk6eX\nAkfAapdMJxoxz6az9SrTMdXHNFMvMel27TWitrViEB9Ute+VEnW2Fe/JvwKBgAbx\nDg7+5qx5DWCD5umXS6E8drX/LwjiJaKuEWAuCNxPR3+FwkiVqrmSa7k9LQjRxqNE\n78vu0Qu2Pc8iIVWzSVtUi8ICKq2g1WPAaEd337UlXA4WGrq/LDEwPTAeeSWqTtWO\nzytfoF6L7lASuvV4IgETMviN8k0Sisgkie+nW7v5AoGBAMr4rftvEEUxOPuu+U7h\newe63BiLTU4F+OMGXhIHQIz88HdQk6w4XTj6Nrb2pivqCGPFNgTr4+WmMfe9DkAH\nxzny8Kdp9HUob6u8DrkWI2OLwKAnReVy3E3wd8MLBeZTahLHVc4vtpMH4L7cXpKv\nYlCS/pHOIfU6R5pPkSXj6ozZ\n-----END PRIVATE KEY-----\n",
+ "client_email": "speekar@able-store-415222.iam.gserviceaccount.com",
+ "client_id": "110531954629070375774",
+ "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+ "token_uri": "https://oauth2.googleapis.com/token",
+ "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+ "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/speekar%40able-store-415222.iam.gserviceaccount.com",
+ "universe_domain": "googleapis.com"
+}
From 5a96a42a9917442427b2f323e77d98f19c36f450 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sun, 25 Feb 2024 13:08:58 +0530
Subject: [PATCH 099/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 0de7f7d..847ae66 100644
--- a/qa.py
+++ b/qa.py
@@ -412,7 +412,7 @@ def readdoc_splittext_pptx(filename):
image_addresses = extract_image_addresses(filename)
print(image_addresses)
bucket_name='BUCKET-NAME'
- os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = '"C:\Users\saikr\Downloads\able-store-415222-3c73cfca4950.json"'
+ os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'able-store-415222-3c73cfca4950.json'
# Authenticate using service account credentials
credentials = service_account.Credentials.from_service_account_info({
"type": "service_account",
From 5a9b34b920f688ddaf3cf0b2cbb288cadd565a8d Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sun, 25 Feb 2024 13:44:44 +0530
Subject: [PATCH 100/129] qa.py
---
qa.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/qa.py b/qa.py
index 847ae66..17cd90d 100644
--- a/qa.py
+++ b/qa.py
@@ -411,7 +411,7 @@ def readdoc_splittext_pptx(filename):
# Extract image addresses from the PowerPoint presentation
image_addresses = extract_image_addresses(filename)
print(image_addresses)
- bucket_name='BUCKET-NAME'
+ bucket_name='aiexplorers'
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'able-store-415222-3c73cfca4950.json'
# Authenticate using service account credentials
credentials = service_account.Credentials.from_service_account_info({
@@ -433,7 +433,10 @@ def readdoc_splittext_pptx(filename):
paragraph_list = []
# Initialize Google Cloud Vision API client
- client = vision.ImageAnnotatorClient(credentials=credentials)
+ storage_client = storage.Client()
+ bucket = storage_client.get_bucket(bucket_name)
+ vision_client = vision.ImageAnnotatorClient(credentials=credentials)
+ feature = vision.Feature(type_=vision.Feature.Type.DOCUMENT_TEXT_DETECTION)
# Perform OCR on images
for image_address in image_addresses:
From bc6a38ce4f06a294e843e67231ce94be1e747fa3 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sun, 25 Feb 2024 13:57:17 +0530
Subject: [PATCH 101/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 17cd90d..81361e7 100644
--- a/qa.py
+++ b/qa.py
@@ -435,7 +435,7 @@ def readdoc_splittext_pptx(filename):
# Initialize Google Cloud Vision API client
storage_client = storage.Client()
bucket = storage_client.get_bucket(bucket_name)
- vision_client = vision.ImageAnnotatorClient(credentials=credentials)
+ vision_client = vision.ImageAnnotatorClient()
feature = vision.Feature(type_=vision.Feature.Type.DOCUMENT_TEXT_DETECTION)
# Perform OCR on images
From 63ffde50aa6fff0c4a88f409c448a8ee52936c8c Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sun, 25 Feb 2024 14:45:21 +0530
Subject: [PATCH 102/129] qa.py
---
qa.py | 170 +++++++++++++++++++++++++++++-----------------------------
1 file changed, 86 insertions(+), 84 deletions(-)
diff --git a/qa.py b/qa.py
index 81361e7..9374b39 100644
--- a/qa.py
+++ b/qa.py
@@ -411,90 +411,92 @@ def readdoc_splittext_pptx(filename):
# Extract image addresses from the PowerPoint presentation
image_addresses = extract_image_addresses(filename)
print(image_addresses)
- bucket_name='aiexplorers'
- os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'able-store-415222-3c73cfca4950.json'
- # Authenticate using service account credentials
- credentials = service_account.Credentials.from_service_account_info({
- "type": "service_account",
- "project_id": "able-store-415222",
- "private_key_id": "3c73cfca4950749ff7d8b9411e28e7db84c89c75",
- "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDZEH33tpJHp4XL\n3+jMloFGguZCzQuT8HwobVK7gI+x/fNfbamU2gzla+vP7GGmIP8JYn0S1cvWY0ki\nyakN7Hxp6DmkIeOpK9piojKriUFUFXu86JiaEmYRUplSeifOCexHM4KIgM4ABKc9\nogGVilJUdoLEigaMXF8/XS7O2uBRaApegUhSCZnYEp64VXVVyPoRtFAcKhmp5y4T\n+x58TjTX/7oEC6nWvfq1opJBzX++svH8f/eQ4FfpCh2WfyDyQk7GNyRxQMM9Xbnx\nwM9aWejuDN+yBufqWjoDI1wOqBNNDAuOr/Gdkpp8E5CqP/mBNwsK5LoMyWaAgrUq\nKkrE/M2fAgMBAAECggEAFXIer/m2zCOOAY2Wx5fbH+dQ7qqpiS1KZwgeE25pRBGu\nGPCRv6YnfgdCniFhqHM8H2Y+qzy6Te0zTZ+U9041IJWUFFZQ0ucFRZd24CHIdZFF\nz20/noFQ2kdJ5K8y/ZVXZdV+ARSQxcmv6FP4ShkHd1baJqbobLe3pWAEfuV7Ii+x\nlW026XwMvMZqGIn5Vw0ra4waJbexmGIHFy29pc7r8RTzgpxXEkSMi3JBPq4q6cfK\nV7ckoUw5qtvhFvhjTH+B2ipPII5+YxAo6MoxTwMrH3aZUrjTVjWzTQ3Fbb72Il77\nlKKAuzNNlNycN5T5cR3WR8rJ9lf2xfIqw9Jw44YDkQKBgQD+63n1aovkv8VLAJeJ\nbb/CRAU3ohrJpx3oRTSdwOkfVRITdJpPzxtZUun8xgFjmR50qptRNobZegbBF+FQ\nRjKu7L69UC0LEEBItn9ud+iQ64JNp+f2Wq7doTRqXHVZBANr3fSwZteQyqh2UHeF\nGQmVk7t35vIj9vXF/4sfAzhFpwKBgQDZ+/O9Y3CmyiHNX7sAJe4n//5W1VEW1nsC\nyO4+fIJPv7NPhJN8AeC//LltKKKm7C87Mc9JTkHQEtTeQoqW8/xlTwGpop068QmE\nVGWM9bfA1hLSgtwp8GoGrT92FX+12BKg5D6UxDslNDzXhrBqXGyVTqIhuPWy0zWw\nL1unWZenSQKBgQC1f2is8Dg8HMHWvcwmv+oo4Y8pZhRWgNLNXgCxVPlmIoalLX70\n1ctuFJeeLkgs5ocFn7bH4t+uFbCbaGo2YlSRsOO5HE2FtANhAfbG6z69d8Clk6eX\nAkfAapdMJxoxz6az9SrTMdXHNFMvMel27TWitrViEB9Ute+VEnW2Fe/JvwKBgAbx\nDg7+5qx5DWCD5umXS6E8drX/LwjiJaKuEWAuCNxPR3+FwkiVqrmSa7k9LQjRxqNE\n78vu0Qu2Pc8iIVWzSVtUi8ICKq2g1WPAaEd337UlXA4WGrq/LDEwPTAeeSWqTtWO\nzytfoF6L7lASuvV4IgETMviN8k0Sisgkie+nW7v5AoGBAMr4rftvEEUxOPuu+U7h\newe63BiLTU4F+OMGXhIHQIz88HdQk6w4XTj6Nrb2pivqCGPFNgTr4+WmMfe9DkAH\nxzny8Kdp9HUob6u8DrkWI2OLwKAnReVy3E3wd8MLBeZTahLHVc4vtpMH4L7cXpKv\nYlCS/pHOIfU6R5pPkSXj6ozZ\n-----END PRIVATE KEY-----\n",
- "client_email": "speekar@able-store-415222.iam.gserviceaccount.com",
- "client_id": "110531954629070375774",
- "auth_uri": "https://accounts.google.com/o/oauth2/auth",
- "token_uri": "https://oauth2.googleapis.com/token",
- "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
- "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/speekar%40able-store-415222.iam.gserviceaccount.com",
- "universe_domain": "googleapis.com"
- })
-
- images_text = ''
- headings_list = []
- paragraph_list = []
-
- # Initialize Google Cloud Vision API client
- storage_client = storage.Client()
- bucket = storage_client.get_bucket(bucket_name)
- vision_client = vision.ImageAnnotatorClient()
- feature = vision.Feature(type_=vision.Feature.Type.DOCUMENT_TEXT_DETECTION)
-
- # Perform OCR on images
- for image_address in image_addresses:
- if os.path.exists(image_address): # Check if the image file exists
- with io.open(image_address, 'rb') as image_file:
- content = image_file.read()
-
- image = vision.Image(content=content)
- response = client.text_detection(image=image)
- texts = response.text_annotations
- for text in texts:
- images_text += text.description + '\n'
-
- # Extract headers and paragraphs from OCR results of images
- pat1 = re.compile(r".+\:")
- pat2 = re.compile(r".+\.\n")
- headings_list_img = pat1.findall(text.description)
- paragraph_list_img = pat2.findall(text.description)
-
- # Append headers and paragraphs from images to existing lists
- headings_list.extend(headings_list_img)
- paragraph_list.extend(paragraph_list_img)
-
- # Extract headers and paragraphs from the PowerPoint text
- pat1 = re.compile(r".+\:")
- pat2 = re.compile(r".+\.\n")
- headings_list_pptx = pat1.findall(docs)
- paragraph_list_pptx = pat2.findall(docs)
- headings_list.extend(headings_list_pptx)
- paragraph_list.extend(paragraph_list_pptx)
- n = 1500
- # Combine all text from PowerPoint and OCR'd images
- all_text = ''.join(paragraph_list_pptx) + images_text
-
- a = glob.glob(filename)
-
- chunk_size = 1024
- chunk_overlap = 10
- text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
-
- texts_isb = []
- texts_raw = []
- documents = []
- for i in range(len(a)):
- documents.extend(loader.load())
- for j in range(len(text_splitter.split_documents(loader.load()))):
- text_chunk = text_splitter.split_documents(loader.load())[j]
- text_chunk.page_content = text_chunk.page_content.replace("\n", " ")
- text_chunk.page_content = text_chunk.page_content.replace("\\n", " ")
- text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
- text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
- texts_isb.append(text_chunk.page_content)
- texts_raw.append(text_chunk)
-
- text_split = texts_isb
-
- return all_text, text_split, texts_raw, headings_list, paragraph_list
-
+ try:
+ bucket_name='aiexplorers'
+ os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'able-store-415222-3c73cfca4950.json'
+ # Authenticate using service account credentials
+ credentials = service_account.Credentials.from_service_account_info({
+ "type": "service_account",
+ "project_id": "able-store-415222",
+ "private_key_id": "3c73cfca4950749ff7d8b9411e28e7db84c89c75",
+ "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDZEH33tpJHp4XL\n3+jMloFGguZCzQuT8HwobVK7gI+x/fNfbamU2gzla+vP7GGmIP8JYn0S1cvWY0ki\nyakN7Hxp6DmkIeOpK9piojKriUFUFXu86JiaEmYRUplSeifOCexHM4KIgM4ABKc9\nogGVilJUdoLEigaMXF8/XS7O2uBRaApegUhSCZnYEp64VXVVyPoRtFAcKhmp5y4T\n+x58TjTX/7oEC6nWvfq1opJBzX++svH8f/eQ4FfpCh2WfyDyQk7GNyRxQMM9Xbnx\nwM9aWejuDN+yBufqWjoDI1wOqBNNDAuOr/Gdkpp8E5CqP/mBNwsK5LoMyWaAgrUq\nKkrE/M2fAgMBAAECggEAFXIer/m2zCOOAY2Wx5fbH+dQ7qqpiS1KZwgeE25pRBGu\nGPCRv6YnfgdCniFhqHM8H2Y+qzy6Te0zTZ+U9041IJWUFFZQ0ucFRZd24CHIdZFF\nz20/noFQ2kdJ5K8y/ZVXZdV+ARSQxcmv6FP4ShkHd1baJqbobLe3pWAEfuV7Ii+x\nlW026XwMvMZqGIn5Vw0ra4waJbexmGIHFy29pc7r8RTzgpxXEkSMi3JBPq4q6cfK\nV7ckoUw5qtvhFvhjTH+B2ipPII5+YxAo6MoxTwMrH3aZUrjTVjWzTQ3Fbb72Il77\nlKKAuzNNlNycN5T5cR3WR8rJ9lf2xfIqw9Jw44YDkQKBgQD+63n1aovkv8VLAJeJ\nbb/CRAU3ohrJpx3oRTSdwOkfVRITdJpPzxtZUun8xgFjmR50qptRNobZegbBF+FQ\nRjKu7L69UC0LEEBItn9ud+iQ64JNp+f2Wq7doTRqXHVZBANr3fSwZteQyqh2UHeF\nGQmVk7t35vIj9vXF/4sfAzhFpwKBgQDZ+/O9Y3CmyiHNX7sAJe4n//5W1VEW1nsC\nyO4+fIJPv7NPhJN8AeC//LltKKKm7C87Mc9JTkHQEtTeQoqW8/xlTwGpop068QmE\nVGWM9bfA1hLSgtwp8GoGrT92FX+12BKg5D6UxDslNDzXhrBqXGyVTqIhuPWy0zWw\nL1unWZenSQKBgQC1f2is8Dg8HMHWvcwmv+oo4Y8pZhRWgNLNXgCxVPlmIoalLX70\n1ctuFJeeLkgs5ocFn7bH4t+uFbCbaGo2YlSRsOO5HE2FtANhAfbG6z69d8Clk6eX\nAkfAapdMJxoxz6az9SrTMdXHNFMvMel27TWitrViEB9Ute+VEnW2Fe/JvwKBgAbx\nDg7+5qx5DWCD5umXS6E8drX/LwjiJaKuEWAuCNxPR3+FwkiVqrmSa7k9LQjRxqNE\n78vu0Qu2Pc8iIVWzSVtUi8ICKq2g1WPAaEd337UlXA4WGrq/LDEwPTAeeSWqTtWO\nzytfoF6L7lASuvV4IgETMviN8k0Sisgkie+nW7v5AoGBAMr4rftvEEUxOPuu+U7h\newe63BiLTU4F+OMGXhIHQIz88HdQk6w4XTj6Nrb2pivqCGPFNgTr4+WmMfe9DkAH\nxzny8Kdp9HUob6u8DrkWI2OLwKAnReVy3E3wd8MLBeZTahLHVc4vtpMH4L7cXpKv\nYlCS/pHOIfU6R5pPkSXj6ozZ\n-----END PRIVATE KEY-----\n",
+ "client_email": "speekar@able-store-415222.iam.gserviceaccount.com",
+ "client_id": "110531954629070375774",
+ "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+ "token_uri": "https://oauth2.googleapis.com/token",
+ "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+ "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/speekar%40able-store-415222.iam.gserviceaccount.com",
+ "universe_domain": "googleapis.com"
+ })
+
+ images_text = ''
+ headings_list = []
+ paragraph_list = []
+
+ # Initialize Google Cloud Vision API client
+ storage_client = storage.Client()
+ bucket = storage_client.get_bucket(bucket_name)
+ vision_client = vision.ImageAnnotatorClient(credentials=credentials)
+ feature = vision.Feature(type_=vision.Feature.Type.DOCUMENT_TEXT_DETECTION)
+
+ # Perform OCR on images
+ for i in range(len(image_addresses)):
+ if os.path.exists(image_addresses[i]): # Check if the image file exists
+ with io.open(image_addresses[i], 'rb') as image_file:
+ content = image_file.read()
+
+ image = vision.Image(content=content)
+ response = vision_client.text_detection(image=image)
+ texts = response.text_annotations
+ for text in texts:
+ images_text += text.description + '\n'
+
+ # Extract headers and paragraphs from OCR results of images
+ pat1 = re.compile(r".+\:")
+ pat2 = re.compile(r".+\.\n")
+ headings_list_img = pat1.findall(text.description)
+ paragraph_list_img = pat2.findall(text.description)
+
+ # Append headers and paragraphs from images to existing lists
+ headings_list.extend(headings_list_img)
+ paragraph_list.extend(paragraph_list_img)
+
+ # Extract headers and paragraphs from the PowerPoint text
+ pat1 = re.compile(r".+\:")
+ pat2 = re.compile(r".+\.\n")
+ headings_list_pptx = pat1.findall(docs)
+ paragraph_list_pptx = pat2.findall(docs)
+ headings_list.extend(headings_list_pptx)
+ paragraph_list.extend(paragraph_list_pptx)
+ n = 1500
+ # Combine all text from PowerPoint and OCR'd images
+ all_text = ''.join(paragraph_list_pptx) + images_text
+
+ a = glob.glob(filename)
+
+ chunk_size = 1024
+ chunk_overlap = 10
+ text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+
+ texts_isb = []
+ texts_raw = []
+ documents = []
+ for i in range(len(a)):
+ documents.extend(loader.load())
+ for j in range(len(text_splitter.split_documents(loader.load()))):
+ text_chunk = text_splitter.split_documents(loader.load())[j]
+ text_chunk.page_content = text_chunk.page_content.replace("\n", " ")
+ text_chunk.page_content = text_chunk.page_content.replace("\\n", " ")
+ text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
+ text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
+ texts_isb.append(text_chunk.page_content)
+ texts_raw.append(text_chunk)
+
+ text_split = texts_isb
+
+ return all_text, text_split, texts_raw, headings_list, paragraph_list
+ except Exception as e:
+ print("An error occurred:", e)
#---------------READ THE .TXT FILE AND GENERATE THE SPLIT--------------------#
@st.cache_resource(show_spinner=True)
def readdoc_splittext_txt(filename):
From 4e0dd209ad30a55a6257cd4b1252aa506abe98ab Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sun, 25 Feb 2024 14:45:59 +0530
Subject: [PATCH 103/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 9374b39..014cde2 100644
--- a/qa.py
+++ b/qa.py
@@ -449,7 +449,7 @@ def readdoc_splittext_pptx(filename):
response = vision_client.text_detection(image=image)
texts = response.text_annotations
for text in texts:
- images_text += text.description + '\n'
+ images_text += text.description + '\n'
# Extract headers and paragraphs from OCR results of images
pat1 = re.compile(r".+\:")
From a9ee78fc128428a0d18f70f0322c025e38f2a2c9 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sun, 25 Feb 2024 14:46:40 +0530
Subject: [PATCH 104/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 014cde2..b1baef9 100644
--- a/qa.py
+++ b/qa.py
@@ -496,7 +496,7 @@ def readdoc_splittext_pptx(filename):
return all_text, text_split, texts_raw, headings_list, paragraph_list
except Exception as e:
- print("An error occurred:", e)
+ print("An error occurred:", e)
#---------------READ THE .TXT FILE AND GENERATE THE SPLIT--------------------#
@st.cache_resource(show_spinner=True)
def readdoc_splittext_txt(filename):
From 0260eaebffddfcb35dc72b8a4b52b88c6d6a8df6 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Fri, 1 Mar 2024 22:40:03 +0530
Subject: [PATCH 105/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index b1baef9..dbfbb10 100644
--- a/qa.py
+++ b/qa.py
@@ -907,7 +907,7 @@ def chatbot_slim(question, context, keywords):
@st.cache_resource(show_spinner=True)
-def create_db(_texts_raw, _uploaded_file_name):
+def create_db(texts_raw, uploaded_file_name):
hf= OpenAIEmbeddings(model="text-embedding-ada-002", openai_api_key=openai.api_key)
From c8d7ebc905327a6450aefab5ecad69c714c38bd3 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Fri, 1 Mar 2024 22:40:28 +0530
Subject: [PATCH 106/129] streamlit_app.py
---
streamlit_app.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 98f1b7f..9700f3c 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -248,7 +248,7 @@ def calculate_rouge_scores(answer,context):
with st.chat_message("assistant"):
st.write("Hi! Getting your contexts ready for query! Please wait!")
- hf, db = create_db(texts_raw, uploaded_file.name)
+ hf, db = create_db(texts_raw, uploaded_file_name)
st.session_state["db_created"] = True
From e10a96074f7209720c12775e5faa0d797e7b4b9e Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Fri, 1 Mar 2024 22:45:00 +0530
Subject: [PATCH 107/129] qa.py
---
qa.py | 84 +++--------------------------------------------------------
1 file changed, 3 insertions(+), 81 deletions(-)
diff --git a/qa.py b/qa.py
index dbfbb10..c4a7cfa 100644
--- a/qa.py
+++ b/qa.py
@@ -599,86 +599,8 @@ def readdoc_splittext_pdf(filename):
texts_isb.append(text_chunk.page_content)
texts_raw.append(text_chunk)
text_split=texts_isb
- return all_text, text_split, texts_raw, headings_list, paragraph_list
+ return all_text, text_split,texts_raw, headings_list, paragraph_list
-# ----------------CREATE CONTEXT-----------------------#
-#def create_context(query, text_split, headings, para_texts):
- # """
- # Create a context for a question by finding the most similar context from the dataframe
- # """
-#
- # # Get the embeddings for the question
- # # Get the distances from the embeddings
- # # i=0
- #kw_model = KeyBERT()
-#
- # sentences = text_split # [i for i in nlp(text).sents]
-#
- # returns = []
- # keystart = time.time()
- # keywords_q = []
-#
- # keywords_query = kw_model.extract_keywords(query)
- # keywords = []
-#
- # for j in range(len(keywords_query)):
- # if keywords_query[j][1] > 0.3:
- # if keywords_query[j][0] not in keywords_q:
- # # print(keywords[j][0], keywords[j][1])
- # keywords_q.append(keywords_query[j][0])
-#
- # i = 0
- # keyword_doc = {}
- # keyend = time.time()
-#
- # for sent in sentences:
- # if isinstance(sent, str) and len(sent) > 6:
- # keywords = kw_model.extract_keywords(sent)
- # keyword_doc_sent = []
-#
- # for j in range(len(keywords)):
- # # Add the heading of the para corresponding to the sentence
- # if j == 0:
- # keyword_doc_sent.append(keywords[j][0])
- # for h, pt in zip(headings, para_texts):
- # pt = pt.lower()
- # index = pt.find(sent.lower())
- # if index != -1:
- # print("contains")
- # # Add the heading of the para corresponding to the sentence
- # keyword_doc_sent.append(h)
-#
- # if keywords[j][1] > 0.3: # and keywords[j][0] not in keyword_doc ):
- # if keywords[j][0] not in keyword_doc_sent:
- # keyword_doc_sent.append(keywords[j][0])
- # keyword_doc[i] = keyword_doc_sent
-#
- # i += 1
-#
- # search_start = time.time()
-#
- # for i in range(len(keyword_doc)):
- # for k in range(len(keywords_q)):
- # match_count = 0
- # if keywords_q[k] in keyword_doc[i]:
- # match_count += 1
- # keywords.append(keywords_q[k])
- # # print(keywords_q[k],keyword_doc[i] )
- # # print("match_count::",match_count)
- # if match_count >= 1 or match_count >= len(keywords_q):
- # # print("Document matched :",i, "::")
- # if remove_newlines(text_split[i]) not in returns:
- # # context_q+=remove_newlines(sent)
- # returns.append(remove_newlines(text_split[i]))
- # # print(returns,match_count )
-#
- # searchend = time.time()
- # search_time = searchend - search_start
-#
- # cur_len = 0
-#
- # Return the context
-# return "\n\n###\n\n".join(returns), keywords
# ----------------CREATE CONTEXT-----------------------#
import time
from keybert import KeyBERT
@@ -907,11 +829,11 @@ def chatbot_slim(question, context, keywords):
@st.cache_resource(show_spinner=True)
-def create_db(texts_raw, uploaded_file_name):
+def create_db(texts_raw,_uploaded_file_name):
hf= OpenAIEmbeddings(model="text-embedding-ada-002", openai_api_key=openai.api_key)
- db = FAISS.from_documents(_texts_raw, hf)
+ db = FAISS.from_documents(texts_raw, hf)
db.save_local("faiss_index_anupam" + _uploaded_file_name)
db=FAISS.load_local("faiss_index_anupam" + _uploaded_file_name, hf)
return hf, db
From 501a52f04b3a872e62d3a884a7c6fd23088e4bf1 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 2 Mar 2024 22:20:15 +0530
Subject: [PATCH 108/129] qa.py
---
qa.py | 59 +----------------------------------------------------------
1 file changed, 1 insertion(+), 58 deletions(-)
diff --git a/qa.py b/qa.py
index c4a7cfa..ae2efd3 100644
--- a/qa.py
+++ b/qa.py
@@ -344,63 +344,6 @@ def extract_image_addresses(filename):
return image_addresses
#---------------READ THE .PPTX FILE AND GENERATE THE SPLIT--------------------#
-#@st.cache_resource(show_spinner=True)
-#def readdoc_splittext_pptx(filename):
- # if ".pptx" in filename:
- # loader = UnstructuredPowerPointLoader(filename)
- # docs = extract_text_from_pptx(filename)
- #image_addresses = extract_image_addresses(filename)
- # Authenticate using service account credentials
- #credentials = service_account.Credentials.from_service_account_file('able-store-415222-3c73cfca4950.json') # Path to your JSON service account key file
- #client = vision.ImageAnnotatorClient(credentials=credentials)
- #images_text=''
- # Perform OCR on an image
- #for image_address in image_addresses:
- # with io.open(image_address, 'rb') as image_file:
- # content = image_file.read()
- #image = vision.Image(content=content)
- #response = client.text_detection(image=image)
- #texts = response.text_annotations
- #for text in texts:
- # print(text.description)
- #pat1= re.compile(r".+\:")
- #pat2=re.compile(r".+\.\n")
- #headings_list=pat1.findall(docs)
- #paragraph_list=pat2.findall(docs)
- #n = 1500 #Number of characters to be included in a single chunk of text
- #all_text=''
- #for text in paragraph_list:
- # all_text+=text
- #all_text+=images_text
- #a=glob.glob(filename)
- # #print(a)
- #chunk_size = 1024
- #chunk_overlap=10
- #text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
- #texts_isb=[]
- #texts_raw = []
- #documents=[]
- #for i in range(len(a)):
- # documents.extend(UnstructuredPowerPointLoader(a[i]).load())
- # for j in range(
- # len(
- # text_splitter.split_documents(
- # UnstructuredPowerPointLoader(a[i]).load()
- # )
- # )
- # ):
- # text_chunk = text_splitter.split_documents(
- # UnstructuredPowerPointLoader(a[i]).load()
- # )[j]
- # text_chunk.page_content = text_chunk.page_content.replace("\n", " ")
- # text_chunk.page_content = text_chunk.page_content.replace("\\n", " ")
- # text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
- # text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
- # texts_isb.append(text_chunk.page_content)
- # texts_raw.append(text_chunk)
-
-# text_split = texts_isb
-# return all_text, text_split, texts_raw, headings_list, paragraph_list
@st.cache_resource(show_spinner=True)
def readdoc_splittext_pptx(filename):
if ".pptx" in filename:
@@ -540,7 +483,7 @@ def readdoc_splittext_txt(filename):
text_split = texts_isb
return all_text, text_split, texts_raw, headings_list, paragraph_list
-
+#---------------READ THE .Pdf FILE AND GENERATE THE SPLIT--------------------#
@st.cache_resource(show_spinner=True)
def readdoc_splittext_pdf(filename):
"""
From dc02c9292dfc8e028bb0ee03a99fc040ee8a7475 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 10:05:47 +0530
Subject: [PATCH 109/129] requirements.txt
---
requirements.txt | 1 -
1 file changed, 1 deletion(-)
diff --git a/requirements.txt b/requirements.txt
index 6daa008..fcf232d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -36,7 +36,6 @@ langchain
sentence_transformers
tiktoken
pdfminer.six
-openai
faiss-cpu
unstructured
keybert[flair]
From 697432b20e2e09d2ce041c211a35954ea7b66b93 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 20:03:25 +0530
Subject: [PATCH 110/129] Update qa.py
---
qa.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/qa.py b/qa.py
index ae2efd3..11d6345 100644
--- a/qa.py
+++ b/qa.py
@@ -203,6 +203,7 @@ def readdoc_splittext(filename):
)
texts_isb = []
texts_chunk = []
+ texts_raw=[]
documents = []
for i in range(len(a)):
documents.extend(UnstructuredWordDocumentLoader(a[i]).load())
@@ -221,9 +222,9 @@ def readdoc_splittext(filename):
text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
texts_isb.append(text_chunk.page_content)
- texts_chunk.append(text_chunk)
+ texts_raw.append(text_chunk)
text_split = texts_isb
- return all_text, text_split, texts_chunk, headings, para_texts
+ return all_text, text_split, texts_raw, headings, para_texts
def remove_newlines(serie):
From daba6c6fa58dc0052ea1d8cb75257c0d9745bdde Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 21:35:34 +0530
Subject: [PATCH 111/129] Update qa.py
---
qa.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 11d6345..25a0026 100644
--- a/qa.py
+++ b/qa.py
@@ -222,8 +222,9 @@ def readdoc_splittext(filename):
text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
texts_isb.append(text_chunk.page_content)
- texts_raw.append(text_chunk)
+ texts_chunk.append(text_chunk)
text_split = texts_isb
+ texts_raw = text_chunk
return all_text, text_split, texts_raw, headings, para_texts
From 607434c4716d05cb92d32146e358b373ff4f8b85 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 21:37:27 +0530
Subject: [PATCH 112/129] streamlit_app.py
---
streamlit_app.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 9700f3c..15cf4b2 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -235,9 +235,9 @@ def calculate_rouge_scores(answer,context):
filename = file_path
if ".docx" in filename: #uploaded_file.name:
- all_text, text_split, text_chunk, headings, para_texts = readdoc_splittext(filename)#uploaded_file.name)
+ all_text, text_split, texts_raw, headings, para_texts = readdoc_splittext(filename)#uploaded_file.name)
elif (".doc" in filename) and (".docx" not in filename): #uploaded_file.name:
- all_text, text_split, text_chunk, headings, para_texts = readdoc_splittext(filename)#uploaded_file.name)
+ all_text, text_split, texts_raw, headings, para_texts = readdoc_splittext(filename)#uploaded_file.name)
elif ".txt" in filename:
all_text, text_split, texts_raw, headings_list, paragraph_list = readdoc_splittext_txt(filename)#uploaded_file.name)
elif ".pptx" in filename: #uploaded_file.name:
From c05331e55fe694649aa4aec7dd9dbe2e41d2cb98 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 21:44:16 +0530
Subject: [PATCH 113/129] streamlit_app.py
---
streamlit_app.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 15cf4b2..4d4b3a2 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -248,7 +248,7 @@ def calculate_rouge_scores(answer,context):
with st.chat_message("assistant"):
st.write("Hi! Getting your contexts ready for query! Please wait!")
- hf, db = create_db(texts_raw, uploaded_file_name)
+ hf, db = create_db(texts_raw,_uploaded_file_name)
st.session_state["db_created"] = True
From 21fd2edcde3893e6a3ca792cffaa6cbf46effb06 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 21:45:00 +0530
Subject: [PATCH 114/129] streamlit_app.py
---
streamlit_app.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index 4d4b3a2..ed7e586 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -248,7 +248,7 @@ def calculate_rouge_scores(answer,context):
with st.chat_message("assistant"):
st.write("Hi! Getting your contexts ready for query! Please wait!")
- hf, db = create_db(texts_raw,_uploaded_file_name)
+ hf, db = create_db(texts_raw,uploaded_file_name)
st.session_state["db_created"] = True
From b102a8ec108b1749f7aac1c6ad4a52c02dc400b2 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 21:49:56 +0530
Subject: [PATCH 115/129] qa.py
From 25a2b780cf9de7c49210efa403d590d800bf7922 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 21:50:13 +0530
Subject: [PATCH 116/129] streamlit_app.py
---
streamlit_app.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index ed7e586..faecac5 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -248,7 +248,7 @@ def calculate_rouge_scores(answer,context):
with st.chat_message("assistant"):
st.write("Hi! Getting your contexts ready for query! Please wait!")
- hf, db = create_db(texts_raw,uploaded_file_name)
+ hf, db = create_db(texts_raw,uploaded_file.name)
st.session_state["db_created"] = True
From acc3a8a82366ededab3d07943c9a8087b751a980 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 21:53:35 +0530
Subject: [PATCH 117/129] qa.py
---
qa.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/qa.py b/qa.py
index 25a0026..b0813a2 100644
--- a/qa.py
+++ b/qa.py
@@ -202,7 +202,7 @@ def readdoc_splittext(filename):
chunk_size=chunk_size, chunk_overlap=chunk_overlap
)
texts_isb = []
- texts_chunk = []
+ text_chunk = []
texts_raw=[]
documents = []
for i in range(len(a)):
@@ -222,9 +222,9 @@ def readdoc_splittext(filename):
text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
texts_isb.append(text_chunk.page_content)
- texts_chunk.append(text_chunk)
+ text_chunk.append(text_chunk)
text_split = texts_isb
- texts_raw = text_chunk
+ texts_raw=text_chunk
return all_text, text_split, texts_raw, headings, para_texts
From a8db894b0e342fb9029822151a501108295b0134 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 21:54:45 +0530
Subject: [PATCH 118/129] qa.py
---
qa.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/qa.py b/qa.py
index b0813a2..946a497 100644
--- a/qa.py
+++ b/qa.py
@@ -202,7 +202,7 @@ def readdoc_splittext(filename):
chunk_size=chunk_size, chunk_overlap=chunk_overlap
)
texts_isb = []
- text_chunk = []
+ texts_chunk = []
texts_raw=[]
documents = []
for i in range(len(a)):
@@ -222,9 +222,9 @@ def readdoc_splittext(filename):
text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
texts_isb.append(text_chunk.page_content)
- text_chunk.append(text_chunk)
+ texts_chunk.append(text_chunk)
text_split = texts_isb
- texts_raw=text_chunk
+ texts_raw=texts_chunk
return all_text, text_split, texts_raw, headings, para_texts
From 318767e42603816fd1a50b96b12842dfc4abb1ca Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 21:55:43 +0530
Subject: [PATCH 119/129] qa.py
---
qa.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/qa.py b/qa.py
index 946a497..1c38da4 100644
--- a/qa.py
+++ b/qa.py
@@ -224,8 +224,7 @@ def readdoc_splittext(filename):
texts_isb.append(text_chunk.page_content)
texts_chunk.append(text_chunk)
text_split = texts_isb
- texts_raw=texts_chunk
- return all_text, text_split, texts_raw, headings, para_texts
+ return all_text, text_split, texts_chunk, headings, para_texts
def remove_newlines(serie):
From eccf04782c52b68f23a0cf1e404e87e295f2ab9e Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 22:01:23 +0530
Subject: [PATCH 120/129] qa.py
---
qa.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/qa.py b/qa.py
index 1c38da4..8d079f5 100644
--- a/qa.py
+++ b/qa.py
@@ -202,7 +202,6 @@ def readdoc_splittext(filename):
chunk_size=chunk_size, chunk_overlap=chunk_overlap
)
texts_isb = []
- texts_chunk = []
texts_raw=[]
documents = []
for i in range(len(a)):
@@ -222,9 +221,9 @@ def readdoc_splittext(filename):
text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
texts_isb.append(text_chunk.page_content)
- texts_chunk.append(text_chunk)
+ texts_raw.append(text_chunk)
text_split = texts_isb
- return all_text, text_split, texts_chunk, headings, para_texts
+ return all_text, text_split, texts_raw, headings, para_texts
def remove_newlines(serie):
From ce44c25f2008076dcfaa3c03da21ff9f46d4ac3e Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 22:02:30 +0530
Subject: [PATCH 121/129] Update qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 8d079f5..1434775 100644
--- a/qa.py
+++ b/qa.py
@@ -772,7 +772,7 @@ def chatbot_slim(question, context, keywords):
@st.cache_resource(show_spinner=True)
-def create_db(texts_raw,_uploaded_file_name):
+def create_db(_texts_raw,_uploaded_file_name):
hf= OpenAIEmbeddings(model="text-embedding-ada-002", openai_api_key=openai.api_key)
From b542ab8cb42bc0e05b47b2a1c53e87559ab94338 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 22:03:14 +0530
Subject: [PATCH 122/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 1434775..1f7ec16 100644
--- a/qa.py
+++ b/qa.py
@@ -776,7 +776,7 @@ def create_db(_texts_raw,_uploaded_file_name):
hf= OpenAIEmbeddings(model="text-embedding-ada-002", openai_api_key=openai.api_key)
- db = FAISS.from_documents(texts_raw, hf)
+ db = FAISS.from_documents(_texts_raw, hf)
db.save_local("faiss_index_anupam" + _uploaded_file_name)
db=FAISS.load_local("faiss_index_anupam" + _uploaded_file_name, hf)
return hf, db
From 5d24ec2f94930eec0e838e68f5cee83cdf7e53da Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 22:04:43 +0530
Subject: [PATCH 123/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 1f7ec16..26eab05 100644
--- a/qa.py
+++ b/qa.py
@@ -771,7 +771,7 @@ def chatbot_slim(question, context, keywords):
-@st.cache_resource(show_spinner=True)
+@st.cache_resource(allow_dangerous_deserialization=True)
def create_db(_texts_raw,_uploaded_file_name):
From 29bc605561a8386b2bd1027a70c206afbc1bfce0 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 22:07:53 +0530
Subject: [PATCH 124/129] \qa.py
---
qa.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/qa.py b/qa.py
index 26eab05..0ed7774 100644
--- a/qa.py
+++ b/qa.py
@@ -202,7 +202,7 @@ def readdoc_splittext(filename):
chunk_size=chunk_size, chunk_overlap=chunk_overlap
)
texts_isb = []
- texts_raw=[]
+ texts_chunk=[]
documents = []
for i in range(len(a)):
documents.extend(UnstructuredWordDocumentLoader(a[i]).load())
@@ -221,9 +221,9 @@ def readdoc_splittext(filename):
text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
text_chunk.page_content = text_chunk.page_content.replace(" ", " ")
texts_isb.append(text_chunk.page_content)
- texts_raw.append(text_chunk)
+ texts_chunk.append(text_chunk)
text_split = texts_isb
- return all_text, text_split, texts_raw, headings, para_texts
+ return all_text, text_split, texts_chunk, headings, para_texts
def remove_newlines(serie):
@@ -772,11 +772,11 @@ def chatbot_slim(question, context, keywords):
@st.cache_resource(allow_dangerous_deserialization=True)
-def create_db(_texts_raw,_uploaded_file_name):
+def create_db(texts_raw,_uploaded_file_name):
hf= OpenAIEmbeddings(model="text-embedding-ada-002", openai_api_key=openai.api_key)
- db = FAISS.from_documents(_texts_raw, hf)
+ db = FAISS.from_documents(texts_raw, hf)
db.save_local("faiss_index_anupam" + _uploaded_file_name)
db=FAISS.load_local("faiss_index_anupam" + _uploaded_file_name, hf)
return hf, db
From 0f0bb4c36102f86b6e6b53590bde9ff367988532 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 22:13:52 +0530
Subject: [PATCH 125/129] streamlit_app.py
---
streamlit_app.py | 19 +++++++++++++------
1 file changed, 13 insertions(+), 6 deletions(-)
diff --git a/streamlit_app.py b/streamlit_app.py
index faecac5..1d8aa2b 100644
--- a/streamlit_app.py
+++ b/streamlit_app.py
@@ -235,21 +235,28 @@ def calculate_rouge_scores(answer,context):
filename = file_path
if ".docx" in filename: #uploaded_file.name:
- all_text, text_split, texts_raw, headings, para_texts = readdoc_splittext(filename)#uploaded_file.name)
+ all_text, text_split, texts_chunk, headings, para_texts = readdoc_splittext(filename)#uploaded_file.name)
elif (".doc" in filename) and (".docx" not in filename): #uploaded_file.name:
- all_text, text_split, texts_raw, headings, para_texts = readdoc_splittext(filename)#uploaded_file.name)
+ all_text, text_split, texts_chunk, headings, para_texts = readdoc_splittext(filename)#uploaded_file.name)
elif ".txt" in filename:
all_text, text_split, texts_raw, headings_list, paragraph_list = readdoc_splittext_txt(filename)#uploaded_file.name)
elif ".pptx" in filename: #uploaded_file.name:
all_text, text_split, texts_raw, headings_list, paragraph_list = readdoc_splittext_pptx(filename)#uploaded_file.name)
elif ".pdf" in filename: #uploaded_file.name:
all_text, text_split, texts_raw, headings_list, paragraph_list = readdoc_splittext_pdf(filename)#uploaded_file.name)
-
with st.chat_message("assistant"):
st.write("Hi! Getting your contexts ready for query! Please wait!")
-
- hf, db = create_db(texts_raw,uploaded_file.name)
-
+ if ".docx" in filename: #uploaded_file.name:
+ hf, db = create_db(texts_chunk,uploaded_file.name)
+ elif (".doc" in filename) and (".docx" not in filename): #uploaded_file.name:
+ hf, db = create_db(texts_chunk,uploaded_file.name)
+ elif ".txt" in filename:
+ hf, db = create_db(texts_raw,uploaded_file.name)
+ elif ".pptx" in filename: #uploaded_file.name:
+ hf, db = create_db(texts_raw,uploaded_file.name)
+ elif ".pdf" in filename: #uploaded_file.name:
+ hf, db = create_db(texts_raw,uploaded_file.name)
+
st.session_state["db_created"] = True
if uploaded_file is not None and st.session_state["db_created"] is True:
From 8a4540d68b39f98dcaf627b40034b7ba59fa35a5 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 22:15:51 +0530
Subject: [PATCH 126/129] Update qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 0ed7774..92d7020 100644
--- a/qa.py
+++ b/qa.py
@@ -771,7 +771,7 @@ def chatbot_slim(question, context, keywords):
-@st.cache_resource(allow_dangerous_deserialization=True)
+@st.cache_resource(show_spinner=True)
def create_db(texts_raw,_uploaded_file_name):
From f531a26f90eb7e21b4cb8dde5a910741d264c363 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 22:28:16 +0530
Subject: [PATCH 127/129] qa.py
---
qa.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/qa.py b/qa.py
index 92d7020..09f3301 100644
--- a/qa.py
+++ b/qa.py
@@ -772,11 +772,11 @@ def chatbot_slim(question, context, keywords):
@st.cache_resource(show_spinner=True)
-def create_db(texts_raw,_uploaded_file_name):
+def create_db(_texts_raw,_uploaded_file_name):
hf= OpenAIEmbeddings(model="text-embedding-ada-002", openai_api_key=openai.api_key)
- db = FAISS.from_documents(texts_raw, hf)
+ db = FAISS.from_documents(_texts_raw, hf)
db.save_local("faiss_index_anupam" + _uploaded_file_name)
db=FAISS.load_local("faiss_index_anupam" + _uploaded_file_name, hf)
return hf, db
From 513dceb6687c7b58952c959f65f5a4cdb677e73e Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 22:40:10 +0530
Subject: [PATCH 128/129] Update qa.py
---
qa.py | 94 +----------------------------------------------------------
1 file changed, 1 insertion(+), 93 deletions(-)
diff --git a/qa.py b/qa.py
index 09f3301..9b61bef 100644
--- a/qa.py
+++ b/qa.py
@@ -492,12 +492,6 @@ def readdoc_splittext_pdf(filename):
splits them based on the chunks needed.
"""
- #print("reading files")
- #print(filename)
- #print(os.path.realpath("./"), os.getcwd())
- #print(os.listdir("./"))
- #for file in os.listdir(os.getcwd()):
- #document = os.path.join("tempDir/", filename)
doc = fitz.open(filename)
@@ -604,92 +598,6 @@ def remove_newlines(s):
print("Context:", context)
print("Keywords:", keywords)"""
-
-
-
-# ------------------------SLIM KAR BASED CHATBOT----------------------------#
-# ------------------------SLIM KAR BASED CHATBOT----------------------------#
-#def chatbot_slim(query, context, keywords):#text_split, headings, para_texts):
-# """
-# Here, this function takes in the textual query, along with the textual context and uses KAR framework to geerate a suitable response
-# with little to almost no hallucinations. Here, openai's davinci-003 has been used to generate the response.
-# """
-#
-# if input:
-# stime = time.time()
-#
-# #context, keywords = create_context(query, text_split, headings, para_texts)
-
-# ctype = ["stuff", "map_reduce", "refine", "map_rerank"]
-# template = """
-# You are a helpful assistant who answers question based on context provided: {context}
-
-# If you don't have enough information to answer the question, say: "Sorry, I cannot answer that".
-
-# """
-# template = """
-# You are a helpful assistant who answers question based on context provided: {context}
-
-# If you don't have enough information to answer the question, say: "I cannot answer".
-
-# """
-# template = """ You answer question based on context below, and if question can't be answered based on context, say \"I don't know\"\n\nContext: {context} """
-
-# system_message_prompt = SystemMessagePromptTemplate.from_template(template)
-
-# # Human question prompt
-
-# human_template = "Answer following question: {question}"
-
-# template = """ Answer question {question} based on context below, and if question can't be answered based on context,
-# say \"I don't know\"\n\nContext: {context}
-
-# Answer:
-# """
-
-# template = """ Use following pieces of context to answer the question. Provide answer in full detail using provided context.
-# If you don't know the answer, say I don't know
-# {context}
-# Question : {question}
-# Answer:"""
-
-# human_message_prompt = HumanMessagePromptTemplate.from_template(human_template)
-
-# chat_prompt = ChatPromptTemplate.from_messages(
-# [system_message_prompt, human_message_prompt]
-# )
-
-# chunk_size = 1500
-# PROMPT = PromptTemplate(
-# input_variables=["context", "question"], template=template
-# )
-#
-# chain_type_kwargs = {"prompt": PROMPT}
-
-# question = query
-# context = context
-# openai.api_key = SECRET_TOKEN
-# model = "gpt-3.5-turbo-instruct"
-# chat = openai.Completion.create(
-# #prompt=f"You answer question based on context below, and if the question can't be answered based on the context,
-# #say \"I don't know\"\n\nContext: {context}\n\n---\n\nQuestion: {question}\nAnswer:",
-# prompt=f"""You are a question answering assistant with no previous information,
-# you answer question based on following context and if question cannot be answered based on context,
-# say \"I don't know\"\n\nContext: {context}\n\n---\n\nQuestion: {question}\nAnswer:""",
-# temperature=0,
-# max_tokens=2000,
-# top_p=1,
-# frequency_penalty=0,
-# presence_penalty=0,
-# stop=None,
-# model=model,
-# )
-
-# reply = chat["choices"][0][
-# "text"
-# ].strip() # ['choices'][0]['message']['content']
-
-# return reply, context, keywords
from keybert import KeyBERT
import openai
@@ -771,7 +679,7 @@ def chatbot_slim(question, context, keywords):
-@st.cache_resource(show_spinner=True)
+@st.cache_resource(show_spinner=True,allow_dangerous_deserialization = True)
def create_db(_texts_raw,_uploaded_file_name):
From 5ee3ff3285766940b8df94816d66b2f1df4798b9 Mon Sep 17 00:00:00 2001
From: Saikrishna-Bulusu <142135673+Saikrishna-Bulusu@users.noreply.github.com>
Date: Sat, 13 Apr 2024 22:41:20 +0530
Subject: [PATCH 129/129] qa.py
---
qa.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/qa.py b/qa.py
index 9b61bef..0831e33 100644
--- a/qa.py
+++ b/qa.py
@@ -679,7 +679,7 @@ def chatbot_slim(question, context, keywords):
-@st.cache_resource(show_spinner=True,allow_dangerous_deserialization = True)
+@st.cache_resource(show_spinner=True)
def create_db(_texts_raw,_uploaded_file_name):