-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathapp.py
More file actions
125 lines (93 loc) · 3.79 KB
/
Copy pathapp.py
File metadata and controls
125 lines (93 loc) · 3.79 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
import os
import sys
# Add utils to path
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
from utils.pdf_loader import load_pdfs
from utils.text_splitter import split_text
from utils.embeddings import create_embeddings, model
from utils.vector_store import create_vector_store, search_vector_store, save_vector_store, load_vector_store
from utils.qa_model import get_answer
def main():
print("\n" + "="*60)
print("📚 DOCQUERY - Document Question Answering System")
print("="*60 + "\n")
# Check if vector store exists
index, chunks = load_vector_store()
if index is None:
print("First time setup: Processing PDF files...")
# STEP 1 — Load PDFs
data_folder = "data"
if not os.path.exists(data_folder):
print(f"Error: '{data_folder}' folder not found!")
print(f"Please create a '{data_folder}' folder and add PDF files.")
return
pdf_files = []
for file in os.listdir(data_folder):
if file.endswith(".pdf"):
pdf_files.append(os.path.join(data_folder, file))
if not pdf_files:
print(f"Error: No PDF files found in '{data_folder}' folder!")
return
print(f"\n📄 PDF Files Found: {len(pdf_files)}")
for f in pdf_files:
print(f" - {os.path.basename(f)}")
# STEP 2 — Load Text
print("\n📖 Extracting text from PDFs...")
text = load_pdfs(pdf_files)
if not text:
print("Error: No text extracted from PDFs!")
return
print(f"✅ Total text length: {len(text)} characters")
# STEP 3 — Split Text
print("\n✂️ Splitting text into chunks...")
chunks = split_text(text)
if not chunks:
print("Error: Text splitting failed!")
return
# STEP 4 — Create Embeddings
print("\n🔢 Creating embeddings...")
embeddings = create_embeddings(chunks)
# STEP 5 — Create Vector Store
print("\n🗄️ Creating FAISS vector store...")
index = create_vector_store(embeddings)
# Save for future use
save_vector_store(index, chunks)
print("\n✅ Vector store saved for future use!")
else:
print("✅ Loaded existing vector store!")
print("\n" + "="*60)
print("💬 Ready to answer your questions!")
print("Type 'quit' or 'exit' to stop")
print("="*60 + "\n")
# Interactive Q&A loop
while True:
query = input("❓ Your question: ").strip()
if query.lower() in ['quit', 'exit', 'q']:
print("\n👋 Goodbye!")
break
if not query:
print("Please enter a valid question.\n")
continue
print("\n🔍 Searching for relevant information...")
# Generate query embedding
query_embedding = model.encode(query)
# Search vector store
results, distances = search_vector_store(index, query_embedding, k=5)
# Build context
top_indexes = results[0]
context_chunks = []
for idx in top_indexes:
chunk = chunks[idx]
# Clean text
chunk = chunk.replace("\n", " ")
chunk = chunk.replace("•", " ")
chunk = " ".join(chunk.split())
context_chunks.append(chunk)
best_context = " ".join(context_chunks)
# Generate answer
print("🤖 Generating answer...\n")
answer = get_answer(query, best_context)
print(f"📝 Answer: {answer}\n")
print("-"*60 + "\n")
if __name__ == "__main__":
main()