From 4a7a943f024a6efabd23ab282ae9aef5fa29dd38 Mon Sep 17 00:00:00 2001 From: Shantanu Date: Thu, 30 Jul 2026 20:33:37 +0530 Subject: [PATCH 1/6] Added config.py as a centeral file via pydantic.settings and made appropraite changes. --- backend/utils/config.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 backend/utils/config.py diff --git a/backend/utils/config.py b/backend/utils/config.py new file mode 100644 index 00000000..bdcbf223 --- /dev/null +++ b/backend/utils/config.py @@ -0,0 +1,26 @@ +from pydantic_settings import BaseSettings, SettingsConfigDict +from pydantic import Field +from typing import List +from pathlib import Path + +class Settings(BaseSettings): + frontend_dist: Path = Field(default=Path("/app/frontend/dist"), alias="FRONTEND_DIST") + cors_origins: str = Field(default="http://localhost:3000,http://127.0.0.1:3000,http://localhost:5173,http://localhost:8000", alias="CORS_ORIGINS") + upload_dir: Path = Field(default=Path("./data/uploads"), alias="UPLOAD_DIR") + settings_api_timeout_seconds: int = Field(default=10, alias="SETTINGS_API_TIMEOUT_SECONDS") + audit_log_dir: Path = Field(default=Path("./data/logs"), alias="AUDIT_LOG_DIR") + db_vacuum_threshold: int = Field(default=500, alias="DB_VACUUM_THRESHOLD") + db_path: Path = Field(default=Path("./data/localmind.db"), alias="DB_PATH") + ollama_host: str = Field(default="http://localhost:11434", alias="OLLAMA_HOST") + chromadb_dir: Path = Field(default=Path("./data/chromadb"), alias="CHROMADB_DIR") + + # Values from .env.example + default_model: str = Field(default="llama3", alias="DEFAULT_MODEL") + exports_dir: Path = Field(default=Path("./data/exports"), alias="EXPORTS_DIR") + max_file_size: int = Field(default=52428800, alias="MAX_FILE_SIZE") + backend_port: int = Field(default=8000, alias="BACKEND_PORT") + frontend_port: int = Field(default=3000, alias="FRONTEND_PORT") + + model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") + +settings = Settings() From bcfc3f5b6756544448554f6d197829bccd436a0a Mon Sep 17 00:00:00 2001 From: Shantanu Date: Sat, 1 Aug 2026 13:07:32 +0530 Subject: [PATCH 2/6] feat: integrate pydantic-settings config across backend --- backend/app.py | 6 +++--- backend/routes/settings.py | 12 ++---------- backend/routes/upload.py | 5 +++-- backend/services/db_service.py | 7 ++++--- backend/services/ollama_service.py | 4 +++- backend/services/rag_service.py | 4 +++- backend/utils/audit_log.py | 3 ++- 7 files changed, 20 insertions(+), 21 deletions(-) diff --git a/backend/app.py b/backend/app.py index 13e3539e..61dd9b33 100644 --- a/backend/app.py +++ b/backend/app.py @@ -27,6 +27,7 @@ from routes.settings import router as settings_router from routes.upload import router as upload_router from services.db_service import get_db, init_db +from utils.config import settings # --- Issue #284 Engine Stability: Contextual Thread-Safe Log Formatter --- @@ -50,7 +51,7 @@ def format(self, record): root_logger.handlers = [stream_handler] logger = logging.getLogger(__name__) -FRONTEND_DIST = Path(os.getenv("FRONTEND_DIST", "/app/frontend/dist")) +FRONTEND_DIST = settings.frontend_dist def run_preflight_checks(): @@ -131,10 +132,9 @@ async def add_request_correlation_id(request: Request, call_next): return response -default_cors_origins = "http://localhost:3000,http://127.0.0.1:3000,http://localhost:5173,http://localhost:8000" cors_origins = [ origin.strip() - for origin in os.getenv("CORS_ORIGINS", default_cors_origins).split(",") + for origin in settings.cors_origins.split(",") if origin.strip() ] diff --git a/backend/routes/settings.py b/backend/routes/settings.py index e1605b64..15036636 100644 --- a/backend/routes/settings.py +++ b/backend/routes/settings.py @@ -10,6 +10,7 @@ from fastapi import APIRouter, HTTPException, status from models.schemas import AppSettings from services.db_service import get_settings, save_setting, save_settings +from utils.config import settings router = APIRouter() logger = logging.getLogger(__name__) @@ -18,16 +19,7 @@ def _resolve_settings_timeout_seconds() -> float: - raw_timeout = os.getenv("SETTINGS_API_TIMEOUT_SECONDS", str(DEFAULT_SETTINGS_API_TIMEOUT_SECONDS)) - try: - parsed_timeout = float(raw_timeout) - except ValueError: - logger.warning( - "settings_timeout_invalid raw_value=%s fallback_timeout_s=%s", - raw_timeout, - DEFAULT_SETTINGS_API_TIMEOUT_SECONDS, - ) - return DEFAULT_SETTINGS_API_TIMEOUT_SECONDS + parsed_timeout = float(settings.settings_api_timeout_seconds) if parsed_timeout <= 0: logger.warning( "settings_timeout_non_positive timeout_s=%s fallback_timeout_s=%s", diff --git a/backend/routes/upload.py b/backend/routes/upload.py index ec9707b9..aeaf1249 100644 --- a/backend/routes/upload.py +++ b/backend/routes/upload.py @@ -4,6 +4,7 @@ import time import traceback from pathlib import Path +from utils.config import settings from fastapi import ( APIRouter, @@ -36,9 +37,9 @@ def _safe_audit(fn, **kwargs): ".htm", ".csv", ".json", ".xml", ".rtf", ".odt", ".epub", ".log", ".tsv", ".ini", ".cfg", ".yaml", ".yml" } -MAX_BYTES = 50 * 1024 * 1024 # 50 MB +MAX_BYTES = settings.max_file_size -UPLOAD_DIR = Path(os.getenv("UPLOAD_DIR", "./data/uploads")) +UPLOAD_DIR = settings.upload_dir UPLOAD_DIR.mkdir(parents=True, exist_ok=True) diff --git a/backend/services/db_service.py b/backend/services/db_service.py index 0619a226..8530d6bc 100644 --- a/backend/services/db_service.py +++ b/backend/services/db_service.py @@ -13,9 +13,10 @@ from sqlite3 import OperationalError import grapheme +from utils.config import settings # ------------------------Vacuum Scheduling-------------------------------------------------------- -VACUUM_THRESHOLD = int(os.getenv("DB_VACUUM_THRESHOLD", "500")) +VACUUM_THRESHOLD = settings.db_vacuum_threshold def _get_deleted_counter(conn) -> int: row = conn.execute( @@ -115,7 +116,7 @@ def restore_db(src_path: str) -> None: -DB_PATH = os.getenv("DB_PATH", "./data/localmind.db") +DB_PATH = str(settings.db_path) os.makedirs(os.path.dirname(DB_PATH) if os.path.dirname(DB_PATH) else ".", exist_ok=True) logger = logging.getLogger(__name__) @@ -247,7 +248,7 @@ def init_db(): ); INSERT OR IGNORE INTO app_settings (key, value) VALUES - ('default_model', '"llama3"'), + ('default_model', '""" + json.dumps(settings.default_model) + """'), ('default_language', '"en"'), ('temperature', '0.7'), ('max_history_turns', '10'), diff --git a/backend/services/ollama_service.py b/backend/services/ollama_service.py index cdb04f81..3ed2f1a7 100644 --- a/backend/services/ollama_service.py +++ b/backend/services/ollama_service.py @@ -12,9 +12,11 @@ from utils.cache import TTLCache from utils.retry import with_retry +from utils.config import settings + logger = logging.getLogger(__name__) -OLLAMA_BASE_URL = os.getenv("OLLAMA_HOST", "http://localhost:11434").rstrip("/") +OLLAMA_BASE_URL = settings.ollama_host.rstrip("/") TIMEOUT = 180.0 SYSTEM_PROMPTS = { diff --git a/backend/services/rag_service.py b/backend/services/rag_service.py index eda47862..d0c81797 100644 --- a/backend/services/rag_service.py +++ b/backend/services/rag_service.py @@ -21,9 +21,11 @@ from services.csv_loader import CleanCSVLoader from services.docx_loader import DocxWithTablesLoader +from utils.config import settings + logger = logging.getLogger(__name__) -CHROMA_PATH = os.getenv("CHROMADB_DIR", "./data/chromadb") +CHROMA_PATH = str(settings.chromadb_dir) EMBED_MODEL = "all-MiniLM-L6-v2" os.makedirs(CHROMA_PATH, exist_ok=True) diff --git a/backend/utils/audit_log.py b/backend/utils/audit_log.py index 0f94ab11..1bae7fb0 100644 --- a/backend/utils/audit_log.py +++ b/backend/utils/audit_log.py @@ -17,9 +17,10 @@ from contextlib import suppress from typing import Any +from utils.config import settings _module_logger = logging.getLogger(__name__) -AUDIT_LOG_DIR = os.getenv("AUDIT_LOG_DIR", "./data/logs") +AUDIT_LOG_DIR = str(settings.audit_log_dir) AUDIT_LOG_FILE = os.path.join(AUDIT_LOG_DIR, "audit.jsonl") _audit_logger = logging.getLogger("audit.upload_queue") From 7b21143f80b523f63d21878f6e2142f1525740e1 Mon Sep 17 00:00:00 2001 From: Shantanu Date: Sat, 1 Aug 2026 13:22:09 +0530 Subject: [PATCH 3/6] fix: parameterize default_model seed and add pydantic-settings to requirements --- backend/requirements.txt | 1 + backend/requirements_fixed.txt | 1 + backend/services/db_service.py | 5 ++++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index 4c8ef293..8e80d9c0 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -10,6 +10,7 @@ python-docx==1.1.2 unstructured==0.18.31 python-multipart==0.0.9 pydantic==2.9.0 +pydantic-settings==2.5.2 python-dotenv==1.0.1 httpx==0.27.0 pytest==8.3.0 diff --git a/backend/requirements_fixed.txt b/backend/requirements_fixed.txt index 38f62834..5b9fc1bc 100644 --- a/backend/requirements_fixed.txt +++ b/backend/requirements_fixed.txt @@ -9,6 +9,7 @@ pypdf==4.3.0 python-docx==1.1.2 python-multipart==0.0.9 pydantic==2.9.0 +pydantic-settings==2.5.2 python-dotenv==1.0.1 httpx==0.27.0 pytest==8.3.0 diff --git a/backend/services/db_service.py b/backend/services/db_service.py index 8530d6bc..535aa359 100644 --- a/backend/services/db_service.py +++ b/backend/services/db_service.py @@ -248,7 +248,6 @@ def init_db(): ); INSERT OR IGNORE INTO app_settings (key, value) VALUES - ('default_model', '""" + json.dumps(settings.default_model) + """'), ('default_language', '"en"'), ('temperature', '0.7'), ('max_history_turns', '10'), @@ -258,6 +257,10 @@ def init_db(): """) + conn.execute( + "INSERT OR IGNORE INTO app_settings (key, value) VALUES (?, ?)", + ("default_model", json.dumps(settings.default_model)) + ) try: conn.execute("ALTER TABLE documents ADD COLUMN status TEXT DEFAULT 'completed'") except sqlite3.OperationalError: From 2fd4f353cef7f82e0efff517d464280caefb4e1b Mon Sep 17 00:00:00 2001 From: Shantanu Date: Sat, 1 Aug 2026 14:11:52 +0530 Subject: [PATCH 4/6] style: apply ruff format and lint fixes --- backend/app.py | 66 +++++---- backend/routes/settings.py | 59 ++++---- backend/routes/upload.py | 120 ++++++++++++----- backend/services/db_service.py | 208 ++++++++++++++++++++--------- backend/services/ollama_service.py | 74 +++++----- backend/services/rag_service.py | 70 ++++++---- backend/utils/audit_log.py | 25 +++- backend/utils/config.py | 24 +++- 8 files changed, 430 insertions(+), 216 deletions(-) diff --git a/backend/app.py b/backend/app.py index 61dd9b33..e8e6e2ad 100644 --- a/backend/app.py +++ b/backend/app.py @@ -10,7 +10,6 @@ import time import uuid from contextlib import asynccontextmanager -from pathlib import Path from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware @@ -38,10 +37,13 @@ def format(self, record): record.correlation_id = "GLOBAL" return super().format(record) + # Initialize a standard console stream log handler stream_handler = logging.StreamHandler() stream_handler.setFormatter( - CorrelationIdFormatter("%(asctime)s | %(levelname)s | [%(correlation_id)s] | %(name)s | %(message)s") + CorrelationIdFormatter( + "%(asctime)s | %(levelname)s | [%(correlation_id)s] | %(name)s | %(message)s" + ) ) # Apply our stream configuration directly onto the base application root logger scope @@ -95,15 +97,16 @@ def run_preflight_checks(): async def lifespan(app: FastAPI): logger.info("Starting LocalMind v2.0...") run_preflight_checks() - + # Start stream cleanup task from routes.chat import clean_expired_streams + cleanup_task = asyncio.create_task(clean_expired_streams()) - + logger.info("LocalMind v2.0 ready!") yield logger.info("👋 Shutting down...") - + # Cancel stream cleanup task cleanup_task.cancel() await asyncio.gather(cleanup_task, return_exceptions=True) @@ -116,26 +119,25 @@ async def lifespan(app: FastAPI): lifespan=lifespan, ) + # --- Issue #284: Custom Correlation Tracking Middleware Interceptor --- @app.middleware("http") async def add_request_correlation_id(request: Request, call_next): correlation_id = request.headers.get("X-Correlation-ID", f"gen-{uuid.uuid4()}") - + # Safely attach tracking state directly onto the request state context loop request.state.correlation_id = correlation_id - + extra = {"correlation_id": correlation_id} logger.info(f"Incoming Request: {request.method} {request.url.path}", extra=extra) - + response = await call_next(request) response.headers["X-Correlation-ID"] = correlation_id return response cors_origins = [ - origin.strip() - for origin in settings.cors_origins.split(",") - if origin.strip() + origin.strip() for origin in settings.cors_origins.split(",") if origin.strip() ] app.add_middleware(GZipMiddleware, minimum_size=1000) @@ -153,37 +155,49 @@ async def add_request_correlation_id(request: Request, call_next): RATE_LIMIT_WINDOW = 60 rate_limits = {} + @app.middleware("http") async def rate_limit_middleware(request: Request, call_next): client_ip = request.client.host if request.client else "127.0.0.1" current_time = time.time() - - if client_ip not in rate_limits or current_time > rate_limits[client_ip]["reset_at"]: - rate_limits[client_ip] = {"count": 0, "reset_at": current_time + RATE_LIMIT_WINDOW} - + + if ( + client_ip not in rate_limits + or current_time > rate_limits[client_ip]["reset_at"] + ): + rate_limits[client_ip] = { + "count": 0, + "reset_at": current_time + RATE_LIMIT_WINDOW, + } + rate_limits[client_ip]["count"] += 1 remaining = max(0, RATE_LIMIT - rate_limits[client_ip]["count"]) reset_time = int(rate_limits[client_ip]["reset_at"]) - + response = await call_next(request) - + response.headers["X-RateLimit-Limit"] = str(RATE_LIMIT) response.headers["X-RateLimit-Remaining"] = str(remaining) response.headers["X-RateLimit-Reset"] = str(reset_time) - + return response -app.include_router(chat_router, prefix="/api/chat", tags=["Chat"]) -app.include_router(upload_router, prefix="/api/upload", tags=["Upload"]) -app.include_router(models_router, prefix="/api/models", tags=["Models"]) + +app.include_router(chat_router, prefix="/api/chat", tags=["Chat"]) +app.include_router(upload_router, prefix="/api/upload", tags=["Upload"]) +app.include_router(models_router, prefix="/api/models", tags=["Models"]) app.include_router(sessions_router, prefix="/api/sessions", tags=["Sessions"]) -app.include_router(plugins_router, prefix="/api/plugins", tags=["Plugins"]) -app.include_router(export_router, prefix="/api/export", tags=["Export"]) +app.include_router(plugins_router, prefix="/api/plugins", tags=["Plugins"]) +app.include_router(export_router, prefix="/api/export", tags=["Export"]) app.include_router(settings_router, prefix="/api/settings", tags=["Settings"]) -app.include_router(prompt_templates_router, prefix="/api/prompt-templates", tags=["Prompt Templates"]) +app.include_router( + prompt_templates_router, prefix="/api/prompt-templates", tags=["Prompt Templates"] +) if (FRONTEND_DIST / "assets").exists(): - app.mount("/assets", StaticFiles(directory=str(FRONTEND_DIST / "assets")), name="assets") + app.mount( + "/assets", StaticFiles(directory=str(FRONTEND_DIST / "assets")), name="assets" + ) @app.get("/", tags=["Health"]) @@ -206,4 +220,4 @@ async def db_health(): conn.execute("SELECT 1") return {"status": "healthy"} except sqlite3.Error: - return {"status": "unhealthy"} \ No newline at end of file + return {"status": "unhealthy"} diff --git a/backend/routes/settings.py b/backend/routes/settings.py index 15036636..02d1e740 100644 --- a/backend/routes/settings.py +++ b/backend/routes/settings.py @@ -2,7 +2,6 @@ import asyncio import logging -import os import time from collections.abc import Callable from typing import Any @@ -33,7 +32,9 @@ def _resolve_settings_timeout_seconds() -> float: SETTINGS_API_TIMEOUT_SECONDS = _resolve_settings_timeout_seconds() -async def _run_with_timeout(operation: str, function: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: +async def _run_with_timeout( + operation: str, function: Callable[..., Any], *args: Any, **kwargs: Any +) -> Any: start_time = time.perf_counter() try: return await asyncio.wait_for( @@ -52,54 +53,65 @@ async def _run_with_timeout(operation: str, function: Callable[..., Any], *args: status_code=status.HTTP_504_GATEWAY_TIMEOUT, detail=f"Settings {operation} operation timed out after {SETTINGS_API_TIMEOUT_SECONDS} seconds.", ) from exc + + @router.get("/") async def get_all(): return await _run_with_timeout("read", get_settings) + @router.put("/") async def update_settings(body: AppSettings): # 1. Enforce safety validation boundary limits on Temperature if body.temperature < 0.0 or body.temperature > 2.0: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail=[{ - "loc": ["body", "temperature"], - "msg": "Temperature must scale cleanly between 0.0 and 2.0.", - "type": "value_error" - }] + detail=[ + { + "loc": ["body", "temperature"], + "msg": "Temperature must scale cleanly between 0.0 and 2.0.", + "type": "value_error", + } + ], ) # 2. Enforce safety validation boundary limits on RAG Context Chunks if body.rag_top_k < 1 or body.rag_top_k > 10: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail=[{ - "loc": ["body", "rag_top_k"], - "msg": "RAG Context chunks selection must stay between 1 and 10.", - "type": "value_error" - }] + detail=[ + { + "loc": ["body", "rag_top_k"], + "msg": "RAG Context chunks selection must stay between 1 and 10.", + "type": "value_error", + } + ], ) # 3. Enforce safety validation boundary limits on RAG Chunk Size if body.rag_chunk_size < 100 or body.rag_chunk_size > 2000: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail=[{ - "loc": ["body", "rag_chunk_size"], - "msg": "RAG chunk size must be between 100 and 2000 characters.", - "type": "value_error" - }] + detail=[ + { + "loc": ["body", "rag_chunk_size"], + "msg": "RAG chunk size must be between 100 and 2000 characters.", + "type": "value_error", + } + ], ) # 4. Enforce safety validation boundary limits on RAG Chunk Overlap if body.rag_chunk_overlap < 0 or body.rag_chunk_overlap > 200: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail=[{ - "loc": ["body", "rag_chunk_overlap"], - "msg": "RAG chunk overlap must be between 0 and 200 characters.", - "type": "value_error" - }] + detail=[ + { + "loc": ["body", "rag_chunk_overlap"], + "msg": "RAG chunk overlap must be between 0 and 200 characters.", + "type": "value_error", + } + ], ) current_settings = get_settings() @@ -118,6 +130,7 @@ async def update_settings(body: AppSettings): await _run_with_timeout("save", save_settings, payload) return await _run_with_timeout("read", get_settings) + @router.put("/{key}") async def update_one(key: str, value: dict): new_value = value.get("value") @@ -132,4 +145,4 @@ async def update_one(key: str, value: dict): ) await _run_with_timeout("save", save_setting, key, new_value) - return {"key": key, "updated": True} \ No newline at end of file + return {"key": key, "updated": True} diff --git a/backend/routes/upload.py b/backend/routes/upload.py index aeaf1249..c4d2b2ba 100644 --- a/backend/routes/upload.py +++ b/backend/routes/upload.py @@ -1,4 +1,5 @@ """Upload routes — /api/upload""" + import logging import os import time @@ -27,15 +28,33 @@ def _safe_audit(fn, **kwargs): try: fn(**kwargs) except Exception as e: # noqa: BLE001 - logger.warning("audit_hook_failed hook=%s error=%s", getattr(fn, "__name__", fn), e) + logger.warning( + "audit_hook_failed hook=%s error=%s", getattr(fn, "__name__", fn), e + ) router = APIRouter() ALLOWED = { - ".txt", ".md", ".pdf", ".docx", ".doc", ".html", - ".htm", ".csv", ".json", ".xml", ".rtf", ".odt", - ".epub", ".log", ".tsv", ".ini", ".cfg", ".yaml", ".yml" + ".txt", + ".md", + ".pdf", + ".docx", + ".doc", + ".html", + ".htm", + ".csv", + ".json", + ".xml", + ".rtf", + ".odt", + ".epub", + ".log", + ".tsv", + ".ini", + ".cfg", + ".yaml", + ".yml", } MAX_BYTES = settings.max_file_size @@ -44,29 +63,46 @@ def _safe_audit(fn, **kwargs): @router.post("/", response_model=UploadResponse) -async def upload(file: UploadFile = File(...), session_id: str = Form(...), background_tasks: BackgroundTasks = None): # noqa: B008 - logger.info("upload_request route=/upload session=%s file=%s", session_id, file.filename) +async def upload( + file: UploadFile = File(...), + session_id: str = Form(...), + background_tasks: BackgroundTasks = None, +): # noqa: B008 + logger.info( + "upload_request route=/upload session=%s file=%s", session_id, file.filename + ) ext = Path(file.filename).suffix.lower() if ext not in ALLOWED: - logger.warning("upload_rejected reason=unsupported_type ext=%s file=%s", ext, file.filename) + logger.warning( + "upload_rejected reason=unsupported_type ext=%s file=%s", ext, file.filename + ) raise HTTPException(status_code=400, detail=f"File type {ext} not allowed.") - + content = await file.read() if len(content) > MAX_BYTES: - logger.warning("upload_rejected reason=file_too_large size_bytes=%s limit=%s", len(content), MAX_BYTES) + logger.warning( + "upload_rejected reason=file_too_large size_bytes=%s limit=%s", + len(content), + MAX_BYTES, + ) raise HTTPException(status_code=413, detail="File too large (max 50MB).") - + file_path = UPLOAD_DIR / f"{session_id}_{file.filename}" file_path.write_bytes(content) size_kb = max(1, len(content) // 1024) - + # Restored original repository database calls: db_service.create_session(session_id) - doc_id = db_service.save_document(session_id, file.filename, str(file_path), 0, size_kb, status="queued") - + doc_id = db_service.save_document( + session_id, file.filename, str(file_path), 0, size_kb, status="queued" + ) + logger.info( "document_queued route=/upload session=%s file=%s doc_id=%s size_kb=%s", - session_id, file.filename, doc_id, size_kb, + session_id, + file.filename, + doc_id, + size_kb, ) # --- Issue #797: structured audit log — UPLOAD_QUEUED --- @@ -78,10 +114,12 @@ async def upload(file: UploadFile = File(...), session_id: str = Form(...), back ) if background_tasks: - background_tasks.add_task(process_document_task, str(file_path), session_id, doc_id) + background_tasks.add_task( + process_document_task, str(file_path), session_id, doc_id + ) else: process_document_task(str(file_path), session_id, doc_id) - + # Fixed Pydantic validation schema matching: return UploadResponse( doc_id=doc_id, @@ -89,7 +127,7 @@ async def upload(file: UploadFile = File(...), session_id: str = Form(...), back file_size_kb=size_kb, chunks_indexed=0, status="queued", - message=f"'{file.filename}' uploaded and processing started." + message=f"'{file.filename}' uploaded and processing started.", ) @@ -101,14 +139,17 @@ def process_document_task(file_path: str, session_id: str, doc_id: int): try: from services import rag_service + db_service.update_document_status(doc_id, "processing") chunks = rag_service.index_document(file_path, session_id, doc_id=doc_id) # Restored original status completion name ("completed"): db_service.update_document_status(doc_id, "completed", chunks_indexed=chunks) - + logger.info( "document_indexed route=/upload session=%s doc_id=%s chunks=%s", - session_id, doc_id, chunks, + session_id, + doc_id, + chunks, ) # --- Issue #797: structured audit log — SUCCESS --- @@ -122,7 +163,9 @@ def process_document_task(file_path: str, session_id: str, doc_id: int): except Exception as e: # noqa: BLE001 logger.error( "document_failed route=/upload session=%s doc_id=%s error=%s", - session_id, doc_id, e, + session_id, + doc_id, + e, ) # --- Issue #797: structured audit log — FAILED --- @@ -141,34 +184,51 @@ def process_document_task(file_path: str, session_id: str, doc_id: int): @router.get("/preview") async def preview_document(filename: str = Query(...), session_id: str = Query(...)): - logger.info("preview_request route=/upload/preview session=%s file=%s", session_id, filename) + logger.info( + "preview_request route=/upload/preview session=%s file=%s", session_id, filename + ) file_path = UPLOAD_DIR / f"{session_id}_{filename}" if not file_path.exists(): logger.warning("preview_failed reason=file_not_found path=%s", file_path) raise HTTPException(status_code=404, detail="Document file not found.") - + try: TEXT_FORMATS = { - ".txt", ".md", ".html", ".htm", ".csv", ".json", - ".xml", ".log", ".tsv", ".ini", ".cfg", ".yaml", - ".yml", ".srt", ".vtt" + ".txt", + ".md", + ".html", + ".htm", + ".csv", + ".json", + ".xml", + ".log", + ".tsv", + ".ini", + ".cfg", + ".yaml", + ".yml", + ".srt", + ".vtt", } ext = Path(file_path).suffix.lower() if ext in TEXT_FORMATS: content = file_path.read_text(encoding="utf-8", errors="ignore") else: from services.rag_service import LOADERS + loader_cls = LOADERS.get(ext) if not loader_cls: content = file_path.read_text(encoding="utf-8", errors="ignore") else: docs = loader_cls(str(file_path)).load() content = "\n".join([doc.page_content for doc in docs]) - + return {"content": content} except Exception as e: # noqa: BLE001 logger.error("preview_failed path=%s error=%s", file_path, e) - raise HTTPException(status_code=500, detail=f"Failed to read document preview: {e!s}") + raise HTTPException( + status_code=500, detail=f"Failed to read document preview: {e!s}" + ) @router.get("/", response_model=list) @@ -181,13 +241,13 @@ async def delete_document(doc_id: int, session_id: str = Query(...)): doc = db_service.get_document_by_id(doc_id) if not doc or doc.get("session_id") != session_id: raise HTTPException(status_code=404, detail="Document not found.") - + file_path = doc.get("file_path", "") if file_path and os.path.exists(file_path): try: os.remove(file_path) except OSError as e: logger.warning("file_delete_failed path=%s error=%s", file_path, e) - + db_service.delete_document(doc_id) - return {"status": "success", "message": f"Document #{doc_id} deleted."} \ No newline at end of file + return {"status": "success", "message": f"Document #{doc_id} deleted."} diff --git a/backend/services/db_service.py b/backend/services/db_service.py index 535aa359..3aa933a4 100644 --- a/backend/services/db_service.py +++ b/backend/services/db_service.py @@ -18,13 +18,15 @@ # ------------------------Vacuum Scheduling-------------------------------------------------------- VACUUM_THRESHOLD = settings.db_vacuum_threshold + def _get_deleted_counter(conn) -> int: row = conn.execute( "SELECT value FROM app_settings WHERE key = 'rows_deleted_since_vacuum'" ).fetchone() return int(row["value"]) if row else 0 -def _increment_deleted_counter(conn,count:int) -> int: + +def _increment_deleted_counter(conn, count: int) -> int: new_value = _get_deleted_counter(conn) + count conn.execute( "INSERT OR REPLACE INTO app_settings (key,value,updated_at) VALUES (?,?, datetime('now'))", @@ -32,9 +34,10 @@ def _increment_deleted_counter(conn,count:int) -> int: ) return new_value + def run_vacuum(): - """ Run VACUUM outside any transaction to reclaim disk space.""" - conn = sqlite3.connect(DB_PATH, timeout=5, isolation_level = None) + """Run VACUUM outside any transaction to reclaim disk space.""" + conn = sqlite3.connect(DB_PATH, timeout=5, isolation_level=None) try: conn.execute("VACUUM") conn.execute( @@ -43,8 +46,9 @@ def run_vacuum(): finally: conn.close() + def _maybe_vacuum(deleted_count: int): - """Track deletions and trigger VACUUM once threshold is crossed.""" + """Track deletions and trigger VACUUM once threshold is crossed.""" if deleted_count <= 0: return with get_db() as conn: @@ -55,6 +59,7 @@ def _maybe_vacuum(deleted_count: int): # ─── Backup / Restore ──────────────────────────────────────────────────────── + def backup_db(dest_path: str) -> None: """Create a consistent backup of the live database at *dest_path*. @@ -97,9 +102,7 @@ def restore_db(src_path: str) -> None: """ src_path = str(src_path) if not os.path.exists(src_path): - raise FileNotFoundError( - f"restore_db: backup file not found at '{src_path}'" - ) + raise FileNotFoundError(f"restore_db: backup file not found at '{src_path}'") try: src = sqlite3.connect(src_path, timeout=5) @@ -115,9 +118,10 @@ def restore_db(src_path: str) -> None: ) from exc - DB_PATH = str(settings.db_path) -os.makedirs(os.path.dirname(DB_PATH) if os.path.dirname(DB_PATH) else ".", exist_ok=True) +os.makedirs( + os.path.dirname(DB_PATH) if os.path.dirname(DB_PATH) else ".", exist_ok=True +) logger = logging.getLogger(__name__) @@ -137,16 +141,15 @@ def get_db(): break except OperationalError as e: - if "locked" in str(e).lower() and attempt < retries - 1: - logger.warning( - "Database locked (attempt %d/%d). Retrying...", - attempt + 1, - retries, - ) - time.sleep(delay) - continue - raise - + if "locked" in str(e).lower() and attempt < retries - 1: + logger.warning( + "Database locked (attempt %d/%d). Retrying...", + attempt + 1, + retries, + ) + time.sleep(delay) + continue + raise try: yield conn @@ -155,9 +158,7 @@ def get_db(): except OperationalError as e: if "locked" in str(e).lower(): conn.rollback() - raise RuntimeError( - "Database is busy. Please try again in a moment." - ) from e + raise RuntimeError("Database is busy. Please try again in a moment.") from e conn.rollback() raise @@ -169,6 +170,7 @@ def get_db(): if conn: conn.close() + def init_db(): """Create all tables on startup.""" with get_db() as conn: @@ -259,22 +261,35 @@ def init_db(): """) conn.execute( "INSERT OR IGNORE INTO app_settings (key, value) VALUES (?, ?)", - ("default_model", json.dumps(settings.default_model)) + ("default_model", json.dumps(settings.default_model)), ) try: - conn.execute("ALTER TABLE documents ADD COLUMN status TEXT DEFAULT 'completed'") + conn.execute( + "ALTER TABLE documents ADD COLUMN status TEXT DEFAULT 'completed'" + ) except sqlite3.OperationalError: pass # column already exists - cols = [row[1] for row in conn.execute("PRAGMA table_info(messages)").fetchall()] + cols = [ + row[1] for row in conn.execute("PRAGMA table_info(messages)").fetchall() + ] if "benchmarks" not in cols: conn.execute("ALTER TABLE messages ADD COLUMN benchmarks TEXT DEFAULT '{}'") - cols_sessions = [row[1] for row in conn.execute("PRAGMA table_info(sessions)").fetchall()] + cols_sessions = [ + row[1] for row in conn.execute("PRAGMA table_info(sessions)").fetchall() + ] if "language" not in cols_sessions: conn.execute("ALTER TABLE sessions ADD COLUMN language TEXT DEFAULT 'en'") + + # ─── Sessions ──────────────────────────────────────────────── -def create_session(session_id: str, title: str = "New Chat", model: str = "llama3", language: str = "en") -> dict: +def create_session( + session_id: str, + title: str = "New Chat", + model: str = "llama3", + language: str = "en", +) -> dict: with get_db() as conn: conn.execute( "INSERT OR IGNORE INTO sessions (id, title, model, language) VALUES (?, ?, ?, ?)", @@ -285,25 +300,43 @@ def create_session(session_id: str, title: str = "New Chat", model: str = "llama def get_session(session_id: str) -> dict | None: with get_db() as conn: - row = conn.execute("SELECT * FROM sessions WHERE id=?", (session_id,)).fetchone() + row = conn.execute( + "SELECT * FROM sessions WHERE id=?", (session_id,) + ).fetchone() return dict(row) if row else None -def update_session(session_id: str, title: str | None = None, model: str | None = None, language: str | None = None): +def update_session( + session_id: str, + title: str | None = None, + model: str | None = None, + language: str | None = None, +): with get_db() as conn: if title is not None: - conn.execute("UPDATE sessions SET title=?, updated_at=datetime('now') WHERE id=?", (title, session_id)) + conn.execute( + "UPDATE sessions SET title=?, updated_at=datetime('now') WHERE id=?", + (title, session_id), + ) if model is not None: - conn.execute("UPDATE sessions SET model=?, updated_at=datetime('now') WHERE id=?", (model, session_id)) + conn.execute( + "UPDATE sessions SET model=?, updated_at=datetime('now') WHERE id=?", + (model, session_id), + ) if language is not None: - conn.execute("UPDATE sessions SET language=?, updated_at=datetime('now') WHERE id=?", (language, session_id)) + conn.execute( + "UPDATE sessions SET language=?, updated_at=datetime('now') WHERE id=?", + (language, session_id), + ) def delete_session(session_id: str): """Deletes a session, clears its physical document assets from disk, and removes database rows.""" with get_db() as conn: # 1. Fetch all physical file paths for documents bound to this session - rows = conn.execute("SELECT file_path FROM documents WHERE session_id=?", (session_id,)).fetchall() + rows = conn.execute( + "SELECT file_path FROM documents WHERE session_id=?", (session_id,) + ).fetchall() for row in rows: if row["file_path"]: physical_path = row["file_path"] @@ -312,7 +345,9 @@ def delete_session(session_id: str): os.remove(physical_path) print(f"Cleaned up session document asset: {physical_path}") except Exception as file_err: # noqa: BLE001 - print(f"Warning: Failed to delete session asset {physical_path}: {file_err!s}") + print( + f"Warning: Failed to delete session asset {physical_path}: {file_err!s}" + ) # 2. Gather counts for vacuum scheduling metric tracking msg_count = conn.execute( @@ -324,7 +359,7 @@ def delete_session(session_id: str): cur = conn.execute("DELETE FROM sessions WHERE id=?", (session_id,)) deleted = cur.rowcount + msg_count + doc_count - _maybe_vacuum(deleted) + _maybe_vacuum(deleted) def clear_all_sessions(): @@ -353,7 +388,7 @@ def toggle_message_reaction(message_id: int, emoji: str) -> str: # Check if this specific emoji reaction already exists for this message row = conn.execute( "SELECT id FROM message_reactions WHERE message_id = ? AND emoji = ?", - (message_id, emoji) + (message_id, emoji), ).fetchone() if row: @@ -362,7 +397,7 @@ def toggle_message_reaction(message_id: int, emoji: str) -> str: else: conn.execute( "INSERT INTO message_reactions (message_id, emoji) VALUES (?, ?)", - (message_id, emoji) + (message_id, emoji), ) return "added" @@ -372,7 +407,7 @@ def get_reactions_for_message(message_id: int) -> list[str]: with get_db() as conn: rows = conn.execute( "SELECT emoji FROM message_reactions WHERE message_id = ? ORDER BY created_at ASC", - (message_id,) + (message_id,), ).fetchall() return [r["emoji"] for r in rows] @@ -383,14 +418,17 @@ def get_session_reactions_map(session_id: str) -> dict[int, list[str]]: Returns a dictionary mapping message_id -> list of emojis. """ with get_db() as conn: - rows = conn.execute(""" + rows = conn.execute( + """ SELECT r.message_id, r.emoji FROM message_reactions r JOIN messages m ON r.message_id = m.id WHERE m.session_id = ? ORDER BY r.created_at ASC - """, (session_id,)).fetchall() - + """, + (session_id,), + ).fetchall() + reactions_map = {} for r in rows: msg_id = r["message_id"] @@ -398,7 +436,15 @@ def get_session_reactions_map(session_id: str) -> dict[int, list[str]]: reactions_map[msg_id] = [] reactions_map[msg_id].append(r["emoji"]) return reactions_map -def save_message(session_id: str, role: str, content: str, sources: list | None = None, benchmarks: dict | None = None): + + +def save_message( + session_id: str, + role: str, + content: str, + sources: list | None = None, + benchmarks: dict | None = None, +): sources = sources or [] with get_db() as conn: conn.execute( @@ -419,7 +465,9 @@ def save_message(session_id: str, role: str, content: str, sources: list | None title = grapheme.slice(content, start=0, end=40) + "..." else: title = content - conn.execute("UPDATE sessions SET title=? WHERE id=?", (title, session_id)) + conn.execute( + "UPDATE sessions SET title=? WHERE id=?", (title, session_id) + ) def get_history(session_id: str, limit: int = 20) -> list[dict]: @@ -444,7 +492,7 @@ def get_messages_full(session_id: str) -> list[dict]: "content": r["content"], "sources": json.loads(r["sources"] or "[]"), "created_at": r["created_at"], - "benchmarks": json.loads(r["benchmarks"] or {}) + "benchmarks": json.loads(r["benchmarks"] or {}), } for r in rows ] @@ -455,7 +503,7 @@ def clear_messages(session_id: str): cur = conn.execute("DELETE FROM messages WHERE session_id=?", (session_id,)) deleted = cur.rowcount conn.execute("UPDATE sessions SET message_count=0 WHERE id=?", (session_id,)) - _maybe_vacuum(deleted) + _maybe_vacuum(deleted) def delete_message(session_id: str, message_id: int) -> int: @@ -480,7 +528,14 @@ def delete_message(session_id: str, message_id: int) -> int: # ─── Documents ─────────────────────────────────────────────── -def save_document(session_id: str, filename: str, file_path: str, chunks: int, size_kb: float, status: str = "completed") -> int: +def save_document( + session_id: str, + filename: str, + file_path: str, + chunks: int, + size_kb: float, + status: str = "completed", +) -> int: with get_db() as conn: cursor = conn.execute( "INSERT INTO documents (session_id, filename, file_path, chunks_indexed, file_size_kb, status) VALUES (?,?,?,?,?,?)", @@ -488,10 +543,14 @@ def save_document(session_id: str, filename: str, file_path: str, chunks: int, s ) return cursor.lastrowid + def update_document_status(doc_id: int, status: str, chunks_indexed: int | None = None): with get_db() as conn: if chunks_indexed is not None: - conn.execute("UPDATE documents SET status=?, chunks_indexed=? WHERE id=?", (status, chunks_indexed, doc_id)) + conn.execute( + "UPDATE documents SET status=?, chunks_indexed=? WHERE id=?", + (status, chunks_indexed, doc_id), + ) else: conn.execute("UPDATE documents SET status=? WHERE id=?", (status, doc_id)) @@ -509,8 +568,10 @@ def delete_document(doc_id: int): """Deletes the physical uploaded file from disk and removes its record entry from SQLite.""" with get_db() as conn: # 1. Fetch the physical file path before deleting the database reference row - row = conn.execute("SELECT file_path FROM documents WHERE id=?", (doc_id,)).fetchone() - + row = conn.execute( + "SELECT file_path FROM documents WHERE id=?", (doc_id,) + ).fetchone() + if row and row["file_path"]: physical_path = row["file_path"] try: @@ -520,13 +581,16 @@ def delete_document(doc_id: int): print(f"Successfully deleted physical file asset: {physical_path}") except Exception as file_err: # noqa: BLE001 # Log the error but continue so the database doesn't lock or desync - print(f"Warning: Failed to clean up disk file {physical_path}: {file_err!s}") + print( + f"Warning: Failed to clean up disk file {physical_path}: {file_err!s}" + ) # 3. Clean up the database record entries cur = conn.execute("DELETE FROM documents WHERE id=?", (doc_id,)) deleted = cur.rowcount - - _maybe_vacuum(deleted) + + _maybe_vacuum(deleted) + # ─── Settings ──────────────────────────────────────────────── def get_settings() -> dict: @@ -556,14 +620,15 @@ def save_settings(settings: dict[str, object]) -> None: # ─── Plugin logs ───────────────────────────────────────────── + def get_plugin_logs(limit: int = 50) -> list[dict]: with get_db() as conn: rows = conn.execute( - "SELECT * FROM plugin_logs ORDER BY created_at DESC LIMIT ?", - (limit,) + "SELECT * FROM plugin_logs ORDER BY created_at DESC LIMIT ?", (limit,) ).fetchall() return [dict(r) for r in rows] + def log_plugin(session_id: str, plugin: str, inp: str, out: str, success: bool = True): with get_db() as conn: conn.execute( @@ -574,9 +639,10 @@ def log_plugin(session_id: str, plugin: str, inp: str, out: str, success: bool = # ─── Shareable Sessions (Issue #270) ───────────────────────── + def create_shared_session(session_id: str) -> str: """ - Captures a frozen snapshot of a chat session's history + Captures a frozen snapshot of a chat session's history and returns a unique, obfuscated sharing ID string. """ # 1. Fetch current session parameters @@ -600,7 +666,7 @@ def create_shared_session(session_id: str) -> str: INSERT INTO shared_sessions (id, session_id, title, model, snapshot_json) VALUES (?, ?, ?, ?, ?) """, - (share_id, session_id, session["title"], session["model"], snapshot_str) + (share_id, session_id, session["title"], session["model"], snapshot_str), ) return share_id @@ -612,7 +678,7 @@ def get_shared_session(share_id: str) -> dict | None: with get_db() as conn: row = conn.execute( "SELECT title, model, snapshot_json, created_at FROM shared_sessions WHERE id = ?", - (share_id,) + (share_id,), ) row = row.fetchone() @@ -623,16 +689,21 @@ def get_shared_session(share_id: str) -> dict | None: "id": share_id, "title": row["title"], "model": row["model"], - "messages": json.loads(row["snapshot_json"]), # Turn string array back into live json dicts - "created_at": row["created_at"] + "messages": json.loads( + row["snapshot_json"] + ), # Turn string array back into live json dicts + "created_at": row["created_at"], } + + # ─── Prompt Templates (Updated Signatures) ─────────────────── + def create_prompt_template(prompt_title: str, prompt: str) -> dict: with get_db() as conn: cursor = conn.execute( "INSERT INTO prompt_templates (name, prompt) VALUES (?, ?)", - (prompt_title, prompt) + (prompt_title, prompt), ) template_id = cursor.lastrowid return get_prompt_template(template_id) @@ -641,8 +712,8 @@ def create_prompt_template(prompt_title: str, prompt: str) -> dict: def get_prompt_template(template_id: int) -> dict | None: with get_db() as conn: row = conn.execute( - "SELECT id, name AS prompt_title, prompt, created_at FROM prompt_templates WHERE id = ?", - (template_id,) + "SELECT id, name AS prompt_title, prompt, created_at FROM prompt_templates WHERE id = ?", + (template_id,), ).fetchone() return dict(row) if row else None @@ -655,12 +726,19 @@ def get_all_prompt_templates() -> list[dict]: return [dict(r) for r in rows] -def update_prompt_template(template_id: int, prompt_title: str | None = None, prompt: str | None = None) -> dict | None: +def update_prompt_template( + template_id: int, prompt_title: str | None = None, prompt: str | None = None +) -> dict | None: with get_db() as conn: if prompt_title: - conn.execute("UPDATE prompt_templates SET name=? WHERE id=?", (prompt_title, template_id)) + conn.execute( + "UPDATE prompt_templates SET name=? WHERE id=?", + (prompt_title, template_id), + ) if prompt: - conn.execute("UPDATE prompt_templates SET prompt=? WHERE id=?", (prompt, template_id)) + conn.execute( + "UPDATE prompt_templates SET prompt=? WHERE id=?", (prompt, template_id) + ) return get_prompt_template(template_id) diff --git a/backend/services/ollama_service.py b/backend/services/ollama_service.py index 3ed2f1a7..fca084dc 100644 --- a/backend/services/ollama_service.py +++ b/backend/services/ollama_service.py @@ -5,7 +5,6 @@ import asyncio import json import logging -import os from collections.abc import AsyncGenerator import httpx @@ -49,6 +48,7 @@ def _build_messages(message: str, context: str, history: list, language: str) -> # Global cache for model metadata (5 minute TTL) model_metadata_cache = TTLCache(ttl_seconds=300) + @with_retry(max_attempts=3, initial_backoff=1.0) async def chat( message: str, @@ -77,6 +77,7 @@ async def chat( response.raise_for_status() return response.json()["message"]["content"] + async def chat_stream( message: str, model: str = "llama3", @@ -94,7 +95,7 @@ async def chat_stream( "stream": True, "options": {"temperature": temperature, "top_p": 0.9, "num_predict": 2048}, } - + max_attempts = 3 actual_max_attempts = max(1, max_attempts) attempt = 1 @@ -105,23 +106,25 @@ async def chat_stream( try: async with ( httpx.AsyncClient(timeout=TIMEOUT) as client, - client.stream("POST", f"{OLLAMA_BASE_URL}/api/chat", json=payload) as resp, + client.stream( + "POST", f"{OLLAMA_BASE_URL}/api/chat", json=payload + ) as resp, ): resp.raise_for_status() async for line in resp.aiter_lines(): - if line.strip(): - try: - data = json.loads(line) - token = data.get("message", {}).get("content", "") - if token: - yield token - if data.get("done"): - break - except json.JSONDecodeError: - continue + if line.strip(): + try: + data = json.loads(line) + token = data.get("message", {}).get("content", "") + if token: + yield token + if data.get("done"): + break + except json.JSONDecodeError: + continue # If we exit the context manager normally, we are done, break out of retry loop break - + except httpx.RequestError as e: is_transient = True error_msg = f"Network Error: {type(e).__name__}" @@ -138,10 +141,14 @@ async def chat_stream( if is_transient: if attempt == actual_max_attempts: - logger.error(f"chat_stream failed after {actual_max_attempts} attempts. Last error: {error_msg}") + logger.error( + f"chat_stream failed after {actual_max_attempts} attempts. Last error: {error_msg}" + ) raise last_exc - - logger.warning(f"chat_stream failed ({error_msg}). Retrying in {backoff}s... (Attempt {attempt}/{actual_max_attempts})") + + logger.warning( + f"chat_stream failed ({error_msg}). Retrying in {backoff}s... (Attempt {attempt}/{actual_max_attempts})" + ) await asyncio.sleep(backoff) attempt += 1 backoff *= 2 @@ -156,12 +163,14 @@ async def list_models() -> list[dict]: models = [] for m in resp.json().get("models", []): size_gb = round(m.get("size", 0) / 1e9, 1) - models.append({ - "name": m["name"], - "size": f"{size_gb} GB", - "status": "available", - "modified_at": m.get("modified_at", ""), - }) + models.append( + { + "name": m["name"], + "size": f"{size_gb} GB", + "status": "available", + "modified_at": m.get("modified_at", ""), + } + ) return models except Exception as e: # noqa: BLE001 logger.warning(f"Could not list models: {e}") @@ -180,18 +189,17 @@ async def get_model_info(model_name: str) -> dict: async with httpx.AsyncClient(timeout=8.0) as client: try: resp = await client.post( - f"{OLLAMA_BASE_URL}/api/show", - json={"name": model_name} + f"{OLLAMA_BASE_URL}/api/show", json={"name": model_name} ) resp.raise_for_status() info = resp.json() - + # Populate cache model_metadata_cache.set(model_name, info) return info except httpx.HTTPStatusError as e: if e.response.status_code == 404: - return {} # Model not found + return {} # Model not found raise except Exception as e: logger.warning(f"Could not fetch metadata for model '{model_name}': {e}") @@ -211,14 +219,15 @@ async def pull_model(model_name: str) -> AsyncGenerator[str, None]: async with ( httpx.AsyncClient(timeout=600.0) as client, client.stream( - "POST", f"{OLLAMA_BASE_URL}/api/pull", - json={"name": model_name, "stream": True} + "POST", + f"{OLLAMA_BASE_URL}/api/pull", + json={"name": model_name, "stream": True}, ) as resp, ): resp.raise_for_status() async for line in resp.aiter_lines(): - if line.strip(): - yield line + "\n" + if line.strip(): + yield line + "\n" break except httpx.RequestError as e: is_transient = True @@ -245,8 +254,7 @@ async def delete_model(model_name: str) -> bool: async with httpx.AsyncClient(timeout=30.0) as client: try: resp = await client.delete( - f"{OLLAMA_BASE_URL}/api/delete", - json={"name": model_name} + f"{OLLAMA_BASE_URL}/api/delete", json={"name": model_name} ) return resp.status_code == 200 except Exception: # noqa: BLE001 diff --git a/backend/services/rag_service.py b/backend/services/rag_service.py index d0c81797..26cae81a 100644 --- a/backend/services/rag_service.py +++ b/backend/services/rag_service.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) CHROMA_PATH = str(settings.chromadb_dir) -EMBED_MODEL = "all-MiniLM-L6-v2" +EMBED_MODEL = "all-MiniLM-L6-v2" os.makedirs(CHROMA_PATH, exist_ok=True) @@ -37,14 +37,14 @@ embedder = SentenceTransformer(EMBED_MODEL) LOADERS = { - ".pdf": PyPDFLoader, - ".txt": TextLoader, - ".md": TextLoader, - ".csv": CleanCSVLoader, + ".pdf": PyPDFLoader, + ".txt": TextLoader, + ".md": TextLoader, + ".csv": CleanCSVLoader, ".docx": DocxWithTablesLoader, ".html": UnstructuredHTMLLoader, - ".srt": TextLoader, # Handle SubRip video/audio transcripts natively - ".vtt": TextLoader, # Handle WebVTT audio transcripts natively + ".srt": TextLoader, # Handle SubRip video/audio transcripts natively + ".vtt": TextLoader, # Handle WebVTT audio transcripts natively } @@ -62,9 +62,10 @@ def index_document(file_path: str, session_id: str, doc_id: int | None = None) - raise ValueError(f"Unsupported file type: {ext}. Supported: {list(LOADERS)}") docs = loader_cls(file_path).load() - + # Fetch live chunk size and overlap bounds configuration directly from database cache settings from services.db_service import get_settings + current_settings = get_settings() chunk_size_val = current_settings.get("rag_chunk_size", 600) overlap_val = current_settings.get("rag_chunk_overlap", 50) @@ -75,48 +76,64 @@ def index_document(file_path: str, session_id: str, doc_id: int | None = None) - chunk_overlap=int(overlap_val), separators=["\n\n", "\n", ". ", " "], ) - + chunks = dynamic_splitter.split_documents(docs) if not chunks: return 0 texts = [c.page_content for c in chunks] ids = [f"{session_id}_{i}" for i in range(len(texts))] - metadatas = [{"source": Path(file_path).name, "chunk": i} for i in range(len(texts))] + metadatas = [ + {"source": Path(file_path).name, "chunk": i} for i in range(len(texts)) + ] col = _collection(session_id) - + batch_size = 200 for i in range(0, len(texts), batch_size): - batch_texts = texts[i:i + batch_size] - batch_ids = ids[i:i + batch_size] - batch_metas = metadatas[i:i + batch_size] - - batch_embeddings = embedder.encode(batch_texts, show_progress_bar=False).tolist() - col.upsert(ids=batch_ids, documents=batch_texts, embeddings=batch_embeddings, metadatas=batch_metas) + batch_texts = texts[i : i + batch_size] + batch_ids = ids[i : i + batch_size] + batch_metas = metadatas[i : i + batch_size] + + batch_embeddings = embedder.encode( + batch_texts, show_progress_bar=False + ).tolist() + col.upsert( + ids=batch_ids, + documents=batch_texts, + embeddings=batch_embeddings, + metadatas=batch_metas, + ) if doc_id is not None: from services import db_service - db_service.update_document_status(doc_id, "processing", chunks_indexed=(i + len(batch_texts))) + + db_service.update_document_status( + doc_id, "processing", chunks_indexed=(i + len(batch_texts)) + ) time.sleep(0.05) # Yield GIL to allow event loop to process other requests - logger.info(f"Indexed {len(chunks)} chunks for session={session_id} using chunk_overlap={overlap_val}") + logger.info( + f"Indexed {len(chunks)} chunks for session={session_id} using chunk_overlap={overlap_val}" + ) return len(chunks) -def retrieve_context(query: str, session_id: str, top_k: int = 4) -> tuple[str, list[dict]]: +def retrieve_context( + query: str, session_id: str, top_k: int = 4 +) -> tuple[str, list[dict]]: col = _collection(session_id) if col.count() == 0: return "", [] - q_emb = embedder.encode([query]).tolist() + q_emb = embedder.encode([query]).tolist() results = col.query( query_embeddings=q_emb, n_results=min(top_k, col.count()), include=["documents", "metadatas"], ) - docs = results["documents"][0] if results["documents"] else [] - metas = results["metadatas"][0] if results["metadatas"] else [] + docs = results["documents"][0] if results["documents"] else [] + metas = results["metadatas"][0] if results["metadatas"] else [] context = "\n\n---\n\n".join(docs) @@ -131,9 +148,10 @@ def delete_session_index(session_id: str): try: chroma_client.delete_collection(f"lm_{session_id.replace('-', '_')}") except Exception as e: # noqa: BLE001 - logger.warning("Could not delete ChromaDB collection for session %s: %s", session_id, e) - + logger.warning( + "Could not delete ChromaDB collection for session %s: %s", session_id, e + ) def get_indexed_count(session_id: str) -> int: - return _collection(session_id).count() \ No newline at end of file + return _collection(session_id).count() diff --git a/backend/utils/audit_log.py b/backend/utils/audit_log.py index 1bae7fb0..664a3a5e 100644 --- a/backend/utils/audit_log.py +++ b/backend/utils/audit_log.py @@ -5,6 +5,7 @@ FAILED) as structured JSON, without ever blocking the calling thread/coroutine and without ever letting a logging failure crash ingestion. """ + from __future__ import annotations import atexit @@ -18,6 +19,7 @@ from typing import Any from utils.config import settings + _module_logger = logging.getLogger(__name__) AUDIT_LOG_DIR = str(settings.audit_log_dir) @@ -36,13 +38,17 @@ class _JsonFormatter(logging.Formatter): def format(self, record: logging.LogRecord) -> str: payload: dict[str, Any] = { "event": getattr(record, "event", record.getMessage()), - "logged_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(record.created)), + "logged_at": time.strftime( + "%Y-%m-%dT%H:%M:%SZ", time.gmtime(record.created) + ), } payload.update(getattr(record, "audit_fields", {})) try: return json.dumps(payload, default=str) except (TypeError, ValueError): - return json.dumps({"event": payload.get("event", "unknown"), "serialization_error": True}) + return json.dumps( + {"event": payload.get("event", "unknown"), "serialization_error": True} + ) def _build_listener() -> logging.handlers.QueueListener: @@ -59,13 +65,17 @@ def _build_listener() -> logging.handlers.QueueListener: file_handler.setFormatter(_JsonFormatter()) handlers.append(file_handler) except OSError as e: - _module_logger.warning("audit_log_file_unavailable path=%s error=%s", AUDIT_LOG_FILE, e) + _module_logger.warning( + "audit_log_file_unavailable path=%s error=%s", AUDIT_LOG_FILE, e + ) log_queue: queue.Queue = queue.Queue(-1) queue_handler = logging.handlers.QueueHandler(log_queue) _audit_logger.handlers = [queue_handler] - listener = logging.handlers.QueueListener(log_queue, *handlers, respect_handler_level=True) + listener = logging.handlers.QueueListener( + log_queue, *handlers, respect_handler_level=True + ) listener.start() return listener @@ -100,6 +110,7 @@ def _now_iso() -> str: # --- Public lifecycle API ------------------------------------------------- + def log_upload_queued(*, file_id: Any, file_size_bytes: int, user_id: str) -> None: _safe_emit( "UPLOAD_QUEUED", @@ -135,7 +146,9 @@ def log_success(*, file_id: Any, user_id: str, duration_ms: float) -> None: ) -def log_failed(*, file_id: Any, user_id: str, duration_ms: float, error: str, stack_trace: str) -> None: +def log_failed( + *, file_id: Any, user_id: str, duration_ms: float, error: str, stack_trace: str +) -> None: _safe_emit( "FAILED", { @@ -146,4 +159,4 @@ def log_failed(*, file_id: Any, user_id: str, duration_ms: float, error: str, st "stack_trace": stack_trace, "timestamp": _now_iso(), }, - ) \ No newline at end of file + ) diff --git a/backend/utils/config.py b/backend/utils/config.py index bdcbf223..e9986a0d 100644 --- a/backend/utils/config.py +++ b/backend/utils/config.py @@ -1,26 +1,36 @@ from pydantic_settings import BaseSettings, SettingsConfigDict from pydantic import Field -from typing import List from pathlib import Path + class Settings(BaseSettings): - frontend_dist: Path = Field(default=Path("/app/frontend/dist"), alias="FRONTEND_DIST") - cors_origins: str = Field(default="http://localhost:3000,http://127.0.0.1:3000,http://localhost:5173,http://localhost:8000", alias="CORS_ORIGINS") + frontend_dist: Path = Field( + default=Path("/app/frontend/dist"), alias="FRONTEND_DIST" + ) + cors_origins: str = Field( + default="http://localhost:3000,http://127.0.0.1:3000,http://localhost:5173,http://localhost:8000", + alias="CORS_ORIGINS", + ) upload_dir: Path = Field(default=Path("./data/uploads"), alias="UPLOAD_DIR") - settings_api_timeout_seconds: int = Field(default=10, alias="SETTINGS_API_TIMEOUT_SECONDS") + settings_api_timeout_seconds: int = Field( + default=10, alias="SETTINGS_API_TIMEOUT_SECONDS" + ) audit_log_dir: Path = Field(default=Path("./data/logs"), alias="AUDIT_LOG_DIR") db_vacuum_threshold: int = Field(default=500, alias="DB_VACUUM_THRESHOLD") db_path: Path = Field(default=Path("./data/localmind.db"), alias="DB_PATH") ollama_host: str = Field(default="http://localhost:11434", alias="OLLAMA_HOST") chromadb_dir: Path = Field(default=Path("./data/chromadb"), alias="CHROMADB_DIR") - + # Values from .env.example default_model: str = Field(default="llama3", alias="DEFAULT_MODEL") exports_dir: Path = Field(default=Path("./data/exports"), alias="EXPORTS_DIR") max_file_size: int = Field(default=52428800, alias="MAX_FILE_SIZE") backend_port: int = Field(default=8000, alias="BACKEND_PORT") frontend_port: int = Field(default=3000, alias="FRONTEND_PORT") - - model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") + + model_config = SettingsConfigDict( + env_file=".env", env_file_encoding="utf-8", extra="ignore" + ) + settings = Settings() From b7192aa6f225836f75761cd2ec96994cbb341952 Mon Sep 17 00:00:00 2001 From: Shantanu Date: Sat, 1 Aug 2026 18:43:00 +0530 Subject: [PATCH 5/6] fix: resolve ruff B008 and test side effects --- backend/app.py | 1 + backend/routes/chat.py | 3 ++- backend/routes/export.py | 3 ++- backend/routes/models.py | 1 + backend/routes/plugins.py | 1 + backend/routes/prompt_templates.py | 1 + backend/routes/settings.py | 1 + backend/routes/upload.py | 7 ++--- backend/services/db_service.py | 1 + backend/services/ollama_service.py | 4 +-- backend/services/rag_service.py | 2 +- backend/tests/test_api.py | 3 ++- backend/tests/test_audit_log.py | 3 ++- backend/tests/test_backup.py | 1 + backend/tests/test_bulk_export.py | 33 ++++++++++++----------- backend/tests/test_cancellation.py | 1 + backend/tests/test_citations.py | 3 ++- backend/tests/test_csrf.py | 3 ++- backend/tests/test_docx_tables.py | 1 + backend/tests/test_model_cache.py | 1 + backend/tests/test_retry_policy.py | 1 + backend/tests/test_settings_timeout.py | 3 ++- backend/tests/test_streaming_recovery.py | 1 + backend/tests_random.log | Bin 0 -> 6128 bytes backend/utils/config.py | 5 ++-- 25 files changed, 53 insertions(+), 31 deletions(-) create mode 100644 backend/tests_random.log diff --git a/backend/app.py b/backend/app.py index e8e6e2ad..874f2df2 100644 --- a/backend/app.py +++ b/backend/app.py @@ -16,6 +16,7 @@ from fastapi.middleware.gzip import GZipMiddleware from fastapi.responses import FileResponse from fastapi.staticfiles import StaticFiles + from middleware.csrf import OriginValidationMiddleware from routes.chat import router as chat_router from routes.export import router as export_router diff --git a/backend/routes/chat.py b/backend/routes/chat.py index 8cc21d3a..fd78ce01 100644 --- a/backend/routes/chat.py +++ b/backend/routes/chat.py @@ -11,8 +11,9 @@ import psutil from fastapi import APIRouter, HTTPException from fastapi.responses import StreamingResponse -from models.schemas import ChatRequest, ChatResponse from pydantic import BaseModel + +from models.schemas import ChatRequest, ChatResponse from services import db_service, ollama_service logger = logging.getLogger(__name__) diff --git a/backend/routes/export.py b/backend/routes/export.py index 5dc9d947..f7629291 100644 --- a/backend/routes/export.py +++ b/backend/routes/export.py @@ -6,8 +6,9 @@ from fastapi import APIRouter, HTTPException from fastapi.responses import Response -from models.schemas import ExportFormat from pydantic import BaseModel, field_validator + +from models.schemas import ExportFormat from services import db_service router = APIRouter() diff --git a/backend/routes/models.py b/backend/routes/models.py index b5773560..4fc9d119 100644 --- a/backend/routes/models.py +++ b/backend/routes/models.py @@ -2,6 +2,7 @@ from fastapi import APIRouter, HTTPException from fastapi.responses import StreamingResponse + from services import ollama_service router = APIRouter() diff --git a/backend/routes/plugins.py b/backend/routes/plugins.py index 39c235d3..64287bfd 100644 --- a/backend/routes/plugins.py +++ b/backend/routes/plugins.py @@ -12,6 +12,7 @@ import tempfile from fastapi import APIRouter, HTTPException + from models.schemas import PluginResult, PluginRun from services import db_service diff --git a/backend/routes/prompt_templates.py b/backend/routes/prompt_templates.py index 003776ad..2489dd7f 100644 --- a/backend/routes/prompt_templates.py +++ b/backend/routes/prompt_templates.py @@ -1,4 +1,5 @@ from fastapi import APIRouter, HTTPException + from models.schemas import PromptTemplateCreate, PromptTemplateUpdate from services import db_service diff --git a/backend/routes/settings.py b/backend/routes/settings.py index 02d1e740..a89a8669 100644 --- a/backend/routes/settings.py +++ b/backend/routes/settings.py @@ -7,6 +7,7 @@ from typing import Any from fastapi import APIRouter, HTTPException, status + from models.schemas import AppSettings from services.db_service import get_settings, save_setting, save_settings from utils.config import settings diff --git a/backend/routes/upload.py b/backend/routes/upload.py index c4d2b2ba..972aa4f2 100644 --- a/backend/routes/upload.py +++ b/backend/routes/upload.py @@ -5,7 +5,6 @@ import time import traceback from pathlib import Path -from utils.config import settings from fastapi import ( APIRouter, @@ -16,9 +15,11 @@ Query, UploadFile, ) + from models.schemas import UploadResponse from services import db_service from utils import audit_log +from utils.config import settings logger = logging.getLogger(__name__) @@ -64,10 +65,10 @@ def _safe_audit(fn, **kwargs): @router.post("/", response_model=UploadResponse) async def upload( - file: UploadFile = File(...), + file: UploadFile = File(...), # noqa: B008 session_id: str = Form(...), background_tasks: BackgroundTasks = None, -): # noqa: B008 +): logger.info( "upload_request route=/upload session=%s file=%s", session_id, file.filename ) diff --git a/backend/services/db_service.py b/backend/services/db_service.py index 3aa933a4..6fbcb62a 100644 --- a/backend/services/db_service.py +++ b/backend/services/db_service.py @@ -13,6 +13,7 @@ from sqlite3 import OperationalError import grapheme + from utils.config import settings # ------------------------Vacuum Scheduling-------------------------------------------------------- diff --git a/backend/services/ollama_service.py b/backend/services/ollama_service.py index fca084dc..ce1f5e35 100644 --- a/backend/services/ollama_service.py +++ b/backend/services/ollama_service.py @@ -8,10 +8,10 @@ from collections.abc import AsyncGenerator import httpx -from utils.cache import TTLCache -from utils.retry import with_retry +from utils.cache import TTLCache from utils.config import settings +from utils.retry import with_retry logger = logging.getLogger(__name__) diff --git a/backend/services/rag_service.py b/backend/services/rag_service.py index 26cae81a..b73372a4 100644 --- a/backend/services/rag_service.py +++ b/backend/services/rag_service.py @@ -17,10 +17,10 @@ ) from langchain_text_splitters import RecursiveCharacterTextSplitter from sentence_transformers import SentenceTransformer + from services.citation_utils import build_sources from services.csv_loader import CleanCSVLoader from services.docx_loader import DocxWithTablesLoader - from utils.config import settings logger = logging.getLogger(__name__) diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index ab964f69..c4e6b89d 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -6,9 +6,10 @@ import tempfile from unittest.mock import AsyncMock, patch +from fastapi.testclient import TestClient + import services.db_service as db from app import app -from fastapi.testclient import TestClient _tmp = tempfile.mktemp(suffix=".db") db.DB_PATH = _tmp diff --git a/backend/tests/test_audit_log.py b/backend/tests/test_audit_log.py index dfbfc1bc..8342c3f1 100644 --- a/backend/tests/test_audit_log.py +++ b/backend/tests/test_audit_log.py @@ -11,10 +11,11 @@ from unittest.mock import patch import pytest +from fastapi.testclient import TestClient + import routes.upload as upload_module import services.db_service as db from app import app -from fastapi.testclient import TestClient from utils import audit_log _tmp = tempfile.mktemp(suffix=".db") diff --git a/backend/tests/test_backup.py b/backend/tests/test_backup.py index fc075e8b..044f23a7 100644 --- a/backend/tests/test_backup.py +++ b/backend/tests/test_backup.py @@ -16,6 +16,7 @@ import uuid import pytest + from services import db_service from services.db_service import ( backup_db, diff --git a/backend/tests/test_bulk_export.py b/backend/tests/test_bulk_export.py index 829c7840..b47379ea 100644 --- a/backend/tests/test_bulk_export.py +++ b/backend/tests/test_bulk_export.py @@ -1,8 +1,9 @@ import tempfile +from fastapi.testclient import TestClient + import services.db_service as db from app import app -from fastapi.testclient import TestClient # Setup temporary database for bulk export tests _tmp = tempfile.mktemp(suffix=".db") @@ -14,19 +15,19 @@ def test_bulk_export_json_success(): # 1. Create two test sessions - db.create_session("sess_1", title="Session One", model="llama3", language="en") - db.create_session("sess_2", title="Session Two", model="mistral", language="fr") + db.create_session("sess_json_1", title="Session One", model="llama3", language="en") + db.create_session("sess_json_2", title="Session Two", model="mistral", language="fr") - db.save_message("sess_1", "user", "Hello from user 1") - db.save_message("sess_1", "assistant", "Response 1") + db.save_message("sess_json_1", "user", "Hello from user 1") + db.save_message("sess_json_1", "assistant", "Response 1") - db.save_message("sess_2", "user", "Bonjour") - db.save_message("sess_2", "assistant", "Salut") + db.save_message("sess_json_2", "user", "Bonjour") + db.save_message("sess_json_2", "assistant", "Salut") # 2. Call bulk export response = client.post( "/api/export/sessions", - json={"session_ids": ["sess_1", "sess_2"], "format": "json"} + json={"session_ids": ["sess_json_1", "sess_json_2"], "format": "json"} ) assert response.status_code == 200 assert response.headers["Content-Type"].startswith("application/json") @@ -41,13 +42,13 @@ def test_bulk_export_json_success(): # Check structure sess_1_data = payload["sessions"][0] - assert sess_1_data["session"]["id"] == "sess_1" + assert sess_1_data["session"]["id"] == "sess_json_1" assert sess_1_data["session"]["title"] == "Session One" assert len(sess_1_data["messages"]) == 2 assert sess_1_data["messages"][0]["content"] == "Hello from user 1" sess_2_data = payload["sessions"][1] - assert sess_2_data["session"]["id"] == "sess_2" + assert sess_2_data["session"]["id"] == "sess_json_2" assert sess_2_data["session"]["title"] == "Session Two" assert len(sess_2_data["messages"]) == 2 assert sess_2_data["messages"][0]["content"] == "Bonjour" @@ -107,7 +108,7 @@ def test_bulk_export_validation_empty_ids(): def test_bulk_export_validation_invalid_format(): response = client.post( "/api/export/sessions", - json={"session_ids": ["sess_1"], "format": "html"} + json={"session_ids": ["sess_invalid_format"], "format": "html"} ) assert response.status_code == 422 errors = response.json()["detail"] @@ -115,18 +116,18 @@ def test_bulk_export_validation_invalid_format(): def test_bulk_export_mixed_valid_invalid(): - # sess_1 exists, sess_invalid does not - db.create_session("sess_1", title="Valid Session") - db.save_message("sess_1", "user", "Hello") + # sess_mixed exists, sess_invalid does not + db.create_session("sess_mixed", title="Valid Session") + db.save_message("sess_mixed", "user", "Hello") response = client.post( "/api/export/sessions", - json={"session_ids": ["sess_1", "sess_invalid"], "format": "json"} + json={"session_ids": ["sess_mixed", "sess_invalid"], "format": "json"} ) assert response.status_code == 200 payload = response.json() assert len(payload["sessions"]) == 1 - assert payload["sessions"][0]["session"]["id"] == "sess_1" + assert payload["sessions"][0]["session"]["id"] == "sess_mixed" def test_bulk_export_all_invalid(): diff --git a/backend/tests/test_cancellation.py b/backend/tests/test_cancellation.py index 3f37c2b3..c412388b 100644 --- a/backend/tests/test_cancellation.py +++ b/backend/tests/test_cancellation.py @@ -3,6 +3,7 @@ from unittest.mock import AsyncMock, patch import pytest + import services.db_service as db from models.schemas import ChatRequest from routes.chat import ACTIVE_STREAMS, cancel_stream, chat_stream diff --git a/backend/tests/test_citations.py b/backend/tests/test_citations.py index 3b941eb5..86f8262a 100644 --- a/backend/tests/test_citations.py +++ b/backend/tests/test_citations.py @@ -12,9 +12,10 @@ import tempfile from unittest.mock import AsyncMock, patch +from fastapi.testclient import TestClient + import services.db_service as db from app import app -from fastapi.testclient import TestClient from models.schemas import ChatMessage, MessageRole, SourceChunk # ─── Shared test client ────────────────────────────────────────── diff --git a/backend/tests/test_csrf.py b/backend/tests/test_csrf.py index 5039f375..a699b3ef 100644 --- a/backend/tests/test_csrf.py +++ b/backend/tests/test_csrf.py @@ -15,9 +15,10 @@ import tempfile import pytest +from fastapi.testclient import TestClient + import services.db_service as db from app import app -from fastapi.testclient import TestClient # ── shared test DB (same pattern as test_api.py) ───────────────────────────── _tmp = tempfile.mktemp(suffix="_csrf.db") diff --git a/backend/tests/test_docx_tables.py b/backend/tests/test_docx_tables.py index b51de038..456842a7 100644 --- a/backend/tests/test_docx_tables.py +++ b/backend/tests/test_docx_tables.py @@ -4,6 +4,7 @@ import tempfile from docx import Document as DocxDocument + from services.docx_loader import DocxWithTablesLoader diff --git a/backend/tests/test_model_cache.py b/backend/tests/test_model_cache.py index 9c1ac36f..1e26d7f1 100644 --- a/backend/tests/test_model_cache.py +++ b/backend/tests/test_model_cache.py @@ -2,6 +2,7 @@ from unittest.mock import patch import pytest + from services.ollama_service import get_model_info, model_metadata_cache from utils.cache import TTLCache diff --git a/backend/tests/test_retry_policy.py b/backend/tests/test_retry_policy.py index b5bd4ace..36d472c5 100644 --- a/backend/tests/test_retry_policy.py +++ b/backend/tests/test_retry_policy.py @@ -2,6 +2,7 @@ import httpx import pytest + from services.ollama_service import chat, chat_stream diff --git a/backend/tests/test_settings_timeout.py b/backend/tests/test_settings_timeout.py index 0b919963..960c3291 100644 --- a/backend/tests/test_settings_timeout.py +++ b/backend/tests/test_settings_timeout.py @@ -6,9 +6,10 @@ import types import pytest +from fastapi.testclient import TestClient + import services.db_service as db from app import app -from fastapi.testclient import TestClient _tmp = tempfile.mktemp(suffix=".db") db.DB_PATH = _tmp diff --git a/backend/tests/test_streaming_recovery.py b/backend/tests/test_streaming_recovery.py index 63b064de..8394d7da 100644 --- a/backend/tests/test_streaming_recovery.py +++ b/backend/tests/test_streaming_recovery.py @@ -4,6 +4,7 @@ from unittest.mock import AsyncMock, patch import pytest + import services.db_service as db from models.schemas import ChatRequest from routes.chat import ACTIVE_STREAMS, StreamBuffer, chat_stream diff --git a/backend/tests_random.log b/backend/tests_random.log new file mode 100644 index 0000000000000000000000000000000000000000..c631a3c0a54082f6767dce88a02f0d1e473c0019 GIT binary patch literal 6128 zcmdUzc~2Wh7>DQYO8pKiS1m-X&5@7;DUk?)7L_zc<1-t@66wS4a0At2+RCd`F|0fg??y<9{(@GR@e`%a1;jde@M?G zzS`jd;|Hv3)ANA;et6BCZaC)qF=O5EE4@4P)x&+(?l9hFA*!*?@W zXRJfd4)Z&FAF@MQu{}kh5qCb~|3%nj);8}x?*Vk`P*3}{!XDppaU4$LSfBsT!)MTL z(SHi{9aeSYj1EZl;*O0te>p6M`EWbR<#A=b+#Z8j?pn}u{ZaUtc8|A+_u`tTNUn`c zcjEXfM)o4F<(h(#<1vLI)TU51kybmrqIZZq2B4P82`y!&#FUmal%)G8*BW~s@Lfbs zhu~40gV2cIE3^vMiL(vPg@{lc_u`k*`I^3D6FuHXd{yOFAmKOk)p-U;{x0uR;6YJBd?lVsh8>V`y<&^ zaWq(^M6%Zu5x3DTd9J&5L6ZEXYU%X~a_7DqUGnzqkyQkjlSJb8708WJInGq+%Gnk= zQv+EplM!QD!g_wzl)9{HtBqY9M_bXh2eeM1t1T*@Uet%$Ot@M|vCF77B*L^hMSOjt zATqSkhx3%3D-*4ZRL-(a#u_aX)P3lUk##B8H@oaOU{#jgbuV%Evyq{HKZoh5sTo>L5OEKo;cy?LD zIf`fr=Ezia8_jdfX`*qfK=X48AF7Q`t<;)Ic>?%gJdx~Le z%=1}^GpzS>jw!Rgk#!Q;%tV}?q(8%Kbzg*;x9mIMe7cfmg0^J zWXwB^Z!>Zo-j+b%I_<}OT2uqaqxJ?nSl6W!nH9Vh=^NyeY+pNh45lq|P!ZlhfCll=+2kGw=pl?$C5Y4==agr0Q@JTHQ#~kDHz|4Jg-^?`yA^NcYWTy zQPudNyy<;;&pD7cB6_)1kB`Flj9-Dba!XDersWPP3BFHLy1e$*R0?OMruzD*uhjJ5HpS-@4Y?b!;{NHrXt;Ciuw zzY6`k(E>_8D1E|{y(bU%Xd>%wl>H80S@O$c|4}oZ1iZm8j=Hi1#z`9DF0;A5p}qZI zZlrRIIDiMEhPPa12>EP9zEccI<5@;pXS!i}0{7bO0rT7E+$I@)J-2&ivhMI^X(s0G zbc^+nG4I$OGI|ncG}zVnYgTZT_eI_zI7^Lq+Uh_f*^u>o@AbI|C5%Db>GW6HIx9O~oqpgP-&l-=_Xj=zXpRBu4f&Mt2b)Li0PN=iF%lXFPH z*)d@^7yp-hiRf}M$JkZgBxdQKjc8c{mp!S7FrRwXkXw1`=%c~B@B2;x)%bStvx?dK w{K`9>+Qn$S Date: Sat, 1 Aug 2026 19:16:12 +0530 Subject: [PATCH 6/6] style: fix import sorting (ruff) --- backend/app.py | 1 - backend/routes/chat.py | 3 +-- backend/routes/export.py | 3 +-- backend/routes/models.py | 1 - backend/routes/plugins.py | 1 - backend/routes/prompt_templates.py | 1 - backend/routes/settings.py | 1 - backend/routes/upload.py | 1 - backend/services/db_service.py | 1 - backend/services/ollama_service.py | 1 - backend/services/rag_service.py | 1 - backend/tests/test_api.py | 3 +-- backend/tests/test_audit_log.py | 3 +-- backend/tests/test_backup.py | 1 - backend/tests/test_bulk_export.py | 3 +-- backend/tests/test_cancellation.py | 1 - backend/tests/test_citations.py | 3 +-- backend/tests/test_csrf.py | 3 +-- backend/tests/test_docx_tables.py | 1 - backend/tests/test_model_cache.py | 1 - backend/tests/test_retry_policy.py | 1 - backend/tests/test_settings_timeout.py | 3 +-- backend/tests/test_streaming_recovery.py | 1 - 23 files changed, 8 insertions(+), 31 deletions(-) diff --git a/backend/app.py b/backend/app.py index 874f2df2..e8e6e2ad 100644 --- a/backend/app.py +++ b/backend/app.py @@ -16,7 +16,6 @@ from fastapi.middleware.gzip import GZipMiddleware from fastapi.responses import FileResponse from fastapi.staticfiles import StaticFiles - from middleware.csrf import OriginValidationMiddleware from routes.chat import router as chat_router from routes.export import router as export_router diff --git a/backend/routes/chat.py b/backend/routes/chat.py index fd78ce01..8cc21d3a 100644 --- a/backend/routes/chat.py +++ b/backend/routes/chat.py @@ -11,9 +11,8 @@ import psutil from fastapi import APIRouter, HTTPException from fastapi.responses import StreamingResponse -from pydantic import BaseModel - from models.schemas import ChatRequest, ChatResponse +from pydantic import BaseModel from services import db_service, ollama_service logger = logging.getLogger(__name__) diff --git a/backend/routes/export.py b/backend/routes/export.py index f7629291..5dc9d947 100644 --- a/backend/routes/export.py +++ b/backend/routes/export.py @@ -6,9 +6,8 @@ from fastapi import APIRouter, HTTPException from fastapi.responses import Response -from pydantic import BaseModel, field_validator - from models.schemas import ExportFormat +from pydantic import BaseModel, field_validator from services import db_service router = APIRouter() diff --git a/backend/routes/models.py b/backend/routes/models.py index 4fc9d119..b5773560 100644 --- a/backend/routes/models.py +++ b/backend/routes/models.py @@ -2,7 +2,6 @@ from fastapi import APIRouter, HTTPException from fastapi.responses import StreamingResponse - from services import ollama_service router = APIRouter() diff --git a/backend/routes/plugins.py b/backend/routes/plugins.py index 64287bfd..39c235d3 100644 --- a/backend/routes/plugins.py +++ b/backend/routes/plugins.py @@ -12,7 +12,6 @@ import tempfile from fastapi import APIRouter, HTTPException - from models.schemas import PluginResult, PluginRun from services import db_service diff --git a/backend/routes/prompt_templates.py b/backend/routes/prompt_templates.py index 2489dd7f..003776ad 100644 --- a/backend/routes/prompt_templates.py +++ b/backend/routes/prompt_templates.py @@ -1,5 +1,4 @@ from fastapi import APIRouter, HTTPException - from models.schemas import PromptTemplateCreate, PromptTemplateUpdate from services import db_service diff --git a/backend/routes/settings.py b/backend/routes/settings.py index a89a8669..02d1e740 100644 --- a/backend/routes/settings.py +++ b/backend/routes/settings.py @@ -7,7 +7,6 @@ from typing import Any from fastapi import APIRouter, HTTPException, status - from models.schemas import AppSettings from services.db_service import get_settings, save_setting, save_settings from utils.config import settings diff --git a/backend/routes/upload.py b/backend/routes/upload.py index 972aa4f2..1834c95b 100644 --- a/backend/routes/upload.py +++ b/backend/routes/upload.py @@ -15,7 +15,6 @@ Query, UploadFile, ) - from models.schemas import UploadResponse from services import db_service from utils import audit_log diff --git a/backend/services/db_service.py b/backend/services/db_service.py index 6fbcb62a..3aa933a4 100644 --- a/backend/services/db_service.py +++ b/backend/services/db_service.py @@ -13,7 +13,6 @@ from sqlite3 import OperationalError import grapheme - from utils.config import settings # ------------------------Vacuum Scheduling-------------------------------------------------------- diff --git a/backend/services/ollama_service.py b/backend/services/ollama_service.py index ce1f5e35..96dae3e0 100644 --- a/backend/services/ollama_service.py +++ b/backend/services/ollama_service.py @@ -8,7 +8,6 @@ from collections.abc import AsyncGenerator import httpx - from utils.cache import TTLCache from utils.config import settings from utils.retry import with_retry diff --git a/backend/services/rag_service.py b/backend/services/rag_service.py index b73372a4..0ac242b8 100644 --- a/backend/services/rag_service.py +++ b/backend/services/rag_service.py @@ -17,7 +17,6 @@ ) from langchain_text_splitters import RecursiveCharacterTextSplitter from sentence_transformers import SentenceTransformer - from services.citation_utils import build_sources from services.csv_loader import CleanCSVLoader from services.docx_loader import DocxWithTablesLoader diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index c4e6b89d..ab964f69 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -6,10 +6,9 @@ import tempfile from unittest.mock import AsyncMock, patch -from fastapi.testclient import TestClient - import services.db_service as db from app import app +from fastapi.testclient import TestClient _tmp = tempfile.mktemp(suffix=".db") db.DB_PATH = _tmp diff --git a/backend/tests/test_audit_log.py b/backend/tests/test_audit_log.py index 8342c3f1..dfbfc1bc 100644 --- a/backend/tests/test_audit_log.py +++ b/backend/tests/test_audit_log.py @@ -11,11 +11,10 @@ from unittest.mock import patch import pytest -from fastapi.testclient import TestClient - import routes.upload as upload_module import services.db_service as db from app import app +from fastapi.testclient import TestClient from utils import audit_log _tmp = tempfile.mktemp(suffix=".db") diff --git a/backend/tests/test_backup.py b/backend/tests/test_backup.py index 044f23a7..fc075e8b 100644 --- a/backend/tests/test_backup.py +++ b/backend/tests/test_backup.py @@ -16,7 +16,6 @@ import uuid import pytest - from services import db_service from services.db_service import ( backup_db, diff --git a/backend/tests/test_bulk_export.py b/backend/tests/test_bulk_export.py index b47379ea..117d85b6 100644 --- a/backend/tests/test_bulk_export.py +++ b/backend/tests/test_bulk_export.py @@ -1,9 +1,8 @@ import tempfile -from fastapi.testclient import TestClient - import services.db_service as db from app import app +from fastapi.testclient import TestClient # Setup temporary database for bulk export tests _tmp = tempfile.mktemp(suffix=".db") diff --git a/backend/tests/test_cancellation.py b/backend/tests/test_cancellation.py index c412388b..3f37c2b3 100644 --- a/backend/tests/test_cancellation.py +++ b/backend/tests/test_cancellation.py @@ -3,7 +3,6 @@ from unittest.mock import AsyncMock, patch import pytest - import services.db_service as db from models.schemas import ChatRequest from routes.chat import ACTIVE_STREAMS, cancel_stream, chat_stream diff --git a/backend/tests/test_citations.py b/backend/tests/test_citations.py index 86f8262a..3b941eb5 100644 --- a/backend/tests/test_citations.py +++ b/backend/tests/test_citations.py @@ -12,10 +12,9 @@ import tempfile from unittest.mock import AsyncMock, patch -from fastapi.testclient import TestClient - import services.db_service as db from app import app +from fastapi.testclient import TestClient from models.schemas import ChatMessage, MessageRole, SourceChunk # ─── Shared test client ────────────────────────────────────────── diff --git a/backend/tests/test_csrf.py b/backend/tests/test_csrf.py index a699b3ef..5039f375 100644 --- a/backend/tests/test_csrf.py +++ b/backend/tests/test_csrf.py @@ -15,10 +15,9 @@ import tempfile import pytest -from fastapi.testclient import TestClient - import services.db_service as db from app import app +from fastapi.testclient import TestClient # ── shared test DB (same pattern as test_api.py) ───────────────────────────── _tmp = tempfile.mktemp(suffix="_csrf.db") diff --git a/backend/tests/test_docx_tables.py b/backend/tests/test_docx_tables.py index 456842a7..b51de038 100644 --- a/backend/tests/test_docx_tables.py +++ b/backend/tests/test_docx_tables.py @@ -4,7 +4,6 @@ import tempfile from docx import Document as DocxDocument - from services.docx_loader import DocxWithTablesLoader diff --git a/backend/tests/test_model_cache.py b/backend/tests/test_model_cache.py index 1e26d7f1..9c1ac36f 100644 --- a/backend/tests/test_model_cache.py +++ b/backend/tests/test_model_cache.py @@ -2,7 +2,6 @@ from unittest.mock import patch import pytest - from services.ollama_service import get_model_info, model_metadata_cache from utils.cache import TTLCache diff --git a/backend/tests/test_retry_policy.py b/backend/tests/test_retry_policy.py index 36d472c5..b5bd4ace 100644 --- a/backend/tests/test_retry_policy.py +++ b/backend/tests/test_retry_policy.py @@ -2,7 +2,6 @@ import httpx import pytest - from services.ollama_service import chat, chat_stream diff --git a/backend/tests/test_settings_timeout.py b/backend/tests/test_settings_timeout.py index 960c3291..0b919963 100644 --- a/backend/tests/test_settings_timeout.py +++ b/backend/tests/test_settings_timeout.py @@ -6,10 +6,9 @@ import types import pytest -from fastapi.testclient import TestClient - import services.db_service as db from app import app +from fastapi.testclient import TestClient _tmp = tempfile.mktemp(suffix=".db") db.DB_PATH = _tmp diff --git a/backend/tests/test_streaming_recovery.py b/backend/tests/test_streaming_recovery.py index 8394d7da..63b064de 100644 --- a/backend/tests/test_streaming_recovery.py +++ b/backend/tests/test_streaming_recovery.py @@ -4,7 +4,6 @@ from unittest.mock import AsyncMock, patch import pytest - import services.db_service as db from models.schemas import ChatRequest from routes.chat import ACTIVE_STREAMS, StreamBuffer, chat_stream