diff --git a/backend/app.py b/backend/app.py index 77bfd872..6618cfdb 100644 --- a/backend/app.py +++ b/backend/app.py @@ -10,7 +10,6 @@ import time import uuid from contextlib import asynccontextmanager -from pathlib import Path from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware @@ -26,7 +25,12 @@ from routes.sessions import router as sessions_router from routes.settings import router as settings_router from routes.upload import router as upload_router -from services.db_service import dedupe_clear_orphaned_processing, get_db, init_db +from services.db_service import ( + dedupe_clear_orphaned_processing, + get_db, + init_db, +) +from utils.config import settings # --- Issue #284 Engine Stability: Contextual Thread-Safe Log Formatter --- @@ -53,7 +57,7 @@ def format(self, record): root_logger.handlers = [stream_handler] logger = logging.getLogger(__name__) -FRONTEND_DIST = Path(os.getenv("FRONTEND_DIST", "/app/frontend/dist")) +FRONTEND_DIST = settings.frontend_dist def run_preflight_checks(): @@ -102,7 +106,6 @@ async def lifespan(app: FastAPI): # server run. They will never be resolved to 'done', so leaving them # would produce false-409 responses for legitimate first retries. dedupe_clear_orphaned_processing() - # Start stream cleanup task from routes.chat import clean_expired_streams @@ -141,11 +144,9 @@ async def add_request_correlation_id(request: Request, call_next): return response -default_cors_origins = "http://localhost:3000,http://127.0.0.1:3000,http://localhost:5173,http://localhost:8000" + cors_origins = [ - origin.strip() - for origin in os.getenv("CORS_ORIGINS", default_cors_origins).split(",") - if origin.strip() + origin.strip() for origin in settings.cors_origins.split(",") if origin.strip() ] app.add_middleware(GZipMiddleware, minimum_size=1000) diff --git a/backend/requirements.txt b/backend/requirements.txt index d29b8864..93e5570f 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -10,6 +10,7 @@ python-docx==1.1.2 unstructured==0.18.31 python-multipart==0.0.9 pydantic==2.9.0 +pydantic-settings==2.5.2 python-dotenv==1.0.1 httpx==0.27.0 pytest==8.3.0 diff --git a/backend/routes/settings.py b/backend/routes/settings.py index e1605b64..02d1e740 100644 --- a/backend/routes/settings.py +++ b/backend/routes/settings.py @@ -2,7 +2,6 @@ import asyncio import logging -import os import time from collections.abc import Callable from typing import Any @@ -10,6 +9,7 @@ from fastapi import APIRouter, HTTPException, status from models.schemas import AppSettings from services.db_service import get_settings, save_setting, save_settings +from utils.config import settings router = APIRouter() logger = logging.getLogger(__name__) @@ -18,16 +18,7 @@ def _resolve_settings_timeout_seconds() -> float: - raw_timeout = os.getenv("SETTINGS_API_TIMEOUT_SECONDS", str(DEFAULT_SETTINGS_API_TIMEOUT_SECONDS)) - try: - parsed_timeout = float(raw_timeout) - except ValueError: - logger.warning( - "settings_timeout_invalid raw_value=%s fallback_timeout_s=%s", - raw_timeout, - DEFAULT_SETTINGS_API_TIMEOUT_SECONDS, - ) - return DEFAULT_SETTINGS_API_TIMEOUT_SECONDS + parsed_timeout = float(settings.settings_api_timeout_seconds) if parsed_timeout <= 0: logger.warning( "settings_timeout_non_positive timeout_s=%s fallback_timeout_s=%s", @@ -41,7 +32,9 @@ def _resolve_settings_timeout_seconds() -> float: SETTINGS_API_TIMEOUT_SECONDS = _resolve_settings_timeout_seconds() -async def _run_with_timeout(operation: str, function: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: +async def _run_with_timeout( + operation: str, function: Callable[..., Any], *args: Any, **kwargs: Any +) -> Any: start_time = time.perf_counter() try: return await asyncio.wait_for( @@ -60,54 +53,65 @@ async def _run_with_timeout(operation: str, function: Callable[..., Any], *args: status_code=status.HTTP_504_GATEWAY_TIMEOUT, detail=f"Settings {operation} operation timed out after {SETTINGS_API_TIMEOUT_SECONDS} seconds.", ) from exc + + @router.get("/") async def get_all(): return await _run_with_timeout("read", get_settings) + @router.put("/") async def update_settings(body: AppSettings): # 1. Enforce safety validation boundary limits on Temperature if body.temperature < 0.0 or body.temperature > 2.0: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail=[{ - "loc": ["body", "temperature"], - "msg": "Temperature must scale cleanly between 0.0 and 2.0.", - "type": "value_error" - }] + detail=[ + { + "loc": ["body", "temperature"], + "msg": "Temperature must scale cleanly between 0.0 and 2.0.", + "type": "value_error", + } + ], ) # 2. Enforce safety validation boundary limits on RAG Context Chunks if body.rag_top_k < 1 or body.rag_top_k > 10: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail=[{ - "loc": ["body", "rag_top_k"], - "msg": "RAG Context chunks selection must stay between 1 and 10.", - "type": "value_error" - }] + detail=[ + { + "loc": ["body", "rag_top_k"], + "msg": "RAG Context chunks selection must stay between 1 and 10.", + "type": "value_error", + } + ], ) # 3. Enforce safety validation boundary limits on RAG Chunk Size if body.rag_chunk_size < 100 or body.rag_chunk_size > 2000: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail=[{ - "loc": ["body", "rag_chunk_size"], - "msg": "RAG chunk size must be between 100 and 2000 characters.", - "type": "value_error" - }] + detail=[ + { + "loc": ["body", "rag_chunk_size"], + "msg": "RAG chunk size must be between 100 and 2000 characters.", + "type": "value_error", + } + ], ) # 4. Enforce safety validation boundary limits on RAG Chunk Overlap if body.rag_chunk_overlap < 0 or body.rag_chunk_overlap > 200: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail=[{ - "loc": ["body", "rag_chunk_overlap"], - "msg": "RAG chunk overlap must be between 0 and 200 characters.", - "type": "value_error" - }] + detail=[ + { + "loc": ["body", "rag_chunk_overlap"], + "msg": "RAG chunk overlap must be between 0 and 200 characters.", + "type": "value_error", + } + ], ) current_settings = get_settings() @@ -126,6 +130,7 @@ async def update_settings(body: AppSettings): await _run_with_timeout("save", save_settings, payload) return await _run_with_timeout("read", get_settings) + @router.put("/{key}") async def update_one(key: str, value: dict): new_value = value.get("value") @@ -140,4 +145,4 @@ async def update_one(key: str, value: dict): ) await _run_with_timeout("save", save_setting, key, new_value) - return {"key": key, "updated": True} \ No newline at end of file + return {"key": key, "updated": True} diff --git a/backend/routes/upload.py b/backend/routes/upload.py index ec9707b9..1834c95b 100644 --- a/backend/routes/upload.py +++ b/backend/routes/upload.py @@ -1,4 +1,5 @@ """Upload routes — /api/upload""" + import logging import os import time @@ -17,6 +18,7 @@ from models.schemas import UploadResponse from services import db_service from utils import audit_log +from utils.config import settings logger = logging.getLogger(__name__) @@ -26,46 +28,81 @@ def _safe_audit(fn, **kwargs): try: fn(**kwargs) except Exception as e: # noqa: BLE001 - logger.warning("audit_hook_failed hook=%s error=%s", getattr(fn, "__name__", fn), e) + logger.warning( + "audit_hook_failed hook=%s error=%s", getattr(fn, "__name__", fn), e + ) router = APIRouter() ALLOWED = { - ".txt", ".md", ".pdf", ".docx", ".doc", ".html", - ".htm", ".csv", ".json", ".xml", ".rtf", ".odt", - ".epub", ".log", ".tsv", ".ini", ".cfg", ".yaml", ".yml" + ".txt", + ".md", + ".pdf", + ".docx", + ".doc", + ".html", + ".htm", + ".csv", + ".json", + ".xml", + ".rtf", + ".odt", + ".epub", + ".log", + ".tsv", + ".ini", + ".cfg", + ".yaml", + ".yml", } -MAX_BYTES = 50 * 1024 * 1024 # 50 MB +MAX_BYTES = settings.max_file_size -UPLOAD_DIR = Path(os.getenv("UPLOAD_DIR", "./data/uploads")) +UPLOAD_DIR = settings.upload_dir UPLOAD_DIR.mkdir(parents=True, exist_ok=True) @router.post("/", response_model=UploadResponse) -async def upload(file: UploadFile = File(...), session_id: str = Form(...), background_tasks: BackgroundTasks = None): # noqa: B008 - logger.info("upload_request route=/upload session=%s file=%s", session_id, file.filename) +async def upload( + file: UploadFile = File(...), # noqa: B008 + session_id: str = Form(...), + background_tasks: BackgroundTasks = None, +): + logger.info( + "upload_request route=/upload session=%s file=%s", session_id, file.filename + ) ext = Path(file.filename).suffix.lower() if ext not in ALLOWED: - logger.warning("upload_rejected reason=unsupported_type ext=%s file=%s", ext, file.filename) + logger.warning( + "upload_rejected reason=unsupported_type ext=%s file=%s", ext, file.filename + ) raise HTTPException(status_code=400, detail=f"File type {ext} not allowed.") - + content = await file.read() if len(content) > MAX_BYTES: - logger.warning("upload_rejected reason=file_too_large size_bytes=%s limit=%s", len(content), MAX_BYTES) + logger.warning( + "upload_rejected reason=file_too_large size_bytes=%s limit=%s", + len(content), + MAX_BYTES, + ) raise HTTPException(status_code=413, detail="File too large (max 50MB).") - + file_path = UPLOAD_DIR / f"{session_id}_{file.filename}" file_path.write_bytes(content) size_kb = max(1, len(content) // 1024) - + # Restored original repository database calls: db_service.create_session(session_id) - doc_id = db_service.save_document(session_id, file.filename, str(file_path), 0, size_kb, status="queued") - + doc_id = db_service.save_document( + session_id, file.filename, str(file_path), 0, size_kb, status="queued" + ) + logger.info( "document_queued route=/upload session=%s file=%s doc_id=%s size_kb=%s", - session_id, file.filename, doc_id, size_kb, + session_id, + file.filename, + doc_id, + size_kb, ) # --- Issue #797: structured audit log — UPLOAD_QUEUED --- @@ -77,10 +114,12 @@ async def upload(file: UploadFile = File(...), session_id: str = Form(...), back ) if background_tasks: - background_tasks.add_task(process_document_task, str(file_path), session_id, doc_id) + background_tasks.add_task( + process_document_task, str(file_path), session_id, doc_id + ) else: process_document_task(str(file_path), session_id, doc_id) - + # Fixed Pydantic validation schema matching: return UploadResponse( doc_id=doc_id, @@ -88,7 +127,7 @@ async def upload(file: UploadFile = File(...), session_id: str = Form(...), back file_size_kb=size_kb, chunks_indexed=0, status="queued", - message=f"'{file.filename}' uploaded and processing started." + message=f"'{file.filename}' uploaded and processing started.", ) @@ -100,14 +139,17 @@ def process_document_task(file_path: str, session_id: str, doc_id: int): try: from services import rag_service + db_service.update_document_status(doc_id, "processing") chunks = rag_service.index_document(file_path, session_id, doc_id=doc_id) # Restored original status completion name ("completed"): db_service.update_document_status(doc_id, "completed", chunks_indexed=chunks) - + logger.info( "document_indexed route=/upload session=%s doc_id=%s chunks=%s", - session_id, doc_id, chunks, + session_id, + doc_id, + chunks, ) # --- Issue #797: structured audit log — SUCCESS --- @@ -121,7 +163,9 @@ def process_document_task(file_path: str, session_id: str, doc_id: int): except Exception as e: # noqa: BLE001 logger.error( "document_failed route=/upload session=%s doc_id=%s error=%s", - session_id, doc_id, e, + session_id, + doc_id, + e, ) # --- Issue #797: structured audit log — FAILED --- @@ -140,34 +184,51 @@ def process_document_task(file_path: str, session_id: str, doc_id: int): @router.get("/preview") async def preview_document(filename: str = Query(...), session_id: str = Query(...)): - logger.info("preview_request route=/upload/preview session=%s file=%s", session_id, filename) + logger.info( + "preview_request route=/upload/preview session=%s file=%s", session_id, filename + ) file_path = UPLOAD_DIR / f"{session_id}_{filename}" if not file_path.exists(): logger.warning("preview_failed reason=file_not_found path=%s", file_path) raise HTTPException(status_code=404, detail="Document file not found.") - + try: TEXT_FORMATS = { - ".txt", ".md", ".html", ".htm", ".csv", ".json", - ".xml", ".log", ".tsv", ".ini", ".cfg", ".yaml", - ".yml", ".srt", ".vtt" + ".txt", + ".md", + ".html", + ".htm", + ".csv", + ".json", + ".xml", + ".log", + ".tsv", + ".ini", + ".cfg", + ".yaml", + ".yml", + ".srt", + ".vtt", } ext = Path(file_path).suffix.lower() if ext in TEXT_FORMATS: content = file_path.read_text(encoding="utf-8", errors="ignore") else: from services.rag_service import LOADERS + loader_cls = LOADERS.get(ext) if not loader_cls: content = file_path.read_text(encoding="utf-8", errors="ignore") else: docs = loader_cls(str(file_path)).load() content = "\n".join([doc.page_content for doc in docs]) - + return {"content": content} except Exception as e: # noqa: BLE001 logger.error("preview_failed path=%s error=%s", file_path, e) - raise HTTPException(status_code=500, detail=f"Failed to read document preview: {e!s}") + raise HTTPException( + status_code=500, detail=f"Failed to read document preview: {e!s}" + ) @router.get("/", response_model=list) @@ -180,13 +241,13 @@ async def delete_document(doc_id: int, session_id: str = Query(...)): doc = db_service.get_document_by_id(doc_id) if not doc or doc.get("session_id") != session_id: raise HTTPException(status_code=404, detail="Document not found.") - + file_path = doc.get("file_path", "") if file_path and os.path.exists(file_path): try: os.remove(file_path) except OSError as e: logger.warning("file_delete_failed path=%s error=%s", file_path, e) - + db_service.delete_document(doc_id) - return {"status": "success", "message": f"Document #{doc_id} deleted."} \ No newline at end of file + return {"status": "success", "message": f"Document #{doc_id} deleted."} diff --git a/backend/services/db_service.py b/backend/services/db_service.py index 4f30f1a1..b9cca5a8 100644 --- a/backend/services/db_service.py +++ b/backend/services/db_service.py @@ -13,9 +13,11 @@ from sqlite3 import OperationalError import grapheme +from utils.config import settings # ------------------------Vacuum Scheduling-------------------------------------------------------- -VACUUM_THRESHOLD = int(os.getenv("DB_VACUUM_THRESHOLD", "500")) +VACUUM_THRESHOLD = settings.db_vacuum_threshold + def _get_deleted_counter(conn) -> int: row = conn.execute( @@ -23,7 +25,8 @@ def _get_deleted_counter(conn) -> int: ).fetchone() return int(row["value"]) if row else 0 -def _increment_deleted_counter(conn,count:int) -> int: + +def _increment_deleted_counter(conn, count: int) -> int: new_value = _get_deleted_counter(conn) + count conn.execute( "INSERT OR REPLACE INTO app_settings (key,value,updated_at) VALUES (?,?, datetime('now'))", @@ -31,9 +34,10 @@ def _increment_deleted_counter(conn,count:int) -> int: ) return new_value + def run_vacuum(): - """ Run VACUUM outside any transaction to reclaim disk space.""" - conn = sqlite3.connect(DB_PATH, timeout=5, isolation_level = None) + """Run VACUUM outside any transaction to reclaim disk space.""" + conn = sqlite3.connect(DB_PATH, timeout=5, isolation_level=None) try: conn.execute("VACUUM") conn.execute( @@ -42,8 +46,9 @@ def run_vacuum(): finally: conn.close() + def _maybe_vacuum(deleted_count: int): - """Track deletions and trigger VACUUM once threshold is crossed.""" + """Track deletions and trigger VACUUM once threshold is crossed.""" if deleted_count <= 0: return with get_db() as conn: @@ -54,6 +59,7 @@ def _maybe_vacuum(deleted_count: int): # ─── Backup / Restore ──────────────────────────────────────────────────────── + def backup_db(dest_path: str) -> None: """Create a consistent backup of the live database at *dest_path*. @@ -96,9 +102,7 @@ def restore_db(src_path: str) -> None: """ src_path = str(src_path) if not os.path.exists(src_path): - raise FileNotFoundError( - f"restore_db: backup file not found at '{src_path}'" - ) + raise FileNotFoundError(f"restore_db: backup file not found at '{src_path}'") try: src = sqlite3.connect(src_path, timeout=5) @@ -114,9 +118,10 @@ def restore_db(src_path: str) -> None: ) from exc - -DB_PATH = os.getenv("DB_PATH", "./data/localmind.db") -os.makedirs(os.path.dirname(DB_PATH) if os.path.dirname(DB_PATH) else ".", exist_ok=True) +DB_PATH = str(settings.db_path) +os.makedirs( + os.path.dirname(DB_PATH) if os.path.dirname(DB_PATH) else ".", exist_ok=True +) logger = logging.getLogger(__name__) @@ -136,16 +141,15 @@ def get_db(): break except OperationalError as e: - if "locked" in str(e).lower() and attempt < retries - 1: - logger.warning( - "Database locked (attempt %d/%d). Retrying...", - attempt + 1, - retries, - ) - time.sleep(delay) - continue - raise - + if "locked" in str(e).lower() and attempt < retries - 1: + logger.warning( + "Database locked (attempt %d/%d). Retrying...", + attempt + 1, + retries, + ) + time.sleep(delay) + continue + raise try: yield conn @@ -154,9 +158,7 @@ def get_db(): except OperationalError as e: if "locked" in str(e).lower(): conn.rollback() - raise RuntimeError( - "Database is busy. Please try again in a moment." - ) from e + raise RuntimeError("Database is busy. Please try again in a moment.") from e conn.rollback() raise @@ -168,6 +170,7 @@ def get_db(): if conn: conn.close() + def init_db(): """Create all tables on startup.""" with get_db() as conn: @@ -257,7 +260,6 @@ def init_db(): CREATE INDEX IF NOT EXISTS idx_dedupe_expires ON dedupe_cache (expires_at); INSERT OR IGNORE INTO app_settings (key, value) VALUES - ('default_model', '"llama3"'), ('default_language', '"en"'), ('temperature', '0.7'), ('max_history_turns', '10'), @@ -267,20 +269,37 @@ def init_db(): """) + conn.execute( + "INSERT OR IGNORE INTO app_settings (key, value) VALUES (?, ?)", + ("default_model", json.dumps(settings.default_model)), + ) try: - conn.execute("ALTER TABLE documents ADD COLUMN status TEXT DEFAULT 'completed'") + conn.execute( + "ALTER TABLE documents ADD COLUMN status TEXT DEFAULT 'completed'" + ) except sqlite3.OperationalError: pass # column already exists - cols = [row[1] for row in conn.execute("PRAGMA table_info(messages)").fetchall()] + cols = [ + row[1] for row in conn.execute("PRAGMA table_info(messages)").fetchall() + ] if "benchmarks" not in cols: conn.execute("ALTER TABLE messages ADD COLUMN benchmarks TEXT DEFAULT '{}'") - cols_sessions = [row[1] for row in conn.execute("PRAGMA table_info(sessions)").fetchall()] + cols_sessions = [ + row[1] for row in conn.execute("PRAGMA table_info(sessions)").fetchall() + ] if "language" not in cols_sessions: conn.execute("ALTER TABLE sessions ADD COLUMN language TEXT DEFAULT 'en'") + + # ─── Sessions ──────────────────────────────────────────────── -def create_session(session_id: str, title: str = "New Chat", model: str = "llama3", language: str = "en") -> dict: +def create_session( + session_id: str, + title: str = "New Chat", + model: str = "llama3", + language: str = "en", +) -> dict: with get_db() as conn: conn.execute( "INSERT OR IGNORE INTO sessions (id, title, model, language) VALUES (?, ?, ?, ?)", @@ -291,25 +310,43 @@ def create_session(session_id: str, title: str = "New Chat", model: str = "llama def get_session(session_id: str) -> dict | None: with get_db() as conn: - row = conn.execute("SELECT * FROM sessions WHERE id=?", (session_id,)).fetchone() + row = conn.execute( + "SELECT * FROM sessions WHERE id=?", (session_id,) + ).fetchone() return dict(row) if row else None -def update_session(session_id: str, title: str | None = None, model: str | None = None, language: str | None = None): +def update_session( + session_id: str, + title: str | None = None, + model: str | None = None, + language: str | None = None, +): with get_db() as conn: if title is not None: - conn.execute("UPDATE sessions SET title=?, updated_at=datetime('now') WHERE id=?", (title, session_id)) + conn.execute( + "UPDATE sessions SET title=?, updated_at=datetime('now') WHERE id=?", + (title, session_id), + ) if model is not None: - conn.execute("UPDATE sessions SET model=?, updated_at=datetime('now') WHERE id=?", (model, session_id)) + conn.execute( + "UPDATE sessions SET model=?, updated_at=datetime('now') WHERE id=?", + (model, session_id), + ) if language is not None: - conn.execute("UPDATE sessions SET language=?, updated_at=datetime('now') WHERE id=?", (language, session_id)) + conn.execute( + "UPDATE sessions SET language=?, updated_at=datetime('now') WHERE id=?", + (language, session_id), + ) def delete_session(session_id: str): """Deletes a session, clears its physical document assets from disk, and removes database rows.""" with get_db() as conn: # 1. Fetch all physical file paths for documents bound to this session - rows = conn.execute("SELECT file_path FROM documents WHERE session_id=?", (session_id,)).fetchall() + rows = conn.execute( + "SELECT file_path FROM documents WHERE session_id=?", (session_id,) + ).fetchall() for row in rows: if row["file_path"]: physical_path = row["file_path"] @@ -318,7 +355,9 @@ def delete_session(session_id: str): os.remove(physical_path) print(f"Cleaned up session document asset: {physical_path}") except Exception as file_err: # noqa: BLE001 - print(f"Warning: Failed to delete session asset {physical_path}: {file_err!s}") + print( + f"Warning: Failed to delete session asset {physical_path}: {file_err!s}" + ) # 2. Gather counts for vacuum scheduling metric tracking msg_count = conn.execute( @@ -330,7 +369,7 @@ def delete_session(session_id: str): cur = conn.execute("DELETE FROM sessions WHERE id=?", (session_id,)) deleted = cur.rowcount + msg_count + doc_count - _maybe_vacuum(deleted) + _maybe_vacuum(deleted) def clear_all_sessions(): @@ -359,7 +398,7 @@ def toggle_message_reaction(message_id: int, emoji: str) -> str: # Check if this specific emoji reaction already exists for this message row = conn.execute( "SELECT id FROM message_reactions WHERE message_id = ? AND emoji = ?", - (message_id, emoji) + (message_id, emoji), ).fetchone() if row: @@ -368,7 +407,7 @@ def toggle_message_reaction(message_id: int, emoji: str) -> str: else: conn.execute( "INSERT INTO message_reactions (message_id, emoji) VALUES (?, ?)", - (message_id, emoji) + (message_id, emoji), ) return "added" @@ -378,7 +417,7 @@ def get_reactions_for_message(message_id: int) -> list[str]: with get_db() as conn: rows = conn.execute( "SELECT emoji FROM message_reactions WHERE message_id = ? ORDER BY created_at ASC", - (message_id,) + (message_id,), ).fetchall() return [r["emoji"] for r in rows] @@ -389,14 +428,17 @@ def get_session_reactions_map(session_id: str) -> dict[int, list[str]]: Returns a dictionary mapping message_id -> list of emojis. """ with get_db() as conn: - rows = conn.execute(""" + rows = conn.execute( + """ SELECT r.message_id, r.emoji FROM message_reactions r JOIN messages m ON r.message_id = m.id WHERE m.session_id = ? ORDER BY r.created_at ASC - """, (session_id,)).fetchall() - + """, + (session_id,), + ).fetchall() + reactions_map = {} for r in rows: msg_id = r["message_id"] @@ -404,7 +446,15 @@ def get_session_reactions_map(session_id: str) -> dict[int, list[str]]: reactions_map[msg_id] = [] reactions_map[msg_id].append(r["emoji"]) return reactions_map -def save_message(session_id: str, role: str, content: str, sources: list | None = None, benchmarks: dict | None = None): + + +def save_message( + session_id: str, + role: str, + content: str, + sources: list | None = None, + benchmarks: dict | None = None, +): sources = sources or [] with get_db() as conn: conn.execute( @@ -425,7 +475,9 @@ def save_message(session_id: str, role: str, content: str, sources: list | None title = grapheme.slice(content, start=0, end=40) + "..." else: title = content - conn.execute("UPDATE sessions SET title=? WHERE id=?", (title, session_id)) + conn.execute( + "UPDATE sessions SET title=? WHERE id=?", (title, session_id) + ) def get_history(session_id: str, limit: int = 20) -> list[dict]: @@ -450,7 +502,7 @@ def get_messages_full(session_id: str) -> list[dict]: "content": r["content"], "sources": json.loads(r["sources"] or "[]"), "created_at": r["created_at"], - "benchmarks": json.loads(r["benchmarks"] or {}) + "benchmarks": json.loads(r["benchmarks"] or {}), } for r in rows ] @@ -461,7 +513,7 @@ def clear_messages(session_id: str): cur = conn.execute("DELETE FROM messages WHERE session_id=?", (session_id,)) deleted = cur.rowcount conn.execute("UPDATE sessions SET message_count=0 WHERE id=?", (session_id,)) - _maybe_vacuum(deleted) + _maybe_vacuum(deleted) def delete_message(session_id: str, message_id: int) -> int: @@ -486,7 +538,14 @@ def delete_message(session_id: str, message_id: int) -> int: # ─── Documents ─────────────────────────────────────────────── -def save_document(session_id: str, filename: str, file_path: str, chunks: int, size_kb: float, status: str = "completed") -> int: +def save_document( + session_id: str, + filename: str, + file_path: str, + chunks: int, + size_kb: float, + status: str = "completed", +) -> int: with get_db() as conn: cursor = conn.execute( "INSERT INTO documents (session_id, filename, file_path, chunks_indexed, file_size_kb, status) VALUES (?,?,?,?,?,?)", @@ -494,10 +553,14 @@ def save_document(session_id: str, filename: str, file_path: str, chunks: int, s ) return cursor.lastrowid + def update_document_status(doc_id: int, status: str, chunks_indexed: int | None = None): with get_db() as conn: if chunks_indexed is not None: - conn.execute("UPDATE documents SET status=?, chunks_indexed=? WHERE id=?", (status, chunks_indexed, doc_id)) + conn.execute( + "UPDATE documents SET status=?, chunks_indexed=? WHERE id=?", + (status, chunks_indexed, doc_id), + ) else: conn.execute("UPDATE documents SET status=? WHERE id=?", (status, doc_id)) @@ -515,8 +578,10 @@ def delete_document(doc_id: int): """Deletes the physical uploaded file from disk and removes its record entry from SQLite.""" with get_db() as conn: # 1. Fetch the physical file path before deleting the database reference row - row = conn.execute("SELECT file_path FROM documents WHERE id=?", (doc_id,)).fetchone() - + row = conn.execute( + "SELECT file_path FROM documents WHERE id=?", (doc_id,) + ).fetchone() + if row and row["file_path"]: physical_path = row["file_path"] try: @@ -526,13 +591,16 @@ def delete_document(doc_id: int): print(f"Successfully deleted physical file asset: {physical_path}") except Exception as file_err: # noqa: BLE001 # Log the error but continue so the database doesn't lock or desync - print(f"Warning: Failed to clean up disk file {physical_path}: {file_err!s}") + print( + f"Warning: Failed to clean up disk file {physical_path}: {file_err!s}" + ) # 3. Clean up the database record entries cur = conn.execute("DELETE FROM documents WHERE id=?", (doc_id,)) deleted = cur.rowcount - - _maybe_vacuum(deleted) + + _maybe_vacuum(deleted) + # ─── Settings ──────────────────────────────────────────────── def get_settings() -> dict: @@ -562,14 +630,15 @@ def save_settings(settings: dict[str, object]) -> None: # ─── Plugin logs ───────────────────────────────────────────── + def get_plugin_logs(limit: int = 50) -> list[dict]: with get_db() as conn: rows = conn.execute( - "SELECT * FROM plugin_logs ORDER BY created_at DESC LIMIT ?", - (limit,) + "SELECT * FROM plugin_logs ORDER BY created_at DESC LIMIT ?", (limit,) ).fetchall() return [dict(r) for r in rows] + def log_plugin(session_id: str, plugin: str, inp: str, out: str, success: bool = True): with get_db() as conn: conn.execute( @@ -580,9 +649,10 @@ def log_plugin(session_id: str, plugin: str, inp: str, out: str, success: bool = # ─── Shareable Sessions (Issue #270) ───────────────────────── + def create_shared_session(session_id: str) -> str: """ - Captures a frozen snapshot of a chat session's history + Captures a frozen snapshot of a chat session's history and returns a unique, obfuscated sharing ID string. """ # 1. Fetch current session parameters @@ -606,7 +676,7 @@ def create_shared_session(session_id: str) -> str: INSERT INTO shared_sessions (id, session_id, title, model, snapshot_json) VALUES (?, ?, ?, ?, ?) """, - (share_id, session_id, session["title"], session["model"], snapshot_str) + (share_id, session_id, session["title"], session["model"], snapshot_str), ) return share_id @@ -618,7 +688,7 @@ def get_shared_session(share_id: str) -> dict | None: with get_db() as conn: row = conn.execute( "SELECT title, model, snapshot_json, created_at FROM shared_sessions WHERE id = ?", - (share_id,) + (share_id,), ) row = row.fetchone() @@ -629,16 +699,21 @@ def get_shared_session(share_id: str) -> dict | None: "id": share_id, "title": row["title"], "model": row["model"], - "messages": json.loads(row["snapshot_json"]), # Turn string array back into live json dicts - "created_at": row["created_at"] + "messages": json.loads( + row["snapshot_json"] + ), # Turn string array back into live json dicts + "created_at": row["created_at"], } + + # ─── Prompt Templates (Updated Signatures) ─────────────────── + def create_prompt_template(prompt_title: str, prompt: str) -> dict: with get_db() as conn: cursor = conn.execute( "INSERT INTO prompt_templates (name, prompt) VALUES (?, ?)", - (prompt_title, prompt) + (prompt_title, prompt), ) template_id = cursor.lastrowid return get_prompt_template(template_id) @@ -647,8 +722,8 @@ def create_prompt_template(prompt_title: str, prompt: str) -> dict: def get_prompt_template(template_id: int) -> dict | None: with get_db() as conn: row = conn.execute( - "SELECT id, name AS prompt_title, prompt, created_at FROM prompt_templates WHERE id = ?", - (template_id,) + "SELECT id, name AS prompt_title, prompt, created_at FROM prompt_templates WHERE id = ?", + (template_id,), ).fetchone() return dict(row) if row else None @@ -661,12 +736,19 @@ def get_all_prompt_templates() -> list[dict]: return [dict(r) for r in rows] -def update_prompt_template(template_id: int, prompt_title: str | None = None, prompt: str | None = None) -> dict | None: +def update_prompt_template( + template_id: int, prompt_title: str | None = None, prompt: str | None = None +) -> dict | None: with get_db() as conn: if prompt_title: - conn.execute("UPDATE prompt_templates SET name=? WHERE id=?", (prompt_title, template_id)) + conn.execute( + "UPDATE prompt_templates SET name=? WHERE id=?", + (prompt_title, template_id), + ) if prompt: - conn.execute("UPDATE prompt_templates SET prompt=? WHERE id=?", (prompt, template_id)) + conn.execute( + "UPDATE prompt_templates SET prompt=? WHERE id=?", (prompt, template_id) + ) return get_prompt_template(template_id) diff --git a/backend/services/ollama_service.py b/backend/services/ollama_service.py index cdb04f81..96dae3e0 100644 --- a/backend/services/ollama_service.py +++ b/backend/services/ollama_service.py @@ -5,16 +5,16 @@ import asyncio import json import logging -import os from collections.abc import AsyncGenerator import httpx from utils.cache import TTLCache +from utils.config import settings from utils.retry import with_retry logger = logging.getLogger(__name__) -OLLAMA_BASE_URL = os.getenv("OLLAMA_HOST", "http://localhost:11434").rstrip("/") +OLLAMA_BASE_URL = settings.ollama_host.rstrip("/") TIMEOUT = 180.0 SYSTEM_PROMPTS = { @@ -47,6 +47,7 @@ def _build_messages(message: str, context: str, history: list, language: str) -> # Global cache for model metadata (5 minute TTL) model_metadata_cache = TTLCache(ttl_seconds=300) + @with_retry(max_attempts=3, initial_backoff=1.0) async def chat( message: str, @@ -75,6 +76,7 @@ async def chat( response.raise_for_status() return response.json()["message"]["content"] + async def chat_stream( message: str, model: str = "llama3", @@ -92,7 +94,7 @@ async def chat_stream( "stream": True, "options": {"temperature": temperature, "top_p": 0.9, "num_predict": 2048}, } - + max_attempts = 3 actual_max_attempts = max(1, max_attempts) attempt = 1 @@ -103,23 +105,25 @@ async def chat_stream( try: async with ( httpx.AsyncClient(timeout=TIMEOUT) as client, - client.stream("POST", f"{OLLAMA_BASE_URL}/api/chat", json=payload) as resp, + client.stream( + "POST", f"{OLLAMA_BASE_URL}/api/chat", json=payload + ) as resp, ): resp.raise_for_status() async for line in resp.aiter_lines(): - if line.strip(): - try: - data = json.loads(line) - token = data.get("message", {}).get("content", "") - if token: - yield token - if data.get("done"): - break - except json.JSONDecodeError: - continue + if line.strip(): + try: + data = json.loads(line) + token = data.get("message", {}).get("content", "") + if token: + yield token + if data.get("done"): + break + except json.JSONDecodeError: + continue # If we exit the context manager normally, we are done, break out of retry loop break - + except httpx.RequestError as e: is_transient = True error_msg = f"Network Error: {type(e).__name__}" @@ -136,10 +140,14 @@ async def chat_stream( if is_transient: if attempt == actual_max_attempts: - logger.error(f"chat_stream failed after {actual_max_attempts} attempts. Last error: {error_msg}") + logger.error( + f"chat_stream failed after {actual_max_attempts} attempts. Last error: {error_msg}" + ) raise last_exc - - logger.warning(f"chat_stream failed ({error_msg}). Retrying in {backoff}s... (Attempt {attempt}/{actual_max_attempts})") + + logger.warning( + f"chat_stream failed ({error_msg}). Retrying in {backoff}s... (Attempt {attempt}/{actual_max_attempts})" + ) await asyncio.sleep(backoff) attempt += 1 backoff *= 2 @@ -154,12 +162,14 @@ async def list_models() -> list[dict]: models = [] for m in resp.json().get("models", []): size_gb = round(m.get("size", 0) / 1e9, 1) - models.append({ - "name": m["name"], - "size": f"{size_gb} GB", - "status": "available", - "modified_at": m.get("modified_at", ""), - }) + models.append( + { + "name": m["name"], + "size": f"{size_gb} GB", + "status": "available", + "modified_at": m.get("modified_at", ""), + } + ) return models except Exception as e: # noqa: BLE001 logger.warning(f"Could not list models: {e}") @@ -178,18 +188,17 @@ async def get_model_info(model_name: str) -> dict: async with httpx.AsyncClient(timeout=8.0) as client: try: resp = await client.post( - f"{OLLAMA_BASE_URL}/api/show", - json={"name": model_name} + f"{OLLAMA_BASE_URL}/api/show", json={"name": model_name} ) resp.raise_for_status() info = resp.json() - + # Populate cache model_metadata_cache.set(model_name, info) return info except httpx.HTTPStatusError as e: if e.response.status_code == 404: - return {} # Model not found + return {} # Model not found raise except Exception as e: logger.warning(f"Could not fetch metadata for model '{model_name}': {e}") @@ -209,14 +218,15 @@ async def pull_model(model_name: str) -> AsyncGenerator[str, None]: async with ( httpx.AsyncClient(timeout=600.0) as client, client.stream( - "POST", f"{OLLAMA_BASE_URL}/api/pull", - json={"name": model_name, "stream": True} + "POST", + f"{OLLAMA_BASE_URL}/api/pull", + json={"name": model_name, "stream": True}, ) as resp, ): resp.raise_for_status() async for line in resp.aiter_lines(): - if line.strip(): - yield line + "\n" + if line.strip(): + yield line + "\n" break except httpx.RequestError as e: is_transient = True @@ -243,8 +253,7 @@ async def delete_model(model_name: str) -> bool: async with httpx.AsyncClient(timeout=30.0) as client: try: resp = await client.delete( - f"{OLLAMA_BASE_URL}/api/delete", - json={"name": model_name} + f"{OLLAMA_BASE_URL}/api/delete", json={"name": model_name} ) return resp.status_code == 200 except Exception: # noqa: BLE001 diff --git a/backend/services/rag_service.py b/backend/services/rag_service.py index eda47862..0ac242b8 100644 --- a/backend/services/rag_service.py +++ b/backend/services/rag_service.py @@ -20,11 +20,12 @@ from services.citation_utils import build_sources from services.csv_loader import CleanCSVLoader from services.docx_loader import DocxWithTablesLoader +from utils.config import settings logger = logging.getLogger(__name__) -CHROMA_PATH = os.getenv("CHROMADB_DIR", "./data/chromadb") -EMBED_MODEL = "all-MiniLM-L6-v2" +CHROMA_PATH = str(settings.chromadb_dir) +EMBED_MODEL = "all-MiniLM-L6-v2" os.makedirs(CHROMA_PATH, exist_ok=True) @@ -35,14 +36,14 @@ embedder = SentenceTransformer(EMBED_MODEL) LOADERS = { - ".pdf": PyPDFLoader, - ".txt": TextLoader, - ".md": TextLoader, - ".csv": CleanCSVLoader, + ".pdf": PyPDFLoader, + ".txt": TextLoader, + ".md": TextLoader, + ".csv": CleanCSVLoader, ".docx": DocxWithTablesLoader, ".html": UnstructuredHTMLLoader, - ".srt": TextLoader, # Handle SubRip video/audio transcripts natively - ".vtt": TextLoader, # Handle WebVTT audio transcripts natively + ".srt": TextLoader, # Handle SubRip video/audio transcripts natively + ".vtt": TextLoader, # Handle WebVTT audio transcripts natively } @@ -60,9 +61,10 @@ def index_document(file_path: str, session_id: str, doc_id: int | None = None) - raise ValueError(f"Unsupported file type: {ext}. Supported: {list(LOADERS)}") docs = loader_cls(file_path).load() - + # Fetch live chunk size and overlap bounds configuration directly from database cache settings from services.db_service import get_settings + current_settings = get_settings() chunk_size_val = current_settings.get("rag_chunk_size", 600) overlap_val = current_settings.get("rag_chunk_overlap", 50) @@ -73,48 +75,64 @@ def index_document(file_path: str, session_id: str, doc_id: int | None = None) - chunk_overlap=int(overlap_val), separators=["\n\n", "\n", ". ", " "], ) - + chunks = dynamic_splitter.split_documents(docs) if not chunks: return 0 texts = [c.page_content for c in chunks] ids = [f"{session_id}_{i}" for i in range(len(texts))] - metadatas = [{"source": Path(file_path).name, "chunk": i} for i in range(len(texts))] + metadatas = [ + {"source": Path(file_path).name, "chunk": i} for i in range(len(texts)) + ] col = _collection(session_id) - + batch_size = 200 for i in range(0, len(texts), batch_size): - batch_texts = texts[i:i + batch_size] - batch_ids = ids[i:i + batch_size] - batch_metas = metadatas[i:i + batch_size] - - batch_embeddings = embedder.encode(batch_texts, show_progress_bar=False).tolist() - col.upsert(ids=batch_ids, documents=batch_texts, embeddings=batch_embeddings, metadatas=batch_metas) + batch_texts = texts[i : i + batch_size] + batch_ids = ids[i : i + batch_size] + batch_metas = metadatas[i : i + batch_size] + + batch_embeddings = embedder.encode( + batch_texts, show_progress_bar=False + ).tolist() + col.upsert( + ids=batch_ids, + documents=batch_texts, + embeddings=batch_embeddings, + metadatas=batch_metas, + ) if doc_id is not None: from services import db_service - db_service.update_document_status(doc_id, "processing", chunks_indexed=(i + len(batch_texts))) + + db_service.update_document_status( + doc_id, "processing", chunks_indexed=(i + len(batch_texts)) + ) time.sleep(0.05) # Yield GIL to allow event loop to process other requests - logger.info(f"Indexed {len(chunks)} chunks for session={session_id} using chunk_overlap={overlap_val}") + logger.info( + f"Indexed {len(chunks)} chunks for session={session_id} using chunk_overlap={overlap_val}" + ) return len(chunks) -def retrieve_context(query: str, session_id: str, top_k: int = 4) -> tuple[str, list[dict]]: +def retrieve_context( + query: str, session_id: str, top_k: int = 4 +) -> tuple[str, list[dict]]: col = _collection(session_id) if col.count() == 0: return "", [] - q_emb = embedder.encode([query]).tolist() + q_emb = embedder.encode([query]).tolist() results = col.query( query_embeddings=q_emb, n_results=min(top_k, col.count()), include=["documents", "metadatas"], ) - docs = results["documents"][0] if results["documents"] else [] - metas = results["metadatas"][0] if results["metadatas"] else [] + docs = results["documents"][0] if results["documents"] else [] + metas = results["metadatas"][0] if results["metadatas"] else [] context = "\n\n---\n\n".join(docs) @@ -129,9 +147,10 @@ def delete_session_index(session_id: str): try: chroma_client.delete_collection(f"lm_{session_id.replace('-', '_')}") except Exception as e: # noqa: BLE001 - logger.warning("Could not delete ChromaDB collection for session %s: %s", session_id, e) - + logger.warning( + "Could not delete ChromaDB collection for session %s: %s", session_id, e + ) def get_indexed_count(session_id: str) -> int: - return _collection(session_id).count() \ No newline at end of file + return _collection(session_id).count() diff --git a/backend/tests/test_bulk_export.py b/backend/tests/test_bulk_export.py index 26a7655a..2b547fd0 100644 --- a/backend/tests/test_bulk_export.py +++ b/backend/tests/test_bulk_export.py @@ -19,19 +19,19 @@ def clear_db(): def test_bulk_export_json_success(): # 1. Create two test sessions - db.create_session("sess_1", title="Session One", model="llama3", language="en") - db.create_session("sess_2", title="Session Two", model="mistral", language="fr") + db.create_session("sess_json_1", title="Session One", model="llama3", language="en") + db.create_session("sess_json_2", title="Session Two", model="mistral", language="fr") - db.save_message("sess_1", "user", "Hello from user 1") - db.save_message("sess_1", "assistant", "Response 1") + db.save_message("sess_json_1", "user", "Hello from user 1") + db.save_message("sess_json_1", "assistant", "Response 1") - db.save_message("sess_2", "user", "Bonjour") - db.save_message("sess_2", "assistant", "Salut") + db.save_message("sess_json_2", "user", "Bonjour") + db.save_message("sess_json_2", "assistant", "Salut") # 2. Call bulk export response = client.post( "/api/export/sessions", - json={"session_ids": ["sess_1", "sess_2"], "format": "json"} + json={"session_ids": ["sess_json_1", "sess_json_2"], "format": "json"} ) assert response.status_code == 200 assert response.headers["Content-Type"].startswith("application/json") @@ -46,13 +46,13 @@ def test_bulk_export_json_success(): # Check structure sess_1_data = payload["sessions"][0] - assert sess_1_data["session"]["id"] == "sess_1" + assert sess_1_data["session"]["id"] == "sess_json_1" assert sess_1_data["session"]["title"] == "Session One" assert len(sess_1_data["messages"]) == 2 assert sess_1_data["messages"][0]["content"] == "Hello from user 1" sess_2_data = payload["sessions"][1] - assert sess_2_data["session"]["id"] == "sess_2" + assert sess_2_data["session"]["id"] == "sess_json_2" assert sess_2_data["session"]["title"] == "Session Two" assert len(sess_2_data["messages"]) == 2 assert sess_2_data["messages"][0]["content"] == "Bonjour" @@ -112,7 +112,7 @@ def test_bulk_export_validation_empty_ids(): def test_bulk_export_validation_invalid_format(): response = client.post( "/api/export/sessions", - json={"session_ids": ["sess_1"], "format": "html"} + json={"session_ids": ["sess_invalid_format"], "format": "html"} ) assert response.status_code == 422 errors = response.json()["detail"] @@ -120,18 +120,18 @@ def test_bulk_export_validation_invalid_format(): def test_bulk_export_mixed_valid_invalid(): - # sess_1 exists, sess_invalid does not - db.create_session("sess_1", title="Valid Session") - db.save_message("sess_1", "user", "Hello") + # sess_mixed exists, sess_invalid does not + db.create_session("sess_mixed", title="Valid Session") + db.save_message("sess_mixed", "user", "Hello") response = client.post( "/api/export/sessions", - json={"session_ids": ["sess_1", "sess_invalid"], "format": "json"} + json={"session_ids": ["sess_mixed", "sess_invalid"], "format": "json"} ) assert response.status_code == 200 payload = response.json() assert len(payload["sessions"]) == 1 - assert payload["sessions"][0]["session"]["id"] == "sess_1" + assert payload["sessions"][0]["session"]["id"] == "sess_mixed" def test_bulk_export_all_invalid(): diff --git a/backend/tests_random.log b/backend/tests_random.log new file mode 100644 index 00000000..c631a3c0 Binary files /dev/null and b/backend/tests_random.log differ diff --git a/backend/utils/audit_log.py b/backend/utils/audit_log.py index 0f94ab11..664a3a5e 100644 --- a/backend/utils/audit_log.py +++ b/backend/utils/audit_log.py @@ -5,6 +5,7 @@ FAILED) as structured JSON, without ever blocking the calling thread/coroutine and without ever letting a logging failure crash ingestion. """ + from __future__ import annotations import atexit @@ -17,9 +18,11 @@ from contextlib import suppress from typing import Any +from utils.config import settings + _module_logger = logging.getLogger(__name__) -AUDIT_LOG_DIR = os.getenv("AUDIT_LOG_DIR", "./data/logs") +AUDIT_LOG_DIR = str(settings.audit_log_dir) AUDIT_LOG_FILE = os.path.join(AUDIT_LOG_DIR, "audit.jsonl") _audit_logger = logging.getLogger("audit.upload_queue") @@ -35,13 +38,17 @@ class _JsonFormatter(logging.Formatter): def format(self, record: logging.LogRecord) -> str: payload: dict[str, Any] = { "event": getattr(record, "event", record.getMessage()), - "logged_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(record.created)), + "logged_at": time.strftime( + "%Y-%m-%dT%H:%M:%SZ", time.gmtime(record.created) + ), } payload.update(getattr(record, "audit_fields", {})) try: return json.dumps(payload, default=str) except (TypeError, ValueError): - return json.dumps({"event": payload.get("event", "unknown"), "serialization_error": True}) + return json.dumps( + {"event": payload.get("event", "unknown"), "serialization_error": True} + ) def _build_listener() -> logging.handlers.QueueListener: @@ -58,13 +65,17 @@ def _build_listener() -> logging.handlers.QueueListener: file_handler.setFormatter(_JsonFormatter()) handlers.append(file_handler) except OSError as e: - _module_logger.warning("audit_log_file_unavailable path=%s error=%s", AUDIT_LOG_FILE, e) + _module_logger.warning( + "audit_log_file_unavailable path=%s error=%s", AUDIT_LOG_FILE, e + ) log_queue: queue.Queue = queue.Queue(-1) queue_handler = logging.handlers.QueueHandler(log_queue) _audit_logger.handlers = [queue_handler] - listener = logging.handlers.QueueListener(log_queue, *handlers, respect_handler_level=True) + listener = logging.handlers.QueueListener( + log_queue, *handlers, respect_handler_level=True + ) listener.start() return listener @@ -99,6 +110,7 @@ def _now_iso() -> str: # --- Public lifecycle API ------------------------------------------------- + def log_upload_queued(*, file_id: Any, file_size_bytes: int, user_id: str) -> None: _safe_emit( "UPLOAD_QUEUED", @@ -134,7 +146,9 @@ def log_success(*, file_id: Any, user_id: str, duration_ms: float) -> None: ) -def log_failed(*, file_id: Any, user_id: str, duration_ms: float, error: str, stack_trace: str) -> None: +def log_failed( + *, file_id: Any, user_id: str, duration_ms: float, error: str, stack_trace: str +) -> None: _safe_emit( "FAILED", { @@ -145,4 +159,4 @@ def log_failed(*, file_id: Any, user_id: str, duration_ms: float, error: str, st "stack_trace": stack_trace, "timestamp": _now_iso(), }, - ) \ No newline at end of file + ) diff --git a/backend/utils/config.py b/backend/utils/config.py new file mode 100644 index 00000000..512bff83 --- /dev/null +++ b/backend/utils/config.py @@ -0,0 +1,37 @@ +from pathlib import Path + +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + frontend_dist: Path = Field( + default=Path("/app/frontend/dist"), alias="FRONTEND_DIST" + ) + cors_origins: str = Field( + default="http://localhost:3000,http://127.0.0.1:3000,http://localhost:5173,http://localhost:8000", + alias="CORS_ORIGINS", + ) + upload_dir: Path = Field(default=Path("./data/uploads"), alias="UPLOAD_DIR") + settings_api_timeout_seconds: int = Field( + default=10, alias="SETTINGS_API_TIMEOUT_SECONDS" + ) + audit_log_dir: Path = Field(default=Path("./data/logs"), alias="AUDIT_LOG_DIR") + db_vacuum_threshold: int = Field(default=500, alias="DB_VACUUM_THRESHOLD") + db_path: Path = Field(default=Path("./data/localmind.db"), alias="DB_PATH") + ollama_host: str = Field(default="http://localhost:11434", alias="OLLAMA_HOST") + chromadb_dir: Path = Field(default=Path("./data/chromadb"), alias="CHROMADB_DIR") + + # Values from .env.example + default_model: str = Field(default="llama3", alias="DEFAULT_MODEL") + exports_dir: Path = Field(default=Path("./data/exports"), alias="EXPORTS_DIR") + max_file_size: int = Field(default=52428800, alias="MAX_FILE_SIZE") + backend_port: int = Field(default=8000, alias="BACKEND_PORT") + frontend_port: int = Field(default=3000, alias="FRONTEND_PORT") + + model_config = SettingsConfigDict( + env_file=".env", env_file_encoding="utf-8", extra="ignore" + ) + + +settings = Settings()