feat: per-user long-term memory and conversation summarization (#41)

.env.example

@@ -12,7 +12,14 @@ GEMINI_API_KEY=your_api_key_here
 # Scholar review (reviewer endpoints are disabled until the token is set)
 # SCHOLAR_REVIEW_TOKEN=generate_a_long_random_value
 # REVIEW_EXPORT_PATH=data/review/reviewed.jsonl
-# REDIS_URL=redis://localhost:6379   # makes the review queue durable
+# Scholar review (reviewer endpoints are disabled until the token is set)
+# SCHOLAR_REVIEW_TOKEN=generate_a_long_random_value
+# REVIEW_EXPORT_PATH=data/review/reviewed.jsonl
+# REDIS_URL=redis://localhost:6379   # durable review queue + memory store
+
+# Per-user memory (optional — defaults shown)
+# MEMORY_TTL_DAYS=90                 # user-profile and chat-summary TTL in days
+# MEMORY_EXTRACTION_ENABLED=true     # background extraction from conversation turns
 
 # Tafsir layer (optional — sensible defaults, no key required)
 # QURAN_API_BASE=https://api.quran.com/api/v4

.github/workflows/ci.yml

@@ -25,13 +25,13 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
-          pip install pytest flake8
+          pip install pytest flake8 pytest-asyncio
 
       - name: Run linting
-        run: flake8 main.py stellar.py nisab.py safety telemetry.py tests/redteam study.py fiqh.py hadith.py confidence.py review.py review_store.py tafsir.py semantic_cache.py tests/test_confidence.py tests/test_review_queue.py tests/test_tafsir.py tests/test_zakat.py tests/test_telemetry.py tests/test_multilingual.py scripts/build_hadith_data.py scripts/build_surah_index.py --max-line-length=120 --ignore=E501,W503
+        run: flake8 main.py memory stellar.py nisab.py safety telemetry.py tests/redteam study.py fiqh.py hadith.py confidence.py review.py review_store.py tafsir.py semantic_cache.py tests/test_confidence.py tests/test_review_queue.py tests/test_tafsir.py tests/test_zakat.py tests/test_telemetry.py tests/test_multilingual.py tests/test_memory_profile.py tests/test_memory_extraction.py tests/test_memory_integration.py scripts/build_hadith_data.py scripts/build_surah_index.py --max-line-length=120 --ignore=E501,W503
 
       - name: Check syntax
-        run: python -m compileall -q main.py stellar.py nisab.py safety telemetry.py tests/redteam study.py fiqh.py hadith.py confidence.py review.py review_store.py tafsir.py semantic_cache.py scripts/build_hadith_data.py scripts/build_surah_index.py
+        run: python -m compileall -q main.py memory stellar.py nisab.py safety telemetry.py tests/redteam study.py fiqh.py hadith.py confidence.py review.py review_store.py tafsir.py semantic_cache.py scripts/build_hadith_data.py scripts/build_surah_index.py
 
       - name: Run offline safety and red-team tests
         run: pytest -q tests/redteam
@@ -66,6 +66,15 @@ jobs:
       - name: Run multilingual support tests
         run: pytest -q tests/test_multilingual.py
 
+      - name: Run memory profile tests
+        run: pytest -q tests/test_memory_profile.py
+
+      - name: Run memory extraction tests
+        run: pytest -q tests/test_memory_extraction.py
+
+      - name: Run memory integration tests
+        run: pytest -q tests/test_memory_integration.py
+
   docker-build:
     name: Docker Build
     runs-on: ubuntu-latest

README.md

@@ -36,6 +36,8 @@ The platform is composed of three services:
 - 🧵 **Conversation history** per chat session
 - 🛡️ **Content safety filters** on model output
 - 🎚️ **Confidence-aware answers** — abstains or hedges instead of guessing, and routes doubtful religious answers to a scholar
+- 🧠 **Per-user long-term memory** — user profiles (knowledge level, madhhab, topics studied, remembered facts) extracted from conversations and injected across sessions; privacy controls with GET/DELETE endpoints and `remember` opt-out per request
+- 📋 **Conversation summarization** — compaction API ready for token-budget-triggered eviction; merges and recompresses summaries when history exceeds budget
 - 📖 **Tafsir-grounded ayah explanations** — retrieved from named classical works, never paraphrased from model memory
 - ⚡ **FastAPI** with automatic OpenAPI docs at `/docs`
 
@@ -45,6 +47,8 @@ The platform is composed of three services:
 |--------|-------|---------|
 | `POST` | `/chat` | Start or continue a chat session |
 | `DELETE` | `/chat/{chat_id}` | Delete a chat session |
+| `GET` | `/memory/{user_id}` | Retrieve a stored user profile (transparency) |
+| `DELETE` | `/memory/{user_id}` | Completely erase a stored user profile |
 | `GET` | `/ping` | Health check |
 | `GET` | `/cache/stats` | Semantic cache metrics (hits, misses, hit rate, etc.) |
 | `POST` | `/tafsir` | Ayah explanation from named tafsir works, with attribution |
@@ -135,7 +139,9 @@ services:
 | `CONFIDENCE_UNVERIFIED_CEILING` | Cap when nothing external corroborated the answer | `0.65` |
 | `SCHOLAR_REVIEW_TOKEN` | Enables the reviewer endpoints; required as `X-Review-Token` | — (endpoints disabled) |
 | `REVIEW_EXPORT_PATH` | JSONL export of reviewed answers | `data/review/reviewed.jsonl` |
-| `REDIS_URL` | Makes the scholar-review queue durable across restarts | — (in-memory) |
+| `REDIS_URL` | Makes the scholar-review queue and memory store durable across restarts | — (in-memory) |
+| `MEMORY_TTL_DAYS` | Time-to-live for stored user profiles and chat summaries in days | `90` |
+| `MEMORY_EXTRACTION_ENABLED` | Background memory extraction from conversation turns | `true` |
 | `STELLAR_NETWORK` | Stellar network for zakat lookups (`testnet` or `public`) | `testnet` |
 | `ZAKAT_NISAB_USD` | Fallback nisab when no gold price can be fetched | `6000` |
 | `NISAB_CACHE_TTL_SECONDS` | How long a fetched gold price is reused | `21600` (6h) |

main.py

@@ -8,7 +8,7 @@
 from fastapi import FastAPI, HTTPException, Request, Response
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import StreamingResponse
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 import google.generativeai as genai
 import time
 
@@ -62,6 +62,15 @@
 from review import enqueue_for_review, router as review_router
 from review_store import get_review_store
 
+from memory import ChatSummary, UserProfile, create_memory_store, render_user_context
+from memory.extraction import (
+    MEMORY_EXTRACTION_ENABLED,
+    apply_updates,
+    extract_updates,
+    merge_summaries,
+    summarize_conversation_turns,
+)
+
 logger = logging.getLogger(__name__)
 
 # Load environment variables
@@ -115,6 +124,8 @@ class ChatRequest(BaseModel):
     context: Optional[str] = None  # Additional context for specific queries
     madhhab: Optional[str] = None  # User's madhhab: hanafi, maliki, shafii, hanbali
     language: Optional[str] = None  # BCP-47 response language (ar, en, ur, etc.); auto-detect when omitted
+    user_id: Optional[str] = Field(default=None, max_length=128)  # Opaque user identifier for personalization
+    remember: bool = True           # When False, existing memory is read but no new data persisted
 
 
 class Message(BaseModel):
@@ -183,6 +194,11 @@ def classify_for_safety(prompt: str, candidate_ids: List[str]):
 # Durable queue for low-confidence religious answers awaiting a scholar
 review_store = get_review_store()
 
+# Per-user memory store (Redis-backed or in-memory)
+memory_store = create_memory_store()
+
+MAX_CHAT_HISTORY_TURNS = 20
+
 # Tafsir retrieval seam: returns None for prompts that are not
 # verse-explanation questions. Offline tests replace this with a stub.
 DEFAULT_TAFSIR_LANGUAGE = "en"
@@ -466,6 +482,13 @@ def _finalize() -> None:
             zakat_context = await zakat_retriever(request.prompt, request.context)
             zakat_info = zakat_context.info if zakat_context else None
 
+        # --- Memory lookup ---
+        profile: Optional[UserProfile] = None
+        summary: Optional[ChatSummary] = None
+        if request.user_id:
+            profile = await memory_store.get_profile(request.user_id)
+            summary = await memory_store.get_chat_summary(f"{request.user_id}:{chat_id}")
+
         # Neither a tafsir-grounded answer nor a zakat answer goes through the
         # semantic response cache: the first is built from retrieved passages
         # (already cached by ayah key), and the second contains one user's real
@@ -475,6 +498,7 @@ def _finalize() -> None:
             and request.context is None
             and tafsir_context is None
             and zakat_context is None
+            and request.user_id is None
             and SEMANTIC_CACHE_ENABLED
         )
 
@@ -526,6 +550,9 @@ async def generate(safety_prompt: str) -> str:
                 system_context += tafsir_system_context(tafsir_context)
             if zakat_context is not None:
                 system_context += zakat_context.prompt_block
+            memory_block = render_user_context(profile, summary)
+            if memory_block:
+                system_context += f"\n\n{memory_block}"
             context = f"Additional context: {extra_context}\n\n" if extra_context else ""
             full_prompt = f"{system_context}\n{context}User question: {safety_prompt}"
             logger.info("Sending message to chat...")
@@ -682,6 +709,33 @@ async def generate(safety_prompt: str) -> str:
         )
         _finalize()
         _succeeded = True
+
+        # --- Background memory extraction and summarization ---
+        # Runs as fire-and-forget tasks after the response is sent.
+        if request.user_id and request.remember and MEMORY_EXTRACTION_ENABLED:
+            asyncio.create_task(
+                _extract_and_update_memory(
+                    request.user_id, prompt, response_text, chat_id, summary, memory_store,
+                )
+            )
+            logger.info("Memory extraction scheduled for user %s", request.user_id[:8])
+
+        # --- Summary eviction ---
+        # After enough turns accumulate, summarize old history and persist.
+        if request.user_id and request.remember and MEMORY_EXTRACTION_ENABLED:
+            chat_session = active_chats.get(chat_id)
+            if chat_session and hasattr(chat_session, "history") and chat_session.history:
+                if len(chat_session.history) >= MAX_CHAT_HISTORY_TURNS:
+                    asyncio.create_task(
+                        _summarize_history(
+                            f"{request.user_id}:{chat_id}",
+                            chat_session.history,
+                            summary,
+                            memory_store,
+                        )
+                    )
+                    logger.info("History summarization triggered for %s", request.user_id[:8])
+
         return response_obj
 
     except ResourceExhausted as exc:
@@ -734,6 +788,47 @@ async def generate(safety_prompt: str) -> str:
         telemetry.current_trace.reset(_ctx_token)
 
 
+async def _extract_and_update_memory(
+    user_id: str, prompt: str, response: str, chat_id: str,
+    existing_summary: Optional[ChatSummary],
+    store: Any,
+) -> None:
+    """Fire-and-forget memory extraction. Runs via asyncio.create_task."""
+    try:
+        updates = await extract_updates(prompt, response)
+        if updates.get("none"):
+            return
+        profile = await store.get_profile(user_id)
+        if profile is None:
+            profile = UserProfile(user_id=user_id)
+        profile = apply_updates(profile, updates)
+        await store.save_profile(user_id, profile)
+        logger.debug("Memory updated for user %s", user_id[:8])
+    except Exception:
+        logger.warning("Memory extraction failed for user %s", user_id[:8], exc_info=True)
+
+
+async def _summarize_history(
+    chat_id: str, history: list, existing_summary: Optional[ChatSummary],
+    store: Any,
+) -> None:
+    """Summarize accumulated conversation turns and persist."""
+    try:
+        turns = [
+            {"role": m.role, "text": m.parts[0].text if m.parts else ""}
+            for m in history
+        ]
+        new_summary_text = await summarize_conversation_turns(turns)
+        if existing_summary:
+            merged = await merge_summaries(existing_summary.content, new_summary_text)
+        else:
+            merged = new_summary_text
+        summary = ChatSummary(chat_id=chat_id, content=merged, turn_count=len(history))
+        await store.save_chat_summary(chat_id, summary)
+    except Exception:
+        logger.warning("History summarization failed for %s", chat_id[:8], exc_info=True)
+
+
 @app.post("/chat/stream")
 async def chat_stream(request: ChatRequest, http_request: Request):
     """Streaming chat endpoint using Server-Sent Events (SSE)."""
@@ -847,9 +942,8 @@ async def event_generator():
         )
 
     except Exception as e:
-        error_msg = f"❌ Streaming Chat API Error: {str(e)}"
-        logger.error(error_msg)
-        raise HTTPException(status_code=500, detail=error_msg)
+        logger.error("Streaming Chat API Error", exc_info=True)
+        raise HTTPException(status_code=500, detail="Internal server error") from e
 
 
 @app.delete("/chat/{chat_id}")
@@ -861,9 +955,8 @@ async def delete_chat(chat_id: str):
             return {"message": "Chat session deleted successfully"}
         return {"message": "Chat session not found"}
     except Exception as e:
-        error_msg = f"❌ Error deleting chat: {str(e)}"
-        logger.error(error_msg)
-        raise HTTPException(status_code=500, detail=error_msg)
+        logger.error("Error deleting chat", exc_info=True)
+        raise HTTPException(status_code=500, detail="Internal server error") from e
 
 
 @app.get("/ping")
@@ -872,6 +965,33 @@ async def ping():
     return {"status": "ok"}
 
 
+@app.get("/memory/{user_id}")
+async def get_memory(user_id: str):
+    """Retrieve the stored user profile for transparency.
+
+    TODO(#9): bind to authenticated principal — anyone who knows a user_id
+    can currently read another user's memory.
+    """
+    profile = await memory_store.get_profile(user_id)
+    if profile is None:
+        raise HTTPException(status_code=404, detail="Memory not found")
+    return profile.model_dump()
+
+
+@app.delete("/memory/{user_id}")
+async def delete_memory(user_id: str):
+    """Completely erase the stored user profile.
+
+    TODO(#9): bind to authenticated principal — anyone who knows a user_id
+    can currently erase another user's memory.
+    """
+    existed = await memory_store.delete_profile(user_id)
+    if existed:
+        logger.info("Deleted memory for user %s", user_id[:8])
+        return {"message": "Memory deleted successfully"}
+    return {"message": "Memory not found"}
+
+
 @app.get("/cache/stats")
 async def cache_stats():
     return semantic_cache.get_stats()

memory/__init__.py

@@ -0,0 +1,88 @@
+"""Per-user long-term memory and conversation summarization.
+
+See README.md for usage; see tests/ for offline-verifiable contracts.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Optional
+
+from memory.models import ChatSummary, UserProfile
+from memory.store import (
+    InMemoryMemoryStore,
+    MemoryStore,
+    RedisMemoryStore,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def create_memory_store() -> MemoryStore:
+    """Factory: ``REDIS_URL`` set → ``RedisMemoryStore``, else in-memory.
+
+    When Redis is configured but fails at startup the error is surfaced
+    (logged + raised) — workers must not silently diverge on user memory.
+    """
+    url = os.getenv("REDIS_URL", "")
+    if url:
+        logger.info("MemoryStore using Redis at %s", url)
+        return RedisMemoryStore(url)
+    logger.info("MemoryStore using in-memory dict (local development)")
+    return InMemoryMemoryStore()
+
+
+def render_user_context(
+    profile: Optional[UserProfile],
+    summary: Optional[ChatSummary],
+) -> str:
+    """Render profile and chat summary as a delimited DATA block.
+
+    Returns an empty string when neither has content so anonymous traffic
+    is completely unaffected.
+    """
+    parts: list[str] = []
+
+    if profile is not None and (
+        profile.knowledge_level
+        or profile.madhhab
+        or profile.preferred_language
+        or profile.topics_studied
+        or profile.remembered_facts
+    ):
+        lines = ["--- Known about this student ---"]
+        if profile.knowledge_level:
+            lines.append(f"Knowledge level: {profile.knowledge_level}")
+        if profile.madhhab:
+            lines.append(f"Madhhab: {profile.madhhab}")
+        if profile.preferred_language:
+            lines.append(f"Preferred language: {profile.preferred_language}")
+        if profile.topics_studied:
+            topics_str = ", ".join(
+                f"{t.topic}" for t in profile.topics_studied[-10:]
+            )
+            lines.append(f"Topics studied: {topics_str}")
+        if profile.remembered_facts:
+            for fact in profile.remembered_facts[-5:]:
+                lines.append(f"- {fact.fact}")
+        parts.append("\n".join(lines))
+
+    if summary is not None and summary.content:
+        parts.append(f"--- Conversation summary ---\n{summary.content}")
+
+    if not parts:
+        return ""
+
+    return "\n\n".join(parts) + "\n---------------------------------\n"
+
+
+__all__ = [
+    "ChatSummary",
+    "InMemoryMemoryStore",
+    "MemoryStore",
+    "RedisMemoryStore",
+    "UserProfile",
+    "create_memory_store",
+    "render_user_context",
+]