diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6db47ce..db216ca 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -8,10 +8,17 @@ on:
jobs:
test:
- runs-on: ubuntu-latest
strategy:
+ fail-fast: false
matrix:
- python-version: ["3.10", "3.11"]
+ os: [ubuntu-latest]
+ python-version: ["3.10", "3.11", "3.12", "3.13"]
+ include:
+ - os: windows-latest
+ python-version: "3.12"
+ - os: macos-latest
+ python-version: "3.12"
+ runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
@@ -20,16 +27,16 @@ jobs:
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
+ cache: pip
- - name: Cache pip
+ - name: Cache embedding model
uses: actions/cache@v4
with:
- path: ~/.cache/pip
- key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
- restore-keys: ${{ runner.os }}-pip-
+ path: ~/.cache/huggingface
+ key: hf-all-MiniLM-L6-v2-${{ runner.os }}
- - name: Install dependencies
- run: pip install --upgrade pip && pip install -r requirements.txt
+ - name: Install package
+ run: pip install -e ".[server,dev]"
- name: Import smoke test
env:
@@ -37,8 +44,25 @@ jobs:
USE_TF: "0"
run: python tests/test_imports.py
- - name: Pipeline integration tests
+ - name: Test suite
env:
TRANSFORMERS_NO_TF: "1"
USE_TF: "0"
- run: python tests/test_pipeline.py
+ run: pytest tests -q
+
+ build:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+ - name: Build sdist and wheel
+ run: |
+ pip install build twine
+ python -m build
+ twine check dist/*
+ - name: Install from wheel and run CLI
+ run: |
+ pip install dist/*.whl
+ memorylens --help
diff --git a/.gitignore b/.gitignore
index df0943d..af478f8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,3 +22,8 @@ results.json
experiment_logs/memorylens.db
*.log
.streamlit/secrets.toml
+benchmark_v04.json
+demo_venv/
+benchmark_v04_200.json
+*.db
+experiment_logs/runs_summary*.csv
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0d6e699..1a9309d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,19 +5,58 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
---
-## [Unreleased]
+## [0.4.0] — 2026-07-04
### Added
-
-- SQLite persistent storage (`utils/storage.py`) — queryable database replacing flat JSON/CSV logs
-- Migration script (`utils/migrate_legacy_logs.py`) — one-shot import of existing JSON logs into SQLite
-- `Storage.compare_runs()` — cross-run recall comparison API
-- `log_run()` now writes to SQLite alongside existing JSON/CSV output (backward compatible)
-- `list_runs()` queries SQLite first, falls back to filesystem scan
+- **PyPI-ready packaging** — all code now lives under a single `memorylens` package
+ (`memorylens.memory`, `memorylens.simulator`, `memorylens.evaluation`, `memorylens.utils`).
+ Fixed an invalid `build-backend` that made the sdist unbuildable, added the
+ `memorylens` console command (`memorylens.cli:main`), and split dependencies into a
+ lean core plus `[dashboard]`, `[server]`, `[faiss]`, `[groq]`, `[openai]`,
+ `[anthropic]`, `[all]`, `[dev]` extras. `python main.py` still works.
+- **GraphMemory** (`graph`) — NetworkX knowledge-graph backend; fact updates replace
+ edges in place so stale values cannot survive (#20)
+- **FAISSMemory** (`faiss`) — FAISS `IndexFlatIP` vector backend as an optional extra (#14)
+- **contradiction_score** — flags contexts that surface both the old and new value of an
+ updated fact; wired into checkpoints, CSV logs, and the dashboard (#21)
+- **Scenario framework** — `Scenario` dataclass + registry (#22) with three domain
+ scenarios: `edtech` (#18), `support` (#23), `medical` (#24); `--scenario` and
+ `--list-scenarios` CLI flags
+- **FastAPI server** — `uvicorn memorylens.api:app`; job-based POST `/v1/benchmarks`,
+ GET `/v1/backends`, `/v1/scenarios`, `/health` (#25)
+- **Dashboard Run History tab** — overlay Recall@T curves from past `experiment_logs/`
+ runs and compare final metrics side-by-side (#27)
+- **SQLite persistent storage** (`memorylens/utils/storage.py`) — queryable database
+ alongside the JSON/CSV logs; `log_run()` writes to it, `list_runs()` queries it
+ first, `Storage.compare_runs()` compares recall across runs, and
+ `python -m memorylens.utils.migrate_legacy_logs` imports legacy JSON logs
+ (#26, contributed by @Sugaria0427)
+- 23 new integration tests (45 total): GraphMemory, contradiction_score, scenarios,
+ FAISS, API lifecycle, SQLite storage, and cascading regressions (#31)
+- CI matrix expanded to Python 3.10–3.13 on Linux plus Windows and macOS, with a
+ package build + `twine check` + wheel-install job (#30)
### Fixed
+- **Cascading cold-tier recall regression** — the newest-first cold-summary merge
+ introduced with the drift fix truncated away the oldest fact summaries, collapsing
+ cascading recall at T=100 from ~75% to ~8%. Merging is oldest-first again (stale
+ values are already rewritten in place by the update patcher) and empty "No key
+ facts." summaries are no longer appended to the cold tier. Regression tests added.
+- Experiment CSV logger crashed on every run since the `has_llm_eval` flag was added
+ (a bool was indexed as a dict); it now skips non-backend keys and rotates the CSV
+ when the metric schema changes
+- Benchmark results in the README were stale and did not reproduce; all tables are
+ regenerated from the current code
+
+### Removed
+- Dead references to an unpublished research paper (`paper/memorylens_paper.md` never
+ existed in the repository); unverifiable "the only framework" marketing claims;
+ fabricated ₹-cost projections in the dashboard (replaced with a clearly labelled
+ illustrative $-projection)
+
+---
-- `_append_csv_summary` now properly filters `has_llm_eval` from display_data (pre-existing bug where `has_llm_eval: True` caused `TypeError` when iterating display_data)
+## [0.3.0] — 2026-05-24
### Documentation — Metric Accuracy Clarifications
@@ -53,8 +92,7 @@ Three research-quality gaps identified and documented across README, docs/, and
- Shows realistic 85–87% recall at T=100 vs ideal RAG's 100% — contrast is the key finding
- Registered as `rag_chunked` backend in benchmark runner and CLI
-**Fix 4 — Research paper**
-- `paper/memorylens_paper.md`: 6-section academic paper with proper citations (Ebbinghaus 1885, MemGPT, RAGAS, Jost 1897, Atkinson & Shiffrin 1968), ablation tables, multi-seed results tables, and related work comparison against RAGAS, TruLens, DeepEval, MemGPT, A-MEM
+**Fix 4 — Research paper** *(never merged into the repository; stale references to it were removed in 0.4.0)*
**Tests**: 10 new tests covering decay functions, ChunkedRAGMemory, stats aggregation, and persona pool structure (24 total, all passing)
diff --git a/CITATION.cff b/CITATION.cff
index d5dcbc0..aeb8f60 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -5,21 +5,22 @@ title: "MemoryLens: A Temporal Decay Benchmark for LLM Memory Architectures"
abstract: >
MemoryLens is an open-source evaluation framework for measuring LLM memory decay
— how AI memory systems forget personal facts across long conversations. It implements
- five memory architectures (Naive, Ideal RAG, Chunked RAG, Cascading Temporal, SummaryMemory),
- five evaluation metrics (Recall@T, Precision@K, Temporal Drift, Memory Noise Ratio,
- Cascade Efficiency), Ebbinghaus-grounded temporal decay with ablation, multi-seed
- statistical validation across five diverse personas, and a dual evaluation pipeline
- (content-based + LLM answer+judge) supporting five provider backends.
+ eight memory architectures (Naive, RAG, Chunked RAG, Cascading Temporal, Summary,
+ Entity, Graph, FAISS), six evaluation metrics (Recall@T, Precision@K, Temporal Drift,
+ Contradiction, Memory Noise Ratio, Cascade Efficiency), Ebbinghaus-grounded temporal
+ decay with ablation, forgetting-curve fitting, multi-seed statistical validation,
+ four domain scenarios, and a dual evaluation pipeline (content-based + LLM
+ answer+judge) supporting five provider backends.
authors:
- - family-names: Srivastava
+ - family-names: Daftary
given-names: Neal
alias: Neal006
orcid: ""
repository-code: "https://github.com/Neal006/memorylens"
url: "https://github.com/Neal006/memorylens"
license: MIT
-version: 0.3.0
-date-released: "2026-05-22"
+version: 0.4.0
+date-released: "2026-07-04"
keywords:
- LLM memory
- memory decay
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 11cc613..ea47b60 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -44,8 +44,8 @@ python -m venv .venv
source .venv/bin/activate # Linux / macOS
.venv\Scripts\activate # Windows
-# 3. Install in editable mode with dev dependencies:
-pip install -e ".[dev]"
+# 3. Install in editable mode with dev + server dependencies:
+pip install -e ".[server,dev]"
# 4. Copy the environment file (API key is optional):
cp .env.example .env
@@ -73,8 +73,9 @@ Simulator → Memory Backend → Evaluator → Dashboard
Each layer is independently extensible. You can add a backend without touching the evaluator, and add a metric without touching the dashboard.
-**Current backends:** `naive` · `rag` · `rag_chunked` · `cascading` · `summary`
-**Current metrics:** Recall@T · Precision@K · Temporal Drift · Memory Noise Ratio · Cascade Efficiency
+**Current backends:** `naive` · `rag` · `rag_chunked` · `cascading` · `summary` · `entity` · `graph` · `faiss`
+**Current metrics:** Recall@T · Precision@K · Temporal Drift · Contradiction · Memory Noise Ratio · Cascade Efficiency
+**Current scenarios:** `default` · `edtech` · `support` · `medical`
**LLM eval providers:** Groq · OpenAI · Anthropic · OpenRouter · Ollama
Set `TRANSFORMERS_NO_TF=1` and `USE_TF=0` if you have TensorFlow installed alongside PyTorch.
@@ -84,33 +85,41 @@ Set `TRANSFORMERS_NO_TF=1` and `USE_TF=0` if you have TensorFlow installed along
## Project layout
```
-memorylens/
-├── memory/ Memory backend implementations — add new backends here
-│ ├── base.py Abstract BaseMemory interface (3 methods every backend must implement)
-│ ├── naive.py Naive full-history backend (simplest example)
-│ ├── rag.py Semantic retrieval backend (sentence-transformers)
-│ ├── cascading.py Three-tier hot/warm/cold with Ebbinghaus temporal decay
-│ ├── entity.py Structured key-value entity extraction (great reference for new backends)
-│ └── decay.py Temporal decay functions (ebbinghaus, exponential, linear)
+memorylens/ The installable package (pip install memorylens)
+├── memory/ Memory backend implementations — add new backends here
+│ ├── base.py Abstract BaseMemory interface (3 methods every backend must implement)
+│ ├── naive.py Naive full-history backend (simplest example)
+│ ├── rag.py Semantic retrieval backend (sentence-transformers)
+│ ├── rag_chunked.py Chunked + bounded-index RAG (production-realistic)
+│ ├── cascading.py Three-tier hot/warm/cold with Ebbinghaus temporal decay
+│ ├── summary.py Rolling-summary compression backend
+│ ├── entity.py Structured key-value entity extraction (great reference for new backends)
+│ ├── graph.py NetworkX knowledge-graph backend
+│ ├── vector_faiss.py FAISS vector index backend (optional dep: memorylens[faiss])
+│ └── decay.py Temporal decay functions (ebbinghaus, exponential, linear)
├── evaluation/
-│ ├── metrics.py All benchmark metrics — add new metrics here
-│ ├── benchmark.py Benchmark orchestrator — registers backends, runs eval loop
-│ ├── stats.py Multi-seed aggregation and forgetting-curve fitting
-│ └── logger.py Experiment logging (JSON + CSV)
+│ ├── metrics.py All benchmark metrics — add new metrics here
+│ ├── benchmark.py Benchmark orchestrator — registers backends, runs eval loop
+│ ├── stats.py Multi-seed aggregation and forgetting-curve fitting
+│ └── logger.py Experiment logging (JSON + CSV)
├── simulator/
-│ ├── facts.py Fact dataclass and BENCHMARK_FACTS
-│ ├── conversation.py generate_conversation() — builds the simulated chat
-│ ├── personas.py 5 diverse personas for multi-seed runs
-│ └── scenarios/ Domain-specific scenarios (edtech, customer_support, medical…)
+│ ├── facts.py Fact dataclass and BENCHMARK_FACTS
+│ ├── conversation.py generate_conversation() — builds the simulated chat
+│ ├── personas.py 5 diverse personas for multi-seed runs
+│ └── scenarios/ Scenario registry (default, edtech, support, medical)
├── utils/
-│ ├── embeddings.py Local sentence-transformer embeddings (no API key needed)
-│ └── providers.py LLM provider abstraction (Groq, OpenAI, Anthropic, Ollama…)
-├── tests/ Integration tests — run with: pytest tests/ -v
-├── dashboard.py Streamlit visualisation dashboard
-├── main.py CLI entry point
-├── quick_demo.py Zero-API-key demo
-├── api/ FastAPI REST server (planned — see Issue #25)
-└── docs/ Guides and comparison docs
+│ ├── embeddings.py Local sentence-transformer embeddings (no API key needed)
+│ ├── providers.py LLM provider abstraction (Groq, OpenAI, Anthropic, Ollama…)
+│ ├── storage.py SQLite store for benchmark runs (experiment_logs/memorylens.db)
+│ └── migrate_legacy_logs.py One-shot legacy JSON → SQLite import
+├── api.py FastAPI REST server (optional dep: memorylens[server])
+└── cli.py CLI entry point (`memorylens` command)
+
+tests/ Integration tests — run with: pytest tests/ -v
+dashboard.py Streamlit visualisation dashboard
+main.py Backward-compatible wrapper around memorylens.cli
+quick_demo.py Zero-API-key demo
+docs/ Guides and comparison docs
```
---
@@ -121,7 +130,7 @@ The most impactful contribution type. Full guide with a worked EntityMemory exam
**Quick version — 4 steps:**
-**Step 1 — Create `memory/your_backend.py`:**
+**Step 1 — Create `memorylens/memory/your_backend.py`:**
```python
from typing import List, Dict
@@ -145,10 +154,10 @@ class YourMemory(BaseMemory):
pass # clear all state
```
-**Step 2 — Register in `evaluation/benchmark.py`:**
+**Step 2 — Register in `memorylens/evaluation/benchmark.py`:**
```python
-from memory.your_backend import YourMemory
+from memorylens.memory.your_backend import YourMemory
def _make_memory(name: str, decay: str = "ebbinghaus") -> BaseMemory:
if name == "your_backend":
@@ -162,7 +171,7 @@ Add `"your_backend"` to `VALID_BACKENDS`.
```python
def test_your_backend_recall_early():
- from memory.your_backend import YourMemory
+ from memorylens.memory.your_backend import YourMemory
mem = YourMemory()
_populate(mem, BENCHMARK_FACTS, 15)
active = [f for f in BENCHMARK_FACTS if f.injected_at < 15]
@@ -184,7 +193,7 @@ Open a PR with the three files changed. A maintainer will review within 48 hours
## How to add a new metric
-All metrics live in `evaluation/metrics.py`. Each is a plain function — no classes.
+All metrics live in `memorylens/evaluation/metrics.py`. Each is a plain function — no classes.
```python
def your_metric(memory: BaseMemory, facts: List[Fact], current_turn: int) -> float:
@@ -196,35 +205,41 @@ def your_metric(memory: BaseMemory, facts: List[Fact], current_turn: int) -> flo
return score
```
-Wire it into the `CheckpointResult` dataclass in `evaluation/benchmark.py` and add a chart in `dashboard.py`.
+Wire it into the `CheckpointResult` dataclass in `memorylens/evaluation/benchmark.py` and add a chart in `dashboard.py`. `contradiction_score` is the most recent worked example — trace it through metrics.py → benchmark.py → dashboard.py.
---
## How to add a new domain scenario
-Copy `simulator/scenarios/edtech.py` as a starting template:
+Copy `memorylens/simulator/scenarios/medical.py` as a starting template:
```python
-# simulator/scenarios/your_scenario.py
-from simulator.facts import Fact
-
-YOUR_FACTS = [
- Fact("name", "Alice Chen", injected_at=0),
- Fact("role", "developer", injected_at=2),
- Fact("city", "Singapore", injected_at=4, updated_at=40, updated_value="Sydney"),
- # 8 facts total; at least 2 should have updated_at set
+# memorylens/simulator/scenarios/your_scenario.py
+from memorylens.simulator.facts import Fact
+from memorylens.simulator.scenarios.base import Scenario
+
+YOUR_PERSONA_POOL = [
+ [
+ Fact("name", "Alice Chen", injected_at=0),
+ Fact("role", "developer", injected_at=2),
+ Fact("city", "Singapore", injected_at=4, updated_at=40, updated_value="Sydney"),
+ # 8 facts total; at least 2 should have updated_at set.
+ # Old and new values must not be substrings of each other.
+ ],
+ # ... 2+ more personas with the same fact keys
]
-YOUR_PERSONA_POOL = [YOUR_FACTS, ...] # 5 different persona fact-lists
-YOUR_FILLER_TURNS = [ # 20+ domain-specific questions
- "Can you help me with...",
- ...
-]
-```
+YOUR_FILLER_TURNS = ["Can you help me with...", ...] # 20 domain questions
-Then add a `--scenario your_scenario` case to `main.py` (copy the existing `edtech` block exactly).
+YOUR_SCENARIO = Scenario(
+ name="your_scenario",
+ description="One-line description shown by --list-scenarios.",
+ persona_pool=YOUR_PERSONA_POOL,
+ filler_turns=YOUR_FILLER_TURNS,
+)
+```
-**Open scenarios:** #23 (CustomerSupport), #24 (Medical) — good first contributions!
+Then register it in the `SCENARIOS` dict in `memorylens/simulator/scenarios/__init__.py` — the CLI (`--scenario your_scenario`), the API, and the tests pick it up automatically.
---
@@ -248,7 +263,7 @@ python main.py --backends naive rag cascading --turns 50
python main.py --seeds 5
```
-CI runs `pytest tests/` on Python 3.10 and 3.11 on every push. All tests must pass without an API key.
+CI runs the suite on Python 3.10–3.13 (Linux) plus Windows and macOS on every push, and builds + validates the PyPI package. All tests must pass without an API key.
---
@@ -281,7 +296,7 @@ refactor: extract _extract_entity() helper from EntityMemory
- [ ] Docstrings added on all new public functions/classes
- [ ] Type hints used on all new function signatures
- [ ] If adding a backend: registered in `VALID_BACKENDS` and `_make_memory()`
-- [ ] If adding a scenario: `--scenario` flag added to `main.py`
+- [ ] If adding a scenario: registered in the `SCENARIOS` dict in `memorylens/simulator/scenarios/__init__.py`
- [ ] README updated if new CLI flags or user-facing features were added
- [ ] No API key required to run any new tests
@@ -304,7 +319,7 @@ refactor: extract _extract_entity() helper from EntityMemory
- **Stuck on an issue?** Comment on it — maintainers respond promptly
- **General questions?** Open a [Discussion](https://github.com/Neal006/memorylens/discussions)
-- **Best reference for new backends:** `memory/entity.py` — the shortest, cleanest example
+- **Best reference for new backends:** `memorylens/memory/entity.py` — the shortest, cleanest example
- **CI failing?** Run `pytest tests/ -v` locally first; the error message is usually self-explanatory
Welcome aboard — we're glad you're here!
diff --git a/README.md b/README.md
index 7bc2349..56c638a 100644
--- a/README.md
+++ b/README.md
@@ -2,421 +2,302 @@
# 🔭 MemoryLens
-### The Open-Source Benchmark for LLM Memory Decay
+### Measure how your AI's memory forgets
-**The only evaluation framework that measures how AI memory systems forget — across architectures, over time, with statistical rigor.**
+**An open-source benchmark for LLM memory decay — 8 memory architectures, 6 metrics, 4 domain scenarios, statistical rigor, zero API keys required.**
[](https://github.com/Neal006/memorylens/actions/workflows/ci.yml)
-[](https://www.python.org/)
+[](https://pypi.org/project/memorylens/)
+[](https://www.python.org/)
[](LICENSE)
[](CONTRIBUTING.md)
-[](https://github.com/Neal006/memorylens/stargazers)
-[](https://github.com/Neal006/memorylens/network/members)
-[**Quick Start**](#quick-start) · [**Results**](#benchmark-results) · [**How It Works**](#how-it-works) · [**vs Other Tools**](#how-memorylens-compares)
+[**Install**](#install) · [**Quick Start**](#quick-start) · [**Results**](#benchmark-results) · [**How It Works**](#how-it-works) · [**Contributing**](#contributing)
---
-## The Problem No One Is Measuring
+## The Problem
-Every LLM application that runs multi-turn conversations has a memory problem. Developers pick a memory strategy — usually "dump everything in the context and hope" — and never measure what actually gets remembered.
+Every LLM application that runs multi-turn conversations has a memory strategy — usually "keep everything in context and hope." Almost nobody measures what that strategy actually remembers after 50, 100, or 200 turns.
-**MemoryLens is the benchmark that measures LLM memory decay.**
+MemoryLens answers three questions:
-It answers three questions no other tool asks:
+- **How much does an AI still recall** after N conversation turns — and at what token cost?
+- **Which memory architecture** (full history, RAG, tiered compression, entity store, knowledge graph…) retains facts most efficiently?
+- **When a user updates a fact** ("I moved to Mumbai"), does the memory surface the new value, the stale one — or contradict itself with both?
-- **How much does an AI actually remember** after 50 conversation turns? After 100?
-- **Which memory architecture retains facts most efficiently** at a given token budget?
-- **When a user updates a fact** ("I moved to Mumbai"), does the AI still give the old answer?
+Every core metric is content-based and deterministic: no API key, fully reproducible. Add any LLM key and a two-stage answer+judge pipeline measures what the model *actually* answers.
---
-## Key Results (multi-seed, n=5 personas, mean ± std)
+## Install
-Run `python main.py` and get statistically valid results like these — **no API key needed:**
-
-| Backend | Recall @ T=100 | Tokens/Query | Cascade Efficiency |
-|---------|:--------------:|:------------:|:-----------------:|
-| Naive (full history eviction) | 62.5 ± 0.0% | 1,189 | 1.0× baseline |
-| Ideal RAG (unbounded, whole-msg) | 100.0 ± 0.0% | 45 | — |
-| **Chunked RAG** (production-realistic) | **85.0 ± 3.8%** | **38** | — |
-| **Cascading Temporal** (Ebbinghaus decay) | **87.5 ± 0.0%** | **218** | **5.67×** |
-| SummaryMemory (extractive mode) | 100.0 ± 0.0% | 318 | — |
-
-> **Why ± 0.0% std for some backends?** Naive and Cascading make eviction and decay decisions based on *position and elapsed time*, not on the content of each persona's values. Since all 5 personas share the same fact injection timing (T=0,1,2,3,4,5,7,9) and update timing (T=40,60), these backends produce identical results across personas — real variance requires varying injection timing, which is a [known v0.4 improvement](#roadmap). Chunked RAG shows real variance (±3.8%) because cosine similarity scoring depends on the actual text of each persona's facts.
+```bash
+pip install memorylens
+```
-> **Why does SummaryMemory show 100% recall?** In zero-API-key mode, SummaryMemory uses *extractive* compression: it keeps only messages containing personal-fact keywords (`name`, `city`, `age`, etc.) verbatim. This is effectively selective full history — fact values are never paraphrased or lost, so substring match recall is always 100%. Set any LLM provider key and compression becomes abstractive; paraphrased facts may not match the substring check, which is the honest production measurement. See [How It Works](#how-it-works) for the full tradeoff.
+| Extra | Installs | For |
+|-------|----------|-----|
+| `memorylens[dashboard]` | streamlit, plotly, pandas | Interactive dashboard |
+| `memorylens[server]` | fastapi, uvicorn | REST API |
+| `memorylens[faiss]` | faiss-cpu | FAISS vector backend |
+| `memorylens[groq]` / `[openai]` / `[anthropic]` | provider SDK | LLM evaluation mode |
+| `memorylens[all]` | everything above | — |
-> **Chunked RAG vs Ideal RAG** shows the gap between a theoretical upper bound and a production-realistic retrieval system. The 15pp difference is the cost of chunking + bounded index eviction. The **Cascading Temporal** backend delivers **5.67× more recall per token** than naive truncation using an Ebbinghaus-grounded forgetting curve.
+Python 3.10–3.13 · Linux, macOS, Windows (all tested in CI).
---
## Quick Start
-### Zero API key — runs in under 60 seconds
+### CLI — results in under a minute, no API key
```bash
-git clone https://github.com/Neal006/memorylens.git
-cd memorylens
-pip install -r requirements.txt
-python main.py
+memorylens # 100-turn benchmark, 3 backends
+memorylens --seeds 5 # multi-seed: mean ± std across 5 personas
+memorylens --scenario medical # domain scenarios: default | edtech | support | medical
+memorylens --backends naive rag_chunked cascading graph
+memorylens --seeds 5 --fit-curves # fit Ebbinghaus + exponential forgetting curves
+memorylens --llm --provider groq # real answer+judge LLM evaluation
```
-### Multi-seed benchmark (statistically valid, mean ± std)
+### Python API
-```bash
-python main.py --seeds 5
+```python
+from memorylens import run_benchmark, results_to_display_dict
+
+raw = run_benchmark(total_turns=100, backends=["naive", "rag", "cascading"])
+results = results_to_display_dict(raw)
+print(results["cascading"]["recall"]) # recall at each checkpoint
```
-### Live LLM evaluation (answer + judge pipeline)
+### Dashboard and REST API
```bash
-cp .env.example .env
-# Add any one key: GROQ_API_KEY, OPENAI_API_KEY, ANTHROPIC_API_KEY, etc.
-python main.py --llm --provider groq
+pip install "memorylens[dashboard]"
+streamlit run dashboard.py # decay curves, run history, cost projection
+
+pip install "memorylens[server]"
+uvicorn memorylens.api:app # POST /v1/benchmarks → job id → poll results
```
-### Decay formula ablation (Ebbinghaus vs exponential vs linear)
+---
-```bash
-python main.py --decay ebbinghaus # default — Ebbinghaus (1885)
-python main.py --decay exponential # Jost (1897)
-python main.py --decay linear # Wickelgren (1972)
-```
+## Benchmark Results
-### Realistic chunked RAG vs ideal RAG
+All numbers below are reproduced from this exact code (v0.4.0) with:
+`memorylens --seeds 5 --backends naive rag rag_chunked cascading summary entity graph faiss`
-```bash
-python main.py --backends naive rag rag_chunked cascading
-```
+### Recall@T — 100 turns (mean ± std, n=5 personas)
-### Interactive dashboard
+| Backend | T=10 | T=50 | T=100 | Tokens/query @ T=100 |
+|---------|:----:|:----:|:-----:|:--------------------:|
+| naive (1,200-token budget) | 100% | 100% | **35.0 ± 5.6%** | 1,193 |
+| rag | 100% | 100% | 100% | 67 |
+| rag_chunked | 100% | 100% | 100% | 56 |
+| cascading | 100% | 100% | 100% | 326 |
+| summary *(extractive)* | 100% | 100% | 100% | 356 |
+| entity | 100% | 100% | 100% | 63 |
+| graph | 100% | 100% | 100% | 62 |
+| faiss | 100% | 100% | 100% | 67 |
-```bash
-streamlit run dashboard.py
-# Select a provider in the sidebar for real LLM recall vs content recall gap charts
-```
+**Read this before quoting numbers.** At 100 turns with 8 templated facts, every retrieval- and extraction-based backend saturates — the differentiation is *token cost at equal recall* (rag_chunked delivers the same recall as naive at ~1/21 the tokens) and naive's collapse once its context budget evicts early turns. Two structural caveats:
+
+1. **entity / graph exploit the fact templates.** Facts are injected as `"My X is Y"`, which the regex extractors match by construction. Their 100% shows structured stores never lose what they capture — it says nothing about free-form conversation. Harder paraphrased injections are on the [roadmap](ROADMAP.md).
+2. **summary is extractive in zero-key mode** — it keeps fact-bearing lines verbatim, so substring recall is guaranteed. With an LLM key, compression becomes abstractive and honest decay appears.
+
+### Stress test — 200 turns
+
+At 200 turns the bounded systems saturate and real decay separates the field
+(`memorylens --turns 200 --checkpoints 10 50 100 150 200 --seeds 5 --backends ...`):
+
+| Backend | T=100 | T=150 | T=200 | Tokens/query @ T=200 |
+|---------|:-----:|:-----:|:-----:|:--------------------:|
+| naive | 35.0 ± 5.6% | 7.5 ± 6.9% | **7.5 ± 6.9%** | 1,189 |
+| rag_chunked (bounded index) | 100% | 17.5 ± 6.9% | **5.0 ± 6.9%** | 70 |
+| cascading | 100% | 100% | **100%** | 326 |
+| rag · summary · entity · graph · faiss | 100% | 100% | 100% | 62–382 |
+
+The two production-realistic constraints fail in opposite ways: naive's fixed token budget evicts old turns wholesale, while rag_chunked's bounded FIFO vector index (200 chunks) silently drops the earliest facts even though each query costs only 70 tokens. Cascading survives because its cold tier compresses fact-bearing lines instead of discarding them.
+
+### Temporal drift and contradiction after fact updates
+
+Two facts update mid-conversation (city at T=40, age at T=60). At T=100:
+
+| Backend | Drift (stale-only retrieval) | Contradiction (old + new surfaced) |
+|---------|:---------------------------:|:----------------------------------:|
+| naive | 0.0 | 1.0 — full history keeps both values |
+| cascading | 0.0 | 0.0 — cold summaries patched in place |
+| entity / graph | 0.0 | 0.0 — updates overwrite in place |
+
+> Drift is a retrieval-layer proxy (worst-case bound). For behavioral measurement — does the model *answer* with the stale value — run `memorylens --llm`.
---
## How It Works
-MemoryLens has three layers:
-
```
┌────────────────────────────────────────────────────────────────────────┐
│ LAYER 1 — SIMULATOR │
-│ Injects personal facts at known turns, fires filler queries in between │
-│ Facts can be updated mid-conversation to test temporal drift │
+│ Injects facts at known turns, fires domain filler queries in between │
+│ Facts can be updated mid-conversation to test drift + contradiction │
│ │
│ T=0 "My name is Arjun Sharma." │
│ T=1 "My city is Bangalore." │
-│ T=40 "My city has changed to Mumbai." ← update event │
-│ T=2–99: generic filler questions (noise) │
+│ T=40 "My city has changed to Mumbai." ← update event │
+│ Scenarios: default (tech Q&A) · edtech · support · medical │
└──────────────────────────────┬─────────────────────────────────────────┘
- │
▼
┌────────────────────────────────────────────────────────────────────────┐
-│ LAYER 2 — MEMORY BACKENDS (5 implementations) │
+│ LAYER 2 — MEMORY BACKENDS (8 implementations) │
│ │
-│ naive Full history, evict oldest at 1,200-token budget │
-│ rag Embed every message, retrieve top-K by cosine similarity │
-│ rag_chunked Chunked + bounded index (production-realistic) │
-│ cascading Hot/Warm/Cold tiers with Ebbinghaus temporal decay │
-│ summary Rolling LLM-generated (or extractive) compression │
+│ naive Full history, evict oldest at a token budget │
+│ rag Embed every message, retrieve top-K (upper bound) │
+│ rag_chunked Chunked + bounded FIFO index (production-realistic) │
+│ cascading Hot/Warm/Cold tiers with Ebbinghaus temporal decay │
+│ summary Rolling compression (LLM or extractive) │
+│ entity Structured key-value fact store │
+│ graph NetworkX knowledge graph, in-place fact updates │
+│ faiss FAISS vector index (optional dependency) │
└──────────────────────────────┬─────────────────────────────────────────┘
- │
▼
┌────────────────────────────────────────────────────────────────────────┐
-│ LAYER 3 — EVALUATOR (5 metrics, dual mode) │
+│ LAYER 3 — EVALUATOR (6 metrics, dual mode) │
│ │
-│ Content mode (no API key): substring match on retrieved chunks │
+│ Content mode (no API key): deterministic substring checks │
│ LLM mode (any provider): answer+judge pipeline — did the LLM │
│ actually answer correctly? │
-│ Gap = content recall − LLM recall │
└────────────────────────────────────────────────────────────────────────┘
```
-### The 5 Evaluation Metrics
-
-| Metric | What It Measures | Formula |
-|--------|-----------------|---------|
-| **Recall@T** | Is the correct fact value in retrieved context at turn T? | `expected_value ∈ context` |
-| **Precision@K** | Of K retrieved chunks, how many contain a real fact? | `relevant_chunks / K` |
-| **Temporal Drift** | After an update, what fraction of retrieved context still contains the stale value? (retrieval-layer proxy — see note) | `old_hits / (old + new hits)` |
-| **Memory Noise Ratio** | What fraction of retrieved context is irrelevant? | `1 − relevant / total` |
-| **Cascade Efficiency** | Recall-per-token ratio vs naive baseline | `(cascading r/t) / (naive r/t)` |
+### The 6 Metrics
-All five metrics are **content-based and deterministic** — no LLM call, fully reproducible.
-
-> **Temporal Drift is a worst-case proxy.** It measures stale-data *contamination in the retrieval layer*, not whether the LLM actually answers with the stale value. If a backend retrieves both "Bangalore" and "Mumbai", drift = 0.5 — but an LLM given both values would likely pick the correct one. This means content drift *overestimates* the real problem. For behavioral measurement (does the LLM answer with the old or new value?), use LLM mode: `python main.py --llm`, which runs `llm_temporal_drift()` — a two-stage answer+judge pipeline.
+| Metric | What It Measures |
+|--------|-----------------|
+| **Recall@T** | Is the correct current fact value in retrieved context at turn T? |
+| **Precision@K** | Of K retrieved chunks, how many contain a real fact? |
+| **Temporal Drift** | After an update, what fraction of retrieval still carries the stale value? *(worst-case proxy)* |
+| **Contradiction** | Does the context surface both old *and* new values at once, forcing the LLM to arbitrate? |
+| **Memory Noise Ratio** | What fraction of retrieved context is irrelevant to any known fact? |
+| **Cascade Efficiency** | Recall-per-token vs the naive baseline |
### The 4 Temporal Decay Functions
-The Cascading backend's warm-tier scoring uses a pluggable forgetting curve:
+The cascading backend's warm-tier scoring uses a pluggable forgetting curve — compare them with `--decay`:
| Name | Formula | Reference |
|------|---------|-----------|
-| `ebbinghaus` *(default)* | `e^{-t / sqrt(1+t)}` | Ebbinghaus (1885) |
+| `ebbinghaus` *(default)* | `e^{-t / (S·√(1+t))}` | Ebbinghaus (1885) |
| `exponential` | `e^{-k·t/window}` | Jost (1897) |
| `linear` | `1 − t/window` | Wickelgren (1972) |
| `default` | `max(0.2, 1 − 0.6·t/w)` | Original heuristic |
-The Ebbinghaus curve produces the highest cascade efficiency (5.67×) because it decays slowly at first — preserving recently-injected facts — then asymptotically approaches zero for ancient context.
-
----
-
-## Benchmark Results
-
-### Recall@T decay (mean ± std, n=5 personas)
-
-| Backend | T=10 | T=25 | T=50 | T=75 | T=100 |
-|---------|:----:|:----:|:----:|:----:|:-----:|
-| Naive | 100±0% | 100±0% | 87.5±0% | 75±0% | 62.5±0% |
-| Ideal RAG | 100±0% | 100±0% | 100±0% | 100±0% | 100±0% |
-| Chunked RAG | 100±0% | 96±2% | 92±3% | 88±4% | 85±4% |
-| Cascading | 100±0% | 100±0% | 87.5±0% | 87.5±0% | 87.5±0% |
-| SummaryMemory *(extractive)* | 100±0% | 100±0% | 100±0% | 100±0% | 100±0% |
-
-*Std=0% for Naive, Ideal RAG, Cascading, and SummaryMemory because their behavior is determined by injection timing, not content values. Chunked RAG's variance comes from embedding similarity differences across persona values. See the callout above for the full explanation.*
-
-### Token cost per query @ T=100
-
-| Backend | Tokens/Query | Relative to Naive |
-|---------|:-----------:|:-----------------:|
-| Naive | 1,189 | 1.0× |
-| Ideal RAG | 45 | 0.038× |
-| Chunked RAG | 38 | 0.032× |
-| Cascading | 218 | 0.183× |
-| SummaryMemory | 318 | 0.268× |
-
-### Cascade Efficiency (recall/token vs naive, Ebbinghaus decay)
-
-| T=10 | T=25 | T=50 | T=75 | T=100 |
-|:----:|:----:|:----:|:----:|:-----:|
-| 1.16× | 1.96× | 2.30× | 3.03× | **5.67×** |
-
-### Decay formula ablation @ T=100
-
-| Decay function | Cascade Efficiency | Reference |
-|----------------|:-----------------:|-----------|
-| Ebbinghaus (default) | **5.67×** | Ebbinghaus (1885) |
-| Exponential | 5.12× | Jost (1897) |
-| Linear | 4.89× | Wickelgren (1972) |
-| Original heuristic | 5.45× | Ad-hoc |
-
----
-
-## How MemoryLens Compares
-
-> Every evaluation framework measures something. MemoryLens is the only one that measures **how memory degrades over conversation turns**.
-
-| Framework | What It Evaluates | Temporal Decay | Multi-Architecture | No-API Mode | Open Source |
-|-----------|------------------|:--------------:|:------------------:|:-----------:|:-----------:|
-| **MemoryLens** | Memory decay over turns | ✅ | ✅ (5 backends) | ✅ | ✅ |
-| [RAGAS](https://github.com/explodinggradients/ragas) | RAG quality (faithfulness, relevance) | ❌ | ❌ | ❌ | ✅ |
-| [TruLens](https://github.com/truera/trulens) | LLM app quality at a single point | ❌ | ❌ | ❌ | ✅ |
-| [DeepEval](https://github.com/confident-ai/deepeval) | LLM answer quality | ❌ | ❌ | Partial | ✅ |
-| [MemGPT](https://github.com/cpacker/MemGPT) | Memory *system* (not evaluator) | N/A | N/A | N/A | ✅ |
-| [LangChain ConversationBuffer](https://python.langchain.com/docs/modules/memory/) | Memory *implementation* | N/A | N/A | N/A | ✅ |
-
-**MemoryLens is the only tool that answers: "How much does my AI forget after N conversation turns?"**
+`--fit-curves` fits both Ebbinghaus and exponential models to your measured Recall@T series and reports half-life, stability, and R².
---
## LLM Provider Support
-MemoryLens works **without any API key** for all content-based metrics. Add any one key to unlock the real LLM evaluation pass:
+All content metrics work with **no API key**. Add any one key for the LLM answer+judge pass:
| Provider | Key | Default Model | Free Tier |
|----------|-----|---------------|-----------|
-| Groq | `GROQ_API_KEY` | llama-3.1-8b-instant | ✅ Yes |
+| Groq | `GROQ_API_KEY` | llama-3.1-8b-instant | ✅ |
| OpenAI | `OPENAI_API_KEY` | gpt-4o-mini | ❌ |
| Anthropic | `ANTHROPIC_API_KEY` | claude-haiku-4-5 | ❌ |
-| OpenRouter | `OPENROUTER_API_KEY` | llama-3.1-8b-instruct:free | ✅ Yes |
-| Ollama | *(none — local)* | llama3.2 | ✅ Always |
+| OpenRouter | `OPENROUTER_API_KEY` | llama-3.1-8b-instruct:free | ✅ |
+| Ollama | *(none — local)* | llama3.2 | ✅ |
```bash
-python main.py --list-providers # see what's available
-python main.py --llm # auto-detect and use
-python main.py --llm --provider groq # force a specific one
+memorylens --list-providers
+memorylens --llm # auto-detect
+memorylens --llm --provider groq # force one
```
---
-## Project Structure
+## How MemoryLens Compares
-```
-memorylens/
-│
-├── simulator/
-│ ├── facts.py # Fact definitions — the ground truth
-│ ├── conversation.py # Turn-by-turn event generator
-│ └── personas.py # 5 diverse personas for multi-seed validation
-│
-├── memory/ # Memory backend implementations
-│ ├── base.py # Abstract base — 3-method interface
-│ ├── naive.py # Naive: full history, evict oldest
-│ ├── rag.py # Ideal RAG: embed + retrieve (upper bound)
-│ ├── rag_chunked.py # Chunked RAG: bounded FIFO index (realistic)
-│ ├── cascading.py # Cascading Temporal: Hot/Warm/Cold tiers
-│ ├── summary.py # SummaryMemory: rolling LLM compression
-│ └── decay.py # 4 temporal decay functions (Ebbinghaus etc.)
-│
-├── evaluation/
-│ ├── metrics.py # 5 metric functions + LLM eval pipeline
-│ ├── benchmark.py # Benchmark runner + multi-seed aggregation
-│ ├── stats.py # Mean ± std + 95% confidence intervals
-│ ├── llm_judge.py # LLM-as-judge helper
-│ └── logger.py # Experiment logger → JSON + CSV
-│
-├── utils/
-│ ├── embeddings.py # sentence-transformers wrapper
-│ ├── providers.py # Unified LLM provider abstraction (5 backends)
-│ └── llm.py # Groq API wrapper (legacy)
-│
-├── paper/
-│ └── memorylens_paper.md # Full research paper with citations
-│
-├── tests/
-│ ├── test_imports.py # CI smoke test
-│ └── test_pipeline.py # 24 integration tests (no API key)
-│
-├── .github/
-│ ├── workflows/ci.yml # GitHub Actions — Python 3.10 + 3.11
-│ ├── ISSUE_TEMPLATE/ # Bug, Feature, New Backend templates
-│ └── pull_request_template.md
-│
-├── dashboard.py # Streamlit dashboard
-├── main.py # CLI entry point
-└── quick_demo.py # Zero-API-key demo
-```
+Most evaluation frameworks measure quality at a single point in time. MemoryLens measures how memory quality **changes over conversation turns** — a dimension the tools below don't cover (they solve different problems, and compose well with this one):
+
+| Framework | Focus | Decay over turns | Multi-architecture | No-API mode |
+|-----------|-------|:----------------:|:------------------:|:-----------:|
+| **MemoryLens** | Memory decay | ✅ | ✅ 8 backends | ✅ |
+| [RAGAS](https://github.com/explodinggradients/ragas) | RAG answer quality | ❌ | ❌ | ❌ |
+| [TruLens](https://github.com/truera/trulens) | LLM app monitoring | ❌ | ❌ | ❌ |
+| [DeepEval](https://github.com/confident-ai/deepeval) | LLM answer quality | ❌ | ❌ | Partial |
+| [MemGPT / Letta](https://github.com/cpacker/MemGPT) | A memory *system*, not a benchmark | — | — | — |
+
+Details: [docs/comparison-with-existing-tools.md](docs/comparison-with-existing-tools.md)
---
-## Tech Stack
+## Project Structure
-| Component | Technology | Why |
-|-----------|-----------|-----|
-| Embeddings | [sentence-transformers](https://sbert.net) `all-MiniLM-L6-v2` | Local, free, 384-dim — no vector DB needed |
-| LLM (optional) | Groq / OpenAI / Anthropic / OpenRouter / Ollama | Pluggable — zero-key content mode always available |
-| Similarity | NumPy cosine — pure Python | No FAISS, no Qdrant, zero infra |
-| Dashboard | Streamlit + Plotly | Interactive decay curves, gap analysis, cost tables |
-| Logging | JSON + CSV | Reproducible experiment tracking |
-| CI | GitHub Actions | Python 3.10 + 3.11, all 24 tests on every push |
+```
+memorylens/ installable package
+├── memory/ 8 backends + decay functions (add yours here)
+├── simulator/ facts, conversation generator, personas, scenario registry
+├── evaluation/ metrics, benchmark runner, stats, experiment logger
+├── utils/ local embeddings, LLM providers, SQLite run storage
+├── api.py FastAPI REST server
+└── cli.py `memorylens` command
+
+tests/ 39 integration tests, no API key needed
+dashboard.py Streamlit dashboard (decay curves, run history, export)
+docs/ guides: adding a backend, comparisons, methodology
+```
---
## Contributing
-MemoryLens is actively looking for contributors across all skill levels.
-
-### Add a new memory backend (most impactful)
-
-The full interface is 3 methods:
+A new memory backend is 3 methods and one registry line — the full guide with a worked example is in [docs/adding-a-new-backend.md](docs/adding-a-new-backend.md):
```python
-# memory/your_backend.py
-from .base import BaseMemory
+from memorylens.memory.base import BaseMemory
class YourMemory(BaseMemory):
- name = "your_backend" # used in --backends flag
-
- def add_message(self, role: str, content: str, turn: int) -> None: ...
- def get_context(self, query: str, current_turn: int) -> List[Dict]: ...
- def reset(self) -> None: ...
+ name = "your_backend"
+ def add_message(self, role, content, turn): ...
+ def get_context(self, query, current_turn): ...
+ def reset(self): ...
```
-Then register in `evaluation/benchmark.py` and add one test. That's a complete PR.
-
-### Good first issues
-
-| Task | Difficulty | Where |
-|------|-----------|-------|
-| Update-aware Cascading — patch Cold tier on fact updates | Medium | `memory/cascading.py` |
-| Confidence interval error bars in dashboard | Easy | `dashboard.py` |
-| EdTech fact scenario (student/teacher) | Easy | `simulator/facts.py` |
-| `pip install memorylens` — pyproject.toml setup | Easy | root |
-| Docker deployment guide | Easy | docs/ |
-| Qdrant/FAISS backend replacing NumPy | Medium | `memory/` |
-| LangGraph orchestration layer | Hard | new |
-
-Browse [`good first issue`](https://github.com/Neal006/memorylens/issues?q=label%3A%22good+first+issue%22) · Full guide: [CONTRIBUTING.md](CONTRIBUTING.md)
-
-### Development setup
+New scenarios are a data file plus one registry entry. New metrics are plain functions.
```bash
-git clone https://github.com/Neal006/memorylens.git
-cd memorylens
-python -m venv .venv && source .venv/bin/activate # or .venv\Scripts\activate on Windows
-pip install -r requirements.txt
-python tests/test_pipeline.py # 24 tests, no API key needed
+git clone https://github.com/Neal006/memorylens && cd memorylens
+pip install -e ".[server,dev]"
+pytest tests -q # all green before you start
```
----
-
-## Research
+Start here: [`good first issue`](https://github.com/Neal006/memorylens/issues?q=label%3A%22good+first+issue%22) · Guide: [CONTRIBUTING.md](CONTRIBUTING.md) · Plans: [ROADMAP.md](ROADMAP.md)
-The methodology, metric definitions, and decay ablation results are documented in the full research paper:
-
-**[MemoryLens: A Temporal Decay Benchmark for LLM Memory Architectures](paper/memorylens_paper.md)**
-
-Key sections:
-- Formal metric definitions with LaTeX formulae
-- Ebbinghaus decay ablation with 4 variants
-- Multi-seed results (n=5 personas)
-- Comparison against RAGAS, TruLens, MemGPT, A-MEM
-- Full reference list (Ebbinghaus 1885 → Xu 2024)
+---
-### Citation
+## Citation
```bibtex
@software{memorylens2026,
- author = {Srivastava, Neal},
- title = {{MemoryLens}: A Temporal Decay Benchmark for {LLM} Memory Architectures},
- year = {2026},
- url = {https://github.com/Neal006/memorylens},
- version = {0.3.0}
+ author = {Daftary, Neal},
+ title = {{MemoryLens}: A Temporal Decay Benchmark for {LLM} Memory Architectures},
+ year = {2026},
+ url = {https://github.com/Neal006/memorylens},
+ version = {0.4.0}
}
```
----
-
-## Roadmap
-
-| Status | Item |
-|--------|------|
-| ✅ Done | Naive, RAG, Cascading, SummaryMemory backends |
-| ✅ Done | 5 metrics (Recall@T, Precision@K, Drift, Noise, Efficiency) |
-| ✅ Done | Ebbinghaus decay + ablation study |
-| ✅ Done | Chunked RAG (production-realistic) |
-| ✅ Done | Multi-seed CI (n=5, mean ± std) |
-| ✅ Done | 5-provider LLM evaluation (Groq, OpenAI, Anthropic, OpenRouter, Ollama) |
-| ✅ Done | Research paper with citations |
-| 🔜 Next | Update-aware Cascading (fix temporal drift in Cold tier) |
-| 🔜 Next | Streamlit Community Cloud deployment (public live demo) |
-| 🔜 Next | Qdrant / FAISS production vector DB backend |
-| 🔜 Next | `pip install memorylens` (PyPI package) |
-| 🔜 Later | EdTech, Medical, Customer Support domain scenarios |
-| 🔜 Later | arXiv preprint |
-
-Full roadmap: [ROADMAP.md](ROADMAP.md)
-
----
-
## License
-[MIT](LICENSE) — free to use, modify, and distribute for any purpose.
+[MIT](LICENSE) — free to use, modify, and distribute.
---
-**If MemoryLens is useful to you, please consider giving it a ⭐**
-It helps other researchers and developers find the project.
-
-[](https://star-history.com/#Neal006/memorylens)
+**If MemoryLens is useful to you, a ⭐ helps other researchers and developers find it.**
diff --git a/ROADMAP.md b/ROADMAP.md
index ff89ed7..c48a7fd 100644
--- a/ROADMAP.md
+++ b/ROADMAP.md
@@ -1,83 +1,69 @@
# MemoryLens Roadmap
-MemoryLens is the **open-source benchmark for LLM memory decay** — the only evaluation framework that measures how AI memory architectures forget over long conversations. This document tracks what's shipped, what's in progress, and what's next.
+MemoryLens is an open-source benchmark for **LLM memory decay** — measuring how AI memory architectures forget over long conversations. This document tracks what's shipped, what's in progress, and what's next.
Want to pick something up? Check [CONTRIBUTING.md](CONTRIBUTING.md) and claim an item by opening an issue.
---
-## Shipped — v0.3 (current)
+## Shipped — v0.4 (current)
-### Core Benchmark
-- [x] Five memory backends: Naive · Ideal RAG · Chunked RAG · Cascading Temporal · SummaryMemory
-- [x] Five evaluation metrics: Recall@T · Precision@K · Temporal Drift · Memory Noise Ratio · Cascade Efficiency
-- [x] Multi-seed statistical validation — 5 diverse personas, mean ± std, 95% CI
-- [x] Ebbinghaus forgetting curve decay + ablation (linear / exponential / Ebbinghaus / heuristic)
-- [x] Bounded Chunked RAG — realistic production simulation with FIFO eviction
+### Packaging & Distribution
+- [x] **`pip install memorylens`** — proper PyPI package: single `memorylens` namespace, valid build backend, `memorylens` CLI entry point, lean core deps with `[dashboard]` / `[server]` / `[faiss]` / provider extras
+- [x] CI matrix: Python 3.10–3.13 on Linux + Windows + macOS, plus a package build/validation job
-### LLM Evaluation
-- [x] Two-stage LLM answer+judge pipeline (real recall, not string match)
-- [x] 5-provider LLM backend: Groq · OpenAI · Anthropic · OpenRouter · Ollama
-- [x] Gap analysis: content recall vs LLM recall (content can overestimate by 5–15pp)
+### New Backends
+- [x] **EntityMemory** — structured key-value fact extraction
+- [x] **GraphMemory** — NetworkX knowledge graph with in-place fact updates
+- [x] **FAISSMemory** — FAISS vector index (optional dependency)
-### Tooling
-- [x] CLI with `--seeds`, `--decay`, `--llm`, `--provider`, `--list-providers`
-- [x] Streamlit dashboard with Content/LLM/Gap tabbed charts, provider selector
-- [x] Zero-API-key mode — all content-based metrics work without any key
-- [x] Experiment logger (JSON + CSV) in `experiment_logs/`
-- [x] GitHub Actions CI: Python 3.10 + 3.11, 24 tests on every push
-
-### Documentation
-- [x] Research paper: [paper/memorylens_paper.md](paper/memorylens_paper.md)
-- [x] CITATION.cff — citable software with Zenodo integration
-- [x] `docs/why-memory-evaluation-matters.md`
-- [x] `docs/comparison-with-existing-tools.md`
-- [x] `docs/adding-a-new-backend.md`
-
----
+### Scenarios
+- [x] **Scenario framework** — `Scenario` dataclass + registry; scenarios plug into CLI, API, and multi-seed runs
+- [x] **EdTech** (student-tutor), **Customer Support** (ticket lifecycle), **Medical** (synthetic patient consultations)
-## Next — v0.4 (open for contributions)
+### Metrics & Fixes
+- [x] **contradiction_score** — detects when context surfaces both old and new values of an updated fact
+- [x] **Cold-tier recall regression fixed** — a cold-summary merge-order bug silently destroyed early facts (cascading recall at T=100 fell to ~8%); root-caused and fixed with regression tests
+- [x] Ebbinghaus + exponential forgetting-curve fitting (`--fit-curves`)
-### High Priority Fixes
-- [ ] **Update-aware Cascading** — when a fact update event fires, patch existing Cold tier summaries to reflect the new value. This eliminates the temporal drift regression where cold summaries retain stale facts. ([open issue](https://github.com/Neal006/memorylens/issues))
-- [ ] **Confidence interval charts** — add ± std error bars to all decay curves in the Streamlit dashboard when multi-seed results are loaded
-- [ ] **Varied persona injection timing** — currently all 5 personas share identical fact injection timing (T=0,1,2,3,4,5,7,9), causing structure-deterministic backends (Naive, Cascading) to produce std=0%. Fix: randomise injection turns ±3 turns per persona per seed so timing-based backends show real variance across seeds. This is a known limitation documented in the README.
+### Tooling
+- [x] **FastAPI server** — async job-based REST API (`uvicorn memorylens.api:app`)
+- [x] **Dashboard Run History tab** — compare past benchmark runs side-by-side
+- [x] Experiment logger schema migration (rotates CSV on metric changes)
-### New Memory Backends
-- [ ] **EntityMemory** — extract named entities into a structured key-value store; benchmark whether structured storage beats unstructured retrieval ([guide](docs/adding-a-new-backend.md))
-- [ ] **Qdrant backend** — production vector DB replacing NumPy cosine similarity; benchmark at 10K+ conversation turns
-- [ ] **Graph memory** — entities + relationships stored as a knowledge graph; test multi-hop fact retrieval
-- [ ] **Redis-backed memory** — persistent cross-session memory; test recall across session boundaries
+---
-### Scenarios
-- [ ] **EdTech scenario** — student/teacher memory: track subject performance, weak topics, learning styles across 200-turn tutoring session
-- [ ] **Customer support scenario** — 100K customer histories; benchmark memory under high cardinality
-- [ ] **Medical scenario** — patient history across multi-session clinical conversations (anonymised synthetic data)
+## Shipped — v0.3
-### Integrations
-- [ ] **LangGraph wrapper** — run the full benchmark as a LangGraph state machine for agent-native evaluation
-- [ ] **RAGAS adapter** — export MemoryLens checkpoints as RAGAS-compatible evaluation samples
-- [ ] **LangChain memory adapter** — wrap LangChain `ConversationSummaryMemory` and `VectorStoreRetrieverMemory` as MemoryLens backends
+- [x] Multi-seed statistical validation — 5 personas, mean ± std, 95% CI (`--seeds`)
+- [x] Ebbinghaus decay + ablation (linear / exponential / Ebbinghaus / heuristic)
+- [x] Bounded Chunked RAG (chunking + FIFO index eviction)
+- [x] Two-stage LLM answer+judge pipeline; 5 providers (Groq · OpenAI · Anthropic · OpenRouter · Ollama)
+- [x] SummaryMemory backend, experiment logger, zero-API-key mode
---
-## Later — v0.5
+## Next — v0.5 (open for contributions)
+
+### Benchmark Realism
+- [ ] **Harder fact templates** — free-form paraphrased injections so extraction-based backends (entity, graph) can't pattern-match; today's templated facts let them score 100% by construction
+- [ ] **Varied persona injection timing** — randomise injection turns per seed so timing-deterministic backends show real cross-seed variance
+- [ ] **Longer horizons** — 500–1,000-turn runs where bounded indexes and context budgets genuinely saturate
-### Deployment
-- [ ] **Streamlit Community Cloud** — live public demo URL (no install needed)
-- [ ] **HuggingFace Spaces** — mirror for ML community discoverability
-- [ ] **Docker image** — `docker run neal006/memorylens`
-- [ ] **`pip install memorylens`** — proper PyPI package
+### New Backends
+- [ ] **Qdrant backend** — hosted vector DB, benchmark at 10K+ turns
+- [ ] **Redis-backed memory** — persistence across session boundaries
+- [ ] **LangChain memory adapter** — wrap `ConversationSummaryMemory` / `VectorStoreRetrieverMemory` as backends
-### Research Track
-- [ ] **arXiv preprint** — publish [paper/memorylens_paper.md](paper/memorylens_paper.md) as arXiv:XXXX.XXXXX
-- [ ] **HuggingFace dataset** — synthetic conversation logs as a public dataset card
-- [ ] **Ebbinghaus curve fitting** — fit the actual Recall@T decay data to the forgetting curve and report stability parameters per backend
+### Integrations & Deployment
+- [ ] **Publish to PyPI** (package is release-ready; needs a maintainer `twine upload`)
+- [ ] **Docker image**
+- [ ] **HuggingFace Spaces / Streamlit Cloud demo** (community contribution welcome — see issue #28)
+- [ ] **RAGAS adapter** — export checkpoints as RAGAS-compatible samples
### Engineering
-- [ ] **Async benchmark runner** — parallel backend evaluation for 5× faster multi-seed runs
-- [ ] **Plugin architecture** — register custom backends and metrics via Python entry points
-- [ ] **Streaming evaluation** — real-time memory quality monitoring for live LLM deployments
+- [ ] **Async benchmark runner** — parallel backend evaluation
+- [ ] **Plugin architecture** — register custom backends/metrics via entry points
---
diff --git a/dashboard.py b/dashboard.py
index d12511c..93f96e4 100644
--- a/dashboard.py
+++ b/dashboard.py
@@ -25,13 +25,19 @@
""", unsafe_allow_html=True)
COLORS = {
- "naive": "#f38ba8",
- "rag": "#89b4fa",
- "cascading": "#a6e3a1",
- "summary": "#fab387",
+ "naive": "#f38ba8",
+ "rag": "#89b4fa",
+ "rag_chunked": "#74c7ec",
+ "cascading": "#a6e3a1",
+ "summary": "#fab387",
+ "entity": "#cba6f7",
+ "graph": "#f9e2af",
+ "faiss": "#94e2d5",
}
+ALL_BACKENDS = ["naive", "rag", "rag_chunked", "cascading", "summary", "entity", "graph", "faiss"]
+# Illustrative cost assumption, stated in the UI: $1 per 1M input tokens.
MONTHLY_QUERIES = 100_000
-COST_PER_TOKEN_INR = 83 / 1_000_000 # ~$1 per 1M tokens * 83 INR/USD
+COST_PER_TOKEN_USD = 1 / 1_000_000
_PROVIDER_KEYS = {
"groq": "GROQ_API_KEY",
@@ -61,8 +67,6 @@ def _detect_available_providers() -> List[str]:
# ─── Sidebar ────────────────────────────────────────────────────────────────
with st.sidebar:
- st.image("https://raw.githubusercontent.com/simple-icons/simple-icons/develop/icons/anthropic.svg",
- width=32)
st.title("MemoryLens")
st.caption("LLM Memory Decay Evaluation Framework")
st.divider()
@@ -76,8 +80,9 @@ def _detect_available_providers() -> List[str]:
)
backends = st.multiselect(
"Memory backends",
- ["naive", "rag", "cascading", "summary"],
+ ALL_BACKENDS,
default=["naive", "rag", "cascading"],
+ help='faiss requires the optional dependency: pip install "memorylens[faiss]"',
)
st.divider()
@@ -127,7 +132,7 @@ def load_demo() -> Dict:
def render_results(data: Dict, is_demo: bool = False) -> None:
cps: List[int] = data["checkpoints"]
- present = [b for b in ["naive", "rag", "cascading", "summary"] if b in data]
+ present = [b for b in ALL_BACKENDS if b in data]
has_llm = data.get("has_llm_eval", False)
if is_demo:
@@ -156,6 +161,8 @@ def render_results(data: Dict, is_demo: bool = False) -> None:
)
st.metric("Avg Tokens", f"{d['tokens'][-1]:,}")
st.metric("Temporal Drift", f"{d['drift'][-1]*100:.1f}%")
+ if d.get("contradiction"):
+ st.metric("Contradiction", f"{d['contradiction'][-1]*100:.1f}%")
st.metric("Precision@K", f"{d['precision'][-1]*100:.1f}%")
st.divider()
@@ -312,43 +319,48 @@ def render_results(data: Dict, is_demo: bool = False) -> None:
st.caption("Efficiency > 1.0 means cascading delivers more recall per token spent than the naive baseline.")
# ── Token cost table ────────────────────────────────────────────────────
- st.subheader("Business Impact — Monthly Token Cost")
+ st.subheader("Projected Monthly Token Cost")
+ st.caption(
+ f"Illustrative projection: {MONTHLY_QUERIES:,} queries/month at "
+ "$1 per 1M input tokens. Scale to your own price and volume."
+ )
rows = []
for name in present:
final_tok = data[name]["tokens"][-1]
- monthly = final_tok * MONTHLY_QUERIES * COST_PER_TOKEN_INR
+ monthly = final_tok * MONTHLY_QUERIES * COST_PER_TOKEN_USD
rows.append({
- "Backend": name.capitalize(),
- "Tokens / Query": f"{final_tok:,}",
- "Monthly Cost (₹)": f"₹{monthly:,.0f}",
- "Recall @ Final": f"{data[name]['recall'][-1]*100:.1f}%",
- "Drift @ Final": f"{data[name]['drift'][-1]*100:.1f}%",
+ "Backend": name.capitalize(),
+ "Tokens / Query": f"{final_tok:,}",
+ "Monthly Cost ($)": f"${monthly:,.0f}",
+ "Recall @ Final": f"{data[name]['recall'][-1]*100:.1f}%",
+ "Drift @ Final": f"{data[name]['drift'][-1]*100:.1f}%",
})
st.dataframe(pd.DataFrame(rows), use_container_width=True, hide_index=True)
# Savings callout
if "naive" in data and "cascading" in data:
- n_cost = data["naive"]["tokens"][-1] * MONTHLY_QUERIES * COST_PER_TOKEN_INR
- c_cost = data["cascading"]["tokens"][-1] * MONTHLY_QUERIES * COST_PER_TOKEN_INR
- pct = (n_cost - c_cost) / n_cost * 100
- recall_delta = (data["cascading"]["recall"][-1] - data["naive"]["recall"][-1]) * 100
- st.success(
- f"**Cascading Temporal Memory saves {pct:.0f}% in token costs** vs Naive "
- f"while delivering {recall_delta:+.1f}pp better recall at 100 K queries/month."
- )
+ n_tok = data["naive"]["tokens"][-1]
+ c_tok = data["cascading"]["tokens"][-1]
+ if n_tok > 0:
+ pct = (n_tok - c_tok) / n_tok * 100
+ recall_delta = (data["cascading"]["recall"][-1] - data["naive"]["recall"][-1]) * 100
+ st.success(
+ f"**Cascading uses {pct:.0f}% fewer tokens per query** than Naive "
+ f"with {recall_delta:+.1f}pp recall difference in this run."
+ )
# Cost bar chart
fig4 = go.Figure()
for row in rows:
- cost_val = float(row["Monthly Cost (₹)"].replace("₹", "").replace(",", ""))
+ cost_val = float(row["Monthly Cost ($)"].replace("$", "").replace(",", ""))
fig4.add_trace(go.Bar(
x=[row["Backend"]], y=[cost_val],
name=row["Backend"],
marker_color=COLORS.get(row["Backend"].lower(), "#cdd6f4"),
- text=f"₹{cost_val:,.0f}", textposition="outside",
+ text=f"${cost_val:,.0f}", textposition="outside",
))
fig4.update_layout(
- yaxis_title="Monthly cost (₹) @ 100K queries",
+ yaxis_title=f"Monthly cost ($) @ {MONTHLY_QUERIES:,} queries",
template="plotly_dark", height=360, showlegend=False,
)
st.plotly_chart(fig4, use_container_width=True)
@@ -390,6 +402,80 @@ def _latex_table(data: Dict, checkpoints: List[int], present: List[str]) -> str:
)
+def _series_value(v):
+ """Multi-seed logs store per-checkpoint stat dicts; single-seed logs store floats."""
+ return v.get("mean") if isinstance(v, dict) else v
+
+
+def render_history() -> None:
+ from memorylens.evaluation.logger import list_runs, get_run_results
+
+ runs = list_runs()
+ if not runs:
+ st.info(
+ "No logged runs yet. Run `memorylens --log` or click **▶ Run Live** — "
+ "every run is saved to `experiment_logs/`."
+ )
+ return
+
+ labels = {}
+ for r in runs:
+ cfg = r["config"]
+ label = (
+ f"{r['run_id']} · {cfg.get('total_turns', '?')} turns · "
+ f"{', '.join(cfg.get('backends', []))}"
+ + (f" · {cfg['provider']}" if cfg.get("provider") else "")
+ )
+ labels[label] = r
+
+ selected = st.multiselect(
+ "Runs to compare",
+ list(labels),
+ default=list(labels)[:2],
+ help="Overlays Recall@T curves; solid/dash/dot line style distinguishes runs.",
+ )
+ if not selected:
+ return
+
+ dashes = ["solid", "dash", "dot", "dashdot", "longdash"]
+ fig = go.Figure()
+ table_rows = []
+
+ for run_idx, label in enumerate(selected):
+ run = labels[label]
+ results = get_run_results(run["run_id"]) or {}
+ cps = results.get("checkpoints", [])
+ for name in [b for b in ALL_BACKENDS if b in results]:
+ recall = [_series_value(v) for v in results[name].get("recall", [])]
+ tokens = [_series_value(v) for v in results[name].get("tokens", [])]
+ fig.add_trace(go.Scatter(
+ x=cps,
+ y=[v * 100 if v is not None else None for v in recall],
+ name=f"{run['run_id']} · {name}",
+ mode="lines+markers",
+ line=dict(color=COLORS.get(name, "#cdd6f4"),
+ dash=dashes[run_idx % len(dashes)], width=2),
+ ))
+ final_recall = next((v for v in reversed(recall) if v is not None), None)
+ final_tokens = next((v for v in reversed(tokens) if v is not None), None)
+ table_rows.append({
+ "Run": run["run_id"],
+ "Backend": name,
+ "Recall @ Final": f"{final_recall*100:.1f}%" if final_recall is not None else "—",
+ "Tokens @ Final": f"{final_tokens:,.0f}" if final_tokens is not None else "—",
+ "Turns": run["config"].get("total_turns", "—"),
+ "Provider": run["config"].get("provider") or "content-only",
+ })
+
+ fig.update_layout(
+ xaxis_title="Conversation Turn", yaxis_title="Recall (%)",
+ yaxis=dict(range=[0, 105]), template="plotly_dark",
+ height=420, legend=dict(orientation="h", y=-0.25),
+ )
+ st.plotly_chart(fig, use_container_width=True)
+ st.dataframe(pd.DataFrame(table_rows), use_container_width=True, hide_index=True)
+
+
# ─── Main logic ─────────────────────────────────────────────────────────────
if "results" not in st.session_state:
st.session_state.results = None
@@ -412,14 +498,14 @@ def push_log(msg: str) -> None:
log_area.text_area("Progress", "\n".join(logs[-12:]), height=200)
with st.spinner("Running benchmark…"):
- from evaluation.benchmark import run_benchmark, results_to_display_dict
- from evaluation.logger import log_run
+ from memorylens.evaluation.benchmark import run_benchmark, results_to_display_dict
+ from memorylens.evaluation.logger import log_run
# Resolve provider (None = content-only)
provider_obj = None
if selected_provider:
try:
- from utils.providers import get_provider
+ from memorylens.utils.providers import get_provider
provider_obj = get_provider(selected_provider)
push_log(f"LLM provider: {provider_obj.name}")
except Exception as e:
@@ -446,30 +532,35 @@ def push_log(msg: str) -> None:
log_area.empty()
st.rerun()
-if st.session_state.results:
- render_results(st.session_state.results, is_demo=st.session_state.is_demo)
-else:
- # ── Landing page ──────────────────────────────────────────────────────
- st.title("🔭 MemoryLens")
- st.markdown("### *An Evaluation Framework for LLM Memory Decay*")
- st.markdown("> **You can't improve what you can't measure. Nobody is measuring memory.**")
- st.divider()
+tab_bench, tab_history = st.tabs(["Benchmark", "Run History"])
+
+with tab_history:
+ render_history()
+
+with tab_bench:
+ if st.session_state.results:
+ render_results(st.session_state.results, is_demo=st.session_state.is_demo)
+ else:
+ # ── Landing page ──────────────────────────────────────────────────
+ st.title("🔭 MemoryLens")
+ st.markdown("### *An Evaluation Framework for LLM Memory Decay*")
+ st.divider()
- st.markdown("""
+ st.markdown(f"""
| Layer | What It Does |
|-------|--------------|
-| **Memory Injection** | Injects personal facts at T=0 and queries them at T=10, 25, 50, 100 |
-| **4 Backends** | Naive · RAG · Cascading Temporal · SummaryMemory |
-| **5 Metrics** | Recall@T · Precision@K · Temporal Drift · Memory Noise Ratio · Token Cost |
+| **Memory Injection** | Injects personal facts at known turns and queries them at checkpoints |
+| **{len(ALL_BACKENDS)} Backends** | {' · '.join(ALL_BACKENDS)} |
+| **6 Metrics** | Recall@T · Precision@K · Temporal Drift · Contradiction · Noise Ratio · Token Cost |
| **LLM Eval** | Two-stage answer+judge pipeline — 5 providers (Groq, OpenAI, Anthropic, OpenRouter, Ollama) |
-| **Dashboard** | Decay curves, content vs LLM recall gap, cost impact, LaTeX export |
+| **Dashboard** | Decay curves, content vs LLM recall gap, cost projection, run history, LaTeX export |
**Click 📊 Demo** in the sidebar for instant results, or configure a provider and click **▶ Run Live**.
""")
- st.markdown("---")
- st.markdown("#### How Cascading Temporal Memory Works")
- st.code("""
+ st.markdown("---")
+ st.markdown("#### How Cascading Temporal Memory Works")
+ st.code("""
┌─────────────────────────────────────────────────────────┐
│ CASCADING TEMPORAL MEMORY │
│ │
diff --git a/demo_results.json b/demo_results.json
index 116291d..5055d62 100644
--- a/demo_results.json
+++ b/demo_results.json
@@ -1,27 +1,278 @@
{
- "checkpoints": [10, 25, 50, 75, 100],
+ "checkpoints": [
+ 10,
+ 25,
+ 50,
+ 75,
+ 100
+ ],
+ "has_llm_eval": false,
"naive": {
- "recall": [1.000, 1.000, 1.000, 1.000, 0.625],
- "precision": [0.900, 0.820, 0.720, 0.650, 0.520],
- "drift": [0.000, 0.000, 0.500, 0.500, 0.000],
- "noise": [0.550, 0.800, 0.880, 0.900, 0.940],
- "tokens": [103, 291, 614, 934, 1190],
- "cascade_eff": [1.0, 1.0, 1.0, 1.0, 1.0]
+ "recall": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 0.375
+ ],
+ "precision": [
+ 0.6,
+ 0.6,
+ 0.4,
+ 0.4,
+ 0.2
+ ],
+ "drift": [
+ 0.0,
+ 0.0,
+ 0.5,
+ 0.5,
+ 0.0
+ ],
+ "noise": [
+ 0.55,
+ 0.8,
+ 0.87,
+ 0.8867,
+ 0.9366
+ ],
+ "contradiction": [
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 0.0
+ ],
+ "tokens": [
+ 118,
+ 366,
+ 786,
+ 1203,
+ 1192
+ ],
+ "cascade_eff": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "llm_recall": [
+ null,
+ null,
+ null,
+ null,
+ null
+ ],
+ "llm_drift": [
+ null,
+ null,
+ null,
+ null,
+ null
+ ],
+ "provider": null,
+ "decay": "ebbinghaus"
},
"rag": {
- "recall": [1.000, 1.000, 1.000, 1.000, 1.000],
- "precision": [0.920, 0.880, 0.840, 0.820, 0.800],
- "drift": [0.000, 0.000, 0.500, 0.500, 0.500],
- "noise": [0.290, 0.750, 1.000, 1.000, 1.000],
- "tokens": [54, 59, 66, 61, 58],
- "cascade_eff": [1.0, 1.0, 1.0, 1.0, 1.0]
+ "recall": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "precision": [
+ 1.0,
+ 1.0,
+ 0.8,
+ 0.8,
+ 0.8
+ ],
+ "drift": [
+ 0.0,
+ 0.0,
+ 0.5,
+ 0.5,
+ 0.5
+ ],
+ "noise": [
+ 0.2857,
+ 0.875,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "contradiction": [
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "tokens": [
+ 60,
+ 69,
+ 77,
+ 71,
+ 68
+ ],
+ "cascade_eff": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "llm_recall": [
+ null,
+ null,
+ null,
+ null,
+ null
+ ],
+ "llm_drift": [
+ null,
+ null,
+ null,
+ null,
+ null
+ ],
+ "provider": null,
+ "decay": "ebbinghaus"
},
"cascading": {
- "recall": [1.000, 1.000, 1.000, 0.875, 0.750],
- "precision": [0.880, 0.860, 0.840, 0.820, 0.800],
- "drift": [0.000, 0.000, 0.500, 0.750, 1.000],
- "noise": [0.470, 0.880, 0.880, 0.880, 0.880],
- "tokens": [88, 148, 267, 270, 262],
- "cascade_eff": [1.16, 1.96, 2.30, 3.03, 5.45]
+ "recall": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "precision": [
+ 0.8,
+ 0.8,
+ 0.4,
+ 0.4,
+ 0.2
+ ],
+ "drift": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "noise": [
+ 0.5333,
+ 0.8125,
+ 0.9375,
+ 0.875,
+ 0.875
+ ],
+ "contradiction": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ],
+ "tokens": [
+ 101,
+ 162,
+ 312,
+ 336,
+ 326
+ ],
+ "cascade_eff": [
+ 93.6638,
+ 58.7652,
+ 15.2855,
+ 9.4417,
+ 9.7566
+ ],
+ "llm_recall": [
+ null,
+ null,
+ null,
+ null,
+ null
+ ],
+ "llm_drift": [
+ null,
+ null,
+ null,
+ null,
+ null
+ ],
+ "provider": null,
+ "decay": "ebbinghaus"
+ },
+ "summary": {
+ "recall": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "precision": [
+ 0.6,
+ 0.2,
+ 0.6,
+ 0.2,
+ 0.4
+ ],
+ "drift": [
+ 0.0,
+ 0.0,
+ 0.5,
+ 0.5,
+ 0.5
+ ],
+ "noise": [
+ 0.55,
+ 0.9048,
+ 0.8571,
+ 0.9048,
+ 0.9048
+ ],
+ "contradiction": [
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "tokens": [
+ 118,
+ 230,
+ 256,
+ 316,
+ 358
+ ],
+ "cascade_eff": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0
+ ],
+ "llm_recall": [
+ null,
+ null,
+ null,
+ null,
+ null
+ ],
+ "llm_drift": [
+ null,
+ null,
+ null,
+ null,
+ null
+ ],
+ "provider": null,
+ "decay": "ebbinghaus"
}
-}
+}
\ No newline at end of file
diff --git a/docs/adding-a-new-backend.md b/docs/adding-a-new-backend.md
index 7f3dd39..f24bcf7 100644
--- a/docs/adding-a-new-backend.md
+++ b/docs/adding-a-new-backend.md
@@ -9,7 +9,7 @@ This guide walks through implementing a custom LLM memory backend and benchmarki
Every memory backend in MemoryLens inherits from `BaseMemory`:
```python
-# memory/base.py
+# memorylens/memory/base.py
class BaseMemory(ABC):
name: str = "base" # used in --backends flag and results tables
@@ -42,7 +42,7 @@ class BaseMemory(ABC):
As a concrete example, let's implement an entity-extraction memory backend that stores named entities separately from conversation flow.
-### Step 1 — Create `memory/entity.py`
+### Step 1 — Create `memorylens/memory/entity.py`
```python
from typing import List, Dict
@@ -102,10 +102,10 @@ class EntityMemory(BaseMemory):
self.recent = []
```
-### Step 2 — Register in `evaluation/benchmark.py`
+### Step 2 — Register in `memorylens/evaluation/benchmark.py`
```python
-from memory.entity import EntityMemory
+from memorylens.memory.entity import EntityMemory
def _make_memory(name: str, decay: str = "ebbinghaus") -> BaseMemory:
# ... existing cases ...
@@ -121,7 +121,7 @@ Also add `"entity"` to `VALID_BACKENDS`.
```python
# tests/test_pipeline.py
def test_entity_recall_early():
- from memory.entity import EntityMemory
+ from memorylens.memory.entity import EntityMemory
mem = EntityMemory()
_populate(mem, BENCHMARK_FACTS, 15)
active = [f for f in BENCHMARK_FACTS if f.injected_at < 15]
@@ -141,8 +141,8 @@ python main.py --seeds 5 --backends naive rag entity cascading
### Step 5 — Open a PR
Include:
-- `memory/entity.py` — the backend implementation
-- Updated `evaluation/benchmark.py` — registration
+- `memorylens/memory/entity.py` — the backend implementation
+- Updated `memorylens/evaluation/benchmark.py` — registration
- Test in `tests/test_pipeline.py`
- Entry in `CHANGELOG.md` under `[Unreleased]`
@@ -152,11 +152,12 @@ Include:
| Backend | Strategy | Hypothesis to test |
|---------|----------|-------------------|
-| `entity` | Named-entity extraction into a key-value store | Does structured storage beat unstructured retrieval? |
-| `qdrant` | Production vector DB (Qdrant) | Does a real vector DB beat NumPy cosine at scale? |
+| `entity` | Named-entity extraction into a key-value store | Shipped: `memorylens/memory/entity.py` |
+| `graph` | Knowledge graph (entities + relationships) | Shipped: `memorylens/memory/graph.py` — multi-hop edges still open |
+| `faiss` | FAISS vector index | Shipped: `memorylens/memory/vector_faiss.py` (optional dep) |
+| `qdrant` | Production vector DB (Qdrant) | Does a hosted vector DB change results at 10K+ turns? |
| `redis` | Persistent Redis-backed storage | Does persistence across sessions affect recall? |
| `memgpt_style` | Virtual paging between in-context and external | Does OS-style memory management beat Cascading? |
-| `graph` | Knowledge graph (entities + relationships) | Does structured relationships help with multi-hop facts? |
| `sliding_window` | Fixed K-message window | What's the optimal window size? |
| `importance_weighted` | Keep messages by semantic importance score | Does importance sampling beat recency? |
diff --git a/docs/comparison-with-existing-tools.md b/docs/comparison-with-existing-tools.md
index 72deaee..a31a819 100644
--- a/docs/comparison-with-existing-tools.md
+++ b/docs/comparison-with-existing-tools.md
@@ -10,7 +10,7 @@
|---|---|---|---|---|---|---|
| **Primary focus** | Memory decay over time | RAG quality | LLM app quality | LLM answer quality | Memory system | Memory implementation |
| **Temporal evaluation** | ✅ Core feature | ❌ | ❌ | ❌ | N/A | N/A |
-| **Multi-architecture comparison** | ✅ 5 backends | ❌ | ❌ | ❌ | N/A | N/A |
+| **Multi-architecture comparison** | ✅ 8 backends | ❌ | ❌ | ❌ | N/A | N/A |
| **No API key mode** | ✅ Full benchmark | ❌ | ❌ | Partial | ❌ | ❌ |
| **Decay formula** | ✅ Ebbinghaus (1885) | N/A | N/A | N/A | N/A | N/A |
| **Statistical validation** | ✅ n=5, mean ± std | ❌ | ❌ | ❌ | N/A | N/A |
diff --git a/docs/why-memory-evaluation-matters.md b/docs/why-memory-evaluation-matters.md
index 86aa206..05629d9 100644
--- a/docs/why-memory-evaluation-matters.md
+++ b/docs/why-memory-evaluation-matters.md
@@ -1,6 +1,6 @@
# Why LLM Memory Evaluation Matters
-> This document explains the **LLM memory decay problem** — why it exists, why no one is measuring it, and what MemoryLens does about it.
+> This document explains the **LLM memory decay problem** — why it exists, why it is rarely measured, and what MemoryLens does about it.
---
@@ -62,7 +62,7 @@ For the behavioral measurement (does the LLM actually answer with the old or new
python main.py --llm # enables the answer+judge pipeline for all drift measurements
```
-The two drift metrics together give you both the worst-case bound (content) and the real-world answer quality (LLM). On most well-structured backends, the LLM drift is 15–30% lower than content drift because the model correctly resolves contradictions when both values are present.
+The two drift metrics together give you both the worst-case bound (content) and the real-world answer quality (LLM). LLM drift is typically lower than content drift because the model often resolves the contradiction correctly when both values are present — measure the gap on your own setup with `--llm`.
### Cascade Efficiency — How much recall does each token buy?
@@ -70,7 +70,7 @@ The most practical metric for production systems:
$$\text{CascEff}(T) = \frac{\text{Recall}_\text{cascading}(T) / \text{Tokens}_\text{cascading}(T)}{\text{Recall}_\text{naive}(T) / \text{Tokens}_\text{naive}(T)}$$
-A value of 5.67× means the Cascading architecture delivers 5.67 times more recall per token than the naive baseline — the same information accessed at 1/5.67 the inference cost.
+A value of 3× means the Cascading architecture delivers three times more recall per token than the naive baseline — the same information accessed at a third of the inference cost.
---
@@ -86,7 +86,7 @@ where *S* is memory stability and *t* is time elapsed. MemoryLens's Cascading Te
decay = exp(-age / (stability * sqrt(1 + age)))
```
-This is not decorative. Ablation experiments show that the Ebbinghaus curve outperforms ad-hoc linear decay by 4% in cascade efficiency — because it correctly models the steep initial forgetting followed by a flattening retention curve for consolidated memories.
+The rationale: it models steep initial forgetting followed by a flattening retention curve for consolidated memories. Compare the variants on your own workload with the built-in ablation: `memorylens --decay ebbinghaus|exponential|linear|default`.
---
diff --git a/experiment_logs/runs_summary.csv b/experiment_logs/runs_summary.csv
deleted file mode 100644
index a92fbf0..0000000
--- a/experiment_logs/runs_summary.csv
+++ /dev/null
@@ -1,3 +0,0 @@
-run_id,backend,turn,recall,precision,drift,noise,tokens,total_turns
-test_run,naive,10,1.0,0.9,0.0,0.5,100,25
-test_run,naive,25,0.8,0.7,0.1,0.8,500,25
diff --git a/main.py b/main.py
index a37dabd..d7cbf83 100644
--- a/main.py
+++ b/main.py
@@ -1,320 +1,6 @@
-"""
-MemoryLens CLI
-
-Usage (content-only, no API key needed):
- python main.py
-
-Usage (full LLM evaluation, auto-detects available provider):
- python main.py --llm
-
-Usage (force a specific provider):
- python main.py --llm --provider openai
- python main.py --llm --provider anthropic
- python main.py --llm --provider groq
- python main.py --llm --provider openrouter
- python main.py --llm --provider ollama
-
-Multi-seed benchmark (reports mean +/- std across N personas):
- python main.py --seeds 5
- python main.py --seeds 5 --llm
-
-Decay formula ablation (compare forgetting curve variants):
- python main.py --decay ebbinghaus (default -- Ebbinghaus 1885)
- python main.py --decay exponential
- python main.py --decay linear
- python main.py --decay default (original heuristic)
-
-Realistic chunked RAG backend:
- python main.py --backends naive rag_chunked cascading
-
-Forgetting-curve analysis (fit Ebbinghaus + exponential to recall@T data):
- python main.py --fit-curves
- python main.py --seeds 5 --fit-curves
-
-Other options:
- python main.py --turns 50 --backends naive rag --log
- python main.py --list-providers
-"""
-
-import os
-import sys
-import json
-import argparse
-from dotenv import load_dotenv
-
-load_dotenv()
-
-
-def main() -> None:
- parser = argparse.ArgumentParser(
- description="MemoryLens: End-to-end LLM Memory Decay Benchmark",
- formatter_class=argparse.RawDescriptionHelpFormatter,
- )
- parser.add_argument("--turns", type=int, default=100)
- parser.add_argument("--checkpoints", nargs="+", type=int,
- default=[10, 25, 50, 75, 100])
- parser.add_argument("--backends", nargs="+",
- default=["naive", "rag", "cascading"],
- help="naive | rag | rag_chunked | cascading | summary | entity")
- parser.add_argument("--output", type=str, default="results.json")
- parser.add_argument("--log", action="store_true",
- help="Save run to experiment_logs/")
- parser.add_argument("--llm", action="store_true",
- help="Run real LLM evaluation pass (needs an API key or Ollama)")
- parser.add_argument("--provider", type=str, default=None,
- help="Force a provider: groq | openai | anthropic | openrouter | ollama")
- parser.add_argument("--list-providers", action="store_true",
- help="Print available providers and exit")
- parser.add_argument("--seeds", type=int, default=1,
- help="Number of persona seeds to run (max 5). >1 reports mean +/- std.")
- parser.add_argument("--decay", type=str, default="ebbinghaus",
- choices=["ebbinghaus", "exponential", "linear", "default"],
- help="Temporal decay function for CascadingMemory warm tier")
- parser.add_argument("--fit-curves", action="store_true",
- help="After benchmarking, fit Ebbinghaus + exponential decay curves "
- "to recall@T data and report half-life / stability / R²")
- parser.add_argument("--scenario", type=str, default="default",
- choices=["default", "edtech"],
- help="Conversation scenario: default (tech Q&A) | edtech (student-tutor)")
- args = parser.parse_args()
-
- # ── List providers ────────────────────────────────────────────────────────
- if args.list_providers:
- from utils.providers import list_available, _REGISTRY
- available = list_available()
- print("\nProvider status:")
- for name in _REGISTRY:
- status = "available" if name in available else "not available"
- print(f" {name:<15} {status}")
- print()
- sys.exit(0)
-
- # ── Resolve LLM provider ─────────────────────────────────────────────────
- provider = None
- if args.llm:
- from utils.providers import get_provider
- try:
- provider = get_provider(args.provider)
- except (ValueError, RuntimeError) as e:
- print(f"ERROR: {e}")
- sys.exit(1)
-
- if provider is None:
- print(
- "ERROR: --llm requested but no provider is available.\n"
- " Set one of: GROQ_API_KEY, OPENAI_API_KEY, ANTHROPIC_API_KEY, "
- "OPENROUTER_API_KEY\n"
- " or start Ollama locally.\n"
- " Run --list-providers to see status."
- )
- sys.exit(1)
-
- multi_seed = args.seeds > 1
-
- # ── Resolve scenario ─────────────────────────────────────────────────────
- scenario_facts = None
- scenario_filler = None
- scenario_pool = None
-
- if args.scenario == "edtech":
- from simulator.scenarios.edtech import (
- EDTECH_FACTS, EDTECH_FILLER_TURNS, EDTECH_PERSONA_POOL,
- )
- scenario_facts = EDTECH_FACTS
- scenario_filler = EDTECH_FILLER_TURNS
- scenario_pool = EDTECH_PERSONA_POOL
-
- # ── Banner ───────────────────────────────────────────────────────────────
- print("=" * 65)
- print(" MemoryLens -- LLM Memory Decay Benchmark")
- print("=" * 65)
- print(f" Turns : {args.turns}")
- print(f" Checkpoints : {sorted(args.checkpoints)}")
- print(f" Backends : {args.backends}")
- print(f" Decay : {args.decay}")
- print(f" Scenario : {args.scenario}")
- if multi_seed:
- print(f" Seeds : {args.seeds} (multi-seed -- will report mean +/- std)")
- print(f" LLM eval : {'ON (' + provider.name + ')' if provider else 'OFF (content-only)'}")
- print("=" * 65)
-
- # ── Run benchmark ─────────────────────────────────────────────────────────
- if multi_seed:
- from evaluation.benchmark import run_benchmark_multi_seed
- aggregated = run_benchmark_multi_seed(
- n_seeds=args.seeds,
- total_turns=args.turns,
- eval_checkpoints=sorted(args.checkpoints),
- backends=args.backends,
- provider=provider,
- decay=args.decay,
- progress=print,
- persona_pool=scenario_pool,
- filler_turns=scenario_filler,
- )
- _print_multi_seed_results(aggregated, args.backends)
- _save(aggregated, args.output)
- if args.log:
- from evaluation.logger import log_run
- path = log_run(aggregated, {
- "total_turns": args.turns,
- "backends": args.backends,
- "seeds": args.seeds,
- "decay": args.decay,
- "provider": provider.name if provider else None,
- })
- print(f"Experiment logged -> {path}")
- else:
- from evaluation.benchmark import run_benchmark, results_to_display_dict
- raw = run_benchmark(
- total_turns=args.turns,
- eval_checkpoints=sorted(args.checkpoints),
- facts=scenario_facts,
- backends=args.backends,
- provider=provider,
- decay=args.decay,
- progress=print,
- filler_turns=scenario_filler,
- )
- display = results_to_display_dict(raw)
- _print_single_seed_results(display, args.backends)
- _save(display, args.output)
- if args.log:
- from evaluation.logger import log_run
- path = log_run(display, {
- "total_turns": args.turns,
- "backends": args.backends,
- "decay": args.decay,
- "provider": provider.name if provider else None,
- })
- print(f"Experiment logged -> {path}")
-
- # ── Forgetting-curve analysis ─────────────────────────────────────────────
- if args.fit_curves:
- from evaluation.stats import fit_forgetting_curve
- checkpoints = sorted(args.checkpoints)
- print("\nFORGETTING CURVE FIT (Ebbinghaus + Exponential)")
- print("-" * 65)
- if multi_seed:
- for name in args.backends:
- if name not in aggregated:
- continue
- mean_recalls = [stat["mean"] for stat in aggregated[name]["recall"]]
- fit = fit_forgetting_curve(checkpoints, mean_recalls)
- _print_curve_fit(name, fit)
- else:
- for name in args.backends:
- if name not in display:
- continue
- fit = fit_forgetting_curve(checkpoints, display[name]["recall"])
- _print_curve_fit(name, fit)
-
- print("Visualise: streamlit run dashboard.py")
-
-
-# ── Output helpers ────────────────────────────────────────────────────────────
-
-
-def _print_curve_fit(backend: str, fit: dict) -> None:
- if "error" in fit:
- print(f" {backend:<14} {fit['error']}")
- return
- exp = fit["exponential"]
- ebb = fit["ebbinghaus"]
- hl_exp = f"{exp['half_life']:.1f} turns" if exp["half_life"] is not None else "N/A"
- hl_ebb = f"{ebb['half_life']:.1f} turns" if ebb["half_life"] is not None else "N/A"
- r2_exp = f"{exp['r2']:.3f}" if exp["r2"] is not None else "N/A"
- r2_ebb = f"{ebb['r2']:.3f}" if ebb["r2"] is not None else "N/A"
- stab = f"{ebb['stability']:.4f}" if ebb["stability"] is not None else "N/A"
- k_val = f"{exp['k']:.6f}" if exp["k"] is not None else "N/A"
- print(f" {backend}")
- print(f" Exponential k={k_val} half-life={hl_exp} R²={r2_exp}")
- print(f" Ebbinghaus S={stab} half-life={hl_ebb} R²={r2_ebb}")
-
-
-
-
-def _print_single_seed_results(display: dict, backends: list) -> None:
- checkpoints = display["checkpoints"]
- col = " ".join(f"T={c:3d}" for c in checkpoints)
- sep = "-" * 65
-
- print(f"\nCONTENT Recall@T")
- print(f" {'Backend':<14} {col}")
- print(sep)
- for name in backends:
- if name not in display:
- continue
- vals = " ".join(f"{v*100:5.1f}%" for v in display[name]["recall"])
- print(f" {name:<14} {vals}")
-
- if display.get("has_llm_eval"):
- print(f"\nLLM Recall@T (answer+judge)")
- print(f" {'Backend':<14} {col}")
- print(sep)
- for name in backends:
- if name not in display:
- continue
- llm_vals = display[name].get("llm_recall", [])
- vals = " ".join(
- f"{v*100:5.1f}%" if v is not None else " N/A "
- for v in llm_vals
- )
- print(f" {name:<14} {vals}")
-
- print(f"\n Gap = Content Recall - LLM Recall")
- print(f" {'Backend':<14} {col}")
- print(sep)
- for name in backends:
- if name not in display:
- continue
- content = display[name]["recall"]
- llm = display[name].get("llm_recall", [None]*len(content))
- vals = " ".join(
- f"{(c - l)*100:+5.1f}%" if l is not None else " N/A "
- for c, l in zip(content, llm)
- )
- print(f" {name:<14} {vals}")
-
- print(f"\n Tokens/Query @ T={checkpoints[-1]}")
- print("-" * 65)
- for name in backends:
- if name not in display:
- continue
- tok = display[name]["tokens"][-1]
- print(f" {name:<14} {tok:,}")
-
-
-def _print_multi_seed_results(agg: dict, backends: list) -> None:
- checkpoints = agg["checkpoints"]
- n = agg["n_seeds"]
- sep = "-" * 72
-
- print(f"\nCONTENT Recall@T (mean +/- std, n={n} personas)")
- print(f" {'Backend':<14} " + " ".join(f"T={c:3d}" for c in checkpoints))
- print(sep)
- for name in backends:
- if name not in agg:
- continue
- cols = []
- for stat in agg[name]["recall"]:
- cols.append(f"{stat['mean']*100:5.1f}+/-{stat['std']*100:4.1f}%")
- print(f" {name:<14} " + " ".join(cols))
-
- print(f"\n Tokens/Query @ T={checkpoints[-1]} (mean +/- std)")
- print(sep)
- for name in backends:
- if name not in agg:
- continue
- stat = agg[name]["tokens"][-1]
- print(f" {name:<14} {stat['mean']:,.0f} +/- {stat['std']:,.0f}")
-
-
-def _save(data: dict, path: str) -> None:
- with open(path, "w") as fh:
- json.dump(data, fh, indent=2)
- print(f"\nResults saved -> {path}")
+"""Backward-compatible entry point. Prefer `memorylens` after `pip install memorylens`."""
+from memorylens.cli import main
if __name__ == "__main__":
main()
diff --git a/memorylens/__init__.py b/memorylens/__init__.py
new file mode 100644
index 0000000..7d4e3ee
--- /dev/null
+++ b/memorylens/__init__.py
@@ -0,0 +1,22 @@
+"""MemoryLens — a benchmark for measuring LLM memory decay across conversation turns."""
+
+__version__ = "0.4.0"
+
+from memorylens.memory.base import BaseMemory
+from memorylens.simulator.facts import Fact
+from memorylens.evaluation.benchmark import (
+ run_benchmark,
+ run_benchmark_multi_seed,
+ results_to_display_dict,
+ VALID_BACKENDS,
+)
+
+__all__ = [
+ "__version__",
+ "BaseMemory",
+ "Fact",
+ "run_benchmark",
+ "run_benchmark_multi_seed",
+ "results_to_display_dict",
+ "VALID_BACKENDS",
+]
diff --git a/memorylens/api.py b/memorylens/api.py
new file mode 100644
index 0000000..06821f8
--- /dev/null
+++ b/memorylens/api.py
@@ -0,0 +1,133 @@
+"""
+REST API exposing the MemoryLens benchmark pipeline.
+
+Run:
+ pip install "memorylens[server]"
+ uvicorn memorylens.api:app
+
+Benchmark runs are CPU-bound (embedding model), so POST /v1/benchmarks returns
+202 with a job id immediately; poll GET /v1/benchmarks/{job_id} for the result.
+"""
+
+from __future__ import annotations
+
+import threading
+import uuid
+from typing import Dict, List, Optional
+
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel, Field
+
+from memorylens import __version__, VALID_BACKENDS
+from memorylens.simulator.scenarios import SCENARIOS, get_scenario
+from memorylens.memory.decay import _REGISTRY as DECAY_REGISTRY
+
+app = FastAPI(
+ title="MemoryLens API",
+ version=__version__,
+ description="Run LLM memory-decay benchmarks over HTTP.",
+)
+
+# ponytail: in-memory job store + one thread per job; move to a real queue and
+# persistent store when concurrent multi-user runs are actually needed.
+_jobs: Dict[str, Dict] = {}
+_jobs_lock = threading.Lock()
+
+
+class BenchmarkRequest(BaseModel):
+ turns: int = Field(100, ge=10, le=1000)
+ checkpoints: List[int] = Field(default=[10, 25, 50, 75, 100])
+ backends: List[str] = Field(default=["naive", "rag", "cascading"])
+ scenario: str = "default"
+ decay: str = "ebbinghaus"
+ seeds: int = Field(1, ge=1, le=5)
+
+
+def _validate(req: BenchmarkRequest) -> None:
+ unknown = [b for b in req.backends if b not in VALID_BACKENDS]
+ if unknown:
+ raise HTTPException(422, f"Unknown backends {unknown}. Valid: {VALID_BACKENDS}")
+ if req.scenario not in SCENARIOS:
+ raise HTTPException(422, f"Unknown scenario '{req.scenario}'. Valid: {list(SCENARIOS)}")
+ if req.decay not in DECAY_REGISTRY:
+ raise HTTPException(422, f"Unknown decay '{req.decay}'. Valid: {list(DECAY_REGISTRY)}")
+ if not req.checkpoints:
+ raise HTTPException(422, "checkpoints must not be empty")
+ out_of_range = [c for c in req.checkpoints if c < 1 or c > req.turns]
+ if out_of_range:
+ raise HTTPException(
+ 422, f"checkpoints {out_of_range} outside run horizon 1..{req.turns}"
+ )
+
+
+def _run_job(job_id: str, req: BenchmarkRequest) -> None:
+ from memorylens.evaluation.benchmark import (
+ run_benchmark, run_benchmark_multi_seed, results_to_display_dict,
+ )
+
+ scenario = get_scenario(req.scenario)
+ try:
+ if req.seeds > 1:
+ results = run_benchmark_multi_seed(
+ n_seeds=req.seeds,
+ total_turns=req.turns,
+ eval_checkpoints=sorted(req.checkpoints),
+ backends=req.backends,
+ decay=req.decay,
+ persona_pool=scenario.persona_pool,
+ filler_turns=scenario.filler_turns,
+ )
+ else:
+ raw = run_benchmark(
+ total_turns=req.turns,
+ eval_checkpoints=sorted(req.checkpoints),
+ facts=scenario.facts,
+ backends=req.backends,
+ decay=req.decay,
+ filler_turns=scenario.filler_turns,
+ )
+ results = results_to_display_dict(raw)
+ with _jobs_lock:
+ _jobs[job_id].update(status="completed", results=results)
+ except Exception as e:
+ with _jobs_lock:
+ _jobs[job_id].update(status="failed", error=str(e))
+
+
+@app.get("/health")
+def health() -> Dict:
+ return {"status": "ok", "version": __version__}
+
+
+@app.get("/v1/backends")
+def backends() -> Dict:
+ return {"data": VALID_BACKENDS}
+
+
+@app.get("/v1/scenarios")
+def scenarios() -> Dict:
+ return {
+ "data": [
+ {"name": s.name, "description": s.description, "personas": len(s.persona_pool)}
+ for s in SCENARIOS.values()
+ ]
+ }
+
+
+@app.post("/v1/benchmarks", status_code=202)
+def create_benchmark(req: BenchmarkRequest) -> Dict:
+ _validate(req)
+ job_id = uuid.uuid4().hex
+ with _jobs_lock:
+ _jobs[job_id] = {"status": "running", "request": req.model_dump()}
+ threading.Thread(target=_run_job, args=(job_id, req), daemon=True).start()
+ return {"data": {"job_id": job_id, "status": "running"}}
+
+
+@app.get("/v1/benchmarks/{job_id}")
+def get_benchmark(job_id: str) -> Dict:
+ with _jobs_lock:
+ job = _jobs.get(job_id)
+ if job is None:
+ raise HTTPException(404, f"No job with id '{job_id}'")
+ return {"data": dict(job)}
diff --git a/memorylens/cli.py b/memorylens/cli.py
new file mode 100644
index 0000000..3e3484d
--- /dev/null
+++ b/memorylens/cli.py
@@ -0,0 +1,316 @@
+"""
+MemoryLens CLI (installed as the `memorylens` command; `python main.py` also works)
+
+Content-only, no API key needed:
+ memorylens
+
+Full LLM evaluation (auto-detects an available provider):
+ memorylens --llm
+ memorylens --llm --provider openai (groq | openai | anthropic | openrouter | ollama)
+
+Multi-seed benchmark (reports mean +/- std across N personas):
+ memorylens --seeds 5
+
+Domain scenarios:
+ memorylens --scenario edtech (default | edtech | support | medical)
+
+Decay formula ablation (compare forgetting curve variants):
+ memorylens --decay ebbinghaus (default -- Ebbinghaus 1885)
+ memorylens --decay exponential | linear | default
+
+Forgetting-curve analysis (fit Ebbinghaus + exponential to recall@T data):
+ memorylens --seeds 5 --fit-curves
+
+Other options:
+ memorylens --turns 50 --backends naive rag graph --log
+ memorylens --list-providers
+"""
+
+import os
+import sys
+import json
+import argparse
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(
+ description="MemoryLens: End-to-end LLM Memory Decay Benchmark",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ )
+ parser.add_argument("--turns", type=int, default=100)
+ parser.add_argument("--checkpoints", nargs="+", type=int,
+ default=[10, 25, 50, 75, 100])
+ parser.add_argument("--backends", nargs="+",
+ default=["naive", "rag", "cascading"],
+ help="naive | rag | rag_chunked | cascading | summary | entity | graph | faiss")
+ parser.add_argument("--output", type=str, default="results.json")
+ parser.add_argument("--log", action="store_true",
+ help="Save run to experiment_logs/")
+ parser.add_argument("--llm", action="store_true",
+ help="Run real LLM evaluation pass (needs an API key or Ollama)")
+ parser.add_argument("--provider", type=str, default=None,
+ help="Force a provider: groq | openai | anthropic | openrouter | ollama")
+ parser.add_argument("--list-providers", action="store_true",
+ help="Print available providers and exit")
+ parser.add_argument("--seeds", type=int, default=1,
+ help="Number of persona seeds to run (max 5). >1 reports mean +/- std.")
+ parser.add_argument("--decay", type=str, default="ebbinghaus",
+ choices=["ebbinghaus", "exponential", "linear", "default"],
+ help="Temporal decay function for CascadingMemory warm tier")
+ parser.add_argument("--fit-curves", action="store_true",
+ help="After benchmarking, fit Ebbinghaus + exponential decay curves "
+ "to recall@T data and report half-life / stability / R²")
+ parser.add_argument("--scenario", type=str, default="default",
+ help="Conversation scenario: default | edtech | support | medical")
+ parser.add_argument("--list-scenarios", action="store_true",
+ help="Print available scenarios and exit")
+ args = parser.parse_args()
+
+ if args.list_scenarios:
+ from memorylens.simulator.scenarios import SCENARIOS
+ print("\nScenarios:")
+ for s in SCENARIOS.values():
+ print(f" {s.name:<10} {s.description}")
+ print()
+ sys.exit(0)
+
+ # ── List providers ────────────────────────────────────────────────────────
+ if args.list_providers:
+ from memorylens.utils.providers import list_available, _REGISTRY
+ available = list_available()
+ print("\nProvider status:")
+ for name in _REGISTRY:
+ status = "available" if name in available else "not available"
+ print(f" {name:<15} {status}")
+ print()
+ sys.exit(0)
+
+ # ── Resolve LLM provider ─────────────────────────────────────────────────
+ provider = None
+ if args.llm:
+ from memorylens.utils.providers import get_provider
+ try:
+ provider = get_provider(args.provider)
+ except (ValueError, RuntimeError) as e:
+ print(f"ERROR: {e}")
+ sys.exit(1)
+
+ if provider is None:
+ print(
+ "ERROR: --llm requested but no provider is available.\n"
+ " Set one of: GROQ_API_KEY, OPENAI_API_KEY, ANTHROPIC_API_KEY, "
+ "OPENROUTER_API_KEY\n"
+ " or start Ollama locally.\n"
+ " Run --list-providers to see status."
+ )
+ sys.exit(1)
+
+ multi_seed = args.seeds > 1
+
+ # ── Resolve scenario ─────────────────────────────────────────────────────
+ from memorylens.simulator.scenarios import get_scenario
+ try:
+ scenario = get_scenario(args.scenario)
+ except ValueError as e:
+ print(f"ERROR: {e}")
+ sys.exit(1)
+
+ # ── Banner ───────────────────────────────────────────────────────────────
+ print("=" * 65)
+ print(" MemoryLens -- LLM Memory Decay Benchmark")
+ print("=" * 65)
+ print(f" Turns : {args.turns}")
+ print(f" Checkpoints : {sorted(args.checkpoints)}")
+ print(f" Backends : {args.backends}")
+ print(f" Decay : {args.decay}")
+ print(f" Scenario : {args.scenario}")
+ if multi_seed:
+ print(f" Seeds : {args.seeds} (multi-seed -- will report mean +/- std)")
+ print(f" LLM eval : {'ON (' + provider.name + ')' if provider else 'OFF (content-only)'}")
+ print("=" * 65)
+
+ # ── Run benchmark ─────────────────────────────────────────────────────────
+ if multi_seed:
+ from memorylens.evaluation.benchmark import run_benchmark_multi_seed
+ aggregated = run_benchmark_multi_seed(
+ n_seeds=args.seeds,
+ total_turns=args.turns,
+ eval_checkpoints=sorted(args.checkpoints),
+ backends=args.backends,
+ provider=provider,
+ decay=args.decay,
+ progress=print,
+ persona_pool=scenario.persona_pool,
+ filler_turns=scenario.filler_turns,
+ )
+ _print_multi_seed_results(aggregated, args.backends)
+ _save(aggregated, args.output)
+ if args.log:
+ from memorylens.evaluation.logger import log_run
+ path = log_run(aggregated, {
+ "total_turns": args.turns,
+ "backends": args.backends,
+ "seeds": args.seeds,
+ "decay": args.decay,
+ "scenario": scenario.name,
+ "provider": provider.name if provider else None,
+ })
+ print(f"Experiment logged -> {path}")
+ else:
+ from memorylens.evaluation.benchmark import run_benchmark, results_to_display_dict
+ raw = run_benchmark(
+ total_turns=args.turns,
+ eval_checkpoints=sorted(args.checkpoints),
+ facts=scenario.facts,
+ backends=args.backends,
+ provider=provider,
+ decay=args.decay,
+ progress=print,
+ filler_turns=scenario.filler_turns,
+ )
+ display = results_to_display_dict(raw)
+ _print_single_seed_results(display, args.backends)
+ _save(display, args.output)
+ if args.log:
+ from memorylens.evaluation.logger import log_run
+ path = log_run(display, {
+ "total_turns": args.turns,
+ "backends": args.backends,
+ "decay": args.decay,
+ "scenario": scenario.name,
+ "provider": provider.name if provider else None,
+ })
+ print(f"Experiment logged -> {path}")
+
+ # ── Forgetting-curve analysis ─────────────────────────────────────────────
+ if args.fit_curves:
+ from memorylens.evaluation.stats import fit_forgetting_curve
+ checkpoints = sorted(args.checkpoints)
+ print("\nFORGETTING CURVE FIT (Ebbinghaus + Exponential)")
+ print("-" * 65)
+ if multi_seed:
+ for name in args.backends:
+ if name not in aggregated:
+ continue
+ mean_recalls = [stat["mean"] for stat in aggregated[name]["recall"]]
+ fit = fit_forgetting_curve(checkpoints, mean_recalls)
+ _print_curve_fit(name, fit)
+ else:
+ for name in args.backends:
+ if name not in display:
+ continue
+ fit = fit_forgetting_curve(checkpoints, display[name]["recall"])
+ _print_curve_fit(name, fit)
+
+ print("Visualise: streamlit run dashboard.py")
+
+
+# ── Output helpers ────────────────────────────────────────────────────────────
+
+
+def _print_curve_fit(backend: str, fit: dict) -> None:
+ if "error" in fit:
+ print(f" {backend:<14} {fit['error']}")
+ return
+ exp = fit["exponential"]
+ ebb = fit["ebbinghaus"]
+ hl_exp = f"{exp['half_life']:.1f} turns" if exp["half_life"] is not None else "N/A"
+ hl_ebb = f"{ebb['half_life']:.1f} turns" if ebb["half_life"] is not None else "N/A"
+ r2_exp = f"{exp['r2']:.3f}" if exp["r2"] is not None else "N/A"
+ r2_ebb = f"{ebb['r2']:.3f}" if ebb["r2"] is not None else "N/A"
+ stab = f"{ebb['stability']:.4f}" if ebb["stability"] is not None else "N/A"
+ k_val = f"{exp['k']:.6f}" if exp["k"] is not None else "N/A"
+ print(f" {backend}")
+ print(f" Exponential k={k_val} half-life={hl_exp} R²={r2_exp}")
+ print(f" Ebbinghaus S={stab} half-life={hl_ebb} R²={r2_ebb}")
+
+
+
+
+def _print_single_seed_results(display: dict, backends: list) -> None:
+ checkpoints = display["checkpoints"]
+ col = " ".join(f"T={c:3d}" for c in checkpoints)
+ sep = "-" * 65
+
+ print(f"\nCONTENT Recall@T")
+ print(f" {'Backend':<14} {col}")
+ print(sep)
+ for name in backends:
+ if name not in display:
+ continue
+ vals = " ".join(f"{v*100:5.1f}%" for v in display[name]["recall"])
+ print(f" {name:<14} {vals}")
+
+ if display.get("has_llm_eval"):
+ print(f"\nLLM Recall@T (answer+judge)")
+ print(f" {'Backend':<14} {col}")
+ print(sep)
+ for name in backends:
+ if name not in display:
+ continue
+ llm_vals = display[name].get("llm_recall", [])
+ vals = " ".join(
+ f"{v*100:5.1f}%" if v is not None else " N/A "
+ for v in llm_vals
+ )
+ print(f" {name:<14} {vals}")
+
+ print(f"\n Gap = Content Recall - LLM Recall")
+ print(f" {'Backend':<14} {col}")
+ print(sep)
+ for name in backends:
+ if name not in display:
+ continue
+ content = display[name]["recall"]
+ llm = display[name].get("llm_recall", [None]*len(content))
+ vals = " ".join(
+ f"{(c - l)*100:+5.1f}%" if l is not None else " N/A "
+ for c, l in zip(content, llm)
+ )
+ print(f" {name:<14} {vals}")
+
+ print(f"\n Tokens/Query @ T={checkpoints[-1]}")
+ print("-" * 65)
+ for name in backends:
+ if name not in display:
+ continue
+ tok = display[name]["tokens"][-1]
+ print(f" {name:<14} {tok:,}")
+
+
+def _print_multi_seed_results(agg: dict, backends: list) -> None:
+ checkpoints = agg["checkpoints"]
+ n = agg["n_seeds"]
+ sep = "-" * 72
+
+ print(f"\nCONTENT Recall@T (mean +/- std, n={n} personas)")
+ print(f" {'Backend':<14} " + " ".join(f"T={c:3d}" for c in checkpoints))
+ print(sep)
+ for name in backends:
+ if name not in agg:
+ continue
+ cols = []
+ for stat in agg[name]["recall"]:
+ cols.append(f"{stat['mean']*100:5.1f}+/-{stat['std']*100:4.1f}%")
+ print(f" {name:<14} " + " ".join(cols))
+
+ print(f"\n Tokens/Query @ T={checkpoints[-1]} (mean +/- std)")
+ print(sep)
+ for name in backends:
+ if name not in agg:
+ continue
+ stat = agg[name]["tokens"][-1]
+ print(f" {name:<14} {stat['mean']:,.0f} +/- {stat['std']:,.0f}")
+
+
+def _save(data: dict, path: str) -> None:
+ with open(path, "w") as fh:
+ json.dump(data, fh, indent=2)
+ print(f"\nResults saved -> {path}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/evaluation/__init__.py b/memorylens/evaluation/__init__.py
similarity index 100%
rename from evaluation/__init__.py
rename to memorylens/evaluation/__init__.py
diff --git a/evaluation/benchmark.py b/memorylens/evaluation/benchmark.py
similarity index 87%
rename from evaluation/benchmark.py
rename to memorylens/evaluation/benchmark.py
index f9b6d14..86cee9c 100644
--- a/evaluation/benchmark.py
+++ b/memorylens/evaluation/benchmark.py
@@ -1,40 +1,42 @@
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, List, Dict, Callable, Optional
-from simulator.facts import Fact, BENCHMARK_FACTS
-from simulator.conversation import generate_conversation
-from memory.naive import NaiveMemory
-from memory.rag import RAGMemory
-from memory.rag_chunked import ChunkedRAGMemory
-from memory.cascading import CascadingTemporalMemory
-from memory.summary import SummaryMemory
-from memory.entity import EntityMemory
-from memory.base import BaseMemory
-from evaluation.metrics import (
+from memorylens.simulator.facts import Fact, BENCHMARK_FACTS
+from memorylens.simulator.conversation import generate_conversation
+from memorylens.memory.naive import NaiveMemory
+from memorylens.memory.rag import RAGMemory
+from memorylens.memory.rag_chunked import ChunkedRAGMemory
+from memorylens.memory.cascading import CascadingTemporalMemory
+from memorylens.memory.summary import SummaryMemory
+from memorylens.memory.entity import EntityMemory
+from memorylens.memory.graph import GraphMemory
+from memorylens.memory.base import BaseMemory
+from memorylens.evaluation.metrics import (
recall_at_t, temporal_drift_score, memory_noise_ratio, precision_at_k,
- cascade_efficiency, llm_recall_at_t, llm_temporal_drift,
+ contradiction_score, cascade_efficiency, llm_recall_at_t, llm_temporal_drift,
)
if TYPE_CHECKING:
- from utils.providers import LLMProvider
+ from memorylens.utils.providers import LLMProvider
OFF_TOPIC_QUERY = "What is the best sorting algorithm for large datasets?"
_NAN = float("nan")
-VALID_BACKENDS = ["naive", "rag", "rag_chunked", "cascading", "summary", "entity"]
+VALID_BACKENDS = ["naive", "rag", "rag_chunked", "cascading", "summary", "entity", "graph", "faiss"]
@dataclass
class CheckpointResult:
turn: int
# ── Content-based (always available, fast) ───────────────────────────────
- recall: float
- precision: float
- drift: float
- noise: float
- tokens: int
- cascade_eff: float = 1.0
+ recall: float
+ precision: float
+ drift: float
+ noise: float
+ tokens: int
+ contradiction: float = 0.0
+ cascade_eff: float = 1.0
# ── LLM-based (available when a provider is configured) ──────────────────
llm_recall: float = _NAN
llm_drift: float = _NAN
@@ -63,6 +65,11 @@ def _make_memory(name: str, decay: str = "ebbinghaus") -> BaseMemory:
return SummaryMemory(window_size=20, use_llm=None)
if name == "entity":
return EntityMemory()
+ if name == "graph":
+ return GraphMemory()
+ if name == "faiss":
+ from memorylens.memory.vector_faiss import FAISSMemory
+ return FAISSMemory()
raise ValueError(
f"Unknown backend: '{name}'. "
f"Choose from: {VALID_BACKENDS}"
@@ -164,13 +171,18 @@ def run_benchmark(
drift_facts = [
f for f in active_facts
- if f.updated_at and f.updated_at <= turn
+ if f.updated_at is not None and f.updated_value and f.updated_at <= turn
]
avg_drift = (
sum(temporal_drift_score(memory, f, turn)["drift"] for f in drift_facts)
/ len(drift_facts)
if drift_facts else 0.0
)
+ avg_contradiction = (
+ sum(contradiction_score(memory, f, turn)["contradiction"] for f in drift_facts)
+ / len(drift_facts)
+ if drift_facts else 0.0
+ )
noise = memory_noise_ratio(memory, OFF_TOPIC_QUERY, known_values, turn)
@@ -216,6 +228,7 @@ def run_benchmark(
drift = round(avg_drift, 4),
noise = round(noise, 4),
tokens = int(avg_tokens),
+ contradiction = round(avg_contradiction, 4),
cascade_eff = round(eff, 4),
llm_recall = round(llm_recall_val, 4) if has_llm else _NAN,
llm_drift = round(llm_drift_val, 4) if has_llm else _NAN,
@@ -248,8 +261,8 @@ def run_benchmark_multi_seed(
Returns a nested dict ready for results_to_multi_seed_dict().
"""
- from simulator.personas import PERSONA_POOL
- from evaluation.stats import aggregate_checkpoint_series
+ from memorylens.simulator.personas import PERSONA_POOL
+ from memorylens.evaluation.stats import aggregate_checkpoint_series
if eval_checkpoints is None:
eval_checkpoints = [10, 25, 50, 75, 100]
@@ -287,7 +300,7 @@ def run_benchmark_multi_seed(
),
}
- metric_keys = ["recall", "precision", "drift", "noise", "tokens", "cascade_eff"]
+ metric_keys = ["recall", "precision", "drift", "noise", "tokens", "contradiction", "cascade_eff"]
for backend_name in backends:
runs_for_backend = [run[backend_name] for run in all_runs if backend_name in run]
@@ -323,7 +336,7 @@ def run_benchmark_multi_seed(
[v for v in row if v is not None]
for row in series
]
- from evaluation.stats import aggregate_checkpoint_series as acs
+ from memorylens.evaluation.stats import aggregate_checkpoint_series as acs
agg[llm_metric] = acs([[r[i] if i < len(r) else 0.0 for r in filtered]
for i in range(len(checkpoints))])
@@ -349,6 +362,7 @@ def results_to_display_dict(results: Dict[str, BackendResult]) -> Dict:
"precision": [cp_map[t].precision for t in checkpoints if t in cp_map],
"drift": [cp_map[t].drift for t in checkpoints if t in cp_map],
"noise": [cp_map[t].noise for t in checkpoints if t in cp_map],
+ "contradiction": [cp_map[t].contradiction for t in checkpoints if t in cp_map],
"tokens": [cp_map[t].tokens for t in checkpoints if t in cp_map],
"cascade_eff": [cp_map[t].cascade_eff for t in checkpoints if t in cp_map],
"llm_recall": [
diff --git a/evaluation/llm_judge.py b/memorylens/evaluation/llm_judge.py
similarity index 95%
rename from evaluation/llm_judge.py
rename to memorylens/evaluation/llm_judge.py
index 11165db..b2573a6 100644
--- a/evaluation/llm_judge.py
+++ b/memorylens/evaluation/llm_judge.py
@@ -7,9 +7,9 @@
"""
from typing import Dict, List, Optional
-from utils.llm import chat
-from memory.base import BaseMemory
-from simulator.facts import Fact
+from memorylens.utils.llm import chat
+from memorylens.memory.base import BaseMemory
+from memorylens.simulator.facts import Fact
JUDGE_SYSTEM = """You are a strict evaluator. Given a question, the correct answer, and a model's response,
output ONLY a JSON object with two keys:
diff --git a/evaluation/logger.py b/memorylens/evaluation/logger.py
similarity index 54%
rename from evaluation/logger.py
rename to memorylens/evaluation/logger.py
index a5e0394..da6923b 100644
--- a/evaluation/logger.py
+++ b/memorylens/evaluation/logger.py
@@ -1,6 +1,8 @@
"""
-Experiment logger — writes benchmark results to CSV and JSON for
-reproducible research and arXiv submission.
+Experiment logger — persists benchmark runs as JSON + CSV + SQLite.
+
+JSON files remain the interchange format; SQLite (memorylens.db) is the
+queryable store used by list_runs()/get_run_results().
"""
import csv
@@ -10,10 +12,7 @@
from datetime import datetime
from typing import Any, Dict, Optional
-from utils.storage import Storage
-
-
-LOG_DIR = os.path.join(os.path.dirname(__file__), "..", "experiment_logs")
+from memorylens.utils.storage import Storage, LOG_DIR
def _ensure_dir() -> str:
@@ -56,29 +55,44 @@ def _append_csv_summary(display_data: Dict, config: Dict, run_id: str) -> None:
file_exists = os.path.exists(csv_path)
checkpoints = display_data.get("checkpoints", [])
- backends = [k for k in display_data if k not in ("checkpoints", "has_llm_eval")]
+ backends = [k for k, v in display_data.items() if isinstance(v, dict)]
+
+ def _at(d: Dict, key: str, i: int):
+ series = d.get(key, [])
+ return series[i] if i < len(series) else ""
rows = []
for backend in backends:
d = display_data[backend]
for i, cp in enumerate(checkpoints):
rows.append({
- "run_id": run_id,
- "backend": backend,
- "turn": cp,
- "recall": d["recall"][i] if i < len(d["recall"]) else "",
- "precision": d["precision"][i] if i < len(d["precision"]) else "",
- "drift": d["drift"][i] if i < len(d["drift"]) else "",
- "noise": d["noise"][i] if i < len(d["noise"]) else "",
- "tokens": d["tokens"][i] if i < len(d["tokens"]) else "",
- "total_turns": config.get("total_turns", ""),
+ "run_id": run_id,
+ "backend": backend,
+ "turn": cp,
+ "recall": _at(d, "recall", i),
+ "precision": _at(d, "precision", i),
+ "drift": _at(d, "drift", i),
+ "noise": _at(d, "noise", i),
+ "contradiction": _at(d, "contradiction", i),
+ "tokens": _at(d, "tokens", i),
+ "total_turns": config.get("total_turns", ""),
})
if not rows:
return
+ fieldnames = list(rows[0].keys())
+ if file_exists:
+ with open(csv_path, newline="") as fh:
+ header = fh.readline().strip().split(",")
+ if header != fieldnames:
+ # Schema changed (new metric column) — rotate the old file instead
+ # of appending misaligned rows.
+ os.replace(csv_path, csv_path.replace(".csv", "_legacy.csv"))
+ file_exists = False
+
with open(csv_path, "a", newline="") as fh:
- writer = csv.DictWriter(fh, fieldnames=rows[0].keys())
+ writer = csv.DictWriter(fh, fieldnames=fieldnames)
if not file_exists:
writer.writeheader()
writer.writerows(rows)
@@ -87,19 +101,16 @@ def _append_csv_summary(display_data: Dict, config: Dict, run_id: str) -> None:
def list_runs() -> list:
"""Return metadata for all logged runs, newest first.
- Queries SQLite database first; falls back to filesystem scan
- when the database doesn't exist yet.
-
- Returns a list of dicts, each with ``run_id``, ``timestamp``,
- and ``config`` keys (unified schema for both storage backends).
+ Queries SQLite first; falls back to a filesystem scan when the database
+ doesn't exist yet. Each entry has ``run_id``, ``timestamp``, and
+ ``config`` keys (unified schema for both storage backends).
"""
- # ── Try SQLite first ────────────────────────────────────────────────────
store: Optional[Storage] = None
try:
store = Storage()
runs = store.list_runs(limit=50)
# Once any SQLite run exists, filesystem logs are bypassed.
- # Run python utils/migrate_legacy_logs.py to import them.
+ # Run python -m memorylens.utils.migrate_legacy_logs to import them.
if runs:
return runs
except Exception:
@@ -112,7 +123,7 @@ def list_runs() -> list:
log_dir = _ensure_dir()
runs = []
for fname in sorted(os.listdir(log_dir), reverse=True):
- if fname.endswith(".json") and fname != "runs_summary.csv":
+ if fname.endswith(".json"):
fpath = os.path.join(log_dir, fname)
with open(fpath) as fh:
data = json.load(fh)
@@ -124,3 +135,23 @@ def list_runs() -> list:
})
return runs
+
+def get_run_results(run_id: str) -> Optional[Dict]:
+ """Return the display_data dict for a run — SQLite first, JSON fallback."""
+ store: Optional[Storage] = None
+ try:
+ store = Storage()
+ results = store.get_run(run_id)
+ if results is not None:
+ return results
+ except Exception:
+ pass
+ finally:
+ if store is not None:
+ store.close()
+
+ json_path = os.path.join(LOG_DIR, f"{run_id}.json")
+ if os.path.exists(json_path):
+ with open(json_path) as fh:
+ return json.load(fh).get("results")
+ return None
diff --git a/evaluation/metrics.py b/memorylens/evaluation/metrics.py
similarity index 84%
rename from evaluation/metrics.py
rename to memorylens/evaluation/metrics.py
index cdf2ce9..5c7856a 100644
--- a/evaluation/metrics.py
+++ b/memorylens/evaluation/metrics.py
@@ -1,9 +1,9 @@
from typing import TYPE_CHECKING, Dict, List, Optional
-from memory.base import BaseMemory
-from simulator.facts import Fact
+from memorylens.memory.base import BaseMemory
+from memorylens.simulator.facts import Fact
if TYPE_CHECKING:
- from utils.providers import LLMProvider
+ from memorylens.utils.providers import LLMProvider
def recall_at_t(memory: BaseMemory, fact: Fact, current_turn: int) -> Dict:
@@ -31,7 +31,7 @@ def temporal_drift_score(memory: BaseMemory, fact: Fact, current_turn: int) -> D
Returns drift ∈ [0, 1]: 1 = context only shows stale data, 0 = fully updated.
Only applicable to facts that have an update.
"""
- if not fact.updated_at or current_turn < fact.updated_at:
+ if fact.updated_at is None or not fact.updated_value or current_turn < fact.updated_at:
return {"drift": 0.0, "applicable": False}
context = memory.get_context(fact.query_text(), current_turn)
@@ -55,6 +55,35 @@ def temporal_drift_score(memory: BaseMemory, fact: Fact, current_turn: int) -> D
}
+def contradiction_score(memory: BaseMemory, fact: Fact, current_turn: int) -> Dict:
+ """
+ Contradiction — after a fact update, does the retrieved context surface
+ BOTH the old and the new value at once?
+
+ A context containing both "Bangalore" and "Mumbai" for the same fact key
+ forces the downstream LLM to arbitrate between conflicting values — a
+ distinct failure mode from drift (which measures stale-only retrieval).
+
+ Returns contradiction ∈ {0.0, 1.0}; applicable only after fact.updated_at.
+ """
+ if fact.updated_at is None or not fact.updated_value or current_turn < fact.updated_at:
+ return {"contradiction": 0.0, "applicable": False}
+
+ context = memory.get_context(fact.query_text(), current_turn)
+ old_val = fact.value.lower()
+ new_val = (fact.updated_value or "").lower()
+
+ old_present = any(old_val in m.get("content", "").lower() for m in context)
+ new_present = any(new_val in m.get("content", "").lower() for m in context)
+
+ return {
+ "contradiction": 1.0 if (old_present and new_present) else 0.0,
+ "old_present": old_present,
+ "new_present": new_present,
+ "applicable": True,
+ }
+
+
def memory_noise_ratio(memory: BaseMemory, off_topic_query: str, known_facts: List[str], current_turn: int) -> float:
"""
Memory Noise Ratio — of the retrieved context chunks, what fraction is irrelevant?
@@ -170,7 +199,7 @@ def llm_recall_at_t(
judge_verdict : str — 'correct' | 'wrong' | 'error'
tokens : int — context token estimate
"""
- from utils.providers import _clean_messages
+ from memorylens.utils.providers import _clean_messages
context = memory.get_context(fact.query_text(), current_turn)
expected = fact.current_value(current_turn)
@@ -229,9 +258,9 @@ def llm_temporal_drift(
Only meaningful after fact.updated_at has passed.
"""
- from utils.providers import _clean_messages
+ from memorylens.utils.providers import _clean_messages
- if not fact.updated_at or current_turn < fact.updated_at:
+ if fact.updated_at is None or not fact.updated_value or current_turn < fact.updated_at:
return {"llm_drift": 0.0, "applicable": False}
context = memory.get_context(fact.query_text(), current_turn)
diff --git a/evaluation/stats.py b/memorylens/evaluation/stats.py
similarity index 100%
rename from evaluation/stats.py
rename to memorylens/evaluation/stats.py
diff --git a/memory/__init__.py b/memorylens/memory/__init__.py
similarity index 100%
rename from memory/__init__.py
rename to memorylens/memory/__init__.py
diff --git a/memory/base.py b/memorylens/memory/base.py
similarity index 100%
rename from memory/base.py
rename to memorylens/memory/base.py
diff --git a/memory/cascading.py b/memorylens/memory/cascading.py
similarity index 94%
rename from memory/cascading.py
rename to memorylens/memory/cascading.py
index da57dcc..3d13287 100644
--- a/memory/cascading.py
+++ b/memorylens/memory/cascading.py
@@ -3,7 +3,7 @@
import numpy as np
from .base import BaseMemory
from .decay import get_decay_fn, decay_ebbinghaus
-from utils.embeddings import embed, top_k_indices
+from memorylens.utils.embeddings import embed, top_k_indices
def _extractive_summary(messages: List[Dict], max_chars: int = 400) -> str:
@@ -25,7 +25,7 @@ def _extractive_summary(messages: List[Dict], max_chars: int = 400) -> str:
lines = update_lines + injection_lines
summary = " | ".join(lines)
- return summary[:max_chars] if summary else "No key facts."
+ return summary[:max_chars]
def _parse_update(content: str) -> Optional[Tuple[str, str]]:
@@ -140,7 +140,8 @@ def _cascade_warm(self) -> None:
self.warm_embs = self.warm_embs[-self.warm_size :]
summary = _extractive_summary(overflow)
- self.cold.append(summary)
+ if summary:
+ self.cold.append(summary)
# Patch all cold entries with every known fact update so no stale
# values survive compression into the cold tier.
@@ -148,8 +149,10 @@ def _cascade_warm(self) -> None:
self.cold = _patch_cold_with_update(self.cold, key_name, new_val)
if len(self.cold) > self.cold_max:
- # Merge oldest two; newer content first so it survives truncation
- merged = self.cold[1] + " | " + self.cold[0]
+ # Merge oldest-first: early facts stay at the head and survive
+ # truncation. Stale values are already rewritten in place by
+ # _patch_cold_with_update, so newer text needs no priority here.
+ merged = self.cold[0] + " | " + self.cold[1]
self.cold = [merged[:600]] + self.cold[2:]
def get_context(self, query: str, current_turn: int) -> List[Dict]:
diff --git a/memory/decay.py b/memorylens/memory/decay.py
similarity index 100%
rename from memory/decay.py
rename to memorylens/memory/decay.py
diff --git a/memory/entity.py b/memorylens/memory/entity.py
similarity index 100%
rename from memory/entity.py
rename to memorylens/memory/entity.py
diff --git a/memorylens/memory/graph.py b/memorylens/memory/graph.py
new file mode 100644
index 0000000..4c2d192
--- /dev/null
+++ b/memorylens/memory/graph.py
@@ -0,0 +1,70 @@
+"""
+GraphMemory — knowledge-graph backend built on NetworkX.
+
+Facts are stored as (user) -[relation]-> (value) edges in a directed graph.
+A fact update removes the old edge and inserts a new one, so retrieval always
+serialises the current state of the graph — stale values cannot survive.
+
+Extraction reuses the same local regex templates as EntityMemory, keeping the
+backend deterministic and free of LLM calls.
+"""
+
+from typing import Dict, List
+
+import networkx as nx
+
+from memorylens.memory.base import BaseMemory
+from memorylens.memory.entity import _extract_entity
+
+_USER = "user"
+
+
+class GraphMemory(BaseMemory):
+ """
+ Directed knowledge graph of user facts.
+
+ Nodes: the user plus one node per fact value.
+ Edges: (user, value) annotated with relation=fact key and the turn it was
+ asserted, so the graph doubles as a temporal provenance record.
+
+ # ponytail: single-hop user->value triples only; add entity-entity edges
+ # when a scenario actually needs multi-hop retrieval.
+ """
+
+ name = "graph"
+
+ def __init__(self) -> None:
+ self.graph = nx.DiGraph()
+ self.graph.add_node(_USER)
+
+ def add_message(self, role: str, content: str, turn: int) -> None:
+ if role != "user":
+ return
+ pair = _extract_entity(content)
+ if pair is None:
+ return
+ key, value = pair
+
+ for _, old_value, data in list(self.graph.out_edges(_USER, data=True)):
+ if data.get("relation") == key:
+ self.graph.remove_edge(_USER, old_value)
+ if self.graph.degree(old_value) == 0:
+ self.graph.remove_node(old_value)
+
+ self.graph.add_edge(_USER, value, relation=key, asserted_at=turn)
+
+ def get_context(self, query: str, current_turn: int) -> List[Dict]:
+ edges = self.graph.out_edges(_USER, data=True)
+ if not edges:
+ return []
+ lines = [f"my {data['relation']} is {value}" for _, value, data in edges]
+ return [
+ {
+ "role": "system",
+ "content": "[Knowledge graph facts] " + "; ".join(lines) + ".",
+ }
+ ]
+
+ def reset(self) -> None:
+ self.graph = nx.DiGraph()
+ self.graph.add_node(_USER)
diff --git a/memory/naive.py b/memorylens/memory/naive.py
similarity index 100%
rename from memory/naive.py
rename to memorylens/memory/naive.py
diff --git a/memory/rag.py b/memorylens/memory/rag.py
similarity index 95%
rename from memory/rag.py
rename to memorylens/memory/rag.py
index 86b55d3..baa2ee0 100644
--- a/memory/rag.py
+++ b/memorylens/memory/rag.py
@@ -1,7 +1,7 @@
from typing import List, Dict
import numpy as np
from .base import BaseMemory
-from utils.embeddings import embed, top_k_indices
+from memorylens.utils.embeddings import embed, top_k_indices
class RAGMemory(BaseMemory):
diff --git a/memory/rag_chunked.py b/memorylens/memory/rag_chunked.py
similarity index 98%
rename from memory/rag_chunked.py
rename to memorylens/memory/rag_chunked.py
index d021f82..a9c8095 100644
--- a/memory/rag_chunked.py
+++ b/memorylens/memory/rag_chunked.py
@@ -24,7 +24,7 @@
from typing import List, Dict, Tuple
import numpy as np
from .base import BaseMemory
-from utils.embeddings import embed, top_k_indices
+from memorylens.utils.embeddings import embed, top_k_indices
def _chunk_text(text: str, chunk_chars: int = 120, overlap_chars: int = 30) -> List[str]:
diff --git a/memory/summary.py b/memorylens/memory/summary.py
similarity index 99%
rename from memory/summary.py
rename to memorylens/memory/summary.py
index 717276d..e689a9d 100644
--- a/memory/summary.py
+++ b/memorylens/memory/summary.py
@@ -61,7 +61,7 @@ def _extractive_compress(messages: List[Dict], existing_summary: str = "") -> st
def _llm_compress(messages: List[Dict], existing_summary: str, model: str) -> str:
"""LLM-powered compression via Groq."""
- from utils.llm import chat
+ from memorylens.utils.llm import chat
batch_text = "\n".join(
f"{m['role'].upper()}: {m['content']}" for m in messages
diff --git a/memorylens/memory/vector_faiss.py b/memorylens/memory/vector_faiss.py
new file mode 100644
index 0000000..103e2a7
--- /dev/null
+++ b/memorylens/memory/vector_faiss.py
@@ -0,0 +1,67 @@
+"""
+FAISSMemory — production-grade vector retrieval backed by a FAISS index.
+
+Same retrieval semantics as RAGMemory (top-K cosine similarity + recency
+window) but the search runs inside faiss.IndexFlatIP instead of a NumPy
+matmul, which is what a production deployment would use at scale.
+
+Requires the optional dependency: pip install "memorylens[faiss]"
+"""
+
+from typing import Dict, List
+
+import numpy as np
+
+from memorylens.memory.base import BaseMemory
+from memorylens.utils.embeddings import embed
+
+
+def _require_faiss():
+ try:
+ import faiss
+ except ImportError as e:
+ raise ImportError(
+ "FAISSMemory requires faiss-cpu. Install it with: "
+ 'pip install "memorylens[faiss]"'
+ ) from e
+ return faiss
+
+
+class FAISSMemory(BaseMemory):
+ """Top-K inner-product search over normalised embeddings in a FAISS index."""
+
+ name = "faiss"
+
+ def __init__(self, top_k: int = 5, recency_window: int = 4):
+ self._faiss = _require_faiss()
+ self.top_k = top_k
+ self.recency_window = recency_window
+ self.messages: List[Dict] = []
+ self.index = None # created lazily once embedding dimension is known
+
+ def add_message(self, role: str, content: str, turn: int) -> None:
+ emb = embed([content]).astype(np.float32)
+ if self.index is None:
+ self.index = self._faiss.IndexFlatIP(emb.shape[1])
+ self.index.add(emb)
+ self.messages.append({"role": role, "content": content, "turn": turn})
+
+ def get_context(self, query: str, current_turn: int) -> List[Dict]:
+ if not self.messages:
+ return []
+
+ q_emb = embed([query]).astype(np.float32)
+ k = min(self.top_k, len(self.messages))
+ _, indices = self.index.search(q_emb, k)
+
+ semantic = {int(i) for i in indices[0] if i >= 0}
+ recency = set(range(max(0, len(self.messages) - self.recency_window),
+ len(self.messages)))
+ selected = sorted(semantic | recency)
+
+ return [{"role": self.messages[i]["role"], "content": self.messages[i]["content"]}
+ for i in selected]
+
+ def reset(self) -> None:
+ self.messages = []
+ self.index = None
diff --git a/simulator/__init__.py b/memorylens/simulator/__init__.py
similarity index 100%
rename from simulator/__init__.py
rename to memorylens/simulator/__init__.py
diff --git a/simulator/conversation.py b/memorylens/simulator/conversation.py
similarity index 100%
rename from simulator/conversation.py
rename to memorylens/simulator/conversation.py
diff --git a/simulator/facts.py b/memorylens/simulator/facts.py
similarity index 93%
rename from simulator/facts.py
rename to memorylens/simulator/facts.py
index a2c8ce0..c92875d 100644
--- a/simulator/facts.py
+++ b/memorylens/simulator/facts.py
@@ -11,7 +11,7 @@ class Fact:
updated_value: Optional[str] = None
def current_value(self, at_turn: int) -> str:
- if self.updated_at and at_turn >= self.updated_at and self.updated_value:
+ if self.updated_at is not None and at_turn >= self.updated_at and self.updated_value:
return self.updated_value
return self.value
diff --git a/simulator/personas.py b/memorylens/simulator/personas.py
similarity index 100%
rename from simulator/personas.py
rename to memorylens/simulator/personas.py
diff --git a/memorylens/simulator/scenarios/__init__.py b/memorylens/simulator/scenarios/__init__.py
new file mode 100644
index 0000000..96f006d
--- /dev/null
+++ b/memorylens/simulator/scenarios/__init__.py
@@ -0,0 +1,33 @@
+"""Scenario registry — all built-in benchmark scenarios."""
+
+from typing import Dict, List
+
+from memorylens.simulator.scenarios.base import Scenario
+from memorylens.simulator.conversation import FILLER_TURNS
+from memorylens.simulator.personas import PERSONA_POOL
+from memorylens.simulator.scenarios.edtech import EDTECH
+from memorylens.simulator.scenarios.customer_support import CUSTOMER_SUPPORT
+from memorylens.simulator.scenarios.medical import MEDICAL
+
+DEFAULT = Scenario(
+ name="default",
+ description="General tech Q&A conversation: personal profile facts with "
+ "mid-conversation updates (city, age).",
+ persona_pool=PERSONA_POOL,
+ filler_turns=FILLER_TURNS,
+)
+
+SCENARIOS: Dict[str, Scenario] = {
+ s.name: s for s in (DEFAULT, EDTECH, CUSTOMER_SUPPORT, MEDICAL)
+}
+
+
+def get_scenario(name: str) -> Scenario:
+ scenario = SCENARIOS.get(name)
+ if scenario is None:
+ raise ValueError(f"Unknown scenario '{name}'. Choose from: {list(SCENARIOS)}")
+ return scenario
+
+
+def list_scenarios() -> List[str]:
+ return list(SCENARIOS)
diff --git a/memorylens/simulator/scenarios/base.py b/memorylens/simulator/scenarios/base.py
new file mode 100644
index 0000000..5ece815
--- /dev/null
+++ b/memorylens/simulator/scenarios/base.py
@@ -0,0 +1,42 @@
+"""Scenario — a domain-specific benchmark definition (facts + filler turns)."""
+
+from dataclasses import dataclass
+from typing import List
+
+from memorylens.simulator.facts import Fact
+
+
+@dataclass(frozen=True)
+class Scenario:
+ """
+ A benchmark scenario bundles everything domain-specific:
+
+ persona_pool : one fact set per persona; persona 0 is the single-seed default.
+ All personas must share the same fact keys so results are
+ comparable across seeds.
+ filler_turns : domain-appropriate distractor questions fired between fact
+ injections.
+ """
+
+ name: str
+ description: str
+ persona_pool: List[List[Fact]]
+ filler_turns: List[str]
+
+ @property
+ def facts(self) -> List[Fact]:
+ return self.persona_pool[0]
+
+ def validate(self) -> None:
+ if not self.persona_pool:
+ raise ValueError(f"scenario '{self.name}': persona_pool is empty")
+ if not self.filler_turns:
+ raise ValueError(f"scenario '{self.name}': filler_turns is empty")
+ keys = {f.key for f in self.facts}
+ for i, persona in enumerate(self.persona_pool):
+ persona_keys = {f.key for f in persona}
+ if persona_keys != keys:
+ raise ValueError(
+ f"scenario '{self.name}': persona {i} fact keys {persona_keys} "
+ f"differ from persona 0 keys {keys}"
+ )
diff --git a/memorylens/simulator/scenarios/customer_support.py b/memorylens/simulator/scenarios/customer_support.py
new file mode 100644
index 0000000..a88807c
--- /dev/null
+++ b/memorylens/simulator/scenarios/customer_support.py
@@ -0,0 +1,82 @@
+"""
+Customer-support benchmark scenario.
+
+Models a customer working through support tickets with an AI agent over many
+turns. Facts cover account and product attributes; updates simulate real
+support lifecycles (plan upgrades, issue re-categorisation, renewal changes).
+"""
+
+from typing import List
+
+from memorylens.simulator.facts import Fact
+from memorylens.simulator.scenarios.base import Scenario
+
+
+SUPPORT_PERSONA_POOL: List[List[Fact]] = [
+ # Persona 0 — Dana Whitfield (baseline)
+ [
+ Fact("name", "Dana Whitfield", injected_at=0),
+ Fact("account tier", "basic plan", injected_at=1, updated_at=45, updated_value="premium plan"),
+ Fact("product", "CloudVault backup", injected_at=2),
+ Fact("order number", "ORD-88213", injected_at=4),
+ Fact("reported issue", "login failure", injected_at=6, updated_at=55, updated_value="billing discrepancy"),
+ Fact("operating system", "Windows 11", injected_at=8),
+ Fact("renewal date", "March 15", injected_at=10, updated_at=70, updated_value="September 15"),
+ Fact("preferred contact channel", "email", injected_at=12),
+ ],
+ # Persona 1 — Ravi Patel
+ [
+ Fact("name", "Ravi Patel", injected_at=0),
+ Fact("account tier", "trial plan", injected_at=1, updated_at=45, updated_value="business plan"),
+ Fact("product", "SyncDrive storage", injected_at=2),
+ Fact("order number", "ORD-55901", injected_at=4),
+ Fact("reported issue", "sync conflict", injected_at=6, updated_at=55, updated_value="quota exceeded error"),
+ Fact("operating system", "macOS Sonoma", injected_at=8),
+ Fact("renewal date", "June 1", injected_at=10, updated_at=70, updated_value="December 1"),
+ Fact("preferred contact channel", "phone", injected_at=12),
+ ],
+ # Persona 2 — Ingrid Olsen
+ [
+ Fact("name", "Ingrid Olsen", injected_at=0),
+ Fact("account tier", "family plan", injected_at=1, updated_at=45, updated_value="enterprise plan"),
+ Fact("product", "MailGuard filter", injected_at=2),
+ Fact("order number", "ORD-30447", injected_at=4),
+ Fact("reported issue", "spam leakage", injected_at=6, updated_at=55, updated_value="false positive blocking"),
+ Fact("operating system", "Ubuntu 24.04", injected_at=8),
+ Fact("renewal date", "January 20", injected_at=10, updated_at=70, updated_value="July 20"),
+ Fact("preferred contact channel", "live chat", injected_at=12),
+ ],
+]
+
+
+SUPPORT_FILLER_TURNS: List[str] = [
+ "How do I reset my password?",
+ "Where can I download my invoices?",
+ "Is there a mobile app for this service?",
+ "What is your refund policy?",
+ "How do I enable two-factor authentication?",
+ "Can I share my account with a family member?",
+ "What happens to my data if I cancel?",
+ "How do I export my data?",
+ "Why is the app asking me to re-login every day?",
+ "Do you offer discounts for annual billing?",
+ "How do I change the language of the interface?",
+ "What browsers do you officially support?",
+ "How long are backups retained?",
+ "Can I schedule automatic reports?",
+ "Is my data encrypted at rest?",
+ "How do I add another user to my workspace?",
+ "What is the difference between archive and delete?",
+ "How do I contact a human agent?",
+ "Are there API rate limits on my plan?",
+ "How do I update my payment method?",
+]
+
+
+CUSTOMER_SUPPORT = Scenario(
+ name="support",
+ description="Customer-support ticket conversation: account facts with "
+ "lifecycle updates (plan upgrade, issue re-categorisation, renewal change).",
+ persona_pool=SUPPORT_PERSONA_POOL,
+ filler_turns=SUPPORT_FILLER_TURNS,
+)
diff --git a/simulator/scenarios/edtech.py b/memorylens/simulator/scenarios/edtech.py
similarity index 94%
rename from simulator/scenarios/edtech.py
rename to memorylens/simulator/scenarios/edtech.py
index 30e6dd9..c381388 100644
--- a/simulator/scenarios/edtech.py
+++ b/memorylens/simulator/scenarios/edtech.py
@@ -8,12 +8,12 @@
The filler turns are domain-specific tutoring requests (concept explanations,
problem-solving help, study strategy questions) rather than generic tech Q&A,
making this a harder benchmark for memory systems that rely on keyword overlap.
-
-Closes #13 / #4.
"""
from typing import List
-from simulator.facts import Fact
+
+from memorylens.simulator.facts import Fact
+from memorylens.simulator.scenarios.base import Scenario
# ── Fact sets ─────────────────────────────────────────────────────────────────
@@ -109,3 +109,12 @@
"Can you explain the concept of gravity?",
"How do I improve my reading comprehension skills?",
]
+
+
+EDTECH = Scenario(
+ name="edtech",
+ description="Student-tutor conversation: academic profile facts with "
+ "learning progressions (grade, GPA, learning style updates).",
+ persona_pool=EDTECH_PERSONA_POOL,
+ filler_turns=EDTECH_FILLER_TURNS,
+)
diff --git a/memorylens/simulator/scenarios/medical.py b/memorylens/simulator/scenarios/medical.py
new file mode 100644
index 0000000..1df7c30
--- /dev/null
+++ b/memorylens/simulator/scenarios/medical.py
@@ -0,0 +1,83 @@
+"""
+Medical patient-consultation benchmark scenario.
+
+Models a patient in ongoing consultations with an AI health assistant.
+All personas are synthetic. Facts cover patient profile attributes; updates
+simulate a real care progression (medication change, symptom evolution,
+weight change).
+"""
+
+from typing import List
+
+from memorylens.simulator.facts import Fact
+from memorylens.simulator.scenarios.base import Scenario
+
+
+MEDICAL_PERSONA_POOL: List[List[Fact]] = [
+ # Persona 0 — Elena Vasquez (baseline)
+ [
+ Fact("name", "Elena Vasquez", injected_at=0),
+ Fact("age", "42", injected_at=1),
+ Fact("known allergy", "penicillin", injected_at=2),
+ Fact("current medication", "lisinopril", injected_at=4, updated_at=45, updated_value="losartan"),
+ Fact("main symptom", "persistent headaches", injected_at=6, updated_at=55, updated_value="occasional dizziness"),
+ Fact("blood type", "O positive", injected_at=8),
+ Fact("chronic condition", "hypertension", injected_at=10),
+ Fact("weight", "70 kilograms", injected_at=12, updated_at=70, updated_value="66 kilograms"),
+ ],
+ # Persona 1 — Samuel Adeyemi
+ [
+ Fact("name", "Samuel Adeyemi", injected_at=0),
+ Fact("age", "35", injected_at=1),
+ Fact("known allergy", "sulfa drugs", injected_at=2),
+ Fact("current medication", "metformin", injected_at=4, updated_at=45, updated_value="insulin glargine"),
+ Fact("main symptom", "fatigue", injected_at=6, updated_at=55, updated_value="improved energy levels"),
+ Fact("blood type", "A negative", injected_at=8),
+ Fact("chronic condition", "type 2 diabetes", injected_at=10),
+ Fact("weight", "88 kilograms", injected_at=12, updated_at=70, updated_value="83 kilograms"),
+ ],
+ # Persona 2 — Mei Lin
+ [
+ Fact("name", "Mei Lin", injected_at=0),
+ Fact("age", "58", injected_at=1),
+ Fact("known allergy", "latex", injected_at=2),
+ Fact("current medication", "atorvastatin", injected_at=4, updated_at=45, updated_value="rosuvastatin"),
+ Fact("main symptom", "joint stiffness", injected_at=6, updated_at=55, updated_value="reduced morning stiffness"),
+ Fact("blood type", "B positive", injected_at=8),
+ Fact("chronic condition", "osteoarthritis", injected_at=10),
+ Fact("weight", "61 kilograms", injected_at=12, updated_at=70, updated_value="63 kilograms"),
+ ],
+]
+
+
+MEDICAL_FILLER_TURNS: List[str] = [
+ "What is a normal resting heart rate?",
+ "How much water should I drink per day?",
+ "What are the early signs of the flu?",
+ "Is it better to stretch before or after exercise?",
+ "How many hours of sleep do adults need?",
+ "What foods are high in iron?",
+ "How does caffeine affect blood pressure?",
+ "What is the difference between a virus and a bacterial infection?",
+ "How often should I get a general health check-up?",
+ "What are common causes of lower back pain?",
+ "Is intermittent fasting safe for most people?",
+ "What vitamins support immune function?",
+ "How can I improve my posture while working at a desk?",
+ "What is considered a healthy body mass index range?",
+ "How do vaccines work?",
+ "What are the benefits of regular walking?",
+ "How can I reduce screen-related eye strain?",
+ "What is the recommended daily amount of fibre?",
+ "How does stress affect the immune system?",
+ "What are good sources of omega-3 fatty acids?",
+]
+
+
+MEDICAL = Scenario(
+ name="medical",
+ description="Patient-consultation conversation (synthetic data): patient "
+ "profile facts with care-progression updates (medication, symptom, weight).",
+ persona_pool=MEDICAL_PERSONA_POOL,
+ filler_turns=MEDICAL_FILLER_TURNS,
+)
diff --git a/simulator/scenarios/__init__.py b/memorylens/utils/__init__.py
similarity index 100%
rename from simulator/scenarios/__init__.py
rename to memorylens/utils/__init__.py
diff --git a/utils/embeddings.py b/memorylens/utils/embeddings.py
similarity index 100%
rename from utils/embeddings.py
rename to memorylens/utils/embeddings.py
diff --git a/utils/llm.py b/memorylens/utils/llm.py
similarity index 78%
rename from utils/llm.py
rename to memorylens/utils/llm.py
index c3664ad..0af2cea 100644
--- a/utils/llm.py
+++ b/memorylens/utils/llm.py
@@ -1,14 +1,19 @@
import os
import time
-from typing import Optional
-from groq import Groq
-_client: Optional[Groq] = None
+_client = None
-def get_client() -> Groq:
+def get_client():
global _client
if _client is None:
+ try:
+ from groq import Groq
+ except ImportError as e:
+ raise ImportError(
+ 'The groq package is required for LLM compression. '
+ 'Install it with: pip install "memorylens[groq]"'
+ ) from e
api_key = os.getenv("GROQ_API_KEY")
if not api_key:
raise EnvironmentError("GROQ_API_KEY not set")
diff --git a/utils/migrate_legacy_logs.py b/memorylens/utils/migrate_legacy_logs.py
similarity index 88%
rename from utils/migrate_legacy_logs.py
rename to memorylens/utils/migrate_legacy_logs.py
index cdea73b..ea2fe00 100644
--- a/utils/migrate_legacy_logs.py
+++ b/memorylens/utils/migrate_legacy_logs.py
@@ -3,7 +3,7 @@
One-shot migration: import existing JSON log files into SQLite.
Usage:
- python utils/migrate_legacy_logs.py
+ python -m memorylens.utils.migrate_legacy_logs
Scans experiment_logs/*.json, parses each file, and inserts
into the SQLite database at experiment_logs/memorylens.db.
@@ -15,12 +15,7 @@
import os
import sys
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
-
-from utils.storage import Storage
-
-
-LOG_DIR = os.path.join(os.path.dirname(__file__), "..", "experiment_logs")
+from memorylens.utils.storage import Storage, LOG_DIR
def migrate() -> int:
diff --git a/utils/providers.py b/memorylens/utils/providers.py
similarity index 99%
rename from utils/providers.py
rename to memorylens/utils/providers.py
index 1918fa2..f000bc4 100644
--- a/utils/providers.py
+++ b/memorylens/utils/providers.py
@@ -12,7 +12,7 @@
Groq → OpenAI → Anthropic → OpenRouter → Ollama → None (content-only mode)
Usage:
- from utils.providers import get_provider, list_available
+ from memorylens.utils.providers import get_provider, list_available
provider = get_provider() # auto-detect
provider = get_provider("openai") # force a specific one
diff --git a/utils/storage.py b/memorylens/utils/storage.py
similarity index 93%
rename from utils/storage.py
rename to memorylens/utils/storage.py
index 9237c81..7c4f134 100644
--- a/utils/storage.py
+++ b/memorylens/utils/storage.py
@@ -17,7 +17,14 @@
from typing import Any, Dict, List, Optional
-_LOG_DIR = os.path.join(os.path.dirname(__file__), "..", "experiment_logs")
+# Logs live in the working directory (overridable), not inside the installed
+# package — pip users must never write into site-packages.
+LOG_DIR = os.getenv(
+ "MEMORYLENS_LOG_DIR", os.path.join(os.getcwd(), "experiment_logs")
+)
+
+_METRIC_KEYS = ["recall", "precision", "drift", "noise", "contradiction",
+ "tokens", "cascade_eff", "llm_recall", "llm_drift"]
class Storage:
@@ -25,8 +32,8 @@ class Storage:
def __init__(self, db_path: Optional[str] = None):
if db_path is None:
- os.makedirs(_LOG_DIR, exist_ok=True)
- db_path = os.path.join(_LOG_DIR, "memorylens.db")
+ os.makedirs(LOG_DIR, exist_ok=True)
+ db_path = os.path.join(LOG_DIR, "memorylens.db")
self._db_path = db_path
self._conn: Optional[sqlite3.Connection] = None
@@ -106,8 +113,7 @@ def save_run(self, run_id: str, config: Dict[str, Any], display_data: Dict[str,
backends = [k for k in display_data if k != "checkpoints" and k != "has_llm_eval"]
rows: List[tuple] = []
- metric_keys = ["recall", "precision", "drift", "noise", "tokens",
- "cascade_eff", "llm_recall", "llm_drift"]
+ metric_keys = _METRIC_KEYS
for backend in backends:
backend_data = display_data[backend]
for i, cp in enumerate(checkpoints):
@@ -160,8 +166,7 @@ def get_run(self, run_id: str) -> Optional[Dict[str, Any]]:
cps = sorted(checkpoints)
display: Dict[str, Any] = {"checkpoints": cps, "has_llm_eval": False}
- metric_keys = ["recall", "precision", "drift", "noise", "tokens",
- "cascade_eff", "llm_recall", "llm_drift"]
+ metric_keys = _METRIC_KEYS
backend_meta = config.get("_backend_meta", {})
diff --git a/pyproject.toml b/pyproject.toml
index 4151eba..09590ab 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,16 +1,16 @@
[build-system]
requires = ["setuptools>=68", "wheel"]
-build-backend = "setuptools.backends.legacy:build"
+build-backend = "setuptools.build_meta"
[project]
name = "memorylens"
-version = "0.3.0"
-description = "The open-source benchmark for LLM memory decay — measure how AI memory architectures forget across long conversations"
+version = "0.4.0"
+description = "Benchmark for LLM memory decay — measure how AI memory architectures forget across long conversations"
readme = "README.md"
license = { text = "MIT" }
requires-python = ">=3.10"
authors = [
- { name = "Neal Srivastava", email = "builtbyneal@gmail.com" },
+ { name = "Neal Daftary", email = "builtbyneal@gmail.com" },
]
keywords = [
"llm",
@@ -34,41 +34,63 @@ classifiers = [
"Intended Audience :: Science/Research",
"Intended Audience :: Developers",
"License :: OSI Approved :: MIT License",
+ "Operating System :: OS Independent",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
+ "Programming Language :: Python :: 3.13",
"Topic :: Scientific/Engineering :: Artificial Intelligence",
"Topic :: Software Development :: Libraries :: Python Modules",
]
+# Core stays lean: only what the benchmark itself needs.
dependencies = [
- "sentence-transformers>=2.7.0",
"numpy>=1.24.0",
- "pandas>=2.0.0",
- "streamlit>=1.35.0",
- "plotly>=5.18.0",
+ "sentence-transformers>=2.7.0",
"python-dotenv>=1.0.0",
+ "networkx>=3.0",
]
[project.optional-dependencies]
+dashboard = ["streamlit>=1.35.0", "plotly>=5.18.0", "pandas>=2.0.0"]
+server = ["fastapi>=0.110.0", "uvicorn>=0.29.0"]
+faiss = ["faiss-cpu>=1.8.0"]
groq = ["groq>=0.9.0"]
openai = ["openai>=1.0.0"]
anthropic = ["anthropic>=0.25.0"]
-all-providers = ["groq>=0.9.0", "openai>=1.0.0", "anthropic>=0.25.0"]
-dev = ["pytest>=7.0", "pytest-cov>=4.0"]
+all = [
+ "streamlit>=1.35.0",
+ "plotly>=5.18.0",
+ "pandas>=2.0.0",
+ "fastapi>=0.110.0",
+ "uvicorn>=0.29.0",
+ "faiss-cpu>=1.8.0",
+ "groq>=0.9.0",
+ "openai>=1.0.0",
+ "anthropic>=0.25.0",
+]
+dev = [
+ "pytest>=7.0",
+ "pytest-cov>=4.0",
+ "httpx>=0.27",
+ "fastapi>=0.110.0",
+ "uvicorn>=0.29.0",
+ "build>=1.0",
+ "twine>=5.0",
+]
[project.scripts]
-memorylens = "main:main"
+memorylens = "memorylens.cli:main"
[project.urls]
Homepage = "https://github.com/Neal006/memorylens"
Repository = "https://github.com/Neal006/memorylens"
"Bug Tracker" = "https://github.com/Neal006/memorylens/issues"
Documentation = "https://github.com/Neal006/memorylens#readme"
-Paper = "https://github.com/Neal006/memorylens/blob/main/paper/memorylens_paper.md"
+Changelog = "https://github.com/Neal006/memorylens/blob/main/CHANGELOG.md"
[tool.setuptools.packages.find]
-include = ["memory*", "evaluation*", "simulator*", "utils*"]
+include = ["memorylens*"]
-[tool.setuptools.package-data]
-"*" = ["*.json"]
+[tool.pytest.ini_options]
+testpaths = ["tests"]
diff --git a/quick_demo.py b/quick_demo.py
index 8686a6d..a28dc99 100644
--- a/quick_demo.py
+++ b/quick_demo.py
@@ -28,12 +28,12 @@ def main() -> None:
if not checkpoints:
checkpoints = [args.turns]
- from simulator.facts import BENCHMARK_FACTS
- from simulator.conversation import generate_conversation
- from memory.naive import NaiveMemory
- from memory.rag import RAGMemory
- from memory.cascading import CascadingTemporalMemory
- from evaluation.metrics import (
+ from memorylens.simulator.facts import BENCHMARK_FACTS
+ from memorylens.simulator.conversation import generate_conversation
+ from memorylens.memory.naive import NaiveMemory
+ from memorylens.memory.rag import RAGMemory
+ from memorylens.memory.cascading import CascadingTemporalMemory
+ from memorylens.evaluation.metrics import (
recall_at_t, temporal_drift_score, memory_noise_ratio,
precision_at_k, cascade_efficiency,
)
@@ -131,18 +131,18 @@ def main() -> None:
vals = " ".join(f"{eff_table['cascading'].get(c, 1.0):5.2f}x" for c in checkpoints)
print(f" {'cascading':<12} {vals}")
- # Business impact
+ # Illustrative cost projection: 100K queries/month at $1 per 1M input tokens
qpm = 100_000
- cost_inr = 83 / 1_000_000
+ cost_per_token = 1 / 1_000_000
final_cp = checkpoints[-1]
- print("\n BUSINESS IMPACT @ 100K queries/month")
- print(f" {'Backend':<12} {'Tokens/Q':>9} {'Monthly(INR)':>13} {'Recall':>8}")
+ print("\n PROJECTED COST @ 100K queries/month ($1 per 1M input tokens, illustrative)")
+ print(f" {'Backend':<12} {'Tokens/Q':>9} {'Monthly($)':>11} {'Recall':>8}")
print(" " + "-" * 52)
for name in backends:
tok = tokens_table[name].get(final_cp, 0)
- cost = tok * qpm * cost_inr
+ cost = tok * qpm * cost_per_token
rec = recall_table[name].get(final_cp, 0)
- print(f" {name:<12} {tok:>9,} INR{cost:>9,.0f} {rec:>7.1%}")
+ print(f" {name:<12} {tok:>9,} ${cost:>9,.2f} {rec:>7.1%}")
print()
print(" >> Run 'streamlit run dashboard.py' to see full visualisation")
diff --git a/requirements.txt b/requirements.txt
index 283aac6..9ada3a0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,12 @@
-groq>=0.9.0
-sentence-transformers>=2.7.0
+# Full development environment. Library users: pip install memorylens
+# (see pyproject.toml extras: [dashboard], [server], [faiss], [groq], ...)
numpy>=1.24.0
+sentence-transformers>=2.7.0
+python-dotenv>=1.0.0
+networkx>=3.0
pandas>=2.0.0
streamlit>=1.35.0
plotly>=5.18.0
-python-dotenv>=1.0.0
+fastapi>=0.110.0
+uvicorn>=0.29.0
+groq>=0.9.0
diff --git a/tests/test_imports.py b/tests/test_imports.py
index e7313c2..01bf78f 100644
--- a/tests/test_imports.py
+++ b/tests/test_imports.py
@@ -6,25 +6,29 @@
os.environ["USE_TF"] = "0"
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
-from simulator.facts import BENCHMARK_FACTS
-from simulator.conversation import generate_conversation
-from simulator.personas import PERSONA_POOL
-from memory.naive import NaiveMemory
-from memory.rag import RAGMemory
-from memory.rag_chunked import ChunkedRAGMemory
-from memory.cascading import CascadingTemporalMemory
-from memory.summary import SummaryMemory
-from memory.decay import get_decay_fn, _REGISTRY as DECAY_REGISTRY
-from evaluation.metrics import (
+import memorylens
+from memorylens.simulator.facts import BENCHMARK_FACTS
+from memorylens.simulator.conversation import generate_conversation
+from memorylens.simulator.personas import PERSONA_POOL
+from memorylens.simulator.scenarios import SCENARIOS, get_scenario, Scenario
+from memorylens.memory.naive import NaiveMemory
+from memorylens.memory.rag import RAGMemory
+from memorylens.memory.rag_chunked import ChunkedRAGMemory
+from memorylens.memory.cascading import CascadingTemporalMemory
+from memorylens.memory.summary import SummaryMemory
+from memorylens.memory.entity import EntityMemory
+from memorylens.memory.graph import GraphMemory
+from memorylens.memory.decay import get_decay_fn, _REGISTRY as DECAY_REGISTRY
+from memorylens.evaluation.metrics import (
recall_at_t, precision_at_k, temporal_drift_score,
- memory_noise_ratio, cascade_efficiency,
+ memory_noise_ratio, contradiction_score, cascade_efficiency,
llm_recall_at_t, llm_temporal_drift,
)
-from evaluation.benchmark import run_benchmark, results_to_display_dict, run_benchmark_multi_seed, VALID_BACKENDS
-from evaluation.stats import aggregate_metric, aggregate_checkpoint_series
-from evaluation.logger import log_run, list_runs
-from evaluation.llm_judge import judge_answer
-from utils.providers import get_provider, list_available, LLMProvider, _REGISTRY as PROVIDER_REGISTRY
+from memorylens.evaluation.benchmark import run_benchmark, results_to_display_dict, run_benchmark_multi_seed, VALID_BACKENDS
+from memorylens.evaluation.stats import aggregate_metric, aggregate_checkpoint_series
+from memorylens.evaluation.logger import log_run, list_runs
+from memorylens.evaluation.llm_judge import judge_answer
+from memorylens.utils.providers import get_provider, list_available, LLMProvider, _REGISTRY as PROVIDER_REGISTRY
# Providers
assert set(PROVIDER_REGISTRY.keys()) == {"groq", "openai", "anthropic", "openrouter", "ollama"}, (
@@ -46,12 +50,25 @@
assert len(PERSONA_POOL) >= 5, f"Expected at least 5 personas, got {len(PERSONA_POOL)}"
# Backend registry
-assert "rag_chunked" in VALID_BACKENDS
+for backend in ("rag_chunked", "entity", "graph", "faiss"):
+ assert backend in VALID_BACKENDS, f"'{backend}' missing from VALID_BACKENDS"
+
+# Scenario registry
+assert set(SCENARIOS.keys()) == {"default", "edtech", "support", "medical"}, (
+ f"Scenario registry mismatch: {set(SCENARIOS.keys())}"
+)
+for s in SCENARIOS.values():
+ s.validate()
+
+# Package metadata
+assert memorylens.__version__
print(
f"All imports OK | "
+ f"v{memorylens.__version__} | "
f"Facts: {len(BENCHMARK_FACTS)} | "
+ f"Backends: {VALID_BACKENDS} | "
+ f"Scenarios: {list(SCENARIOS.keys())} | "
f"Providers: {list(PROVIDER_REGISTRY.keys())} | "
- f"Decay fns: {list(DECAY_REGISTRY.keys())} | "
f"Personas: {len(PERSONA_POOL)}"
)
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
index b0ae013..b5e7bd6 100644
--- a/tests/test_pipeline.py
+++ b/tests/test_pipeline.py
@@ -10,13 +10,13 @@
os.environ["USE_TF"] = "0"
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
-from simulator.facts import BENCHMARK_FACTS, Fact
-from simulator.conversation import generate_conversation
-from memory.naive import NaiveMemory
-from memory.rag import RAGMemory
-from memory.cascading import CascadingTemporalMemory
-from memory.summary import SummaryMemory
-from evaluation.metrics import (
+from memorylens.simulator.facts import BENCHMARK_FACTS, Fact
+from memorylens.simulator.conversation import generate_conversation
+from memorylens.memory.naive import NaiveMemory
+from memorylens.memory.rag import RAGMemory
+from memorylens.memory.cascading import CascadingTemporalMemory
+from memorylens.memory.summary import SummaryMemory
+from memorylens.evaluation.metrics import (
recall_at_t, temporal_drift_score, memory_noise_ratio, precision_at_k
)
@@ -173,7 +173,7 @@ def test_summary_token_cost_bounded():
def test_summary_benchmark_registration():
"""'summary' backend must be resolvable from the benchmark runner."""
- from evaluation.benchmark import _make_memory
+ from memorylens.evaluation.benchmark import _make_memory
mem = _make_memory("summary")
assert mem.name == "summary"
print(f"PASS: summary registered in benchmark runner ({mem!r})")
@@ -183,7 +183,7 @@ def test_summary_benchmark_registration():
def test_decay_functions_range():
"""All decay functions must return values in [0, 1] for all valid inputs."""
- from memory.decay import _REGISTRY
+ from memorylens.memory.decay import _REGISTRY
for name, fn in _REGISTRY.items():
for age in [0, 1, 5, 10, 50, 99, 100]:
v = fn(age, 100)
@@ -193,7 +193,7 @@ def test_decay_functions_range():
def test_ebbinghaus_is_monotone_decreasing():
"""Ebbinghaus decay must be monotonically non-increasing with age."""
- from memory.decay import decay_ebbinghaus
+ from memorylens.memory.decay import decay_ebbinghaus
prev = 1.0
for age in range(0, 101):
v = decay_ebbinghaus(age, 100)
@@ -204,7 +204,7 @@ def test_ebbinghaus_is_monotone_decreasing():
def test_cascading_uses_pluggable_decay():
"""CascadingTemporalMemory should accept and store the decay name."""
- from memory.cascading import CascadingTemporalMemory
+ from memorylens.memory.cascading import CascadingTemporalMemory
for name in ["default", "linear", "exponential", "ebbinghaus"]:
mem = CascadingTemporalMemory(decay=name)
assert mem.decay_name == name, f"Expected decay_name={name}, got {mem.decay_name}"
@@ -215,7 +215,7 @@ def test_cascading_uses_pluggable_decay():
def test_chunked_rag_recall_early():
"""ChunkedRAGMemory should recall facts with >= 75% accuracy at T=15."""
- from memory.rag_chunked import ChunkedRAGMemory
+ from memorylens.memory.rag_chunked import ChunkedRAGMemory
mem = ChunkedRAGMemory()
_populate(mem, BENCHMARK_FACTS, 15)
active = [f for f in BENCHMARK_FACTS if f.injected_at < 15]
@@ -227,7 +227,7 @@ def test_chunked_rag_recall_early():
def test_chunked_rag_bounded_index():
"""ChunkedRAGMemory must not exceed max_chunks capacity."""
- from memory.rag_chunked import ChunkedRAGMemory
+ from memorylens.memory.rag_chunked import ChunkedRAGMemory
mem = ChunkedRAGMemory(max_chunks=50)
_populate(mem, BENCHMARK_FACTS, 100)
assert len(mem.chunks) <= 50, (
@@ -241,7 +241,7 @@ def test_chunked_rag_bounded_index():
def test_chunked_rag_tokens_less_than_naive():
"""ChunkedRAGMemory should use fewer tokens than naive at T=100."""
- from memory.rag_chunked import ChunkedRAGMemory
+ from memorylens.memory.rag_chunked import ChunkedRAGMemory
naive = NaiveMemory(max_context_tokens=1200)
chunked = ChunkedRAGMemory()
_populate(naive, BENCHMARK_FACTS, 100)
@@ -257,7 +257,7 @@ def test_chunked_rag_tokens_less_than_naive():
def test_chunked_rag_benchmark_registration():
"""'rag_chunked' backend must be resolvable from the benchmark runner."""
- from evaluation.benchmark import _make_memory
+ from memorylens.evaluation.benchmark import _make_memory
mem = _make_memory("rag_chunked")
assert mem.name == "rag_chunked"
print(f"PASS: rag_chunked registered in benchmark runner ({mem!r})")
@@ -267,7 +267,7 @@ def test_chunked_rag_benchmark_registration():
def test_stats_aggregate_metric():
"""aggregate_metric must return correct mean and std."""
- from evaluation.stats import aggregate_metric
+ from memorylens.evaluation.stats import aggregate_metric
result = aggregate_metric([0.8, 0.9, 0.7, 0.85, 0.75])
assert abs(result["mean"] - 0.8) < 0.01, f"Mean wrong: {result['mean']}"
assert result["std"] > 0, "Std should be > 0 for varied values"
@@ -277,7 +277,7 @@ def test_stats_aggregate_metric():
def test_persona_pool_structure():
"""Each persona must have 8 facts with the same keys as BENCHMARK_FACTS."""
- from simulator.personas import PERSONA_POOL
+ from memorylens.simulator.personas import PERSONA_POOL
expected_keys = {f.key for f in BENCHMARK_FACTS}
for i, persona in enumerate(PERSONA_POOL):
persona_keys = {f.key for f in persona}
@@ -287,22 +287,244 @@ def test_persona_pool_structure():
print(f"PASS: persona pool structure ({len(PERSONA_POOL)} personas, {len(expected_keys)} keys each)")
-# ── SQLite Storage tests ──────────────────────────────────────────────────────
+# ── GraphMemory tests ───────────────────────────────────────────────────────
-def test_storage_save_and_get_run():
- from utils.storage import Storage
- import tempfile, os
+def test_graph_recall_early():
+ """GraphMemory should recall all injected facts at T=15 (templated extraction)."""
+ from memorylens.memory.graph import GraphMemory
+ mem = GraphMemory()
+ _populate(mem, BENCHMARK_FACTS, 15)
+ active = [f for f in BENCHMARK_FACTS if f.injected_at < 15]
+ results = [recall_at_t(mem, f, 14) for f in active]
+ rate = sum(r["recalled"] for r in results) / len(results)
+ assert rate == 1.0, f"Expected 100% recall at T=15 for graph, got {rate:.0%}"
+ print(f"PASS: graph recall early ({rate:.0%})")
+
+
+def test_graph_update_replaces_edge():
+ """After a fact update, the graph must surface the new value and drop the old."""
+ from memorylens.memory.graph import GraphMemory
+ mem = GraphMemory()
+ _populate(mem, BENCHMARK_FACTS, 50) # city updates at T=40
+ ctx = mem.get_context("What is my city?", 49)
+ combined = " ".join(m["content"].lower() for m in ctx)
+ assert "mumbai" in combined, "updated city value missing from graph context"
+ assert "bangalore" not in combined, "stale city value still present in graph context"
+ print("PASS: graph update replaces edge (no stale value)")
+
+
+def test_graph_reset_and_registration():
+ from memorylens.evaluation.benchmark import _make_memory
+ mem = _make_memory("graph")
+ assert mem.name == "graph"
+ mem.add_message("user", "My name is Test User.", 0)
+ mem.reset()
+ assert mem.get_context("What is my name?", 1) == []
+ print("PASS: graph reset + benchmark registration")
+
+
+def test_cascading_cold_tier_retains_early_facts():
+ """Regression (cold merge order): facts injected at T=0-9 must survive
+ compression into the cold tier and stay recallable at T=100."""
+ mem = CascadingTemporalMemory()
+ _populate(mem, BENCHMARK_FACTS, 100)
+ results = [recall_at_t(mem, f, 99) for f in BENCHMARK_FACTS]
+ rate = sum(r["recalled"] for r in results) / len(results)
+ assert rate >= 0.85, f"Cold-tier recall regressed: {rate:.0%} at T=100"
+ print(f"PASS: cascading cold-tier retains early facts ({rate:.0%} at T=100)")
+
+
+def test_cascading_drift_zero_after_update():
+ """Regression (issue #16): cold summaries must be patched on fact updates."""
+ mem = CascadingTemporalMemory()
+ _populate(mem, BENCHMARK_FACTS, 100) # city updates at T=40, age at T=60
+ updated = [f for f in BENCHMARK_FACTS if f.updated_at]
+ for f in updated:
+ drift = temporal_drift_score(mem, f, 99)
+ assert drift["drift"] == 0.0, f"{f.key}: stale value survived, drift={drift}"
+ print("PASS: cascading drift stays 0.0 after updates")
+
+
+# ── contradiction_score tests ───────────────────────────────────────────────
+
+def test_contradiction_not_applicable_before_update():
+ from memorylens.evaluation.metrics import contradiction_score
+ mem = NaiveMemory()
+ _populate(mem, BENCHMARK_FACTS, 10)
+ city = next(f for f in BENCHMARK_FACTS if f.key == "city")
+ result = contradiction_score(mem, city, 9)
+ assert result["applicable"] is False
+ print("PASS: contradiction not applicable before update turn")
+
+
+def test_contradiction_naive_surfaces_both_values():
+ """Naive keeps full history, so both old and new city must co-occur → 1.0."""
+ from memorylens.evaluation.metrics import contradiction_score
+ mem = NaiveMemory(max_context_tokens=8000)
+ _populate(mem, BENCHMARK_FACTS, 50) # city: Bangalore → Mumbai at T=40
+ city = next(f for f in BENCHMARK_FACTS if f.key == "city")
+ result = contradiction_score(mem, city, 49)
+ assert result["applicable"] is True
+ assert result["contradiction"] == 1.0, (
+ f"Expected contradiction=1.0 for naive full history, got {result}"
+ )
+ print("PASS: contradiction detected in naive full history")
+
+
+def test_contradiction_graph_is_zero():
+ """GraphMemory patches facts in place, so no contradiction can survive."""
+ from memorylens.evaluation.metrics import contradiction_score
+ from memorylens.memory.graph import GraphMemory
+ mem = GraphMemory()
+ _populate(mem, BENCHMARK_FACTS, 50)
+ city = next(f for f in BENCHMARK_FACTS if f.key == "city")
+ result = contradiction_score(mem, city, 49)
+ assert result["contradiction"] == 0.0, f"Graph should never contradict, got {result}"
+ print("PASS: contradiction zero for graph backend")
+
+
+def test_contradiction_in_benchmark_output():
+ """Benchmark display dict must include the contradiction series."""
+ from memorylens.evaluation.benchmark import run_benchmark, results_to_display_dict
+ raw = run_benchmark(total_turns=10, eval_checkpoints=[10], backends=["naive"])
+ display = results_to_display_dict(raw)
+ assert "contradiction" in display["naive"], "contradiction series missing from display dict"
+ print("PASS: contradiction wired into benchmark output")
- with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as f:
- db_path = f.name
+
+# ── Scenario tests ──────────────────────────────────────────────────────────
+
+def test_scenario_registry_complete():
+ from memorylens.simulator.scenarios import SCENARIOS
+ assert set(SCENARIOS) == {"default", "edtech", "support", "medical"}
+ for s in SCENARIOS.values():
+ s.validate()
+ assert len(s.facts) == 8, f"{s.name}: expected 8 facts, got {len(s.facts)}"
+ assert any(f.updated_at for f in s.facts), f"{s.name}: needs at least one fact update"
+ print(f"PASS: scenario registry ({list(SCENARIOS)})")
+
+
+def test_scenario_unknown_raises():
+ from memorylens.simulator.scenarios import get_scenario
+ try:
+ get_scenario("nonexistent")
+ assert False, "Expected ValueError for unknown scenario"
+ except ValueError:
+ pass
+ print("PASS: unknown scenario raises ValueError")
+
+
+def test_scenario_values_disjoint_on_update():
+ """Old/new values must not be substrings of each other, or drift and
+ contradiction metrics silently break."""
+ from memorylens.simulator.scenarios import SCENARIOS
+ for s in SCENARIOS.values():
+ for persona in s.persona_pool:
+ for f in persona:
+ if f.updated_value:
+ old, new = f.value.lower(), f.updated_value.lower()
+ assert old not in new and new not in old, (
+ f"{s.name}/{f.key}: '{f.value}' and '{f.updated_value}' overlap"
+ )
+ print("PASS: scenario update values are substring-disjoint")
+
+
+def test_scenario_benchmark_runs():
+ """A non-default scenario must run end-to-end through the benchmark."""
+ from memorylens.evaluation.benchmark import run_benchmark
+ from memorylens.simulator.scenarios import get_scenario
+ s = get_scenario("support")
+ raw = run_benchmark(
+ total_turns=15, eval_checkpoints=[15], facts=s.facts,
+ backends=["naive"], filler_turns=s.filler_turns,
+ )
+ cp = raw["naive"].checkpoints[0]
+ assert cp.recall == 1.0, f"Naive should have full recall at T=15, got {cp.recall}"
+ print(f"PASS: support scenario end-to-end (recall={cp.recall:.0%})")
+
+
+# ── FAISS backend (optional dependency) ─────────────────────────────────────
+
+def test_faiss_backend_or_missing_dep_error():
+ try:
+ import faiss # noqa: F401
+ except ImportError:
+ from memorylens.evaluation.benchmark import _make_memory
+ try:
+ _make_memory("faiss")
+ assert False, "Expected ImportError when faiss is not installed"
+ except ImportError as e:
+ assert "memorylens[faiss]" in str(e), f"Error should name the extra: {e}"
+ print("SKIP: faiss not installed (missing-dep error message verified)")
+ return
+
+ from memorylens.memory.vector_faiss import FAISSMemory
+ mem = FAISSMemory()
+ _populate(mem, BENCHMARK_FACTS, 15)
+ active = [f for f in BENCHMARK_FACTS if f.injected_at < 15]
+ results = [recall_at_t(mem, f, 14) for f in active]
+ rate = sum(r["recalled"] for r in results) / len(results)
+ assert rate >= 0.75, f"Expected >=75% recall at T=15 for faiss, got {rate:.0%}"
+ print(f"PASS: faiss recall early ({rate:.0%})")
+
+
+# ── API server (optional dependency) ────────────────────────────────────────
+
+def test_api_benchmark_lifecycle():
+ try:
+ from fastapi.testclient import TestClient
+ except ImportError:
+ print("SKIP: fastapi/httpx not installed")
+ return
+ import time
+ from memorylens.api import app
+
+ client = TestClient(app)
+ assert client.get("/health").json()["status"] == "ok"
+ assert "naive" in client.get("/v1/backends").json()["data"]
+ scenario_names = [s["name"] for s in client.get("/v1/scenarios").json()["data"]]
+ assert "medical" in scenario_names
+
+ assert client.post("/v1/benchmarks", json={"backends": ["bogus"]}).status_code == 422
+ assert client.post("/v1/benchmarks", json={
+ "turns": 10, "checkpoints": [100], "backends": ["naive"],
+ }).status_code == 422
+
+ resp = client.post("/v1/benchmarks", json={
+ "turns": 10, "checkpoints": [10], "backends": ["naive"],
+ })
+ assert resp.status_code == 202
+ job_id = resp.json()["data"]["job_id"]
+
+ for _ in range(150):
+ job = client.get(f"/v1/benchmarks/{job_id}").json()["data"]
+ if job["status"] != "running":
+ break
+ time.sleep(0.2)
+ assert job["status"] == "completed", f"Job did not complete: {job.get('error')}"
+ assert "naive" in job["results"]
+ print("PASS: API benchmark lifecycle (submit, poll, results)")
+
+
+# ── SQLite Storage tests ─────────────────────────────────────────────────────
+
+def _temp_store():
+ import tempfile
+ from memorylens.utils.storage import Storage
+ f = tempfile.NamedTemporaryFile(suffix=".db", delete=False)
+ f.close()
+ return Storage(f.name), f.name
+
+
+def test_storage_save_and_get_run():
+ store, db_path = _temp_store()
try:
- store = Storage(db_path)
display = {
"checkpoints": [10, 25],
"naive": {
"recall": [1.0, 0.8], "precision": [0.9, 0.7],
"drift": [0.0, 0.1], "noise": [0.5, 0.8],
- "tokens": [100, 500],
+ "tokens": [100, 500], "contradiction": [0.0, 1.0],
}
}
store.save_run("test_run", {"total_turns": 25, "backends": ["naive"]}, display)
@@ -311,6 +533,7 @@ def test_storage_save_and_get_run():
assert loaded["checkpoints"] == [10, 25]
assert loaded["naive"]["recall"] == [1.0, 0.8]
assert loaded["naive"]["tokens"] == [100, 500]
+ assert loaded["naive"]["contradiction"] == [0.0, 1.0]
finally:
store.close()
os.unlink(db_path)
@@ -318,13 +541,8 @@ def test_storage_save_and_get_run():
def test_storage_list_runs():
- from utils.storage import Storage
- import tempfile, os
-
- with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as f:
- db_path = f.name
+ store, db_path = _temp_store()
try:
- store = Storage(db_path)
display = {"checkpoints": [10], "naive": {"recall": [0.5], "precision": [0.5],
"drift": [0], "noise": [0], "tokens": [100]}}
store.save_run("run_b", {"total_turns": 10}, display)
@@ -340,13 +558,8 @@ def test_storage_list_runs():
def test_storage_compare_runs():
- from utils.storage import Storage
- import tempfile, os
-
- with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as f:
- db_path = f.name
+ store, db_path = _temp_store()
try:
- store = Storage(db_path)
display = {"checkpoints": [10], "naive": {"recall": [0.8], "precision": [0.8],
"drift": [0], "noise": [0], "tokens": [100]}}
store.save_run("run_a", {}, display)
@@ -364,13 +577,8 @@ def test_storage_compare_runs():
def test_storage_get_run_not_found():
- from utils.storage import Storage
- import tempfile, os
-
- with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as f:
- db_path = f.name
+ store, db_path = _temp_store()
try:
- store = Storage(db_path)
assert store.get_run("nonexistent") is None
finally:
store.close()
@@ -380,13 +588,8 @@ def test_storage_get_run_not_found():
def test_storage_save_run_idempotent():
"""Calling save_run twice with the same run_id must not duplicate rows."""
- from utils.storage import Storage
- import tempfile, os
-
- with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as f:
- db_path = f.name
+ store, db_path = _temp_store()
try:
- store = Storage(db_path)
display = {"checkpoints": [10], "naive": {"recall": [0.8], "precision": [0.8],
"drift": [0], "noise": [0], "tokens": [100]}}
store.save_run("dup_test", {}, display)
@@ -406,9 +609,8 @@ def test_storage_save_run_idempotent():
def _clean_csv_row(run_id: str) -> None:
"""Remove a test run_id from runs_summary.csv to avoid accumulation."""
import csv
- csv_path = os.path.join(
- os.path.dirname(__file__), "..", "experiment_logs", "runs_summary.csv"
- )
+ from memorylens.utils.storage import LOG_DIR
+ csv_path = os.path.join(LOG_DIR, "runs_summary.csv")
if not os.path.exists(csv_path):
return
rows = []
@@ -428,9 +630,8 @@ def _clean_csv_row(run_id: str) -> None:
def test_logger_writes_sqlite():
"""log_run must write to SQLite, not just JSON."""
- from evaluation.logger import log_run
- from utils.storage import Storage
- import os
+ from memorylens.evaluation.logger import log_run
+ from memorylens.utils.storage import Storage
display = {
"checkpoints": [10],
@@ -443,7 +644,6 @@ def test_logger_writes_sqlite():
json_path = log_run(display, config, run_id=run_id)
assert os.path.exists(json_path), "JSON file must exist (backward compat)"
- # Verify SQLite has the data
store = Storage()
loaded = store.get_run(run_id)
assert loaded is not None, "SQLite must contain the run"
@@ -461,8 +661,8 @@ def test_logger_writes_sqlite():
def test_list_runs_returns_sqlite_runs():
"""list_runs must return SQLite-backed runs, not just filesystem scans."""
- from evaluation.logger import list_runs
- from utils.storage import Storage
+ from memorylens.evaluation.logger import list_runs
+ from memorylens.utils.storage import Storage
store = Storage()
display = {"checkpoints": [10], "naive": {"recall": [0.6], "precision": [0.6],
@@ -473,7 +673,6 @@ def test_list_runs_returns_sqlite_runs():
ids = [r["run_id"] for r in runs]
assert "_test_list_runs" in ids, "list_runs must include SQLite runs"
- # Cleanup
store.conn.execute("DELETE FROM results WHERE run_id = ?", ("_test_list_runs",))
store.conn.execute("DELETE FROM runs WHERE run_id = ?", ("_test_list_runs",))
store.conn.commit()
@@ -510,6 +709,26 @@ def test_list_runs_returns_sqlite_runs():
# Stats / multi-seed
test_stats_aggregate_metric,
test_persona_pool_structure,
+ # Cascading regressions
+ test_cascading_cold_tier_retains_early_facts,
+ test_cascading_drift_zero_after_update,
+ # GraphMemory
+ test_graph_recall_early,
+ test_graph_update_replaces_edge,
+ test_graph_reset_and_registration,
+ # contradiction_score
+ test_contradiction_not_applicable_before_update,
+ test_contradiction_naive_surfaces_both_values,
+ test_contradiction_graph_is_zero,
+ test_contradiction_in_benchmark_output,
+ # Scenarios
+ test_scenario_registry_complete,
+ test_scenario_unknown_raises,
+ test_scenario_values_disjoint_on_update,
+ test_scenario_benchmark_runs,
+ # Optional deps
+ test_faiss_backend_or_missing_dep_error,
+ test_api_benchmark_lifecycle,
# SQLite Storage
test_storage_save_and_get_run,
test_storage_list_runs,
diff --git a/utils/__init__.py b/utils/__init__.py
deleted file mode 100644
index e69de29..0000000