feat: MemoryBench evaluation suite (Epic #283)

.github/workflows/ci.yml

@@ -64,11 +64,13 @@ jobs:
           python evals/runners/run_scholar_pipeline_smoke.py
           python evals/runners/run_track_pipeline_smoke.py
           python evals/runners/run_eventlog_replay_smoke.py
+          python evals/runners/run_retrieval_benchmark_smoke.py
 
       - name: Run memory range and acceptance testing
         env:
           PYTHONPATH: src
         run: |
           python evals/memory/test_deletion_compliance.py
           python evals/memory/test_retrieval_hit_rate.py
-
+          python evals/memory/test_scope_isolation.py
+          python evals/memory/test_injection_robustness.py

Makefile

@@ -0,0 +1,8 @@
+.PHONY: bench-roi
+
+bench-roi:
+	@if [ -z "$$OPENAI_API_KEY$$ANTHROPIC_API_KEY$$OPENROUTER_API_KEY$$NVIDIA_MINIMAX_API_KEY$$NVIDIA_GLM_API_KEY" ]; then \
+		echo "Missing API key: set OPENAI_API_KEY, ANTHROPIC_API_KEY, OPENROUTER_API_KEY, NVIDIA_MINIMAX_API_KEY, or NVIDIA_GLM_API_KEY"; \
+		exit 1; \
+	fi
+	PYTHONPATH=src python evals/memory/bench_roi.py --output evals/reports/memory_roi_benchmark.json

README.md

@@ -1,17 +1,27 @@
 <!-- markdownlint-disable MD001 MD041 -->
 
-<h1 align="center">PaperBot</h1>
+<h1 align="center">Oh, God! My idea comes true.</h1>
 
 <h3 align="center">
 AI-powered research workflow: paper discovery → LLM analysis → scholar tracking → Paper2Code → multi-agent studio
 </h3>
 
 <p align="center">
-  <a href="https://github.com/jerry609/PaperBot/actions"><img src="https://github.com/jerry609/PaperBot/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
-  <a href="https://github.com/jerry609/PaperBot/issues/232"><img src="https://img.shields.io/badge/roadmap-Q1%2FQ2%202026-blue" alt="Roadmap"></a>
-  <a href="https://opensource.org/licenses/MIT"><img src="https://img.shields.io/badge/License-MIT-green.svg" alt="License: MIT"></a>
-  <img src="https://img.shields.io/badge/python-3.10%2B-blue" alt="Python 3.10+">
-  <img src="https://img.shields.io/badge/Next.js-16-black" alt="Next.js 16">
+  <a href="https://github.com/jerry609/PaperBot/actions">
+    <img src="https://img.shields.io/github/actions/workflow/status/jerry609/PaperBot/ci.yml?branch=dev&label=CI&logo=github" alt="CI">
+  </a>
+  <a href="https://github.com/jerry609/PaperBot/issues/232">
+    <img src="https://img.shields.io/badge/roadmap-2026-blue?logo=roadmap" alt="Roadmap">
+  </a>
+  <img src="https://img.shields.io/badge/version-0.1.0-007ec6?logo=github" alt="Version 0.1.0">
+  <img src="https://img.shields.io/badge/license-MIT-green" alt="License">
+</p>
+
+<p align="center">
+  <img src="https://img.shields.io/badge/Python-3.10+-blue?logo=python&logoColor=white" alt="Python">
+  <img src="https://img.shields.io/badge/Next.js-16-black?logo=nextdotjs" alt="Next.js">
+  <img src="https://img.shields.io/badge/Platform-Win%20|%20Mac%20|%20Linux-lightgrey" alt="Platform">
+  <img src="https://img.shields.io/badge/Downloads-xx-lightgrey?style=social&logo=icloud" alt="Downloads">
 </p>
 
 <p align="center">
@@ -26,7 +36,7 @@ AI-powered research workflow: paper discovery → LLM analysis → scholar track
 
 ## About
 
-PaperBot is an end-to-end research assistant that automates the paper discovery → analysis → reproduction pipeline. It combines multi-source search, LLM-powered evaluation, scholar tracking, and code generation into a unified workflow with Web, CLI, and API interfaces.
+"Oh, God! My idea comes true." is an end-to-end research assistant that automates the paper discovery → analysis → reproduction pipeline. It combines multi-source search, LLM-powered evaluation, scholar tracking, and code generation into a unified workflow with Web, CLI, and API interfaces.
 
 **Backend** Python + FastAPI (SSE streaming) · **Frontend** Next.js + Ink CLI · **Sources** arXiv / Semantic Scholar / OpenAlex / HuggingFace Daily Papers / papers.cool
 
@@ -71,6 +81,7 @@ PaperBot is an end-to-end research assistant that automates the paper discovery
 - **Structured Cards** — LLM-extracted method / dataset / conclusion / limitations with DB caching
 - **Related Work** — Draft generation from saved papers with [AuthorYear] citation format
 - **Memory System** — Research memory with FTS5 + BM25 search, context engine for personalized recommendations
+- **MemoryBench Suite** — Retrieval / context / isolation / injection / performance / ROI / effectiveness benchmarks for the memory and Paper2Code stack
 
 ### Reproduction & Studio
 
@@ -84,6 +95,7 @@ PaperBot is an end-to-end research assistant that automates the paper discovery
 ### Install
 
 ```bash
+# Use python3 for macOS/Linux
 python -m venv .venv && source .venv/bin/activate
 pip install -e .
 ```
@@ -136,6 +148,7 @@ PAPERBOT_NOTIFY_EMAIL_TO=recipient@example.com
 alembic upgrade head
 
 # API server
+# Use python3 for macOS/Linux
 python -m uvicorn src.paperbot.api.main:app --reload --port 8000
 
 # Web dashboard (separate terminal)
@@ -197,6 +210,7 @@ Active Epics:
 | [#153](https://github.com/jerry609/PaperBot/issues/153) | Memory & Context | P0-P1 done |
 | [#154](https://github.com/jerry609/PaperBot/issues/154) | Agentic Research | Design done |
 | [#179](https://github.com/jerry609/PaperBot/issues/179) | Daily Push | Complete |
+| [#283](https://github.com/jerry609/PaperBot/issues/283) | MemoryBench | Complete |
 | [#159](https://github.com/jerry609/PaperBot/issues/159) | Obsidian CLI | Not started |
 
 ## Contributing
@@ -222,6 +236,13 @@ python -m black . && python -m isort .
 | [`docs/PLAN.md`](docs/PLAN.md) | Architecture assessment |
 | [`docs/PAPERSCOOL_WORKFLOW.md`](docs/PAPERSCOOL_WORKFLOW.md) | Topic Workflow guide |
 | [`docs/p2c/`](docs/p2c/) | Paper2Context design docs |
+| [`docs/benchmark/MEMORYBENCH_EPIC_283_COMPLETION.md`](docs/benchmark/MEMORYBENCH_EPIC_283_COMPLETION.md) | MemoryBench Epic completion report |
+| [`docs/benchmark/MEMORYBENCH_RUNTIME_REPORT_2026-03-07.md`](docs/benchmark/MEMORYBENCH_RUNTIME_REPORT_2026-03-07.md) | Live ROI + 1M memory runtime report |
+| [`docs/search_eval.md`](docs/search_eval.md) | Retrieval benchmark guide |
+| [`docs/context_engine_eval.md`](docs/context_engine_eval.md) | Context extraction benchmark guide |
+| [`docs/memory_performance_eval.md`](docs/memory_performance_eval.md) | Memory performance benchmark guide |
+| [`docs/p2c/P2C_ROI_BENCHMARK.md`](docs/p2c/P2C_ROI_BENCHMARK.md) | ROI benchmark guide |
+| [`docs/memory_effectiveness_eval.md`](docs/memory_effectiveness_eval.md) | Multi-session memory effectiveness benchmark guide |
 | [`docs/memory_system.md`](docs/memory_system.md) | Memory system design |
 | [`docs/anchor_system.md`](docs/anchor_system.md) | Anchor author system |
 | [`docs/AGENTIC_RESEARCH_EVOLUTION.md`](docs/AGENTIC_RESEARCH_EVOLUTION.md) | Agentic Research evolution plan |

alembic/versions/0019_memory_fts5.py

@@ -0,0 +1,89 @@
+"""memory_items FTS5 full-text index
+
+Revision ID: 0019_memory_fts5
+Revises: b94c1a2be26e
+Create Date: 2026-03-03
+
+Creates a SQLite FTS5 virtual table for memory_items.content and three triggers
+(after insert / after delete / after update) to keep it in sync.
+
+FTS5 provides BM25-ranked full-text search; falls back to LIKE on non-SQLite
+databases (Postgres) where FTS5 is not available.
+"""
+from __future__ import annotations
+
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0019_memory_fts5"
+down_revision = "b94c1a2be26e"
+branch_labels = None
+depends_on = None
+
+# --------------------------------------------------------------------------- #
+# SQLite-only helpers                                                          #
+# --------------------------------------------------------------------------- #
+
+_CREATE_FTS = """\
+CREATE VIRTUAL TABLE IF NOT EXISTS memory_items_fts
+USING fts5(content, tokenize='porter ascii');
+"""
+
+_POPULATE_FTS = """\
+INSERT INTO memory_items_fts(rowid, content)
+SELECT id, content FROM memory_items
+WHERE deleted_at IS NULL;
+"""
+
+_TRIGGER_INSERT = """\
+CREATE TRIGGER IF NOT EXISTS memory_items_fts_ai
+AFTER INSERT ON memory_items BEGIN
+    INSERT INTO memory_items_fts(rowid, content) VALUES (new.id, new.content);
+END;
+"""
+
+_TRIGGER_DELETE = """\
+CREATE TRIGGER IF NOT EXISTS memory_items_fts_ad
+AFTER DELETE ON memory_items BEGIN
+    DELETE FROM memory_items_fts WHERE rowid = old.id;
+END;
+"""
+
+_TRIGGER_UPDATE = """\
+CREATE TRIGGER IF NOT EXISTS memory_items_fts_au
+AFTER UPDATE OF content ON memory_items BEGIN
+    DELETE FROM memory_items_fts WHERE rowid = old.id;
+    INSERT INTO memory_items_fts(rowid, content) VALUES (new.id, new.content);
+END;
+"""
+
+_DROP_TRIGGERS = """\
+DROP TRIGGER IF EXISTS memory_items_fts_ai;
+DROP TRIGGER IF EXISTS memory_items_fts_ad;
+DROP TRIGGER IF EXISTS memory_items_fts_au;
+"""
+
+_DROP_FTS = "DROP TABLE IF EXISTS memory_items_fts;"
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    dialect = bind.dialect.name
+    if dialect != "sqlite":
+        # FTS5 is SQLite-only; Postgres uses pg_trgm / tsvector instead.
+        return
+
+    bind.execute(sa.text(_CREATE_FTS))
+    bind.execute(sa.text(_POPULATE_FTS))
+    bind.execute(sa.text(_TRIGGER_INSERT))
+    bind.execute(sa.text(_TRIGGER_DELETE))
+    bind.execute(sa.text(_TRIGGER_UPDATE))
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    if bind.dialect.name != "sqlite":
+        return
+
+    bind.execute(sa.text(_DROP_TRIGGERS))
+    bind.execute(sa.text(_DROP_FTS))

alembic/versions/0020_memory_embedding.py

@@ -0,0 +1,116 @@
+"""memory_items embedding column + sqlite-vec virtual table
+
+Revision ID: 0020_memory_embedding
+Revises: 0019_memory_fts5
+Create Date: 2026-03-03
+
+Phase B of issue #161:
+- Adds `embedding BLOB` column to memory_items for storing float32 vectors.
+- Creates `vec_items` sqlite-vec virtual table (SQLite + sqlite-vec only).
+- Creates sync triggers to keep vec_items in step with memory_items.embedding.
+
+Graceful degradation: if sqlite-vec is not installed the column migration still
+runs; only the virtual table and triggers are skipped.
+"""
+from __future__ import annotations
+
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0020_memory_embedding"
+down_revision = "0019_memory_fts5"
+branch_labels = None
+depends_on = None
+
+_EMBEDDING_DIM = 1536  # text-embedding-3-small
+
+_ADD_COLUMN = "ALTER TABLE memory_items ADD COLUMN embedding BLOB"
+
+_CREATE_VEC = f"""\
+CREATE VIRTUAL TABLE IF NOT EXISTS vec_items
+USING vec0(embedding float[{_EMBEDDING_DIM}])
+"""
+
+_POPULATE_VEC = """\
+INSERT OR IGNORE INTO vec_items(rowid, embedding)
+SELECT id, embedding FROM memory_items
+WHERE embedding IS NOT NULL AND deleted_at IS NULL
+"""
+
+_TRIGGER_VEC_INSERT = """\
+CREATE TRIGGER IF NOT EXISTS memory_items_vec_ai
+AFTER INSERT ON memory_items
+WHEN new.embedding IS NOT NULL
+BEGIN
+    INSERT OR REPLACE INTO vec_items(rowid, embedding) VALUES (new.id, new.embedding);
+END
+"""
+
+_TRIGGER_VEC_UPDATE = """\
+CREATE TRIGGER IF NOT EXISTS memory_items_vec_au
+AFTER UPDATE OF embedding ON memory_items
+BEGIN
+    DELETE FROM vec_items WHERE rowid = old.id;
+    INSERT OR IGNORE INTO vec_items(rowid, embedding)
+    SELECT new.id, new.embedding WHERE new.embedding IS NOT NULL;
+END
+"""
+
+_TRIGGER_VEC_DELETE = """\
+CREATE TRIGGER IF NOT EXISTS memory_items_vec_ad
+AFTER DELETE ON memory_items BEGIN
+    DELETE FROM vec_items WHERE rowid = old.id;
+END
+"""
+
+_DROP_VEC_TRIGGERS = """\
+DROP TRIGGER IF EXISTS memory_items_vec_ai;
+DROP TRIGGER IF EXISTS memory_items_vec_au;
+DROP TRIGGER IF EXISTS memory_items_vec_ad;
+"""
+_DROP_VEC_TABLE = "DROP TABLE IF EXISTS vec_items"
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    dialect = bind.dialect.name
+
+    # 1. Add embedding column (all dialects — column is used for BLOB storage).
+    try:
+        bind.execute(sa.text(_ADD_COLUMN))
+    except Exception:
+        pass  # Column already exists.
+
+    if dialect != "sqlite":
+        return  # sqlite-vec is SQLite-only.
+
+    # 2. Load sqlite-vec extension (best-effort).
+    try:
+        import sqlite_vec  # type: ignore
+        raw_conn = bind.connection.dbapi_connection  # type: ignore
+        raw_conn.enable_load_extension(True)
+        sqlite_vec.load(raw_conn)
+        raw_conn.enable_load_extension(False)
+    except Exception:
+        return  # sqlite-vec not installed — skip virtual table creation.
+
+    # 3. Create vec_items virtual table + triggers.
+    bind.execute(sa.text(_CREATE_VEC))
+    bind.execute(sa.text(_POPULATE_VEC))
+    bind.execute(sa.text(_TRIGGER_VEC_INSERT))
+    bind.execute(sa.text(_TRIGGER_VEC_UPDATE))
+    bind.execute(sa.text(_TRIGGER_VEC_DELETE))
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+
+    # Drop vec infrastructure (SQLite only; ignore errors).
+    if bind.dialect.name == "sqlite":
+        try:
+            bind.execute(sa.text(_DROP_VEC_TRIGGERS))
+            bind.execute(sa.text(_DROP_VEC_TABLE))
+        except Exception:
+            pass
+
+    # NOTE: SQLite doesn't support DROP COLUMN; leave the embedding column in place.

alembic/versions/0021_repro_code_experience.py

@@ -0,0 +1,42 @@
+"""repro_code_experience table for CodeMemory persistence
+
+Revision ID: 0021_repro_code_experience
+Revises: 0020_memory_embedding
+Create Date: 2026-03-03
+
+Issue #162: Persist CodeMemory experience data so it survives process restarts.
+Creates the repro_code_experience table with indexes on paper_id and pack_id.
+"""
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0021_repro_code_experience"
+down_revision = "0020_memory_embedding"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "repro_code_experience",
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column("pack_id", sa.String(64), nullable=True),
+        sa.Column("paper_id", sa.String(256), nullable=True),
+        sa.Column("pattern_type", sa.String(32), nullable=False),
+        sa.Column("content", sa.Text(), nullable=False, server_default=""),
+        sa.Column("code_snippet", sa.Text(), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index("ix_repro_code_experience_paper_id", "repro_code_experience", ["paper_id"])
+    op.create_index("ix_repro_code_experience_pack_id", "repro_code_experience", ["pack_id"])
+    op.create_index("ix_repro_code_experience_pattern_type", "repro_code_experience", ["pattern_type"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_repro_code_experience_pattern_type", table_name="repro_code_experience")
+    op.drop_index("ix_repro_code_experience_pack_id", table_name="repro_code_experience")
+    op.drop_index("ix_repro_code_experience_paper_id", table_name="repro_code_experience")
+    op.drop_table("repro_code_experience")