Stabilize memory context output (roadmap Sprint 1)

lib/services/memory-distillation-service.ts

@@ -39,19 +39,77 @@ export function extractRisks(events: MemoryEvent[]) {
   return topEvents(events.filter((event) => includesAny(event.raw_text, riskWords)), 10).map((event) => ({ event_id: event.id, text: asSentence(event.raw_text), source: event.source, sensitivity: event.sensitivity ?? "medium" }));
 }
 
-export function extractPeopleMentions(events: MemoryEvent[]) {
-  const people = new Map<string, { name: string; event_ids: string[]; notes: string[] }>();
+// Roadmap Sprint 1 (#1 — stabilize output): stoplist junk entities, dedupe event ids, merge
+// single-token aliases into their full name, and cap sizes so people_map stays sharp instead
+// of dumping every capitalized sentence-opener as a "person".
+const MAX_PEOPLE = 12;
+const MAX_EVENT_IDS_PER_PERSON = 8;
+
+// Capitalized words that are pronouns / imperatives / sentence-openers / domain labels and
+// must never be treated as a person's name.
+const PERSON_NAME_STOPWORDS = new Set<string>([
+  "the", "a", "an", "this", "that", "these", "those", "it", "its",
+  "he", "him", "his", "she", "her", "hers", "they", "them", "their", "we", "us", "our", "ours", "you", "your", "yours", "i", "me", "my", "mine",
+  "do", "don", "dont", "does", "did", "doing", "done", "use", "using", "used", "add", "set", "make", "made", "ask", "treat", "note", "source",
+  "keep", "avoid", "preserve", "allow", "allowed", "required", "reinforced", "working", "current", "best", "direction", "essential", "future", "important",
+  "if", "when", "then", "else", "and", "but", "or", "nor", "for", "so", "yet", "not", "no", "yes", "never", "always", "only", "also",
+  "before", "during", "after", "every", "each", "all", "any", "some", "main", "core", "both", "new", "old",
+  "rule", "rules", "story", "scene", "scenes", "style", "canon", "user", "users", "taglish", "status",
+  "pandora", "chatgpt", "memory",
+]);
+
+type PersonEntry = { name: string; event_ids: string[]; notes: string[] };
+
+function isLikelyPersonName(name: string): boolean {
+  const tokens = name.split(/\s+/).filter(Boolean);
+  if (tokens.length === 0) return false;
+  if (PERSON_NAME_STOPWORDS.has(tokens[0].toLowerCase())) return false;
+  if (tokens.length === 1) return tokens[0].length >= 3;
+  return true;
+}
+
+// Merge a single-token name (e.g. "Janine") into a multi-token name that starts with it
+// (e.g. "Janine Tan"). Distinct aliases with no shared full name (e.g. "Jana") stay separate.
+function canonicalizePeople(people: Map<string, PersonEntry>): PersonEntry[] {
+  const entries = [...people.values()];
+  const multi = entries.filter((entry) => entry.name.includes(" "));
+  const kept: PersonEntry[] = [];
+  for (const entry of entries) {
+    if (!entry.name.includes(" ")) {
+      const target = multi.find((m) => m.name.split(/\s+/)[0].toLowerCase() === entry.name.toLowerCase());
+      if (target && target !== entry) {
+        for (const id of entry.event_ids) if (!target.event_ids.includes(id)) target.event_ids.push(id);
+        for (const note of entry.notes) if (target.notes.length < 2 && !target.notes.includes(note)) target.notes.push(note);
+        continue;
+      }
+    }
+    kept.push(entry);
+  }
+  return kept;
+}
+
+export function extractPeopleMentions(events: MemoryEvent[], opts: { maxPeople?: number; maxEventIdsPerPerson?: number } = {}) {
+  const maxPeople = opts.maxPeople ?? MAX_PEOPLE;
+  const maxIds = opts.maxEventIdsPerPerson ?? MAX_EVENT_IDS_PER_PERSON;
+  const people = new Map<string, PersonEntry>();
   for (const event of events) {
-    for (const match of event.raw_text.matchAll(/\b[A-Z][a-z]+(?:\s[A-Z][a-z]+)?\b/g)) {
-      const name = match[0];
-      if (["Pandora", "ChatGPT", "Memory"].includes(name)) continue;
+    // One event contributes each distinct name at most once (no per-occurrence id duplication).
+    const namesInEvent = new Set<string>();
+    for (const match of event.raw_text.matchAll(/\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+){0,2}\b/g)) {
+      const name = match[0].trim();
+      if (isLikelyPersonName(name)) namesInEvent.add(name);
+    }
+    for (const name of namesInEvent) {
       const entry = people.get(name) ?? { name, event_ids: [], notes: [] };
-      entry.event_ids.push(event.id);
+      if (!entry.event_ids.includes(event.id)) entry.event_ids.push(event.id);
       if (entry.notes.length < 2) entry.notes.push(asSentence(event.raw_text));
       people.set(name, entry);
     }
   }
-  return [...people.values()].sort((a, b) => b.event_ids.length - a.event_ids.length).slice(0, 12);
+  return canonicalizePeople(people)
+    .sort((a, b) => b.event_ids.length - a.event_ids.length || a.name.localeCompare(b.name))
+    .slice(0, maxPeople)
+    .map((entry) => ({ name: entry.name, event_ids: entry.event_ids.slice(0, maxIds), notes: entry.notes }));
 }
 
 export function extractProjectMentions(events: MemoryEvent[]) {
@@ -91,8 +149,31 @@ export function buildMasterContextPack(namespace: MemoryBridgeNamespace, userId:
   };
 }
 
-export function compactContextResponse(pack: MemoryContextPack | null, events: MemoryEvent[], input: { include_risks?: boolean; include_people?: boolean; include_projects?: boolean }) {
-  return {
+const DEFAULT_MAX_PAYLOAD_CHARS = 12000;
+function payloadChars(value: unknown): number { return JSON.stringify(value).length; }
+
+// Progressive, deterministic slimming so a context response never dumps a giant payload.
+// Trims the heaviest fields first (people event ids/notes), then list lengths, then the summary.
+function slimContextResponse(response: Record<string, unknown>, maxChars: number): Record<string, unknown> {
+  const list = (value: unknown): unknown[] => (Array.isArray(value) ? value : []);
+  if (payloadChars(response) <= maxChars) return response;
+  response.people_map = list(response.people_map).map((person) => {
+    const p = (person ?? {}) as Record<string, unknown>;
+    return { ...p, event_ids: list(p.event_ids).slice(0, 3), notes: list(p.notes).slice(0, 1) };
+  });
+  if (payloadChars(response) <= maxChars) return response;
+  response.key_points = list(response.key_points).slice(0, 6);
+  response.open_loops = list(response.open_loops).slice(0, 6);
+  response.risks = list(response.risks).slice(0, 6);
+  response.active_projects = list(response.active_projects).slice(0, 6);
+  if (payloadChars(response) <= maxChars) return response;
+  response.people_map = list(response.people_map).slice(0, 6);
+  response.summary = String(response.summary ?? "").slice(0, 1200);
+  return response;
+}
+
+export function compactContextResponse(pack: MemoryContextPack | null, events: MemoryEvent[], input: { include_risks?: boolean; include_people?: boolean; include_projects?: boolean; max_payload_chars?: number; debug?: boolean }) {
+  const response = {
     title: pack?.title ?? "Pandora context pack unavailable",
     summary: pack?.summary ?? summarizeEventsDeterministically(events),
     key_points: pack?.key_points ?? keyPoints(events),
@@ -107,4 +188,7 @@ export function compactContextResponse(pack: MemoryContextPack | null, events: M
       "Ask before storing new long-term memories.",
     ],
   };
+  // debug mode returns the full payload; default responses are capped to stay compact.
+  if (input.debug) return response;
+  return slimContextResponse(response, Math.max(2000, Number(input.max_payload_chars ?? DEFAULT_MAX_PAYLOAD_CHARS)));
 }

lib/services/pandora-mcp-tools.ts

@@ -16,7 +16,7 @@ import { disabledPostAnswer, runPostAnswerTurn, runPreAnswerTurn } from "@/lib/s
 
 const namespaceSchema = z.enum(["real_life", "au"]);
 export const latestContextPackInputSchema = z.object({ namespace: namespaceSchema, pack_type: z.enum(["daily", "master"]).optional() });
-export const memoryContextInputSchema = z.object({ namespace: namespaceSchema, query: z.string().optional(), current_task: z.string().optional(), max_items: z.number().int().positive().max(100).optional(), include_risks: z.boolean().optional(), include_people: z.boolean().optional(), include_projects: z.boolean().optional() });
+export const memoryContextInputSchema = z.object({ namespace: namespaceSchema, query: z.string().optional(), current_task: z.string().optional(), max_items: z.number().int().positive().max(100).optional(), include_risks: z.boolean().optional(), include_people: z.boolean().optional(), include_projects: z.boolean().optional(), max_payload_chars: z.number().int().positive().max(200000).optional(), debug: z.boolean().optional() });
 export const captureMemoryEventInputSchema = z.object({ namespace: namespaceSchema, raw_text: z.string().trim().min(1).max(8000), source: z.string().trim().max(120).optional(), source_ref: z.string().trim().max(500).optional(), importance: z.number().int().min(1).max(10).optional(), sensitivity: z.enum(["low", "medium", "high", "private"]).optional() });
 export const distillContextPackInputSchema = z.object({ namespace: namespaceSchema, pack_type: z.enum(["daily", "master"]) });
 const runtime = (capture = false, distill = false) => ({ config: { memoryCaptureApiEnabled: capture, memoryContextApiEnabled: true, memoryDistillationEnabled: distill }, gates: { memoryCaptureApiEnabled: { envVar: "PANDORA_ENABLE_MCP_CAPTURE" }, memoryContextApiEnabled: { envVar: "PANDORA_ENABLE_MCP" }, memoryDistillationEnabled: { envVar: "PANDORA_ENABLE_MCP_DISTILLATION" } } }) as never;

tests/unit/context-stabilization.test.ts

@@ -0,0 +1,75 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { describe, expect, it } from "vitest";
+import { extractPeopleMentions, compactContextResponse } from "@/lib/services/memory-distillation-service";
+
+function ev(id: string, raw_text: string): any {
+  return { id, namespace: "au", user_id: "u", source: "chatgpt_user_direct", raw_text, status: "captured", created_by: "u", created_at: "2026-07-03T00:00:00Z" };
+}
+
+describe("Sprint 1 — stabilize output (people_map + payload)", () => {
+  it("drops junk capitalized sentence-openers and keeps real names", () => {
+    const events = [
+      ev("e1", "REINFORCED AU MEMORY RULE. Janine Tan is the character. She is agentic. Mang Jun is raw. Do not link real identity. Keep this. User asked to save."),
+    ];
+    const people = extractPeopleMentions(events);
+    const names = people.map((p) => p.name);
+
+    expect(names).toContain("Janine Tan");
+    expect(names).toContain("Mang Jun");
+    for (const junk of ["The", "Do", "She", "He", "Keep", "User", "Rule", "Status", "Source"]) {
+      expect(names).not.toContain(junk);
+    }
+  });
+
+  it("counts each event once per person (no per-occurrence id duplication)", () => {
+    const events = [ev("e1", "Janine Tan. Janine Tan. Janine Tan smiled at Janine Tan.")];
+    const [person] = extractPeopleMentions(events);
+    expect(person.name).toBe("Janine Tan");
+    expect(person.event_ids).toEqual(["e1"]);
+  });
+
+  it("merges a single-token alias into its full name but keeps distinct aliases separate", () => {
+    const events = [
+      ev("e1", "Janine Tan arrived."),
+      ev("e2", "Janine waited."),
+      ev("e3", "Jana ran off."),
+    ];
+    const names = extractPeopleMentions(events).map((p) => p.name);
+    expect(names).toContain("Janine Tan");
+    expect(names).not.toContain("Janine"); // merged into "Janine Tan"
+    expect(names).toContain("Jana"); // distinct alias kept
+    const janine = extractPeopleMentions(events).find((p) => p.name === "Janine Tan")!;
+    expect(janine.event_ids.sort()).toEqual(["e1", "e2"]);
+  });
+
+  it("caps people count and event ids per person", () => {
+    const firsts = ["Aaron", "Bella", "Cara", "Dana", "Ella", "Faye", "Gina", "Hana", "Iris", "Jane", "Kira", "Lena", "Mona", "Nora", "Opal"];
+    const manyPeople = firsts.map((first, i) => ev(`p${i}`, `${first} Zeta did a thing.`));
+    expect(extractPeopleMentions(manyPeople).length).toBe(12);
+
+    const manyEvents = Array.from({ length: 12 }, (_, i) => ev(`e${i}`, "Janine Tan noted something."));
+    const [person] = extractPeopleMentions(manyEvents);
+    expect(person.event_ids.length).toBe(8);
+  });
+
+  it("slims an oversized context response under the payload budget, and debug bypasses it", () => {
+    const pack: any = {
+      title: "Pandora master context pack",
+      summary: "short summary",
+      key_points: [],
+      active_projects: [],
+      people_map: [{ name: "Janine Tan", event_ids: Array.from({ length: 500 }, (_, i) => `event-${i}`), notes: ["a note"] }],
+      decisions: [],
+      risks: [],
+      open_loops: [],
+    };
+
+    const slim = compactContextResponse(pack, [], { max_payload_chars: 3000 });
+    expect(JSON.stringify(slim).length).toBeLessThanOrEqual(3000);
+    expect(slim.people_map[0].event_ids.length).toBeLessThanOrEqual(3);
+
+    const full = compactContextResponse(pack, [], { debug: true });
+    expect(JSON.stringify(full).length).toBeGreaterThan(3000);
+    expect(full.people_map[0].event_ids.length).toBe(500);
+  });
+});