diff --git a/lib/services/memory-distillation-service.ts b/lib/services/memory-distillation-service.ts index a3ba67b..9161cd5 100644 --- a/lib/services/memory-distillation-service.ts +++ b/lib/services/memory-distillation-service.ts @@ -39,19 +39,77 @@ export function extractRisks(events: MemoryEvent[]) { return topEvents(events.filter((event) => includesAny(event.raw_text, riskWords)), 10).map((event) => ({ event_id: event.id, text: asSentence(event.raw_text), source: event.source, sensitivity: event.sensitivity ?? "medium" })); } -export function extractPeopleMentions(events: MemoryEvent[]) { - const people = new Map(); +// Roadmap Sprint 1 (#1 — stabilize output): stoplist junk entities, dedupe event ids, merge +// single-token aliases into their full name, and cap sizes so people_map stays sharp instead +// of dumping every capitalized sentence-opener as a "person". +const MAX_PEOPLE = 12; +const MAX_EVENT_IDS_PER_PERSON = 8; + +// Capitalized words that are pronouns / imperatives / sentence-openers / domain labels and +// must never be treated as a person's name. +const PERSON_NAME_STOPWORDS = new Set([ + "the", "a", "an", "this", "that", "these", "those", "it", "its", + "he", "him", "his", "she", "her", "hers", "they", "them", "their", "we", "us", "our", "ours", "you", "your", "yours", "i", "me", "my", "mine", + "do", "don", "dont", "does", "did", "doing", "done", "use", "using", "used", "add", "set", "make", "made", "ask", "treat", "note", "source", + "keep", "avoid", "preserve", "allow", "allowed", "required", "reinforced", "working", "current", "best", "direction", "essential", "future", "important", + "if", "when", "then", "else", "and", "but", "or", "nor", "for", "so", "yet", "not", "no", "yes", "never", "always", "only", "also", + "before", "during", "after", "every", "each", "all", "any", "some", "main", "core", "both", "new", "old", + "rule", "rules", "story", "scene", "scenes", "style", "canon", "user", "users", "taglish", "status", + "pandora", "chatgpt", "memory", +]); + +type PersonEntry = { name: string; event_ids: string[]; notes: string[] }; + +function isLikelyPersonName(name: string): boolean { + const tokens = name.split(/\s+/).filter(Boolean); + if (tokens.length === 0) return false; + if (PERSON_NAME_STOPWORDS.has(tokens[0].toLowerCase())) return false; + if (tokens.length === 1) return tokens[0].length >= 3; + return true; +} + +// Merge a single-token name (e.g. "Janine") into a multi-token name that starts with it +// (e.g. "Janine Tan"). Distinct aliases with no shared full name (e.g. "Jana") stay separate. +function canonicalizePeople(people: Map): PersonEntry[] { + const entries = [...people.values()]; + const multi = entries.filter((entry) => entry.name.includes(" ")); + const kept: PersonEntry[] = []; + for (const entry of entries) { + if (!entry.name.includes(" ")) { + const target = multi.find((m) => m.name.split(/\s+/)[0].toLowerCase() === entry.name.toLowerCase()); + if (target && target !== entry) { + for (const id of entry.event_ids) if (!target.event_ids.includes(id)) target.event_ids.push(id); + for (const note of entry.notes) if (target.notes.length < 2 && !target.notes.includes(note)) target.notes.push(note); + continue; + } + } + kept.push(entry); + } + return kept; +} + +export function extractPeopleMentions(events: MemoryEvent[], opts: { maxPeople?: number; maxEventIdsPerPerson?: number } = {}) { + const maxPeople = opts.maxPeople ?? MAX_PEOPLE; + const maxIds = opts.maxEventIdsPerPerson ?? MAX_EVENT_IDS_PER_PERSON; + const people = new Map(); for (const event of events) { - for (const match of event.raw_text.matchAll(/\b[A-Z][a-z]+(?:\s[A-Z][a-z]+)?\b/g)) { - const name = match[0]; - if (["Pandora", "ChatGPT", "Memory"].includes(name)) continue; + // One event contributes each distinct name at most once (no per-occurrence id duplication). + const namesInEvent = new Set(); + for (const match of event.raw_text.matchAll(/\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+){0,2}\b/g)) { + const name = match[0].trim(); + if (isLikelyPersonName(name)) namesInEvent.add(name); + } + for (const name of namesInEvent) { const entry = people.get(name) ?? { name, event_ids: [], notes: [] }; - entry.event_ids.push(event.id); + if (!entry.event_ids.includes(event.id)) entry.event_ids.push(event.id); if (entry.notes.length < 2) entry.notes.push(asSentence(event.raw_text)); people.set(name, entry); } } - return [...people.values()].sort((a, b) => b.event_ids.length - a.event_ids.length).slice(0, 12); + return canonicalizePeople(people) + .sort((a, b) => b.event_ids.length - a.event_ids.length || a.name.localeCompare(b.name)) + .slice(0, maxPeople) + .map((entry) => ({ name: entry.name, event_ids: entry.event_ids.slice(0, maxIds), notes: entry.notes })); } export function extractProjectMentions(events: MemoryEvent[]) { @@ -91,8 +149,31 @@ export function buildMasterContextPack(namespace: MemoryBridgeNamespace, userId: }; } -export function compactContextResponse(pack: MemoryContextPack | null, events: MemoryEvent[], input: { include_risks?: boolean; include_people?: boolean; include_projects?: boolean }) { - return { +const DEFAULT_MAX_PAYLOAD_CHARS = 12000; +function payloadChars(value: unknown): number { return JSON.stringify(value).length; } + +// Progressive, deterministic slimming so a context response never dumps a giant payload. +// Trims the heaviest fields first (people event ids/notes), then list lengths, then the summary. +function slimContextResponse(response: Record, maxChars: number): Record { + const list = (value: unknown): unknown[] => (Array.isArray(value) ? value : []); + if (payloadChars(response) <= maxChars) return response; + response.people_map = list(response.people_map).map((person) => { + const p = (person ?? {}) as Record; + return { ...p, event_ids: list(p.event_ids).slice(0, 3), notes: list(p.notes).slice(0, 1) }; + }); + if (payloadChars(response) <= maxChars) return response; + response.key_points = list(response.key_points).slice(0, 6); + response.open_loops = list(response.open_loops).slice(0, 6); + response.risks = list(response.risks).slice(0, 6); + response.active_projects = list(response.active_projects).slice(0, 6); + if (payloadChars(response) <= maxChars) return response; + response.people_map = list(response.people_map).slice(0, 6); + response.summary = String(response.summary ?? "").slice(0, 1200); + return response; +} + +export function compactContextResponse(pack: MemoryContextPack | null, events: MemoryEvent[], input: { include_risks?: boolean; include_people?: boolean; include_projects?: boolean; max_payload_chars?: number; debug?: boolean }) { + const response = { title: pack?.title ?? "Pandora context pack unavailable", summary: pack?.summary ?? summarizeEventsDeterministically(events), key_points: pack?.key_points ?? keyPoints(events), @@ -107,4 +188,7 @@ export function compactContextResponse(pack: MemoryContextPack | null, events: M "Ask before storing new long-term memories.", ], }; + // debug mode returns the full payload; default responses are capped to stay compact. + if (input.debug) return response; + return slimContextResponse(response, Math.max(2000, Number(input.max_payload_chars ?? DEFAULT_MAX_PAYLOAD_CHARS))); } diff --git a/lib/services/pandora-mcp-tools.ts b/lib/services/pandora-mcp-tools.ts index 960c41d..ff246f1 100644 --- a/lib/services/pandora-mcp-tools.ts +++ b/lib/services/pandora-mcp-tools.ts @@ -16,7 +16,7 @@ import { disabledPostAnswer, runPostAnswerTurn, runPreAnswerTurn } from "@/lib/s const namespaceSchema = z.enum(["real_life", "au"]); export const latestContextPackInputSchema = z.object({ namespace: namespaceSchema, pack_type: z.enum(["daily", "master"]).optional() }); -export const memoryContextInputSchema = z.object({ namespace: namespaceSchema, query: z.string().optional(), current_task: z.string().optional(), max_items: z.number().int().positive().max(100).optional(), include_risks: z.boolean().optional(), include_people: z.boolean().optional(), include_projects: z.boolean().optional() }); +export const memoryContextInputSchema = z.object({ namespace: namespaceSchema, query: z.string().optional(), current_task: z.string().optional(), max_items: z.number().int().positive().max(100).optional(), include_risks: z.boolean().optional(), include_people: z.boolean().optional(), include_projects: z.boolean().optional(), max_payload_chars: z.number().int().positive().max(200000).optional(), debug: z.boolean().optional() }); export const captureMemoryEventInputSchema = z.object({ namespace: namespaceSchema, raw_text: z.string().trim().min(1).max(8000), source: z.string().trim().max(120).optional(), source_ref: z.string().trim().max(500).optional(), importance: z.number().int().min(1).max(10).optional(), sensitivity: z.enum(["low", "medium", "high", "private"]).optional() }); export const distillContextPackInputSchema = z.object({ namespace: namespaceSchema, pack_type: z.enum(["daily", "master"]) }); const runtime = (capture = false, distill = false) => ({ config: { memoryCaptureApiEnabled: capture, memoryContextApiEnabled: true, memoryDistillationEnabled: distill }, gates: { memoryCaptureApiEnabled: { envVar: "PANDORA_ENABLE_MCP_CAPTURE" }, memoryContextApiEnabled: { envVar: "PANDORA_ENABLE_MCP" }, memoryDistillationEnabled: { envVar: "PANDORA_ENABLE_MCP_DISTILLATION" } } }) as never; diff --git a/tests/unit/context-stabilization.test.ts b/tests/unit/context-stabilization.test.ts new file mode 100644 index 0000000..ac8ed96 --- /dev/null +++ b/tests/unit/context-stabilization.test.ts @@ -0,0 +1,75 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { describe, expect, it } from "vitest"; +import { extractPeopleMentions, compactContextResponse } from "@/lib/services/memory-distillation-service"; + +function ev(id: string, raw_text: string): any { + return { id, namespace: "au", user_id: "u", source: "chatgpt_user_direct", raw_text, status: "captured", created_by: "u", created_at: "2026-07-03T00:00:00Z" }; +} + +describe("Sprint 1 — stabilize output (people_map + payload)", () => { + it("drops junk capitalized sentence-openers and keeps real names", () => { + const events = [ + ev("e1", "REINFORCED AU MEMORY RULE. Janine Tan is the character. She is agentic. Mang Jun is raw. Do not link real identity. Keep this. User asked to save."), + ]; + const people = extractPeopleMentions(events); + const names = people.map((p) => p.name); + + expect(names).toContain("Janine Tan"); + expect(names).toContain("Mang Jun"); + for (const junk of ["The", "Do", "She", "He", "Keep", "User", "Rule", "Status", "Source"]) { + expect(names).not.toContain(junk); + } + }); + + it("counts each event once per person (no per-occurrence id duplication)", () => { + const events = [ev("e1", "Janine Tan. Janine Tan. Janine Tan smiled at Janine Tan.")]; + const [person] = extractPeopleMentions(events); + expect(person.name).toBe("Janine Tan"); + expect(person.event_ids).toEqual(["e1"]); + }); + + it("merges a single-token alias into its full name but keeps distinct aliases separate", () => { + const events = [ + ev("e1", "Janine Tan arrived."), + ev("e2", "Janine waited."), + ev("e3", "Jana ran off."), + ]; + const names = extractPeopleMentions(events).map((p) => p.name); + expect(names).toContain("Janine Tan"); + expect(names).not.toContain("Janine"); // merged into "Janine Tan" + expect(names).toContain("Jana"); // distinct alias kept + const janine = extractPeopleMentions(events).find((p) => p.name === "Janine Tan")!; + expect(janine.event_ids.sort()).toEqual(["e1", "e2"]); + }); + + it("caps people count and event ids per person", () => { + const firsts = ["Aaron", "Bella", "Cara", "Dana", "Ella", "Faye", "Gina", "Hana", "Iris", "Jane", "Kira", "Lena", "Mona", "Nora", "Opal"]; + const manyPeople = firsts.map((first, i) => ev(`p${i}`, `${first} Zeta did a thing.`)); + expect(extractPeopleMentions(manyPeople).length).toBe(12); + + const manyEvents = Array.from({ length: 12 }, (_, i) => ev(`e${i}`, "Janine Tan noted something.")); + const [person] = extractPeopleMentions(manyEvents); + expect(person.event_ids.length).toBe(8); + }); + + it("slims an oversized context response under the payload budget, and debug bypasses it", () => { + const pack: any = { + title: "Pandora master context pack", + summary: "short summary", + key_points: [], + active_projects: [], + people_map: [{ name: "Janine Tan", event_ids: Array.from({ length: 500 }, (_, i) => `event-${i}`), notes: ["a note"] }], + decisions: [], + risks: [], + open_loops: [], + }; + + const slim = compactContextResponse(pack, [], { max_payload_chars: 3000 }); + expect(JSON.stringify(slim).length).toBeLessThanOrEqual(3000); + expect(slim.people_map[0].event_ids.length).toBeLessThanOrEqual(3); + + const full = compactContextResponse(pack, [], { debug: true }); + expect(JSON.stringify(full).length).toBeGreaterThan(3000); + expect(full.people_map[0].event_ids.length).toBe(500); + }); +});