Adaptive profile extraction + confidence_score 400 fix

lib/services/adaptive-chatgpt-context-service.ts

@@ -1,4 +1,53 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import type { MemoryBridgeDbClient, MemoryBridgeNamespace } from "@/lib/services/memory-bridge-service";
 import { getHybridMemoryContext } from "@/lib/services/memory-hybrid-retrieval-service";
-export async function buildAdaptiveChatGptContext(client:MemoryBridgeDbClient,input:{user_id:string;namespace:MemoryBridgeNamespace;query?:string;current_task?:string;max_items?:number}){ const ctx=await getHybridMemoryContext(client,input); return {identity_context:"Private Pandora memory context for Joven; keep real_life and au namespaces separate.",answer_style:"Blunt, execution-focused, concise but complete. Do not overpraise. Separate coded, deployed, connected, authenticated, tool-discovered, tool-called successfully, and fully proven.",current_priorities:ctx.latest_context_pack?.key_points??[],active_projects:ctx.project_context,risk_warnings:ctx.risk_warnings,relationship_loops:ctx.open_loops.filter((l:any)=>String(l.loop_type).includes("relationship")),business_rules:[],technical_rules:["Do not call a task done without verification.","Do not save or expose secrets."],writing_rules:input.namespace==="au"?ctx.adaptive_profile:[],decision_rules:["Ask for review before saving sensitive/private memory.","Keep public read/write disabled."],do_not_forget:ctx.recent_events,do_not_do:["Do not retrieve across users or namespaces.","Do not store raw secrets.","Do not use public memory reads/writes."],retrieval_hints:ctx.retrieval_reasoning_summary,warnings:ctx.warnings,updated_at:new Date().toISOString()}; }
+
+// Flatten a profile array field (facts/preferences/...) into plain text lines for the
+// ChatGPT-facing context. Items may be strings or { text, ... } objects.
+function profileLines(items: unknown): string[] {
+  return (Array.isArray(items) ? items : [])
+    .map((item: any) => (typeof item === "string" ? item : String(item?.text ?? "")))
+    .filter((text) => text.trim().length > 0);
+}
+
+export async function buildAdaptiveChatGptContext(
+  client: MemoryBridgeDbClient,
+  input: { user_id: string; namespace: MemoryBridgeNamespace; query?: string; current_task?: string; max_items?: number },
+) {
+  const ctx = await getHybridMemoryContext(client, input);
+  // Latest active operating profile produced by refresh_adaptive_profiles.
+  const operating = (ctx.adaptive_profile ?? [])[0] ?? null;
+  const preferences = profileLines(operating?.preferences);
+  const facts = profileLines(operating?.facts);
+  const decisions = profileLines(operating?.decisions);
+  return {
+    identity_context: "Private Pandora memory context for Joven; keep real_life and au namespaces separate.",
+    answer_style:
+      "Blunt, execution-focused, concise but complete. Do not overpraise. Separate coded, deployed, connected, authenticated, tool-discovered, tool-called successfully, and fully proven.",
+    current_priorities: ctx.latest_context_pack?.key_points ?? [],
+    active_projects: ctx.project_context,
+    risk_warnings: ctx.risk_warnings,
+    relationship_loops: ctx.open_loops.filter((l: any) => String(l.loop_type).includes("relationship")),
+    // Extracted profile preferences surface as the namespace-appropriate rule set.
+    business_rules: input.namespace === "real_life" ? preferences : [],
+    technical_rules: ["Do not call a task done without verification.", "Do not save or expose secrets."],
+    writing_rules: input.namespace === "au" ? preferences : [],
+    decision_rules: [
+      ...decisions,
+      "Ask for review before saving sensitive/private memory.",
+      "Keep public read/write disabled.",
+    ],
+    // Extracted durable facts are surfaced alongside recent events.
+    do_not_forget: [...facts, ...ctx.recent_events],
+    do_not_do: [
+      "Do not retrieve across users or namespaces.",
+      "Do not store raw secrets.",
+      "Do not use public memory reads/writes.",
+    ],
+    adaptive_profile_summary: operating?.summary ?? null,
+    adaptive_profile_confidence: operating?.confidence ?? null,
+    retrieval_hints: ctx.retrieval_reasoning_summary,
+    warnings: ctx.warnings,
+    updated_at: new Date().toISOString(),
+  };
+}

lib/services/adaptive-profile-extractor.ts

@@ -0,0 +1,128 @@
+import type { MemoryBridgeNamespace } from "@/lib/services/memory-bridge-service";
+import { classifyContent, clamp01 } from "@/lib/services/memory-usefulness-scoring-service";
+
+// Deterministic adaptive-profile extraction.
+//
+// Turns a user's namespace-scoped memory events into a structured operating profile
+// (preferences, facts, decisions, open loops, risks) with a deterministic confidence.
+// Intentionally model-free and embedding-free: every output is a pure function of the
+// events' text/importance/sensitivity, so it is safe to run without PANDORA_ENABLE_MODEL_CALLS
+// or PANDORA_ENABLE_EMBEDDINGS. Reuses the Phase 5D deterministic content classifier.
+
+export type AdaptiveProfileSourceEvent = {
+  id?: string | null;
+  source?: string | null;
+  extracted_summary?: string | null;
+  raw_text?: string | null;
+  importance?: number | null;
+  sensitivity?: string | null;
+  status?: string | null;
+  created_at?: string | null;
+  memory_type?: string | null;
+};
+
+export type AdaptiveProfileItem = { text: string; event_id: string | null; importance: number | null };
+
+export type ExtractedAdaptiveProfile = {
+  summary: string;
+  facts: AdaptiveProfileItem[];
+  preferences: AdaptiveProfileItem[];
+  patterns: AdaptiveProfileItem[];
+  risks: AdaptiveProfileItem[];
+  open_loops: AdaptiveProfileItem[];
+  decisions: AdaptiveProfileItem[];
+  evidence_refs: Array<{ event_id: string | null; source: string | null }>;
+  confidence: number;
+  event_count: number;
+};
+
+const DECISION_RE = /\b(decided|decision|approved|chose|choosing|merged|agreed|confirm(?:ed)?)\b/i;
+const RISK_RE = /\b(risk|danger|concern|blocker|warning|do not|don't|avoid|must not|never|boundary|consent)\b/i;
+
+function eventText(event: AdaptiveProfileSourceEvent): string {
+  return String(event.extracted_summary ?? event.raw_text ?? "").replace(/\s+/g, " ").trim();
+}
+
+function clip(text: string, max = 240): string {
+  return text.length > max ? text.slice(0, max) : text;
+}
+
+export function extractAdaptiveProfile(
+  events: AdaptiveProfileSourceEvent[],
+  namespace: MemoryBridgeNamespace,
+): ExtractedAdaptiveProfile {
+  const facts: AdaptiveProfileItem[] = [];
+  const preferences: AdaptiveProfileItem[] = [];
+  const patterns: AdaptiveProfileItem[] = [];
+  const risks: AdaptiveProfileItem[] = [];
+  const openLoops: AdaptiveProfileItem[] = [];
+  const decisions: AdaptiveProfileItem[] = [];
+  const evidenceRefs: Array<{ event_id: string | null; source: string | null }> = [];
+  const seen = new Set<string>();
+
+  for (const event of events) {
+    if (event.status === "archived") continue;
+    const raw = eventText(event);
+    if (!raw) continue;
+    const key = raw.toLowerCase();
+    if (seen.has(key)) continue;
+    seen.add(key);
+
+    const item: AdaptiveProfileItem = { text: clip(raw), event_id: event.id ?? null, importance: event.importance ?? null };
+    evidenceRefs.push({ event_id: event.id ?? null, source: event.source ?? null });
+
+    const category = classifyContent({
+      text: raw,
+      memory_type: event.memory_type ?? null,
+      importance: event.importance ?? null,
+      source: event.source ?? null,
+    });
+    const sensitive = event.sensitivity === "high" || event.sensitivity === "private";
+    const isRisk = RISK_RE.test(raw) || sensitive;
+    const isDecision = DECISION_RE.test(raw);
+
+    if (isRisk) risks.push(item);
+    if (isDecision) decisions.push(item);
+
+    switch (category) {
+      case "durable_preference":
+        preferences.push(item);
+        break;
+      case "production_fact":
+        facts.push(item);
+        break;
+      case "task_state":
+        openLoops.push(item);
+        break;
+      default:
+        if (!isRisk && !isDecision) patterns.push(item);
+        break;
+    }
+  }
+
+  const evidenceCount = evidenceRefs.length;
+  const preferenceShare = evidenceCount ? preferences.length / evidenceCount : 0;
+  // Deterministic confidence: grows with evidence volume and preference share, capped at 0.95.
+  const confidence = evidenceCount === 0
+    ? 0.4
+    : clamp01(0.5 + Math.min(evidenceCount, 10) * 0.03 + preferenceShare * 0.15);
+
+  const summary = evidenceCount === 0
+    ? `No ${namespace} memories available to build an adaptive profile.`
+    : `Adaptive ${namespace} operating profile from ${evidenceCount} memories: `
+      + `${preferences.length} preferences, ${facts.length} facts, ${decisions.length} decisions, `
+      + `${openLoops.length} open loops, ${risks.length} risks.`;
+
+  return {
+    summary,
+    facts,
+    preferences,
+    patterns,
+    risks,
+    open_loops: openLoops,
+    decisions,
+    evidence_refs: evidenceRefs,
+    confidence,
+    event_count: evidenceCount,
+  };
+}

lib/services/memory-hybrid-retrieval-service.ts

@@ -3,8 +3,8 @@ import type { MemoryBridgeDbClient, MemoryBridgeNamespace } from "@/lib/services
 import { rankMemoriesByRetrievalWeight } from "@/lib/services/memory-usefulness-scoring-service";
 export async function getHybridMemoryContext(client:MemoryBridgeDbClient,input:{user_id:string;namespace:MemoryBridgeNamespace;query?:string;current_task?:string;max_items?:number;include_semantic?:boolean;include_profiles?:boolean;include_recent?:boolean;include_open_loops?:boolean},env:Partial<NodeJS.ProcessEnv>=process.env){ const max=Math.min(Math.max(input.max_items??8,1),Number(env.PANDORA_RETRIEVAL_MAX_ITEMS??20)); const [packs,events,profiles,loops]=await Promise.all([
  client.from("memory_context_packs").select("*").eq("user_id",input.user_id).eq("namespace",input.namespace).eq("status","active").order("created_at",{ascending:false}).limit(1) as unknown as Promise<any>,
- client.from("memory_events").select("id,source,source_ref,extracted_summary,raw_text,importance,sensitivity,status,created_at,updated_at,confidence,retrieval_weight,retrieval_count,positive_feedback_count,negative_feedback_count,superseded_by_memory_id").eq("user_id",input.user_id).eq("namespace",input.namespace).neq("status","archived").order("created_at",{ascending:false}).limit(max) as unknown as Promise<any>,
+ client.from("memory_events").select("id,source,source_ref,extracted_summary,raw_text,importance,sensitivity,status,created_at,updated_at,confidence_score,retrieval_weight,retrieval_count,positive_feedback_count,negative_feedback_count,superseded_by_memory_id").eq("user_id",input.user_id).eq("namespace",input.namespace).neq("status","archived").order("created_at",{ascending:false}).limit(max) as unknown as Promise<any>,
  input.include_profiles!==false ? client.from("memory_profiles").select("*").eq("user_id",input.user_id).eq("namespace",input.namespace).eq("status","active").order("updated_at",{ascending:false}).limit(max) as unknown as Promise<any> : Promise.resolve({data:[]}),
  input.include_open_loops!==false ? client.from("memory_open_loops").select("*").eq("user_id",input.user_id).eq("namespace",input.namespace).eq("status","open").order("updated_at",{ascending:false}).limit(max) as unknown as Promise<any> : Promise.resolve({data:[]})]);
  const warnings:string[]=[]; if(!(packs.data??[])[0]) warnings.push("no_active_context_pack"); if(env.PANDORA_ENABLE_SEMANTIC_RETRIEVAL!=="true") warnings.push("semantic_retrieval_disabled"); if(env.PANDORA_ENABLE_EMBEDDINGS!=="true") warnings.push("embeddings_disabled"); if(env.PANDORA_ENABLE_MODEL_CALLS!=="true") warnings.push("model_calls_disabled");
- return {namespace:input.namespace,current_task:input.current_task??null,adaptive_profile:(profiles.data??[]).filter((p:any)=>p.profile_type==="operating_profile"),style_profile:(profiles.data??[]).filter((p:any)=>p.profile_type==="style_profile"),project_context:(profiles.data??[]).filter((p:any)=>p.profile_type==="project_profile"),people_context:(profiles.data??[]).filter((p:any)=>p.profile_type==="person_profile"),risk_warnings:[...(profiles.data??[]).filter((p:any)=>p.profile_type==="risk_profile"),...(loops.data??[])],open_loops:loops.data??[],latest_context_pack:(packs.data??[])[0]??null,recent_events:rankMemoriesByRetrievalWeight((events.data??[]).map((e:any)=>({...e,text:e.extracted_summary??e.raw_text??""}))).map((e:any)=>({...e,text:undefined,raw_text:undefined,summary:e.extracted_summary??String(e.raw_text??"").slice(0,240)})),semantic_matches:[],retrieval_reasoning_summary:"Hybrid retrieval used active packs, recent events, active profiles, open loops, and gated semantic matches.",warnings}; }
+ return {namespace:input.namespace,current_task:input.current_task??null,adaptive_profile:(profiles.data??[]).filter((p:any)=>p.profile_type==="operating_profile"),style_profile:(profiles.data??[]).filter((p:any)=>p.profile_type==="style_profile"),project_context:(profiles.data??[]).filter((p:any)=>p.profile_type==="project_profile"),people_context:(profiles.data??[]).filter((p:any)=>p.profile_type==="person_profile"),risk_warnings:[...(profiles.data??[]).filter((p:any)=>p.profile_type==="risk_profile"),...(loops.data??[])],open_loops:loops.data??[],latest_context_pack:(packs.data??[])[0]??null,recent_events:rankMemoriesByRetrievalWeight((events.data??[]).map((e:any)=>({...e,text:e.extracted_summary??e.raw_text??"",confidence:e.confidence_score}))).map((e:any)=>({...e,text:undefined,raw_text:undefined,summary:e.extracted_summary??String(e.raw_text??"").slice(0,240)})),semantic_matches:[],retrieval_reasoning_summary:"Hybrid retrieval used active packs, recent events, active profiles, open loops, and gated semantic matches.",warnings}; }

lib/services/memory-profile-service.ts

@@ -1,5 +1,6 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import type { MemoryBridgeDbClient, MemoryBridgeNamespace } from "@/lib/services/memory-bridge-service";
+import { extractAdaptiveProfile, type AdaptiveProfileSourceEvent } from "@/lib/services/adaptive-profile-extractor";
 
 export type MemoryProfileInput = {
   user_id: string;
@@ -62,6 +63,58 @@ export async function upsertVersionedMemoryProfile(client: MemoryBridgeDbClient,
 export async function upsertProfileFromMemoryEvents(client: MemoryBridgeDbClient, input: { user_id: string; namespace: MemoryBridgeNamespace; profile_type: string; subject_key: string; summary: string; evidence_refs: unknown[]; dry_run?: boolean }) {
   return upsertVersionedMemoryProfile(client, { ...input, title: input.subject_key, confidence: input.evidence_refs.length ? 0.65 : 0.4 });
 }
+
+// Reads the user's namespace-scoped memory events, extracts a structured adaptive profile
+// deterministically (no model calls / embeddings), and upserts a versioned profile row.
+// This is what makes refresh_adaptive_profiles actually populate real profile content
+// instead of writing an empty stub.
+export async function refreshAdaptiveProfileFromEvents(
+  client: MemoryBridgeDbClient,
+  input: { user_id: string; namespace: MemoryBridgeNamespace; profile_type?: string; subject_key?: string; dry_run?: boolean; max_events?: number },
+) {
+  const limit = Math.min(Math.max(input.max_events ?? 200, 1), 500);
+  const read = await (client
+    .from("memory_events")
+    .select("id,source,source_ref,extracted_summary,raw_text,importance,sensitivity,status,created_at")
+    .eq("user_id", input.user_id)
+    .eq("namespace", input.namespace)
+    .neq("status", "archived")
+    .order("created_at", { ascending: false })
+    .limit(limit) as any as Promise<{ data: AdaptiveProfileSourceEvent[] | null; error: { message: string } | null }>);
+  const emptyCounts = { event_count: 0, preferences: 0, facts: 0, decisions: 0, open_loops: 0, risks: 0 };
+  if (read.error) {
+    return { ok: false, dry_run: !!input.dry_run, blockers: ["event_read_failed"], warnings: [read.error.message], next_step: "Check memory_events RLS and schema.", extracted: emptyCounts };
+  }
+  const extracted = extractAdaptiveProfile(read.data ?? [], input.namespace);
+  const result = await upsertVersionedMemoryProfile(client, {
+    user_id: input.user_id,
+    namespace: input.namespace,
+    profile_type: input.profile_type ?? "operating_profile",
+    subject_key: input.subject_key ?? "global",
+    title: input.subject_key ?? "global",
+    summary: extracted.summary,
+    facts: extracted.facts,
+    preferences: extracted.preferences,
+    patterns: extracted.patterns,
+    risks: extracted.risks,
+    open_loops: extracted.open_loops,
+    decisions: extracted.decisions,
+    evidence_refs: extracted.evidence_refs,
+    confidence: extracted.confidence,
+    dry_run: input.dry_run,
+  });
+  return {
+    ...result,
+    extracted: {
+      event_count: extracted.event_count,
+      preferences: extracted.preferences.length,
+      facts: extracted.facts.length,
+      decisions: extracted.decisions.length,
+      open_loops: extracted.open_loops.length,
+      risks: extracted.risks.length,
+    },
+  };
+}
 export const updateOperatingProfile = upsertProfileFromMemoryEvents;
 export const updateStyleProfile = upsertProfileFromMemoryEvents;
 export const updateRiskProfile = upsertProfileFromMemoryEvents;