From 31d9f9593c9f8dc17ba864e5ac5000e045b7aba1 Mon Sep 17 00:00:00 2001 From: TriDefender Date: Tue, 11 Aug 2026 01:12:23 +0800 Subject: [PATCH 01/29] =?UTF-8?q?Fix:=20=E6=B7=BB=E5=8A=A0=E4=BA=86?= =?UTF-8?q?=E6=9C=AA=E6=8F=90=E5=8F=96=E8=AE=B0=E5=BF=86=E8=A1=A5=E5=81=BF?= =?UTF-8?q?=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 可以重新尝试提取之前未提取的记忆节点，在正式提取前会让用户确认。 --- index.ts | 1 + package.json | 3 +- src/cli-extract.ts | 288 +++++++++++++++++++++++++++ src/cli.ts | 55 ++++++ src/store/store.ts | 31 +++ test/cli-extract.test.ts | 417 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 794 insertions(+), 1 deletion(-) create mode 100644 src/cli-extract.ts create mode 100644 test/cli-extract.test.ts diff --git a/index.ts b/index.ts index 8397aa5..2df04fd 100755 --- a/index.ts +++ b/index.ts @@ -258,6 +258,7 @@ const graphMemoryProPlugin = { pluginId: "graph-memory-pro", pluginConfig: raw as Record | undefined, resolveConfigPath: (p: string) => api.resolvePath?.(p) ?? p, + defaultModel: readDefaultModel(api.config), }), { commands: ["graph-memory"] }, ); diff --git a/package.json b/package.json index 4ea628f..2c0112d 100755 --- a/package.json +++ b/package.json @@ -24,8 +24,9 @@ "test:watch": "vitest --passWithNoTests" }, "dependencies": { + "@sinclair/typebox": "^0.34.48", "neo4j-driver": "^5.27.0", - "@sinclair/typebox": "^0.34.48" + "opencode-ai": "^1.18.16" }, "devDependencies": { "@types/node": "^20.0.0", diff --git a/src/cli-extract.ts b/src/cli-extract.ts new file mode 100644 index 0000000..cfe9d18 --- /dev/null +++ b/src/cli-extract.ts @@ -0,0 +1,288 @@ +/** + * graph-memory-pro CLI — `openclaw graph-memory extract` + * + * 对未被提取的会话消息做批量图谱提取，补齐因 compact 未触发、提取失败或 + * 进程退出而残留的 GmMessage。流程镜像 index.ts 的 compact() 路径： + * getUnextracted → extractor.extract → upsertNode + syncEmbed → upsertEdge → markExtracted + * + * 命令在 cli-metadata 模式下运行（register() 早早 return），所以这里必须自行 + * 完成 Neo4j driver / schema / LLM / embedder / Extractor / Recaller 的初始化。 + */ + +import readline from "node:readline/promises"; +import { stdin as input, stdout as output } from "node:process"; + +import type { Driver } from "neo4j-driver"; +import type { GmConfig } from "./types.ts"; +import { getDriver, initSchema, closeDriver } from "./store/db.ts"; +import { + listUnextractedSessions, + getUnextracted, + markExtracted, + upsertNode, + upsertEdge, + findByName, + getBySession, + type UnextractedSessionInfo, +} from "./store/store.ts"; +import { createCompleteFn, resolveProvider } from "./engine/llm.ts"; +import { createEmbedFn } from "./engine/embed.ts"; +import { Recaller } from "./recaller/recall.ts"; +import { Extractor } from "./extractor/extract.ts"; + +const AFFIRMATIVE = new Set(["y", "yes", "yeah", "yep", "ok", "okay", "true", "1", "confirm"]); + +export function isAffirmative(answer: string): boolean { + return AFFIRMATIVE.has(answer.trim().toLowerCase()); +} + +export interface BackfillExtractOptions { + yes?: boolean; + limit?: number; + session?: string; + dryRun?: boolean; +} + +export interface BackfillExtractParams { + cfg: GmConfig; + effectiveModel: string; + options: BackfillExtractOptions; + log?: (msg: string) => void; + prompt?: (question: string) => Promise; +} + +export interface BackfillExtractResult { + sessionsTotal: number; + sessionsProcessed: number; + sessionsSkipped: number; + nodesCreated: number; + edgesCreated: number; + batches: number; + durationMs: number; +} + +const DEFAULT_BATCH_LIMIT_MULTIPLIER = 3; + +function defaultLog(msg: string): void { + console.log(msg); +} + +function formatSessionLine(info: UnextractedSessionInfo, index: number): string { + const created = info.minCreatedAt > 0 + ? new Date(info.minCreatedAt).toISOString().replace("T", " ").slice(0, 19) + : "?"; + return ` ${String(index + 1).padStart(3, " ")}. sid=${info.sessionId.slice(0, 12)}… msgs=${info.messageCount} maxTurn=${info.maxTurn} since=${created}`; +} + +export async function runBackfillExtraction( + params: BackfillExtractParams, +): Promise { + const start = Date.now(); + const log = params.log ?? defaultLog; + const opts = params.options; + const cfg = params.cfg; + + const result: BackfillExtractResult = { + sessionsTotal: 0, + sessionsProcessed: 0, + sessionsSkipped: 0, + nodesCreated: 0, + edgesCreated: 0, + batches: 0, + durationMs: 0, + }; + + if (!cfg.neo4j?.uri) { + throw new Error( + "[graph-memory-pro] extract 需要 neo4j.uri 配置。请在 graph-memory-pro 插件配置中设置 neo4j.uri / neo4j.user / neo4j.password。", + ); + } + + if (!params.effectiveModel) { + throw new Error( + "[graph-memory-pro] extract 需要一个 LLM model。请在 config.llm.model 或 agents.defaults.model 中设置。", + ); + } + + const providerInfo = resolveProvider(cfg.llm); + if (providerInfo.provider === "anthropic" && !cfg.llm?.apiKey) { + throw new Error("[graph-memory-pro] llm.provider=anthropic 但未配 llm.apiKey，无法提取。"); + } + if (providerInfo.provider === "openai" && (!cfg.llm?.apiKey || !cfg.llm?.baseURL)) { + throw new Error("[graph-memory-pro] llm.provider=openai 需要 llm.apiKey + llm.baseURL，无法提取。"); + } + if (providerInfo.provider === "oauth" && !cfg.llm?.oauthPath) { + throw new Error( + "[graph-memory-pro] llm.provider=oauth 但未配 llm.oauthPath。请先运行 `openclaw graph-memory auth login`。", + ); + } + + const driver: Driver = getDriver(cfg.neo4j); + + try { + log("[graph-memory-pro] 正在初始化 Neo4j schema..."); + await initSchema(driver, cfg.embedding); + + log("[graph-memory-pro] 正在初始化 LLM 与 embedder..."); + const llm = createCompleteFn(params.effectiveModel, cfg.llm); + const extractor = new Extractor(llm); + const recaller = new Recaller(driver, cfg); + const embedFn = await createEmbedFn(cfg.embedding); + if (embedFn) { + recaller.setEmbedFn(embedFn); + log("[graph-memory-pro] embedding 已就绪，新节点将同步向量。"); + } else { + log("[graph-memory-pro] 未配置 embedding，跳过向量同步（dual-path recall 会降级为文本搜索）。"); + } + + let sessions = await listUnextractedSessions(driver); + if (opts.session) { + sessions = sessions.filter(s => s.sessionId === opts.session); + if (!sessions.length) { + log(`[graph-memory-pro] --session=${opts.session} 没有匹配到含未提取消息的会话。`); + result.durationMs = Date.now() - start; + return result; + } + } + result.sessionsTotal = sessions.length; + + if (sessions.length === 0) { + log("[graph-memory-pro] 没有需要提取的会话。"); + result.durationMs = Date.now() - start; + return result; + } + + const totalMessages = sessions.reduce((s, info) => s + info.messageCount, 0); + log(`[graph-memory-pro] 发现 ${sessions.length} 个会话共 ${totalMessages} 条未提取消息：`); + sessions.forEach((info, i) => log(formatSessionLine(info, i))); + + if (opts.dryRun) { + log("[graph-memory-pro] --dry-run 模式，未执行提取。"); + result.sessionsSkipped = sessions.length; + result.durationMs = Date.now() - start; + return result; + } + + if (!opts.yes) { + const prompt = params.prompt ?? ((q: string) => defaultPrompt(q)); + const answer = await prompt(`\n将对以上 ${sessions.length} 个会话发起 LLM 提取，继续？[y/N] `); + if (!isAffirmative(answer)) { + log("[graph-memory-pro] 已取消。"); + result.sessionsSkipped = sessions.length; + result.durationMs = Date.now() - start; + return result; + } + } + + const batchLimit = opts.limit && opts.limit > 0 + ? opts.limit + : Math.max(1, cfg.compactTurnCount) * DEFAULT_BATCH_LIMIT_MULTIPLIER; + + log(`\n[graph-memory-pro] 开始提取（每批最多 ${batchLimit} 条消息）...`); + + for (const info of sessions) { + log(`\n[graph-memory-pro] 会话 ${info.sessionId.slice(0, 12)}… (${info.messageCount} 条消息)`); + try { + const processed = await extractSessionLoop(driver, extractor, recaller, info.sessionId, batchLimit, log); + result.nodesCreated += processed.nodes; + result.edgesCreated += processed.edges; + result.batches += processed.batches; + result.sessionsProcessed += 1; + log(` -> 完成：${processed.nodes} 节点 / ${processed.edges} 边 / ${processed.batches} 批`); + } catch (err) { + result.sessionsSkipped += 1; + log(` -> 失败：${err instanceof Error ? err.message : String(err)}`); + } + } + + result.durationMs = Date.now() - start; + + log( + `\n[graph-memory-pro] 提取完成：${result.sessionsProcessed}/${result.sessionsTotal} 会话，` + + `${result.nodesCreated} 节点，${result.edgesCreated} 边，${result.batches} 批，` + + `用时 ${(result.durationMs / 1000).toFixed(1)}s`, + ); + return result; + } finally { + await closeDriver(); + } +} + +interface SessionExtractStats { + nodes: number; + edges: number; + batches: number; +} + +async function extractSessionLoop( + driver: Driver, + extractor: Extractor, + recaller: Recaller, + sessionId: string, + batchLimit: number, + log: (msg: string) => void, +): Promise { + const stats: SessionExtractStats = { nodes: 0, edges: 0, batches: 0 }; + const hardBatchCeiling = 50; + let exhausted = false; + + for (let i = 0; i < hardBatchCeiling; i++) { + const msgs = await getUnextracted(driver, sessionId, batchLimit); + if (!msgs.length) break; + + stats.batches += 1; + const existing = (await getBySession(driver, sessionId)).map(n => n.name); + const extraction = await extractor.extract({ messages: msgs, existingNames: existing }); + + const nameToId = new Map(); + for (const nc of extraction.nodes) { + const { node } = await upsertNode(driver, { + type: nc.type, name: nc.name, + description: nc.description, content: nc.content, + }, sessionId); + nameToId.set(node.name, node.id); + stats.nodes += 1; + void recaller.syncEmbed(node).catch(() => {}); + } + + for (const ec of extraction.edges) { + const fromNode = await findByName(driver, ec.from); + const toNode = await findByName(driver, ec.to); + const fromId = nameToId.get(ec.from) ?? fromNode?.id; + const toId = nameToId.get(ec.to) ?? toNode?.id; + if (fromId && toId) { + await upsertEdge(driver, { + fromId, toId, type: ec.type, + instruction: ec.instruction, condition: ec.condition, sessionId, + }); + stats.edges += 1; + } + } + + const maxTurn = msgs.reduce((m, msg) => Math.max(m, msg.turn_index ?? 0), 0); + await markExtracted(driver, sessionId, maxTurn); + log(` batch ${stats.batches}: ${msgs.length} 消息 -> ${extraction.nodes.length} 节点 / ${extraction.edges.length} 边（累计 ${stats.nodes}/${stats.edges}）`); + + if (msgs.length < batchLimit) break; + if (i === hardBatchCeiling - 1) exhausted = true; + } + + if (exhausted) { + log(` 警告：达到批数上限 ${hardBatchCeiling}，会话 ${sessionId.slice(0, 12)}… 仍有未提取消息，请再次运行。`); + } + + return stats; +} + +async function defaultPrompt(question: string): Promise { + if (!process.stdin.isTTY && process.env.GRAPH_MEMORY_EXTRACT_CONFIRM === undefined) { + return ""; + } + const rl = readline.createInterface({ input, output }); + try { + const answer = await rl.question(question); + return answer; + } finally { + rl.close(); + } +} diff --git a/src/cli.ts b/src/cli.ts index f4c5a25..a7faeaf 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -25,6 +25,8 @@ import { type OAuthProviderId, } from "./engine/oauth.ts"; import type { ReasoningEffort } from "./engine/llm.ts"; +import { runBackfillExtraction } from "./cli-extract.ts"; +import { DEFAULT_CONFIG, type GmConfig } from "./types.ts"; // ─── 最小 Commander 鸭子类型（避免引入 commander 依赖） ─────────── // host 运行时注入真正的 commander.Command 实例，结构兼容此接口即可。 @@ -45,6 +47,7 @@ export interface GraphMemoryCliDeps { pluginId?: string; pluginConfig?: Record | undefined; resolveConfigPath?: (input: string) => string; + defaultModel?: string; oauthTestHooks?: { openUrl?: (url: string) => void | Promise; authorizeUrl?: (url: string) => void | Promise; @@ -375,5 +378,57 @@ export function createGraphMemoryCli(deps: GraphMemoryCliDeps) { throw new Error(`[graph-memory-pro] OAuth login failed: ${message}`); } }); + + root + .command("extract") + .description( + "扫描 Neo4j 中未提取的会话消息，按 compact 流程批量补提知识图谱，并同步节点 embedding", + ) + .option("--yes", "跳过确认提示，直接执行提取", false) + .option("--dry-run", "只列出待提取会话，不调用 LLM", false) + .option("--limit ", "每个会话每批最多提取的消息条数（默认 compactTurnCount * 3）", undefined) + .option("--session ", "仅提取指定 sessionId（默认全部含未提取消息的会话）", undefined) + .option("--model ", "本次提取使用的 LLM 模型（覆盖配置中的 llm.model / agents.defaults.model）", undefined) + .action(async (options: Record) => { + try { + const rawCfg = isPlainObject(deps.pluginConfig) + ? (deps.pluginConfig as Record) + : {}; + const cfg: GmConfig = { + ...DEFAULT_CONFIG, + ...(rawCfg as Partial), + }; + if (isPlainObject(rawCfg.neo4j)) { + cfg.neo4j = { ...DEFAULT_CONFIG.neo4j, ...(rawCfg.neo4j as any) }; + } + + const cfgLlm = isPlainObject(rawCfg.llm) ? (rawCfg.llm as any) : undefined; + const flagModel = typeof options.model === "string" && options.model.trim() + ? options.model.trim() + : undefined; + const effectiveModel = flagModel ?? cfgLlm?.model ?? deps.defaultModel ?? ""; + + const limitFlag = typeof options.limit === "string" + ? Number.parseInt(options.limit, 10) + : (typeof options.limit === "number" ? options.limit : undefined); + + await runBackfillExtraction({ + cfg, + effectiveModel, + options: { + yes: options.yes === true, + dryRun: options.dryRun === true, + session: typeof options.session === "string" ? options.session : undefined, + limit: limitFlag !== undefined && Number.isFinite(limitFlag) && limitFlag > 0 + ? Math.floor(limitFlag) + : undefined, + }, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error("[graph-memory-pro] extract 失败：", message); + throw new Error(`[graph-memory-pro] extract failed: ${message}`); + } + }); }; } diff --git a/src/store/store.ts b/src/store/store.ts index e08787c..992d259 100755 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -888,6 +888,37 @@ export async function getUnextracted(driver: Driver, sid: string, limit: number) } } +export interface UnextractedSessionInfo { + sessionId: string; + messageCount: number; + maxTurn: number; + minCreatedAt: number; +} + +export async function listUnextractedSessions(driver: Driver): Promise { + const session = getSession(driver); + try { + const result = await session.run(` + MATCH (m:GmMessage {extracted: false}) + WITH m.sessionId AS sid, + count(*) AS msgCount, + max(m.turnIndex) AS maxTurn, + min(coalesce(m.createdAt, 0)) AS minCreated + WHERE sid IS NOT NULL + RETURN sid, msgCount, maxTurn, minCreated + ORDER BY minCreated ASC, sid ASC + `); + return result.records.map(r => ({ + sessionId: r.get("sid"), + messageCount: toInt(r.get("msgCount")), + maxTurn: toInt(r.get("maxTurn")), + minCreatedAt: toInt(r.get("minCreated")), + })); + } finally { + await session.close(); + } +} + export async function markExtracted(driver: Driver, sid: string, upToTurn: number): Promise { const session = getSession(driver); try { diff --git a/test/cli-extract.test.ts b/test/cli-extract.test.ts new file mode 100644 index 0000000..902016b --- /dev/null +++ b/test/cli-extract.test.ts @@ -0,0 +1,417 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + listUnextractedSessions: vi.fn(async () => [] as any[]), + getUnextracted: vi.fn(async (_d: any, _sid: any, _limit: any) => [] as any[]), + markExtracted: vi.fn(async () => {}), + upsertNode: vi.fn(async (_driver: any, c: any) => ({ + node: { + id: `n-${c.name}`, + type: c.type, + name: c.name, + description: c.description ?? "", + content: c.content, + status: "active", + validatedCount: 1, + sourceSessions: [], + communityId: null, + pagerank: 0, + createdAt: 0, + updatedAt: 0, + }, + isNew: true, + })), + upsertEdge: vi.fn(async () => {}), + findByName: vi.fn(async () => null), + getBySession: vi.fn(async () => [] as any[]), + extract: vi.fn(async () => ({ nodes: [] as any[], edges: [] as any[] })), + initSchema: vi.fn(async () => {}), + closeDriver: vi.fn(async () => {}), +})); + +vi.mock("../src/store/db.ts", () => ({ + getDriver: () => ({}), + initSchema: mocks.initSchema, + getSession: () => ({ close: async () => {} }), + closeDriver: mocks.closeDriver, +})); + +vi.mock("../src/store/store.ts", () => ({ + listUnextractedSessions: mocks.listUnextractedSessions, + getUnextracted: mocks.getUnextracted, + markExtracted: mocks.markExtracted, + upsertNode: mocks.upsertNode, + upsertEdge: mocks.upsertEdge, + findByName: mocks.findByName, + getBySession: mocks.getBySession, +})); + +vi.mock("../src/engine/llm.ts", () => ({ + createCompleteFn: () => async () => "", + resolveProvider: () => ({ provider: "openai", inferred: false }), +})); + +vi.mock("../src/engine/embed.ts", () => ({ + createEmbedFn: async () => null, +})); + +vi.mock("../src/recaller/recall.ts", () => ({ + Recaller: class { + setEmbedFn(): void {} + async syncEmbed(): Promise {} + }, +})); + +vi.mock("../src/extractor/extract.ts", () => ({ + Extractor: class { + async extract() { + return mocks.extract(); + } + }, +})); + +import { isAffirmative, runBackfillExtraction } from "../src/cli-extract.ts"; +import { DEFAULT_CONFIG } from "../src/types.ts"; + +function makeCfg(overrides: Record = {}) { + return { + ...DEFAULT_CONFIG, + neo4j: { uri: "bolt://localhost:7687", user: "neo4j", password: "x" }, + llm: { provider: "openai", apiKey: "k", baseURL: "https://api.openai.com/v1", model: "gpt-test" }, + ...overrides, + } as any; +} + +const SAMPLE_SESSION = { + sessionId: "sid-abc-1234567890", + messageCount: 5, + maxTurn: 5, + minCreatedAt: 1700000000000, +}; + +describe("isAffirmative", () => { + it.each([ + ["y", true], + ["Y", true], + ["yes", true], + ["YES", true], + [" yes ", true], + ["yeah", true], + ["ok", true], + ["confirm", true], + ["1", true], + ["true", true], + ["n", false], + ["no", false], + ["", false], + ["maybe", false], + ["nope", false], + ["0", false], + ])("isAffirmative(%j) -> %s", (input, expected) => { + expect(isAffirmative(input)).toBe(expected); + }); +}); + +describe("runBackfillExtraction", () => { + beforeEach(() => { + mocks.listUnextractedSessions.mockReset(); + mocks.getUnextracted.mockReset(); + mocks.markExtracted.mockReset(); + mocks.upsertNode.mockReset(); + mocks.upsertEdge.mockReset(); + mocks.findByName.mockReset(); + mocks.getBySession.mockReset(); + mocks.extract.mockReset(); + mocks.initSchema.mockReset(); + mocks.closeDriver.mockReset(); + + mocks.initSchema.mockResolvedValue(undefined); + mocks.closeDriver.mockResolvedValue(undefined); + mocks.getBySession.mockResolvedValue([]); + mocks.extract.mockResolvedValue({ nodes: [], edges: [] }); + mocks.upsertNode.mockImplementation(async (_d: any, c: any) => ({ + node: { + id: `n-${c.name}`, + type: c.type, + name: c.name, + description: c.description ?? "", + content: c.content, + status: "active", + validatedCount: 1, + sourceSessions: [], + communityId: null, + pagerank: 0, + createdAt: 0, + updatedAt: 0, + }, + isNew: true, + })); + mocks.upsertEdge.mockResolvedValue(undefined); + mocks.findByName.mockResolvedValue(null); + mocks.markExtracted.mockResolvedValue(undefined); + }); + + it("returns sessionsTotal=0 and skips everything when no unextracted sessions", async () => { + mocks.listUnextractedSessions.mockResolvedValue([]); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: {}, + log, + }); + + expect(result.sessionsTotal).toBe(0); + expect(result.sessionsProcessed).toBe(0); + expect(mocks.extract).not.toHaveBeenCalled(); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + expect(log).toHaveBeenCalledWith(expect.stringContaining("没有需要提取的会话")); + }); + + it("requires an LLM model and throws a clear error when missing", async () => { + mocks.listUnextractedSessions.mockResolvedValue([]); + await expect( + runBackfillExtraction({ cfg: makeCfg(), effectiveModel: "", options: {}, log: vi.fn() }), + ).rejects.toThrow(/LLM model/); + expect(mocks.closeDriver).not.toHaveBeenCalled(); + }); + + it("requires neo4j.uri and throws a clear error when missing", async () => { + mocks.listUnextractedSessions.mockResolvedValue([]); + await expect( + runBackfillExtraction({ + cfg: { ...makeCfg(), neo4j: { uri: "", user: "", password: "" } } as any, + effectiveModel: "gpt-test", + options: {}, + log: vi.fn(), + }), + ).rejects.toThrow(/neo4j\.uri/); + }); + + it("aborts when the user declines the confirmation prompt", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + const prompt = vi.fn().mockResolvedValue("n"); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: {}, + log, + prompt, + }); + + expect(prompt).toHaveBeenCalledTimes(1); + expect(result.sessionsProcessed).toBe(0); + expect(result.sessionsSkipped).toBe(1); + expect(mocks.extract).not.toHaveBeenCalled(); + expect(mocks.markExtracted).not.toHaveBeenCalled(); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("does not prompt when --yes is set and runs extraction", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + mocks.getUnextracted.mockResolvedValueOnce([ + { role: "user", content: "hello", turn_index: 1 }, + { role: "assistant", content: "hi", turn_index: 2 }, + ]).mockResolvedValueOnce([]); + mocks.extract.mockResolvedValueOnce({ + nodes: [ + { type: "TASK", name: "t1", description: "d", content: "c" }, + { type: "SKILL", name: "s1", description: "d", content: "c" }, + ], + edges: [ + { from: "t1", to: "s1", type: "USED_SKILL", instruction: "i" }, + ], + }); + const prompt = vi.fn(); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { yes: true }, + log, + prompt, + }); + + expect(prompt).not.toHaveBeenCalled(); + expect(result.sessionsProcessed).toBe(1); + expect(result.nodesCreated).toBe(2); + expect(result.edgesCreated).toBe(1); + expect(result.batches).toBe(1); + expect(mocks.extract).toHaveBeenCalledTimes(1); + expect(mocks.markExtracted).toHaveBeenCalledWith(expect.anything(), "sid-abc-1234567890", 2); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("filters sessions to the one specified by --session", async () => { + mocks.listUnextractedSessions.mockResolvedValue([ + { ...SAMPLE_SESSION, sessionId: "aaa" }, + { ...SAMPLE_SESSION, sessionId: "bbb" }, + ]); + mocks.getUnextracted.mockResolvedValue([]); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { yes: true, session: "bbb" }, + log, + }); + + expect(result.sessionsTotal).toBe(1); + expect(result.sessionsProcessed).toBe(1); + expect(mocks.getUnextracted).toHaveBeenCalledWith(expect.anything(), "bbb", expect.any(Number)); + }); + + it("exits cleanly when --session matches no sessions", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { session: "does-not-exist" }, + log, + }); + + expect(result.sessionsTotal).toBe(0); + expect(mocks.extract).not.toHaveBeenCalled(); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("lists sessions but does not extract under --dry-run", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + const prompt = vi.fn(); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { dryRun: true }, + log, + prompt, + }); + + expect(prompt).not.toHaveBeenCalled(); + expect(result.sessionsSkipped).toBe(1); + expect(result.sessionsProcessed).toBe(0); + expect(mocks.extract).not.toHaveBeenCalled(); + expect(log).toHaveBeenCalledWith(expect.stringContaining("--dry-run")); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("loops multiple batches until getUnextracted returns empty", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + mocks.getUnextracted + .mockResolvedValueOnce([ + { role: "user", content: "m1", turn_index: 1 }, + ]) + .mockResolvedValueOnce([ + { role: "user", content: "m2", turn_index: 2 }, + ]) + .mockResolvedValueOnce([]); + mocks.extract.mockResolvedValue({ nodes: [], edges: [] }); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { yes: true, limit: 1 }, + log, + }); + + expect(result.batches).toBe(2); + expect(mocks.markExtracted).toHaveBeenCalledTimes(2); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("records a session as skipped when getUnextracted throws", async () => { + mocks.listUnextractedSessions.mockResolvedValue([ + { ...SAMPLE_SESSION, sessionId: "good" }, + { ...SAMPLE_SESSION, sessionId: "bad" }, + ]); + const callCount = new Map(); + mocks.getUnextracted.mockImplementation(async (_d: any, sid: string) => { + if (sid === "bad") throw new Error("boom"); + const n = (callCount.get(sid) ?? 0) + 1; + callCount.set(sid, n); + if (n === 1) return [{ role: "user", content: "x", turn_index: 1 }]; + return []; + }); + mocks.extract.mockResolvedValue({ nodes: [], edges: [] }); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { yes: true }, + log, + }); + + expect(result.sessionsProcessed).toBe(1); + expect(result.sessionsSkipped).toBe(1); + expect(result.sessionsTotal).toBe(2); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("calls closeDriver even when listUnextractedSessions throws (try/finally)", async () => { + mocks.listUnextractedSessions.mockRejectedValue(new Error("neo4j down")); + const log = vi.fn(); + + await expect( + runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { yes: true }, + log, + }), + ).rejects.toThrow("neo4j down"); + + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("does not warn about batch ceiling when session completes before the ceiling", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + const fullBatches = 3; + mocks.getUnextracted.mockImplementation(async () => { + const call = mocks.getUnextracted.mock.calls.length; + if (call < fullBatches) return Array.from({ length: 5 }, (_, i) => ({ role: "user", content: `m${i}`, turn_index: call * 5 + i })); + return [{ role: "user", content: "last", turn_index: fullBatches * 5 }]; + }); + mocks.extract.mockResolvedValue({ nodes: [], edges: [] }); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg({ compactTurnCount: 1 }), + effectiveModel: "gpt-test", + options: { yes: true, limit: 5 }, + log, + }); + + expect(result.batches).toBe(fullBatches); + expect(result.sessionsProcessed).toBe(1); + const warningCalls = log.mock.calls.filter(c => typeof c[0] === "string" && c[0].includes("达到批数上限")); + expect(warningCalls).toHaveLength(0); + }); + + it("warns about batch ceiling when the session genuinely has more messages than the ceiling allows", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + mocks.getUnextracted.mockResolvedValue(Array.from({ length: 5 }, (_, i) => ({ role: "user", content: `m${i}`, turn_index: i }))); + mocks.extract.mockResolvedValue({ nodes: [], edges: [] }); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg({ compactTurnCount: 1 }), + effectiveModel: "gpt-test", + options: { yes: true, limit: 5 }, + log, + }); + + expect(result.batches).toBe(50); + const warningCalls = log.mock.calls.filter(c => typeof c[0] === "string" && c[0].includes("达到批数上限")); + expect(warningCalls).toHaveLength(1); + }); +}); From 3df36142bfb538374539453e15dfc3e9f9cb778e Mon Sep 17 00:00:00 2001 From: TriDefender Date: Tue, 11 Aug 2026 11:22:06 +0800 Subject: [PATCH 02/29] =?UTF-8?q?Feat:=20=E6=B7=BB=E5=8A=A0=E9=81=97?= =?UTF-8?q?=E5=BF=98=E6=9B=B2=E7=BA=BF=E9=97=A8=E6=8E=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 添加模仿艾宾浩斯遗忘曲线的门控机制，长期不用的节点会被deprecate掉（非硬性删除，可被重新激活）避免过时噪声影响检索结果 --- README.md | 25 +++- docs/decay.md | 175 ++++++++++++++++++++++++ index.ts | 13 +- openclaw.plugin.json | 22 ++++ src/graph/decay.ts | 263 ++++++++++++++++++++++++++++++++++++ src/graph/maintenance.ts | 8 +- src/store/store.ts | 13 +- src/types.ts | 63 +++++++++ test/decay.test.ts | 278 +++++++++++++++++++++++++++++++++++++++ 9 files changed, 853 insertions(+), 7 deletions(-) create mode 100644 docs/decay.md create mode 100644 src/graph/decay.ts create mode 100644 test/decay.test.ts diff --git a/README.md b/README.md index fbf1c23..79bdc64 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,29 @@ Anthropic direct (Claude) — drop `baseURL`, switch `provider`: `embedding` is optional. When present, `dimensions` must match the Neo4j vector index dimension. For a fresh database, the plugin creates matching indexes during startup. If you change dimensions later, recreate the vector indexes or the Neo4j database. +### Memory decay (forgetting curve) + +Each maintenance cycle scores every active node with a three-factor weighted model (recency + frequency + intrinsic) and bidirectionally transitions nodes across three tiers: `core` / `working` / `peripheral`. Nodes never get `status=deprecated` from decay — only manual deprecate / merge does that. Decay only adjusts `tier`, so all active nodes remain searchable. + +The full formula, field mapping from the reference implementation, default-value rationale, and tuning guide live in **[`docs/decay.md`](docs/decay.md)**. + +Minimal config (all fields optional, defaults shown): + +```json +"decay": { "enabled": true } +``` + +Common overrides — for fuller control see `docs/decay.md` §4: + +```json +"decay": { + "enabled": true, + "recencyHalfLifeDays": 30, + "peripheralCompositeThreshold": 0.15, + "workingAccessThreshold": 3 +} +``` + ### OAuth login (experimental) ```bash @@ -127,7 +150,7 @@ conversation messages -> GmMessage nodes -> LLM triple extraction -> embeddings -> vector recall + community expansion + GDS PPR -> XML context injection -session end -> dedup -> global PageRank -> communities -> summaries +session end -> decay (forgetting curve) -> dedup -> global PageRank -> communities -> summaries ``` ## Verify diff --git a/docs/decay.md b/docs/decay.md new file mode 100644 index 0000000..ba57d5f --- /dev/null +++ b/docs/decay.md @@ -0,0 +1,175 @@ +# Memory Decay — 柔性评分模型 + +graph-memory-pro 的衰减机制采用**三因子加权评分 + tier 双向转换**，参考 [memory-lancedb-pro](https://github.com/CortexReach/memory-lancedb-pro) 的设计并映射到本仓库的图模型信号。 + +- **decay 不动 `status`**——只调整 `tier`（`core` / `working` / `peripheral`）。`status=deprecated` 仅由手动弃用（`gm_update mode=deprecate` / merge）触发。 +- 每次 `gm_maintain` 或 `session_end` 维护的第 0 步执行：扫描所有 active 节点 → 评分 → tier 转换 → 写回 `decayScore` / `tier` / `decayComputedAt`。 +- 评分结果可通过 `gm_stats` / CRUD API 查看；外层搜索目前**不读 decayScore 排序**（已由 PageRank + tier 隐含分层）。 + +--- + +## 1. 评分公式 + +``` +composite = wR · recency + wF · frequency + wI · intrinsic +``` + +三个权重默认 `0.4 / 0.3 / 0.3`，**推荐**和为 1。运行时若和≠1 会自动按比例归一化（`wR' = wR / (wR+wF+wI)`），保证 `composite ∈ [0,1]`，避免用户覆盖单个权重导致评分越界。归一化在 `scoreNode()` 内进行，原始 `cfg.*Weight` 值不被修改。 + +### 1.1 Recency（时间衰减，权重 0.4） + +Weibull 拉伸指数： + +``` +recency = exp( −λ · daysSinceLastAccess^β ) + +λ = ln(2) / effectiveHL +effectiveHL = recencyHalfLifeDays · exp( importanceModulation · importance ) +``` + +- **半衰期调制**：重要记忆（高 `importance`）的 `effectiveHL` 更大 → 衰减更慢。对应艾宾浩斯曲线"重要事件保留更久"。 +- **tier-β**：曲线形状随 tier 变化，反馈式调整衰减速度： + + | tier | β | 效果 | + |---|---|---| + | `core` | 0.8 | 尾部衰减缓（核心知识保得久） | + | `working` | 1.0 | 标准指数衰减 | + | `peripheral` | 1.3 | 加速衰减（边缘知识更快被遗忘） | + +### 1.2 Frequency（访问频率，权重 0.3） + +``` +frequency = base · ( 0.5 + 0.5 · recentnessBonus ) + +base = 1 − exp( −validatedCount / 5 ) +recentnessBonus = exp( −avgAccessGapDays / 30 ) # 仅当 validatedCount > 1 +avgAccessGapDays = ( lastAccessedAt − createdAt ) / ( validatedCount − 1 ) +``` + +- 用 `validatedCount`（LLM 重新提取的次数）替代 lancedb-pro 的 `accessCount`（manual recall 触发的次数）。前者是更强的"重新确认"信号。 +- `validatedCount ≤ 1` 时跳过 `recentnessBonus`，只返回 `base`（无法算平均间隔）。 + +### 1.3 Intrinsic（内在价值，权重 0.3） + +``` +intrinsic = importance · confidence + +importance = pagerank / maxPagerank # 每次扫描时按当前批次归一化到 [0,1] +confidence = 1 − 1 / ( 1 + validatedCount ) # 饱和函数，收敛到 1 +``` + +--- + +## 2. 字段映射（lancedb-pro → graph-memory-pro） + +| lancedb-pro 字段 | 本仓库替代 | 说明 | +|---|---|---| +| `accessCount` | `validatedCount` | LLM 重新提取次数（强信号，原为 manual recall 触发） | +| `lastAccessedAt` | `lastAccessedAt` | 由 `upsertNode` 在任意写入路径刷新（重新提取、`gm_record`、`gm_update`、CRUD POST）。`mergeNodes` 故意不刷新（合并 ≠ 用户重新激活） | +| `importance` | `pagerank / maxPagerank` | 图结构重要性，每次扫描归一化 | +| `confidence` | `1 − 1/(1+validatedCount)` | 饱和置信度 | +| `tier` | `tier`（新增字段） | 与 `status` 正交 | + +--- + +## 3. Tier 双向转换 + +| 转换 | 条件 | +|---|---| +| **core → working** | `composite < peripheralCompositeThreshold` **AND** `count < workingAccessThreshold` | +| **working → peripheral** | `composite < peripheralCompositeThreshold` **OR**（`ageDays > peripheralAgeDays` **AND** `count < workingAccessThreshold`） | +| **peripheral → working** | `count >= workingAccessThreshold` **AND** `composite >= workingCompositeThreshold` | +| **working → core** | `count >= coreAccessThreshold` **AND** `composite >= coreCompositeThreshold` **AND** `importance >= coreImportanceThreshold` | + +- 新节点默认 `tier = "working"`。 +- 节点保持 `status = active` 不变；tier 变化时仅更新 `updatedAt`，不改变搜索过滤行为。 +- 不存在的"core→peripheral"和"peripheral→core"由两次相邻转换实现（经过 working）。 + +--- + +## 4. 默认值与调参指南 + +### 4.1 默认配置 + +```json +{ + "decay": { + "enabled": true, + "recencyHalfLifeDays": 30, + "recencyWeight": 0.4, + "importanceModulation": 1.5, + "frequencyWeight": 0.3, + "intrinsicWeight": 0.3, + "betaCore": 0.8, + "betaWorking": 1.0, + "betaPeripheral": 1.3, + "coreAccessThreshold": 10, + "coreCompositeThreshold": 0.7, + "coreImportanceThreshold": 0.8, + "peripheralCompositeThreshold": 0.15, + "peripheralAgeDays": 60, + "workingAccessThreshold": 3, + "workingCompositeThreshold": 0.4 + } +} +``` + +### 4.2 数值来源 + +| 参数 | 默认值 | 来源 | +|---|---|---| +| `recencyHalfLifeDays` | 30 | 艾宾浩斯曲线 ~25% 保留率拐点；同时与 lancedb-pro 的 `recencyHalfLifeDays` + `ACCESS_DECAY_HALF_LIFE_DAYS` 一致 | +| `importanceModulation` | 1.5 | lancedb-pro：`effectiveHL = 30 · exp(1.5 · importance)`，importance=1 时半衰期延长到 ~134 天 | +| `betaCore/Working/Peripheral` | 0.8 / 1.0 / 1.3 | lancedb-pro Weibull 形状参数 | +| 7 个 tier 转换阈值 | — | lancedb-pro `tier-manager` 默认值 | +| `recencyWeight / frequencyWeight / intrinsicWeight` | 0.4 / 0.3 / 0.3 | lancedb-pro 三因子权重，和为 1 | +| `validatedCount` 分母 | 5 | lancedb-pro 的 `1 − exp(−count/5)` 基础频率项（未改） | + +### 4.3 常见调参场景 + +| 想要的效果 | 调整方向 | +|---|---| +| 记忆整体保留更久 | 调高 `recencyHalfLifeDays`（如 60）或调低 `peripheralCompositeThreshold`（更难降级） | +| 更激进遗忘 | 调低 `recencyHalfLifeDays`（如 14）或调高 `peripheralCompositeThreshold` | +| 重要知识显著保得久 | 调高 `importanceModulation`（半衰期调制更强） | +| 核心知识不易降级 | 调低 `betaCore`（更缓的尾部）或调高 `coreCompositeThreshold`（更难升 core，留在 working 也保得久） | +| 单次曝光更易遗忘 | 调高 `workingAccessThreshold`（promote 到 working 需要更多确认） | +| 永久禁用衰减 | `"enabled": false` | + +### 4.4 与原布尔阈值方案的对照（向后兼容） + +旧版本（`maxAgeDays` + `minCalls`）的布尔规则已被这套柔性评分取代。原默认值 `maxAgeDays=30, minCalls=2` 在新模型下大致对应于： + +- 一个 `validatedCount=1`、`tier=working`、低 pagerank 的节点，约 30 天后 `recency` 跌破 0.15 → `composite` 跌破 `peripheralCompositeThreshold` → demote 到 `peripheral`。 +- 关键差别：新模型**不会 deprecate**，只是降到 `peripheral` tier，搜索过滤仍包含它（只是 decayScore 较低）。 + +--- + +## 5. 数据库字段 + +| 字段 | 类型 | 写入者 | 说明 | +|---|---|---|---| +| `tier` | string | `applyDecay` / `upsertNode`（创建时初始化为 `working`） | `core` / `working` / `peripheral` | +| `lastAccessedAt` | int (epoch ms) | `upsertNode`（重新提取时） | decay 评分的时间基准 | +| `decayScore` | float (0~1) | `applyDecay` | 最近一次评分结果 | +| `decayComputedAt` | int (epoch ms) | `applyDecay` | 评分时间戳 | + +旧节点缺这些字段时： +- `tier` 缺失 → 评分按 `working` 处理；首次 `applyDecay` 时自动写入 `working` +- `lastAccessedAt` 缺失 → 回退到 `updatedAt` / `createdAt` +- `decayScore` / `decayComputedAt` 缺失 → 在首次 `applyDecay` 前为 undefined，不影响评分 + +**Backfill 时机**：新字段在第一次 `applyDecay` 运行时为每个 active 节点批量写入。如果部署初始用 `decay.enabled=false`，字段会一直缺失直到切换为 `true` 后的第一次维护周期。在切换前的窗口期，对 raw DB 直接做 `tier` 过滤查询会返回 null/missing 而非 `"working"`——目前搜索路径不读 `tier`，但自定义查询需要留意。 + +--- + +## 6. 实现位置 + +| 文件 | 内容 | +|---|---| +| `src/graph/decay.ts` | 评分函数 + tier 决策 + `applyDecay()` 批处理 | +| `src/types.ts` | `DecayConfig` 接口、`NodeTier` 类型、`GmNode` 新字段、`DEFAULT_CONFIG.decay` | +| `src/store/store.ts` | `toNode` 字段映射、`upsertNode` 初始化 `tier` / `lastAccessedAt` | +| `src/graph/maintenance.ts` | 调用入口（step 0） | +| `test/decay.test.ts` | 评分函数 + tier 决策纯函数单元测试 | +| `openclaw.plugin.json` | 用户可见的配置 schema | diff --git a/index.ts b/index.ts index 2df04fd..4799841 100755 --- a/index.ts +++ b/index.ts @@ -272,6 +272,7 @@ const graphMemoryProPlugin = { const cfg: GmConfig = { ...DEFAULT_CONFIG, ...raw }; if (raw.neo4j) cfg.neo4j = { ...DEFAULT_CONFIG.neo4j, ...raw.neo4j }; + if (raw.decay) cfg.decay = { ...DEFAULT_CONFIG.decay, ...raw.decay }; const providerModel = readDefaultModel(api.config); @@ -1121,13 +1122,21 @@ const graphMemoryProPlugin = { (_ctx: any) => ({ name: "gm_maintain", label: "Graph Memory Maintenance", - description: "手动触发图维护：去重、PageRank、社区检测。", + description: "手动触发图维护：衰减评分 + tier 转换、去重、PageRank、社区检测。", parameters: Type.Object({}), async execute() { const embedFn = (recaller as any).embed ?? undefined; const result = await runMaintenance(driver, cfg, llm, embedFn); + const t = result.decay.tierTransitions; + const totalTransitions = t.coreToWorking + t.workingToPeripheral + t.peripheralToWorking + t.workingToCore; const text = [ `🔧 图维护完成（${result.durationMs}ms）`, + result.decay.enabled + ? `衰减：扫描 ${result.decay.scanned} 个节点，tier 转换 ${totalTransitions} 次` + + (totalTransitions > 0 + ? `（core→working ${t.coreToWorking}，working→peripheral ${t.workingToPeripheral}，peripheral→working ${t.peripheralToWorking}，working→core ${t.workingToCore}）` + : "") + : `衰减：已禁用`, `去重：${result.dedup.pairs.length} 对相似，合并 ${result.dedup.merged} 对`, ...(result.dedup.pairs.length > 0 ? result.dedup.pairs.slice(0, 5).map(p => ` "${p.nameA}" ≈ "${p.nameB}" (${(p.similarity * 100).toFixed(1)}%)`) @@ -1137,7 +1146,7 @@ const graphMemoryProPlugin = { `PageRank Top 5：`, ...result.pagerank.topK.slice(0, 5).map((n, i) => ` ${i + 1}. ${n.name} (${n.score.toFixed(4)})`), ].join("\n"); - return { content: [{ type: "text", text }], details: { durationMs: result.durationMs, dedupMerged: result.dedup.merged, communities: result.community.count } }; + return { content: [{ type: "text", text }], details: { durationMs: result.durationMs, decayTransitions: totalTransitions, dedupMerged: result.dedup.merged, communities: result.community.count } }; }, }), { name: "gm_maintain" }, diff --git a/openclaw.plugin.json b/openclaw.plugin.json index f54c9d0..1a6b474 100755 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -29,6 +29,28 @@ "dedupThreshold": { "type": "number", "default": 0.90 }, "pagerankDamping": { "type": "number", "default": 0.85 }, "pagerankIterations": { "type": "number", "default": 20 }, + "decay": { + "type": "object", + "description": "柔性衰减：三因子加权评分（recency+frequency+intrinsic）+ tier 双向转换（core/working/peripheral）。完整公式与调参指南见 docs/decay.md。recencyWeight + frequencyWeight + intrinsicWeight 推荐和为 1（运行时会自动归一化）。", + "properties": { + "enabled": { "type": "boolean", "default": true, "description": "是否启用自动衰减。关闭后 tier 永久保持初始 working 状态。" }, + "recencyHalfLifeDays": { "type": "number", "default": 30, "description": "Recency 半衰期（天）。effectiveHL = halfLife * exp(importanceModulation * importance)。" }, + "recencyWeight": { "type": "number", "default": 0.4, "description": "Recency 在 composite 中的权重。三个权重推荐和为 1。" }, + "importanceModulation": { "type": "number", "default": 1.5, "description": "半衰期调制系数；越大则高 importance 节点衰减越慢。" }, + "frequencyWeight": { "type": "number", "default": 0.3, "description": "Frequency 在 composite 中的权重。三个权重推荐和为 1。" }, + "intrinsicWeight": { "type": "number", "default": 0.3, "description": "Intrinsic（importance × confidence）在 composite 中的权重。三个权重推荐和为 1。" }, + "betaCore": { "type": "number", "default": 0.8, "description": "core tier 的 Weibull 形状参数；<1 = 缓衰。" }, + "betaWorking": { "type": "number", "default": 1.0, "description": "working tier 的 Weibull 形状参数；=1 = 标准指数衰减。" }, + "betaPeripheral": { "type": "number", "default": 1.3, "description": "peripheral tier 的 Weibull 形状参数；>1 = 加速衰减。" }, + "coreAccessThreshold": { "type": "number", "default": 10, "description": "working→core 所需的最低 validatedCount。" }, + "coreCompositeThreshold": { "type": "number", "default": 0.7, "description": "working→core 所需的最低 composite 分数。" }, + "coreImportanceThreshold": { "type": "number", "default": 0.8, "description": "working→core 所需的最低归一化 importance。" }, + "peripheralCompositeThreshold": { "type": "number", "default": 0.15, "description": "composite 低于此值触发 demote（core→working 或 working→peripheral）。" }, + "peripheralAgeDays": { "type": "number", "default": 60, "description": "working→peripheral 的年龄阈值（同时 validatedCount < workingAccessThreshold 才触发）。" }, + "workingAccessThreshold": { "type": "number", "default": 3, "description": "demote（count 不足时）/ promote（count 充足时）的 access 次数分界。" }, + "workingCompositeThreshold": { "type": "number", "default": 0.4, "description": "peripheral→working 所需的最低 composite 分数。" } + } + }, "llm": { "type": "object", "properties": { diff --git a/src/graph/decay.ts b/src/graph/decay.ts new file mode 100644 index 0000000..6de5c07 --- /dev/null +++ b/src/graph/decay.ts @@ -0,0 +1,263 @@ +/** + * graph-memory-pro — 柔性衰减（三因子加权评分 + tier 双向转换） + * + * 完整公式、字段映射、默认值来源、调参指南见 docs/decay.md。 + * 评分 / tier 决策 / applyDecay 的入口均在本文件。 + * + * 调用时机：runMaintenance 的第 0 步（去重/PageRank/社区之前）。 + * decay 不动 status，只动 tier。 + */ + +import type { Driver } from "neo4j-driver"; +import type { GmConfig, DecayConfig, GmNode, NodeTier } from "../types.ts"; +import { getSession } from "../store/db.ts"; +import { allActiveNodes } from "../store/store.ts"; + +const MS_PER_DAY = 86_400_000; + +export interface CompositeScore { + composite: number; + recency: number; + frequency: number; + intrinsic: number; +} + +export interface TierTransition { + coreToWorking: number; + workingToPeripheral: number; + peripheralToWorking: number; + workingToCore: number; +} + +export interface DecayResult { + enabled: boolean; + scanned: number; + tierTransitions: TierTransition; + durationMs: number; +} + +// ─── 归一化辅助（纯函数，便于单元测试） ────────────────────── + +/** importance ∈ [0,1]：当前批次的 pagerank 归一化值。 */ +export function normalizeImportance(pagerank: number, maxPagerank: number): number { + if (maxPagerank <= 0) return 0; + return Math.min(1, Math.max(0, pagerank / maxPagerank)); +} + +/** confidence ∈ [0,1)：validatedCount 越高越可信，饱和收敛到 1。 */ +export function computeConfidence(validatedCount: number): number { + const c = Math.max(0, validatedCount); + return 1 - 1 / (1 + c); +} + +// ─── 三因子评分（纯函数） ──────────────────────────────────── + +/** β 随 tier 变化：core 缓衰、peripheral 促衰。 */ +export function computeBeta(tier: NodeTier, cfg: DecayConfig): number { + switch (tier) { + case "core": return cfg.betaCore; + case "working": return cfg.betaWorking; + case "peripheral": return cfg.betaPeripheral; + } +} + +/** + * Recency 分量：Weibull 拉伸指数衰减。 + * tier 决定 β；importance 调制半衰期（高重要性 → 慢衰减）。 + */ +export function scoreRecency( + node: Pick, + importance: number, + now: number, + cfg: DecayConfig, +): number { + const lastActive = node.lastAccessedAt > 0 + ? node.lastAccessedAt + : (node.updatedAt > 0 ? node.updatedAt : node.createdAt); + const daysSince = Math.max(0, (now - lastActive) / MS_PER_DAY); + + const effectiveHL = cfg.recencyHalfLifeDays * Math.exp(cfg.importanceModulation * importance); + const lambda = Math.LN2 / effectiveHL; + const beta = computeBeta(node.tier ?? "working", cfg); + + return Math.exp(-lambda * Math.pow(daysSince, beta)); +} + +/** + * Frequency 分量：基础饱和项 × 平均访问间隔新鲜度。 + * validatedCount ≤ 1 时只返回基础项（无法算平均间隔）。 + */ +export function scoreFrequency( + node: Pick, +): number { + const count = Math.max(0, node.validatedCount); + const base = 1 - Math.exp(-count / 5); + if (count <= 1) return base; + + const lastActive = node.lastAccessedAt > 0 + ? node.lastAccessedAt + : (node.updatedAt > 0 ? node.updatedAt : node.createdAt); + const accessSpanDays = Math.max(1, (lastActive - node.createdAt) / MS_PER_DAY); + const avgGapDays = accessSpanDays / Math.max(count - 1, 1); + const recentnessBonus = Math.exp(-avgGapDays / 30); + + return base * (0.5 + 0.5 * recentnessBonus); +} + +/** Intrinsic 分量：importance × confidence。 */ +export function scoreIntrinsic(importance: number, confidence: number): number { + return importance * confidence; +} + +/** 三因子加权汇总。权重和在运行时归一化到 1，避免用户配置偏差导致 composite > 1。 */ +export function scoreNode( + node: Pick, + maxPagerank: number, + now: number, + cfg: DecayConfig, +): CompositeScore { + const importance = normalizeImportance(node.pagerank, maxPagerank); + const confidence = computeConfidence(node.validatedCount); + const recency = scoreRecency(node, importance, now, cfg); + const frequency = scoreFrequency(node); + const intrinsic = scoreIntrinsic(importance, confidence); + + const wSum = cfg.recencyWeight + cfg.frequencyWeight + cfg.intrinsicWeight; + const safeSum = wSum > 0 ? wSum : 1; + const wR = cfg.recencyWeight / safeSum; + const wF = cfg.frequencyWeight / safeSum; + const wI = cfg.intrinsicWeight / safeSum; + + const composite = wR * recency + wF * frequency + wI * intrinsic; + + return { composite, recency, frequency, intrinsic }; +} + +// ─── Tier 转换决策（纯函数） ───────────────────────────────── + +/** + * 决定节点的下一个 tier。返回 null 表示保持不变。 + * importance 已归一化（调用方须先 normalizeImportance）。 + */ +export function decideTierTransition( + node: Pick, + score: CompositeScore, + importance: number, + cfg: DecayConfig, + now: number = Date.now(), +): NodeTier | null { + const current = node.tier ?? "working"; + const count = node.validatedCount; + const ageDays = Math.max(0, (now - node.createdAt) / MS_PER_DAY); + const composite = score.composite; + + if (current === "core" + && composite < cfg.peripheralCompositeThreshold + && count < cfg.workingAccessThreshold) { + return "working"; + } + + if (current === "working") { + if (composite < cfg.peripheralCompositeThreshold) return "peripheral"; + if (ageDays > cfg.peripheralAgeDays && count < cfg.workingAccessThreshold) { + return "peripheral"; + } + } + + if (current === "peripheral" + && count >= cfg.workingAccessThreshold + && composite >= cfg.workingCompositeThreshold) { + return "working"; + } + + if (current === "working" + && count >= cfg.coreAccessThreshold + && composite >= cfg.coreCompositeThreshold + && importance >= cfg.coreImportanceThreshold) { + return "core"; + } + + return null; +} + +// ─── 应用层：扫描 + 评分 + 转换 ────────────────────────────── + +const EMPTY_TRANSITIONS: TierTransition = { + coreToWorking: 0, + workingToPeripheral: 0, + peripheralToWorking: 0, + workingToCore: 0, +}; + +function bumpTransition(transitions: TierTransition, from: NodeTier, to: NodeTier): void { + if (from === "core" && to === "working") transitions.coreToWorking++; + else if (from === "working" && to === "peripheral") transitions.workingToPeripheral++; + else if (from === "peripheral" && to === "working") transitions.peripheralToWorking++; + else if (from === "working" && to === "core") transitions.workingToCore++; +} + +/** + * 扫描所有 active 节点：评分 + tier 转换 + 写回 decayScore / tier。 + * 不动 status（status=deprecated 仅由手动弃用触发）。 + */ +export async function applyDecay(driver: Driver, cfg: Pick): Promise { + const start = Date.now(); + const d = cfg.decay; + if (!d?.enabled) { + return { enabled: false, scanned: 0, tierTransitions: { ...EMPTY_TRANSITIONS }, durationMs: 0 }; + } + + const nodes = await allActiveNodes(driver); + if (nodes.length === 0) { + return { enabled: true, scanned: 0, tierTransitions: { ...EMPTY_TRANSITIONS }, durationMs: 0 }; + } + + const maxPagerank = Math.max(...nodes.map(n => n.pagerank), 0.0001); + + const updates: Array<{ id: string; tier: NodeTier; composite: number; tierChanged: boolean }> = []; + const transitions: TierTransition = { ...EMPTY_TRANSITIONS }; + + for (const node of nodes) { + const score = scoreNode(node, maxPagerank, start, d); + const importance = normalizeImportance(node.pagerank, maxPagerank); + const currentTier = node.tier ?? "working"; + const nextTier = decideTierTransition(node, score, importance, d, start); + const finalTier = nextTier ?? currentTier; + const tierChanged = nextTier !== null; + + if (tierChanged) bumpTransition(transitions, currentTier, finalTier); + + updates.push({ + id: node.id, + tier: finalTier, + composite: score.composite, + tierChanged, + }); + } + + if (updates.length > 0) { + const session = getSession(driver); + try { + await session.run( + `UNWIND $updates AS u + MATCH (n:Task|Skill|Event {id: u.id}) + SET n.tier = u.tier, + n.decayScore = u.composite, + n.decayComputedAt = $now, + n.updatedAt = CASE WHEN u.tierChanged THEN $now ELSE n.updatedAt END`, + { updates, now: start }, + ); + } finally { + await session.close(); + } + } + + return { + enabled: true, + scanned: nodes.length, + tierTransitions: transitions, + durationMs: Date.now() - start, + }; +} diff --git a/src/graph/maintenance.ts b/src/graph/maintenance.ts index 64cd4fa..a2e68b8 100755 --- a/src/graph/maintenance.ts +++ b/src/graph/maintenance.ts @@ -2,7 +2,7 @@ * graph-memory-pro — 图谱维护 * * 调用时机：session_end（finalize 之后） - * 执行顺序：去重 → 全局 PageRank → 社区检测 → 社区描述 + * 执行顺序：衰减 → 去重 → 全局 PageRank → 社区检测 → 社区描述 */ import type { Driver } from "neo4j-driver"; @@ -12,8 +12,10 @@ import type { EmbedFn } from "../engine/embed.ts"; import { computeGlobalPageRank, type GlobalPageRankResult } from "./pagerank.ts"; import { detectCommunities, summarizeCommunities, type CommunityResult } from "./community.ts"; import { dedup, type DedupResult } from "./dedup.ts"; +import { applyDecay, type DecayResult } from "./decay.ts"; export interface MaintenanceResult { + decay: DecayResult; dedup: DedupResult; pagerank: GlobalPageRankResult; community: CommunityResult; @@ -26,6 +28,9 @@ export async function runMaintenance( ): Promise { const start = Date.now(); + // 0. 衰减（柔性评分 + tier 转换）—— 先于其他步骤，让后续基于最新 tier 集合运算 + const decayResult = await applyDecay(driver, cfg); + // 1. 去重 const dedupResult = await dedup(driver, cfg); @@ -44,6 +49,7 @@ export async function runMaintenance( } return { + decay: decayResult, dedup: dedupResult, pagerank: pagerankResult, community: communityResult, diff --git a/src/store/store.ts b/src/store/store.ts index 992d259..d85541c 100755 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -8,7 +8,7 @@ import type { Driver } from "neo4j-driver"; import neo4j from "neo4j-driver"; import { createHash } from "crypto"; -import type { GmNode, GmEdge, EdgeType, NodeType } from "../types.ts"; +import type { GmNode, GmEdge, EdgeType, NodeType, NodeTier } from "../types.ts"; import { NODE_TYPE_TO_LABEL, isValidEdgeDirection } from "../types.ts"; import { getSession } from "./db.ts"; @@ -32,6 +32,8 @@ function toNode(r: any): GmNode { description: n.description ?? "", content: n.content, status: n.status, + tier: (n.tier === "core" || n.tier === "working" || n.tier === "peripheral" + ? n.tier : "working") as NodeTier, validatedCount: toInt(n.validatedCount ?? n.validated_count ?? 1), sourceSessions: typeof n.sourceSessions === "string" ? JSON.parse(n.sourceSessions) @@ -40,6 +42,9 @@ function toNode(r: any): GmNode { pagerank: toFloat(n.pagerank ?? 0), createdAt: toInt(n.createdAt ?? n.created_at ?? 0), updatedAt: toInt(n.updatedAt ?? n.updated_at ?? 0), + lastAccessedAt: toInt(n.lastAccessedAt ?? n.last_accessed_at ?? n.updatedAt ?? n.updated_at ?? n.createdAt ?? 0), + decayScore: typeof n.decayScore === "number" ? n.decayScore : undefined, + decayComputedAt: n.decayComputedAt ? toInt(n.decayComputedAt) : undefined, }; } @@ -164,6 +169,7 @@ export async function upsertNode( THEN n.sourceSessions + $sessionId ELSE n.sourceSessions END, + n.lastAccessedAt = $now, n.updatedAt = $now RETURN n `, { name, content: c.content, description: c.description, sessionId, now: Date.now() }); @@ -180,9 +186,10 @@ export async function upsertNode( CREATE (n:MemoryNode:${label} { id: $id, name: $name, type: $type, description: $description, content: $content, - status: 'active', validatedCount: 1, + status: 'active', tier: 'working', validatedCount: 1, sourceSessions: $sessions, communityId: null, - pagerank: 0.0, createdAt: $now, updatedAt: $now + pagerank: 0.0, createdAt: $now, updatedAt: $now, + lastAccessedAt: $now }) RETURN n `, { diff --git a/src/types.ts b/src/types.ts index 4ae637d..47f1596 100755 --- a/src/types.ts +++ b/src/types.ts @@ -10,6 +10,13 @@ export type NodeType = "TASK" | "SKILL" | "EVENT"; export type NodeStatus = "active" | "deprecated"; +/** + * 记忆分层 tier（与 NodeStatus 正交）。 + * decay 评分模型据此双向转换：core↔working↔peripheral。 + * 节点仍保持 status=active，仅 tier 变化；status=deprecated 只由手动弃用触发。 + */ +export type NodeTier = "core" | "working" | "peripheral"; + /** Neo4j label 映射：TASK->Task, SKILL->Skill, EVENT->Event */ export const NODE_TYPE_TO_LABEL: Record = { TASK: "Task", @@ -24,12 +31,24 @@ export interface GmNode { description: string; content: string; status: NodeStatus; + tier: NodeTier; validatedCount: number; sourceSessions: string[]; communityId: string | null; pagerank: number; createdAt: number; updatedAt: number; + /** + * 最近一次"相关性活动"时间戳（epoch ms），由 upsertNode 在任意写入路径刷新 + * （重新提取、gm_record、gm_update、CRUD POST）。是衰减判定的基准。 + * 与 updatedAt 的区别：updatedAt 在 deprecate/merge 时也会变，不能代表相关性； + * 而 mergeNodes 故意不更新 lastAccessedAt（合并 ≠ 用户重新激活）。 + */ + lastAccessedAt: number; + /** 最近一次 decay 评分（0~1，越大越鲜活/重要）。仅 applyDecay 写入。 */ + decayScore?: number; + /** decayScore 的计算时间戳（epoch ms）。 */ + decayComputedAt?: number; } // ─── 边 ─────────────────────────────────────────────────────── @@ -137,6 +156,30 @@ export interface Neo4jConfig { password: string; } +// ─── 衰减（柔性评分模型）配置 ───────────────────────────────── +// +// 完整公式、字段映射、默认值来源、调参指南见 docs/decay.md。 +// 评分和 tier 转换逻辑实现在 src/graph/decay.ts。 + +export interface DecayConfig { + enabled: boolean; + recencyHalfLifeDays: number; + recencyWeight: number; + importanceModulation: number; + frequencyWeight: number; + intrinsicWeight: number; + betaCore: number; + betaWorking: number; + betaPeripheral: number; + coreAccessThreshold: number; + coreCompositeThreshold: number; + coreImportanceThreshold: number; + peripheralCompositeThreshold: number; + peripheralAgeDays: number; + workingAccessThreshold: number; + workingCompositeThreshold: number; +} + // ─── 插件配置 ───────────────────────────────────────────────── export interface GmConfig { @@ -163,6 +206,8 @@ export interface GmConfig { dedupThreshold: number; pagerankDamping: number; pagerankIterations: number; + /** 遗忘曲线衰减配置；未提供时使用 DEFAULT_CONFIG.decay。 */ + decay?: DecayConfig; } export const DEFAULT_CONFIG: GmConfig = { @@ -178,4 +223,22 @@ export const DEFAULT_CONFIG: GmConfig = { dedupThreshold: 0.90, pagerankDamping: 0.85, pagerankIterations: 20, + decay: { + enabled: true, + recencyHalfLifeDays: 30, + recencyWeight: 0.4, + importanceModulation: 1.5, + frequencyWeight: 0.3, + intrinsicWeight: 0.3, + betaCore: 0.8, + betaWorking: 1.0, + betaPeripheral: 1.3, + coreAccessThreshold: 10, + coreCompositeThreshold: 0.7, + coreImportanceThreshold: 0.8, + peripheralCompositeThreshold: 0.15, + peripheralAgeDays: 60, + workingAccessThreshold: 3, + workingCompositeThreshold: 0.4, + }, }; diff --git a/test/decay.test.ts b/test/decay.test.ts new file mode 100644 index 0000000..c6ab7a6 --- /dev/null +++ b/test/decay.test.ts @@ -0,0 +1,278 @@ +import { describe, it, expect } from "vitest"; +import { + normalizeImportance, + computeConfidence, + computeBeta, + scoreRecency, + scoreFrequency, + scoreIntrinsic, + scoreNode, + decideTierTransition, +} from "../src/graph/decay.ts"; +import { DEFAULT_CONFIG, type DecayConfig, type GmNode } from "../src/types.ts"; + +const cfg: DecayConfig = { ...DEFAULT_CONFIG.decay! }; +const NOW = Date.UTC(2026, 0, 15, 0, 0, 0); +const MS_PER_DAY = 86_400_000; + +function makeNode(overrides: Partial = {}): GmNode { + return { + id: "test-id", + type: "SKILL", + name: "test", + description: "", + content: "", + status: "active", + tier: "working", + validatedCount: 1, + sourceSessions: [], + communityId: null, + pagerank: 0, + createdAt: NOW - 10 * MS_PER_DAY, + updatedAt: NOW - 10 * MS_PER_DAY, + lastAccessedAt: NOW - 10 * MS_PER_DAY, + ...overrides, + }; +} + +describe("normalizeImportance", () => { + it("pagerank=0 时返回 0（即使 maxPagerank>0）", () => { + expect(normalizeImportance(0, 1.0)).toBe(0); + }); + + it("maxPagerank≤0 时返回 0（避免除零）", () => { + expect(normalizeImportance(5, 0)).toBe(0); + expect(normalizeImportance(5, -1)).toBe(0); + }); + + it("pagerank = maxPagerank 时返回 1", () => { + expect(normalizeImportance(0.5, 0.5)).toBe(1); + }); + + it("截断到 [0,1]", () => { + expect(normalizeImportance(2.0, 1.0)).toBe(1); + expect(normalizeImportance(-1, 1.0)).toBe(0); + }); +}); + +describe("computeConfidence", () => { + it("count=0 时 confidence=0", () => { + expect(computeConfidence(0)).toBe(0); + }); + + it("count=1 时 confidence=0.5", () => { + expect(computeConfidence(1)).toBeCloseTo(0.5, 6); + }); + + it("count 增大时饱和收敛到 1（永不达到）", () => { + expect(computeConfidence(10)).toBeLessThan(1); + expect(computeConfidence(100)).toBeLessThan(1); + expect(computeConfidence(100)).toBeGreaterThan(computeConfidence(10)); + }); + + it("负数按 0 处理", () => { + expect(computeConfidence(-5)).toBe(0); + }); +}); + +describe("computeBeta", () => { + it("core < working < peripheral（缓衰 → 促衰）", () => { + expect(computeBeta("core", cfg)).toBe(0.8); + expect(computeBeta("working", cfg)).toBe(1.0); + expect(computeBeta("peripheral", cfg)).toBe(1.3); + }); +}); + +describe("scoreRecency", () => { + it("刚刚访问（daysSince=0）→ 1.0", () => { + const node = makeNode({ lastAccessedAt: NOW }); + expect(scoreRecency(node, 0, NOW, cfg)).toBeCloseTo(1, 6); + }); + + it("importance=0 + working tier + 30 天 → recency ≈ 0.5（半衰期）", () => { + const node = makeNode({ tier: "working", lastAccessedAt: NOW - 30 * MS_PER_DAY }); + expect(scoreRecency(node, 0, NOW, cfg)).toBeCloseTo(0.5, 2); + }); + + it("高 importance 拉长 effectiveHL（衰减更慢）", () => { + const node = makeNode({ tier: "working", lastAccessedAt: NOW - 30 * MS_PER_DAY }); + const highImp = scoreRecency(node, 1.0, NOW, cfg); + const zeroImp = scoreRecency(node, 0, NOW, cfg); + expect(highImp).toBeGreaterThan(zeroImp); + expect(highImp).toBeGreaterThan(0.5); + }); + + it("tier=peripheral 比 tier=working 衰减更快", () => { + const days = 10; + const w = scoreRecency(makeNode({ tier: "working", lastAccessedAt: NOW - days * MS_PER_DAY }), 0, NOW, cfg); + const p = scoreRecency(makeNode({ tier: "peripheral", lastAccessedAt: NOW - days * MS_PER_DAY }), 0, NOW, cfg); + expect(p).toBeLessThan(w); + }); + + it("lastAccessedAt 缺失时回退到 updatedAt", () => { + const viaFallback = makeNode({ lastAccessedAt: 0, updatedAt: NOW - 5 * MS_PER_DAY }); + const direct = makeNode({ lastAccessedAt: NOW - 5 * MS_PER_DAY }); + expect(scoreRecency(viaFallback, 0, NOW, cfg)) + .toBeCloseTo(scoreRecency(direct, 0, NOW, cfg), 6); + }); +}); + +describe("scoreFrequency", () => { + it("count=0 时 base=0", () => { + expect(scoreFrequency(makeNode({ validatedCount: 0 }))).toBe(0); + }); + + it("count=1 时只返回 base（无 recentnessBonus）", () => { + const expected = 1 - Math.exp(-1 / 5); + expect(scoreFrequency(makeNode({ validatedCount: 1 }))).toBeCloseTo(expected, 6); + }); + + it("count > 1 时 base × (0.5 + 0.5*recentnessBonus)，结果 ≤ base", () => { + const node = makeNode({ + validatedCount: 3, + createdAt: NOW - 30 * MS_PER_DAY, + lastAccessedAt: NOW, + }); + const base = 1 - Math.exp(-3 / 5); + const score = scoreFrequency(node); + expect(score).toBeLessThanOrEqual(base); + expect(score).toBeGreaterThan(0); + }); + + it("访问越紧凑（avgGapDays 越小）recentnessBonus 越大", () => { + const tight = makeNode({ + validatedCount: 5, + createdAt: NOW - 4 * MS_PER_DAY, + lastAccessedAt: NOW, + }); + const sparse = makeNode({ + validatedCount: 5, + createdAt: NOW - 100 * MS_PER_DAY, + lastAccessedAt: NOW, + }); + expect(scoreFrequency(tight)).toBeGreaterThan(scoreFrequency(sparse)); + }); +}); + +describe("scoreIntrinsic", () => { + it("= importance × confidence", () => { + expect(scoreIntrinsic(0.5, 0.5)).toBeCloseTo(0.25, 6); + expect(scoreIntrinsic(1, 1)).toBe(1); + expect(scoreIntrinsic(0, 0.5)).toBe(0); + }); +}); + +describe("scoreNode", () => { + it("权重和为 1 时 composite 落在 [0,1]", () => { + const node = makeNode({ pagerank: 0.5, validatedCount: 5, lastAccessedAt: NOW }); + const r = scoreNode(node, 1.0, NOW, cfg); + expect(r.composite).toBeGreaterThanOrEqual(0); + expect(r.composite).toBeLessThanOrEqual(1); + }); + + it("新鲜高 PR 节点 composite 显著高于陈旧低 PR 节点", () => { + const fresh = makeNode({ pagerank: 1.0, validatedCount: 1, lastAccessedAt: NOW }); + const stale = makeNode({ + pagerank: 0, + validatedCount: 1, + lastAccessedAt: NOW - 90 * MS_PER_DAY, + }); + expect(scoreNode(fresh, 1.0, NOW, cfg).composite) + .toBeGreaterThan(scoreNode(stale, 1.0, NOW, cfg).composite); + }); + + it("权重和≠1 时自动归一化，composite 仍落在 [0,1]", () => { + const skewedCfg: DecayConfig = { + ...cfg, + recencyWeight: 0.5, + frequencyWeight: 0.5, + intrinsicWeight: 0.5, // 和=1.5 + }; + const node = makeNode({ + pagerank: 1.0, + validatedCount: 10, + lastAccessedAt: NOW, + updatedAt: NOW, + createdAt: NOW, + }); + const r = scoreNode(node, 1.0, NOW, skewedCfg); + expect(r.composite).toBeLessThanOrEqual(1); + expect(r.composite).toBeGreaterThanOrEqual(0); + }); + + it("权重和为 0 时回退到等权重，不抛错", () => { + const zeroCfg: DecayConfig = { + ...cfg, + recencyWeight: 0, + frequencyWeight: 0, + intrinsicWeight: 0, + }; + const node = makeNode({ pagerank: 0.5, validatedCount: 1, lastAccessedAt: NOW }); + const r = scoreNode(node, 1.0, NOW, zeroCfg); + expect(Number.isFinite(r.composite)).toBe(true); + }); +}); + +describe("decideTierTransition", () => { + const scoreLow = { composite: 0.1, recency: 0, frequency: 0, intrinsic: 0 }; + const scoreHigh = { composite: 0.9, recency: 0.9, frequency: 0.9, intrinsic: 0.9 }; + const scoreMid = { composite: 0.5, recency: 0.5, frequency: 0.5, intrinsic: 0 }; + + it("core + composite 低 + count 低 → working", () => { + const node = makeNode({ tier: "core", validatedCount: 1 }); + expect(decideTierTransition(node, scoreLow, 0, cfg, NOW)).toBe("working"); + }); + + it("core + composite 高 → 保持 core", () => { + const node = makeNode({ tier: "core", validatedCount: 20 }); + expect(decideTierTransition(node, scoreHigh, 0.9, cfg, NOW)).toBeNull(); + }); + + it("working + composite < pct → peripheral", () => { + const node = makeNode({ tier: "working", validatedCount: 1 }); + expect(decideTierTransition(node, scoreLow, 0, cfg, NOW)).toBe("peripheral"); + }); + + it("working + 陈旧（age > peripheralAgeDays）+ count 低 → peripheral", () => { + const node = makeNode({ + tier: "working", + validatedCount: 1, + createdAt: NOW - (cfg.peripheralAgeDays + 1) * MS_PER_DAY, + }); + expect(decideTierTransition(node, scoreMid, 0, cfg, NOW)).toBe("peripheral"); + }); + + it("working + 陈旧但 count 充足 → 保持 working", () => { + const node = makeNode({ + tier: "working", + validatedCount: 5, + createdAt: NOW - (cfg.peripheralAgeDays + 1) * MS_PER_DAY, + }); + expect(decideTierTransition(node, scoreMid, 0, cfg, NOW)).toBeNull(); + }); + + it("peripheral + count 充足 + composite 高 → working", () => { + const node = makeNode({ tier: "peripheral", validatedCount: 5 }); + expect(decideTierTransition(node, scoreMid, 0, cfg, NOW)).toBe("working"); + }); + + it("peripheral + count 不足 → 保持 peripheral", () => { + const node = makeNode({ tier: "peripheral", validatedCount: 1 }); + expect(decideTierTransition(node, scoreHigh, 0, cfg, NOW)).toBeNull(); + }); + + it("working + count + composite + importance 都高 → core", () => { + const node = makeNode({ tier: "working", validatedCount: 15 }); + expect(decideTierTransition(node, scoreHigh, 0.9, cfg, NOW)).toBe("core"); + }); + + it("working + count + composite 高但 importance 不足 → 保持 working", () => { + const node = makeNode({ tier: "working", validatedCount: 15 }); + expect(decideTierTransition(node, scoreHigh, 0.5, cfg, NOW)).toBeNull(); + }); + + it("tier undefined 按 working 处理", () => { + const node = makeNode({ tier: undefined as unknown as GmNode["tier"], validatedCount: 1 }); + expect(decideTierTransition(node, scoreLow, 0, cfg, NOW)).toBe("peripheral"); + }); +}); From 2f7fa58c71c617e5c5e51db925b93930480330c6 Mon Sep 17 00:00:00 2001 From: TriDefender Date: Tue, 11 Aug 2026 13:56:57 +0800 Subject: [PATCH 03/29] Fix: Fixed tests --- src/graph/decay.ts | 17 +++++++++++------ src/types.ts | 6 ++++-- test/assemble-context.test.ts | 2 ++ test/integration.assemble.test.ts | 2 ++ 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/graph/decay.ts b/src/graph/decay.ts index 6de5c07..9859c88 100644 --- a/src/graph/decay.ts +++ b/src/graph/decay.ts @@ -52,6 +52,15 @@ export function computeConfidence(validatedCount: number): number { // ─── 三因子评分（纯函数） ──────────────────────────────────── +/** 选 lastAccessedAt → updatedAt → createdAt 中第一个 > 0 的，用于回退旧节点缺字段。 */ +function pickLastActive(node: Pick): number { + const la = node.lastAccessedAt ?? 0; + const up = node.updatedAt ?? 0; + if (la > 0) return la; + if (up > 0) return up; + return node.createdAt ?? 0; +} + /** β 随 tier 变化：core 缓衰、peripheral 促衰。 */ export function computeBeta(tier: NodeTier, cfg: DecayConfig): number { switch (tier) { @@ -71,9 +80,7 @@ export function scoreRecency( now: number, cfg: DecayConfig, ): number { - const lastActive = node.lastAccessedAt > 0 - ? node.lastAccessedAt - : (node.updatedAt > 0 ? node.updatedAt : node.createdAt); + const lastActive = pickLastActive(node); const daysSince = Math.max(0, (now - lastActive) / MS_PER_DAY); const effectiveHL = cfg.recencyHalfLifeDays * Math.exp(cfg.importanceModulation * importance); @@ -94,9 +101,7 @@ export function scoreFrequency( const base = 1 - Math.exp(-count / 5); if (count <= 1) return base; - const lastActive = node.lastAccessedAt > 0 - ? node.lastAccessedAt - : (node.updatedAt > 0 ? node.updatedAt : node.createdAt); + const lastActive = pickLastActive(node); const accessSpanDays = Math.max(1, (lastActive - node.createdAt) / MS_PER_DAY); const avgGapDays = accessSpanDays / Math.max(count - 1, 1); const recentnessBonus = Math.exp(-avgGapDays / 30); diff --git a/src/types.ts b/src/types.ts index 47f1596..4fd1097 100755 --- a/src/types.ts +++ b/src/types.ts @@ -31,7 +31,8 @@ export interface GmNode { description: string; content: string; status: NodeStatus; - tier: NodeTier; + /** 与 NodeStatus 正交的衰减分层；旧节点/新节点缺省时按 working 处理。 */ + tier?: NodeTier; validatedCount: number; sourceSessions: string[]; communityId: string | null; @@ -43,8 +44,9 @@ export interface GmNode { * （重新提取、gm_record、gm_update、CRUD POST）。是衰减判定的基准。 * 与 updatedAt 的区别：updatedAt 在 deprecate/merge 时也会变，不能代表相关性； * 而 mergeNodes 故意不更新 lastAccessedAt（合并 ≠ 用户重新激活）。 + * 缺省时回退到 updatedAt / createdAt。 */ - lastAccessedAt: number; + lastAccessedAt?: number; /** 最近一次 decay 评分（0~1，越大越鲜活/重要）。仅 applyDecay 写入。 */ decayScore?: number; /** decayScore 的计算时间戳（epoch ms）。 */ diff --git a/test/assemble-context.test.ts b/test/assemble-context.test.ts index e025e90..0c2340c 100644 --- a/test/assemble-context.test.ts +++ b/test/assemble-context.test.ts @@ -11,12 +11,14 @@ function makeNode(overrides: Partial): GmNode { description: "description", content: "content", status: "active", + tier: "working", validatedCount: 1, sourceSessions: ["test"], communityId: null, pagerank: 0, createdAt: now, updatedAt: now, + lastAccessedAt: now, ...overrides, }; } diff --git a/test/integration.assemble.test.ts b/test/integration.assemble.test.ts index 594458e..705985b 100644 --- a/test/integration.assemble.test.ts +++ b/test/integration.assemble.test.ts @@ -24,12 +24,14 @@ function makeNode(over: Partial): GmNode { description: over.description ?? "desc", content: over.content ?? "content body", status: over.status ?? "active", + tier: over.tier ?? "working", validatedCount: over.validatedCount ?? 1, sourceSessions: over.sourceSessions ?? ["s1"], communityId: over.communityId ?? null, pagerank: over.pagerank ?? 0, createdAt: over.createdAt ?? Date.now(), updatedAt: over.updatedAt ?? Date.now(), + lastAccessedAt: over.lastAccessedAt ?? Date.now(), }; } From dcfad1d2b2e8739eb7db877d2aafafccf746dfc8 Mon Sep 17 00:00:00 2001 From: TriDefender Date: Wed, 12 Aug 2026 16:23:34 +0800 Subject: [PATCH 04/29] =?UTF-8?q?Fix:=20=E7=A1=AE=E4=BF=9D=E8=B7=A8?= =?UTF-8?q?=E6=8F=92=E4=BB=B6=E9=87=8D=E5=90=AF=E5=8F=AF=E4=BB=A5=E7=BB=A7?= =?UTF-8?q?=E6=89=BF=E7=8A=B6=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- index.ts | 135 +++++++++++++++++++++++++------------- setup-graph-memory-pro.sh | 1 + src/store/store.ts | 19 ++++++ 3 files changed, 110 insertions(+), 45 deletions(-) diff --git a/index.ts b/index.ts index 4b1aa7b..cd88dde 100755 --- a/index.ts +++ b/index.ts @@ -9,7 +9,7 @@ import type { OpenClawPluginApi } from "openclaw/plugin-sdk"; import { Type } from "@sinclair/typebox"; import { getDriver, initSchema, getSession } from "./src/store/db.ts"; import { - saveMessage, getUnextracted, + saveMessage, getUnextracted, getMaxTurnIndex, markExtracted, isTurnExtracted, upsertNode, upsertEdge, findByName, updateNode, deleteNode, deprecateNodeAndDisconnect, @@ -352,13 +352,30 @@ const graphMemoryProPlugin = { /** * 每轮结束后直接从原始消息提取知识图谱 * 一轮 = 用户发一条消息 → agent 不管调了多少工具 → 最终回复用户 + * + * compact() 与本函数对同一 session 存在 TOCTOU 竞争：两条路径都先 + * isTurnExtracted/getUnextracted → 调 LLM → 最后 markExtracted，中间窗口 + * 允许另一条路径重复提取同一批消息（重复 LLM 调用 + validatedCount 双递增）。 + * 用 per-session async 互斥锁串行化两条路径的提取体。 */ + const extractLocks = new Map>(); + function withExtractLock(sessionId: string, fn: () => Promise): Promise { + const prev = extractLocks.get(sessionId) ?? Promise.resolve(); + const chain = prev.catch(() => {}); + const result = chain.then(() => fn()); + // 链上只保留"上一轮是否结束"的状态，丢弃返回值并吞掉错误， + // 否则一次失败会永久污染链 → 后续 acquire 直接 reject。 + extractLocks.set(sessionId, result.then(() => undefined, () => undefined)); + return result; + } + async function extractTurnKnowledge(sessionId: string, turnNum: number, rawMessages: any[]): Promise { - try { - if (await isTurnExtracted(driver, sessionId, turnNum)) { - api.logger.info(`[graph-memory-pro] turn ${turnNum}: already extracted (compact), skipping`); - return; - } + return withExtractLock(sessionId, async () => { + try { + if (await isTurnExtracted(driver, sessionId, turnNum)) { + api.logger.info(`[graph-memory-pro] turn ${turnNum}: already extracted (compact), skipping`); + return; + } const existing = (await getBySession(driver, sessionId)).map(n => n.name); const result = await extractor.extract({ messages: rawMessages, @@ -401,10 +418,12 @@ const graphMemoryProPlugin = { } catch (err) { api.logger.error(`[graph-memory-pro] turn ${turnNum} extract failed: ${err}`); } + }); } // ── Session 运行时状态 ────────────────────────────────── const msgSeq = new Map(); + const msgSeqLoaders = new Map>(); const recalled = new Map(); const sessionIdsByKey = new Map(); const pendingSubagentRecall = new Map(); @@ -421,6 +440,24 @@ const graphMemoryProPlugin = { } async function ingestMessage(sessionId: string, message: any): Promise { + if (!msgSeq.has(sessionId)) { + // 插件重启后内存 Map 会丢，必须从 DB 恢复 MAX(turnIndex)，否则下一条消息 + // turnIndex=1 → MERGE 命中旧行 → ON CREATE 被跳过 → 新消息静默丢失。 + // in-flight Promise 去重，避免并发 ingest 同时查询 + 互相覆盖 seq。 + let loader = msgSeqLoaders.get(sessionId); + if (!loader) { + loader = getMaxTurnIndex(driver, sessionId).then(max => { + msgSeq.set(sessionId, max); + msgSeqLoaders.delete(sessionId); + return max; + }).catch(err => { + msgSeqLoaders.delete(sessionId); + throw err; + }); + msgSeqLoaders.set(sessionId, loader); + } + await loader; + } const seq = (msgSeq.get(sessionId) ?? 0) + 1; msgSeq.set(sessionId, seq); await saveMessage(driver, sessionId, seq, message.role ?? "unknown", message); @@ -548,51 +585,53 @@ const graphMemoryProPlugin = { async compact({ sessionId, sessionKey, currentTokenCount }: { sessionId: string; sessionKey?: string; sessionFile: string; tokenBudget?: number; force?: boolean; currentTokenCount?: number }) { bindSessionIdentity(sessionId, sessionKey); - const msgs = await getUnextracted(driver, sessionId, cfg.compactTurnCount * 3); + return withExtractLock(sessionId, async () => { + const msgs = await getUnextracted(driver, sessionId, cfg.compactTurnCount * 3); - if (!msgs.length) return { ok: true, compacted: false, reason: "no messages" }; + if (!msgs.length) return { ok: true, compacted: false, reason: "no messages" }; - try { - const existing = (await getBySession(driver, sessionId)).map(n => n.name); - const result = await extractor.extract({ messages: msgs, existingNames: existing }); - - const nameToId = new Map(); - for (const nc of result.nodes) { - const { node } = await upsertNode(driver, { - type: nc.type, name: nc.name, - description: nc.description, content: nc.content, - }, sessionId); - nameToId.set(node.name, node.id); - recaller.syncEmbed(node).catch(() => {}); - } + try { + const existing = (await getBySession(driver, sessionId)).map(n => n.name); + const result = await extractor.extract({ messages: msgs, existingNames: existing }); + + const nameToId = new Map(); + for (const nc of result.nodes) { + const { node } = await upsertNode(driver, { + type: nc.type, name: nc.name, + description: nc.description, content: nc.content, + }, sessionId); + nameToId.set(node.name, node.id); + recaller.syncEmbed(node).catch(() => {}); + } - for (const ec of result.edges) { - const fromNode = await findByName(driver, ec.from); - const toNode = await findByName(driver, ec.to); - const fromId = nameToId.get(ec.from) ?? fromNode?.id; - const toId = nameToId.get(ec.to) ?? toNode?.id; - if (fromId && toId) { - await upsertEdge(driver, { - fromId, toId, type: ec.type, - instruction: ec.instruction, condition: ec.condition, sessionId, - }); + for (const ec of result.edges) { + const fromNode = await findByName(driver, ec.from); + const toNode = await findByName(driver, ec.to); + const fromId = nameToId.get(ec.from) ?? fromNode?.id; + const toId = nameToId.get(ec.to) ?? toNode?.id; + if (fromId && toId) { + await upsertEdge(driver, { + fromId, toId, type: ec.type, + instruction: ec.instruction, condition: ec.condition, sessionId, + }); + } } - } - const maxTurn = Math.max(...msgs.map((m: any) => m.turn_index)); - await markExtracted(driver, sessionId, maxTurn); + const maxTurn = Math.max(...msgs.map((m: any) => m.turn_index)); + await markExtracted(driver, sessionId, maxTurn); - return { - ok: true, compacted: true, - result: { - summary: `extracted ${result.nodes.length} nodes, ${result.edges.length} edges`, - tokensBefore: currentTokenCount ?? 0, - }, - }; - } catch (err) { - api.logger.error(`[graph-memory-pro] compact failed: ${err}`); - return { ok: false, compacted: false, reason: String(err) }; - } + return { + ok: true, compacted: true, + result: { + summary: `extracted ${result.nodes.length} nodes, ${result.edges.length} edges`, + tokensBefore: currentTokenCount ?? 0, + }, + }; + } catch (err) { + api.logger.error(`[graph-memory-pro] compact failed: ${err}`); + return { ok: false, compacted: false, reason: String(err) }; + } + }); }, async afterTurn({ sessionId, sessionKey, messages, prePromptMessageCount, isHeartbeat }: { @@ -649,6 +688,8 @@ const graphMemoryProPlugin = { if (childSessionId) { recalled.delete(childSessionId); msgSeq.delete(childSessionId); + msgSeqLoaders.delete(childSessionId); + extractLocks.delete(childSessionId); ingestedSinceTurn.delete(childSessionId); } sessionIdsByKey.delete(childSessionKey); @@ -657,6 +698,8 @@ const graphMemoryProPlugin = { async dispose() { msgSeq.clear(); + msgSeqLoaders.clear(); + extractLocks.clear(); recalled.clear(); sessionIdsByKey.clear(); pendingSubagentRecall.clear(); @@ -733,6 +776,8 @@ const graphMemoryProPlugin = { api.logger.error(`[graph-memory-pro] session_end error: ${err}`); } finally { msgSeq.delete(sid); + msgSeqLoaders.delete(sid); + extractLocks.delete(sid); recalled.delete(sid); ingestedSinceTurn.delete(sid); if (sessionKey && sessionIdsByKey.get(sessionKey) === sid) { diff --git a/setup-graph-memory-pro.sh b/setup-graph-memory-pro.sh index 6307a09..1e9baf8 100644 --- a/setup-graph-memory-pro.sh +++ b/setup-graph-memory-pro.sh @@ -79,6 +79,7 @@ NEO4J_USER="neo4j" NEO4J_URI="" # 留空 → 根据是否自建 Neo4j 自动决定 PLUGIN_REF="" INTERACTIVE=true +PC="" # 嵌入式 provider 选择（1-7）；交互模式由 read 赋值，非交互留空 AUTOSTART_METHODS=() # configure_autostart 写入；卸载与完成提示读取 while [[ $# -gt 0 ]]; do case "$1" in diff --git a/src/store/store.ts b/src/store/store.ts index d85541c..8e9eb7b 100755 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -859,6 +859,9 @@ export async function saveMessage( m.content = $content, m.extracted = false, m.createdAt = $now + ON MATCH SET + m.role = $role, + m.content = $content `, { id: uid("m"), sid, @@ -872,6 +875,22 @@ export async function saveMessage( } } +/** 该会话当前最大 turnIndex（无消息返回 0）。用于插件重启后恢复内存 msgSeq； + * 否则 turnIndex 从 1 重计 → MERGE 命中旧行 → ON CREATE 被跳过 → 新消息被静默丢弃。 */ +export async function getMaxTurnIndex(driver: Driver, sid: string): Promise { + const session = getSession(driver); + try { + const result = await session.run( + `MATCH (m:GmMessage {sessionId: $sid}) + RETURN coalesce(max(m.turnIndex), 0) AS maxTurn`, + { sid }, + ); + return toInt(result.records[0].get("maxTurn")); + } finally { + await session.close(); + } +} + export async function getUnextracted(driver: Driver, sid: string, limit: number): Promise { const session = getSession(driver); try { From 8ccfb793d383fb5eb5331c1d04a4419af226eba8 Mon Sep 17 00:00:00 2001 From: TriDefender Date: Sat, 15 Aug 2026 19:47:29 +0000 Subject: [PATCH 05/29] =?UTF-8?q?=E7=A7=BB=E6=A4=8D=E4=B8=BB=E5=B9=B2?= =?UTF-8?q?=E6=94=B9=E5=8A=A81fdec04=EF=BC=8C=E9=81=BF=E5=85=8D=E5=85=A8?= =?UTF-8?q?=E9=87=8F=E6=9B=B4=E6=96=B0communities?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 移植了https://github.com/adoresever/graph-memory/commit/1fdec04a8d49ffee1a4585c20c7162ee3e204370 src/store/store.ts - CommunitySummary 接口新增 memberSignature: string | null - upsertCommunitySummary() 增加 memberSignature 参数，MERGE 时写入（沿用 embedding 的 CASE 保留模式：传 null 不清空旧值） - getCommunitySummary() / getAllCommunitySummaries() 返回 memberSignature - 新增 getCommunitySummaryBySignature()：按签名查社区（ORDER BY updatedAt DESC LIMIT 1），带回 embedding 供复用 src/graph/community.ts - 新增导出 buildCommunityMemberSignature()：成员 ID 排序后 sha1（与上游逐字等价） - summarizeCommunities() 循环内两层短路： 1. 签名未变且摘要非空 → 跳过，不调 LLM 2. 其他社区存在相同签名 + 非空摘要 → 复用其 summary + embedding，不调 LLM 3. 否则走原有 LLM 生成路径，upsert 时写入签名 --- src/graph/community.ts | 29 ++++++++++++++++++++++++- src/store/db.ts | 1 + src/store/store.ts | 35 ++++++++++++++++++++++++++++++- test/community-signature.test.ts | 36 ++++++++++++++++++++++++++++++++ test/integration.graph.test.ts | 34 ++++++++++++++++++++++++++++-- 5 files changed, 131 insertions(+), 4 deletions(-) create mode 100644 test/community-signature.test.ts diff --git a/src/graph/community.ts b/src/graph/community.ts index d5f5dd5..71b021a 100755 --- a/src/graph/community.ts +++ b/src/graph/community.ts @@ -6,12 +6,15 @@ * 保留 summarizeCommunities()（需要 LLM） */ +import { createHash } from "node:crypto"; import type { Driver } from "neo4j-driver"; import { getSession } from "../store/db.ts"; import { clearCommunities, updateCommunities, upsertCommunitySummary, + getCommunitySummary, + getCommunitySummaryBySignature, pruneCommunitySummaries, } from "../store/store.ts"; import { getExistingActiveRelTypes, projectActiveGraph } from "./projection.ts"; @@ -142,6 +145,10 @@ const COMMUNITY_SUMMARY_SYS = `你是知识图谱社区摘要引擎。根据社 - 不要使用"社区"这个词 - 不要加引号或标点以外的格式`; +export function buildCommunityMemberSignature(memberIds: string[]): string { + return createHash("sha1").update([...memberIds].sort().join(",")).digest("hex"); +} + export async function summarizeCommunities( driver: Driver, communities: Map, @@ -154,6 +161,26 @@ export async function summarizeCommunities( for (const [communityId, memberIds] of communities) { if (memberIds.length === 0) continue; + const memberSignature = buildCommunityMemberSignature(memberIds); + + const current = await getCommunitySummary(driver, communityId); + if (current?.memberSignature === memberSignature && current.summary.trim()) { + continue; + } + + const reusable = await getCommunitySummaryBySignature(driver, memberSignature); + if (reusable?.summary.trim()) { + await upsertCommunitySummary( + driver, + communityId, + reusable.summary, + memberIds.length, + reusable.embedding, + memberSignature, + ); + continue; + } + const session = getSession(driver); let members: any[]; try { @@ -204,7 +231,7 @@ export async function summarizeCommunities( } catch {} } - await upsertCommunitySummary(driver, communityId, cleaned, memberIds.length, embedding); + await upsertCommunitySummary(driver, communityId, cleaned, memberIds.length, embedding, memberSignature); generated++; } catch (err) { console.log(` [WARN] community summary failed for ${communityId}: ${err}`); diff --git a/src/store/db.ts b/src/store/db.ts index 620441c..d5d7315 100755 --- a/src/store/db.ts +++ b/src/store/db.ts @@ -75,6 +75,7 @@ export async function initSchema(driver: Driver, embedding?: EmbeddingConfig): P // Community await session.run("CREATE CONSTRAINT community_id IF NOT EXISTS FOR (c:Community) REQUIRE c.id IS UNIQUE"); + await session.run("CREATE INDEX community_member_signature IF NOT EXISTS FOR (c:Community) ON (c.memberSignature)"); // Message (temporary extraction buffer) await session.run("CREATE CONSTRAINT gm_msg_id IF NOT EXISTS FOR (m:GmMessage) REQUIRE m.id IS UNIQUE"); diff --git a/src/store/store.ts b/src/store/store.ts index 8e9eb7b..b5d6f6a 100755 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -1029,12 +1029,15 @@ export interface CommunitySummary { id: string; summary: string; nodeCount: number; + /** 成员 ID 排序后的 sha1 — 用于识别"成员构成未变"的社区（复用摘要） */ + memberSignature: string | null; createdAt: number; updatedAt: number; } export async function upsertCommunitySummary( - driver: Driver, id: string, summary: string, nodeCount: number, embedding?: number[], + driver: Driver, id: string, summary: string, nodeCount: number, + embedding?: number[], memberSignature?: string, ): Promise { const session = getSession(driver); try { @@ -1044,18 +1047,21 @@ export async function upsertCommunitySummary( c.summary = $summary, c.nodeCount = $nodeCount, c.embedding = $embedding, + c.memberSignature = $memberSignature, c.createdAt = $now, c.updatedAt = $now ON MATCH SET c.summary = $summary, c.nodeCount = $nodeCount, c.embedding = CASE WHEN $embedding IS NOT NULL THEN $embedding ELSE c.embedding END, + c.memberSignature = CASE WHEN $memberSignature IS NOT NULL THEN $memberSignature ELSE c.memberSignature END, c.updatedAt = $now `, { id, summary, nodeCount, embedding: embedding ?? null, + memberSignature: memberSignature ?? null, now: Date.now(), }); } finally { @@ -1076,6 +1082,32 @@ export async function getCommunitySummary(driver: Driver, id: string): Promise { + const session = getSession(driver); + try { + const result = await session.run( + "MATCH (c:Community {memberSignature: $memberSignature}) RETURN c ORDER BY c.updatedAt DESC LIMIT 1", + { memberSignature }, + ); + if (result.records.length === 0) return null; + const c = result.records[0].get("c").properties; + return { + id: c.id, + summary: c.summary, + nodeCount: toInt(c.nodeCount), + memberSignature: c.memberSignature ?? null, + embedding: Array.isArray(c.embedding) ? (c.embedding as number[]) : undefined, createdAt: toInt(c.createdAt), updatedAt: toInt(c.updatedAt), }; @@ -1096,6 +1128,7 @@ export async function getAllCommunitySummaries(driver: Driver): Promise { + it("成员顺序不影响签名（排序后哈希）", () => { + expect(buildCommunityMemberSignature(["a", "b", "c"])) + .toBe(buildCommunityMemberSignature(["c", "a", "b"])); + }); + + it("相同成员恒生成相同签名", () => { + expect(buildCommunityMemberSignature(["x", "y"])) + .toBe(buildCommunityMemberSignature(["x", "y"])); + }); + + it("成员构成不同则签名不同", () => { + expect(buildCommunityMemberSignature(["a", "b"])) + .not.toBe(buildCommunityMemberSignature(["a", "c"])); + expect(buildCommunityMemberSignature(["a", "b"])) + .not.toBe(buildCommunityMemberSignature(["a", "b", "c"])); + }); + + it("输出为 40 位小写 hex（sha1）", () => { + expect(buildCommunityMemberSignature(["a"])).toMatch(/^[0-9a-f]{40}$/); + }); + + it("不修改入参数组", () => { + const input = ["b", "a"]; + buildCommunityMemberSignature(input); + expect(input).toEqual(["b", "a"]); + }); +}); diff --git a/test/integration.graph.test.ts b/test/integration.graph.test.ts index 9dc2e90..db43d45 100644 --- a/test/integration.graph.test.ts +++ b/test/integration.graph.test.ts @@ -12,12 +12,14 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import type { Driver } from "neo4j-driver"; import { getDriver, initSchema, closeDriver, getSession } from "../src/store/db.ts"; import { - upsertNode, upsertEdge, saveVector, findById, deprecate, + upsertNode, upsertEdge, saveVector, findById, deprecate, getCommunitySummary, } from "../src/store/store.ts"; import { personalizedPageRank, computeGlobalPageRank, } from "../src/graph/pagerank.ts"; -import { detectCommunities, getCommunityPeers } from "../src/graph/community.ts"; +import { + detectCommunities, getCommunityPeers, summarizeCommunities, buildCommunityMemberSignature, +} from "../src/graph/community.ts"; import { detectDuplicates, dedup } from "../src/graph/dedup.ts"; import { runMaintenance } from "../src/graph/maintenance.ts"; import { DEFAULT_CONFIG, type GmConfig } from "../src/types.ts"; @@ -187,6 +189,34 @@ describe.skipIf(!ENABLED)("graph layer integration (GDS, Docker)", () => { } }); + it("summarizeCommunities：社区成员未变时复用摘要，不重调 LLM", async () => { + const memberIds = [nodeIds["gmpsrc-deploy"], nodeIds["gmpsrc-compose"]]; + const communities = new Map([["c-reuse-test", memberIds]]); + let llmCalls = 0; + const llm = async () => { + llmCalls += 1; + return "容器部署与编排技能"; + }; + + const first = await summarizeCommunities(driver, communities, llm); + const second = await summarizeCommunities(driver, communities, llm); + + expect(first).toBe(1); + expect(second).toBe(0); + expect(llmCalls).toBe(1); + + const summary = await getCommunitySummary(driver, "c-reuse-test"); + expect(summary?.summary).toBe("容器部署与编排技能"); + expect(summary?.memberSignature).toBe(buildCommunityMemberSignature(memberIds)); + + const cleanup = getSession(driver); + try { + await cleanup.run("MATCH (c:Community {id: 'c-reuse-test'}) DELETE c"); + } finally { + await cleanup.close(); + } + }); + it("detectDuplicates：gmpsrc-* 无 embedding，函数不抛错", async () => { let passed = false; await expectDimSafe(async () => { From 492923cbd81da541f7e2e7fcf5b50102a14c263f Mon Sep 17 00:00:00 2001 From: TriDefender Date: Sat, 15 Aug 2026 19:58:54 +0000 Subject: [PATCH 06/29] Update integration.graph.test.ts --- test/integration.graph.test.ts | 36 ++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/test/integration.graph.test.ts b/test/integration.graph.test.ts index db43d45..8b52d10 100644 --- a/test/integration.graph.test.ts +++ b/test/integration.graph.test.ts @@ -191,15 +191,27 @@ describe.skipIf(!ENABLED)("graph layer integration (GDS, Docker)", () => { it("summarizeCommunities：社区成员未变时复用摘要，不重调 LLM", async () => { const memberIds = [nodeIds["gmpsrc-deploy"], nodeIds["gmpsrc-compose"]]; - const communities = new Map([["c-reuse-test", memberIds]]); + + // 生产不变量：detectCommunities 会先给成员节点写入 communityId， + // pruneCommunitySummaries 只保留仍被 active 成员引用的社区 — 不先 SET 会被 prune 删掉 + const prepare = getSession(driver); + try { + await prepare.run( + "MATCH (n:MemoryNode) WHERE n.id IN $ids SET n.communityId = $cid", + { ids: memberIds, cid: "c-reuse-test" }, + ); + } finally { + await prepare.close(); + } + let llmCalls = 0; const llm = async () => { llmCalls += 1; return "容器部署与编排技能"; }; - const first = await summarizeCommunities(driver, communities, llm); - const second = await summarizeCommunities(driver, communities, llm); + const first = await summarizeCommunities(driver, new Map([["c-reuse-test", memberIds]]), llm); + const second = await summarizeCommunities(driver, new Map([["c-reuse-test", memberIds]]), llm); expect(first).toBe(1); expect(second).toBe(0); @@ -209,9 +221,25 @@ describe.skipIf(!ENABLED)("graph layer integration (GDS, Docker)", () => { expect(summary?.summary).toBe("容器部署与编排技能"); expect(summary?.memberSignature).toBe(buildCommunityMemberSignature(memberIds)); + // detectCommunities 每轮按成员数重编号（c-1..c-N），ID 变但成员相同 → 按签名跨社区复用 + const third = await summarizeCommunities( + driver, new Map([["c-reuse-renumbered", memberIds]]), llm, + ); + expect(third).toBe(0); + expect(llmCalls).toBe(1); + const renumbered = await getCommunitySummary(driver, "c-reuse-renumbered"); + expect(renumbered?.summary).toBe("容器部署与编排技能"); + expect(renumbered?.memberSignature).toBe(buildCommunityMemberSignature(memberIds)); + const cleanup = getSession(driver); try { - await cleanup.run("MATCH (c:Community {id: 'c-reuse-test'}) DELETE c"); + await cleanup.run( + "MATCH (c:Community) WHERE c.id IN ['c-reuse-test', 'c-reuse-renumbered'] DELETE c", + ); + await cleanup.run( + "MATCH (n:MemoryNode) WHERE n.id IN $ids SET n.communityId = null", + { ids: memberIds }, + ); } finally { await cleanup.close(); } From 499cb767485aeaf2fe71d8732024ef4cd650cf4b Mon Sep 17 00:00:00 2001 From: TriDefender Date: Sat, 15 Aug 2026 21:24:07 +0000 Subject: [PATCH 07/29] Added finegrain memory control on cron This commit aims to tackle more finegrained memory control on cron sessions: Does a repeating cron session generates 'false popularity' on specific memory nodes? Now it enables you to: - disable memory extraction for cron runs - disable entirely graph functions, so it will not inject context information (it may change workflows, use this with caution since models may lack crucial information. you're advised to test the workflow before hand) - skip session end actions on cron --- README.md | 24 +++++ README_CN.md | 22 ++++ index.ts | 51 ++++++++- openclaw.plugin.json | 9 ++ src/types.ts | 27 +++++ test/session-identity.test.ts | 197 +++++++++++++++++++++++++++++++++- 6 files changed, 324 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 79bdc64..5ecb143 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,30 @@ Common overrides — for fuller control see `docs/decay.md` §4: } ``` +### Cron sessions + +Sessions created by OpenClaw scheduled tasks can be configured independently of normal sessions. The host places the cron marker on the **sessionKey** (`sessionId` is a random UUID); real shapes are `cron:`, `agent::cron:`, or `agent::cron::run:`: + +```json +"cron": { + "enabled": true, + "extract": true, + "finalizeAndMaintain": true +} +``` + +| Option | Default | Description | +| --- | --- | --- | +| `enabled` | `false` | Enable graph functionality inside cron sessions (recall injection + message buffering). When `false`, cron sessions skip automatic recall and message persistence; the `gm_*` tools remain available for explicit calls (manual escape hatch). | +| `extract` | `false` | Trigger knowledge extraction (LLM triples) in cron sessions via `afterTurn` / `compact`. When `false`, messages are still buffered and can be backfilled later with `openclaw graph-memory extract`. | +| `finalizeAndMaintain` | `false` | Run finalize (EVENT→SKILL promotion) and graph maintenance (decay / PageRank / communities) when a cron session ends. Disable when frequent cron runs make end-of-session global maintenance too costly. | + +All three options default to **`false`**: cron sessions skip the graph entirely (no recall, no buffering, no extraction, no maintenance) unless explicitly enabled. `enabled: false` is the master switch — even with `extract` / `finalizeAndMaintain` set to `true`, nothing runs. Non-cron sessions are never affected by these options. + +All three sub-options are optional; omitted fields keep the default `false` (e.g. with `"cron": { "enabled": true }` only recall and message buffering are enabled — extraction and end-of-session maintenance stay off). + +Caveat: when a cron job sets an explicit custom `sessionKey`, the host does not append the `cron` segment — such sessions cannot be detected and are treated as normal sessions. + ### OAuth login (experimental) ```bash diff --git a/README_CN.md b/README_CN.md index eabc330..2851ca7 100644 --- a/README_CN.md +++ b/README_CN.md @@ -81,6 +81,28 @@ bash setup-graph-memory-pro.sh --uninstall `embedding` 可选。设置时，`dimensions` 必须与 Neo4j 向量索引维度一致。新数据库会在插件启动时按配置创建索引；更换维度后需要重建向量索引或 Neo4j 数据库。 +### cron 会话行为控制 + +OpenClaw 定时任务创建的会话可以独立配置图谱行为。host 把 cron 标记放在 **sessionKey** 上（`sessionId` 是随机 UUID），实际形状为 `cron:`、`agent::cron:` 或 `agent::cron::run:`： + +```json +"cron": { + "enabled": true, + "extract": true, + "finalizeAndMaintain": true +} +``` + +| 选项 | 默认 | 说明 | +| --- | --- | --- | +| `enabled` | `true` | 是否在 cron 会话内启用图谱功能（召回注入 + 消息入库）。关闭后 cron 会话不自动召回、不自动入库；`gm_*` 工具仍可手动调用（作为显式逃生通道）。 | +| `extract` | `true` | 是否在 cron 会话内触发知识提取（afterTurn / compact 的 LLM 三元组提取）。关闭后消息仍入库缓冲，之后可用 `openclaw graph-memory extract` 手动回填。 | +| `finalizeAndMaintain` | `true` | cron 会话结束时是否执行 finalize（EVENT→SKILL 晋升）和图维护（decay / PageRank / 社区检测）。定时任务频繁时可关闭，避免每次会话结束都跑全局维护。 | + +三个选项**默认全部开启**：cron 会话默认使用图谱，需按需显式关闭。`enabled=true` 是总开关：即使 `extract`/`finalizeAndMaintain` 设为 `false` 也不生效。非 cron 会话不受这些选项影响。 + +注意：若 cron 任务显式设置了自定义 `sessionKey`，host 不再附加 `cron` 段，此类会话无法被识别，将按普通会话处理。 + ### OAuth 登录（实验性） ```bash diff --git a/index.ts b/index.ts index cd88dde..9973e7b 100755 --- a/index.ts +++ b/index.ts @@ -24,7 +24,7 @@ import { Extractor } from "./src/extractor/extract.ts"; import { assembleContext } from "./src/format/assemble.ts"; import { sanitizeToolUseResultPairing } from "./src/format/transcript-repair.ts"; import { runMaintenance } from "./src/graph/maintenance.ts"; -import { DEFAULT_CONFIG, type GmConfig, type RecallResult, type EdgeType } from "./src/types.ts"; +import { DEFAULT_CONFIG, DEFAULT_CRON_CONFIG, isCronSessionKey, type GmConfig, type RecallResult, type EdgeType } from "./src/types.ts"; import { registerCrudRoutes } from "./src/routes/crud.ts"; import { createGraphMemoryCli } from "./src/cli.ts"; @@ -279,6 +279,8 @@ const graphMemoryProPlugin = { const cfg: GmConfig = { ...DEFAULT_CONFIG, ...raw }; if (raw.neo4j) cfg.neo4j = { ...DEFAULT_CONFIG.neo4j, ...raw.neo4j }; if (raw.decay) cfg.decay = { ...DEFAULT_CONFIG.decay, ...raw.decay }; + if (raw.cron) cfg.cron = { ...DEFAULT_CONFIG.cron, ...raw.cron }; + const cronCfg = cfg.cron ?? DEFAULT_CRON_CONFIG; const providerModel = readDefaultModel(api.config); @@ -467,6 +469,9 @@ const graphMemoryProPlugin = { api.on("before_agent_start", async (event: any, ctx: any) => { try { + // cron session 关闭图谱功能时不召回（cron 标记在 sessionKey 上，sessionId 是随机 UUID） + if (isCronSessionKey(typeof ctx?.sessionKey === "string" ? ctx.sessionKey : null) && !cronCfg.enabled) return; + const rawPrompt = typeof event?.prompt === "string" ? event.prompt : ""; const prompt = cleanPrompt(rawPrompt); if (!prompt) return; @@ -506,6 +511,10 @@ const graphMemoryProPlugin = { async ingest({ sessionId, sessionKey, message, isHeartbeat }: { sessionId: string; sessionKey?: string; message: any; isHeartbeat?: boolean }) { if (isHeartbeat) return { ingested: false }; bindSessionIdentity(sessionId, sessionKey); + // cron session 关闭图谱功能：消息不入库 + if (isCronSessionKey(sessionKey) && !cronCfg.enabled) { + return { ingested: false }; + } await ingestMessage(sessionId, message); ingestedSinceTurn.set(sessionId, (ingestedSinceTurn.get(sessionId) ?? 0) + 1); return { ingested: true }; @@ -517,6 +526,21 @@ const graphMemoryProPlugin = { bindSessionIdentity(sessionId, sessionKey); const budget = tokenBudget ?? 128_000; + // cron session 关闭图谱功能：仅做消息裁剪与配对修复，不注入图谱上下文 + if (isCronSessionKey(sessionKey) && !cronCfg.enabled) { + const prepared = prepareAssemblyMessages(messages); + if (prepared.dropped > 0) { + api.logger.info( + `[graph-memory-pro] assemble: ${prepared.messages.length} msgs (~${prepared.tokens} tok), ` + + `dropped ${prepared.dropped} older msgs, graph skipped (cron session)`, + ); + } + return { + messages: prepared.messages, + estimatedTokens: prepared.tokens, + }; + } + const activeNodes = await getBySession(driver, sessionId); const activeEdges: any[] = []; for (const n of activeNodes) { @@ -585,6 +609,13 @@ const graphMemoryProPlugin = { async compact({ sessionId, sessionKey, currentTokenCount }: { sessionId: string; sessionKey?: string; sessionFile: string; tokenBudget?: number; force?: boolean; currentTokenCount?: number }) { bindSessionIdentity(sessionId, sessionKey); + // cron session 关闭图谱功能或知识提取：不触发 LLM 提取 + if (isCronSessionKey(sessionKey) && !(cronCfg.enabled && cronCfg.extract)) { + return { + ok: true, compacted: false, + reason: cronCfg.enabled ? "cron session extraction disabled" : "cron session graph disabled", + }; + } return withExtractLock(sessionId, async () => { const msgs = await getUnextracted(driver, sessionId, cfg.compactTurnCount * 3); @@ -648,6 +679,12 @@ const graphMemoryProPlugin = { return; } + // cron session 关闭图谱功能：跳过入库回填与知识提取 + if (isCronSessionKey(sessionKey) && !cronCfg.enabled) { + ingestedSinceTurn.delete(sessionId); + return; + } + // Official OpenClaw delivers ingest() and afterTurn() as separate // lifecycle phases. Older downstream builds incorrectly call only // afterTurn(). Persist just the missing suffix so neither host loses @@ -668,6 +705,12 @@ const graphMemoryProPlugin = { api.logger.info(`[graph-memory-pro] afterTurn sid=${sessionId.slice(0, 8)} turn=${turnNum} rawMsgs=${newMessages.length}`); + // cron session 关闭知识提取：消息仅入库缓冲，可稍后用 `graph-memory extract` 手动回填 + if (isCronSessionKey(sessionKey) && !cronCfg.extract) { + api.logger.info("[graph-memory-pro] cron session: extraction skipped (cron.extract=false)"); + return; + } + // 直接用原始消息提取知识图谱（异步，不阻塞） extractTurnKnowledge(sessionId, turnNum, newMessages).catch(err => { api.logger.error(`[graph-memory-pro] extract failed: ${err}`); @@ -722,6 +765,12 @@ const graphMemoryProPlugin = { : typeof ctx?.sessionKey === "string" ? ctx.sessionKey : undefined; try { + // cron session：图谱功能关闭或明确禁用时，跳过 finalize 与图维护（finally 清理仍执行） + if (isCronSessionKey(sessionKey) && !(cronCfg.enabled && cronCfg.finalizeAndMaintain)) { + api.logger.info(`[graph-memory-pro] cron session ${sid.slice(0, 12)}…: finalize + maintenance skipped (cron config)`); + return; + } + const nodes = await getBySession(driver, sid); if (nodes.length) { // 获取图谱摘要 diff --git a/openclaw.plugin.json b/openclaw.plugin.json index 1a6b474..8022351 100755 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -51,6 +51,15 @@ "workingCompositeThreshold": { "type": "number", "default": 0.4, "description": "peripheral→working 所需的最低 composite 分数。" } } }, + "cron": { + "type": "object", + "description": "cron 定时会话的图谱行为开关（host 将 cron 标记放在 sessionKey 上，形如 agent::cron:）。默认全部启用，与普通会话一致。注意：cron 任务若显式设置自定义 sessionKey，则无法被识别为 cron 会话。", + "properties": { + "enabled": { "type": "boolean", "default": true, "description": "是否在 cron session 内启用图谱功能（召回注入 + 消息入库）。关闭后 cron 会话不自动召回、不自动入库；gm_* 工具仍可手动调用。" }, + "extract": { "type": "boolean", "default": true, "description": "是否在 cron session 内触发知识提取（afterTurn / compact 的 LLM 三元组提取）。关闭后消息仍入库缓冲，可用 openclaw graph-memory extract 手动回填。" }, + "finalizeAndMaintain": { "type": "boolean", "default": true, "description": "cron session 结束时是否执行 finalize（EVENT→SKILL 晋升）和图维护（decay/PageRank/社区检测）。频繁的 cron 任务可关闭以避免每次结束都跑全局维护。" } + } + }, "llm": { "type": "object", "properties": { diff --git a/src/types.ts b/src/types.ts index 4fd1097..ce30fca 100755 --- a/src/types.ts +++ b/src/types.ts @@ -182,6 +182,31 @@ export interface DecayConfig { workingCompositeThreshold: number; } +// ─── cron 会话（定时任务）的图谱行为配置 ───────────────────── + +/** + * 判断是否为 cron 定时会话。host 把 cron 标记放在 sessionKey 上（sessionId 是随机 UUID）， + * 实际形状：cron: / agent::cron: / agent::cron::run:。 + * 按段匹配（split(":") 后包含 "cron"），避免误匹配 "cron-daily" 这类自定义段。 + * 注意：cron 任务若显式设置了自定义 sessionKey，host 不再附加 cron 段，此类会话无法识别（见 README）。 + * cron session 的图谱行为（召回/消息入库、知识提取、结束维护）可由 `cron` 配置独立开关；非 cron session 不受影响。 + */ +export function isCronSessionKey(sessionKey: string | undefined | null): boolean { + return typeof sessionKey === "string" && sessionKey.split(":").includes("cron"); +} + +export interface CronConfig { + enabled: boolean; + extract: boolean; + finalizeAndMaintain: boolean; +} + +export const DEFAULT_CRON_CONFIG: CronConfig = { + enabled: true, + extract: true, + finalizeAndMaintain: true, +}; + // ─── 插件配置 ───────────────────────────────────────────────── export interface GmConfig { @@ -210,6 +235,7 @@ export interface GmConfig { pagerankIterations: number; /** 遗忘曲线衰减配置；未提供时使用 DEFAULT_CONFIG.decay。 */ decay?: DecayConfig; + cron?: CronConfig; } export const DEFAULT_CONFIG: GmConfig = { @@ -243,4 +269,5 @@ export const DEFAULT_CONFIG: GmConfig = { workingAccessThreshold: 3, workingCompositeThreshold: 0.4, }, + cron: DEFAULT_CRON_CONFIG, }; diff --git a/test/session-identity.test.ts b/test/session-identity.test.ts index c2ec70c..1078531 100644 --- a/test/session-identity.test.ts +++ b/test/session-identity.test.ts @@ -1,7 +1,12 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +import { isCronSessionKey } from "../src/types.ts"; const mocks = vi.hoisted(() => ({ getBySession: vi.fn(async () => []), + saveMessage: vi.fn(async () => {}), + getMaxTurnIndex: vi.fn(async () => 0), + getUnextracted: vi.fn(async () => []), + isTurnExtracted: vi.fn(async () => false), recall: vi.fn(async () => ({ nodes: [{ id: "recalled-node" }], edges: [], @@ -25,10 +30,11 @@ vi.mock("../src/store/db.ts", () => ({ })); vi.mock("../src/store/store.ts", () => ({ - saveMessage: async () => {}, - getUnextracted: async () => [], + saveMessage: mocks.saveMessage, + getUnextracted: mocks.getUnextracted, + getMaxTurnIndex: mocks.getMaxTurnIndex, markExtracted: async () => {}, - isTurnExtracted: async () => false, + isTurnExtracted: mocks.isTurnExtracted, upsertNode: async () => ({ node: {}, isNew: false }), upsertEdge: async () => {}, findByName: async () => null, @@ -81,11 +87,28 @@ import graphMemoryProPlugin from "../index.ts"; type HookHandler = (event: Record, context: Record) => Promise; type EngineHarness = { readonly bootstrap: (params: { readonly sessionId: string; readonly sessionKey?: string }) => Promise; + readonly ingest: (params: { + readonly sessionId: string; + readonly sessionKey?: string; + readonly message: unknown; + readonly isHeartbeat?: boolean; + }) => Promise<{ readonly ingested: boolean }>; readonly assemble: (params: { readonly sessionId: string; readonly sessionKey?: string; readonly messages: readonly unknown[]; }) => Promise; + readonly compact: (params: { readonly sessionId: string; readonly sessionKey?: string }) => Promise<{ + readonly ok: boolean; + readonly compacted: boolean; + readonly reason?: string; + }>; + readonly afterTurn: (params: { + readonly sessionId: string; + readonly sessionKey?: string; + readonly messages: readonly unknown[]; + readonly prePromptMessageCount: number; + }) => Promise; readonly prepareSubagentSpawn: (params: { readonly parentSessionKey: string; readonly childSessionKey: string; @@ -93,7 +116,7 @@ type EngineHarness = { }) => Promise<{ readonly rollback: () => void }>; }; -function registerPlugin(): { readonly hooks: Map; readonly engine: EngineHarness } { +function registerPlugin(pluginConfig: Record = {}): { readonly hooks: Map; readonly engine: EngineHarness } { const hooks = new Map(); let engine: EngineHarness | undefined; graphMemoryProPlugin.register({ @@ -104,7 +127,7 @@ function registerPlugin(): { readonly hooks: Map; readonly error: () => {}, }, config: {}, - pluginConfig: {}, + pluginConfig, resolvePath: (path: string) => path, on: (event: string, handler: HookHandler) => { hooks.set(event, handler); }, registerContextEngine: (_id: string, factory: () => EngineHarness) => { engine = factory(); }, @@ -165,3 +188,167 @@ describe("session identity", () => { ); }); }); + +describe("cron session gating (cron 配置)", () => { + beforeEach(() => { + mocks.getBySession.mockClear(); + mocks.saveMessage.mockClear(); + mocks.getMaxTurnIndex.mockClear(); + mocks.getUnextracted.mockClear(); + mocks.isTurnExtracted.mockClear(); + mocks.recall.mockClear(); + mocks.assembleContext.mockClear(); + mocks.runMaintenance.mockClear(); + }); + + // host 契约：sessionId 是随机 transcript UUID，cron 标记在 sessionKey 上 + const CRON_KEY = "agent:agent-1:cron:daily-report"; + const CRON_SID = "0f1e2d3c-4b5a-6978-8976-543210fedcba"; + + it("isCronSessionKey 按 sessionKey 段匹配 cron 标记", () => { + expect(isCronSessionKey("cron:job-1")).toBe(true); + expect(isCronSessionKey("agent:agent-1:cron:daily")).toBe(true); + expect(isCronSessionKey("agent:agent-1:cron:daily:run:r1")).toBe(true); + expect(isCronSessionKey("agent:main")).toBe(false); + expect(isCronSessionKey("agent:cron-daily:main")).toBe(false); + expect(isCronSessionKey("scheduled-cron:x")).toBe(false); + expect(isCronSessionKey("")).toBe(false); + expect(isCronSessionKey(undefined)).toBe(false); + expect(isCronSessionKey(null)).toBe(false); + }); + + it("默认配置（全 false）下 cron session_end 跳过 finalize 与图维护", async () => { + const handler = registerPlugin().hooks.get("session_end"); + if (!handler) throw new Error("session_end hook was not registered"); + + await handler({ sessionId: CRON_SID, sessionKey: CRON_KEY }, {}); + + expect(mocks.getBySession).not.toHaveBeenCalled(); + expect(mocks.runMaintenance).not.toHaveBeenCalled(); + }); + + it("默认配置下 cron session 不入库（默认关闭）", async () => { + const { engine } = registerPlugin(); + + await expect(engine.ingest({ sessionId: CRON_SID, sessionKey: CRON_KEY, message: { role: "user", content: "hi" } })) + .resolves.toEqual({ ingested: false }); + expect(mocks.saveMessage).not.toHaveBeenCalled(); + }); + + it("finalizeAndMaintain=false 时 cron session 跳过 finalize 与图维护", async () => { + const handler = registerPlugin({ cron: { enabled: true, finalizeAndMaintain: false } }).hooks.get("session_end"); + if (!handler) throw new Error("session_end hook was not registered"); + + await handler({ sessionId: CRON_SID, sessionKey: CRON_KEY }, {}); + + expect(mocks.getBySession).not.toHaveBeenCalled(); + expect(mocks.runMaintenance).not.toHaveBeenCalled(); + }); + + it("finalizeAndMaintain=true 时 cron session 执行 finalize 与图维护", async () => { + const handler = registerPlugin({ cron: { enabled: true, finalizeAndMaintain: true } }).hooks.get("session_end"); + if (!handler) throw new Error("session_end hook was not registered"); + + await handler({ sessionId: CRON_SID, sessionKey: CRON_KEY }, {}); + + expect(mocks.getBySession).toHaveBeenCalledWith({}, CRON_SID); + expect(mocks.runMaintenance).toHaveBeenCalledTimes(1); + }); + + it("finalizeAndMaintain=true 不影响普通会话的既有行为", async () => { + const handler = registerPlugin({ cron: { enabled: true, finalizeAndMaintain: false } }).hooks.get("session_end"); + if (!handler) throw new Error("session_end hook was not registered"); + + await handler({ sessionId: "normal-session", sessionKey: "agent:main" }, {}); + + expect(mocks.runMaintenance).toHaveBeenCalledTimes(1); + }); + + it("enabled=false 时 cron session 不召回、不入库、不注入图谱上下文", async () => { + const { hooks, engine } = registerPlugin({ cron: { enabled: false } }); + + const beforeAgentStart = hooks.get("before_agent_start"); + if (!beforeAgentStart) throw new Error("before_agent_start hook was not registered"); + await beforeAgentStart({ prompt: "daily digest" }, { sessionKey: CRON_KEY }); + expect(mocks.recall).not.toHaveBeenCalled(); + + await expect(engine.ingest({ sessionId: CRON_SID, sessionKey: CRON_KEY, message: { role: "user", content: "hi" } })) + .resolves.toEqual({ ingested: false }); + expect(mocks.saveMessage).not.toHaveBeenCalled(); + + await engine.assemble({ sessionId: CRON_SID, sessionKey: CRON_KEY, messages: [] }); + expect(mocks.assembleContext).not.toHaveBeenCalled(); + }); + + it("enabled=false 总开关：cron afterTurn 跳过入库回填，compact 跳过提取", async () => { + const { engine } = registerPlugin({ cron: { enabled: false } }); + + await engine.afterTurn({ + sessionId: CRON_SID, + sessionKey: CRON_KEY, + messages: [{ role: "user", content: "hi" }], + prePromptMessageCount: 0, + }); + expect(mocks.saveMessage).not.toHaveBeenCalled(); + expect(mocks.isTurnExtracted).not.toHaveBeenCalled(); + + const res = await engine.compact({ sessionId: CRON_SID, sessionKey: CRON_KEY }); + expect(res).toEqual({ ok: true, compacted: false, reason: "cron session graph disabled" }); + expect(mocks.getUnextracted).not.toHaveBeenCalled(); + }); + + it("enabled=true 时 cron session 正常入库", async () => { + const { engine } = registerPlugin({ cron: { enabled: true } }); + + await engine.ingest({ sessionId: CRON_SID, sessionKey: CRON_KEY, message: { role: "user", content: "hi" } }); + + expect(mocks.saveMessage).toHaveBeenCalledTimes(1); + }); + + it("extract=false 时 cron session 消息仍入库缓冲但不触发提取", async () => { + const { engine } = registerPlugin({ cron: { enabled: true, extract: false } }); + + await engine.afterTurn({ + sessionId: CRON_SID, + sessionKey: CRON_KEY, + messages: [{ role: "user", content: "hi" }], + prePromptMessageCount: 0, + }); + + expect(mocks.saveMessage).toHaveBeenCalledTimes(1); + expect(mocks.isTurnExtracted).not.toHaveBeenCalled(); + }); + + it("extract=true 时 cron session 触发提取", async () => { + const { engine } = registerPlugin({ cron: { enabled: true, extract: true } }); + + await engine.afterTurn({ + sessionId: CRON_SID, + sessionKey: CRON_KEY, + messages: [{ role: "user", content: "hi" }], + prePromptMessageCount: 0, + }); + // afterTurn 内的 extractTurnKnowledge 是 fire-and-forget，flush 微任务后再断言 + await new Promise(resolve => setTimeout(resolve, 0)); + + expect(mocks.isTurnExtracted).toHaveBeenCalledTimes(1); + }); + + it("extract=false 时 cron session compact 直接跳过提取", async () => { + const { engine } = registerPlugin({ cron: { enabled: true, extract: false } }); + + const res = await engine.compact({ sessionId: CRON_SID, sessionKey: CRON_KEY }); + + expect(res).toEqual({ ok: true, compacted: false, reason: "cron session extraction disabled" }); + expect(mocks.getUnextracted).not.toHaveBeenCalled(); + }); + + it("cron 任务设置自定义 sessionKey（无 cron 段）时按普通会话处理", async () => { + const { engine } = registerPlugin({ cron: { enabled: false } }); + + await expect(engine.ingest({ sessionId: CRON_SID, sessionKey: "agent:my-custom-key", message: { role: "user", content: "hi" } })) + .resolves.toEqual({ ingested: true }); + + expect(mocks.saveMessage).toHaveBeenCalledTimes(1); + }); +}); From 02c4bb21168e2dc8b55de273de3ae23ad3e73199 Mon Sep 17 00:00:00 2001 From: TriDefender Date: Sat, 15 Aug 2026 21:39:14 +0000 Subject: [PATCH 08/29] Fix: fixed incoherent tests --- README.md | 10 +++++----- README_CN.md | 2 +- test/session-identity.test.ts | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 5ecb143..cd95ab5 100644 --- a/README.md +++ b/README.md @@ -145,13 +145,13 @@ Sessions created by OpenClaw scheduled tasks can be configured independently of | Option | Default | Description | | --- | --- | --- | -| `enabled` | `false` | Enable graph functionality inside cron sessions (recall injection + message buffering). When `false`, cron sessions skip automatic recall and message persistence; the `gm_*` tools remain available for explicit calls (manual escape hatch). | -| `extract` | `false` | Trigger knowledge extraction (LLM triples) in cron sessions via `afterTurn` / `compact`. When `false`, messages are still buffered and can be backfilled later with `openclaw graph-memory extract`. | -| `finalizeAndMaintain` | `false` | Run finalize (EVENT→SKILL promotion) and graph maintenance (decay / PageRank / communities) when a cron session ends. Disable when frequent cron runs make end-of-session global maintenance too costly. | +| `enabled` | `true` | Enable graph functionality inside cron sessions (recall injection + message buffering). When `false`, cron sessions skip automatic recall and message persistence; the `gm_*` tools remain available for explicit calls (manual escape hatch). | +| `extract` | `true` | Trigger knowledge extraction (LLM triples) in cron sessions via `afterTurn` / `compact`. When `false`, messages are still buffered and can be backfilled later with `openclaw graph-memory extract`. | +| `finalizeAndMaintain` | `true` | Run finalize (EVENT→SKILL promotion) and graph maintenance (decay / PageRank / communities) when a cron session ends. Disable when frequent cron runs make end-of-session global maintenance too costly. | -All three options default to **`false`**: cron sessions skip the graph entirely (no recall, no buffering, no extraction, no maintenance) unless explicitly enabled. `enabled: false` is the master switch — even with `extract` / `finalizeAndMaintain` set to `true`, nothing runs. Non-cron sessions are never affected by these options. +All three options default to **`true`**: cron sessions behave like normal sessions (recall, buffering, extraction, and end-of-session maintenance all enabled) unless explicitly disabled. `enabled: false` is the master switch — even with `extract` / `finalizeAndMaintain` set to `true`, nothing runs. Non-cron sessions are never affected by these options. -All three sub-options are optional; omitted fields keep the default `false` (e.g. with `"cron": { "enabled": true }` only recall and message buffering are enabled — extraction and end-of-session maintenance stay off). +All three sub-options are optional; omitted fields keep the default `true` (e.g. with `"cron": { "extract": false }` only extraction is disabled — recall, buffering, and end-of-session maintenance stay on). Caveat: when a cron job sets an explicit custom `sessionKey`, the host does not append the `cron` segment — such sessions cannot be detected and are treated as normal sessions. diff --git a/README_CN.md b/README_CN.md index 2851ca7..a67305b 100644 --- a/README_CN.md +++ b/README_CN.md @@ -99,7 +99,7 @@ OpenClaw 定时任务创建的会话可以独立配置图谱行为。host 把 cr | `extract` | `true` | 是否在 cron 会话内触发知识提取（afterTurn / compact 的 LLM 三元组提取）。关闭后消息仍入库缓冲，之后可用 `openclaw graph-memory extract` 手动回填。 | | `finalizeAndMaintain` | `true` | cron 会话结束时是否执行 finalize（EVENT→SKILL 晋升）和图维护（decay / PageRank / 社区检测）。定时任务频繁时可关闭，避免每次会话结束都跑全局维护。 | -三个选项**默认全部开启**：cron 会话默认使用图谱，需按需显式关闭。`enabled=true` 是总开关：即使 `extract`/`finalizeAndMaintain` 设为 `false` 也不生效。非 cron 会话不受这些选项影响。 +三个选项**默认全部开启**：cron 会话默认使用图谱，需按需显式关闭。`enabled=false` 是总开关：即使 `extract`/`finalizeAndMaintain` 设为 `true` 也不生效。非 cron 会话不受这些选项影响。三个子项均可省略，未写的字段取默认值 `true`。 注意：若 cron 任务显式设置了自定义 `sessionKey`，host 不再附加 `cron` 段，此类会话无法被识别，将按普通会话处理。 diff --git a/test/session-identity.test.ts b/test/session-identity.test.ts index 1078531..8b069bf 100644 --- a/test/session-identity.test.ts +++ b/test/session-identity.test.ts @@ -217,22 +217,22 @@ describe("cron session gating (cron 配置)", () => { expect(isCronSessionKey(null)).toBe(false); }); - it("默认配置（全 false）下 cron session_end 跳过 finalize 与图维护", async () => { + it("默认配置（全 true）下 cron session_end 仍执行 finalize 与图维护（向后兼容）", async () => { const handler = registerPlugin().hooks.get("session_end"); if (!handler) throw new Error("session_end hook was not registered"); await handler({ sessionId: CRON_SID, sessionKey: CRON_KEY }, {}); - expect(mocks.getBySession).not.toHaveBeenCalled(); - expect(mocks.runMaintenance).not.toHaveBeenCalled(); + expect(mocks.getBySession).toHaveBeenCalledWith({}, CRON_SID); + expect(mocks.runMaintenance).toHaveBeenCalledTimes(1); }); - it("默认配置下 cron session 不入库（默认关闭）", async () => { + it("默认配置下 cron session 正常入库", async () => { const { engine } = registerPlugin(); await expect(engine.ingest({ sessionId: CRON_SID, sessionKey: CRON_KEY, message: { role: "user", content: "hi" } })) - .resolves.toEqual({ ingested: false }); - expect(mocks.saveMessage).not.toHaveBeenCalled(); + .resolves.toEqual({ ingested: true }); + expect(mocks.saveMessage).toHaveBeenCalledTimes(1); }); it("finalizeAndMaintain=false 时 cron session 跳过 finalize 与图维护", async () => { From d1a760432f479956b9af4ac79773c5afad9d4f9d Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:44:58 +0000 Subject: [PATCH 09/29] =?UTF-8?q?Fix:=20=E4=BF=AE=E4=BA=86=E4=B8=80?= =?UTF-8?q?=E4=BA=9B=E9=98=BB=E5=A1=9E=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 修掉了原来多会话并发结束会并发跑两次全局维护的竞态 - 防止插件启动时检测不到embeddings就整个生命周期回退到FTS --- index.ts | 278 ++++++++++++++++++++++++++++------ src/recaller/recall.ts | 6 + src/store/db.ts | 10 +- src/store/gate.ts | 60 ++++++++ test/neo4j-gate.test.ts | 68 +++++++++ test/session-identity.test.ts | 2 + 6 files changed, 370 insertions(+), 54 deletions(-) create mode 100644 src/store/gate.ts create mode 100644 test/neo4j-gate.test.ts diff --git a/index.ts b/index.ts index 9973e7b..6c05142 100755 --- a/index.ts +++ b/index.ts @@ -8,6 +8,7 @@ import type { OpenClawPluginApi } from "openclaw/plugin-sdk"; import { Type } from "@sinclair/typebox"; import { getDriver, initSchema, getSession } from "./src/store/db.ts"; +import { Neo4jGate } from "./src/store/gate.ts"; import { saveMessage, getUnextracted, getMaxTurnIndex, markExtracted, isTurnExtracted, @@ -328,10 +329,16 @@ const graphMemoryProPlugin = { // ── 初始化 Neo4j ──────────────────────────────────────── const driver = getDriver(cfg.neo4j); + // Neo4j 熔断门控：掉线时快速降级（跳图谱注入 / 缓冲消息），避免每轮吃满 driver 超时 + const neo4jGate = new Neo4jGate(); + // Schema 初始化（异步，不阻塞启动） initSchema(driver, cfg.embedding) .then(() => api.logger.info("[graph-memory-pro] Neo4j schema initialized")) - .catch(err => api.logger.error(`[graph-memory-pro] schema init failed: ${err}`)); + .catch(err => { + neo4jGate.recordFailure(); + api.logger.error(`[graph-memory-pro] schema init failed: ${err}`); + }); const llm = createCompleteFn(effectiveModel, cfg.llm); const recaller = new Recaller(driver, cfg); @@ -372,6 +379,11 @@ const graphMemoryProPlugin = { } async function extractTurnKnowledge(sessionId: string, turnNum: number, rawMessages: any[]): Promise { + // 熔断开启时跳过本轮提取：消息保持未标记，恢复后由 compact / extract 补提取 + if (!neo4jGate.isAvailable()) { + api.logger.info(`[graph-memory-pro] turn ${turnNum}: extraction skipped (neo4j circuit open)`); + return; + } return withExtractLock(sessionId, async () => { try { if (await isTurnExtracted(driver, sessionId, turnNum)) { @@ -465,6 +477,155 @@ const graphMemoryProPlugin = { await saveMessage(driver, sessionId, seq, message.role ?? "unknown", message); } + // ── 消息持久化：门控 + 内存缓冲（Neo4j 掉线时兜底） ──── + + interface BufferedMessage { sessionId: string; seq: number; role: string; message: any } + const messageBuffer: BufferedMessage[] = []; + const MESSAGE_BUFFER_CAP = 2000; + let bufferFlushInFlight = false; + + /** + * 缓冲一条消息：seq 在缓冲时预分配（不走 getMaxTurnIndex —— 那也是 DB 查询）， + * 保证恢复后落库顺序与到达顺序一致。 + */ + function bufferMessage(sessionId: string, message: any): void { + const seq = (msgSeq.get(sessionId) ?? 0) + 1; + msgSeq.set(sessionId, seq); + if (messageBuffer.length >= MESSAGE_BUFFER_CAP) { + messageBuffer.shift(); + api.logger.warn("[graph-memory-pro] message buffer full, dropping oldest buffered message"); + } + messageBuffer.push({ sessionId, seq, role: message?.role ?? "unknown", message }); + } + + /** 恢复后把缓冲消息刷回 Neo4j（single-flight；失败保留余量稍后重试）。 */ + async function flushMessageBuffer(): Promise { + if (bufferFlushInFlight || !messageBuffer.length || !neo4jGate.isAvailable()) return; + bufferFlushInFlight = true; + let flushed = 0; + try { + while (messageBuffer.length) { + const next = messageBuffer[0]; + try { + await saveMessage(driver, next.sessionId, next.seq, next.role, next.message); + messageBuffer.shift(); + flushed += 1; + } catch (err) { + neo4jGate.recordFailure(); + api.logger.warn(`[graph-memory-pro] buffered message flush failed, will retry later: ${err}`); + break; + } + } + if (flushed > 0) { + api.logger.info(`[graph-memory-pro] flushed ${flushed} buffered message(s) to neo4j`); + } + } finally { + bufferFlushInFlight = false; + } + } + + /** + * ingest / afterTurn 共用的落库入口： + * 可用 → 直接写；不可用或写失败 → 缓冲并吞掉错误（不向 host 抛）， + * 恢复后由 flushMessageBuffer 补写。返回的 ingested=true 语义为"引擎已接管该消息"。 + */ + async function persistMessage(sessionId: string, message: any): Promise { + if (!neo4jGate.isAvailable()) { + bufferMessage(sessionId, message); + return; + } + try { + await ingestMessage(sessionId, message); + neo4jGate.recordSuccess(); + void flushMessageBuffer(); + } catch (err) { + neo4jGate.recordFailure(); + bufferMessage(sessionId, message); + api.logger.warn(`[graph-memory-pro] neo4j write failed, message buffered (${messageBuffer.length} pending): ${err}`); + } + } + + // ── recall 超时预算：慢查询不拖回合，回退缓存/降级 ────── + + const RECALL_BUDGET_MS = 5_000; + + /** + * 给 Promise 加等待上限。不取消底层操作（Neo4j 查询会在后台自然完成、 + * 连接归还连接池），只是放弃等待 —— 慢 != 死。 + */ + function withBudget(p: Promise, ms: number, label: string): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms); + p.then( + v => { clearTimeout(timer); resolve(v); }, + e => { clearTimeout(timer); reject(e); }, + ); + }); + } + + // ── 图维护：后台单飞 + trailing rerun（A1） ───────────── + // session_end 不再 await 维护链（衰减→去重→PR→社区→LLM 摘要可能耗时数分钟）； + // 全局单飞修掉多会话并发跑维护的竞态；运行期间的再次请求只标记 rerun， + // 当前一轮结束后最多补跑一次（覆盖"最后一个结束的会话"）。 + + let maintenanceRun: Promise | null = null; + let maintenanceRerunRequested = false; + + function scheduleMaintenance(): void { + if (!neo4jGate.isAvailable()) { + api.logger.info("[graph-memory-pro] maintenance skipped: neo4j unavailable (circuit open)"); + return; + } + if (maintenanceRun) { + maintenanceRerunRequested = true; + api.logger.info("[graph-memory-pro] maintenance already running, rerun queued"); + return; + } + maintenanceRun = (async () => { + try { + do { + maintenanceRerunRequested = false; + const embedFn = recaller.embedFn ?? undefined; + const result = await runMaintenance(driver, cfg, llm, embedFn); + neo4jGate.recordSuccess(); + api.logger.info( + `[graph-memory-pro] maintenance: ${result.durationMs}ms, ` + + `dedup=${result.dedup.merged}, communities=${result.community.count}, ` + + `summaries=${result.communitySummaries}, ` + + `top_pr=${result.pagerank.topK.slice(0, 3).map(n => `${n.name}(${n.score.toFixed(3)})`).join(",")}`, + ); + } while (maintenanceRerunRequested && neo4jGate.isAvailable()); + } catch (err) { + neo4jGate.recordFailure(); + api.logger.error(`[graph-memory-pro] maintenance failed: ${err}`); + } finally { + maintenanceRun = null; + maintenanceRerunRequested = false; + } + })(); + } + + // ── embedding 会话级 re-probe ────────────────────────── + // 启动 probe 失败会让插件停在文本搜索模式直到重启；这里在每个会话开始时 + // 重试一次（single-flight），临时性故障恢复后自动回到向量召回。 + + const embeddingConfigured = !!(cfg.embedding && (cfg.embedding.apiKey || cfg.embedding.baseURL)); + let embedProbeInFlight = false; + + function ensureEmbeddingReady(): void { + if (!embeddingConfigured || recaller.hasEmbedFn() || embedProbeInFlight) return; + embedProbeInFlight = true; + createEmbedFn(cfg.embedding) + .then(fn => { + if (fn) { + recaller.setEmbedFn(fn); + api.logger.info("[graph-memory-pro] embedding re-probe succeeded — vector search re-enabled"); + } + }) + .catch(() => {}) + .finally(() => { embedProbeInFlight = false; }); + } + // ── before_agent_start：召回 ──────────────────────────── api.on("before_agent_start", async (event: any, ctx: any) => { @@ -476,10 +637,12 @@ const graphMemoryProPlugin = { const prompt = cleanPrompt(rawPrompt); if (!prompt) return; if (prompt.includes("/new or /reset") || prompt.includes("new session was started")) return; + // 熔断开启时跳过召回 —— assemble 也会走降级路径（仅转录文本） + if (!neo4jGate.isAvailable()) return; api.logger.info(`[graph-memory-pro] recall query: "${prompt.slice(0, 80)}"`); - const res = await recaller.recall(prompt); + const res = await withBudget(recaller.recall(prompt), RECALL_BUDGET_MS, "[graph-memory-pro] recall"); if (res.nodes.length) { const sessionId = typeof ctx?.sessionId === "string" ? ctx.sessionId : undefined; const sessionKey = typeof ctx?.sessionKey === "string" ? ctx.sessionKey : undefined; @@ -505,6 +668,8 @@ const graphMemoryProPlugin = { async bootstrap({ sessionId, sessionKey }: { sessionId: string; sessionKey?: string }) { bindSessionIdentity(sessionId, sessionKey); + // 每个会话开始时尝试恢复 embedding（启动 probe 失败后的会话级 re-probe） + ensureEmbeddingReady(); return { bootstrapped: true }; }, @@ -515,7 +680,7 @@ const graphMemoryProPlugin = { if (isCronSessionKey(sessionKey) && !cronCfg.enabled) { return { ingested: false }; } - await ingestMessage(sessionId, message); + await persistMessage(sessionId, message); ingestedSinceTurn.set(sessionId, (ingestedSinceTurn.get(sessionId) ?? 0) + 1); return { ingested: true }; }, @@ -541,20 +706,13 @@ const graphMemoryProPlugin = { }; } - const activeNodes = await getBySession(driver, sessionId); - const activeEdges: any[] = []; - for (const n of activeNodes) { - activeEdges.push(...await edgesFrom(driver, n.id)); - activeEdges.push(...await edgesTo(driver, n.id)); - } - // prompt-aware recall：优先用当前 prompt 做新鲜召回，回退到 before_agent_start 缓存 let rec = recalled.get(sessionId) ?? { nodes: [], edges: [] }; - if (prompt) { + if (prompt && neo4jGate.isAvailable()) { const cleaned = cleanPrompt(prompt); if (cleaned) { try { - const freshRec = await recaller.recall(cleaned); + const freshRec = await withBudget(recaller.recall(cleaned), RECALL_BUDGET_MS, "[graph-memory-pro] assemble recall"); if (freshRec.nodes.length) { rec = freshRec; recalled.set(sessionId, freshRec); @@ -564,45 +722,52 @@ const graphMemoryProPlugin = { } } } - const totalGmNodes = activeNodes.length + rec.nodes.length; const prepared = prepareAssemblyMessages(messages); - if (totalGmNodes === 0) { - if (prepared.dropped > 0) { - api.logger.info( - `[graph-memory-pro] assemble: ${prepared.messages.length} msgs (~${prepared.tokens} tok), ` + - `dropped ${prepared.dropped} older msgs, graph ~0 tok`, - ); + // 图谱段：门控 + 降级 —— Neo4j 掉线/超时时只返回裁剪后的转录， + // 不让错误抛回 host（原实现无 catch，getBySession 失败会炸掉 assemble） + let graphTokens = 0; + let systemPromptAddition: string | undefined; + if (neo4jGate.isAvailable()) { + try { + const activeNodes = await getBySession(driver, sessionId); + const activeEdges: any[] = []; + for (const n of activeNodes) { + activeEdges.push(...await edgesFrom(driver, n.id)); + activeEdges.push(...await edgesTo(driver, n.id)); + } + + if (activeNodes.length + rec.nodes.length > 0) { + const { xml, systemPrompt, tokens } = await assembleContext(driver, { + tokenBudget: budget, + activeNodes, + activeEdges, + recalledNodes: rec.nodes, + recalledEdges: rec.edges, + }); + graphTokens = tokens; + if (xml) { + systemPromptAddition = systemPrompt ? `${systemPrompt}\n\n${xml}` : xml; + } + } + neo4jGate.recordSuccess(); + void flushMessageBuffer(); + } catch (err) { + neo4jGate.recordFailure(); + api.logger.warn(`[graph-memory-pro] assemble: graph context unavailable, transcript-only: ${err}`); } - return { - messages: prepared.messages, - estimatedTokens: prepared.tokens, - }; } - const { xml, systemPrompt, tokens: gmTokens } = await assembleContext(driver, { - tokenBudget: budget, - activeNodes, - activeEdges, - recalledNodes: rec.nodes, - recalledEdges: rec.edges, - }); - if (prepared.dropped > 0) { api.logger.info( `[graph-memory-pro] assemble: ${prepared.messages.length} msgs (~${prepared.tokens} tok), ` + - `dropped ${prepared.dropped} older msgs, graph ~${gmTokens} tok`, + `dropped ${prepared.dropped} older msgs, graph ~${graphTokens} tok`, ); } - let systemPromptAddition: string | undefined; - if (xml) { - systemPromptAddition = systemPrompt ? `${systemPrompt}\n\n${xml}` : xml; - } - return { messages: prepared.messages, - estimatedTokens: gmTokens + prepared.tokens, + estimatedTokens: graphTokens + prepared.tokens, ...(systemPromptAddition ? { systemPromptAddition } : {}), }; }, @@ -616,6 +781,10 @@ const graphMemoryProPlugin = { reason: cronCfg.enabled ? "cron session extraction disabled" : "cron session graph disabled", }; } + // 熔断开启时跳过提取：未提取消息保留，恢复后下一次 compact / extract 补上 + if (!neo4jGate.isAvailable()) { + return { ok: true, compacted: false, reason: "neo4j unavailable (circuit open)" }; + } return withExtractLock(sessionId, async () => { const msgs = await getUnextracted(driver, sessionId, cfg.compactTurnCount * 3); @@ -692,7 +861,7 @@ const graphMemoryProPlugin = { const ingestedCount = ingestedSinceTurn.get(sessionId) ?? 0; const missingMessages = missingIngestMessages(newMessages, ingestedCount); for (const message of missingMessages) { - await ingestMessage(sessionId, message); + await persistMessage(sessionId, message); } if (missingMessages.length > 0) { api.logger.warn( @@ -771,7 +940,22 @@ const graphMemoryProPlugin = { return; } - const nodes = await getBySession(driver, sid); + // 熔断开启时跳过 finalize（全是 Neo4j 写）—— 消息已缓冲，恢复后补齐 + if (!neo4jGate.isAvailable()) { + api.logger.warn(`[graph-memory-pro] session_end ${sid.slice(0, 12)}…: neo4j unavailable (circuit open), finalize + maintenance skipped`); + return; + } + + let nodes: Awaited>; + try { + nodes = await getBySession(driver, sid); + neo4jGate.recordSuccess(); + void flushMessageBuffer(); + } catch (err) { + neo4jGate.recordFailure(); + api.logger.error(`[graph-memory-pro] session_end error: ${err}`); + return; + } if (nodes.length) { // 获取图谱摘要 const session = getSession(driver); @@ -812,15 +996,9 @@ const graphMemoryProPlugin = { for (const id of fin.invalidations) await deprecate(driver, id); } - // 图维护 - const embedFn = (recaller as any).embed ?? undefined; - const result = await runMaintenance(driver, cfg, llm, embedFn); - api.logger.info( - `[graph-memory-pro] maintenance: ${result.durationMs}ms, ` + - `dedup=${result.dedup.merged}, communities=${result.community.count}, ` + - `summaries=${result.communitySummaries}, ` + - `top_pr=${result.pagerank.topK.slice(0, 3).map(n => `${n.name}(${n.score.toFixed(3)})`).join(",")}`, - ); + // 图维护：后台单飞（A1）—— 衰减→去重→PR→社区→LLM 摘要可能耗时数分钟， + // 不再阻塞 session_end；结果只进日志，host 对其零依赖 + scheduleMaintenance(); } catch (err) { api.logger.error(`[graph-memory-pro] session_end error: ${err}`); } finally { @@ -1244,7 +1422,7 @@ const graphMemoryProPlugin = { description: "手动触发图维护：衰减评分 + tier 转换、去重、PageRank、社区检测。", parameters: Type.Object({}), async execute() { - const embedFn = (recaller as any).embed ?? undefined; + const embedFn = recaller.embedFn ?? undefined; const result = await runMaintenance(driver, cfg, llm, embedFn); const t = result.decay.tierTransitions; const totalTransitions = t.coreToWorking + t.workingToPeripheral + t.peripheralToWorking + t.workingToCore; diff --git a/src/recaller/recall.ts b/src/recaller/recall.ts index 87ec456..e7fa6f3 100755 --- a/src/recaller/recall.ts +++ b/src/recaller/recall.ts @@ -88,6 +88,12 @@ export class Recaller { setEmbedFn(fn: EmbedFn): void { this.embed = fn; } + /** 是否已接入 embedding（启动 probe 成功或会话级 re-probe 成功）。 */ + hasEmbedFn(): boolean { return this.embed !== null; } + + /** 只读暴露 embedFn（maintenance / gm_maintain 需要），替代 (recaller as any).embed。 */ + get embedFn(): EmbedFn | null { return this.embed; } + async recall(query: string, options?: RecallOptions): Promise { const limit = this.cfg.recallMaxNodes; const timeRange = options ? parseTimeRange(options) : null; diff --git a/src/store/db.ts b/src/store/db.ts index d5d7315..01631f8 100755 --- a/src/store/db.ts +++ b/src/store/db.ts @@ -21,8 +21,9 @@ export function getDriver(cfg: Neo4jConfig): Driver { if (_driver) return _driver; _driver = neo4j.driver(cfg.uri, neo4j.auth.basic(cfg.user, cfg.password), { maxConnectionPoolSize: 50, - connectionAcquisitionTimeout: 60000, - maxTransactionRetryTime: 30000, + // 快速失败配合 gate.ts 熔断：掉线时 ~10s 内报错跳闸，而不是每次卡 30-60s + connectionAcquisitionTimeout: 15_000, + maxTransactionRetryTime: 10_000, }); return _driver; } @@ -40,8 +41,9 @@ export function getSession(driver: Driver): Session { console.log("[graph-memory-pro] reconnecting Neo4j driver..."); _driver = neo4j.driver(_cfg.uri, neo4j.auth.basic(_cfg.user, _cfg.password), { maxConnectionPoolSize: 50, - connectionAcquisitionTimeout: 60000, - maxTransactionRetryTime: 30000, + // 与 getDriver 保持一致：快速失败，让熔断门控尽早接手 + connectionAcquisitionTimeout: 15_000, + maxTransactionRetryTime: 10_000, }); return _driver.session({ database: "neo4j" }); } diff --git a/src/store/gate.ts b/src/store/gate.ts new file mode 100644 index 0000000..7a8e9c3 --- /dev/null +++ b/src/store/gate.ts @@ -0,0 +1,60 @@ +/** + * graph-memory-pro — Neo4j 熔断门控（circuit breaker） + * + * 解决的问题：Neo4j 掉线时，每次 ingest / assemble / recall 都要吃满 + * driver 的重试与连接获取超时（最坏数十秒），对话每轮都被拖住， + * 体验上等同于"卡死"。 + * + * 语义： + * - closed（正常）：isAvailable() === true，所有操作放行。 + * - open（跳闸）：连续失败 >= failureThreshold 次后进入；冷却期内 + * isAvailable() === false，调用方应立即降级（跳过图谱、缓冲消息）， + * 而不是等 driver 超时。 + * - half-open（半开探测）：冷却期结束后 isAvailable() 恢复 true， + * 下一个真实操作充当探测 —— 成功则复位 closed，失败则重新计时冷却。 + * + * 注意：失败计数只应从"纯 Neo4j 调用点"记录（saveMessage / getBySession + * 等）。混合了 LLM / embedding 的调用点（recall、compact）不要记录， + * 否则 LLM 超时会误跳闸。 + */ + +export class Neo4jGate { + private consecutiveFailures = 0; + private open = false; + private openedAt = 0; + + constructor( + /** 连续失败多少次后跳闸 */ + private readonly failureThreshold: number = 2, + /** 跳闸后的冷却时长（ms），到期进入半开 */ + private readonly cooldownMs: number = 120_000, + ) {} + + /** 操作成功：复位计数并闭合熔断。 */ + recordSuccess(): void { + this.consecutiveFailures = 0; + this.open = false; + } + + /** + * 操作失败：累计计数；达到阈值跳闸。 + * 已处于 open 时再次失败（半开探测失败 / 在途请求迟到失败）会 + * 重置冷却计时 —— 但被门控的调用方在 open 期间不会发起操作， + * 所以不会出现"永远无法恢复"的抖动。 + */ + recordFailure(): void { + this.consecutiveFailures += 1; + if (!this.open && this.consecutiveFailures >= this.failureThreshold) { + this.open = true; + } + if (this.open) { + this.openedAt = Date.now(); + } + } + + /** 当前是否放行操作（closed 或 冷却到期的 half-open）。 */ + isAvailable(): boolean { + if (!this.open) return true; + return Date.now() - this.openedAt >= this.cooldownMs; + } +} diff --git a/test/neo4j-gate.test.ts b/test/neo4j-gate.test.ts new file mode 100644 index 0000000..4a430d0 --- /dev/null +++ b/test/neo4j-gate.test.ts @@ -0,0 +1,68 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { Neo4jGate } from "../src/store/gate.ts"; + +describe("Neo4j 熔断门控 (Neo4jGate)", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("closed 状态放行所有操作，成功复位失败计数", () => { + const gate = new Neo4jGate(2, 120_000); + expect(gate.isAvailable()).toBe(true); + + gate.recordFailure(); + expect(gate.isAvailable()).toBe(true); + + gate.recordSuccess(); + gate.recordFailure(); + expect(gate.isAvailable()).toBe(true); + }); + + it("连续失败达到阈值后跳闸，冷却期内不可用", () => { + vi.useFakeTimers(); + const gate = new Neo4jGate(2, 120_000); + + gate.recordFailure(); + gate.recordFailure(); + expect(gate.isAvailable()).toBe(false); + + vi.advanceTimersByTime(119_999); + expect(gate.isAvailable()).toBe(false); + + vi.advanceTimersByTime(1); + expect(gate.isAvailable()).toBe(true); + }); + + it("半开后一次成功即完全恢复（计数归零）", () => { + vi.useFakeTimers(); + const gate = new Neo4jGate(2, 120_000); + + gate.recordFailure(); + gate.recordFailure(); + vi.advanceTimersByTime(120_000); + expect(gate.isAvailable()).toBe(true); + + gate.recordSuccess(); + gate.recordFailure(); + expect(gate.isAvailable()).toBe(true); + }); + + it("半开探测失败重新进入冷却", () => { + vi.useFakeTimers(); + const gate = new Neo4jGate(2, 120_000); + + gate.recordFailure(); + gate.recordFailure(); + vi.advanceTimersByTime(120_000); + + gate.recordFailure(); + expect(gate.isAvailable()).toBe(false); + + vi.advanceTimersByTime(119_999); + expect(gate.isAvailable()).toBe(false); + + vi.advanceTimersByTime(1); + expect(gate.isAvailable()).toBe(true); + }); +}); diff --git a/test/session-identity.test.ts b/test/session-identity.test.ts index 8b069bf..c2ba55e 100644 --- a/test/session-identity.test.ts +++ b/test/session-identity.test.ts @@ -58,6 +58,8 @@ vi.mock("../src/engine/embed.ts", () => ({ vi.mock("../src/recaller/recall.ts", () => ({ Recaller: class { setEmbedFn(): void {} + hasEmbedFn(): boolean { return false; } + get embedFn() { return null; } async recall() { return mocks.recall(); } async syncEmbed(): Promise {} }, From 5101f7cddfc8a46c3d68ac0cddfc0bf5c132f546 Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:03:11 +0000 Subject: [PATCH 10/29] =?UTF-8?q?Fix:=20=E8=BF=9B=E8=A1=8C=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E5=AE=A1=E8=AE=A1=E5=90=8E=E6=8D=89=E4=BA=86=E5=87=A0?= =?UTF-8?q?=E4=B8=AAbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- index.ts | 90 ++++++++++++++++++++++------------- test/session-identity.test.ts | 82 ++++++++++++++++++++++++++++++- 2 files changed, 138 insertions(+), 34 deletions(-) diff --git a/index.ts b/index.ts index 6c05142..26f6fd2 100755 --- a/index.ts +++ b/index.ts @@ -345,16 +345,24 @@ const graphMemoryProPlugin = { const extractor = new Extractor(llm); // ── 初始化 embedding ──────────────────────────────────── + // re-probe 状态提前声明：启动 probe 失败时记录时间戳，bootstrap 的 + // 会话级 re-probe 据此退避（端点宕机时不逐会话重试刷日志/打 API） + const embeddingConfigured = !!(cfg.embedding && (cfg.embedding.apiKey || cfg.embedding.baseURL)); + let embedProbeInFlight = false; + let lastEmbedProbeAt = 0; + createEmbedFn(cfg.embedding) .then((fn) => { if (fn) { recaller.setEmbedFn(fn); api.logger.info("[graph-memory-pro] vector search ready"); } else { + lastEmbedProbeAt = Date.now(); api.logger.info("[graph-memory-pro] text search mode (配置 embedding 可启用语义搜索)"); } }) .catch(() => { + lastEmbedProbeAt = Date.now(); api.logger.info("[graph-memory-pro] text search mode"); }); @@ -386,6 +394,9 @@ const graphMemoryProPlugin = { } return withExtractLock(sessionId, async () => { try { + // 先等掉线期间缓冲的消息落库，再判断/标记 extracted——否则行落库晚于 + // markExtracted 时会以 extracted=false 重现，被下一轮 compact 重复提取 + if (messageBuffer.length) await flushMessageBuffer(); if (await isTurnExtracted(driver, sessionId, turnNum)) { api.logger.info(`[graph-memory-pro] turn ${turnNum}: already extracted (compact), skipping`); return; @@ -479,49 +490,61 @@ const graphMemoryProPlugin = { // ── 消息持久化：门控 + 内存缓冲（Neo4j 掉线时兜底） ──── - interface BufferedMessage { sessionId: string; seq: number; role: string; message: any } + interface BufferedMessage { sessionId: string; message: any } const messageBuffer: BufferedMessage[] = []; const MESSAGE_BUFFER_CAP = 2000; - let bufferFlushInFlight = false; + let flushRun: Promise | null = null; /** - * 缓冲一条消息：seq 在缓冲时预分配（不走 getMaxTurnIndex —— 那也是 DB 查询）， - * 保证恢复后落库顺序与到达顺序一致。 + * 缓冲一条消息。不在缓冲时分配 seq：内存 msgSeq 在 session_end 清理 / + * DB 故障时与 DB 脱节，预分配的 seq 会与已有行撞号，saveMessage 的 + * ON MATCH SET 会静默覆盖旧行内容。seq 统一在 flush 时由 ingestMessage + * 分配（那时 DB 可达，getMaxTurnIndex 恢复能正确兜底）。 */ function bufferMessage(sessionId: string, message: any): void { - const seq = (msgSeq.get(sessionId) ?? 0) + 1; - msgSeq.set(sessionId, seq); + // 不可序列化的消息（循环引用 / BigInt）永远写不进 DB——当场丢弃， + // 否则它会永久堵在 flush 队列头并反复重跳熔断 + try { JSON.stringify(message); } catch (err) { + api.logger.warn(`[graph-memory-pro] message not serializable, dropped from outage buffer: ${err}`); + return; + } if (messageBuffer.length >= MESSAGE_BUFFER_CAP) { messageBuffer.shift(); api.logger.warn("[graph-memory-pro] message buffer full, dropping oldest buffered message"); } - messageBuffer.push({ sessionId, seq, role: message?.role ?? "unknown", message }); + messageBuffer.push({ sessionId, message }); } - /** 恢复后把缓冲消息刷回 Neo4j（single-flight；失败保留余量稍后重试）。 */ - async function flushMessageBuffer(): Promise { - if (bufferFlushInFlight || !messageBuffer.length || !neo4jGate.isAvailable()) return; - bufferFlushInFlight = true; - let flushed = 0; - try { - while (messageBuffer.length) { - const next = messageBuffer[0]; - try { - await saveMessage(driver, next.sessionId, next.seq, next.role, next.message); - messageBuffer.shift(); - flushed += 1; - } catch (err) { - neo4jGate.recordFailure(); - api.logger.warn(`[graph-memory-pro] buffered message flush failed, will retry later: ${err}`); - break; + /** + * 恢复后把缓冲消息刷回 Neo4j。single-flight：返回同一个 in-flight + * promise，让 extract / compact 路径能真正等它完成再继续。 + */ + function flushMessageBuffer(): Promise { + if (flushRun) return flushRun; + if (!messageBuffer.length || !neo4jGate.isAvailable()) return Promise.resolve(); + flushRun = (async () => { + let flushed = 0; + try { + while (messageBuffer.length) { + const next = messageBuffer[0]; + try { + await ingestMessage(next.sessionId, next.message); + messageBuffer.shift(); + flushed += 1; + } catch (err) { + neo4jGate.recordFailure(); + api.logger.warn(`[graph-memory-pro] buffered message flush failed, will retry later: ${err}`); + break; + } } + if (flushed > 0) { + api.logger.info(`[graph-memory-pro] flushed ${flushed} buffered message(s) to neo4j`); + } + } finally { + flushRun = null; } - if (flushed > 0) { - api.logger.info(`[graph-memory-pro] flushed ${flushed} buffered message(s) to neo4j`); - } - } finally { - bufferFlushInFlight = false; - } + })(); + return flushRun; } /** @@ -607,14 +630,15 @@ const graphMemoryProPlugin = { // ── embedding 会话级 re-probe ────────────────────────── // 启动 probe 失败会让插件停在文本搜索模式直到重启；这里在每个会话开始时 - // 重试一次（single-flight），临时性故障恢复后自动回到向量召回。 + // 重试（single-flight + 5 分钟退避），临时性故障恢复后自动回到向量召回。 - const embeddingConfigured = !!(cfg.embedding && (cfg.embedding.apiKey || cfg.embedding.baseURL)); - let embedProbeInFlight = false; + const EMBED_REPROBE_INTERVAL_MS = 300_000; function ensureEmbeddingReady(): void { if (!embeddingConfigured || recaller.hasEmbedFn() || embedProbeInFlight) return; + if (Date.now() - lastEmbedProbeAt < EMBED_REPROBE_INTERVAL_MS) return; embedProbeInFlight = true; + lastEmbedProbeAt = Date.now(); createEmbedFn(cfg.embedding) .then(fn => { if (fn) { @@ -786,6 +810,8 @@ const graphMemoryProPlugin = { return { ok: true, compacted: false, reason: "neo4j unavailable (circuit open)" }; } return withExtractLock(sessionId, async () => { + // compact 是掉线恢复后的补提取路径：先把缓冲消息刷进 DB 再读未提取集 + if (messageBuffer.length) await flushMessageBuffer(); const msgs = await getUnextracted(driver, sessionId, cfg.compactTurnCount * 3); if (!msgs.length) return { ok: true, compacted: false, reason: "no messages" }; diff --git a/test/session-identity.test.ts b/test/session-identity.test.ts index c2ba55e..3253762 100644 --- a/test/session-identity.test.ts +++ b/test/session-identity.test.ts @@ -2,8 +2,10 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { isCronSessionKey } from "../src/types.ts"; const mocks = vi.hoisted(() => ({ - getBySession: vi.fn(async () => []), - saveMessage: vi.fn(async () => {}), + getBySession: vi.fn(async () => [] as unknown[]), + saveMessage: vi.fn(async ( + _driver: unknown, _sid: string, _turn: number, _role: string, _content: unknown, + ): Promise => {}), getMaxTurnIndex: vi.fn(async () => 0), getUnextracted: vi.fn(async () => []), isTurnExtracted: vi.fn(async () => false), @@ -354,3 +356,79 @@ describe("cron session gating (cron 配置)", () => { expect(mocks.saveMessage).toHaveBeenCalledTimes(1); }); }); + +describe("outage buffering (Neo4j 掉线缓冲)", () => { + beforeEach(() => { + // mockReset：清掉上一个测试可能设置的 mockRejectedValue 等持久实现， + // 否则持续拒绝会泄漏到后续测试，把所有消息都打进缓冲 + mocks.saveMessage.mockReset().mockImplementation(async () => {}); + mocks.saveMessage.mockClear(); + mocks.getUnextracted.mockClear(); + mocks.getMaxTurnIndex.mockClear(); + }); + + it("写失败时消息被缓冲且不向 host 抛错", async () => { + const { engine } = registerPlugin(); + mocks.saveMessage.mockRejectedValue(new Error("neo4j down")); + + await expect( + engine.ingest({ sessionId: "outage-1", message: { role: "user", content: "hello" } }), + ).resolves.toEqual({ ingested: true }); + + expect(mocks.saveMessage).toHaveBeenCalledTimes(1); + }); + + it("恢复后缓冲消息经 ingestMessage 补写（seq 由 DB 状态分配，不撞号）", async () => { + const { engine } = registerPlugin(); + mocks.saveMessage.mockRejectedValueOnce(new Error("neo4j down")); + + // 第一条：写失败 → 缓冲（seq 1 被失败尝试消耗） + await engine.ingest({ sessionId: "outage-2", message: { role: "user", content: "first message" } }); + // 第二条：写成功 → 触发 flush → 第一条补写（分配新 seq，绕开撞号） + await engine.ingest({ sessionId: "outage-2", message: { role: "user", content: "second message" } }); + await new Promise(resolve => setTimeout(resolve, 0)); + + expect(mocks.saveMessage).toHaveBeenCalledTimes(3); + const flushed = mocks.saveMessage.mock.calls[2]; + expect(flushed[1]).toBe("outage-2"); + expect(flushed[2]).toBe(3); // seq 3 = max(已写 2) + 1，而非缓冲期的 1 + expect((flushed[4] as { content: string }).content).toContain("first message"); + }); + + it("不可序列化的消息被丢弃，不堵塞后续消息（队头防堵）", async () => { + const { engine } = registerPlugin(); + // 先让写失败一次，把毒消息逼进缓冲路径（真实 saveMessage 会在 stringify 时抛错） + mocks.saveMessage.mockRejectedValueOnce(new Error("neo4j down")); + + const poison: any = { role: "user", content: "ok" }; + poison.self = poison; // 循环引用 → JSON.stringify 抛错 + + await expect( + engine.ingest({ sessionId: "outage-3", message: poison }), + ).resolves.toEqual({ ingested: true }); + await expect( + engine.ingest({ sessionId: "outage-3", message: { role: "user", content: "after poison" } }), + ).resolves.toEqual({ ingested: true }); + await new Promise(resolve => setTimeout(resolve, 0)); + + // 第一次 saveMessage 因序列化失败抛错 → 消息进缓冲即被丢弃； + // 第二条正常直写后 flush 无积压 → 总共 2 次调用，毒消息不重试 + expect(mocks.saveMessage).toHaveBeenCalledTimes(2); + }); + + it("compact 在读取未提取消息前先刷缓冲（恢复补提取顺序）", async () => { + const { engine } = registerPlugin(); + mocks.saveMessage.mockRejectedValueOnce(new Error("neo4j down")); + + await engine.ingest({ sessionId: "outage-4", message: { role: "user", content: "buffered turn" } }); + expect(mocks.saveMessage).toHaveBeenCalledTimes(1); + + const res = await engine.compact({ sessionId: "outage-4" }); + expect(res).toEqual({ ok: true, compacted: false, reason: "no messages" }); + + // flush 先于 getUnextracted：第二条 saveMessage 是缓冲补写，然后才查未提取集 + expect(mocks.saveMessage).toHaveBeenCalledTimes(2); + expect((mocks.saveMessage.mock.calls[1][4] as { content: string }).content).toContain("buffered turn"); + expect(mocks.getUnextracted).toHaveBeenCalledTimes(1); + }); +}); From bb124b76b78b193e1dfec93f21e70546a04f5fe7 Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:17:28 +0000 Subject: [PATCH 11/29] =?UTF-8?q?Fix=20=E4=BF=AE=E5=A4=8D=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E5=AE=A1=E8=AE=A1=E5=8F=91=E7=8E=B0=E7=9A=84=E5=B9=B6?= =?UTF-8?q?=E5=8F=91=E3=80=81=E6=95=B0=E6=8D=AE=E6=8D=9F=E5=9D=8F=E4=B8=8E?= =?UTF-8?q?=E6=80=A7=E8=83=BD=E7=BC=BA=E9=99=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 数据损坏/丢失： - gm_maintain 与后台维护链并发（validatedCount 双计、communityId 互踩）： scheduleMaintenance 改为类型化单飞 promise，全部维护入口收敛同一互斥点 - computeGlobalPageRank：GDS write 成功后 drop/topK 失败不再回落 1/(i+1) 覆盖真实 PageRank（drop 失败单独吞，宁泄漏临时投影） - summarizeCommunities：prune 挪到签名复用之后——先 prune 会删掉重编号 场景下的捐赠者社区，导致每轮维护全量重算 LLM 摘要+embedding - graph-memory extract：批边界 await 所有 syncEmbed，closeDriver 不再 丢失最后一批在途向量（markExtracted 后不可自愈） - PUT /nodes：改 type 与非法边清理（EDGE_DIRECTION_RULES 生成谓词）包进 单 executeWrite 事务；name 重名预检 409、空名 400 功能失效： - db.ts：删除不可达的 driver 重连分支（驱动 5.x session() 从不抛错）， getSession 永远优先模块级 _driver 单例 - llm.ts：OAuth 缓存按 oauthPath mtime 失效（CLI 刷新 token 后网关 下一次调用即生效，无需重启）；Anthropic 遍历找 text 块（thinking 块 在前不再误报 empty）；openai/anthropic 路径补 fetchRetry（429/5xx） - recall：双路径 Promise.all 并行 + query 向量算一次共享 + assemble 复用 未变 prompt 的缓存（4 embed/4 投影 → 1 embed/2 并行遍历） - flushMessageBuffer 成功后补排 maintenance（熔断跳过的一轮不再丢失） 竞态/一致性： - upsertNode：撞 *_name 唯一约束退回更新路径（幂等）；records[0] 守卫 - upsertEdge：三段查询合并为单条 apoc.merge.relationship（消除并发重复边） - session_end finalize 包进 withExtractLock；gm_record/gm_link 溯源统一 sessionId；crud 内联 normalizeName 改用 store 导入（受一致性测试保护） 性能： - detectDuplicates：N 次向量查询折叠为单条 UNWIND+CALL - applyDecay：Math.max spread 改 reduce（防超大节点集爆栈） - assemble：2N 次串行边查询改 edgesTouching 单次批量；社区摘要并发预取 - communityRepresentatives 加 totalLimit 封顶；graphWalk clamp maxDepth - recall 超时 30s 退避（withBudget 不取消底层查询，防堆积） 配置/测试： - schema recallMaxNodes 3→6 对齐代码默认；freshTailCount 接线到 sliceLastTurn（默认 5=原硬编码 KEEP_TURNS，行为不变） - integration.recall/routes 支持 NEO4J_TEST_URI（消除 7687 硬编码） - 新增集成用例：TASK→EVENT 改 type 边清理、name 重名 409、社区摘要 --- index.ts | 180 ++++++++++++++++++++------------ openclaw.plugin.json | 4 +- src/cli-extract.ts | 9 +- src/engine/embed.ts | 4 +- src/engine/llm.ts | 27 +++-- src/format/assemble.ts | 14 +-- src/graph/community.ts | 10 +- src/graph/decay.ts | 3 +- src/graph/dedup.ts | 55 +++++----- src/graph/pagerank.ts | 54 ++++++---- src/recaller/recall.ts | 24 +++-- src/routes/crud.ts | 66 +++++++++--- src/store/db.ts | 33 ++---- src/store/store.ts | 175 ++++++++++++++++++++----------- src/types.ts | 5 +- test/integration.graph.test.ts | 14 ++- test/integration.recall.test.ts | 3 +- test/integration.routes.test.ts | 49 ++++++++- test/session-identity.test.ts | 1 + 19 files changed, 480 insertions(+), 250 deletions(-) diff --git a/index.ts b/index.ts index 26f6fd2..4a29eb4 100755 --- a/index.ts +++ b/index.ts @@ -14,7 +14,7 @@ import { markExtracted, isTurnExtracted, upsertNode, upsertEdge, findByName, updateNode, deleteNode, deprecateNodeAndDisconnect, - getBySession, edgesFrom, edgesTo, + getBySession, edgesTouching, deleteEdges, mergeNodes, deprecate, getStats, } from "./src/store/store.ts"; @@ -173,17 +173,18 @@ export function extractUserText(msg: any): string { export function sliceLastTurn( messages: any[], + keepTurns: number = KEEP_TURNS, ): { messages: any[]; tokens: number; dropped: number } { if (!messages.length) { return { messages: [], tokens: 0, dropped: 0 }; } - // 找到最近 N 个 user 消息的位置 + // 找到最近 N 个 user 消息的位置（N = keepTurns，由 cfg.freshTailCount 注入） const userIndices: number[] = []; for (let i = messages.length - 1; i >= 0; i--) { if (messages[i].role === "user") { userIndices.push(i); - if (userIndices.length >= KEEP_TURNS) break; + if (userIndices.length >= keepTurns) break; } } if (!userIndices.length) { @@ -232,8 +233,9 @@ export function sliceLastTurn( /** 图谱为空时也必须执行相同的裁剪、工具配对修复和 content 规范化。 */ export function prepareAssemblyMessages( messages: any[], + keepTurns: number = KEEP_TURNS, ): { messages: any[]; tokens: number; dropped: number } { - const sliced = sliceLastTurn(messages); + const sliced = sliceLastTurn(messages, keepTurns); return { messages: normalizeMessageContent(sanitizeToolUseResultPairing(sliced.messages)), tokens: sliced.tokens, @@ -450,6 +452,8 @@ const graphMemoryProPlugin = { const msgSeq = new Map(); const msgSeqLoaders = new Map>(); const recalled = new Map(); + // recalled 结果对应的 recall prompt（assemble 复用缓存判定用；继承路径不写，查不到则照常新鲜召回） + const recalledPrompt = new Map(); const sessionIdsByKey = new Map(); const pendingSubagentRecall = new Map(); const ingestedSinceTurn = new Map(); @@ -539,6 +543,9 @@ const graphMemoryProPlugin = { } if (flushed > 0) { api.logger.info(`[graph-memory-pro] flushed ${flushed} buffered message(s) to neo4j`); + // 补偿掉线期间被熔断跳过的维护：缓冲消息已补录，趁 gate 可用重排一轮 + // （scheduleMaintenance 自带单飞 + gate 检查，无会话时它是安全的 no-op 调用） + scheduleMaintenance(); } } finally { flushRun = null; @@ -572,6 +579,14 @@ const graphMemoryProPlugin = { const RECALL_BUDGET_MS = 5_000; + // 超时退避：withBudget 只放弃等待、不取消底层查询，反复超时会在后台堆积 + // 占连接的 Neo4j 查询链；冷却窗口内跳过新 recall，直接走缓存/降级。 + const RECALL_BACKOFF_MS = 30_000; + let recallBackoffUntil = 0; + function markRecallBackoffOnTimeout(err: unknown): void { + if (String(err).includes("timed out")) recallBackoffUntil = Date.now() + RECALL_BACKOFF_MS; + } + /** * 给 Promise 加等待上限。不取消底层操作（Neo4j 查询会在后台自然完成、 * 连接归还连接池），只是放弃等待 —— 慢 != 死。 @@ -591,25 +606,34 @@ const graphMemoryProPlugin = { // 全局单飞修掉多会话并发跑维护的竞态；运行期间的再次请求只标记 rerun， // 当前一轮结束后最多补跑一次（覆盖"最后一个结束的会话"）。 - let maintenanceRun: Promise | null = null; + let maintenanceRun: Promise> | { skipped: string } | { failed: string }> | null = null; let maintenanceRerunRequested = false; - function scheduleMaintenance(): void { + /** + * 唯一的维护入口（session_end 与 gm_maintain 共用）： + * - 在跑 → 标记 rerun（保留 session_end 的 trailing 补跑语义）并 join + * 同一个 in-flight promise（gm_maintain 据此拿到结果而非并发裸跑） + * - gate 打开（熔断）→ 返回 skipped 标记，不触碰数据库 + * - 空闲 → 自己成为那一轮 + * 并发跑两条维护链会导致 dedup 双计 validatedCount、communityId 互相覆盖。 + */ + function scheduleMaintenance(): Promise> | { skipped: string } | { failed: string }> { if (!neo4jGate.isAvailable()) { api.logger.info("[graph-memory-pro] maintenance skipped: neo4j unavailable (circuit open)"); - return; + return Promise.resolve({ skipped: "neo4j unavailable (circuit open)" }); } if (maintenanceRun) { maintenanceRerunRequested = true; - api.logger.info("[graph-memory-pro] maintenance already running, rerun queued"); - return; + api.logger.info("[graph-memory-pro] maintenance already running, rerun queued + joining in-flight run"); + return maintenanceRun; } maintenanceRun = (async () => { try { + let result: Awaited>; do { maintenanceRerunRequested = false; const embedFn = recaller.embedFn ?? undefined; - const result = await runMaintenance(driver, cfg, llm, embedFn); + result = await runMaintenance(driver, cfg, llm, embedFn); neo4jGate.recordSuccess(); api.logger.info( `[graph-memory-pro] maintenance: ${result.durationMs}ms, ` + @@ -618,14 +642,17 @@ const graphMemoryProPlugin = { `top_pr=${result.pagerank.topK.slice(0, 3).map(n => `${n.name}(${n.score.toFixed(3)})`).join(",")}`, ); } while (maintenanceRerunRequested && neo4jGate.isAvailable()); + return result; } catch (err) { neo4jGate.recordFailure(); api.logger.error(`[graph-memory-pro] maintenance failed: ${err}`); + return { failed: String(err) }; } finally { maintenanceRun = null; maintenanceRerunRequested = false; } })(); + return maintenanceRun; } // ── embedding 会话级 re-probe ────────────────────────── @@ -663,6 +690,8 @@ const graphMemoryProPlugin = { if (prompt.includes("/new or /reset") || prompt.includes("new session was started")) return; // 熔断开启时跳过召回 —— assemble 也会走降级路径（仅转录文本） if (!neo4jGate.isAvailable()) return; + // 超时冷却窗口内跳过（后台可能仍有在途查询，不再叠加） + if (Date.now() < recallBackoffUntil) return; api.logger.info(`[graph-memory-pro] recall query: "${prompt.slice(0, 80)}"`); @@ -673,10 +702,12 @@ const graphMemoryProPlugin = { if (sessionId) { bindSessionIdentity(sessionId, sessionKey); recalled.set(sessionId, res); + recalledPrompt.set(sessionId, prompt); } api.logger.info(`[graph-memory-pro] recalled ${res.nodes.length} nodes, ${res.edges.length} edges`); } } catch (err) { + markRecallBackoffOnTimeout(err); api.logger.warn(`[graph-memory-pro] recall failed: ${err}`); } }); @@ -717,7 +748,7 @@ const graphMemoryProPlugin = { // cron session 关闭图谱功能：仅做消息裁剪与配对修复，不注入图谱上下文 if (isCronSessionKey(sessionKey) && !cronCfg.enabled) { - const prepared = prepareAssemblyMessages(messages); + const prepared = prepareAssemblyMessages(messages, cfg.freshTailCount); if (prepared.dropped > 0) { api.logger.info( `[graph-memory-pro] assemble: ${prepared.messages.length} msgs (~${prepared.tokens} tok), ` + @@ -730,23 +761,25 @@ const graphMemoryProPlugin = { }; } - // prompt-aware recall：优先用当前 prompt 做新鲜召回，回退到 before_agent_start 缓存 + // prompt-aware recall：clean 后的 prompt 与缓存命中同一查询时直接复用 + // before_agent_start 的结果，只有变化才发起第二次召回 let rec = recalled.get(sessionId) ?? { nodes: [], edges: [] }; - if (prompt && neo4jGate.isAvailable()) { - const cleaned = cleanPrompt(prompt); - if (cleaned) { - try { - const freshRec = await withBudget(recaller.recall(cleaned), RECALL_BUDGET_MS, "[graph-memory-pro] assemble recall"); - if (freshRec.nodes.length) { - rec = freshRec; - recalled.set(sessionId, freshRec); - } - } catch (err) { - api.logger.warn(`[graph-memory-pro] assemble recall failed: ${err}`); + const cachedPrompt = recalledPrompt.get(sessionId); + const cleanedPrompt = prompt ? cleanPrompt(prompt) : ""; + if (cleanedPrompt && neo4jGate.isAvailable() && Date.now() >= recallBackoffUntil && cleanedPrompt !== cachedPrompt) { + try { + const freshRec = await withBudget(recaller.recall(cleanedPrompt), RECALL_BUDGET_MS, "[graph-memory-pro] assemble recall"); + if (freshRec.nodes.length) { + rec = freshRec; + recalled.set(sessionId, freshRec); + recalledPrompt.set(sessionId, cleanedPrompt); } + } catch (err) { + markRecallBackoffOnTimeout(err); + api.logger.warn(`[graph-memory-pro] assemble recall failed: ${err}`); } } - const prepared = prepareAssemblyMessages(messages); + const prepared = prepareAssemblyMessages(messages, cfg.freshTailCount); // 图谱段：门控 + 降级 —— Neo4j 掉线/超时时只返回裁剪后的转录， // 不让错误抛回 host（原实现无 catch，getBySession 失败会炸掉 assemble） @@ -755,11 +788,8 @@ const graphMemoryProPlugin = { if (neo4jGate.isAvailable()) { try { const activeNodes = await getBySession(driver, sessionId); - const activeEdges: any[] = []; - for (const n of activeNodes) { - activeEdges.push(...await edgesFrom(driver, n.id)); - activeEdges.push(...await edgesTo(driver, n.id)); - } + // 单次批量查询替代逐节点 edgesFrom+edgesTo 的 2N 次串行往返 + const activeEdges = await edgesTouching(driver, activeNodes.map(n => n.id)); if (activeNodes.length + rec.nodes.length > 0) { const { xml, systemPrompt, tokens } = await assembleContext(driver, { @@ -925,6 +955,7 @@ const graphMemoryProPlugin = { const childSessionId = sessionIdsByKey.get(childSessionKey); if (childSessionId) { recalled.delete(childSessionId); + recalledPrompt.delete(childSessionId); msgSeq.delete(childSessionId); msgSeqLoaders.delete(childSessionId); extractLocks.delete(childSessionId); @@ -939,6 +970,7 @@ const graphMemoryProPlugin = { msgSeqLoaders.clear(); extractLocks.clear(); recalled.clear(); + recalledPrompt.clear(); sessionIdsByKey.clear(); pendingSubagentRecall.clear(); ingestedSinceTurn.clear(); @@ -983,43 +1015,47 @@ const graphMemoryProPlugin = { return; } if (nodes.length) { - // 获取图谱摘要 - const session = getSession(driver); - let summary = ""; - try { - const summaryResult = await session.run(` - MATCH (n:Task|Skill|Event {status: 'active'}) - RETURN n.name AS name, n.type AS type, n.validatedCount AS vc, n.pagerank AS pr - ORDER BY n.pagerank DESC LIMIT 20 - `); - summary = summaryResult.records - .map(r => `${r.get("type")}:${r.get("name")}(v${r.get("vc")},pr${(r.get("pr") ?? 0).toFixed?.(3) ?? "0"})`) - .join(", "); - } finally { - await session.close(); - } + // finalize 的 upsert 与 afterTurn/compact 的提取共用 per-session 互斥锁： + // 最后一轮的 afterTurn 提取可能仍在途，不串行化会重复 upsert（validatedCount 双递增） + await withExtractLock(sid, async () => { + // 获取图谱摘要 + const session = getSession(driver); + let summary = ""; + try { + const summaryResult = await session.run(` + MATCH (n:Task|Skill|Event {status: 'active'}) + RETURN n.name AS name, n.type AS type, n.validatedCount AS vc, n.pagerank AS pr + ORDER BY n.pagerank DESC LIMIT 20 + `); + summary = summaryResult.records + .map(r => `${r.get("type")}:${r.get("name")}(v${r.get("vc")},pr${(r.get("pr") ?? 0).toFixed?.(3) ?? "0"})`) + .join(", "); + } finally { + await session.close(); + } - const fin = await extractor.finalize({ sessionNodes: nodes, graphSummary: summary }); + const fin = await extractor.finalize({ sessionNodes: nodes, graphSummary: summary }); - for (const nc of fin.promotedSkills) { - if (nc.name && nc.content) { - await upsertNode(driver, { - type: "SKILL", name: nc.name, - description: nc.description ?? "", content: nc.content, - }, sid); + for (const nc of fin.promotedSkills) { + if (nc.name && nc.content) { + await upsertNode(driver, { + type: "SKILL", name: nc.name, + description: nc.description ?? "", content: nc.content, + }, sid); + } } - } - for (const ec of fin.newEdges) { - const fromNode = await findByName(driver, ec.from); - const toNode = await findByName(driver, ec.to); - if (fromNode && toNode) { - await upsertEdge(driver, { - fromId: fromNode.id, toId: toNode.id, type: ec.type, - instruction: ec.instruction, sessionId: sid, - }); + for (const ec of fin.newEdges) { + const fromNode = await findByName(driver, ec.from); + const toNode = await findByName(driver, ec.to); + if (fromNode && toNode) { + await upsertEdge(driver, { + fromId: fromNode.id, toId: toNode.id, type: ec.type, + instruction: ec.instruction, sessionId: sid, + }); + } } - } - for (const id of fin.invalidations) await deprecate(driver, id); + for (const id of fin.invalidations) await deprecate(driver, id); + }); } // 图维护：后台单飞（A1）—— 衰减→去重→PR→社区→LLM 摘要可能耗时数分钟， @@ -1032,6 +1068,7 @@ const graphMemoryProPlugin = { msgSeqLoaders.delete(sid); extractLocks.delete(sid); recalled.delete(sid); + recalledPrompt.delete(sid); ingestedSinceTurn.delete(sid); if (sessionKey && sessionIdsByKey.get(sessionKey) === sid) { sessionIdsByKey.delete(sessionKey); @@ -1123,7 +1160,8 @@ const graphMemoryProPlugin = { relatedSkill: Type.Optional(Type.String({ description: "关联的已有技能名" })), }), async execute(_toolCallId: string, p: any) { - const sid = ctx?.sessionKey ?? ctx?.sessionId ?? "manual"; + // 溯源统一用 sessionId（与 getBySession 的会话视图对齐）；无会话上下文才落 "manual" + const sid = ctx?.sessionId ?? "manual"; if (!["TASK", "SKILL", "EVENT"].includes(p.type)) { throw new Error(`[graph-memory-pro] 无效节点类型：${String(p.type)}`); } @@ -1257,7 +1295,7 @@ const graphMemoryProPlugin = { ); api.registerTool( - (_ctx: any) => ({ + (ctx: any) => ({ name: "gm_link", label: "Link Graph Memory Nodes", description: @@ -1282,7 +1320,7 @@ const graphMemoryProPlugin = { const stored = await upsertEdge(driver, { fromId: fromNode.id, toId: toNode.id, type: p.type, - instruction: p.instruction, condition: p.condition, sessionId: "manual", + instruction: p.instruction, condition: p.condition, sessionId: ctx?.sessionId ?? "manual", }); if (!stored) { throw new Error( @@ -1448,8 +1486,16 @@ const graphMemoryProPlugin = { description: "手动触发图维护：衰减评分 + tier 转换、去重、PageRank、社区检测。", parameters: Type.Object({}), async execute() { - const embedFn = recaller.embedFn ?? undefined; - const result = await runMaintenance(driver, cfg, llm, embedFn); + // 走 scheduleMaintenance 单飞入口：后台维护在跑时 join 而非并发裸跑 + // （并发会导致 dedup 双计 validatedCount、communityId 互相覆盖） + const result = await scheduleMaintenance(); + if (!("decay" in result)) { + const reason = "skipped" in result ? result.skipped : result.failed; + return { + content: [{ type: "text", text: `⚠️ 图维护未完成：${reason}` }], + details: result, + }; + } const t = result.decay.tierTransitions; const totalTransitions = t.coreToWorking + t.workingToPeripheral + t.peripheralToWorking + t.workingToCore; const text = [ diff --git a/openclaw.plugin.json b/openclaw.plugin.json index 8022351..9a40a38 100755 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -23,9 +23,9 @@ } }, "compactTurnCount": { "type": "number", "default": 6 }, - "recallMaxNodes": { "type": "number", "default": 3 }, + "recallMaxNodes": { "type": "number", "default": 6 }, "recallMaxDepth": { "type": "number", "default": 2 }, - "freshTailCount": { "type": "number", "default": 10 }, + "freshTailCount": { "type": "number", "default": 5, "description": "assemble 转录保留的最近轮数（裁剪窗口）" }, "dedupThreshold": { "type": "number", "default": 0.90 }, "pagerankDamping": { "type": "number", "default": 0.85 }, "pagerankIterations": { "type": "number", "default": 20 }, diff --git a/src/cli-extract.ts b/src/cli-extract.ts index cfe9d18..04a821c 100644 --- a/src/cli-extract.ts +++ b/src/cli-extract.ts @@ -235,6 +235,11 @@ async function extractSessionLoop( const extraction = await extractor.extract({ messages: msgs, existingNames: existing }); const nameToId = new Map(); + // 批内 fire-and-forget 的 syncEmbed 收集到批边界统一 await： + // closeDriver 在 finally 里执行，若不等待，最后一批在途的 embedding + // HTTP 请求会撞上已关闭的 driver 且错误被吞——向量丢失且不可自愈 + // （markExtracted 已执行，重跑 extract 不会补）。 + const pendingEmbeds: Promise[] = []; for (const nc of extraction.nodes) { const { node } = await upsertNode(driver, { type: nc.type, name: nc.name, @@ -242,7 +247,7 @@ async function extractSessionLoop( }, sessionId); nameToId.set(node.name, node.id); stats.nodes += 1; - void recaller.syncEmbed(node).catch(() => {}); + pendingEmbeds.push(recaller.syncEmbed(node).catch(() => {})); } for (const ec of extraction.edges) { @@ -259,6 +264,8 @@ async function extractSessionLoop( } } + await Promise.allSettled(pendingEmbeds); + const maxTurn = msgs.reduce((m, msg) => Math.max(m, msg.turn_index ?? 0), 0); await markExtracted(driver, sessionId, maxTurn); log(` batch ${stats.batches}: ${msgs.length} 消息 -> ${extraction.nodes.length} 节点 / ${extraction.edges.length} 边（累计 ${stats.nodes}/${stats.edges}）`); diff --git a/src/engine/embed.ts b/src/engine/embed.ts index fa98dc7..740a35d 100755 --- a/src/engine/embed.ts +++ b/src/engine/embed.ts @@ -128,8 +128,8 @@ export async function createEmbedFn(cfg: EmbeddingConfig | undefined): Promise => { return callEmbedding(text.slice(0, 8000), mode); }; - } catch (err) { - console.error(`[graph-memory-pro] embedding probe failed:`, err); + } catch { + // probe 失败返回 null（调用方日志已有 "text search mode" 降级提示），不在库代码里写 stdout return null; } } diff --git a/src/engine/llm.ts b/src/engine/llm.ts index c474dc7..2d7cce2 100755 --- a/src/engine/llm.ts +++ b/src/engine/llm.ts @@ -23,6 +23,7 @@ * 超时：AbortController 强制；默认 60s，cfg.llm.timeoutMs 可调（慢速 API 用户可调大）。 */ +import { stat } from "node:fs/promises"; import { loadOAuthSession, needsRefresh, @@ -144,13 +145,19 @@ export function createCompleteFn( // ── OAuth 会话缓存：单飞刷新，避免并发请求同时触发 refresh ── const oauthPath = provider === "oauth" ? llmConfig?.oauthPath : undefined; let cachedSessionPromise: Promise | null = null; + let cachedSessionMtimeMs: number | null = null; let refreshPromise: Promise | null = null; async function getOAuthSession(): Promise { if (!oauthPath) { throw new Error("[graph-memory] provider=oauth 需要 llm.oauthPath"); } - if (!cachedSessionPromise) { + // oauthPath 可能被运行中的其他进程重写（CLI auth login / CLI extract 刷新 token）。 + // 进程内缓存按 mtime 失效：文件更新后下一次调用即重载，无需重启网关。 + let mtimeMs: number | null = null; + try { mtimeMs = (await stat(oauthPath)).mtimeMs; } catch { /* 文件暂不可达：沿用缓存 */ } + if (!cachedSessionPromise || (mtimeMs !== null && mtimeMs !== cachedSessionMtimeMs)) { + cachedSessionMtimeMs = mtimeMs; cachedSessionPromise = loadOAuthSession(oauthPath).catch((error) => { cachedSessionPromise = null; throw error; @@ -163,6 +170,8 @@ export function createCompleteFn( .then(async (s) => { await saveOAuthSession(oauthPath, s); cachedSessionPromise = Promise.resolve(s); + // 同步 mtime 标记，避免下次调用因文件刚写入而多余重载一次 + try { cachedSessionMtimeMs = (await stat(oauthPath)).mtimeMs; } catch {} refreshPromise = null; return s; }) @@ -251,7 +260,7 @@ export function createCompleteFn( ); } const baseURL = (llmConfig?.baseURL ?? ANTHROPIC_DEFAULT_BASE_URL).replace(/\/+$/, ""); - const res = await fetchWithTimeout(`${baseURL}/v1/messages`, { + const res = await fetchRetry(`${baseURL}/v1/messages`, { method: "POST", headers: { "Content-Type": "application/json", @@ -264,13 +273,19 @@ export function createCompleteFn( system, messages: [{ role: "user", content: user }], }), - }, timeoutMs); + }, 3, timeoutMs); if (!res.ok) { const errText = await res.text().catch(() => ""); throw new Error(`[graph-memory] Anthropic API ${res.status}: ${errText.slice(0, 200)}`); } const data = await res.json() as any; - const text = data.content?.[0]?.text; + // 遍历 content 找 text 块：只看 content[0] 时，thinking 块在前会误报 empty content + const text = Array.isArray(data.content) + ? data.content + .filter((b: any) => b?.type === "text" && typeof b.text === "string") + .map((b: any) => b.text) + .join("") + : ""; if (text) return text; const stop = data.choices?.[0]?.finish_reason ?? data.stop_reason; throw new Error( @@ -288,7 +303,7 @@ export function createCompleteFn( ); } const url = `${baseURL.replace(/\/+$/, "")}/chat/completions`; - const res = await fetchWithTimeout(url, { + const res = await fetchRetry(url, { method: "POST", headers: { "Content-Type": "application/json", @@ -303,7 +318,7 @@ export function createCompleteFn( max_tokens: maxTokens, temperature: 0.1, }), - }, timeoutMs); + }, 3, timeoutMs); if (!res.ok) { const errText = await res.text().catch(() => ""); throw new Error(`[graph-memory] LLM API ${res.status}: ${errText.slice(0, 200)}`); diff --git a/src/format/assemble.ts b/src/format/assemble.ts index 00c516d..29838ec 100755 --- a/src/format/assemble.ts +++ b/src/format/assemble.ts @@ -117,13 +117,15 @@ export async function assembleContext( selectedIds.has(e.fromId) && selectedIds.has(e.toId) && !seen.has(e.id) && seen.add(e.id) ); - // 预加载所有需要的社区摘要 - const communityIds = new Set(selected.map(n => n.communityId).filter(Boolean) as string[]); + // 预加载所有需要的社区摘要（并发拉取，避免逐个 await 的串行往返） + const communityIds = Array.from(new Set(selected.map(n => n.communityId).filter(Boolean) as string[])); + const summaries = await Promise.all( + communityIds.map(cid => getCommunitySummary(driver, cid)), + ); const communitySummaries = new Map(); - for (const cid of communityIds) { - const summary = await getCommunitySummary(driver, cid); - if (summary) communitySummaries.set(cid, summary); - } + communityIds.forEach((cid, i) => { + if (summaries[i]) communitySummaries.set(cid, summaries[i]!); + }); // 按社区分组 const byCommunity = new Map(); diff --git a/src/graph/community.ts b/src/graph/community.ts index 71b021a..af76335 100755 --- a/src/graph/community.ts +++ b/src/graph/community.ts @@ -155,7 +155,6 @@ export async function summarizeCommunities( llm: CompleteFn, embedFn?: EmbedFn, ): Promise { - await pruneCommunitySummaries(driver); let generated = 0; for (const [communityId, memberIds] of communities) { @@ -233,10 +232,15 @@ export async function summarizeCommunities( await upsertCommunitySummary(driver, communityId, cleaned, memberIds.length, embedding, memberSignature); generated++; - } catch (err) { - console.log(` [WARN] community summary failed for ${communityId}: ${err}`); + } catch { + // 单社区摘要失败静默跳过（与 syncEmbed 的吞错策略一致）——库代码不直接写 stdout } } + // prune 必须在复用查找之后：detectCommunities 每轮按成员数重编号 c-1..c-N， + // 旧 id 社区（summary/memberSignature/embedding 的持有者）在新编号下"无人引用"， + // 先 prune 会把捐赠者删掉，签名复用永远不生效 → 每轮维护全量重算 LLM 摘要。 + await pruneCommunitySummaries(driver); + return generated; } diff --git a/src/graph/decay.ts b/src/graph/decay.ts index 9859c88..0f3b2dd 100644 --- a/src/graph/decay.ts +++ b/src/graph/decay.ts @@ -219,7 +219,8 @@ export async function applyDecay(driver: Driver, cfg: Pick): return { enabled: true, scanned: 0, tierTransitions: { ...EMPTY_TRANSITIONS }, durationMs: 0 }; } - const maxPagerank = Math.max(...nodes.map(n => n.pagerank), 0.0001); + // reduce 而非 Math.max(...map)：spread 在超大节点集（>10 万）会爆调用栈 + const maxPagerank = nodes.reduce((m, n) => Math.max(m, n.pagerank), 0.0001); const updates: Array<{ id: string; tier: NodeTier; composite: number; tierChanged: boolean }> = []; const transitions: TierTransition = { ...EMPTY_TRANSITIONS }; diff --git a/src/graph/dedup.ts b/src/graph/dedup.ts index 9fac7b5..94e1f4b 100755 --- a/src/graph/dedup.ts +++ b/src/graph/dedup.ts @@ -39,36 +39,37 @@ export async function detectDuplicates(driver: Driver, cfg: GmConfig): Promise ({ + id: r.get("id"), + name: r.get("name"), + embedding: r.get("embedding"), + })); + const searchResult = await session.run(` + UNWIND $nodes AS n + CALL db.index.vector.queryNodes('gm_node_embedding', 5, n.embedding) YIELD node, score + WHERE node.id <> n.id AND node.status = 'active' AND score >= $threshold + RETURN n.id AS nodeA, n.name AS nameA, node.id AS nodeB, node.name AS nameB, score AS similarity + `, { nodes, threshold: cfg.dedupThreshold }); + const pairs: DuplicatePair[] = []; const seenPairs = new Set(); - // 对每个节点做向量搜索 - for (const record of nodesResult.records) { - const nodeId = record.get("id"); - const nodeName = record.get("name"); - const embedding = record.get("embedding"); - - const searchResult = await session.run(` - CALL db.index.vector.queryNodes('gm_node_embedding', 5, $vec) - YIELD node, score - WHERE node.id <> $nodeId AND node.status = 'active' AND score >= $threshold - RETURN node.id AS id, node.name AS name, score - `, { vec: embedding, nodeId, threshold: cfg.dedupThreshold }); - - for (const sr of searchResult.records) { - const otherId = sr.get("id"); - const pairKey = [nodeId, otherId].sort().join("|"); - if (seenPairs.has(pairKey)) continue; - seenPairs.add(pairKey); - - pairs.push({ - nodeA: nodeId, - nodeB: otherId, - nameA: nodeName, - nameB: sr.get("name"), - similarity: sr.get("score"), - }); - } + for (const sr of searchResult.records) { + const nodeId = sr.get("nodeA"); + const otherId = sr.get("nodeB"); + const pairKey = [nodeId, otherId].sort().join("|"); + if (seenPairs.has(pairKey)) continue; + seenPairs.add(pairKey); + + pairs.push({ + nodeA: nodeId, + nodeB: otherId, + nameA: sr.get("nameA"), + nameB: sr.get("nameB"), + similarity: sr.get("similarity"), + }); } return pairs.sort((a, b) => b.similarity - a.similarity); diff --git a/src/graph/pagerank.ts b/src/graph/pagerank.ts index 2cb7030..a39edd6 100755 --- a/src/graph/pagerank.ts +++ b/src/graph/pagerank.ts @@ -122,30 +122,40 @@ export async function computeGlobalPageRank(driver: Driver, cfg: GmConfig): Prom await projectActiveGraph(session, graphName, existingTypes); - await session.run(` - CALL gds.pageRank.write('${graphName}', { - writeProperty: 'pagerank', - dampingFactor: $damping, - maxIterations: toInteger($iterations) - }) - `, { damping: cfg.pagerankDamping, iterations: cfg.pagerankIterations }); - - await session.run(`CALL gds.graph.drop('${graphName}')`); - - const topResult = await session.run(` - MATCH (n:Task|Skill|Event {status: 'active'}) RETURN n.id AS id, n.name AS name, n.pagerank AS score - ORDER BY n.pagerank DESC LIMIT 20 - `); + try { + await session.run(` + CALL gds.pageRank.write('${graphName}', { + writeProperty: 'pagerank', + dampingFactor: $damping, + maxIterations: toInteger($iterations) + }) + `, { damping: cfg.pagerankDamping, iterations: cfg.pagerankIterations }); + } finally { + // drop 失败只能吞掉（宁泄漏一个临时投影）：此分支若抛错落入外层 catch， + // fallback 会用 1/(i+1) 覆盖刚 write 成功的真实 PageRank —— 数据损坏远重于投影泄漏 + try { await session.run(`CALL gds.graph.drop('${graphName}')`); } catch {} + } - const scores = new Map(); - const topK: Array<{ id: string; name: string; score: number }> = []; - for (const r of topResult.records) { - const rawScore = r.get("score"); - const score = typeof rawScore === "number" ? rawScore : (rawScore?.toNumber?.() ?? 0); - scores.set(r.get("id"), score); - topK.push({ id: r.get("id"), name: r.get("name"), score }); + // write 已成功：pagerank 属性已是真值，后续读取失败只返回空排序， + // 绝不回落外层 catch 的 fallback（那会覆盖全图正确分数） + try { + const topResult = await session.run(` + MATCH (n:Task|Skill|Event {status: 'active'}) RETURN n.id AS id, n.name AS name, n.pagerank AS score + ORDER BY n.pagerank DESC LIMIT 20 + `); + + const scores = new Map(); + const topK: Array<{ id: string; name: string; score: number }> = []; + for (const r of topResult.records) { + const rawScore = r.get("score"); + const score = typeof rawScore === "number" ? rawScore : (rawScore?.toNumber?.() ?? 0); + scores.set(r.get("id"), score); + topK.push({ id: r.get("id"), name: r.get("name"), score }); + } + return { scores, topK }; + } catch { + return { scores: new Map(), topK: [] }; } - return { scores, topK }; } catch { try { await session.run(`CALL gds.graph.drop('${graphName}')`); } catch {} // GDS 不可用时降级为确定性 fallback（与 PPR 一致：按稳定排序赋 1/(i+1)） diff --git a/src/recaller/recall.ts b/src/recaller/recall.ts index e7fa6f3..1c1f515 100755 --- a/src/recaller/recall.ts +++ b/src/recaller/recall.ts @@ -98,8 +98,17 @@ export class Recaller { const limit = this.cfg.recallMaxNodes; const timeRange = options ? parseTimeRange(options) : null; - const precise = await this.recallPrecise(query, limit, timeRange); - const generalized = await this.recallGeneralized(query, limit, timeRange); + // query 向量只算一次，两条路径共享；失败统一落 null（各路径走文本兜底）。 + // 双路径并行执行 —— 原串行 + 各自 embed 会把 2 次调用放大成 4 次 API 调用 + // 与 4 次图遍历，全部压在调用方的预算窗口内。 + const embedPromise: Promise = this.embed + ? this.embed(query, "query").catch(() => null) + : Promise.resolve(null); + + const [precise, generalized] = await Promise.all([ + this.recallPrecise(query, limit, timeRange, embedPromise), + this.recallGeneralized(limit, timeRange, embedPromise), + ]); const merged = this.mergeResults(precise, generalized); return merged; @@ -112,12 +121,13 @@ export class Recaller { query: string, limit: number, timeRange: ParsedTimeRange | null, + embedPromise: Promise, ): Promise { let seeds: GmNode[] = []; - if (this.embed) { + const vec = await embedPromise; + if (vec) { try { - const vec = await this.embed(query, "query"); const scored = await vectorSearchWithScore(this.driver, vec, Math.ceil(limit / 2)); seeds = scored.map(s => s.node); @@ -180,15 +190,15 @@ export class Recaller { * 泛化召回：社区向量搜索 → 图遍历 → PPR 排序 */ private async recallGeneralized( - query: string, limit: number, timeRange: ParsedTimeRange | null, + embedPromise: Promise, ): Promise { let seeds: GmNode[] = []; - if (this.embed) { + const vec = await embedPromise; + if (vec) { try { - const vec = await this.embed(query, "query"); const scoredCommunities = await communityVectorSearch(this.driver, vec); if (scoredCommunities.length > 0) { diff --git a/src/routes/crud.ts b/src/routes/crud.ts index c837ea3..9d4b961 100644 --- a/src/routes/crud.ts +++ b/src/routes/crud.ts @@ -16,11 +16,11 @@ import type { Driver } from "neo4j-driver"; import type { OpenClawPluginApi } from "openclaw/plugin-sdk"; import type { Recaller } from "../recaller/recall.ts"; import type { NodeType, EdgeType } from "../types.ts"; -import { NODE_TYPE_TO_LABEL, isValidEdgeDirection } from "../types.ts"; +import { NODE_TYPE_TO_LABEL, isValidEdgeDirection, EDGE_DIRECTION_RULES, EDGE_TYPES } from "../types.ts"; import { - upsertNode, findById, allActiveNodes, allEdges, + upsertNode, findById, findByName, allActiveNodes, allEdges, upsertEdge, edgesFrom, edgesTo, deprecate, mergeNodes, - searchNodes, getStats, + searchNodes, getStats, normalizeName, } from "../store/store.ts"; import { getSession } from "../store/db.ts"; @@ -262,12 +262,20 @@ async function handleUpdateNode( params.content = body.content as string; } if (body.name !== undefined) { - // Name change — normalize - const newName = (body.name as string).trim().toLowerCase() - .replace(/[\s_]+/g, "-") - .replace(/[^a-z0-9\u4e00-\u9fff\-]/g, "") - .replace(/-{2,}/g, "-") - .replace(/^-|-$/g, ""); + // 与 store/extractor 的 normalizeName 同源（受 normalize-name.test.ts 跨文件一致性保护） + const newName = normalizeName(body.name as string); + if (!newName) { + json(res, 400, { error: "name normalizes to empty string" }); + return true; + } + if (newName !== existing.name) { + // 重名预检：撞 *_name 唯一约束会让裸 session.run 抛成 500，提前回 409 + const conflict = await findByName(driver, newName); + if (conflict && conflict.id !== existing.id) { + json(res, 409, { error: `Node name already exists: ${newName}` }); + return true; + } + } updates.push("n.name = $newName"); params.newName = newName; } @@ -287,12 +295,40 @@ async function handleUpdateNode( if (updates.length > 1) { // always has updatedAt const session = getSession(driver); try { - await session.run( - `MATCH (n:Task|Skill|Event {id: $id}) - SET ${updates.join(", ")} - ${newType ? `REMOVE n:Task, n:Skill, n:Event SET n:${NODE_TYPE_TO_LABEL[newType]}` : ""}`, - params, - ); + // 单事务：改 type 与非法边清理必须原子 —— 若 DELETE 瞬时失败而 SET 已提交， + // 节点会停留在"新 type + 违反白名单的存量边"状态（正是本端点要修复的不变量） + await session.executeWrite(async tx => { + await tx.run( + `MATCH (n:Task|Skill|Event {id: $id}) + SET ${updates.join(", ")} + ${newType ? `REMOVE n:Task, n:Skill, n:Event SET n:${NODE_TYPE_TO_LABEL[newType]}` : ""}`, + params, + ); + + if (newType) { + // 清理方向白名单外的存量边（gm_* 工具链不允许改 type，仅此端点允许—— + // 必须自己恢复图谱不变量）。合法谓词从 EDGE_DIRECTION_RULES 生成（与 + // isValidEdgeDirection 同一事实源），出/入边各一条定向查询——无向匹配无法区分 + // source/target，from/to 集合不对称时会误删合法边。 + const legal = EDGE_TYPES + .map(t => { + const rule = EDGE_DIRECTION_RULES[t]; + return `(type(r) = '${t}' AND source.type IN ${JSON.stringify(rule.from)} AND target.type IN ${JSON.stringify(rule.to)})`; + }) + .join(" OR "); + const types = JSON.stringify([...EDGE_TYPES]); + await tx.run(` + MATCH (source:Task|Skill|Event {id: $id})-[r]->(target:Task|Skill|Event) + WHERE type(r) IN ${types} AND NOT (${legal}) + DELETE r + `, { id }); + await tx.run(` + MATCH (source:Task|Skill|Event)-[r]->(target:Task|Skill|Event {id: $id}) + WHERE type(r) IN ${types} AND NOT (${legal}) + DELETE r + `, { id }); + } + }); } finally { await session.close(); } diff --git a/src/store/db.ts b/src/store/db.ts index 01631f8..e5a43be 100755 --- a/src/store/db.ts +++ b/src/store/db.ts @@ -1,23 +1,19 @@ /** * graph-memory-pro — Neo4j 连接管理（加固版） * - * 解决 "Pool is closed" 问题： * - driver 是长生命周期单例，不在 dispose 时关闭 - * - getSession 在 driver 被意外关闭时自动重建 + * - getSession 永远优先模块级单例（见函数注释） */ import neo4j, { type Driver, type Session } from "neo4j-driver"; import type { EmbeddingConfig, Neo4jConfig } from "../types.ts"; let _driver: Driver | null = null; -let _cfg: Neo4jConfig | null = null; /** * 获取 Neo4j Driver 单例 - * 保存配置，支持自动重连 */ export function getDriver(cfg: Neo4jConfig): Driver { - _cfg = cfg; if (_driver) return _driver; _driver = neo4j.driver(cfg.uri, neo4j.auth.basic(cfg.user, cfg.password), { maxConnectionPoolSize: 50, @@ -30,25 +26,16 @@ export function getDriver(cfg: Neo4jConfig): Driver { /** * 获取一个 Session（用完必须 close） - * 如果 driver 被关闭了，自动用保存的配置重建 + * + * 永远优先模块级 _driver 单例：调用方（register() 启动时捕获一次并四处传递） + * 持有的旧引用在单例重建后会指向已关闭的池。入参仅作 getDriver 未初始化时 + * 的回退兼容。 + * 注：neo4j-driver 5.x 的 driver.session() 构造阶段不抛错，"Pool is closed" + * 在 session.run() 才报——掉线恢复由 gate 熔断 + 驱动自身连接池重连负责， + * 这里不做（也做不了）session 级重连。 */ -export function getSession(driver: Driver): Session { - try { - return driver.session({ database: "neo4j" }); - } catch (err) { - // Pool is closed — 尝试重建 driver - if (_cfg && String(err).includes("closed")) { - console.log("[graph-memory-pro] reconnecting Neo4j driver..."); - _driver = neo4j.driver(_cfg.uri, neo4j.auth.basic(_cfg.user, _cfg.password), { - maxConnectionPoolSize: 50, - // 与 getDriver 保持一致：快速失败，让熔断门控尽早接手 - connectionAcquisitionTimeout: 15_000, - maxTransactionRetryTime: 10_000, - }); - return _driver.session({ database: "neo4j" }); - } - throw err; - } +export function getSession(passedDriver: Driver): Session { + return (_driver ?? passedDriver).session({ database: "neo4j" }); } /** diff --git a/src/store/store.ts b/src/store/store.ts index b5d6f6a..0538bbc 100755 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -9,7 +9,7 @@ import type { Driver } from "neo4j-driver"; import neo4j from "neo4j-driver"; import { createHash } from "crypto"; import type { GmNode, GmEdge, EdgeType, NodeType, NodeTier } from "../types.ts"; -import { NODE_TYPE_TO_LABEL, isValidEdgeDirection } from "../types.ts"; +import { NODE_TYPE_TO_LABEL, isValidEdgeDirection, EDGE_TYPES } from "../types.ts"; import { getSession } from "./db.ts"; /** Neo4j LIMIT/索引参数必须是 Integer */ @@ -141,6 +141,12 @@ export async function allEdges(driver: Driver): Promise { } } +/** 判断错误是否为 *_name 唯一约束冲突（CREATE 撞上并发创建时用于幂等回退） */ +function isNameConstraintViolation(err: unknown): boolean { + const s = String(err); + return s.includes("ConstraintValidationFailed") || s.includes("already exists with label"); +} + export async function upsertNode( driver: Driver, c: { type: NodeType; name: string; description: string; content: string }, @@ -150,6 +156,35 @@ export async function upsertNode( const label = NODE_TYPE_TO_LABEL[c.type as NodeType]; if (!label) throw new Error(`[graph-memory-pro] Invalid node type: ${String(c.type)}`); const session = getSession(driver); + /** 按 name 更新已存在节点（find 命中与撞约束回退两条路径共用） */ + const updateExisting = async (): Promise<{ node: GmNode; isNew: boolean }> => { + await session.run(` + MATCH (n:Task|Skill|Event {name: $name}) + SET n.content = CASE WHEN size($content) > size(n.content) THEN $content ELSE n.content END, + n.description = CASE WHEN size($description) > size(n.description) THEN $description ELSE n.description END, + n.validatedCount = n.validatedCount + 1, + n.sourceSessions = CASE + WHEN NOT $sessionId IN n.sourceSessions + THEN n.sourceSessions + $sessionId + ELSE n.sourceSessions + END, + n.lastAccessedAt = $now, + n.updatedAt = $now + RETURN n + `, { name, content: c.content, description: c.description, sessionId, now: Date.now() }); + + const updated = await session.run( + "MATCH (n:Task|Skill|Event {name: $name}) RETURN n", + { name }, + ); + // 撞约束回退路径存在窄窗口：并发创建的同名节点可能在 MATCH 前被删除 + //（如 maintenance mergeNodes）——守卫让单个节点失败而不是 TypeError 炸整批 + const record = updated.records[0]?.get("n"); + if (!record) { + throw new Error(`[graph-memory-pro] upsertNode: node "${name}" disappeared during update`); + } + return { node: toNode(record), isNew: false }; + }; try { // Try to find existing node with this name across all knowledge labels const existing = await session.run( @@ -158,29 +193,12 @@ export async function upsertNode( ); if (existing.records.length > 0) { - // Update existing node - await session.run(` - MATCH (n:Task|Skill|Event {name: $name}) - SET n.content = CASE WHEN size($content) > size(n.content) THEN $content ELSE n.content END, - n.description = CASE WHEN size($description) > size(n.description) THEN $description ELSE n.description END, - n.validatedCount = n.validatedCount + 1, - n.sourceSessions = CASE - WHEN NOT $sessionId IN n.sourceSessions - THEN n.sourceSessions + $sessionId - ELSE n.sourceSessions - END, - n.lastAccessedAt = $now, - n.updatedAt = $now - RETURN n - `, { name, content: c.content, description: c.description, sessionId, now: Date.now() }); - - const updated = await session.run( - "MATCH (n:Task|Skill|Event {name: $name}) RETURN n", - { name }, - ); - return { node: toNode(updated.records[0].get("n")), isNew: false }; - } else { - // Create new node with specific label + // 必须 return await：否则 finally 的 session.close() 会与闭包内的 + // 第二次 session.run 竞态（closed session 错误） + return await updateExisting(); + } + // Create new node with specific label + try { const now = Date.now(); const result = await session.run(` CREATE (n:MemoryNode:${label} { @@ -198,6 +216,11 @@ export async function upsertNode( sessions: [sessionId], now, }); return { node: toNode(result.records[0].get("n")), isNew: true }; + } catch (err) { + // find-then-create 窗口内并发路径抢先创建了同名节点（撞 *_name 唯一约束） + // → 退回更新路径保持幂等，而不是让整轮提取失败重试 + if (isNameConstraintViolation(err)) return await updateExisting(); + throw err; } } finally { await session.close(); @@ -458,42 +481,25 @@ export async function upsertEdge( const toType = endpoints.records[0].get("toType"); if (!isValidEdgeDirection(e.type, fromType, toType)) return false; - // 检查是否已存在同 from+to+type 的边 - const existing = await session.run(` - MATCH (a:Task|Skill|Event {id: $fromId})-[r]->(b:Task|Skill|Event {id: $toId}) - WHERE type(r) = $type - RETURN r - `, { fromId: e.fromId, toId: e.toId, type: e.type }); - - if (existing.records.length > 0) { - await session.run(` - MATCH (a:Task|Skill|Event {id: $fromId})-[r]->(b:Task|Skill|Event {id: $toId}) - WHERE type(r) = $type - SET r.instruction = $instruction - `, { fromId: e.fromId, toId: e.toId, type: e.type, instruction: e.instruction }); - } else { - // 用 APOC 动态创建关系（type 是变量） - await session.run(` - MATCH (a:Task|Skill|Event {id: $fromId}), (b:Task|Skill|Event {id: $toId}) - CALL apoc.create.relationship(a, $type, { - id: $id, - instruction: $instruction, - condition: $condition, - sessionId: $sessionId, - createdAt: $now - }, b) YIELD rel - RETURN rel - `, { - fromId: e.fromId, - toId: e.toId, - type: e.type, - id: uid("e"), - instruction: e.instruction, - condition: e.condition ?? null, - sessionId: e.sessionId, - now: Date.now(), - }); - } + // MERGE 语义：查重 + 创建/更新合并为单条原子语句，消除并发下绕过查重产生重复边的窗口。 + // onCreate 写入全部属性；onMatch 仅刷新 instruction（与原查重-更新分支行为一致）。 + await session.run(` + MATCH (a:Task|Skill|Event {id: $fromId}), (b:Task|Skill|Event {id: $toId}) + CALL apoc.merge.relationship(a, $type, {}, { + id: $id, instruction: $instruction, condition: $condition, + sessionId: $sessionId, createdAt: $now + }, b, { instruction: $instruction }) YIELD rel + RETURN rel + `, { + fromId: e.fromId, + toId: e.toId, + type: e.type, + id: uid("e"), + instruction: e.instruction, + condition: e.condition ?? null, + sessionId: e.sessionId, + now: Date.now(), + }); return true; } finally { await session.close(); @@ -550,6 +556,34 @@ export async function edgesTo(driver: Driver, id: string): Promise { } } +/** 批量查询至少一端在 ids 内的知识边 —— 一次往返替代逐节点 edgesFrom+edgesTo 的 2N 次往返。 */ +export async function edgesTouching(driver: Driver, ids: string[]): Promise { + if (!ids.length) return []; + const session = getSession(driver); + try { + const result = await session.run(` + MATCH (a:Task|Skill|Event)-[r]->(b:Task|Skill|Event) + WHERE (a.id IN $ids OR b.id IN $ids) + AND type(r) IN ${JSON.stringify([...EDGE_TYPES])} + RETURN r.id AS id, a.id AS fromId, b.id AS toId, type(r) AS type, + r.instruction AS instruction, r.condition AS condition, + r.sessionId AS sessionId, r.createdAt AS createdAt + `, { ids }); + return result.records.map(r => ({ + id: r.get("id"), + fromId: r.get("fromId"), + toId: r.get("toId"), + type: r.get("type") as EdgeType, + instruction: r.get("instruction"), + condition: r.get("condition") ?? undefined, + sessionId: r.get("sessionId"), + createdAt: toInt(r.get("createdAt")), + })); + } finally { + await session.close(); + } +} + /** 删除 from→to 之间的边；type 省略时删除所有类型。返回删除条数。 */ export async function deleteEdges( driver: Driver, @@ -743,6 +777,10 @@ export async function graphWalk( ): Promise<{ nodes: GmNode[]; edges: GmEdge[] }> { if (!seedIds.length) return { nodes: [], edges: [] }; + // maxDepth 来自配置且直接内插进 Cypher —— clamp 到 [1,4]，非法值只会得到安全深度而非语法错误 + const parsedDepth = Number(maxDepth); + const depth = Math.max(1, Math.min(4, Number.isFinite(parsedDepth) ? Math.floor(parsedDepth) : 2)); + const session = getSession(driver); try { // 用 Neo4j 的变长路径匹配做图遍历 @@ -751,7 +789,7 @@ export async function graphWalk( WHERE seed.id IN $seedIds AND seed.status = 'active' CALL { WITH seed - MATCH path = (seed)-[*0..${maxDepth}]-(neighbor:Task|Skill|Event {status: 'active'}) + MATCH path = (seed)-[*0..${depth}]-(neighbor:Task|Skill|Event {status: 'active'}) WHERE all(node IN nodes(path) WHERE node.status = 'active') RETURN DISTINCT neighbor } @@ -807,7 +845,16 @@ export async function getBySession(driver: Driver, sessionId: string): Promise { +/** + * 每社区取最近更新的 perCommunity 个代表节点。 + * totalLimit 封顶总返回数（按社区规模降序截断）—— recall 兜底路径用它做 + * graphWalk 种子，社区很多时无上限种子会把遍历放大成全图扫描。 + */ +export async function communityRepresentatives( + driver: Driver, + perCommunity = 2, + totalLimit = 20, +): Promise { const session = getSession(driver); try { const result = await session.run(` @@ -816,9 +863,11 @@ export async function communityRepresentatives(driver: Driver, perCommunity = 2) WITH n.communityId AS cid, n ORDER BY n.updatedAt DESC WITH cid, collect(n) AS members + ORDER BY size(members) DESC UNWIND members[0..toInteger($perCommunity)] AS m RETURN m AS n - `, { perCommunity }); + LIMIT toInteger($totalLimit) + `, { perCommunity, totalLimit }); return result.records.map(r => toNode(r.get("n"))); } finally { await session.close(); diff --git a/src/types.ts b/src/types.ts index ce30fca..a264a1f 100755 --- a/src/types.ts +++ b/src/types.ts @@ -65,7 +65,7 @@ export const EDGE_TYPES = [ export type EdgeType = (typeof EDGE_TYPES)[number]; -const EDGE_DIRECTION_RULES: Record = { @@ -214,6 +214,7 @@ export interface GmConfig { compactTurnCount: number; recallMaxNodes: number; recallMaxDepth: number; + /** assemble 保留的最近轮数（裁剪窗口）；默认 5，与 sliceLastTurn 的回退值一致。 */ freshTailCount: number; embedding?: EmbeddingConfig; llm?: { @@ -247,7 +248,7 @@ export const DEFAULT_CONFIG: GmConfig = { compactTurnCount: 6, recallMaxNodes: 6, recallMaxDepth: 2, - freshTailCount: 10, + freshTailCount: 5, dedupThreshold: 0.90, pagerankDamping: 0.85, pagerankIterations: 20, diff --git a/test/integration.graph.test.ts b/test/integration.graph.test.ts index 8b52d10..61cc0ca 100644 --- a/test/integration.graph.test.ts +++ b/test/integration.graph.test.ts @@ -221,7 +221,19 @@ describe.skipIf(!ENABLED)("graph layer integration (GDS, Docker)", () => { expect(summary?.summary).toBe("容器部署与编排技能"); expect(summary?.memberSignature).toBe(buildCommunityMemberSignature(memberIds)); - // detectCommunities 每轮按成员数重编号（c-1..c-N），ID 变但成员相同 → 按签名跨社区复用 + // detectCommunities 每轮按成员数重编号（c-1..c-N），ID 变但成员相同 → 按签名跨社区复用。 + // 生产链路里 updateCommunities 会先把成员 communityId 改写到新 id 再进 summarize —— + // 这里同样 SET 成员指向新 id（保持生产不变量），旧 id 成为"无人引用"的捐赠者， + // 复用查找发生在 prune 之前，捐赠者复制完摘要后才被 prune 清理。 + const renumber = getSession(driver); + try { + await renumber.run( + "MATCH (n:MemoryNode) WHERE n.id IN $ids SET n.communityId = $cid", + { ids: memberIds, cid: "c-reuse-renumbered" }, + ); + } finally { + await renumber.close(); + } const third = await summarizeCommunities( driver, new Map([["c-reuse-renumbered", memberIds]]), llm, ); diff --git a/test/integration.recall.test.ts b/test/integration.recall.test.ts index 1cf5390..6bb51b5 100644 --- a/test/integration.recall.test.ts +++ b/test/integration.recall.test.ts @@ -17,6 +17,7 @@ import { Recaller, buildNodeEmbeddingText } from "../src/recaller/recall.ts"; import { DEFAULT_CONFIG, type GmConfig } from "../src/types.ts"; const ENABLED = !!process.env.NEO4J_INTEGRATION; +const NEO4J_URI = process.env.NEO4J_TEST_URI ?? "bolt://localhost:7687"; let driver: Driver; const TEST_SID = `recall-${Date.now()}`; @@ -24,7 +25,7 @@ const cfg: GmConfig = { ...DEFAULT_CONFIG, recallMaxNodes: 5, recallMaxDepth: 2 describe.skipIf(!ENABLED)("Recaller integration", () => { beforeAll(async () => { - driver = getDriver({ uri: "bolt://localhost:7687", user: "neo4j", password: "graphmemory" }); + driver = getDriver({ uri: NEO4J_URI, user: "neo4j", password: "graphmemory" }); await initSchema(driver); // 构造可被关键词召回的图 diff --git a/test/integration.routes.test.ts b/test/integration.routes.test.ts index 9337726..5077ad1 100644 --- a/test/integration.routes.test.ts +++ b/test/integration.routes.test.ts @@ -6,6 +6,7 @@ import { closeDriver, getDriver, getSession, initSchema } from "../src/store/db. import { findById, upsertNode } from "../src/store/store.ts"; const ENABLED = !!process.env.NEO4J_INTEGRATION; +const NEO4J_URI = process.env.NEO4J_TEST_URI ?? "bolt://localhost:7687"; const TEST_SID = `routes-${Date.now()}`; let driver: Driver; @@ -30,7 +31,7 @@ async function request(method: string, path: string, body?: Record { beforeAll(async () => { - driver = getDriver({ uri: "bolt://localhost:7687", user: "neo4j", password: "graphmemory" }); + driver = getDriver({ uri: NEO4J_URI, user: "neo4j", password: "graphmemory" }); await initSchema(driver); const api = { @@ -109,4 +110,50 @@ describe.skipIf(!ENABLED)("CRUD route integration", () => { }); expect(response.status).toBe(400); }); + + it("drops direction-violating edges when the node type changes (TASK→EVENT)", async () => { + const { node: skill } = await upsertNode(driver, { + type: "SKILL", name: "route-typechange-skill", description: "d", content: "c", + }, TEST_SID); + const { node: task } = await upsertNode(driver, { + type: "TASK", name: "route-typechange-task", description: "d", content: "c", + }, TEST_SID); + + // TASK→SKILL 的 USED_SKILL 合法；节点改成 EVENT 后 USED_SKILL 出边违反白名单 + const created = await request("POST", "edges", { + fromId: task.id, + toId: skill.id, + type: "USED_SKILL", + instruction: "legal before type change", + }); + expect(created.status).toBe(201); + + const changed = await request("PUT", `nodes?id=${task.id}`, { type: "EVENT" }); + expect(changed.status).toBe(200); + + const session = getSession(driver); + try { + const outEdges = await session.run( + "MATCH (n {id: $id})-[r]->() RETURN type(r) AS type", { id: task.id }, + ); + expect(outEdges.records).toHaveLength(0); + } finally { + await session.close(); + } + }); + + it("rejects renaming a node to an existing name with 409", async () => { + const { node: keeper } = await upsertNode(driver, { + type: "SKILL", name: "route-name-keeper", description: "d", content: "c", + }, TEST_SID); + const { node: victim } = await upsertNode(driver, { + type: "TASK", name: "route-name-victim", description: "d", content: "c", + }, TEST_SID); + + const response = await request("PUT", `nodes?id=${victim.id}`, { name: keeper.name }); + expect(response.status).toBe(409); + // 自身同名（标准化后未变）不算冲突 + const selfRename = await request("PUT", `nodes?id=${victim.id}`, { name: "route-name-victim" }); + expect(selfRename.status).toBe(200); + }); }); diff --git a/test/session-identity.test.ts b/test/session-identity.test.ts index 3253762..4068ec3 100644 --- a/test/session-identity.test.ts +++ b/test/session-identity.test.ts @@ -44,6 +44,7 @@ vi.mock("../src/store/store.ts", () => ({ getBySession: mocks.getBySession, edgesFrom: async () => [], edgesTo: async () => [], + edgesTouching: async () => [], deprecate: async () => {}, getStats: async () => ({}), })); From 6009c28046049e73ded0fd66b857b1b587401a48 Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Mon, 31 Aug 2026 00:10:46 +0000 Subject: [PATCH 12/29] chore: updated docs --- README.md | 4 ++-- README_CN.md | 38 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index cd95ab5..9de115e 100644 --- a/README.md +++ b/README.md @@ -210,13 +210,13 @@ Inspect the graph with the bundled Cypher shell: | `gm_unlink` | Remove edges between two nodes by name; optional `type` filter, otherwise all from→to edges | | `gm_merge` | Merge two same-type duplicate nodes: keep absorbs content/validatedCount/sessions + dedup-aware edge migration; merge is soft-deleted | | `gm_stats` | Show node, relationship, community, and PageRank statistics | -| `gm_maintain` | Run deduplication, PageRank, and community maintenance | +| `gm_maintain` | Run decay scoring, deduplication, PageRank, and community maintenance | ## Development ```bash npm install -npm run build # tsc --noEmit +npm run build # compiles to dist/ npm test # unit tests only (no Neo4j required) ``` diff --git a/README_CN.md b/README_CN.md index a67305b..f5adea1 100644 --- a/README_CN.md +++ b/README_CN.md @@ -62,6 +62,7 @@ bash setup-graph-memory-pro.sh --uninstall "password": "你的 Neo4j 密码" }, "llm": { + "provider": "openai", "apiKey": "你的 LLM API Key", "baseURL": "https://api.openai.com/v1", "model": "gpt-4o-mini" @@ -79,8 +80,41 @@ bash setup-graph-memory-pro.sh --uninstall } ``` +Anthropic 直连（Claude）——去掉 `baseURL`，切换 `provider`： + +```json +"llm": { + "provider": "anthropic", + "apiKey": "sk-ant-...", + "model": "claude-3-5-sonnet-20241022" +} +``` + `embedding` 可选。设置时，`dimensions` 必须与 Neo4j 向量索引维度一致。新数据库会在插件启动时按配置创建索引；更换维度后需要重建向量索引或 Neo4j 数据库。 +### 记忆衰减（遗忘曲线） + +每个维护周期对全部 active 节点做三因子加权评分（recency + frequency + intrinsic），并在三个 tier 之间双向转换：`core` / `working` / `peripheral`。衰减不会把节点置为 `status=deprecated`——只有手动弃用 / 合并才会。decay 只调整 `tier`，所有 active 节点始终保持可搜索。 + +完整公式、字段映射、默认值依据与调参指南见 **[`docs/decay.md`](docs/decay.md)**。 + +最小配置（所有字段可选，默认值如下）： + +```json +"decay": { "enabled": true } +``` + +常用覆盖——完整参数见 `docs/decay.md` §4： + +```json +"decay": { + "enabled": true, + "recencyHalfLifeDays": 30, + "peripheralCompositeThreshold": 0.15, + "workingAccessThreshold": 3 +} +``` + ### cron 会话行为控制 OpenClaw 定时任务创建的会话可以独立配置图谱行为。host 把 cron 标记放在 **sessionKey** 上（`sessionId` 是随机 UUID），实际形状为 `cron:`、`agent::cron:` 或 `agent::cron::run:`： @@ -122,7 +156,7 @@ openclaw graph-memory auth login -> embedding -> 向量召回 + 社区扩展 + GDS PPR -> XML 上下文注入 -会话结束 -> 去重 -> 全局 PageRank -> 社区 -> 社区摘要 +会话结束 -> 衰减（遗忘曲线）-> 去重 -> 全局 PageRank -> 社区 -> 社区摘要 ``` ## 验证 @@ -156,7 +190,7 @@ openclaw gateway --verbose | `gm_unlink` | 按名称删除两节点之间的关系边；可选 type 过滤，不传则删除 from→to 之间所有边 | | `gm_merge` | 合并两个同类型重复节点：keep 吸收 content/validatedCount/sessions + 去重边迁移；merge 节点被软删除（deprecated） | | `gm_stats` | 查看节点、关系、社区和 PageRank 统计 | -| `gm_maintain` | 执行去重、PageRank 和社区维护 | +| `gm_maintain` | 执行衰减评分、去重、PageRank 和社区维护 | ## 开发 From c3f6748278bd61516dda208e6dcb429533f65f31 Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Mon, 31 Aug 2026 00:38:58 +0000 Subject: [PATCH 13/29] feat: add LLM failure cooldown guard (persistent config errors only) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 仅 401/403/404 等持久性配置错误触发 10 分钟冷却；429/5xx 瞬时故障与 400/422 单条坏 prompt 不触发。冷却期内 LLM 调用快速失败，避免凭证失效、 模型名错误后每轮照付完整请求 + 超时。与 Neo4jGate 对偶：gate 保护 DB， guard 保护 LLM。三条 provider 路径（openai/anthropic/oauth）统一在 createCompleteFn 出口包裹。 移植自上游 pr/43 (2ad19c1 + 4d636d2)。 --- src/engine/llm-guard.ts | 64 +++++++++++++++++++++++++++++++++ src/engine/llm.ts | 23 +++++++++++- test/llm-guard.test.ts | 80 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 src/engine/llm-guard.ts create mode 100644 test/llm-guard.test.ts diff --git a/src/engine/llm-guard.ts b/src/engine/llm-guard.ts new file mode 100644 index 0000000..8090209 --- /dev/null +++ b/src/engine/llm-guard.ts @@ -0,0 +1,64 @@ +/** + * graph-memory-pro — LLM 失败冷却守卫 + * + * 与 Neo4jGate（DB 熔断）对偶：保护 LLM 依赖。 + * 仅对"持久性配置错误"触发冷却 —— 401/403/404（凭证失效、无权限、模型名/端点错误）， + * 这类错误不会自愈，逐轮重试只会浪费每次完整的请求 + 超时等待。 + * + * 不触发的情况： + * - 429/5xx：瞬时故障，fetchRetry 已内部重试，冷却反而放大抖动 + * - 400/422：可能只是单条坏 prompt（超长/格式问题），不能殃及后续正常调用 + * - 无状态码的错误（超时、空返回、缺配置）：交给各自的正常失败路径 + */ + +const RETRYABLE_STATUSES = new Set([429, 500, 502, 503, 529]); +const PAUSING_STATUSES = new Set([401, 403, 404]); + +/** + * 从错误消息中提取 HTTP 状态码。 + * 覆盖三条 provider 路径的报错格式： + * "[graph-memory] LLM API 401: …" （openai 兼容） + * "[graph-memory] Anthropic API 403: …" （anthropic） + * "[graph-memory] OAuth LLM API 401: …" （oauth，含 "LLM API 401" 子串） + */ +export function extractLlmStatus(error: unknown): number | null { + const text = String(error ?? ""); + const match = text.match(/\b(?:LLM|Anthropic) API (\d{3})\b/); + if (!match) return null; + return Number(match[1]); +} + +export class LlmFailureGuard { + private pausedUntil = 0; + + constructor( + private readonly cooldownMs = 10 * 60_000, + private readonly now: () => number = () => Date.now(), + ) {} + + canRun(): boolean { + return this.now() >= this.pausedUntil; + } + + remainingMs(): number { + return Math.max(0, this.pausedUntil - this.now()); + } + + /** 成功调用后清除冷却，下一次失败重新计时。 */ + reset(): void { + this.pausedUntil = 0; + } + + /** + * 按错误类型决定是否触发冷却。 + * 返回是否触发（调用方仅用于日志/测试；冷却本身幂等，重复触发取更晚到期时间）。 + */ + tripIfNeeded(error: unknown): boolean { + const status = extractLlmStatus(error); + if (status == null || RETRYABLE_STATUSES.has(status) || !PAUSING_STATUSES.has(status)) { + return false; + } + this.pausedUntil = Math.max(this.pausedUntil, this.now() + this.cooldownMs); + return true; + } +} diff --git a/src/engine/llm.ts b/src/engine/llm.ts index 2d7cce2..c0c5c67 100755 --- a/src/engine/llm.ts +++ b/src/engine/llm.ts @@ -24,6 +24,7 @@ */ import { stat } from "node:fs/promises"; +import { LlmFailureGuard } from "./llm-guard.ts"; import { loadOAuthSession, needsRefresh, @@ -185,7 +186,7 @@ export function createCompleteFn( return session; } - return async (system, user) => { + const complete = async (system: string, user: string): Promise => { // ── 路径 C：OAuth Codex Responses API ── if (provider === "oauth") { if (!oauthPath) { @@ -335,4 +336,24 @@ export function createCompleteFn( `. Raise llm.maxTokens if recurring.`, ); }; + + // ── 失败冷却守卫：持久性配置错误（401/403/404）后冷却 10 分钟，快速失败 ── + // 避免凭证失效/模型名错误时每轮照付一次完整请求 + 超时等待。成功调用即清除。 + const guard = new LlmFailureGuard(); + return async (system: string, user: string): Promise => { + if (!guard.canRun()) { + const seconds = Math.max(1, Math.ceil(guard.remainingMs() / 1000)); + throw new Error( + `[graph-memory] LLM paused for ${seconds}s after a previous permanent API error`, + ); + } + try { + const text = await complete(system, user); + guard.reset(); + return text; + } catch (err) { + guard.tripIfNeeded(err); + throw err; + } + }; } diff --git a/test/llm-guard.test.ts b/test/llm-guard.test.ts new file mode 100644 index 0000000..75e9df7 --- /dev/null +++ b/test/llm-guard.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; + +import { extractLlmStatus, LlmFailureGuard } from "../src/engine/llm-guard.ts"; + +describe("LlmFailureGuard", () => { + it("pauses after permanent 4xx API errors and recovers after cooldown", () => { + let now = 1_000; + const guard = new LlmFailureGuard(60_000, () => now); + + expect(guard.canRun()).toBe(true); + expect( + guard.tripIfNeeded(new Error('[graph-memory] LLM API 403: {"error":"User not found or inactive"}')), + ).toBe(true); + expect(guard.canRun()).toBe(false); + expect(guard.remainingMs()).toBe(60_000); + + now += 59_000; + expect(guard.canRun()).toBe(false); + + now += 2_000; + expect(guard.canRun()).toBe(true); + }); + + it("ignores retryable errors (fetchRetry already retried them)", () => { + const guard = new LlmFailureGuard(60_000, () => 1_000); + + expect(guard.tripIfNeeded(new Error("[graph-memory] LLM API 429: rate limited"))).toBe(false); + expect(guard.tripIfNeeded(new Error("[graph-memory] LLM API 503: upstream down"))).toBe(false); + expect(guard.canRun()).toBe(true); + }); + + it("does not pause for request-specific 400 and 422 errors", () => { + const guard = new LlmFailureGuard(60_000, () => 1_000); + + expect(guard.tripIfNeeded(new Error("[graph-memory] LLM API 400: prompt too long"))).toBe(false); + expect(guard.tripIfNeeded(new Error("[graph-memory] LLM API 422: invalid message"))).toBe(false); + expect(guard.canRun()).toBe(true); + }); + + it("does not pause for errors without an HTTP status (timeout, empty content, config)", () => { + const guard = new LlmFailureGuard(60_000, () => 1_000); + + expect(guard.tripIfNeeded(new Error("[graph-memory] LLM request timed out after 60000ms"))).toBe(false); + expect(guard.tripIfNeeded(new Error("[graph-memory] LLM returned empty content"))).toBe(false); + expect(guard.tripIfNeeded(new Error("[graph-memory] llm.provider=anthropic 但未配 llm.apiKey"))).toBe(false); + expect(guard.canRun()).toBe(true); + }); + + it("recognizes all three provider error formats (openai / anthropic / oauth)", () => { + expect(extractLlmStatus(new Error("[graph-memory] LLM API 401: invalid key"))).toBe(401); + expect(extractLlmStatus(new Error("[graph-memory] Anthropic API 403: forbidden"))).toBe(403); + expect(extractLlmStatus(new Error("[graph-memory] OAuth LLM API 404: model not found"))).toBe(404); + }); + + it("pauses Anthropic authentication failures", () => { + const guard = new LlmFailureGuard(60_000, () => 1_000); + + expect(guard.tripIfNeeded(new Error("[graph-memory] Anthropic API 401: invalid x-api-key"))).toBe(true); + expect(guard.canRun()).toBe(false); + }); + + it("repeated trips extend the pause instead of shortening it", () => { + let now = 1_000; + const guard = new LlmFailureGuard(60_000, () => now); + + guard.tripIfNeeded(new Error("[graph-memory] LLM API 401: expired")); + now += 30_000; + guard.tripIfNeeded(new Error("[graph-memory] LLM API 403: revoked")); + expect(guard.remainingMs()).toBe(60_000); + }); + + it("reset() clears the pause after a successful call", () => { + const guard = new LlmFailureGuard(60_000, () => 1_000); + + guard.tripIfNeeded(new Error("[graph-memory] LLM API 401: expired")); + expect(guard.canRun()).toBe(false); + guard.reset(); + expect(guard.canRun()).toBe(true); + }); +}); From 0afefdcd1de27a13cad4c005aab9ddb5de03f28c Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Mon, 31 Aug 2026 00:44:50 +0000 Subject: [PATCH 14/29] fix: guard duplicate register() and accept lowercase llm.baseUrl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 某些宿主版本会在未 dispose 的情况下重复调用 register()（热重载/配置变更）： 重复注册 hook/工具/路由让每轮工作翻倍，且新引擎实例持有自己那份 per-session 提取锁，与旧实例并行时同一 session 的提取互斥被打破。现在重复 register() 只重绑 ContextEngine 工厂并告警；dispose() 清空标记后才允许真正的重载。 守卫放在 CLI 元数据注册之后，保持 cli-metadata 模式不加载运行时的契约。 顺带：llm.baseUrl / embedding.baseUrl 小写拼写归一到 baseURL（显式 baseURL 优先），与上游 52db397 对齐。 --- index.ts | 35 +++++++++++++ test/register-guard.test.ts | 95 +++++++++++++++++++++++++++++++++++ test/session-identity.test.ts | 17 ++++++- 3 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 test/register-guard.test.ts diff --git a/index.ts b/index.ts index 4a29eb4..0389676 100755 --- a/index.ts +++ b/index.ts @@ -245,6 +245,15 @@ export function prepareAssemblyMessages( // ─── 插件对象 ───────────────────────────────────────────────── +/** + * 当前活跃引擎（防重复注册守卫）。 + * 某些宿主版本会在未调用 dispose 的情况下重复调用 register()（热重载、配置变更）： + * 重新注册全套 hook/工具/路由会让每轮工作翻倍，且新引擎实例持有自己那份 + * per-session 提取锁，与旧实例并行时同一 session 的提取互斥被打破。 + * 只有 dispose() 清空标记（真正的重载）后，下一次 register() 才走完整初始化。 + */ +let activeEngine: { dispose: () => Promise | void } | null = null; + const graphMemoryProPlugin = { id: "graph-memory-pro", name: "Graph Memory Pro", @@ -279,10 +288,32 @@ const graphMemoryProPlugin = { return; } + // 防重复注册：CLI 元数据仍可重复注册（幂等），但运行时只允许一份。 + // 复用现有引擎，只重绑 ContextEngine 工厂（见 activeEngine 上的说明）。 + if (activeEngine) { + api.registerContextEngine("graph-memory-pro", () => activeEngine); + api.logger.warn("[graph-memory-pro] duplicate register() ignored; reusing active engine"); + return; + } + const cfg: GmConfig = { ...DEFAULT_CONFIG, ...raw }; if (raw.neo4j) cfg.neo4j = { ...DEFAULT_CONFIG.neo4j, ...raw.neo4j }; if (raw.decay) cfg.decay = { ...DEFAULT_CONFIG.decay, ...raw.decay }; if (raw.cron) cfg.cron = { ...DEFAULT_CONFIG.cron, ...raw.cron }; + + // 拼写兼容：接受小写 baseUrl（部分宿主/用户的配置习惯），统一归一到 baseURL。 + // 显式 baseURL 优先；trim 后为空视为未配置。 + const rawLlm = (raw.llm ?? {}) as Record; + if (cfg.llm && !cfg.llm.baseURL && typeof rawLlm.baseUrl === "string" && rawLlm.baseUrl.trim()) { + cfg.llm = { ...cfg.llm, baseURL: rawLlm.baseUrl.trim() }; + } + const rawEmbedding = (raw.embedding ?? {}) as Record; + if ( + cfg.embedding && !cfg.embedding.baseURL && + typeof rawEmbedding.baseUrl === "string" && rawEmbedding.baseUrl.trim() + ) { + cfg.embedding = { ...cfg.embedding, baseURL: rawEmbedding.baseUrl.trim() }; + } const cronCfg = cfg.cron ?? DEFAULT_CRON_CONFIG; const providerModel = readDefaultModel(api.config); @@ -974,10 +1005,14 @@ const graphMemoryProPlugin = { sessionIdsByKey.clear(); pendingSubagentRecall.clear(); ingestedSinceTurn.clear(); + // 仅当释放的是当前活跃引擎时清空标记 —— 真正的重载（dispose 后重新 + // register）才会走完整初始化路径 + if (activeEngine === engine) activeEngine = null; // 不关闭 Neo4j driver — 连接池自管理生命周期，进程退出时由 OS 回收 }, }; + activeEngine = engine; api.registerContextEngine("graph-memory-pro", () => engine); // ── session_end：finalize + 图维护 ────────────────────── diff --git a/test/register-guard.test.ts b/test/register-guard.test.ts new file mode 100644 index 0000000..9656f77 --- /dev/null +++ b/test/register-guard.test.ts @@ -0,0 +1,95 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import graphMemoryProPlugin from "../index.ts"; +import { closeDriver } from "../src/store/db.ts"; + +/** 完整运行时模式的 fake api（覆盖 register() 用到的全部方法）。 */ +function fullApi(pluginConfig: Record = {}) { + return { + pluginConfig, + config: {}, + resolvePath: (v: string) => v, + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + registerCli: vi.fn(), + registerTool: vi.fn(), + registerContextEngine: vi.fn(), + registerHttpRoute: vi.fn(), + on: vi.fn(), + } as any; +} + +describe("duplicate register() guard", () => { + let createdEngine: { dispose: () => Promise } | null = null; + + afterEach(async () => { + if (createdEngine) { + await createdEngine.dispose(); + createdEngine = null; + } + await closeDriver(); + }); + + it("reuses the active engine when the host registers twice without dispose", () => { + const api1 = fullApi(); + graphMemoryProPlugin.register(api1); + expect(api1.registerContextEngine).toHaveBeenCalledTimes(1); + createdEngine = api1.registerContextEngine.mock.calls[0][1](); + + // 第二次 register：只重绑引擎工厂，不再注册工具/路由/hook + const api2 = fullApi(); + graphMemoryProPlugin.register(api2); + expect(api2.registerContextEngine).toHaveBeenCalledTimes(1); + expect(api2.registerTool).not.toHaveBeenCalled(); + expect(api2.registerHttpRoute).not.toHaveBeenCalled(); + expect(api2.on).not.toHaveBeenCalled(); + expect(api2.registerContextEngine.mock.calls[0][1]()).toBe(createdEngine); + expect(api2.logger.warn).toHaveBeenCalledWith( + expect.stringContaining("duplicate register() ignored"), + ); + }); + + it("creates a fresh engine after dispose() (genuine reload)", async () => { + const api1 = fullApi(); + graphMemoryProPlugin.register(api1); + const engineA = api1.registerContextEngine.mock.calls[0][1](); + createdEngine = engineA; + await engineA.dispose(); + + const api2 = fullApi(); + graphMemoryProPlugin.register(api2); + const engineB = api2.registerContextEngine.mock.calls[0][1](); + expect(engineB).not.toBe(engineA); + expect(api2.registerTool).toHaveBeenCalled(); + // afterEach 清理的是"最后创建"的引擎（engineB 才是当前活跃的） + createdEngine = engineB; + }); + + it("normalizes lowercase llm.baseUrl to baseURL before provider resolution", () => { + // 只有 apiKey 时启发式推断为 anthropic；归一生效后 baseURL 存在 → 推断为 openai + const api = fullApi({ llm: { apiKey: "k", baseUrl: "http://localhost:8080/v1/" } }); + graphMemoryProPlugin.register(api); + createdEngine = api.registerContextEngine.mock.calls[0]?.[1]?.() ?? null; + + expect(api.logger.warn).toHaveBeenCalledWith( + expect.stringContaining('推断为 "openai"'), + ); + }); + + it("explicit baseURL wins over baseUrl spelling", () => { + const api = fullApi({ + llm: { apiKey: "k", baseURL: "http://explicit/v1", baseUrl: "http://lowercase/v1" }, + }); + graphMemoryProPlugin.register(api); + createdEngine = api.registerContextEngine.mock.calls[0]?.[1]?.() ?? null; + + // 显式配置优先 —— 这里只验证 register 未被拼写兼容逻辑破坏（推断告警产生一次） + expect(api.logger.warn).toHaveBeenCalledWith( + expect.stringContaining('推断为 "openai"'), + ); + expect( + api.logger.warn.mock.calls.filter((args: unknown[]) => + String(args[0]).includes("llm.provider 未显式设置"), + ), + ).toHaveLength(1); + }); +}); diff --git a/test/session-identity.test.ts b/test/session-identity.test.ts index 4068ec3..9df14e7 100644 --- a/test/session-identity.test.ts +++ b/test/session-identity.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; import { isCronSessionKey } from "../src/types.ts"; const mocks = vi.hoisted(() => ({ @@ -121,9 +121,16 @@ type EngineHarness = { }) => Promise<{ readonly rollback: () => void }>; }; +// register() 带防重复注册守卫（模块级 activeEngine）：同一进程内未 dispose 的 +// 二次 register 会被拦截复用。本文件每个用例都注册一个带独立 pluginConfig +// 的隔离引擎 —— 在重复注册前先 dispose 上一个，保持逐用例隔离。 +let previousEngine: { dispose?: () => Promise | void } | null = null; + function registerPlugin(pluginConfig: Record = {}): { readonly hooks: Map; readonly engine: EngineHarness } { const hooks = new Map(); let engine: EngineHarness | undefined; + previousEngine?.dispose?.(); + previousEngine = null; graphMemoryProPlugin.register({ logger: { debug: () => {}, @@ -140,9 +147,17 @@ function registerPlugin(pluginConfig: Record = {}): { readonly registerHttpRoute: () => {}, }); if (!engine) throw new Error("context engine was not registered"); + previousEngine = engine; return { hooks, engine }; } +afterAll(async () => { + // 释放最后一个引擎，清掉模块级 activeEngine —— vitest singleFork 下 + // 所有测试文件共享进程，不能把注册状态泄漏给后续文件 + await previousEngine?.dispose?.(); + previousEngine = null; +}); + describe("session identity", () => { beforeEach(() => { mocks.getBySession.mockClear(); From bba42eb143060410b5762146d319e7ba48588329 Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Mon, 31 Aug 2026 01:18:26 +0000 Subject: [PATCH 15/29] feat: add opt-in bounded raw-message retention (messageRetention) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 移植自上游 #96（SQLite → Neo4j Cypher）。原则不变：上下文压缩不构成 删除持久证据的授权 —— 默认 keep=all 零行为变化，必须显式 opt-in。 - keep=referenced：只删已提取完成的消息（知识已固化进图谱）；v2.0 无消息级 出处边，extracted=true 即等价上游'无 gm_node_sources 引用'前置条件 - keep=recent：叠加时间窗（每 session 最近 N 轮真实用户发言 + 最近 N 天） - 每 batch 有界（默认 500 行/维护周期），dryRun 只报候选不删 - 候选选择与删除同一写事务，DELETE 前按 extracted 重新校验 - 挂在 runMaintenance 尾部；register 时预校验配置（非法 fail closed 回退 keep=all） - openclaw.plugin.json configSchema 声明 messageRetention 测试安全加固（本地误连真实库事故的整改）： - integration.retention.test.ts 必须显式提供 NEO4J_TEST_URI（不默认 7687） - 隔离守卫：库内存在非测试前缀 GmMessage 即拒绝运行 - ci.yml 显式设置 NEO4J_TEST_URI（ephemeral runner 安全） - 本地测试容器固定 7688 端口 --- .github/workflows/ci.yml | 3 + index.ts | 18 +++ openclaw.plugin.json | 16 ++ src/graph/maintenance.ts | 19 ++- src/store/retention.ts | 243 +++++++++++++++++++++++++++++ src/types.ts | 27 ++++ test/integration.retention.test.ts | 198 +++++++++++++++++++++++ test/retention-policy.test.ts | 60 +++++++ 8 files changed, 583 insertions(+), 1 deletion(-) create mode 100644 src/store/retention.ts create mode 100644 test/integration.retention.test.ts create mode 100644 test/retention-policy.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8207acb..2c8706b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,6 +91,9 @@ jobs: run: npm test env: NEO4J_INTEGRATION: "1" + # 集成测试（尤其删除类的 retention 测试）要求显式指定 URI； + # runner 上的 service container 是一次性实例，7687 安全 + NEO4J_TEST_URI: bolt://localhost:7687 CI: "true" shellcheck: diff --git a/index.ts b/index.ts index 0389676..2b49665 100755 --- a/index.ts +++ b/index.ts @@ -25,6 +25,7 @@ import { Extractor } from "./src/extractor/extract.ts"; import { assembleContext } from "./src/format/assemble.ts"; import { sanitizeToolUseResultPairing } from "./src/format/transcript-repair.ts"; import { runMaintenance } from "./src/graph/maintenance.ts"; +import { normalizeMessageRetentionPolicy } from "./src/store/retention.ts"; import { DEFAULT_CONFIG, DEFAULT_CRON_CONFIG, isCronSessionKey, type GmConfig, type RecallResult, type EdgeType } from "./src/types.ts"; import { registerCrudRoutes } from "./src/routes/crud.ts"; import { createGraphMemoryCli } from "./src/cli.ts"; @@ -314,6 +315,19 @@ const graphMemoryProPlugin = { ) { cfg.embedding = { ...cfg.embedding, baseURL: rawEmbedding.baseUrl.trim() }; } + + // messageRetention 配置预校验：非法策略在启动时报错并回退 keep=all（fail closed）， + // 而不是等到 session_end 维护链里每次抛错 + if (cfg.messageRetention) { + try { + normalizeMessageRetentionPolicy(cfg.messageRetention); + } catch (err) { + api.logger.error( + `[graph-memory-pro] messageRetention 配置非法，保留策略回退为 keep=all（不删除任何消息）：${err}`, + ); + delete cfg.messageRetention; + } + } const cronCfg = cfg.cron ?? DEFAULT_CRON_CONFIG; const providerModel = readDefaultModel(api.config); @@ -670,6 +684,10 @@ const graphMemoryProPlugin = { `[graph-memory-pro] maintenance: ${result.durationMs}ms, ` + `dedup=${result.dedup.merged}, communities=${result.community.count}, ` + `summaries=${result.communitySummaries}, ` + + (result.retention + ? `retention=${result.retention.dryRun ? "dryRun:" : ""}` + + `${result.retention.deletedRows}/${result.retention.selectedRows} msgs, ` + : "") + `top_pr=${result.pagerank.topK.slice(0, 3).map(n => `${n.name}(${n.score.toFixed(3)})`).join(",")}`, ); } while (maintenanceRerunRequested && neo4jGate.isAvailable()); diff --git a/openclaw.plugin.json b/openclaw.plugin.json index 9a40a38..2f0e275 100755 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -60,6 +60,22 @@ "finalizeAndMaintain": { "type": "boolean", "default": true, "description": "cron session 结束时是否执行 finalize（EVENT→SKILL 晋升）和图维护（decay/PageRank/社区检测）。频繁的 cron 任务可关闭以避免每次结束都跑全局维护。" } } }, + "messageRetention": { + "type": "object", + "description": "原始消息（GmMessage）有界保留策略。默认 keep=all 永不删除；上下文压缩不构成删除持久证据的授权。挂在图维护链尾部，每个维护周期最多处理 batchSize 行。建议先配 dryRun=true 观察候选集再实际启用。", + "properties": { + "keep": { + "type": "string", + "enum": ["all", "referenced", "recent"], + "default": "all", + "description": "all=全部保留（默认）；referenced=只删已提取完成的消息（知识已固化进图谱）；recent=在 referenced 基础上按时间窗保护最近内容（需配 recentTurns 或 retentionDays）" + }, + "recentTurns": { "type": "number", "default": 0, "description": "keep=recent：每 session 保留最近 N 轮真实用户发言（该轮及其后的全部消息保留）。" }, + "retentionDays": { "type": "number", "default": 0, "description": "keep=recent：保留最近 N 天内入库的消息。" }, + "batchSize": { "type": "number", "default": 500, "description": "单个维护周期最多处理的行数（1~10000），保证维护链工作有界。" }, + "dryRun": { "type": "boolean", "default": false, "description": "true 时只报告候选集不删除 —— 启用前先跑一轮 dryRun 验证。" } + } + }, "llm": { "type": "object", "properties": { diff --git a/src/graph/maintenance.ts b/src/graph/maintenance.ts index a2e68b8..c2fc1c3 100755 --- a/src/graph/maintenance.ts +++ b/src/graph/maintenance.ts @@ -2,7 +2,7 @@ * graph-memory-pro — 图谱维护 * * 调用时机：session_end（finalize 之后） - * 执行顺序：衰减 → 去重 → 全局 PageRank → 社区检测 → 社区描述 + * 执行顺序：衰减 → 去重 → 全局 PageRank → 社区检测 → 社区描述 → 消息保留（opt-in） */ import type { Driver } from "neo4j-driver"; @@ -13,6 +13,10 @@ import { computeGlobalPageRank, type GlobalPageRankResult } from "./pagerank.ts" import { detectCommunities, summarizeCommunities, type CommunityResult } from "./community.ts"; import { dedup, type DedupResult } from "./dedup.ts"; import { applyDecay, type DecayResult } from "./decay.ts"; +import { + normalizeMessageRetentionPolicy, runMessageRetention, + type MessageRetentionResult, +} from "../store/retention.ts"; export interface MaintenanceResult { decay: DecayResult; @@ -20,6 +24,8 @@ export interface MaintenanceResult { pagerank: GlobalPageRankResult; community: CommunityResult; communitySummaries: number; + /** 原始消息保留（opt-in）；未配置 messageRetention 或 keep=all 时为 undefined。 */ + retention?: MessageRetentionResult; durationMs: number; } @@ -48,12 +54,23 @@ export async function runMaintenance( } catch {} } + // 5. 原始消息保留（opt-in；keep=all 零开销直返）。 + // 非法策略 fail closed：抛错终止维护、不删任何东西，由调用方记录。 + let retention: MessageRetentionResult | undefined; + if (cfg.messageRetention) { + const policy = normalizeMessageRetentionPolicy(cfg.messageRetention); + if (policy.keep !== "all") { + retention = await runMessageRetention(driver, policy); + } + } + return { decay: decayResult, dedup: dedupResult, pagerank: pagerankResult, community: communityResult, communitySummaries, + ...(retention ? { retention } : {}), durationMs: Date.now() - start, }; } diff --git a/src/store/retention.ts b/src/store/retention.ts new file mode 100644 index 0000000..6c1914c --- /dev/null +++ b/src/store/retention.ts @@ -0,0 +1,243 @@ +/** + * graph-memory-pro — 原始消息有界保留（opt-in） + * + * 移植自上游 #96（SQLite → Neo4j Cypher）。设计原则不变： + * "上下文压缩改变的是模型可见面，不构成删除持久证据的授权。" + * + * - 默认 keep=all：不做任何删除，本模块在维护链中零开销直返。 + * - referenced：只删"已提取完成"的消息 —— 知识已固化进图谱节点/边，原始文本退役。 + * - recent：referenced 之上叠加时间窗保护（每 session 最近 N 轮真实用户发言 + * 及其后消息、最近 N 天内入库的消息），验证时要求至少配置一个窗口参数。 + * + * 与上游的差异：v2.0 schema 没有消息级出处边（节点仅记 sourceSessions， + * 粒度为 session），上游 "无 gm_node_sources 引用" 的前置条件在这里等价于 + * extracted=true。DELETE 仍按 extracted 重新校验，防止候选查询与删除语义 + * 未来漂移 —— 候选集不能成为删除的授权。 + */ + +import { createHash } from "node:crypto"; +import { int } from "neo4j-driver"; +import type { Driver } from "neo4j-driver"; +import { getSession } from "./db.ts"; +import type { MessageRetentionConfig } from "../types.ts"; + +export type NormalizedMessageRetentionPolicy = Required; + +export interface MessageRetentionResult { + policy: string; + policyRevision: string; + dryRun: boolean; + selectedRows: number; + selectedBytes: number; + deletedRows: number; + deletedBytes: number; + selectedSessions: number; + byRole: Record; + oldestCreatedAt: number | null; + newestCreatedAt: number | null; + hasMore: boolean; + cutoffAt: number; + durationMs: number; +} + +interface CandidateRow { + id: string; + sessionId: string; + role: string; + createdAt: number; + contentBytes: number; +} + +function boundedInteger( + value: unknown, + name: string, + fallback: number, + minimum: number, + maximum: number, +): number { + if (value === undefined) return fallback; + if (!Number.isInteger(value) || Number(value) < minimum || Number(value) > maximum) { + throw new TypeError( + `[graph-memory] messageRetention.${name} must be an integer between ${minimum} and ${maximum}, received ${String(value)}`, + ); + } + return Number(value); +} + +/** 配置校验。非法策略 fail closed（抛错 → 不删任何东西）；register 时提前验证以给出友好报错。 */ +export function normalizeMessageRetentionPolicy( + input: MessageRetentionConfig | undefined, +): NormalizedMessageRetentionPolicy { + if (input === undefined) return { keep: "all", recentTurns: 0, retentionDays: 0, batchSize: 500, dryRun: false }; + if (!input || typeof input !== "object" || Array.isArray(input)) { + throw new TypeError("[graph-memory] messageRetention must be an object"); + } + + const keep = input.keep ?? "all"; + if (keep !== "all" && keep !== "referenced" && keep !== "recent") { + throw new TypeError( + `[graph-memory] messageRetention.keep must be all, referenced, or recent, received ${String(keep)}`, + ); + } + const recentTurns = boundedInteger(input.recentTurns, "recentTurns", 0, 0, 100_000); + const retentionDays = boundedInteger(input.retentionDays, "retentionDays", 0, 0, 36_500); + const batchSize = boundedInteger(input.batchSize, "batchSize", 500, 1, 10_000); + const dryRun = input.dryRun ?? false; + if (typeof dryRun !== "boolean") { + throw new TypeError( + `[graph-memory] messageRetention.dryRun must be a boolean, received ${String(dryRun)}`, + ); + } + if (keep === "recent" && recentTurns === 0 && retentionDays === 0) { + throw new TypeError( + "[graph-memory] messageRetention.keep=recent requires recentTurns or retentionDays", + ); + } + + return { keep, recentTurns, retentionDays, batchSize, dryRun }; +} + +export function messageRetentionPolicyRevision( + policy: NormalizedMessageRetentionPolicy, +): string { + return createHash("sha256") + .update(JSON.stringify(policy)) + .digest("hex") + .slice(0, 12); +} + +function emptyResult(policy: NormalizedMessageRetentionPolicy, cutoffAt: number, start: number): MessageRetentionResult { + return { + policy: policy.keep, + policyRevision: messageRetentionPolicyRevision(policy), + dryRun: policy.dryRun, + selectedRows: 0, + selectedBytes: 0, + deletedRows: 0, + deletedBytes: 0, + selectedSessions: 0, + byRole: {}, + oldestCreatedAt: null, + newestCreatedAt: null, + hasMore: false, + cutoffAt, + durationMs: Date.now() - start, + }; +} + +/** + * 构造候选查询。 + * + * recentTurns 保护：以 session 内最近 N 条 user 消息里最旧一条的 turnIndex 为界， + * 只候选更早的消息（该轮的 assistant/tool 消息随所属轮次保留）； + * 没有任何 user 消息的 session 完全保护 —— 与上游 LEFT JOIN 的 NULL 语义一致。 + * createdAt 缺失/为零/为负/超前一律 fail closed（不进候选）。 + */ +function buildCandidateQuery( + policy: NormalizedMessageRetentionPolicy, + now: number, +): { cypher: string; params: Record } { + const params: Record = { + // JS number 会以 Float 发送 —— LIMIT 与列表下标要求整数，必须用 neo4j int() + recentTurns: int(policy.recentTurns), + batchLimit: int(policy.batchSize + 1), + }; + + let match = "MATCH (m:GmMessage)\nWHERE m.extracted = true\n"; + if (policy.recentTurns > 0) { + match = + "MATCH (u:GmMessage {role: 'user'})\n" + + "WITH u.sessionId AS sid, u ORDER BY sid, u.turnIndex DESC\n" + + "WITH sid, collect(u.turnIndex) AS turns\n" + + "WITH sid, CASE WHEN size(turns) >= $recentTurns THEN turns[$recentTurns - 1] ELSE -1 END AS cutoffTurn\n" + + "MATCH (m:GmMessage {sessionId: sid})\n" + + "WHERE m.extracted = true\n" + + " AND m.turnIndex < cutoffTurn\n"; + } + if (policy.retentionDays > 0) { + params.ageCutoff = now - policy.retentionDays * 86_400_000; + match += " AND m.createdAt > 0 AND m.createdAt < $ageCutoff\n"; + } + + const cypher = + match + + "WITH m ORDER BY m.createdAt, m.sessionId, m.turnIndex\n" + + "LIMIT $batchLimit\n" + + "RETURN m.id AS id, m.sessionId AS sessionId, m.role AS role, " + + "m.createdAt AS createdAt, size(m.content) AS contentBytes"; + return { cypher, params }; +} + +/** + * 跑一个有界批次。候选选择与删除放在同一个写事务里冻结边界； + * 删除前按 extracted 重新校验（候选查询不是删除的授权）。 + * size(m.content) 返回字符数（近似体积），与上游 BLOB 字节数略有差异，仅用于统计。 + */ +export async function runMessageRetention( + driver: Driver, + policy: NormalizedMessageRetentionPolicy, + now: number = Date.now(), +): Promise { + const start = Date.now(); + if (policy.keep === "all") return emptyResult(policy, now, start); + + const { cypher, params } = buildCandidateQuery(policy, now); + + const session = getSession(driver); + try { + const { rows, deletedRows, hasMore } = await session.executeWrite(async (tx) => { + const candRes = await tx.run(cypher, params); + const rows: CandidateRow[] = candRes.records.map((r) => ({ + id: r.get("id"), + sessionId: r.get("sessionId"), + role: r.get("role"), + createdAt: Number(r.get("createdAt")), + contentBytes: Number(r.get("contentBytes")), + })); + + let hasMore = false; + if (rows.length > policy.batchSize) { + hasMore = true; + rows.pop(); + } + + let deletedRows = 0; + if (!policy.dryRun && rows.length) { + const delRes = await tx.run( + "UNWIND $ids AS mid " + + "MATCH (m:GmMessage {id: mid}) " + + "WHERE m.extracted = true " + + "DETACH DELETE m " + + "RETURN count(m) AS deleted", + { ids: rows.map((r) => r.id) }, + ); + deletedRows = delRes.records[0]?.get("deleted").toNumber() ?? 0; + } + return { rows, deletedRows, hasMore }; + }); + + const selectedBytes = rows.reduce((total, r) => total + r.contentBytes, 0); + const byRole: Record = {}; + for (const r of rows) byRole[r.role] = (byRole[r.role] ?? 0) + 1; + const created = rows.map((r) => r.createdAt).filter(Number.isFinite); + + return { + policy: policy.keep, + policyRevision: messageRetentionPolicyRevision(policy), + dryRun: policy.dryRun, + selectedRows: rows.length, + selectedBytes, + deletedRows, + deletedBytes: deletedRows === rows.length ? selectedBytes : 0, + selectedSessions: new Set(rows.map((r) => r.sessionId)).size, + byRole, + oldestCreatedAt: created.length ? Math.min(...created) : null, + newestCreatedAt: created.length ? Math.max(...created) : null, + hasMore, + cutoffAt: now, + durationMs: Date.now() - start, + }; + } finally { + await session.close(); + } +} diff --git a/src/types.ts b/src/types.ts index a264a1f..280752a 100755 --- a/src/types.ts +++ b/src/types.ts @@ -207,6 +207,31 @@ export const DEFAULT_CRON_CONFIG: CronConfig = { finalizeAndMaintain: true, }; +// ─── 原始消息保留（opt-in 有界清理）────────────────────────── +// +// 设计原则（移植自上游 #96）："上下文压缩改变的是模型可见面， +// 不构成删除持久证据的授权。" 默认 keep=all 零行为变化； +// 清理逻辑实现在 src/store/retention.ts，挂在 runMaintenance 尾部。 + +export type MessageRetentionMode = "all" | "referenced" | "recent"; + +export interface MessageRetentionConfig { + /** + * all（默认）保留全部原始消息，零开销； + * referenced 只删"已提取完成"的消息（知识已固化进图谱，原始文本退役）； + * recent 在 referenced 的基础上按时间窗保护最近内容。 + */ + keep?: MessageRetentionMode; + /** keep=recent：每个 session 保留最近 N 轮真实用户发言（该轮及其后全部保留）。 */ + recentTurns?: number; + /** keep=recent：保留最近 N 天内入库的消息。 */ + retentionDays?: number; + /** 单个维护周期最多处理的行数（保证维护链工作有界）。默认 500。 */ + batchSize?: number; + /** true 时只报告候选集不删除 —— 启用前先跑一轮 dryRun 验证。 */ + dryRun?: boolean; +} + // ─── 插件配置 ───────────────────────────────────────────────── export interface GmConfig { @@ -237,6 +262,8 @@ export interface GmConfig { /** 遗忘曲线衰减配置；未提供时使用 DEFAULT_CONFIG.decay。 */ decay?: DecayConfig; cron?: CronConfig; + /** 原始消息保留策略；未提供时等价 keep=all（永不删除）。 */ + messageRetention?: MessageRetentionConfig; } export const DEFAULT_CONFIG: GmConfig = { diff --git a/test/integration.retention.test.ts b/test/integration.retention.test.ts new file mode 100644 index 0000000..397a797 --- /dev/null +++ b/test/integration.retention.test.ts @@ -0,0 +1,198 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import type { Driver } from "neo4j-driver"; +import { getDriver, initSchema, closeDriver, getSession } from "../src/store/db.ts"; +import { saveMessage, markExtracted } from "../src/store/store.ts"; +import { normalizeMessageRetentionPolicy, runMessageRetention } from "../src/store/retention.ts"; + +// 仅在显式提供 NEO4J_TEST_URI 时运行（需要 Docker Neo4j，CI 执行）。 +// 刻意不默认 bolt://localhost:7687：本测试会删除数据，绝不允许 +// 因缺省值静默落到可能是真实数据库的端口上。 +const NEO4J_URI = process.env.NEO4J_TEST_URI; +const ENABLED = !!process.env.NEO4J_INTEGRATION && !!NEO4J_URI; +const TEST_PREFIX = "retention-"; + +let driver: Driver; +const SID_A = `retention-a-${Date.now()}`; // referenced 场景：前半提取、后半未提取 +const SID_B = `retention-b-${Date.now()}`; // recentTurns 场景：全部提取 +const SID_C = `retention-c-${Date.now()}`; // retentionDays 场景：无 user 消息（recentTurns 下全保护） +const SID_D = `retention-d-${Date.now()}`; // 无 user 消息的 session +const DAY = 86_400_000; + +async function seedSession(sid: string, turns: number, roles: (t: number) => string): Promise { + for (let t = 0; t < turns; t++) { + await saveMessage(driver, sid, t, roles(t), { text: `msg-${sid}-${t}` }); + } +} + +async function survivingTurns(sid: string): Promise { + const session = getSession(driver); + try { + const res = await session.run( + "MATCH (m:GmMessage {sessionId: $sid}) RETURN m.turnIndex AS t ORDER BY t", + { sid }, + ); + return res.records.map((r) => Number(r.get("t"))); + } finally { + await session.close(); + } +} + +async function setCreatedAt(sid: string, turn: number, ts: number): Promise { + const session = getSession(driver); + try { + await session.run( + "MATCH (m:GmMessage {sessionId: $sid, turnIndex: $turn}) SET m.createdAt = $ts", + { sid, turn, ts }, + ); + } finally { + await session.close(); + } +} + +describe.skipIf(!ENABLED)("GmMessage retention (Docker)", () => { + beforeAll(async () => { + driver = getDriver({ uri: NEO4J_URI!, user: "neo4j", password: "graphmemory" }); + await initSchema(driver); + + // 隔离守卫：库内存在任何非本测试前缀的 GmMessage 即中止 —— + // 这是删除类测试，绝不能在有真实数据的库上执行 + const guard = getSession(driver); + try { + const res = await guard.run( + "MATCH (m:GmMessage) WHERE NOT m.sessionId STARTS WITH $prefix RETURN count(m) AS n", + { prefix: TEST_PREFIX }, + ); + const foreign = res.records[0].get("n").toNumber(); + if (foreign > 0) { + throw new Error( + `[retention-test] 目标数据库含 ${foreign} 条非测试前缀消息，疑似真实数据库 —— 拒绝运行。` + + `请确认 NEO4J_TEST_URI 指向一次性测试实例。`, + ); + } + // 清理历史残留（前次中断的运行），保证候选集可预测 + await guard.run( + "MATCH (m:GmMessage) WHERE m.sessionId STARTS WITH $prefix DETACH DELETE m", + { prefix: TEST_PREFIX }, + ); + } finally { + await guard.close(); + } + + // A：10 条（偶数轮 user / 奇数轮 assistant），turns 0-5 已提取，6-9 未提取 + await seedSession(SID_A, 10, (t) => (t % 2 === 0 ? "user" : "assistant")); + await markExtracted(driver, SID_A, 5); + // B：10 条全部提取，角色同 A（user 轮 0/2/4/6/8） + await seedSession(SID_B, 10, (t) => (t % 2 === 0 ? "user" : "assistant")); + await markExtracted(driver, SID_B, 9); + // C：4 条全部 assistant（无 user 消息）、全部提取；turns 0/1 一百天前，2/3 昨天 + await seedSession(SID_C, 4, () => "assistant"); + await markExtracted(driver, SID_C, 3); + await setCreatedAt(SID_C, 0, Date.now() - 100 * DAY); + await setCreatedAt(SID_C, 1, Date.now() - 100 * DAY); + await setCreatedAt(SID_C, 2, Date.now() - 1 * DAY); + await setCreatedAt(SID_C, 3, Date.now() - 1 * DAY); + // D：2 条 assistant、全部提取、无 user 消息 + await seedSession(SID_D, 2, () => "assistant"); + await markExtracted(driver, SID_D, 1); + }); + + afterAll(async () => { + const s = getSession(driver); + try { + await s.run( + "MATCH (m:GmMessage) WHERE m.sessionId STARTS WITH $prefix DETACH DELETE m", + { prefix: TEST_PREFIX }, + ); + } finally { + await s.close(); + } + await closeDriver(); + }); + + it("keep=all is a no-op that never touches the database", async () => { + const policy = normalizeMessageRetentionPolicy({ keep: "all" }); + const result = await runMessageRetention(driver, policy); + expect(result.selectedRows).toBe(0); + expect(result.deletedRows).toBe(0); + expect(await survivingTurns(SID_A)).toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + }); + + it("retentionDays window only removes extracted messages older than the cutoff", async () => { + const policy = normalizeMessageRetentionPolicy({ keep: "recent", retentionDays: 30 }); + const result = await runMessageRetention(driver, policy); + + expect(result.policy).toBe("recent"); + expect(result.deletedRows).toBeGreaterThanOrEqual(2); // 至少 C 的 turns 0/1 + // C：只有一百天前的 turns 0/1 被删，昨天的 2/3 保留 + expect(await survivingTurns(SID_C)).toEqual([2, 3]); + // A/B/D 全部保留（createdAt 都是新写入的） + expect(await survivingTurns(SID_A)).toHaveLength(10); + expect(await survivingTurns(SID_B)).toHaveLength(10); + expect(await survivingTurns(SID_D)).toHaveLength(2); + }); + + it("recentTurns window protects the newest user turns and unextracted messages", async () => { + const policy = normalizeMessageRetentionPolicy({ keep: "recent", recentTurns: 2 }); + const result = await runMessageRetention(driver, policy); + + // cutoff = 第 2 新 user 轮的 turnIndex（user 轮 0/2/4/6/8 → cutoff=6），该轮及其后保留 + expect(result.deletedRows).toBeGreaterThanOrEqual(12); // A/B 各删 turns 0-5 + // A：turns 0-5（已提取、窗口外）删；6-9 留（6/8 在窗口内，7/9 未提取） + expect(await survivingTurns(SID_A)).toEqual([6, 7, 8, 9]); + // B：同结构全提取，turns 0-5 删、6-9 留 + expect(await survivingTurns(SID_B)).toEqual([6, 7, 8, 9]); + // C/D 没有 user 消息 → 完全保护（与上游 NULL cutoff 语义一致） + expect(await survivingTurns(SID_C)).toEqual([2, 3]); + expect(await survivingTurns(SID_D)).toEqual([0, 1]); + }); + + it("dryRun reports candidates without deleting", async () => { + const policy = normalizeMessageRetentionPolicy({ keep: "referenced", dryRun: true }); + const result = await runMessageRetention(driver, policy); + + expect(result.dryRun).toBe(true); + expect(result.deletedRows).toBe(0); + // 候选 = 剩余全部已提取：B(6-9) + C(2,3) + D(0,1) = 8（A 的 6-9 未提取不算） + expect(result.selectedRows).toBe(8); + expect(await survivingTurns(SID_A)).toEqual([6, 7, 8, 9]); + expect(await survivingTurns(SID_B)).toEqual([6, 7, 8, 9]); + }); + + it("keep=referenced removes remaining extracted messages and never unextracted ones", async () => { + const policy = normalizeMessageRetentionPolicy({ keep: "referenced" }); + const result = await runMessageRetention(driver, policy); + + expect(result.deletedRows).toBe(8); + // A 只剩未提取的 turns 6-9 —— 未提取消息永远不进候选 + expect(await survivingTurns(SID_A)).toEqual([6, 7, 8, 9]); + expect(await survivingTurns(SID_B)).toEqual([]); + expect(await survivingTurns(SID_C)).toEqual([]); + expect(await survivingTurns(SID_D)).toEqual([]); + }); + + it("reports per-role stats and session spread", async () => { + // 上一测试后 A 仅剩 turns 6-9（user 6/8, assistant 7/9）——再跑一轮 referenced 空转 + const policy = normalizeMessageRetentionPolicy({ keep: "referenced" }); + const result = await runMessageRetention(driver, policy); + expect(result.selectedRows).toBeGreaterThanOrEqual(0); + expect(result.cutoffAt).toBeGreaterThan(0); + expect(result.durationMs).toBeGreaterThanOrEqual(0); + expect(result.policyRevision).toMatch(/^[0-9a-f]{12}$/); + }); + + it("batches deletion and reports hasMore when candidates exceed batchSize", async () => { + // 专用 session：6 条全部提取、无 user 消息（referenced 下全部可候选） + const sid = `retention-e-${Date.now()}`; + await seedSession(sid, 6, () => "assistant"); + await markExtracted(driver, sid, 5); + + // batchSize=2：一个维护周期只处理最旧的 2 条，hasMore=true 提示还有剩余 + const policy = normalizeMessageRetentionPolicy({ keep: "referenced", batchSize: 2 }); + const result = await runMessageRetention(driver, policy); + + expect(result.selectedRows).toBe(2); + expect(result.deletedRows).toBe(2); + expect(result.hasMore).toBe(true); + expect(await survivingTurns(sid)).toEqual([2, 3, 4, 5]); + }); +}); diff --git a/test/retention-policy.test.ts b/test/retention-policy.test.ts new file mode 100644 index 0000000..77139c4 --- /dev/null +++ b/test/retention-policy.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; + +import { + normalizeMessageRetentionPolicy, + messageRetentionPolicyRevision, +} from "../src/store/retention.ts"; + +describe("normalizeMessageRetentionPolicy", () => { + it("defaults to keep=all with no pruning and bounded batches", () => { + expect(normalizeMessageRetentionPolicy(undefined)).toEqual({ + keep: "all", recentTurns: 0, retentionDays: 0, batchSize: 500, dryRun: false, + }); + expect(normalizeMessageRetentionPolicy({})).toEqual({ + keep: "all", recentTurns: 0, retentionDays: 0, batchSize: 500, dryRun: false, + }); + }); + + it("accepts a valid recent policy", () => { + const policy = normalizeMessageRetentionPolicy({ + keep: "recent", recentTurns: 3, retentionDays: 30, batchSize: 100, dryRun: true, + }); + expect(policy).toEqual({ + keep: "recent", recentTurns: 3, retentionDays: 30, batchSize: 100, dryRun: true, + }); + }); + + it("accepts keep=referenced without window params", () => { + expect(normalizeMessageRetentionPolicy({ keep: "referenced" }).keep).toBe("referenced"); + }); + + it("rejects invalid keep mode and non-object input", () => { + expect(() => normalizeMessageRetentionPolicy({ keep: "aggressive" } as any)).toThrow(TypeError); + expect(() => normalizeMessageRetentionPolicy("recent" as any)).toThrow(TypeError); + expect(() => normalizeMessageRetentionPolicy([] as any)).toThrow(TypeError); + }); + + it("keep=recent requires at least one window parameter", () => { + expect(() => normalizeMessageRetentionPolicy({ keep: "recent" })).toThrow(/requires recentTurns or retentionDays/); + }); + + it("rejects out-of-bounds and non-integer numeric fields", () => { + expect(() => normalizeMessageRetentionPolicy({ batchSize: 0 })).toThrow(/batchSize/); + expect(() => normalizeMessageRetentionPolicy({ batchSize: 10_001 })).toThrow(/batchSize/); + expect(() => normalizeMessageRetentionPolicy({ batchSize: 12.5 })).toThrow(/batchSize/); + expect(() => normalizeMessageRetentionPolicy({ recentTurns: -1 })).toThrow(/recentTurns/); + expect(() => normalizeMessageRetentionPolicy({ retentionDays: 40_000 })).toThrow(/retentionDays/); + }); + + it("rejects non-boolean dryRun", () => { + expect(() => normalizeMessageRetentionPolicy({ dryRun: "yes" as any })).toThrow(/dryRun/); + }); + + it("revision is stable for identical policies and differs across policies", () => { + const a = normalizeMessageRetentionPolicy({ keep: "referenced", batchSize: 100 }); + const b = normalizeMessageRetentionPolicy({ keep: "referenced", batchSize: 100 }); + const c = normalizeMessageRetentionPolicy({ keep: "referenced", batchSize: 200 }); + expect(messageRetentionPolicyRevision(a)).toBe(messageRetentionPolicyRevision(b)); + expect(messageRetentionPolicyRevision(a)).not.toBe(messageRetentionPolicyRevision(c)); + }); +}); From d2476242a69464871e17110511d5f76049e9c248 Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Tue, 1 Sep 2026 07:59:51 +0000 Subject: [PATCH 16/29] =?UTF-8?q?Fix:=20=E6=8C=89=E7=85=A7Openclaw=20?= =?UTF-8?q?=E8=A7=84=E8=8C=83=E5=A3=B0=E6=98=8E=20transcript=20fencing=20?= =?UTF-8?q?=E8=AF=AD=E4=B9=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .video_agent/plugin_root | 1 + README.md | 29 +++- README_CN.md | 29 +++- index.ts | 91 +++++++++- openclaw.plugin.json | 2 +- src/cli-extract.ts | 5 +- src/engine/llm-guard.ts | 21 +++ src/engine/llm.ts | 31 +++- src/graph/maintenance.ts | 21 ++- src/store/db.ts | 14 ++ src/store/retention.ts | 18 +- src/store/store.ts | 56 ++++++- src/types.ts | 4 +- test/cli-extract.test.ts | 24 ++- test/commit-turn.test.ts | 240 +++++++++++++++++++++++++++ test/integration.retention.test.ts | 40 ++++- test/integration.turn-commit.test.ts | 77 +++++++++ test/llm-cooldown-recovery.test.ts | 98 +++++++++++ test/llm-guard.test.ts | 17 ++ test/register-guard.test.ts | 76 +++++++++ test/session-identity.test.ts | 1 + 21 files changed, 858 insertions(+), 37 deletions(-) create mode 100644 .video_agent/plugin_root create mode 100644 test/commit-turn.test.ts create mode 100644 test/integration.turn-commit.test.ts create mode 100644 test/llm-cooldown-recovery.test.ts diff --git a/.video_agent/plugin_root b/.video_agent/plugin_root new file mode 100644 index 0000000..76fc7ab --- /dev/null +++ b/.video_agent/plugin_root @@ -0,0 +1 @@ +C:\Users\18913\.zcode\cli\plugins\cache\zcode-plugins-official\video-agent-kit\0.4.3 \ No newline at end of file diff --git a/README.md b/README.md index 9de115e..2ee7a54 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ The installer configures Neo4j to start at boot with a 3-tier no-sudo fallback: ## Manual Configuration -Install the local plugin, then make it the OpenClaw context engine: +Install the local plugin, then make it the OpenClaw context engine (**restart the gateway after changing config**: the plugin guards against duplicate `register()` — a second registration without `dispose()` reuses the active engine, so new config is never hot-reloaded): ```json { @@ -155,6 +155,33 @@ All three sub-options are optional; omitted fields keep the default `true` (e.g. Caveat: when a cron job sets an explicit custom `sessionKey`, the host does not append the `cron` segment — such sessions cannot be detected and are treated as normal sessions. +### Raw message retention (messageRetention, opt-in) + +Context compaction only changes what the model sees — it is **not authorization to delete persistent evidence**. The default `keep=all` never deletes any raw message (GmMessage) at zero overhead. To bound database growth, opt into bounded pruning; it runs at the tail of the graph maintenance chain and processes at most `batchSize` rows per cycle: + +```json +"messageRetention": { + "keep": "referenced", + "batchSize": 500, + "dryRun": false +} +``` + +| Option | Default | Description | +| --- | --- | --- | +| `keep` | `"all"` | `all` = keep everything (default, zero behavior change); `referenced` = delete only messages that were extracted **and** actually produced knowledge; `recent` = `referenced` plus a time-window guard (requires at least one window option). | +| `recentTurns` | `0` | `keep=recent`: keep the newest N real user turns per session (that turn and everything after it). Sessions with no user messages are fully protected. | +| `retentionDays` | `0` | `keep=recent`: keep messages ingested within the last N days. | +| `batchSize` | `500` | Maximum rows processed per maintenance cycle (1–10000), keeping the chain bounded. | +| `dryRun` | `false` | `true` reports the candidate set without deleting — **run one `dryRun` cycle to validate candidates before enabling pruning**. | + +Deletion semantics (fail-closed): + +- Only messages with `extracted=true` **and** `producedKnowledge=true` (the extraction actually produced nodes/edges) enter the candidate set. +- Turns where the LLM returned zero nodes and zero edges are marked `producedKnowledge=false` — the raw evidence stays in the database. Such turns are never re-extracted automatically; to re-mine them with a better prompt/model, manually reset their `extracted` flag to `false` and re-run `openclaw graph-memory extract`. +- Unextracted messages and legacy rows (extracted before this flag existed, no `producedKnowledge` property) are never deleted. +- A failure in the retention step itself (invalid policy fails closed with zero deletions / Neo4j error) never invalidates the other maintenance steps; it retries next cycle. + ### OAuth login (experimental) ```bash diff --git a/README_CN.md b/README_CN.md index f5adea1..87bb74d 100644 --- a/README_CN.md +++ b/README_CN.md @@ -44,7 +44,7 @@ bash setup-graph-memory-pro.sh --uninstall ## 手动配置 -安装插件后，在 `~/.openclaw/openclaw.json` 中配置： +安装插件后，在 `~/.openclaw/openclaw.json` 中配置（**修改配置后需重启网关生效**：插件内置防重复注册守卫，宿主未 dispose 的二次 `register()` 会复用现有引擎，新配置不会热加载）： ```json { @@ -137,6 +137,33 @@ OpenClaw 定时任务创建的会话可以独立配置图谱行为。host 把 cr 注意：若 cron 任务显式设置了自定义 `sessionKey`，host 不再附加 `cron` 段，此类会话无法被识别，将按普通会话处理。 +### 原始消息保留（messageRetention，opt-in） + +上下文压缩只改变模型可见面，**不构成删除持久证据的授权**——默认 `keep=all` 永不删除任何原始消息（GmMessage），零开销。需要控制库体积时可显式开启有界清理，挂在图维护链尾部，每个维护周期最多处理 `batchSize` 行： + +```json +"messageRetention": { + "keep": "referenced", + "batchSize": 500, + "dryRun": false +} +``` + +| 选项 | 默认 | 说明 | +| --- | --- | --- | +| `keep` | `"all"` | `all`=全部保留（默认，零行为变化）；`referenced`=只删"已提取完成且实际产出知识"的消息；`recent`=在 `referenced` 基础上叠加时间窗保护（需至少配一个窗口参数）。 | +| `recentTurns` | `0` | `keep=recent`：每 session 保留最近 N 轮真实用户发言（该轮及其后的全部消息保留）。无 user 消息的 session 完全保护。 | +| `retentionDays` | `0` | `keep=recent`：保留最近 N 天内入库的消息。 | +| `batchSize` | `500` | 单个维护周期最多处理的行数（1~10000），保证维护链工作有界。 | +| `dryRun` | `false` | `true` 时只报告候选集不删除——**启用清理前建议先跑一轮 `dryRun` 验证候选集**。 | + +删除语义（fail-closed）： + +- 只有 `extracted=true` **且** `producedKnowledge=true`（该轮 LLM 提取实际产出节点/边）的消息才会进入候选。 +- LLM 空提取（成功返回零节点零边）的轮次标记 `producedKnowledge=false`，原始证据保留在库中。此类轮次不会自动重提；如需用更好的 prompt/模型重挖，需手动将这些行的 `extracted` 重置为 `false`，再运行 `openclaw graph-memory extract` 回填。 +- 未提取消息、以及标记机制上线前的遗留行（无 `producedKnowledge` 属性）一律不删。 +- 保留步骤自身失败（非法策略 fail-closed 不删 / Neo4j 故障）不影响维护链的其他步骤，下一周期自动重试。 + ### OAuth 登录（实验性） ```bash diff --git a/index.ts b/index.ts index 2b49665..ddba531 100755 --- a/index.ts +++ b/index.ts @@ -11,7 +11,7 @@ import { getDriver, initSchema, getSession } from "./src/store/db.ts"; import { Neo4jGate } from "./src/store/gate.ts"; import { saveMessage, getUnextracted, getMaxTurnIndex, - markExtracted, isTurnExtracted, + markExtracted, isTurnExtracted, commitTurnAdvance, upsertNode, upsertEdge, findByName, updateNode, deleteNode, deprecateNodeAndDisconnect, getBySession, edgesTouching, @@ -455,8 +455,11 @@ const graphMemoryProPlugin = { }); if (!result.nodes.length && !result.edges.length) { - await markExtracted(driver, sessionId, turnNum); - api.logger.info(`[graph-memory-pro] turn ${turnNum}: no knowledge extracted (marked extracted)`); + // 空提取也要标记 extracted（防止重复提取），但 producedKnowledge=false： + // 该轮没有知识固化进图谱，原始证据保留（retention 不删； + // 重挖需手动重置 extracted 后跑 graph-memory extract） + await markExtracted(driver, sessionId, turnNum, false); + api.logger.info(`[graph-memory-pro] turn ${turnNum}: no knowledge extracted (marked extracted, evidence retained)`); return; } @@ -685,8 +688,10 @@ const graphMemoryProPlugin = { `dedup=${result.dedup.merged}, communities=${result.community.count}, ` + `summaries=${result.communitySummaries}, ` + (result.retention - ? `retention=${result.retention.dryRun ? "dryRun:" : ""}` + - `${result.retention.deletedRows}/${result.retention.selectedRows} msgs, ` + ? ("error" in result.retention + ? `retention=failed: ${result.retention.error.slice(0, 120)}, ` + : `retention=${result.retention.dryRun ? "dryRun:" : ""}` + + `${result.retention.deletedRows}/${result.retention.selectedRows} msgs, `) : "") + `top_pr=${result.pagerank.topK.slice(0, 3).map(n => `${n.name}(${n.score.toFixed(3)})`).join(",")}`, ); @@ -768,6 +773,14 @@ const graphMemoryProPlugin = { id: "graph-memory-pro", name: "Graph Memory Pro", ownsCompaction: true, + // OpenClaw 2026.3.7+ transcript fencing 契约。未声明时 host 会把引擎 + // 逐回合降级到 legacy（"current-turn transcript fencing is not declared"）。 + // 两项承诺：轮前读取只看到 admitted user entry 之前的精确前缀； + // 轮推进经 commitTurn 以 advancementKey 原子幂等落盘。 + transcriptSemantics: { + currentTurnFence: "before-current-turn-entry-v1", + turnAdvancementIdempotency: "atomic-idempotent-v1", + }, }, async bootstrap({ sessionId, sessionKey }: { sessionId: string; sessionKey?: string }) { @@ -923,7 +936,10 @@ const graphMemoryProPlugin = { } const maxTurn = Math.max(...msgs.map((m: any) => m.turn_index)); - await markExtracted(driver, sessionId, maxTurn); + await markExtracted( + driver, sessionId, maxTurn, + result.nodes.length > 0 || result.edges.length > 0, + ); return { ok: true, compacted: true, @@ -991,6 +1007,69 @@ const graphMemoryProPlugin = { }); }, + /** + * OpenClaw 2026.3.7+ transcript fencing 契约的轮推进点（见 info.transcriptSemantics）。 + * host 只对成功接受的轮次调用；重试携带同一 advancementKey。 + * 义务 = 一次以 advancementKey 为键的原子幂等写（GmTurnCommit 唯一约束 + CREATE）： + * 首次 → committed；重试撞约束 → duplicate，副作用不再重放。 + * + * 消息持久化仍归 ingest/afterTurn（契约保证 fenced 路径下 ingest 照常逐条触发； + * afterTurn 的 backfill 依赖 ingestedSinceTurn 计数器，commitTurn 若也回填， + * 两生命周期点并存时会以新 seq 重写整轮消息 → GmMessage 重复行）。 + * 此处只做：标记落盘 + 触发本轮提取 —— 提取与 afterTurn 共用同一条幂等管线 + * （withExtractLock + isTurnExtracted 双守卫），两点并存时后进入者自动空转。 + */ + async commitTurn({ sessionId, sessionKey, advancementKey, messages, isHeartbeat }: { + sessionId?: string; sessionKey?: string; advancementKey?: string; messages?: any[]; isHeartbeat?: boolean; + }) { + if (isHeartbeat || !advancementKey) return { status: "committed" as const }; + if (sessionId) bindSessionIdentity(sessionId, sessionKey); + const sid = sessionId ?? (sessionKey ? sessionIdsByKey.get(sessionKey) : undefined); + + // 宿主只传 advancementKey 且 sessionKey 无历史绑定（缺 bootstrap/ingest） + // 时无法归属会话：标记与提取都不可用。明确告警而非静默 no-op —— + // 返回 committed 只是"不阻塞回合"的降级，不代表已落盘。 + if (!sid) { + api.logger.warn(`[graph-memory-pro] commitTurn: cannot resolve session for advancementKey=${advancementKey.slice(0, 12)}…, marker + extraction skipped`); + return { status: "committed" as const }; + } + + // cron session 关闭图谱功能：整个提交按 no-op 处理（不写标记、不提取） + if (isCronSessionKey(sessionKey) && !cronCfg.enabled) return { status: "committed" as const }; + + let advance: "committed" | "duplicate" = "committed"; + if (neo4jGate.isAvailable()) { + try { + advance = await commitTurnAdvance(driver, sid, advancementKey, messages?.length ?? 0); + neo4jGate.recordSuccess(); + } catch (err) { + // 标记写失败不向 host 抛错：提取副作用自带幂等守卫，host 重试 + // 最多多跑一次空检查；沿用 ingest/assemble 的"降级不炸回合"哲学 + neo4jGate.recordFailure(); + api.logger.warn(`[graph-memory-pro] commitTurn marker write failed (side effects remain idempotent): ${err}`); + } + } + // duplicate 短路：若首次提交的提取本身失败过（LLM 错误只记日志）， + // 这里也不会重放 —— 恢复路径是 compact() / `graph-memory extract` 回填 + if (advance === "duplicate") return { status: "duplicate" as const }; + + if (messages?.length) { + if (isCronSessionKey(sessionKey) && !cronCfg.extract) { + api.logger.info("[graph-memory-pro] cron session: extraction skipped (cron.extract=false)"); + } else { + // 读 turn 编号前先等掉线缓冲落库（与 afterTurn 的 backfill-then-read + // 对齐）：flush 会给缓冲消息分配新 seq，预读的旧值会让 markExtracted + // 只覆盖旧前缀 → 本轮消息保持未提取 → compact 重复提取 + if (messageBuffer.length) await flushMessageBuffer(); + const turnNum = msgSeq.get(sid) ?? 0; + extractTurnKnowledge(sid, turnNum, messages).catch(err => { + api.logger.error(`[graph-memory-pro] extract failed: ${err}`); + }); + } + } + return { status: "committed" as const }; + }, + async prepareSubagentSpawn({ parentSessionKey, childSessionKey, parentSessionId }: { parentSessionKey: string; childSessionKey: string; parentSessionId?: string; }) { diff --git a/openclaw.plugin.json b/openclaw.plugin.json index 2f0e275..e14582f 100755 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -68,7 +68,7 @@ "type": "string", "enum": ["all", "referenced", "recent"], "default": "all", - "description": "all=全部保留（默认）；referenced=只删已提取完成的消息（知识已固化进图谱）；recent=在 referenced 基础上按时间窗保护最近内容（需配 recentTurns 或 retentionDays）" + "description": "all=全部保留（默认）；referenced=只删已提取完成且实际产出知识的消息（producedKnowledge=true，知识已固化进图谱；LLM 空提取的轮次保留原始证据，重挖需手动重置 extracted 后跑 graph-memory extract）；recent=在 referenced 基础上按时间窗保护最近内容（需配 recentTurns 或 retentionDays）" }, "recentTurns": { "type": "number", "default": 0, "description": "keep=recent：每 session 保留最近 N 轮真实用户发言（该轮及其后的全部消息保留）。" }, "retentionDays": { "type": "number", "default": 0, "description": "keep=recent：保留最近 N 天内入库的消息。" }, diff --git a/src/cli-extract.ts b/src/cli-extract.ts index 04a821c..a34f3d7 100644 --- a/src/cli-extract.ts +++ b/src/cli-extract.ts @@ -267,7 +267,10 @@ async function extractSessionLoop( await Promise.allSettled(pendingEmbeds); const maxTurn = msgs.reduce((m, msg) => Math.max(m, msg.turn_index ?? 0), 0); - await markExtracted(driver, sessionId, maxTurn); + await markExtracted( + driver, sessionId, maxTurn, + extraction.nodes.length > 0 || extraction.edges.length > 0, + ); log(` batch ${stats.batches}: ${msgs.length} 消息 -> ${extraction.nodes.length} 节点 / ${extraction.edges.length} 边（累计 ${stats.nodes}/${stats.edges}）`); if (msgs.length < batchLimit) break; diff --git a/src/engine/llm-guard.ts b/src/engine/llm-guard.ts index 8090209..b6d8386 100644 --- a/src/engine/llm-guard.ts +++ b/src/engine/llm-guard.ts @@ -9,10 +9,15 @@ * - 429/5xx：瞬时故障，fetchRetry 已内部重试，冷却反而放大抖动 * - 400/422：可能只是单条坏 prompt（超长/格式问题），不能殃及后续正常调用 * - 无状态码的错误（超时、空返回、缺配置）：交给各自的正常失败路径 + * + * 例外：OAuth token 端点的 400 = invalid_grant（refresh token 被吊销/过期）， + * 与聊天端点的 400（可能只是单条坏 prompt）语义不同 —— 属于持久性凭证故障， + * 同样触发冷却；恢复路径是重新 auth login（llm.ts 会按会话文件 mtime 自动解除）。 */ const RETRYABLE_STATUSES = new Set([429, 500, 502, 503, 529]); const PAUSING_STATUSES = new Set([401, 403, 404]); +const OAUTH_REFRESH_PAUSING_STATUSES = new Set([400, 401, 403]); /** * 从错误消息中提取 HTTP 状态码。 @@ -28,6 +33,16 @@ export function extractLlmStatus(error: unknown): number | null { return Number(match[1]); } +/** + * 识别 oauth.ts refreshOAuthSession 的报错格式："OAuth refresh failed (400): …"。 + * 独立于 extractLlmStatus —— token 端点与聊天端点对同一状态码的语义不同。 + */ +export function extractOAuthRefreshStatus(error: unknown): number | null { + const match = String(error ?? "").match(/\bOAuth refresh failed \((\d{3})\)/); + if (!match) return null; + return Number(match[1]); +} + export class LlmFailureGuard { private pausedUntil = 0; @@ -54,6 +69,12 @@ export class LlmFailureGuard { * 返回是否触发（调用方仅用于日志/测试；冷却本身幂等，重复触发取更晚到期时间）。 */ tripIfNeeded(error: unknown): boolean { + const refreshStatus = extractOAuthRefreshStatus(error); + if (refreshStatus != null) { + if (!OAUTH_REFRESH_PAUSING_STATUSES.has(refreshStatus)) return false; + this.pausedUntil = Math.max(this.pausedUntil, this.now() + this.cooldownMs); + return true; + } const status = extractLlmStatus(error); if (status == null || RETRYABLE_STATUSES.has(status) || !PAUSING_STATUSES.has(status)) { return false; diff --git a/src/engine/llm.ts b/src/engine/llm.ts index c0c5c67..6442368 100755 --- a/src/engine/llm.ts +++ b/src/engine/llm.ts @@ -339,20 +339,41 @@ export function createCompleteFn( // ── 失败冷却守卫：持久性配置错误（401/403/404）后冷却 10 分钟，快速失败 ── // 避免凭证失效/模型名错误时每轮照付一次完整请求 + 超时等待。成功调用即清除。 + // OAuth 例外自愈：oauthPath 可能被外部进程重写（CLI auth login / CLI extract + // 刷新 token）。冷却触发时记录会话文件 mtime，后续调用发现文件已变化 = + // 凭证已被修复 → 立即解除冷却重试（401 的常见诱因是时钟偏移导致缓存的 + // access token 提前过期，重登即可恢复，不应被迫等满 10 分钟）。 const guard = new LlmFailureGuard(); + let oauthTripMtimeMs: number | null | undefined; // undefined = 冷却非 oauth 路径触发 return async (system: string, user: string): Promise => { if (!guard.canRun()) { - const seconds = Math.max(1, Math.ceil(guard.remainingMs() / 1000)); - throw new Error( - `[graph-memory] LLM paused for ${seconds}s after a previous permanent API error`, - ); + if (provider === "oauth" && oauthPath && oauthTripMtimeMs !== undefined) { + let currentMtimeMs: number | null = null; + try { currentMtimeMs = (await stat(oauthPath)).mtimeMs; } catch { /* 文件暂不可达：维持冷却 */ } + // 仅在"确实读到不同的 mtime"或"trip 时读不到、现在读得到"时解除； + // 瞬时 stat 失败（null）不解除 —— 避免 AV/EBUSY 类抖动白白放行一次必败请求 + const fileReplaced = oauthTripMtimeMs === null + ? currentMtimeMs !== null + : currentMtimeMs !== null && currentMtimeMs !== oauthTripMtimeMs; + if (fileReplaced) guard.reset(); + } + if (!guard.canRun()) { + const seconds = Math.max(1, Math.ceil(guard.remainingMs() / 1000)); + throw new Error( + `[graph-memory] LLM paused for ${seconds}s after a previous permanent API error` + + (provider === "oauth" ? " — 重新 auth login 或等待 token 文件刷新后自动解除" : ""), + ); + } } try { const text = await complete(system, user); guard.reset(); return text; } catch (err) { - guard.tripIfNeeded(err); + if (guard.tripIfNeeded(err) && provider === "oauth" && oauthPath) { + try { oauthTripMtimeMs = (await stat(oauthPath)).mtimeMs; } + catch { oauthTripMtimeMs = null; } + } throw err; } }; diff --git a/src/graph/maintenance.ts b/src/graph/maintenance.ts index c2fc1c3..edbe6f2 100755 --- a/src/graph/maintenance.ts +++ b/src/graph/maintenance.ts @@ -24,8 +24,9 @@ export interface MaintenanceResult { pagerank: GlobalPageRankResult; community: CommunityResult; communitySummaries: number; - /** 原始消息保留（opt-in）；未配置 messageRetention 或 keep=all 时为 undefined。 */ - retention?: MessageRetentionResult; + /** 原始消息保留（opt-in）；未配置 messageRetention 或 keep=all 时为 undefined， + * 执行失败时为 { error }（fail-soft，不否定整轮维护）。 */ + retention?: MessageRetentionResult | { error: string }; durationMs: number; } @@ -55,12 +56,18 @@ export async function runMaintenance( } // 5. 原始消息保留（opt-in；keep=all 零开销直返）。 - // 非法策略 fail closed：抛错终止维护、不删任何东西，由调用方记录。 - let retention: MessageRetentionResult | undefined; + // fail-soft：链尾清理失败（非法策略 fail-closed 不删 / Neo4j 故障）不应否定 + // 已完成的 decay/dedup/PageRank/社区步骤 —— 前面步骤成功说明连接健康， + // 也不计入 Neo4j 熔断。错误记入结果，由调用方记日志，下一周期重试。 + let retention: MessageRetentionResult | { error: string } | undefined; if (cfg.messageRetention) { - const policy = normalizeMessageRetentionPolicy(cfg.messageRetention); - if (policy.keep !== "all") { - retention = await runMessageRetention(driver, policy); + try { + const policy = normalizeMessageRetentionPolicy(cfg.messageRetention); + if (policy.keep !== "all") { + retention = await runMessageRetention(driver, policy); + } + } catch (err) { + retention = { error: String(err) }; } } diff --git a/src/store/db.ts b/src/store/db.ts index e5a43be..f2fdc6a 100755 --- a/src/store/db.ts +++ b/src/store/db.ts @@ -87,6 +87,20 @@ export async function initSchema(driver: Driver, embedding?: EmbeddingConfig): P FOR (c:Community) ON (c.embedding) OPTIONS {indexConfig: {\`vector.dimensions\`: ${dimensions}, \`vector.similarity_function\`: 'cosine'}} `); + + // Turn commit marker (OpenClaw transcript fencing contract). + // 放在 DDL 链末尾：升级后首启若约束尚未建好，并发重试的 CREATE 可能 + // 已写入重复 advancementKey —— 带冲突数据的 CREATE CONSTRAINT 会失败， + // 先清重复行再建约束；即使这里失败也不能波及上面的向量索引。 + // 唯一约束是幂等提交的原子性来源：重试的 CREATE 撞约束报 + // ConstraintValidationFailed，commitTurnAdvance 据此区分 committed/duplicate。 + await session.run(` + MATCH (t:GmTurnCommit) + WITH t.advancementKey AS key, collect(t) AS marks + WHERE size(marks) > 1 + FOREACH (n IN marks[1..] | DELETE n) + `); + await session.run("CREATE CONSTRAINT gm_turn_commit_key IF NOT EXISTS FOR (t:GmTurnCommit) REQUIRE t.advancementKey IS UNIQUE"); } finally { await session.close(); } diff --git a/src/store/retention.ts b/src/store/retention.ts index 6c1914c..8415c6a 100644 --- a/src/store/retention.ts +++ b/src/store/retention.ts @@ -5,14 +5,18 @@ * "上下文压缩改变的是模型可见面，不构成删除持久证据的授权。" * * - 默认 keep=all：不做任何删除，本模块在维护链中零开销直返。 - * - referenced：只删"已提取完成"的消息 —— 知识已固化进图谱节点/边，原始文本退役。 + * - referenced：只删"已提取完成且实际产出知识"的消息（extracted=true 且 + * producedKnowledge=true）—— 知识已固化进图谱节点/边，原始文本退役。 + * LLM 空提取（零节点零边）的轮次标记 producedKnowledge=false，原始证据 + * 保留；重挖需手动重置 extracted 后跑 graph-memory extract。 * - recent：referenced 之上叠加时间窗保护（每 session 最近 N 轮真实用户发言 * 及其后消息、最近 N 天内入库的消息），验证时要求至少配置一个窗口参数。 * * 与上游的差异：v2.0 schema 没有消息级出处边（节点仅记 sourceSessions， * 粒度为 session），上游 "无 gm_node_sources 引用" 的前置条件在这里等价于 - * extracted=true。DELETE 仍按 extracted 重新校验，防止候选查询与删除语义 - * 未来漂移 —— 候选集不能成为删除的授权。 + * extracted=true AND producedKnowledge=true。遗留行（producedKnowledge 属性 + * 缺失，标记机制上线前已提取）fail-closed 不删。DELETE 前按同条件重新校验， + * 防止候选查询与删除语义未来漂移 —— 候选集不能成为删除的授权。 */ import { createHash } from "node:crypto"; @@ -143,7 +147,7 @@ function buildCandidateQuery( batchLimit: int(policy.batchSize + 1), }; - let match = "MATCH (m:GmMessage)\nWHERE m.extracted = true\n"; + let match = "MATCH (m:GmMessage)\nWHERE m.extracted = true AND m.producedKnowledge = true\n"; if (policy.recentTurns > 0) { match = "MATCH (u:GmMessage {role: 'user'})\n" + @@ -151,7 +155,7 @@ function buildCandidateQuery( "WITH sid, collect(u.turnIndex) AS turns\n" + "WITH sid, CASE WHEN size(turns) >= $recentTurns THEN turns[$recentTurns - 1] ELSE -1 END AS cutoffTurn\n" + "MATCH (m:GmMessage {sessionId: sid})\n" + - "WHERE m.extracted = true\n" + + "WHERE m.extracted = true AND m.producedKnowledge = true\n" + " AND m.turnIndex < cutoffTurn\n"; } if (policy.retentionDays > 0) { @@ -170,7 +174,7 @@ function buildCandidateQuery( /** * 跑一个有界批次。候选选择与删除放在同一个写事务里冻结边界； - * 删除前按 extracted 重新校验（候选查询不是删除的授权）。 + * 删除前按 extracted + producedKnowledge 重新校验（候选查询不是删除的授权）。 * size(m.content) 返回字符数（近似体积），与上游 BLOB 字节数略有差异，仅用于统计。 */ export async function runMessageRetention( @@ -206,7 +210,7 @@ export async function runMessageRetention( const delRes = await tx.run( "UNWIND $ids AS mid " + "MATCH (m:GmMessage {id: mid}) " + - "WHERE m.extracted = true " + + "WHERE m.extracted = true AND m.producedKnowledge = true " + "DETACH DELETE m " + "RETURN count(m) AS deleted", { ids: rows.map((r) => r.id) }, diff --git a/src/store/store.ts b/src/store/store.ts index 0538bbc..a764508 100755 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -994,14 +994,29 @@ export async function listUnextractedSessions(driver: Driver): Promise { +/** + * 标记 sid 会话内 turnIndex ≤ upToTurn 的消息为已提取。 + * + * producedKnowledge：该次提取是否实际产出节点/边。LLM 成功返回零节点零边时 + * 传 false —— 原始证据保留（retention 只删 producedKnowledge=true 的行）。 + * 注意空提取轮次不会自动重提：重挖需手动将其 extracted 重置为 false 再跑 + * `openclaw graph-memory extract`。 + * + * 只作用于尚未提取的行（extracted=false）：producedKnowledge 在提取转换时刻 + * 一次性写入，不会被后续更大范围的批量标记覆盖（compact 的范围标记可能横跨 + * 早期已标记的轮次）。遗留行（无 producedKnowledge 属性）保持 null， + * retention 对其 fail-closed 不删。 + */ +export async function markExtracted( + driver: Driver, sid: string, upToTurn: number, producedKnowledge = true, +): Promise { const session = getSession(driver); try { await session.run(` MATCH (m:GmMessage {sessionId: $sid}) - WHERE m.turnIndex <= $upToTurn - SET m.extracted = true - `, { sid, upToTurn }); + WHERE m.turnIndex <= $upToTurn AND m.extracted = false + SET m.extracted = true, m.producedKnowledge = $pk + `, { sid, upToTurn, pk: producedKnowledge }); } finally { await session.close(); } @@ -1021,6 +1036,39 @@ export async function isTurnExtracted(driver: Driver, sid: string, turn: number) } } +// ─── 轮次提交标记（OpenClaw transcript fencing 契约） ───────── + +/** neo4j-driver 约束冲突错误的 code 形如 Neo.ClientError.Schema.ConstraintValidationFailed */ +function isConstraintViolation(err: unknown): boolean { + const code = (err as { code?: unknown } | null | undefined)?.code; + if (typeof code === "string" && code.includes("ConstraintValidation")) return true; + return String(err).includes("ConstraintValidation"); +} + +/** + * 幂等提交一个 logical turn（host 只对成功接受的轮次调用，重试携带同一 advancementKey）。 + * 唯一约束 + CREATE 保证原子幂等：首次 → "committed"；重试撞约束 → "duplicate"。 + * 标记节点本身即"一次原子幂等写"的全部载荷 —— 消息已由 ingest/afterTurn 逐条落库， + * 不在标记里重复存（否则与 GmMessage 行双写、outage 缓冲路径无法对齐）。 + */ +export async function commitTurnAdvance( + driver: Driver, sid: string, advancementKey: string, messageCount: number, +): Promise<"committed" | "duplicate"> { + const session = getSession(driver); + try { + await session.run( + `CREATE (t:GmTurnCommit {advancementKey: $key, sessionId: $sid, messageCount: $count, createdAt: $now})`, + { key: advancementKey, sid, count: messageCount, now: Date.now() }, + ); + return "committed"; + } catch (err) { + if (isConstraintViolation(err)) return "duplicate"; + throw err; + } finally { + await session.close(); + } +} + // ─── 统计 ──────────────────────────────────────────────────── export async function getStats(driver: Driver): Promise<{ diff --git a/src/types.ts b/src/types.ts index 280752a..5d0c3ca 100755 --- a/src/types.ts +++ b/src/types.ts @@ -218,7 +218,9 @@ export type MessageRetentionMode = "all" | "referenced" | "recent"; export interface MessageRetentionConfig { /** * all（默认）保留全部原始消息，零开销； - * referenced 只删"已提取完成"的消息（知识已固化进图谱，原始文本退役）； + * referenced 只删"已提取完成且实际产出知识"的消息（producedKnowledge=true， + * 知识已固化进图谱，原始文本退役；LLM 空提取的轮次保留原始证据， + * 重挖需手动重置 extracted 后跑 graph-memory extract）； * recent 在 referenced 的基础上按时间窗保护最近内容。 */ keep?: MessageRetentionMode; diff --git a/test/cli-extract.test.ts b/test/cli-extract.test.ts index 902016b..7ba66db 100644 --- a/test/cli-extract.test.ts +++ b/test/cli-extract.test.ts @@ -242,10 +242,32 @@ describe("runBackfillExtraction", () => { expect(result.edgesCreated).toBe(1); expect(result.batches).toBe(1); expect(mocks.extract).toHaveBeenCalledTimes(1); - expect(mocks.markExtracted).toHaveBeenCalledWith(expect.anything(), "sid-abc-1234567890", 2); + // 提取产出 2 节点 → producedKnowledge=true（空提取时应为 false，证据保留） + expect(mocks.markExtracted).toHaveBeenCalledWith(expect.anything(), "sid-abc-1234567890", 2, true); expect(mocks.closeDriver).toHaveBeenCalledTimes(1); }); + it("marks empty extractions as producedKnowledge=false (evidence retained)", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + mocks.getUnextracted.mockResolvedValueOnce([ + { role: "user", content: "hello", turn_index: 1 }, + ]).mockResolvedValueOnce([]); + mocks.extract.mockResolvedValueOnce({ nodes: [], edges: [] }); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { yes: true }, + log, + prompt: vi.fn(), + }); + + expect(result.sessionsProcessed).toBe(1); + expect(result.nodesCreated).toBe(0); + expect(mocks.markExtracted).toHaveBeenCalledWith(expect.anything(), "sid-abc-1234567890", 1, false); + }); + it("filters sessions to the one specified by --session", async () => { mocks.listUnextractedSessions.mockResolvedValue([ { ...SAMPLE_SESSION, sessionId: "aaa" }, diff --git a/test/commit-turn.test.ts b/test/commit-turn.test.ts new file mode 100644 index 0000000..9acd688 --- /dev/null +++ b/test/commit-turn.test.ts @@ -0,0 +1,240 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +// ── 模块隔离：register() 的完整初始化路径绝不能触碰真实 Neo4j ── +// 不 mock 的话 getDriver() 会连 bolt://localhost:7687，initSchema 还会执行 +// 真实 DDL —— 任何 7687 端口有真实库的机器上裸跑 `npm test` 都有写风险。 +vi.mock("../src/store/db.ts", () => ({ + getDriver: () => ({}), + initSchema: async () => {}, + getSession: () => ({ close: async () => {} }), + closeDriver: async () => {}, +})); + +vi.mock("../src/store/store.ts", () => ({ + saveMessage: vi.fn(async () => {}), + getUnextracted: async () => [], + getMaxTurnIndex: async () => 0, + markExtracted: vi.fn(async () => {}), + isTurnExtracted: vi.fn(async () => false), + commitTurnAdvance: vi.fn(async () => "committed" as const), + upsertNode: async () => ({ node: {}, isNew: false }), + upsertEdge: async () => {}, + findByName: async () => null, + updateNode: async () => null, + deleteNode: async () => {}, + deprecateNodeAndDisconnect: async () => {}, + getBySession: async () => [], + edgesTouching: async () => [], + deleteEdges: async () => {}, + mergeNodes: async () => {}, + deprecate: async () => {}, + getStats: async () => ({}), +})); + +vi.mock("../src/engine/llm.ts", () => ({ + createCompleteFn: () => async () => "", + resolveProvider: () => ({ provider: "anthropic", effectiveModel: "m", inferred: false }), +})); + +vi.mock("../src/engine/embed.ts", () => ({ + createEmbedFn: async () => null, +})); + +vi.mock("../src/recaller/recall.ts", () => ({ + Recaller: class { + setEmbedFn(): void {} + hasEmbedFn(): boolean { return false; } + get embedFn() { return null; } + async recall() { return { nodes: [], edges: [], tokenEstimate: 0 }; } + async syncEmbed(): Promise {} + }, + parseTimeRange: () => null, +})); + +vi.mock("../src/extractor/extract.ts", () => ({ + Extractor: class { + async extract() { return { nodes: [], edges: [] }; } + async finalize() { return { promotedSkills: [], newEdges: [], invalidations: [] }; } + }, +})); + +vi.mock("../src/format/assemble.ts", () => ({ + assembleContext: async () => ({ xml: "", systemPrompt: "", tokens: 0 }), +})); + +vi.mock("../src/graph/maintenance.ts", () => ({ + runMaintenance: async () => ({ durationMs: 0 }), +})); + +vi.mock("../src/routes/crud.ts", () => ({ + registerCrudRoutes: () => {}, +})); + +import graphMemoryProPlugin from "../index.ts"; +import { closeDriver } from "../src/store/db.ts"; +import { commitTurnAdvance, isTurnExtracted, markExtracted, saveMessage } from "../src/store/store.ts"; + +const commitMock = vi.mocked(commitTurnAdvance); +const isTurnExtractedMock = vi.mocked(isTurnExtracted); +const markExtractedMock = vi.mocked(markExtracted); +const saveMessageMock = vi.mocked(saveMessage); + +function fullApi(pluginConfig: Record = {}) { + return { + pluginConfig, + config: {}, + resolvePath: (v: string) => v, + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + registerCli: vi.fn(), + registerTool: vi.fn(), + registerContextEngine: vi.fn(), + registerHttpRoute: vi.fn(), + on: vi.fn(), + } as any; +} + +function buildEngine(pluginConfig: Record = {}) { + const api = fullApi(pluginConfig); + graphMemoryProPlugin.register(api); + const engine = api.registerContextEngine.mock.calls[0][1]() as any; + return { api, engine }; +} + +/** 轮询等待 fire-and-forget 的提取管线真正跑起来。 */ +async function until(cond: () => boolean, ms = 2000): Promise { + const start = Date.now(); + while (!cond()) { + if (Date.now() - start > ms) throw new Error("condition not met in time"); + await new Promise((r) => setTimeout(r, 10)); + } +} + +describe("commitTurn (OpenClaw transcript fencing contract)", () => { + let engine: any; + let api: any; + + afterEach(async () => { + if (engine) { + await engine.dispose(); + engine = null; + } + await closeDriver(); + commitMock.mockReset(); + commitMock.mockResolvedValue("committed"); + isTurnExtractedMock.mockClear(); + markExtractedMock.mockClear(); + saveMessageMock.mockClear(); + }); + + it("declares current-turn transcript fencing semantics in engine.info", () => { + ({ api, engine } = buildEngine()); + expect(engine.info.transcriptSemantics).toEqual({ + currentTurnFence: "before-current-turn-entry-v1", + turnAdvancementIdempotency: "atomic-idempotent-v1", + }); + }); + + it("returns committed on first write and schedules turn extraction", async () => { + ({ api, engine } = buildEngine()); + const res = await engine.commitTurn({ + sessionId: "s-commit-1", + sessionKey: "agent:t:session-1", + advancementKey: "adv-key-1", + messages: [{ role: "user", content: "hi" }, { role: "assistant", content: "yo" }], + }); + expect(res).toEqual({ status: "committed" }); + expect(commitMock).toHaveBeenCalledTimes(1); + expect(commitMock.mock.calls[0][2]).toBe("adv-key-1"); + expect(commitMock.mock.calls[0][3]).toBe(2); + + // 提取是 fire-and-forget，但最终必须进入 isTurnExtracted 幂等检查 + await until(() => isTurnExtractedMock.mock.calls.length > 0); + }); + + it("returns duplicate on host retry without replaying side effects", async () => { + ({ api, engine } = buildEngine()); + commitMock.mockResolvedValueOnce("committed"); + commitMock.mockResolvedValueOnce("duplicate"); + + const first = await engine.commitTurn({ + sessionId: "s-commit-2", advancementKey: "adv-key-2", messages: [{ role: "user", content: "q" }], + }); + const retry = await engine.commitTurn({ + sessionId: "s-commit-2", advancementKey: "adv-key-2", messages: [{ role: "user", content: "q" }], + }); + expect(first).toEqual({ status: "committed" }); + expect(retry).toEqual({ status: "duplicate" }); + expect(commitMock).toHaveBeenCalledTimes(2); + + // duplicate 短路后不得再触发提取：先等首次提交的提取管线完整落地 + // （markExtracted 是管线最后一步），再确认 retry 没有安排新的提取 + await until(() => markExtractedMock.mock.calls.length > 0); + const settledCalls = markExtractedMock.mock.calls.length; + isTurnExtractedMock.mockClear(); + await new Promise((r) => setTimeout(r, 100)); + expect(isTurnExtractedMock).not.toHaveBeenCalled(); + expect(markExtractedMock.mock.calls.length).toBe(settledCalls); + }); + + it("degrades to committed when the marker write fails (non-constraint error)", async () => { + ({ api, engine } = buildEngine()); + commitMock.mockRejectedValueOnce(new Error("pool closed")); + const res = await engine.commitTurn({ + sessionId: "s-commit-3", advancementKey: "adv-key-3", messages: [], + }); + expect(res).toEqual({ status: "committed" }); + expect(api.logger.warn).toHaveBeenCalledWith( + expect.stringContaining("commitTurn marker write failed"), + ); + }); + + it("no-ops for heartbeats and missing advancementKey", async () => { + ({ api, engine } = buildEngine()); + expect(await engine.commitTurn({ sessionId: "s", advancementKey: "k", isHeartbeat: true })) + .toEqual({ status: "committed" }); + expect(await engine.commitTurn({ sessionId: "s", messages: [] })) + .toEqual({ status: "committed" }); + expect(commitMock).not.toHaveBeenCalled(); + }); + + it("skips the marker entirely for cron sessions with cron.enabled=false", async () => { + ({ api, engine } = buildEngine({ cron: { enabled: false } })); + const res = await engine.commitTurn({ + sessionId: "s-cron-1", + sessionKey: "agent:main:cron:job-1", + advancementKey: "adv-cron-1", + messages: [{ role: "user", content: "tick" }], + }); + expect(res).toEqual({ status: "committed" }); + expect(commitMock).not.toHaveBeenCalled(); + expect(isTurnExtractedMock).not.toHaveBeenCalled(); + }); + + it("never persists messages itself — persistence stays with ingest/afterTurn", async () => { + // 设计决策的回归钉子：commitTurn 若也开始落消息，与 afterTurn 并存时 + // 会以新 seq 重写整轮 → GmMessage 重复行 + ({ api, engine } = buildEngine()); + await engine.commitTurn({ + sessionId: "s-nopersist", + advancementKey: "adv-nopersist", + messages: [{ role: "user", content: "hello" }], + }); + await until(() => isTurnExtractedMock.mock.calls.length > 0); + await new Promise((r) => setTimeout(r, 50)); + expect(saveMessageMock).not.toHaveBeenCalled(); + }); + + it("warns and skips marker + extraction when the session cannot be resolved", async () => { + ({ api, engine } = buildEngine()); + const res = await engine.commitTurn({ + advancementKey: "adv-no-session", + messages: [{ role: "user", content: "orphan" }], + }); + expect(res).toEqual({ status: "committed" }); + expect(commitMock).not.toHaveBeenCalled(); + expect(isTurnExtractedMock).not.toHaveBeenCalled(); + expect(api.logger.warn).toHaveBeenCalledWith( + expect.stringContaining("cannot resolve session"), + ); + }); +}); diff --git a/test/integration.retention.test.ts b/test/integration.retention.test.ts index 397a797..c9a6c60 100644 --- a/test/integration.retention.test.ts +++ b/test/integration.retention.test.ts @@ -16,6 +16,8 @@ const SID_A = `retention-a-${Date.now()}`; // referenced 场景：前半提取 const SID_B = `retention-b-${Date.now()}`; // recentTurns 场景：全部提取 const SID_C = `retention-c-${Date.now()}`; // retentionDays 场景：无 user 消息（recentTurns 下全保护） const SID_D = `retention-d-${Date.now()}`; // 无 user 消息的 session +const SID_E = `retention-e-${Date.now()}`; // 空提取：extracted=true 但 producedKnowledge=false +const SID_F = `retention-f-${Date.now()}`; // 遗留行：直接 SET extracted=true，无 producedKnowledge 属性 const DAY = 86_400_000; async function seedSession(sid: string, turns: number, roles: (t: number) => string): Promise { @@ -49,6 +51,19 @@ async function setCreatedAt(sid: string, turn: number, ts: number): Promise { + const session = getSession(driver); + try { + await session.run( + "MATCH (m:GmMessage {sessionId: $sid}) SET m.extracted = true", + { sid }, + ); + } finally { + await session.close(); + } +} + describe.skipIf(!ENABLED)("GmMessage retention (Docker)", () => { beforeAll(async () => { driver = getDriver({ uri: NEO4J_URI!, user: "neo4j", password: "graphmemory" }); @@ -94,6 +109,12 @@ describe.skipIf(!ENABLED)("GmMessage retention (Docker)", () => { // D：2 条 assistant、全部提取、无 user 消息 await seedSession(SID_D, 2, () => "assistant"); await markExtracted(driver, SID_D, 1); + // E：4 条空提取 —— LLM 成功返回零节点零边：extracted=true, producedKnowledge=false + await seedSession(SID_E, 4, () => "assistant"); + await markExtracted(driver, SID_E, 3, false); + // F：遗留行（标记机制上线前已提取）：无 producedKnowledge 属性 + await seedSession(SID_F, 2, () => "assistant"); + await setExtractedLegacy(SID_F); }); afterAll(async () => { @@ -152,7 +173,8 @@ describe.skipIf(!ENABLED)("GmMessage retention (Docker)", () => { expect(result.dryRun).toBe(true); expect(result.deletedRows).toBe(0); - // 候选 = 剩余全部已提取：B(6-9) + C(2,3) + D(0,1) = 8（A 的 6-9 未提取不算） + // 候选 = 剩余全部 producedKnowledge=true：B(6-9) + C(2,3) + D(0,1) = 8 + // （E 空提取 pk=false、F 遗留行无属性 —— 均不进候选） expect(result.selectedRows).toBe(8); expect(await survivingTurns(SID_A)).toEqual([6, 7, 8, 9]); expect(await survivingTurns(SID_B)).toEqual([6, 7, 8, 9]); @@ -170,6 +192,20 @@ describe.skipIf(!ENABLED)("GmMessage retention (Docker)", () => { expect(await survivingTurns(SID_D)).toEqual([]); }); + it("keeps empty-extraction turns and legacy rows (producedKnowledge gate)", async () => { + const policy = normalizeMessageRetentionPolicy({ keep: "referenced" }); + const result = await runMessageRetention(driver, policy); + + // E：空提取（extracted=true, producedKnowledge=false）—— 原始证据保留， + // 日后换更好的 prompt/模型仍可回捞 + expect(await survivingTurns(SID_E)).toEqual([0, 1, 2, 3]); + // F：遗留行（无 producedKnowledge 属性）—— fail-closed 不删 + expect(await survivingTurns(SID_F)).toEqual([0, 1]); + // 上一测试已清空全部 producedKnowledge=true 候选，本轮零删除 + expect(result.selectedRows).toBe(0); + expect(result.deletedRows).toBe(0); + }); + it("reports per-role stats and session spread", async () => { // 上一测试后 A 仅剩 turns 6-9（user 6/8, assistant 7/9）——再跑一轮 referenced 空转 const policy = normalizeMessageRetentionPolicy({ keep: "referenced" }); @@ -182,7 +218,7 @@ describe.skipIf(!ENABLED)("GmMessage retention (Docker)", () => { it("batches deletion and reports hasMore when candidates exceed batchSize", async () => { // 专用 session：6 条全部提取、无 user 消息（referenced 下全部可候选） - const sid = `retention-e-${Date.now()}`; + const sid = `retention-batch-${Date.now()}`; await seedSession(sid, 6, () => "assistant"); await markExtracted(driver, sid, 5); diff --git a/test/integration.turn-commit.test.ts b/test/integration.turn-commit.test.ts new file mode 100644 index 0000000..57d3635 --- /dev/null +++ b/test/integration.turn-commit.test.ts @@ -0,0 +1,77 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import type { Driver } from "neo4j-driver"; +import { getDriver, initSchema, closeDriver, getSession } from "../src/store/db.ts"; +import { commitTurnAdvance } from "../src/store/store.ts"; + +// 仅在显式提供 NEO4J_TEST_URI 时运行（需要 Docker Neo4j，CI 执行）。 +// 刻意不默认 bolt://localhost:7687：本测试会写/删数据，绝不允许 +// 因缺省值静默落到可能是真实数据库的端口上。 +const NEO4J_URI = process.env.NEO4J_TEST_URI; +const ENABLED = !!process.env.NEO4J_INTEGRATION && !!NEO4J_URI; +const SID = `turncommit-${Date.now()}`; + +let driver: Driver; + +async function markerCount(advancementKey: string): Promise { + const session = getSession(driver); + try { + const res = await session.run( + "MATCH (t:GmTurnCommit {advancementKey: $key}) RETURN count(t) AS c", + { key: advancementKey }, + ); + return Number(res.records[0].get("c")); + } finally { + await session.close(); + } +} + +describe.skipIf(!ENABLED)("GmTurnCommit marker (Docker)", () => { + beforeAll(async () => { + driver = getDriver({ + uri: NEO4J_URI!, + user: process.env.NEO4J_TEST_USER ?? "neo4j", + password: process.env.NEO4J_TEST_PASSWORD ?? "testpassword", + }); + await initSchema(driver); + }); + + afterAll(async () => { + // 只清理本测试写入的标记，不动其他数据 + const session = getSession(driver); + try { + await session.run( + "MATCH (t:GmTurnCommit {sessionId: $sid}) DETACH DELETE t", + { sid: SID }, + ); + } finally { + await session.close(); + } + await closeDriver(); + }); + + it("initSchema creates the uniqueness constraint backing atomic idempotent commits", async () => { + const session = getSession(driver); + try { + const res = await session.run( + "SHOW CONSTRAINTS WHERE type = 'UNIQUENESS' AND entitiesLabel = 'GmTurnCommit'", + ); + expect(res.records.length).toBeGreaterThanOrEqual(1); + } finally { + await session.close(); + } + }); + + it("first write commits, retry with the same advancementKey reports duplicate", async () => { + const key = `${SID}-retry`; + await expect(commitTurnAdvance(driver, SID, key, 3)).resolves.toBe("committed"); + await expect(commitTurnAdvance(driver, SID, key, 3)).resolves.toBe("duplicate"); + expect(await markerCount(key)).toBe(1); + }); + + it("distinct advancementKeys commit independently", async () => { + await expect(commitTurnAdvance(driver, SID, `${SID}-a`, 1)).resolves.toBe("committed"); + await expect(commitTurnAdvance(driver, SID, `${SID}-b`, 2)).resolves.toBe("committed"); + expect(await markerCount(`${SID}-a`)).toBe(1); + expect(await markerCount(`${SID}-b`)).toBe(1); + }); +}); diff --git a/test/llm-cooldown-recovery.test.ts b/test/llm-cooldown-recovery.test.ts new file mode 100644 index 0000000..c8669a5 --- /dev/null +++ b/test/llm-cooldown-recovery.test.ts @@ -0,0 +1,98 @@ +import { mkdtemp, rm, utimes, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { createCompleteFn } from "../src/engine/llm.ts"; + +/** + * P2 修复回归：LLM 冷却期间，oauthPath 会话文件被外部进程重写 + * （`openclaw graph-memory auth login` / CLI extract 刷新 token）必须 + * 立即解除冷却并重试，而不是等满 10 分钟。 + * fetch 全程 mock，绝不触网；会话文件用临时目录 + utimes 精确控制 mtime。 + */ + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + vi.unstubAllGlobals(); + await Promise.all(temporaryDirectories.splice(0).map((dir) => rm(dir, { + recursive: true, + force: true, + }))); +}); + +/** 无 expiresAt → needsRefresh=false，跳过刷新直接进入请求（fetch mock 拦截）。 */ +async function writeOAuthFile(authPath: string, accessToken: string, mtime: Date): Promise { + await writeFile(authPath, JSON.stringify({ + accessToken, + accountId: "test-account", + providerId: "openai-codex", + })); + await utimes(authPath, mtime, mtime); +} + +function codexSuccessResponse(): Response { + return new Response( + JSON.stringify({ output: [{ type: "message", content: [{ type: "output_text", text: "ok" }] }] }), + { status: 200, headers: { "content-type": "application/json" } }, + ); +} + +describe("LLM cooldown recovers when the OAuth session file is rewritten", () => { + it("fast-fails during cooldown, then auto-lifts after the file changes", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "graph-memory-cooldown-")); + temporaryDirectories.push(directory); + const oauthPath = path.join(directory, "oauth.json"); + + const fetchMock = vi.fn() + .mockResolvedValueOnce(new Response("unauthorized", { status: 401 })) + .mockResolvedValueOnce(codexSuccessResponse()); + vi.stubGlobal("fetch", fetchMock); + + const complete = createCompleteFn("gpt-5.6-luna", { provider: "oauth", oauthPath }); + + // 1) 首次调用：401（持久性凭证错误）→ 触发冷却 + await writeOAuthFile(oauthPath, "stale-token", new Date(1_700_000_000_000)); + await expect(complete("sys", "user")).rejects.toThrow(/OAuth LLM API 401/); + expect(fetchMock).toHaveBeenCalledTimes(1); + + // 2) 文件未变：冷却期快速失败，不再发请求 + await expect(complete("sys", "user")).rejects.toThrow(/LLM paused for \d+s/); + expect(fetchMock).toHaveBeenCalledTimes(1); + + // 3) 外部进程重写会话文件（mtime 变化）→ 冷却自动解除并重试 + await writeOAuthFile(oauthPath, "fresh-token", new Date(1_700_000_060_000)); + await expect(complete("sys", "user")).resolves.toBe("ok"); + expect(fetchMock).toHaveBeenCalledTimes(2); + // 新 token 确实被使用（第 2 次请求的 Authorization 头） + const authHeader = fetchMock.mock.calls[1][1].headers["Authorization"]; + expect(authHeader).toBe("Bearer fresh-token"); + }); + + it("keeps the cooldown while the file stays untouched", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "graph-memory-cooldown-")); + temporaryDirectories.push(directory); + const oauthPath = path.join(directory, "oauth.json"); + + const fetchMock = vi.fn() + .mockResolvedValueOnce(new Response("unauthorized", { status: 401 })) + .mockResolvedValueOnce(codexSuccessResponse()); + vi.stubGlobal("fetch", fetchMock); + + const complete = createCompleteFn("gpt-5.6-luna", { provider: "oauth", oauthPath }); + + await writeOAuthFile(oauthPath, "stale-token", new Date(1_700_000_000_000)); + await expect(complete("sys", "user")).rejects.toThrow(/OAuth LLM API 401/); + + // 相同 mtime 重写内容（异常场景）：mtime 未变 → 维持冷却 + await writeFile(oauthPath, JSON.stringify({ + accessToken: "same-mtime-token", + accountId: "test-account", + providerId: "openai-codex", + })); + await utimes(oauthPath, new Date(1_700_000_000_000), new Date(1_700_000_000_000)); + await expect(complete("sys", "user")).rejects.toThrow(/LLM paused for \d+s/); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/test/llm-guard.test.ts b/test/llm-guard.test.ts index 75e9df7..935f426 100644 --- a/test/llm-guard.test.ts +++ b/test/llm-guard.test.ts @@ -52,6 +52,23 @@ describe("LlmFailureGuard", () => { expect(extractLlmStatus(new Error("[graph-memory] OAuth LLM API 404: model not found"))).toBe(404); }); + it("pauses when the OAuth refresh token is rejected (persistent credential failure)", () => { + // token 端点的 400 = invalid_grant（refresh token 被吊销/过期）—— 持久性故障 + const guard = new LlmFailureGuard(60_000, () => 1_000); + expect(guard.tripIfNeeded(new Error("OAuth refresh failed (400): invalid_grant"))).toBe(true); + expect(guard.tripIfNeeded(new Error("OAuth refresh failed (401): unauthorized"))).toBe(true); + expect(guard.canRun()).toBe(false); + }); + + it("does not pause for transient OAuth refresh failures", () => { + const guard = new LlmFailureGuard(60_000, () => 1_000); + expect(guard.tripIfNeeded(new Error("OAuth refresh failed (429): slow down"))).toBe(false); + expect(guard.tripIfNeeded(new Error("OAuth refresh failed (500): upstream error"))).toBe(false); + expect(guard.tripIfNeeded(new Error("OAuth refresh returned no access token"))).toBe(false); + expect(guard.tripIfNeeded(new Error("OAuth session from /x is expired and has no refresh token"))).toBe(false); + expect(guard.canRun()).toBe(true); + }); + it("pauses Anthropic authentication failures", () => { const guard = new LlmFailureGuard(60_000, () => 1_000); diff --git a/test/register-guard.test.ts b/test/register-guard.test.ts index 9656f77..6bc4478 100644 --- a/test/register-guard.test.ts +++ b/test/register-guard.test.ts @@ -1,5 +1,81 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +// ── 模块隔离：register() 的完整初始化路径绝不能触碰真实 Neo4j ── +// 不 mock 的话 getDriver() 会连 bolt://localhost:7687，initSchema 还会执行 +// 真实 DDL —— 任何 7687 端口有真实库的机器上裸跑 `npm test` 都有写风险。 +vi.mock("../src/store/db.ts", () => ({ + getDriver: () => ({}), + initSchema: async () => {}, + getSession: () => ({ close: async () => {} }), + closeDriver: async () => {}, +})); + +vi.mock("../src/store/store.ts", () => ({ + saveMessage: async () => {}, + getUnextracted: async () => [], + getMaxTurnIndex: async () => 0, + markExtracted: async () => {}, + isTurnExtracted: async () => false, + upsertNode: async () => ({ node: {}, isNew: false }), + upsertEdge: async () => {}, + findByName: async () => null, + updateNode: async () => null, + deleteNode: async () => {}, + deprecateNodeAndDisconnect: async () => {}, + getBySession: async () => [], + edgesFrom: async () => [], + edgesTo: async () => [], + edgesTouching: async () => [], + deleteEdges: async () => {}, + mergeNodes: async () => {}, + deprecate: async () => {}, + getStats: async () => ({}), +})); + +// resolveProvider 必须是真实实现：baseUrl 归一化的用例断言的就是它的推断告警。 +// 只替换 createCompleteFn（真实实现会发起 HTTP / 触发 OAuth 文件读取）。 +vi.mock("../src/engine/llm.ts", async (importActual) => { + const actual = await importActual(); + return { + ...actual, + createCompleteFn: () => async () => "", + }; +}); + +vi.mock("../src/engine/embed.ts", () => ({ + createEmbedFn: async () => null, +})); + +vi.mock("../src/recaller/recall.ts", () => ({ + Recaller: class { + setEmbedFn(): void {} + hasEmbedFn(): boolean { return false; } + get embedFn() { return null; } + async recall() { return { nodes: [], edges: [], tokenEstimate: 0 }; } + async syncEmbed(): Promise {} + }, + parseTimeRange: () => null, +})); + +vi.mock("../src/extractor/extract.ts", () => ({ + Extractor: class { + async extract() { return { nodes: [], edges: [] }; } + async finalize() { return { promotedSkills: [], newEdges: [], invalidations: [] }; } + }, +})); + +vi.mock("../src/format/assemble.ts", () => ({ + assembleContext: async () => ({ xml: "", systemPrompt: "", tokens: 0 }), +})); + +vi.mock("../src/graph/maintenance.ts", () => ({ + runMaintenance: async () => ({ durationMs: 0 }), +})); + +vi.mock("../src/routes/crud.ts", () => ({ + registerCrudRoutes: () => {}, +})); + import graphMemoryProPlugin from "../index.ts"; import { closeDriver } from "../src/store/db.ts"; diff --git a/test/session-identity.test.ts b/test/session-identity.test.ts index 9df14e7..976a8df 100644 --- a/test/session-identity.test.ts +++ b/test/session-identity.test.ts @@ -119,6 +119,7 @@ type EngineHarness = { readonly childSessionKey: string; readonly parentSessionId?: string; }) => Promise<{ readonly rollback: () => void }>; + readonly dispose: () => Promise; }; // register() 带防重复注册守卫（模块级 activeEngine）：同一进程内未 dispose 的 From fcfec617235723311805b2877cd2f084d8c0ab5a Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Tue, 1 Sep 2026 08:08:39 +0000 Subject: [PATCH 17/29] Fix: Test synced passwords --- .github/workflows/ci.yml | 3 +++ test/integration.turn-commit.test.ts | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c8706b..6f04e6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,6 +94,9 @@ jobs: # 集成测试（尤其删除类的 retention 测试）要求显式指定 URI； # runner 上的 service container 是一次性实例，7687 安全 NEO4J_TEST_URI: bolt://localhost:7687 + # 与 service 的 NEO4J_AUTH 保持同一来源，供可配置凭据的测试使用。 + NEO4J_TEST_USER: neo4j + NEO4J_TEST_PASSWORD: graphmemory CI: "true" shellcheck: diff --git a/test/integration.turn-commit.test.ts b/test/integration.turn-commit.test.ts index 57d3635..8a61751 100644 --- a/test/integration.turn-commit.test.ts +++ b/test/integration.turn-commit.test.ts @@ -30,7 +30,8 @@ describe.skipIf(!ENABLED)("GmTurnCommit marker (Docker)", () => { driver = getDriver({ uri: NEO4J_URI!, user: process.env.NEO4J_TEST_USER ?? "neo4j", - password: process.env.NEO4J_TEST_PASSWORD ?? "testpassword", + // Keep the local fallback aligned with the disposable Neo4j instance in CI. + password: process.env.NEO4J_TEST_PASSWORD ?? "graphmemory", }); await initSchema(driver); }); @@ -53,7 +54,9 @@ describe.skipIf(!ENABLED)("GmTurnCommit marker (Docker)", () => { const session = getSession(driver); try { const res = await session.run( - "SHOW CONSTRAINTS WHERE type = 'UNIQUENESS' AND entitiesLabel = 'GmTurnCommit'", + "SHOW CONSTRAINTS YIELD type, labelsOrTypes " + + "WHERE type = 'UNIQUENESS' AND 'GmTurnCommit' IN labelsOrTypes " + + "RETURN type", ); expect(res.records.length).toBeGreaterThanOrEqual(1); } finally { From 6bd0d7ebc0ede3d245a19be032d996a35c9333ef Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:52:28 +0000 Subject: [PATCH 18/29] Fix: Context engine "graph-memory-pro" factory returned null instead of a ContextEngine object.. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复如下问题： 07:54:21 [context-engine] Context engine "graph-memory-pro" degraded to "legacy" for this logical turn: Context engine "graph-memory-pro" factory returned null instead of a ContextEngine object.. The "legacy" engine will handle only this turn; configuration is unchanged, and "graph-memory-pro" will be retried next turn. --- index.ts | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/index.ts b/index.ts index ddba531..f77044c 100755 --- a/index.ts +++ b/index.ts @@ -39,6 +39,28 @@ export function isGraphMemoryCliInvocation(argv: readonly string[] = process.arg return argv.slice(2).includes("graph-memory"); } +/** + * Host registration modes that never serve logical turns ("readOnlyDiscovery" + * lifecycle in the host registry — turn resolution degrades such entries to + * legacy by design, and runtime entries are adopted from the composition root + * instead of being replaced). Running the full runtime init (second Neo4j + * driver, embed probe, schema DDL) in these loads is pure waste, and their + * registerContextEngine calls are no-ops against a runtime registry entry. + * Modes: cli-metadata (CLI discovery), discovery / tool-discovery (scoped + * loads, e.g. config hot reload inspection), setup-only (setup contract with + * empty pluginConfig). Undefined mode (OpenClaw < 2026.7) counts as full. + */ +const METADATA_ONLY_REGISTRATION_MODES = new Set([ + "cli-metadata", + "discovery", + "tool-discovery", + "setup-only", +]); + +export function isMetadataOnlyRegistration(mode: unknown): boolean { + return typeof mode === "string" && METADATA_ONLY_REGISTRATION_MODES.has(mode); +} + // ─── 从 OpenClaw config 读默认 model 名 ────────────────────── /** @@ -283,7 +305,7 @@ const graphMemoryProPlugin = { ); } if ( - api.registrationMode === "cli-metadata" || + isMetadataOnlyRegistration(api.registrationMode) || isGraphMemoryCliInvocation() ) { return; @@ -292,7 +314,14 @@ const graphMemoryProPlugin = { // 防重复注册：CLI 元数据仍可重复注册（幂等），但运行时只允许一份。 // 复用现有引擎，只重绑 ContextEngine 工厂（见 activeEngine 上的说明）。 if (activeEngine) { - api.registerContextEngine("graph-memory-pro", () => activeEngine); + // 工厂必须捕获引擎对象本身，绝不能写成 `() => activeEngine`： + // host 逐逻辑 turn 惰性调用工厂，dispose() 清空 activeEngine 之后 + // 该闭包会返回 null —— host 按契约判定 "factory returned null" 并 + // 逐回合降级 legacy，直到 gateway 重启（2026-09-01 事故）。返回已 + // dispose 的引擎对象是安全的：方法仍满足 ContextEngine 契约，且 + // per-session 状态自愈（msgSeq 经 getMaxTurnIndex 从 DB 恢复）。 + const reusableEngine = activeEngine; + api.registerContextEngine("graph-memory-pro", () => reusableEngine); api.logger.warn("[graph-memory-pro] duplicate register() ignored; reusing active engine"); return; } From 42b18e0831580084beef0223bb6aa74b750916b3 Mon Sep 17 00:00:00 2001 From: TriDefender Date: Wed, 2 Sep 2026 14:36:46 +0000 Subject: [PATCH 19/29] =?UTF-8?q?chore:=20remove=20dead=20code=20=E2=80=94?= =?UTF-8?q?=20legacy=20store=20APIs=20bypassed=20by=20Neo4j=20versioning?= =?UTF-8?q?=20(#1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dead code found by cross-referencing every export (production, CLI, tests) on the desktop-2.0 codebase: Remove entirely (zero callers): - vectorSearch(): legacy compat wrapper, only ever wrapped vectorSearchWithScore(); nothing (production or tests) calls it - getAllVectors(): superseded by the Neo4j vector index — dedup now uses db.index.vector.queryNodes instead of pulling all embeddings - getAllCommunitySummaries(): no callers anywhere - updatePageranks(): production writes pagerank via GDS (gds.pageRank.write / uniform fallback) directly in pagerank.ts; this batch writer was only referenced by one integration test, which now sets pagerank via Cypher directly (same coverage of the topNodes read path) De-export (still used internally, no external importers): - ScoredNode / ScoredCommunity return types (store.ts) - LlmConfig (llm.ts — cli.ts only imports ReasoningEffort) - RecallOptions / RecallTimeField (recall.ts) Verification: tsc clean; vitest 239 passed / 76 skipped (Neo4j integration gated behind NEO4J_INTEGRATION=1). The one failing test (installer-upgrade, spawnSync bash ETIMEDOUT) fails identically on the unmodified branch — sandbox networking, unrelated to this change. Co-authored-by: TriDefender <173548745+TriDefender@users.noreply.github.com> --- src/engine/llm.ts | 2 +- src/recaller/recall.ts | 4 +- src/store/store.ts | 67 +--------------------------------- test/integration.neo4j.test.ts | 11 ++++-- 4 files changed, 13 insertions(+), 71 deletions(-) diff --git a/src/engine/llm.ts b/src/engine/llm.ts index 6442368..eeae6ba 100755 --- a/src/engine/llm.ts +++ b/src/engine/llm.ts @@ -43,7 +43,7 @@ export type ReasoningEffort = "low" | "medium" | "high"; const DEFAULT_REASONING_EFFORT: ReasoningEffort = "medium"; -export interface LlmConfig { +interface LlmConfig { /** 显式 provider 切换。未设时按 baseURL 是否存在推断（向后兼容，仅产生 openai/anthropic）。 */ provider?: LlmProvider; apiKey?: string; diff --git a/src/recaller/recall.ts b/src/recaller/recall.ts index 1c1f515..460bad4 100755 --- a/src/recaller/recall.ts +++ b/src/recaller/recall.ts @@ -25,9 +25,9 @@ export function buildNodeEmbeddingText( // ─── 时间筛选 ─────────────────────────────────────────────── -export type RecallTimeField = "createdAt" | "updatedAt"; +type RecallTimeField = "createdAt" | "updatedAt"; -export interface RecallOptions { +interface RecallOptions { /** ISO 8601 字符串；只返回 timeField 对应时刻 >= after 的节点 */ after?: string; /** ISO 8601 字符串；只返回 timeField 对应时刻 <= before 的节点 */ diff --git a/src/store/store.ts b/src/store/store.ts index a764508..1dad863 100755 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -415,22 +415,6 @@ export async function mergeNodes(driver: Driver, keepId: string, mergeId: string } } -/** 批量更新 PageRank 分数 */ -export async function updatePageranks(driver: Driver, scores: Map): Promise { - if (scores.size === 0) return; - const session = getSession(driver); - try { - const entries = Array.from(scores.entries()).map(([id, score]) => ({ id, score })); - await session.run(` - UNWIND $entries AS entry - MATCH (n:Task|Skill|Event {id: entry.id}) - SET n.pagerank = entry.score - `, { entries }); - } finally { - await session.close(); - } -} - /** 批量更新社区 ID */ export async function updateCommunities(driver: Driver, labels: Map): Promise { if (labels.size === 0) return; @@ -664,7 +648,7 @@ export async function topNodes(driver: Driver, limit = 6): Promise { // ─── 向量搜索 ─────────────────────────────────────────────── -export type ScoredNode = { node: GmNode; score: number }; +type ScoredNode = { node: GmNode; score: number }; export async function vectorSearchWithScore( driver: Driver, queryVec: number[], limit: number, minScore = 0.35, @@ -688,15 +672,8 @@ export async function vectorSearchWithScore( } } -export async function vectorSearch( - driver: Driver, queryVec: number[], limit: number, minScore = 0.35, -): Promise { - const scored = await vectorSearchWithScore(driver, queryVec, limit, minScore); - return scored.map(s => s.node); -} - /** 社区向量搜索 */ -export type ScoredCommunity = { id: string; summary: string; score: number; nodeCount: number }; +type ScoredCommunity = { id: string; summary: string; score: number; nodeCount: number }; export async function communityVectorSearch( driver: Driver, queryVec: number[], minScore = 0.15, @@ -750,24 +727,6 @@ export async function getVectorHash(driver: Driver, nodeId: string): Promise> { - const session = getSession(driver); - try { - const result = await session.run(` - MATCH (n:Task|Skill|Event {status: 'active'}) - WHERE n.embedding IS NOT NULL - RETURN n.id AS nodeId, n.embedding AS embedding - `); - return result.records.map(r => ({ - nodeId: r.get("nodeId"), - embedding: r.get("embedding"), - })); - } finally { - await session.close(); - } -} - // ─── 图遍历 ──────────────────────────────────────────────── export async function graphWalk( @@ -1213,28 +1172,6 @@ export async function getCommunitySummaryBySignature( } } -export async function getAllCommunitySummaries(driver: Driver): Promise { - const session = getSession(driver); - try { - const result = await session.run( - "MATCH (c:Community) RETURN c ORDER BY c.nodeCount DESC" - ); - return result.records.map(r => { - const c = r.get("c").properties; - return { - id: c.id, - summary: c.summary, - nodeCount: toInt(c.nodeCount), - memberSignature: c.memberSignature ?? null, - createdAt: toInt(c.createdAt), - updatedAt: toInt(c.updatedAt), - }; - }); - } finally { - await session.close(); - } -} - export async function pruneCommunitySummaries(driver: Driver): Promise { const session = getSession(driver); try { diff --git a/test/integration.neo4j.test.ts b/test/integration.neo4j.test.ts index 3b12b2c..6b7f211 100644 --- a/test/integration.neo4j.test.ts +++ b/test/integration.neo4j.test.ts @@ -8,7 +8,7 @@ import { saveMessage, getUnextracted, markExtracted, isTurnExtracted, deprecate, getStats, mergeNodes, searchNodes, topNodes, getBySession, saveVector, vectorSearchWithScore, getVectorHash, - updateCommunities, updatePageranks, + updateCommunities, deleteNode, deprecateNodeAndDisconnect, deleteEdges, } from "../src/store/store.ts"; @@ -520,11 +520,16 @@ describe.skipIf(!ENABLED)("Neo4j integration (Docker)", () => { expect(hits.length).toBeGreaterThanOrEqual(2); expect(hits.every(n => n.name.includes("kubernetes") || n.description.includes("k8s") || n.content.includes("kubectl"))).toBe(true); - // topNodes：先 updatePageranks，再查 top + // topNodes：直接用 Cypher 写 pagerank（生产由 gds.pageRank.write 写入），再查 top const { node: top } = await upsertNode(driver, { type: "TASK", name: "Top Ranked Task", description: "high", content: "important", }, TEST_SID); - await updatePageranks(driver, new Map([[top.id, 999]])); + const w = driver.session(); + try { + await w.run("MATCH (n:Task|Skill|Event {id: $id}) SET n.pagerank = 999", { id: top.id }); + } finally { + await w.close(); + } const topHits = await topNodes(driver, 3); expect(topHits.length).toBeGreaterThanOrEqual(1); expect(topHits[0].id).toBe(top.id); From b5a18b12e9f11fc823c0b2fa64b15edc02def697 Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:24:00 +0000 Subject: [PATCH 20/29] Removed plugin residuals --- .video_agent/plugin_root | 1 - test/plugin-cli-lifecycle.test.ts | 12 +++++++++++ test/register-guard.test.ts | 35 +++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) delete mode 100644 .video_agent/plugin_root diff --git a/.video_agent/plugin_root b/.video_agent/plugin_root deleted file mode 100644 index 76fc7ab..0000000 --- a/.video_agent/plugin_root +++ /dev/null @@ -1 +0,0 @@ -C:\Users\18913\.zcode\cli\plugins\cache\zcode-plugins-official\video-agent-kit\0.4.3 \ No newline at end of file diff --git a/test/plugin-cli-lifecycle.test.ts b/test/plugin-cli-lifecycle.test.ts index 22de604..b0aa0bf 100644 --- a/test/plugin-cli-lifecycle.test.ts +++ b/test/plugin-cli-lifecycle.test.ts @@ -84,6 +84,18 @@ describe("plugin CLI lifecycle", () => { expect(api.registerHttpRoute).not.toHaveBeenCalled(); }); + it("registers only CLI metadata in discovery/tool-discovery/setup-only modes", () => { + for (const mode of ["discovery", "tool-discovery", "setup-only"]) { + const api = metadataApi(mode); + graphMemoryProPlugin.register(api as any); + expect(api.registerCli).toHaveBeenCalledOnce(); + expect(api.registerTool).not.toHaveBeenCalled(); + expect(api.registerContextEngine).not.toHaveBeenCalled(); + expect(api.registerHttpRoute).not.toHaveBeenCalled(); + expect(api.on).not.toHaveBeenCalled(); + } + }); + it("recognizes the plugin command for legacy OpenClaw discovery", () => { expect(isGraphMemoryCliInvocation(["node", "openclaw", "graph-memory", "auth", "login"])).toBe(true); expect(isGraphMemoryCliInvocation(["node", "openclaw", "gateway"])).toBe(false); diff --git a/test/register-guard.test.ts b/test/register-guard.test.ts index 6bc4478..4e3ec6a 100644 --- a/test/register-guard.test.ts +++ b/test/register-guard.test.ts @@ -124,6 +124,41 @@ describe("duplicate register() guard", () => { ); }); + // 2026-09-01 事故回归：host 逐逻辑 turn 惰性调用工厂。守卫曾把工厂重绑为 + // `() => activeEngine`（可变模块变量），dispose() 清空标记后工厂返回 null， + // host 按契约判定 "factory returned null" 并逐回合降级 legacy 直到重启。 + it("rebound factory still returns the engine after dispose clears the module flag", async () => { + const api1 = fullApi(); + graphMemoryProPlugin.register(api1); + const engineA = api1.registerContextEngine.mock.calls[0][1](); + createdEngine = engineA; + + const api2 = fullApi(); + graphMemoryProPlugin.register(api2); // 守卫路径重绑工厂 + const reboundFactory = api2.registerContextEngine.mock.calls[0][1]; + + await engineA.dispose(); // activeEngine → null + + expect(reboundFactory()).toBe(engineA); + expect(reboundFactory()).not.toBeNull(); + }); + + it("skips runtime init for discovery-mode loads (readOnlyDiscovery never serves turns)", () => { + const api1 = fullApi(); + graphMemoryProPlugin.register(api1); + createdEngine = api1.registerContextEngine.mock.calls[0][1](); + + // 配置热重载的 discovery 加载：只重注册 CLI 元数据，不碰运行时， + // 也不与活跃引擎守卫交互（mode 检查在守卫之前） + const api2 = fullApi(); + api2.registrationMode = "discovery"; + graphMemoryProPlugin.register(api2); + expect(api2.registerCli).toHaveBeenCalledOnce(); + expect(api2.registerContextEngine).not.toHaveBeenCalled(); + expect(api2.registerTool).not.toHaveBeenCalled(); + expect(api2.on).not.toHaveBeenCalled(); + }); + it("creates a fresh engine after dispose() (genuine reload)", async () => { const api1 = fullApi(); graphMemoryProPlugin.register(api1); From db20f0a5356ca2c0373ce0c4a3d6d7da08c75476 Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:21:34 +0000 Subject: [PATCH 21/29] refactor: dedupe shared implementations (Fixes #2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A1. Consolidate duplicated fetchRetry into src/engine/http.ts - llm.ts / embed.ts each carried a near-identical copy that had already drifted: the llm side lost the network-error retry branch during an earlier refactor. Shared module restores it for both. - Timeout errors are now a typed HttpTimeoutError; per-caller semantics preserved via retryOnTimeout (llm: false — a 60s timeout must not quadruple worst-case latency; embed: true). - Exhausted status retries now return the last response instead of a generic "failed after retries" error, so callers surface status-specific messages. - Add unit tests (8) covering status retry, network-error retry, timeout semantics, and label formatting. A2. extract.ts imports normalizeName from store.ts instead of carrying a byte-identical copy guarded by a cross-file consistency test. A3. New src/tokens.ts (CHARS_PER_TOKEN + estimateTokens); index.ts, recall.ts and assemble.ts now share the /3 heuristic. A4. Drop RecallResult.tokenEstimate — computed in three places, never read by anyone (updated 4 test fixtures accordingly). B1. store.ts uid(): crypto.randomUUID() replaces the hand-rolled Date.now()+Math.random() suffix. B3(partial). index.ts: extractAssistantText/extractUserText share textFromBlocks(); llm.ts/store.ts block-parsing left as-is (different input shapes). Deliberately out of scope: oauth.ts toBase64Url wrapper (cosmetic), JWT/SSE hand-rolled parsers (zero-dependency by design). tsc clean; vitest 241 passed / 76 skipped. The one failure (installer-upgrade, spawnSync bash ETIMEDOUT) is a pre-existing sandbox networking limitation, reproduced on the unmodified branch. Fixes #2 --- index.ts | 23 ++--- src/engine/embed.ts | 25 +----- src/engine/http.ts | 82 ++++++++++++++++++ src/engine/llm.ts | 42 +--------- src/extractor/extract.ts | 11 +-- src/format/assemble.ts | 5 +- src/recaller/recall.ts | 15 ++-- src/store/store.ts | 4 +- src/tokens.ts | 13 +++ src/types.ts | 1 - test/commit-turn.test.ts | 2 +- test/http-retry.test.ts | 143 ++++++++++++++++++++++++++++++++ test/integration.recall.test.ts | 7 -- test/normalize-name.test.ts | 21 ----- test/register-guard.test.ts | 2 +- test/session-identity.test.ts | 1 - 16 files changed, 268 insertions(+), 129 deletions(-) create mode 100644 src/engine/http.ts create mode 100644 src/tokens.ts create mode 100644 test/http-retry.test.ts diff --git a/index.ts b/index.ts index f77044c..4eee362 100755 --- a/index.ts +++ b/index.ts @@ -20,6 +20,7 @@ import { } from "./src/store/store.ts"; import { createCompleteFn, resolveProvider } from "./src/engine/llm.ts"; import { createEmbedFn } from "./src/engine/embed.ts"; +import { estimateTokens } from "./src/tokens.ts"; import { Recaller, parseTimeRange } from "./src/recaller/recall.ts"; import { Extractor } from "./src/extractor/extract.ts"; import { assembleContext } from "./src/format/assemble.ts"; @@ -158,17 +159,21 @@ function estimateMsgTokens(msg: any): number { const text = typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content ?? ""); - return Math.ceil(text.length / 3); + return estimateTokens(text.length); +} + +/** 从 content blocks 数组抽取纯文本（text block 拼接） */ +function textFromBlocks(blocks: any[]): string { + return blocks + .filter((b: any) => b && typeof b === "object" && b.type === "text" && typeof b.text === "string") + .map((b: any) => b.text) + .join("\n"); } export function extractAssistantText(msg: any): string { if (typeof msg.content === "string") return msg.content; if (!Array.isArray(msg.content)) return ""; - return msg.content - .filter((b: any) => b && typeof b === "object" && b.type === "text" && typeof b.text === "string") - .map((b: any) => b.text) - .join("\n") - .trim(); + return textFromBlocks(msg.content).trim(); } export function extractUserText(msg: any): string { @@ -178,11 +183,7 @@ export function extractUserText(msg: any): string { } else if (!Array.isArray(msg.content)) { raw = String(msg.content ?? ""); } else { - raw = msg.content - .filter((b: any) => b && typeof b === "object" && b.type === "text" && typeof b.text === "string") - .map((b: any) => b.text) - .join("\n") - .trim(); + raw = textFromBlocks(msg.content).trim(); } // 去掉 OpenClaw metadata（Sender JSON block、命令前缀、时间戳） const fenceEnd = raw.lastIndexOf("```"); diff --git a/src/engine/embed.ts b/src/engine/embed.ts index 740a35d..9626068 100755 --- a/src/engine/embed.ts +++ b/src/engine/embed.ts @@ -15,32 +15,11 @@ */ import type { EmbeddingConfig } from "../types.ts"; +import { fetchRetry } from "./http.ts"; export type EmbedMode = "db" | "query"; export type EmbedFn = (text: string, mode?: EmbedMode) => Promise; -// ─── 带重试 + 超时的 fetch ───────────────────────────────────── - -const RETRYABLE = new Set([429, 500, 502, 503, 529]); - -async function fetchRetry(url: string, init: RequestInit, retries = 3, timeoutMs = 10_000): Promise { - for (let i = 0; i <= retries; i++) { - const ctrl = new AbortController(); - const t = setTimeout(() => ctrl.abort(), timeoutMs); - try { - const res = await fetch(url, { ...init, signal: ctrl.signal }); - clearTimeout(t); - if (res.ok || i >= retries || !RETRYABLE.has(res.status)) return res; - await new Promise(r => setTimeout(r, 1000 * Math.pow(2, i))); - } catch (err: any) { - clearTimeout(t); - if (i >= retries) throw err; - await new Promise(r => setTimeout(r, 1000 * (i + 1))); - } - } - throw new Error("[graph-memory-pro] embed fetch failed after retries"); -} - // ─── Provider 识别 ─────────────────────────────────────────── /** @@ -105,7 +84,7 @@ export async function createEmbedFn(cfg: EmbeddingConfig | undefined): Promise ""); diff --git a/src/engine/http.ts b/src/engine/http.ts new file mode 100644 index 0000000..723b43d --- /dev/null +++ b/src/engine/http.ts @@ -0,0 +1,82 @@ +/** + * graph-memory-pro — 共享 HTTP 客户端工具 + * + * 统一 LLM / Embedding 调用的超时与重试语义（此前 llm.ts / embed.ts 各有一份 + * fetchRetry，且已发生行为分叉：llm 侧在重构中丢失了网络异常重试分支）。 + * + * 语义： + * - 可重试 HTTP 状态码（429/5xx）：指数退避重试 + * - 网络级异常（连接失败、连接被重置等）：同样重试 + * - 超时：默认立即抛出 HttpTimeoutError；仅当 retryOnTimeout=true 时计入重试 + * （LLM 调用耗时长，超时重试会成倍拉长最坏阻塞时间；embedding 调用短，重试无害） + */ + +const RETRYABLE_STATUS = new Set([429, 500, 502, 503, 529]); + +/** 请求超时（AbortError 的友好化包装；instanceof 可与网络异常区分） */ +export class HttpTimeoutError extends Error { + constructor(message: string) { + super(message); + this.name = "HttpTimeoutError"; + } +} + +export interface FetchRetryOptions { + /** 最大重试次数（不含首次），默认 3 */ + retries?: number; + /** 单次请求超时，默认 30_000ms */ + timeoutMs?: number; + /** 错误信息前缀（如 "[graph-memory] LLM"），保持各调用方原报错格式 */ + label?: string; + /** 超时是否计入重试；默认 false（立即抛出） */ + retryOnTimeout?: boolean; + /** 重试退避函数，attempt 从 0 开始；默认指数退避 1s/2s/4s…（测试可注入 0） */ + backoffMs?: (attempt: number) => number; +} + +export async function fetchWithTimeout( + url: string, + init: RequestInit, + timeoutMs: number, + label = "[graph-memory]", +): Promise { + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), timeoutMs); + try { + return await fetch(url, { ...init, signal: ctrl.signal }); + } catch (err: any) { + if (err?.name === "AbortError") { + throw new HttpTimeoutError(`${label} request timed out after ${timeoutMs}ms`); + } + throw err; + } finally { + clearTimeout(timer); + } +} + +export async function fetchRetry( + url: string, + init: RequestInit, + opts: FetchRetryOptions = {}, +): Promise { + const { + retries = 3, + timeoutMs = 30_000, + label = "[graph-memory]", + retryOnTimeout = false, + backoffMs = (attempt: number) => 1000 * Math.pow(2, attempt), + } = opts; + + for (let i = 0; i <= retries; i++) { + try { + const res = await fetchWithTimeout(url, init, timeoutMs, label); + if (res.ok || i >= retries || !RETRYABLE_STATUS.has(res.status)) return res; + await new Promise((r) => setTimeout(r, backoffMs(i))); + } catch (err) { + if (err instanceof HttpTimeoutError && !retryOnTimeout) throw err; + if (i >= retries) throw err; + await new Promise((r) => setTimeout(r, backoffMs(i))); + } + } + throw new Error(`${label} request failed after retries`); +} diff --git a/src/engine/llm.ts b/src/engine/llm.ts index eeae6ba..a6c67ac 100755 --- a/src/engine/llm.ts +++ b/src/engine/llm.ts @@ -35,6 +35,7 @@ import { extractOutputTextFromSse, } from "./oauth.ts"; import type { OAuthSession } from "./oauth.ts"; +import { fetchRetry } from "./http.ts"; export type LlmProvider = "openai" | "anthropic" | "oauth"; @@ -84,41 +85,6 @@ export function resolveProvider(cfg: LlmConfig | undefined): { return { provider: inferred, inferred: true }; } -async function fetchWithTimeout( - url: string, - init: RequestInit, - timeoutMs: number, -): Promise { - const ctrl = new AbortController(); - const timer = setTimeout(() => ctrl.abort(), timeoutMs); - try { - return await fetch(url, { ...init, signal: ctrl.signal }); - } catch (err: any) { - if (err?.name === "AbortError") { - throw new Error(`[graph-memory] LLM request timed out after ${timeoutMs}ms`); - } - throw err; - } finally { - clearTimeout(timer); - } -} - -const RETRYABLE = new Set([429, 500, 502, 503, 529]); - -async function fetchRetry( - url: string, - init: RequestInit, - retries: number, - timeoutMs: number, -): Promise { - for (let i = 0; i <= retries; i++) { - const res = await fetchWithTimeout(url, init, timeoutMs); - if (res.ok || i >= retries || !RETRYABLE.has(res.status)) return res; - await new Promise((r) => setTimeout(r, 1000 * Math.pow(2, i))); - } - throw new Error("[graph-memory] fetch failed after retries"); -} - /** * 构造 LLM CompleteFn。 * @@ -220,7 +186,7 @@ export function createCompleteFn( stream: false, text: { format: { type: "text" } }, }), - }, 3, timeoutMs); + }, { retries: 3, timeoutMs, label: "[graph-memory] LLM" }); if (!res.ok) { const errText = await res.text().catch(() => ""); @@ -274,7 +240,7 @@ export function createCompleteFn( system, messages: [{ role: "user", content: user }], }), - }, 3, timeoutMs); + }, { retries: 3, timeoutMs, label: "[graph-memory] LLM" }); if (!res.ok) { const errText = await res.text().catch(() => ""); throw new Error(`[graph-memory] Anthropic API ${res.status}: ${errText.slice(0, 200)}`); @@ -319,7 +285,7 @@ export function createCompleteFn( max_tokens: maxTokens, temperature: 0.1, }), - }, 3, timeoutMs); + }, { retries: 3, timeoutMs, label: "[graph-memory] LLM" }); if (!res.ok) { const errText = await res.text().catch(() => ""); throw new Error(`[graph-memory] LLM API ${res.status}: ${errText.slice(0, 200)}`); diff --git a/src/extractor/extract.ts b/src/extractor/extract.ts index 8e4f95d..f0bc367 100755 --- a/src/extractor/extract.ts +++ b/src/extractor/extract.ts @@ -8,6 +8,7 @@ import type { ExtractionResult, FinalizeResult } from "../types.ts"; import { EDGE_TYPES, isValidEdgeDirection } from "../types.ts"; import type { CompleteFn } from "../engine/llm.ts"; +import { normalizeName } from "../store/store.ts"; // ─── 节点/边合法值 ────────────────────────────────────────────── @@ -159,16 +160,6 @@ ${JSON.stringify(nodes.map(n => ({ ${summary}`; -// ─── 名称标准化（与 store.ts 一致）──────────────────────────── - -export function normalizeName(name: string): string { - return name.trim().toLowerCase() - .replace(/[\s_]+/g, "-") - .replace(/[^a-z0-9\u4e00-\u9fff\-]/g, "") - .replace(/-{2,}/g, "-") - .replace(/^-|-$/g, ""); -} - // ─── 边类型自动修正 ───────────────────────────────────────────── /** diff --git a/src/format/assemble.ts b/src/format/assemble.ts index 29838ec..37050ce 100755 --- a/src/format/assemble.ts +++ b/src/format/assemble.ts @@ -8,8 +8,7 @@ import type { Driver } from "neo4j-driver"; import type { GmNode, GmEdge } from "../types.ts"; import { getCommunitySummary, type CommunitySummary } from "../store/store.ts"; - -const CHARS_PER_TOKEN = 3; +import { CHARS_PER_TOKEN, estimateTokens } from "../tokens.ts"; export function buildSystemPromptAddition(params: { selectedNodes: Array<{ type: string; src: "active" | "recalled" }>; @@ -180,7 +179,7 @@ export async function assembleContext( }); const fullContent = systemPrompt + "\n\n" + xml; - return { xml, systemPrompt, tokens: Math.ceil(fullContent.length / CHARS_PER_TOKEN) }; + return { xml, systemPrompt, tokens: estimateTokens(fullContent.length) }; } function escapeXml(s: string): string { diff --git a/src/recaller/recall.ts b/src/recaller/recall.ts index 460bad4..7625a60 100755 --- a/src/recaller/recall.ts +++ b/src/recaller/recall.ts @@ -143,7 +143,7 @@ export class Recaller { seeds = await searchNodes(this.driver, query, limit); } - if (!seeds.length) return { nodes: [], edges: [], tokenEstimate: 0 }; + if (!seeds.length) return { nodes: [], edges: [] }; const seedIds = seeds.map(n => n.id); @@ -161,7 +161,7 @@ export class Recaller { this.cfg.recallMaxDepth, ); - if (!nodes.length) return { nodes: [], edges: [], tokenEstimate: 0 }; + if (!nodes.length) return { nodes: [], edges: [] }; // PPR 排序 const candidateIds = nodes.map(n => n.id); @@ -182,7 +182,6 @@ export class Recaller { return { nodes: filtered, edges: edges.filter(e => ids.has(e.fromId) && ids.has(e.toId)), - tokenEstimate: this.estimateTokens(filtered), }; } @@ -213,11 +212,11 @@ export class Recaller { seeds = await communityRepresentatives(this.driver, 2); } - if (!seeds.length) return { nodes: [], edges: [], tokenEstimate: 0 }; + if (!seeds.length) return { nodes: [], edges: [] }; const seedIds = seeds.map(n => n.id); const { nodes, edges } = await graphWalk(this.driver, seedIds, 1); - if (!nodes.length) return { nodes: [], edges: [], tokenEstimate: 0 }; + if (!nodes.length) return { nodes: [], edges: [] }; const candidateIds = nodes.map(n => n.id); const { scores: pprScores } = await personalizedPageRank( @@ -237,7 +236,6 @@ export class Recaller { return { nodes: filtered, edges: edges.filter(e => ids.has(e.fromId) && ids.has(e.toId)), - tokenEstimate: this.estimateTokens(filtered), }; } @@ -261,12 +259,9 @@ export class Recaller { const nodes = Array.from(nodeMap.values()); const edges = Array.from(edgeMap.values()); - return { nodes, edges, tokenEstimate: this.estimateTokens(nodes) }; + return { nodes, edges }; } - private estimateTokens(nodes: GmNode[]): number { - return Math.ceil(nodes.reduce((s, n) => s + n.content.length + n.description.length, 0) / 3); - } async syncEmbed(node: GmNode): Promise { if (!this.embed) return; diff --git a/src/store/store.ts b/src/store/store.ts index 1dad863..7838972 100755 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -7,7 +7,7 @@ import type { Driver } from "neo4j-driver"; import neo4j from "neo4j-driver"; -import { createHash } from "crypto"; +import { createHash, randomUUID } from "crypto"; import type { GmNode, GmEdge, EdgeType, NodeType, NodeTier } from "../types.ts"; import { NODE_TYPE_TO_LABEL, isValidEdgeDirection, EDGE_TYPES } from "../types.ts"; import { getSession } from "./db.ts"; @@ -20,7 +20,7 @@ function nint(v: number): any { // ─── 工具 ───────────────────────────────────────────────────── function uid(p: string): string { - return `${p}-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`; + return `${p}-${randomUUID()}`; } function toNode(r: any): GmNode { diff --git a/src/tokens.ts b/src/tokens.ts new file mode 100644 index 0000000..2938d9b --- /dev/null +++ b/src/tokens.ts @@ -0,0 +1,13 @@ +/** + * graph-memory-pro — token 估算 + * + * 统一的字符→token 粗估换算（约 3 字符 = 1 token，中英混合文本的经验值）。 + * 全仓库所有 token 估算必须经由本模块，避免系数多处漂移。 + */ + +export const CHARS_PER_TOKEN = 3; + +/** 按字符数粗估 token 数（向上取整） */ +export function estimateTokens(chars: number): number { + return Math.ceil(chars / CHARS_PER_TOKEN); +} diff --git a/src/types.ts b/src/types.ts index 5d0c3ca..d55a7b0 100755 --- a/src/types.ts +++ b/src/types.ts @@ -138,7 +138,6 @@ export interface FinalizeResult { export interface RecallResult { nodes: GmNode[]; edges: GmEdge[]; - tokenEstimate: number; } // ─── Embedding 配置 ────────────────────────────────────────── diff --git a/test/commit-turn.test.ts b/test/commit-turn.test.ts index 9acd688..422b267 100644 --- a/test/commit-turn.test.ts +++ b/test/commit-turn.test.ts @@ -45,7 +45,7 @@ vi.mock("../src/recaller/recall.ts", () => ({ setEmbedFn(): void {} hasEmbedFn(): boolean { return false; } get embedFn() { return null; } - async recall() { return { nodes: [], edges: [], tokenEstimate: 0 }; } + async recall() { return { nodes: [], edges: [] }; } async syncEmbed(): Promise {} }, parseTimeRange: () => null, diff --git a/test/http-retry.test.ts b/test/http-retry.test.ts new file mode 100644 index 0000000..1f9ecfb --- /dev/null +++ b/test/http-retry.test.ts @@ -0,0 +1,143 @@ +/** + * graph-memory — src/engine/http.ts 单元测试 + * + * 覆盖统一的超时/重试语义： + * - 可重试状态码（429/5xx）指数退避重试 + * - 网络异常重试（llm.ts 侧曾在此丢失，回归保护） + * - 超时默认立即抛出（HttpTimeoutError），retryOnTimeout=true 时计入重试 + * - 不可重试状态码（401 等）立即返回响应 + */ + +import { describe, it, expect, vi, afterEach } from "vitest"; +import { fetchRetry, HttpTimeoutError } from "../src/engine/http.ts"; + +const NO_BACKOFF = { backoffMs: () => 0 }; + +function jsonResponse(status: number): Response { + return new Response(JSON.stringify({ ok: false }), { status }); +} + +/** 永远挂起、仅在 abort 信号触发时以 AbortError 拒绝的 mock fetch（模拟超时） */ +function hangingFetch() { + return vi.fn((_url: string, init?: RequestInit) => + new Promise((_, reject) => { + init?.signal?.addEventListener("abort", () => { + const err = new Error("The operation was aborted"); + err.name = "AbortError"; + reject(err); + }); + }), + ); +} + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("fetchRetry 状态码重试", () => { + it("429 两次后 200：重试并成功，共请求 3 次", async () => { + const mock = vi.fn() + .mockResolvedValueOnce(jsonResponse(429)) + .mockResolvedValueOnce(jsonResponse(429)) + .mockResolvedValueOnce(new Response("{}", { status: 200 })); + vi.stubGlobal("fetch", mock); + + const res = await fetchRetry("https://x.test/v1", { method: "POST" }, NO_BACKOFF); + + expect(res.ok).toBe(true); + expect(mock).toHaveBeenCalledTimes(3); + }); + + it("持续 500：重试耗尽后返回最后一次响应（由调用方产出带状态码的报错）", async () => { + const mock = vi.fn().mockResolvedValue(jsonResponse(500)); + vi.stubGlobal("fetch", mock); + + const res = await fetchRetry("https://x.test/v1", { method: "POST" }, NO_BACKOFF); + + expect(res.status).toBe(500); + expect(mock).toHaveBeenCalledTimes(4); // 首次 + 3 次重试 + }); + + it("401 不可重试：立即返回，仅请求 1 次", async () => { + const mock = vi.fn().mockResolvedValue(jsonResponse(401)); + vi.stubGlobal("fetch", mock); + + const res = await fetchRetry("https://x.test/v1", { method: "POST" }, NO_BACKOFF); + + expect(res.status).toBe(401); + expect(mock).toHaveBeenCalledTimes(1); + }); +}); + +describe("fetchRetry 网络异常重试（llm.ts 侧曾丢失的行为）", () => { + it("网络错误后成功：重试生效", async () => { + const err = new Error("socket hang up"); + const mock = vi.fn() + .mockRejectedValueOnce(err) + .mockResolvedValueOnce(new Response("{}", { status: 200 })); + vi.stubGlobal("fetch", mock); + + const res = await fetchRetry("https://x.test/v1", { method: "POST" }, NO_BACKOFF); + + expect(res.ok).toBe(true); + expect(mock).toHaveBeenCalledTimes(2); + }); + + it("持续网络错误：重试耗尽后抛出最后一次错误", async () => { + const mock = vi.fn().mockRejectedValue(new Error("ECONNRESET")); + vi.stubGlobal("fetch", mock); + + await expect( + fetchRetry("https://x.test/v1", { method: "POST" }, NO_BACKOFF), + ).rejects.toThrow("ECONNRESET"); + expect(mock).toHaveBeenCalledTimes(4); + }); +}); + +describe("fetchRetry 超时语义", () => { + it("默认超时不重试：立即抛 HttpTimeoutError，仅请求 1 次", async () => { + const mock = hangingFetch(); + vi.stubGlobal("fetch", mock); + + await expect( + fetchRetry("https://x.test/v1", { method: "POST" }, { timeoutMs: 30, ...NO_BACKOFF }), + ).rejects.toBeInstanceOf(HttpTimeoutError); + expect(mock).toHaveBeenCalledTimes(1); + }); + + it("retryOnTimeout=true：超时计入重试，第 2 次成功", async () => { + const mock = vi.fn() + .mockImplementationOnce((_url: string, init?: RequestInit) => + new Promise((_, reject) => { + init?.signal?.addEventListener("abort", () => { + const err = new Error("aborted"); + err.name = "AbortError"; + reject(err); + }); + }), + ) + .mockResolvedValueOnce(new Response("{}", { status: 200 })); + vi.stubGlobal("fetch", mock); + + const res = await fetchRetry( + "https://x.test/v1", + { method: "POST" }, + { timeoutMs: 30, retryOnTimeout: true, ...NO_BACKOFF }, + ); + + expect(res.ok).toBe(true); + expect(mock).toHaveBeenCalledTimes(2); + }); + + it("超时错误信息包含 label 与时长", async () => { + vi.stubGlobal("fetch", hangingFetch()); + + await expect( + fetchRetry( + "https://x.test/v1", + { method: "POST" }, + { timeoutMs: 25, label: "[graph-memory] LLM", ...NO_BACKOFF }, + ), + ).rejects.toThrow("[graph-memory] LLM request timed out after 25ms"); + }); +}); diff --git a/test/integration.recall.test.ts b/test/integration.recall.test.ts index 6bb51b5..4eb1e0f 100644 --- a/test/integration.recall.test.ts +++ b/test/integration.recall.test.ts @@ -63,13 +63,8 @@ describe.skipIf(!ENABLED)("Recaller integration", () => { // 结构验证（不依赖具体节点返回 —— PPR 排序受共享 Neo4j 现有数据影响） expect(result).toHaveProperty("nodes"); expect(result).toHaveProperty("edges"); - expect(result).toHaveProperty("tokenEstimate"); expect(Array.isArray(result.nodes)).toBe(true); expect(Array.isArray(result.edges)).toBe(true); - // 如果有节点返回，tokenEstimate 应 > 0 - if (result.nodes.length > 0) { - expect(result.tokenEstimate).toBeGreaterThan(0); - } }); it("recall 带 mock embedFn：走向量搜索路径，不抛错", async () => { @@ -86,7 +81,6 @@ describe.skipIf(!ENABLED)("Recaller integration", () => { const result = await recaller.recall("docker"); expect(result).toHaveProperty("nodes"); expect(result).toHaveProperty("edges"); - expect(result).toHaveProperty("tokenEstimate"); }); it("recall 空查询：降级到 topNodes，返回合法结构", async () => { @@ -94,7 +88,6 @@ describe.skipIf(!ENABLED)("Recaller integration", () => { const result = await recaller.recall(" "); expect(result).toHaveProperty("nodes"); expect(result).toHaveProperty("edges"); - expect(result).toHaveProperty("tokenEstimate"); expect(result.nodes.length).toBeGreaterThanOrEqual(0); }); diff --git a/test/normalize-name.test.ts b/test/normalize-name.test.ts index 45c7358..d8cc705 100644 --- a/test/normalize-name.test.ts +++ b/test/normalize-name.test.ts @@ -1,6 +1,5 @@ import { describe, it, expect } from "vitest"; import { normalizeName as normalize } from "../src/store/store.ts"; -import { normalizeName as normalizeNameExtract } from "../src/extractor/extract.ts"; describe("normalizeName", () => { it("小写化", () => { @@ -53,23 +52,3 @@ describe("normalizeName", () => { expect(normalize("")).toBe(""); }); }); - -describe("normalizeName 跨文件一致性（store.ts 与 extract.ts 必须相同）", () => { - const corpus = [ - "Docker Build", - "API_KEY", - "React 18!", - "数据库迁移", - " mixed_Case Name! ", - "a---b__c d", - "", - "已经-标准化", - "Neovis 3D 可视化", - ]; - - for (const input of corpus) { - it(`相同输入相同输出: ${JSON.stringify(input)}`, () => { - expect(normalizeNameExtract(input)).toBe(normalize(input)); - }); - } -}); diff --git a/test/register-guard.test.ts b/test/register-guard.test.ts index 4e3ec6a..f5c7809 100644 --- a/test/register-guard.test.ts +++ b/test/register-guard.test.ts @@ -51,7 +51,7 @@ vi.mock("../src/recaller/recall.ts", () => ({ setEmbedFn(): void {} hasEmbedFn(): boolean { return false; } get embedFn() { return null; } - async recall() { return { nodes: [], edges: [], tokenEstimate: 0 }; } + async recall() { return { nodes: [], edges: [] }; } async syncEmbed(): Promise {} }, parseTimeRange: () => null, diff --git a/test/session-identity.test.ts b/test/session-identity.test.ts index 976a8df..1f04900 100644 --- a/test/session-identity.test.ts +++ b/test/session-identity.test.ts @@ -12,7 +12,6 @@ const mocks = vi.hoisted(() => ({ recall: vi.fn(async () => ({ nodes: [{ id: "recalled-node" }], edges: [], - tokenEstimate: 1, })), assembleContext: vi.fn(async () => ({ xml: "", systemPrompt: "", tokens: 0 })), runMaintenance: vi.fn(async () => ({ From 0575239884f587cfe9d1e8879d8398f41fdf93fd Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:49:29 +0000 Subject: [PATCH 22/29] =?UTF-8?q?feat:=20reembed=20pipeline=20=E2=80=94=20?= =?UTF-8?q?void=20stale=20vectors=20+=20batch=20rebuild=20(graph-memory=20?= =?UTF-8?q?reembed)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switching embedding models leaves old-model vectors in place: recall/dedup guard with `WHERE embedding IS NOT NULL`, so stale vectors silently degrade (or break the dedup UNWIND on dimension change), and `graph-memory extract` backfill only covers new nodes. - embed.ts: `createEmbedder()` returns single `embed` + batched `embedBatch` (shared probe; input[]/texts[] bodies, 2s-per-text timeout scaling); pure `parseBatchEmbeddingResponse()` reassembles data[] by index and validates count/shape. `createEmbedFn` kept as a single-text wrapper. - store.ts: embedding stats, clearAllEmbeddings (removes embedding AND contentHash so syncEmbed cannot short-circuit on the stale hash), ID-cursor-paginated embedding targets (SKIP pagination breaks once the NULL set shrinks), saveCommunityEmbedding, getVectorIndexDimensions, dropVectorIndexes. - cli-reembed.ts: `graph-memory reembed` — probe dims vs vector index dims (abort on mismatch, `--recreate-index` to drop/recreate via initSchema), void, cursor-paginated batch rebuild (buildNodeEmbeddingText reused so contentHash semantics match runtime syncEmbed), per-item fallback when a batch call fails. Flags: --dry-run / --yes / --batch / --recreate-index. - cli.ts: register the reembed subcommand. - tests: reembed.test.ts (planReembed + batch response parsing, 14 cases); integration.neo4j.test.ts +2 live-DB cases (void/cursor/community/index dims). - docs: README.md / README_CN.md re-embedding sections. --- README.md | 19 ++ README_CN.md | 16 ++ src/cli-reembed.ts | 331 +++++++++++++++++++++++++++++++++ src/cli.ts | 52 ++++++ src/engine/embed.ts | 94 +++++++++- src/store/store.ts | 176 ++++++++++++++++++ test/integration.neo4j.test.ts | 69 +++++++ test/reembed.test.ts | 105 +++++++++++ 8 files changed, 853 insertions(+), 9 deletions(-) create mode 100644 src/cli-reembed.ts create mode 100644 test/reembed.test.ts diff --git a/README.md b/README.md index 2ee7a54..e939bb4 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,25 @@ Deletion semantics (fail-closed): - Unextracted messages and legacy rows (extracted before this flag existed, no `producedKnowledge` property) are never deleted. - A failure in the retention step itself (invalid policy fails closed with zero deletions / Neo4j error) never invalidates the other maintenance steps; it retries next cycle. +### Re-embedding after switching embedding models + +Embedding vectors are model-specific — vectors produced by the old model are not comparable +(different dimensions break dedup and vector search outright), and vectors are write-once with +no automatic migration. After changing `embedding.model` / `embedding.dimensions`, run: + +```bash +openclaw graph-memory reembed --dry-run # report dimension match + vector coverage, no writes +openclaw graph-memory reembed # void all vectors and rebuild them in batches +``` + +The command voids every `MemoryNode.embedding` (and its `contentHash`, so the runtime +`syncEmbed` hash guard cannot short-circuit), then re-embeds all active nodes and community +summaries with the current model using batched requests (`--batch `, default 32; failed +batches automatically fall back to per-item requests). If the vector index dimensions no longer +match the configured model, the run aborts — add `--recreate-index` to drop and recreate +`gm_node_embedding` / `gm_community_embedding` with the new dimensions. Items that fail stay +vectorless and are picked up by a re-run. + ### OAuth login (experimental) ```bash diff --git a/README_CN.md b/README_CN.md index 87bb74d..75c495c 100644 --- a/README_CN.md +++ b/README_CN.md @@ -164,6 +164,22 @@ OpenClaw 定时任务创建的会话可以独立配置图谱行为。host 把 cr - 未提取消息、以及标记机制上线前的遗留行（无 `producedKnowledge` 属性）一律不删。 - 保留步骤自身失败（非法策略 fail-closed 不删 / Neo4j 故障）不影响维护链的其他步骤，下一周期自动重试。 +### 更换 embedding 模型后的重嵌入 + +embedding 向量与模型绑定——旧模型产出的向量与新模型不可比（维度不同时去重与向量搜索直接失效）， +且向量只写不删、没有自动迁移。更改 `embedding.model` / `embedding.dimensions` 后请执行： + +```bash +openclaw graph-memory reembed --dry-run # 只报告维度对照与向量覆盖情况，不写入 +openclaw graph-memory reembed # 清空全部向量并按批重建 +``` + +该命令清空所有 `MemoryNode.embedding`（连同 `contentHash`，避免运行时 `syncEmbed` +被旧文本 hash 短路），然后用当前模型按批重建全部活跃节点与社区摘要向量（`--batch `， +默认 32；批失败自动退化为逐条请求）。若向量索引维度与当前模型不符，命令会中止——加 +`--recreate-index` 删除并按新维度重建 `gm_node_embedding` / `gm_community_embedding`。 +失败条目保持无向量，修复端点后重跑本命令即可增量补齐。 + ### OAuth 登录（实验性） ```bash diff --git a/src/cli-reembed.ts b/src/cli-reembed.ts new file mode 100644 index 0000000..a4c158e --- /dev/null +++ b/src/cli-reembed.ts @@ -0,0 +1,331 @@ +/** + * graph-memory-pro CLI — `openclaw graph-memory reembed` + * + * 换 embedding 模型后的一次性重建管线。旧模型的向量与新模型不可比（维度甚至 + * 不同），而向量只写不删 —— recall/dedup 的 `WHERE embedding IS NOT NULL` 防御 + * 会让旧向量静默失效（dedup 维度不符时整个向量查询直接报错）。流程： + * 1. 探测当前模型输出维度，对照 gm_node_embedding / gm_community_embedding 索引维度 + * （不符时默认中止；--recreate-index 删除索引并按配置重建） + * 2. 清空（void）所有 MemoryNode.embedding + contentHash、Community.embedding + * 3. 用当前 embedding 模型按批重建（标准端点走批量 input[]；批失败退化为逐条） + * 文本格式复用 buildNodeEmbeddingText，hash 语义与运行时 syncEmbed 一致。 + * + * 命令在 cli-metadata 模式下运行（register() 早早 return），所以这里自行完成 + * Neo4j driver / schema / embedder 的初始化，并在 finally 中 closeDriver。 + */ + +import readline from "node:readline/promises"; +import { stdin as input, stdout as output } from "node:process"; + +import type { Driver } from "neo4j-driver"; +import type { GmConfig } from "./types.ts"; +import { getDriver, initSchema, closeDriver } from "./store/db.ts"; +import { + getEmbeddingStats, + clearAllEmbeddings, + listNodeEmbeddingTargets, + listCommunityEmbeddingTargets, + saveVector, + saveCommunityEmbedding, + getVectorIndexDimensions, + dropVectorIndexes, +} from "./store/store.ts"; +import { createEmbedder, type Embedder } from "./engine/embed.ts"; +import { buildNodeEmbeddingText } from "./recaller/recall.ts"; +import { isAffirmative } from "./cli-extract.ts"; + +export const DEFAULT_REEMBED_BATCH = 32; +const MAX_REEMBED_BATCH = 256; + +export interface ReembedOptions { + yes?: boolean; + dryRun?: boolean; + batch?: number; + recreateIndex?: boolean; +} + +export interface ReembedParams { + cfg: GmConfig; + options: ReembedOptions; + log?: (msg: string) => void; + prompt?: (question: string) => Promise; +} + +export interface ReembedResult { + clearedNodes: number; + clearedCommunities: number; + nodesEmbedded: number; + nodesFailed: number; + communitiesEmbedded: number; + communitiesFailed: number; + batches: number; + recreatedIndex: boolean; + durationMs: number; +} + +// ─── 维度决策（纯函数，可测） ───────────────────────────────── + +export type ReembedPlan = + | { action: "run" } + | { action: "recreate-index"; reason: string } + | { action: "abort"; reason: string }; + +/** + * 对照"当前模型输出维度"与"两个向量索引的实际维度"决定行动。 + * 索引不存在（全新库 / 索引被删）→ run：initSchema 已按当前配置重建。 + */ +export function planReembed(params: { + probeDim: number; + nodeIndexDim: number | null; + communityIndexDim: number | null; + recreateIndex?: boolean; +}): ReembedPlan { + const { probeDim, nodeIndexDim, communityIndexDim, recreateIndex } = params; + const dims = [...new Set([nodeIndexDim, communityIndexDim].filter( + (d): d is number => typeof d === "number", + ))]; + if (!dims.length) return { action: "run" }; + if (!dims.some(d => d !== probeDim)) return { action: "run" }; + + const dimText = dims.join("/"); + if (recreateIndex) { + return { + action: "recreate-index", + reason: `向量索引维度（${dimText}）与当前模型输出（${probeDim}）不符，将删除索引并按配置重建`, + }; + } + return { + action: "abort", + reason: + `向量索引维度（${dimText}）与当前 embedding 模型输出维度（${probeDim}）不一致。` + + `换模型后必须重建索引：加 --recreate-index 让本命令删除并按新维度重建（旧向量会一并作废），` + + `或把 embedding.dimensions 改回 ${dimText}。`, + }; +} + +function clampBatch(batch: number | undefined): number { + if (!batch || !Number.isFinite(batch) || batch < 1) return DEFAULT_REEMBED_BATCH; + return Math.min(Math.floor(batch), MAX_REEMBED_BATCH); +} + +function defaultLog(msg: string): void { + console.log(msg); +} + +async function defaultPrompt(question: string): Promise { + if (!process.stdin.isTTY && process.env.GRAPH_MEMORY_REEMBED_CONFIRM === undefined) { + return ""; + } + const rl = readline.createInterface({ input, output }); + try { + return await rl.question(question); + } finally { + rl.close(); + } +} + +/** 单批嵌入：优先批量调用，失败退化为逐条请求（兼容批量响应结构未知的 provider） */ +async function embedTexts( + embedder: Embedder, + texts: string[], + log: (msg: string) => void, +): Promise<{ vecs: (number[] | null)[]; batchCalls: number }> { + try { + const vecs = await embedder.embedBatch(texts, "db"); + return { vecs, batchCalls: 1 }; + } catch (err) { + if (texts.length === 1) throw err; + log(` 批量调用失败（${err instanceof Error ? err.message : String(err)}），本批退化为逐条请求`); + } + const vecs: (number[] | null)[] = []; + for (const text of texts) { + try { + vecs.push(await embedder.embed(text, "db")); + } catch { + vecs.push(null); + } + } + return { vecs, batchCalls: texts.length }; +} + +export async function runReembed(params: ReembedParams): Promise { + const start = Date.now(); + const log = params.log ?? defaultLog; + const opts = params.options; + const cfg = params.cfg; + const batch = clampBatch(opts.batch); + + const result: ReembedResult = { + clearedNodes: 0, + clearedCommunities: 0, + nodesEmbedded: 0, + nodesFailed: 0, + communitiesEmbedded: 0, + communitiesFailed: 0, + batches: 0, + recreatedIndex: false, + durationMs: 0, + }; + + if (!cfg.neo4j?.uri) { + throw new Error( + "[graph-memory-pro] reembed 需要 neo4j.uri 配置。请在 graph-memory-pro 插件配置中设置 neo4j.uri / neo4j.user / neo4j.password。", + ); + } + if (!cfg.embedding || (!cfg.embedding.apiKey && !cfg.embedding.baseURL)) { + throw new Error( + "[graph-memory-pro] reembed 需要 embedding 配置（embedding.apiKey 或 embedding.baseURL）。" + + "请先在插件配置中指向新的 embedding 模型。", + ); + } + + const driver: Driver = getDriver(cfg.neo4j); + + try { + log("[graph-memory-pro] 正在初始化 Neo4j schema..."); + await initSchema(driver, cfg.embedding); + + log("[graph-memory-pro] 正在探测 embedding 端点..."); + const embedder = await createEmbedder(cfg.embedding); + if (!embedder) { + throw new Error("[graph-memory-pro] embedding 端点探测失败，无法重嵌入。请检查 embedding.apiKey / baseURL / model。"); + } + const probeDim = (await embedder.embed("ping", "query")).length; + + // ── 维度对照 ── + const indexDims = await getVectorIndexDimensions(driver); + const plan = planReembed({ + probeDim, + nodeIndexDim: indexDims.gm_node_embedding, + communityIndexDim: indexDims.gm_community_embedding, + recreateIndex: opts.recreateIndex, + }); + + const stats = await getEmbeddingStats(driver); + log(`\n[graph-memory-pro] 当前模型输出维度: ${probeDim}`); + log(`[graph-memory-pro] 向量索引维度: node=${indexDims.gm_node_embedding ?? "(未建)"} community=${indexDims.gm_community_embedding ?? "(未建)"}`); + log(`[graph-memory-pro] 活跃节点: ${stats.nodesTotal}（已有向量 ${stats.nodesEmbedded}，待重建 ${stats.nodesTotal - stats.nodesEmbedded}）`); + log(`[graph-memory-pro] 社区: ${stats.communitiesTotal}（已有向量 ${stats.communitiesEmbedded}）`); + + if (plan.action === "abort") { + throw new Error(`[graph-memory-pro] ${plan.reason}`); + } + if (plan.action === "recreate-index") { + log(`[graph-memory-pro] ${plan.reason}`); + if (opts.dryRun) { + log("[graph-memory-pro] --dry-run 模式，未执行任何更改。"); + result.durationMs = Date.now() - start; + return result; + } + await dropVectorIndexes(driver); + await initSchema(driver, cfg.embedding); + result.recreatedIndex = true; + log("[graph-memory-pro] 向量索引已按当前配置重建。"); + } + + if (opts.dryRun) { + const pending = stats.nodesTotal - stats.nodesEmbedded + stats.communitiesTotal - stats.communitiesEmbedded; + log(`\n[graph-memory-pro] --dry-run：将清除全部现有向量并按每批 ${batch} 条重建约 ${pending} 条 embedding，未执行任何更改。`); + result.durationMs = Date.now() - start; + return result; + } + + if (!opts.yes) { + const prompt = params.prompt ?? ((q: string) => defaultPrompt(q)); + const answer = await prompt( + `\n将清除现有节点/社区向量并用当前模型重建（每批 ${batch} 条），继续？[y/N] `, + ); + if (!isAffirmative(answer)) { + log("[graph-memory-pro] 已取消。"); + result.durationMs = Date.now() - start; + return result; + } + } + + // ── 清空旧向量 ── + const cleared = await clearAllEmbeddings(driver); + result.clearedNodes = cleared.nodes; + result.clearedCommunities = cleared.communities; + log(`\n[graph-memory-pro] 已清除 ${cleared.nodes} 个节点 / ${cleared.communities} 个社区的旧向量。`); + + // ── 节点重建（游标分页：重建会让 NULL 集合收缩，SKIP 分页会跳号） ── + log(`[graph-memory-pro] 开始重建节点向量（每批 ${batch} 条）...`); + let nodeCursor = ""; + for (;;) { + const targets = await listNodeEmbeddingTargets(driver, nodeCursor, batch); + if (!targets.length) break; + nodeCursor = targets[targets.length - 1].id; + + const texts = targets.map(t => buildNodeEmbeddingText(t)); + const embeddable = targets + .map((t, i) => ({ t, text: texts[i] })) + .filter(x => x.text.trim().length > 0); + + if (embeddable.length < targets.length) { + const skipped = targets.length - embeddable.length; + result.nodesFailed += skipped; + log(` 跳过 ${skipped} 个文本为空的节点（保持无向量，不影响文本搜索）`); + } + + if (embeddable.length) { + const { vecs, batchCalls } = await embedTexts( + embedder, + embeddable.map(x => x.text), + log, + ); + result.batches += batchCalls; + for (let i = 0; i < embeddable.length; i++) { + const vec = vecs[i]; + if (!vec) { + result.nodesFailed += 1; + continue; + } + await saveVector(driver, embeddable[i].t.id, embeddable[i].text, vec); + result.nodesEmbedded += 1; + } + } + + log(` 进度: 已重建 ${result.nodesEmbedded}，失败/跳过 ${result.nodesFailed}（游标 ${nodeCursor.slice(0, 12)}…）`); + } + + // ── 社区重建 ── + log("[graph-memory-pro] 开始重建社区向量..."); + let communityCursor = ""; + for (;;) { + const targets = await listCommunityEmbeddingTargets(driver, communityCursor, batch); + if (!targets.length) break; + communityCursor = targets[targets.length - 1].id; + + const { vecs, batchCalls } = await embedTexts( + embedder, + targets.map(t => t.summary), + log, + ); + result.batches += batchCalls; + for (let i = 0; i < targets.length; i++) { + const vec = vecs[i]; + if (!vec) { + result.communitiesFailed += 1; + continue; + } + await saveCommunityEmbedding(driver, targets[i].id, vec); + result.communitiesEmbedded += 1; + } + log(` 进度: 已重建 ${result.communitiesEmbedded}，失败 ${result.communitiesFailed}`); + } + + result.durationMs = Date.now() - start; + log( + `\n[graph-memory-pro] 重嵌入完成：节点 ${result.nodesEmbedded}（失败/跳过 ${result.nodesFailed}），` + + `社区 ${result.communitiesEmbedded}（失败 ${result.communitiesFailed}），` + + `${result.batches} 次 embedding 调用${result.recreatedIndex ? "，索引已重建" : ""}，` + + `用时 ${(result.durationMs / 1000).toFixed(1)}s`, + ); + if (result.nodesFailed > 0 || result.communitiesFailed > 0) { + log("[graph-memory-pro] 提示：失败条目保持无向量，修复端点后再次运行本命令即可增量补齐。"); + } + return result; + } finally { + await closeDriver(); + } +} diff --git a/src/cli.ts b/src/cli.ts index a7faeaf..14eef92 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -26,6 +26,7 @@ import { } from "./engine/oauth.ts"; import type { ReasoningEffort } from "./engine/llm.ts"; import { runBackfillExtraction } from "./cli-extract.ts"; +import { runReembed } from "./cli-reembed.ts"; import { DEFAULT_CONFIG, type GmConfig } from "./types.ts"; // ─── 最小 Commander 鸭子类型（避免引入 commander 依赖） ─────────── @@ -430,5 +431,56 @@ export function createGraphMemoryCli(deps: GraphMemoryCliDeps) { throw new Error(`[graph-memory-pro] extract failed: ${message}`); } }); + + root + .command("reembed") + .description( + "清除现有向量并用当前 embedding 模型批量重建（换 embedding 模型后必须执行，否则旧向量静默失效）", + ) + .option("--yes", "跳过确认提示，直接执行", false) + .option("--dry-run", "只报告向量覆盖情况与维度对照，不写入", false) + .option("--batch ", "每次 embedding 请求携带的文本条数（默认 32，上限 256）", undefined) + .option( + "--recreate-index", + "向量索引维度与当前模型输出不符时，删除索引并按 embedding.dimensions 重建", + false, + ) + .action(async (options: Record) => { + try { + const rawCfg = isPlainObject(deps.pluginConfig) + ? (deps.pluginConfig as Record) + : {}; + const cfg: GmConfig = { + ...DEFAULT_CONFIG, + ...(rawCfg as Partial), + }; + if (isPlainObject(rawCfg.neo4j)) { + cfg.neo4j = { ...DEFAULT_CONFIG.neo4j, ...(rawCfg.neo4j as any) }; + } + if (isPlainObject(rawCfg.embedding)) { + cfg.embedding = { ...(rawCfg.embedding as any) }; + } + + const batchFlag = typeof options.batch === "string" + ? Number.parseInt(options.batch, 10) + : (typeof options.batch === "number" ? options.batch : undefined); + + await runReembed({ + cfg, + options: { + yes: options.yes === true, + dryRun: options.dryRun === true, + recreateIndex: options.recreateIndex === true, + batch: batchFlag !== undefined && Number.isFinite(batchFlag) && batchFlag > 0 + ? Math.floor(batchFlag) + : undefined, + }, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error("[graph-memory-pro] reembed 失败：", message); + throw new Error(`[graph-memory-pro] reembed failed: ${message}`); + } + }); }; } diff --git a/src/engine/embed.ts b/src/engine/embed.ts index 9626068..4564282 100755 --- a/src/engine/embed.ts +++ b/src/engine/embed.ts @@ -19,6 +19,12 @@ import { fetchRetry } from "./http.ts"; export type EmbedMode = "db" | "query"; export type EmbedFn = (text: string, mode?: EmbedMode) => Promise; +export type EmbedBatchFn = (texts: string[], mode?: EmbedMode) => Promise; + +export interface Embedder { + embed: EmbedFn; + embedBatch: EmbedBatchFn; +} // ─── Provider 识别 ─────────────────────────────────────────── @@ -43,9 +49,49 @@ export function isMinimaxEndpoint(baseURL: string): boolean { ); } +// ─── 批量响应解析（纯函数，可测） ───────────────────────────── + +/** + * 从 /embeddings 响应的 data 数组重建与输入等长、顺序一致的向量数组。 + * OpenAI 兼容端点按 item.index 归位；缺 index 时按响应位置对齐。 + * MiniMax 用 item.vector 字段。条数不符 / 缺向量 / index 非法都抛错—— + * 上层（CLI 重嵌入）会退化为逐条请求兜底。 + */ +export function parseBatchEmbeddingResponse( + data: unknown, + expectedCount: number, + minimax: boolean, +): number[][] { + if (!Array.isArray(data)) { + throw new Error("[graph-memory-pro] Embedding batch response missing data array"); + } + if (data.length !== expectedCount) { + throw new Error( + `[graph-memory-pro] Embedding batch returned ${data.length} vectors for ${expectedCount} inputs`, + ); + } + const out: number[][] = new Array(expectedCount); + data.forEach((item: any, position: number) => { + const vec = minimax ? item?.vector : item?.embedding; + if (!Array.isArray(vec) || !vec.length) { + throw new Error(`[graph-memory-pro] Embedding batch response item ${position} has no vector`); + } + const index = typeof item?.index === "number" ? item.index : position; + if (index < 0 || index >= expectedCount || out[index]) { + throw new Error(`[graph-memory-pro] Embedding batch response has invalid index ${index}`); + } + out[index] = vec; + }); + return out; +} + // ─── EmbedFn 工厂 ─────────────────────────────────────────── -export async function createEmbedFn(cfg: EmbeddingConfig | undefined): Promise { +/** + * 构造单发 + 批量两个 embed 函数（共享 probe 与配置解析）。 + * 运行时路径只用单发 embed；`graph-memory reembed` 额外消费批量接口。 + */ +export async function createEmbedder(cfg: EmbeddingConfig | undefined): Promise { // Local OpenAI-compatible servers commonly do not require a key. A key by // itself still selects the default OpenAI endpoint; a URL by itself selects // an unauthenticated local/custom endpoint. @@ -62,12 +108,13 @@ export async function createEmbedFn(cfg: EmbeddingConfig | undefined): Promise { + function buildBody(input: string | string[], mode: EmbedMode): Record { if (minimax) { return { model, - texts: [input], + texts: Array.isArray(input) ? input : [input], type: mode, }; } @@ -76,15 +123,19 @@ export async function createEmbedFn(cfg: EmbeddingConfig | undefined): Promise { - const res = await fetchRetry(`${baseURL}/embeddings`, { + async function postEmbedding(body: Record, timeoutMs: number): Promise { + return fetchRetry(`${baseURL}/embeddings`, { method: "POST", headers: { "Content-Type": "application/json", ...(apiKey ? { "Authorization": `Bearer ${apiKey}` } : {}), }, - body: JSON.stringify(buildBody(input, mode)), - }, { timeoutMs: 10_000, label: "[graph-memory-pro] Embedding", retryOnTimeout: true }); + body: JSON.stringify(body), + }, { timeoutMs, label: "[graph-memory-pro] Embedding", retryOnTimeout: true }); + } + + async function callEmbedding(input: string, mode: EmbedMode): Promise { + const res = await postEmbedding(buildBody(input, mode), 10_000); if (!res.ok) { const errText = await res.text().catch(() => ""); @@ -100,15 +151,40 @@ export async function createEmbedFn(cfg: EmbeddingConfig | undefined): Promise { + if (!texts.length) return []; + // 批量请求服务端耗时随条数线性增长：按 2s/条 推算超时，下限沿用单发的 10s + const res = await postEmbedding(buildBody(texts, mode), Math.max(10_000, 2_000 * texts.length)); + + if (!res.ok) { + const errText = await res.text().catch(() => ""); + throw new Error(`[graph-memory-pro] Embedding API ${res.status}: ${errText.slice(0, 200)}`); + } + + const data = await res.json() as any; + return parseBatchEmbeddingResponse(data?.data, texts.length, minimax); + } + try { const probe = await callEmbedding("ping", "query"); if (!probe.length) return null; - return async (text: string, mode: EmbedMode = "db"): Promise => { - return callEmbedding(text.slice(0, 8000), mode); + return { + embed: async (text: string, mode: EmbedMode = "db"): Promise => { + return callEmbedding(text.slice(0, 8000), mode); + }, + embedBatch: async (texts: string[], mode: EmbedMode = "db"): Promise => { + return callEmbeddingBatch(texts.map(t => t.slice(0, 8000)), mode); + }, }; } catch { // probe 失败返回 null（调用方日志已有 "text search mode" 降级提示），不在库代码里写 stdout return null; } } + +/** 兼容包装：只需单发 embed 的调用方（index.ts / cli-extract.ts） */ +export async function createEmbedFn(cfg: EmbeddingConfig | undefined): Promise { + const embedder = await createEmbedder(cfg); + return embedder?.embed ?? null; +} diff --git a/src/store/store.ts b/src/store/store.ts index 7838972..1d30956 100755 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -727,6 +727,182 @@ export async function getVectorHash(driver: Driver, nodeId: string): Promise { + const session = getSession(driver); + try { + const nodeRes = await session.run(` + MATCH (n:MemoryNode {status: 'active'}) + RETURN count(n) AS total, count(n.embedding) AS embedded + `); + const commRes = await session.run(` + MATCH (c:Community) + RETURN count(c) AS total, count(c.embedding) AS embedded + `); + const nodeRec = nodeRes.records[0]; + const commRec = commRes.records[0]; + return { + nodesTotal: nodeRec ? toInt(nodeRec.get("total")) : 0, + nodesEmbedded: nodeRec ? toInt(nodeRec.get("embedded")) : 0, + communitiesTotal: commRec ? toInt(commRec.get("total")) : 0, + communitiesEmbedded: commRec ? toInt(commRec.get("embedded")) : 0, + }; + } finally { + await session.close(); + } +} + +/** + * 清空（void）全部向量。contentHash 必须与 embedding 一同清除： + * syncEmbed 以"文本 hash 未变"短路，只清 embedding 会让重嵌入被旧 hash 跳过。 + */ +export async function clearAllEmbeddings( + driver: Driver, +): Promise<{ nodes: number; communities: number }> { + const session = getSession(driver); + try { + const nodeRes = await session.run(` + MATCH (n:MemoryNode) + WHERE n.embedding IS NOT NULL OR n.contentHash IS NOT NULL + REMOVE n.embedding, n.contentHash + RETURN count(n) AS cleared + `); + const commRes = await session.run(` + MATCH (c:Community) + WHERE c.embedding IS NOT NULL + REMOVE c.embedding + RETURN count(c) AS cleared + `); + return { + nodes: nodeRes.records[0] ? toInt(nodeRes.records[0].get("cleared")) : 0, + communities: commRes.records[0] ? toInt(commRes.records[0].get("cleared")) : 0, + }; + } finally { + await session.close(); + } +} + +/** 待重嵌入节点（游标分页 —— 重建过程会让集合收缩，SKIP 分页会跳号） */ +export interface NodeEmbeddingTarget { + id: string; + name: string; + description: string; + content: string; +} + +export async function listNodeEmbeddingTargets( + driver: Driver, + afterId: string, + limit: number, +): Promise { + const session = getSession(driver); + try { + const result = await session.run(` + MATCH (n:MemoryNode {status: 'active'}) + WHERE n.embedding IS NULL AND n.id > $afterId + RETURN n.id AS id, n.name AS name, n.description AS description, n.content AS content + ORDER BY n.id + LIMIT toInteger($limit) + `, { afterId, limit: nint(limit) }); + return result.records.map(r => ({ + id: r.get("id") ?? "", + name: r.get("name") ?? "", + description: r.get("description") ?? "", + content: r.get("content") ?? "", + })); + } finally { + await session.close(); + } +} + +/** 待重嵌入社区（摘要非空才可嵌入） */ +export interface CommunityEmbeddingTarget { + id: string; + summary: string; +} + +export async function listCommunityEmbeddingTargets( + driver: Driver, + afterId: string, + limit: number, +): Promise { + const session = getSession(driver); + try { + const result = await session.run(` + MATCH (c:Community) + WHERE c.embedding IS NULL AND c.summary IS NOT NULL AND trim(c.summary) <> '' + RETURN c.id AS id, c.summary AS summary + ORDER BY c.id + LIMIT toInteger($limit) + `, { afterId, limit: nint(limit) }); + return result.records.map(r => ({ + id: r.get("id") ?? "", + summary: r.get("summary") ?? "", + })); + } finally { + await session.close(); + } +} + +export async function saveCommunityEmbedding(driver: Driver, id: string, vec: number[]): Promise { + const session = getSession(driver); + try { + await session.run( + "MATCH (c:Community {id: $id}) SET c.embedding = $vec", + { id, vec }, + ); + } finally { + await session.close(); + } +} + +/** 读取两个向量索引的维度（索引不存在时为 null —— 全新库场景） */ +export async function getVectorIndexDimensions(driver: Driver): Promise> { + const session = getSession(driver); + try { + const result = await session.run(` + SHOW INDEXES YIELD name, options + WHERE name IN ['gm_node_embedding', 'gm_community_embedding'] + RETURN name, options.indexConfig AS indexConfig + `); + const out: Record = { + gm_node_embedding: null, + gm_community_embedding: null, + }; + for (const r of result.records) { + const name = r.get("name"); + const indexConfig = r.get("indexConfig") as any; + const dim = indexConfig?.["vector.dimensions"]; + if (typeof name === "string" && name in out && typeof dim === "number") { + out[name] = dim; + } + } + return out; + } finally { + await session.close(); + } +} + +/** 删除两个向量索引（--recreate-index 重建路径；initSchema 会按配置重新创建） */ +export async function dropVectorIndexes(driver: Driver): Promise { + const session = getSession(driver); + try { + await session.run("DROP INDEX gm_node_embedding IF EXISTS"); + await session.run("DROP INDEX gm_community_embedding IF EXISTS"); + } finally { + await session.close(); + } +} + // ─── 图遍历 ──────────────────────────────────────────────── export async function graphWalk( diff --git a/test/integration.neo4j.test.ts b/test/integration.neo4j.test.ts index 6b7f211..b360b59 100644 --- a/test/integration.neo4j.test.ts +++ b/test/integration.neo4j.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import type { Driver } from "neo4j-driver"; import graphMemoryProPlugin from "../index.ts"; import { getDriver, initSchema, closeDriver, getSession } from "../src/store/db.ts"; +import { buildNodeEmbeddingText } from "../src/recaller/recall.ts"; import { upsertNode, findByName, findById, updateNode, upsertEdge, edgesFrom, edgesTo, graphWalk, @@ -11,6 +12,8 @@ import { updateCommunities, deleteNode, deprecateNodeAndDisconnect, deleteEdges, + clearAllEmbeddings, listNodeEmbeddingTargets, listCommunityEmbeddingTargets, + saveCommunityEmbedding, getVectorIndexDimensions, } from "../src/store/store.ts"; // 仅在 NEO4J_INTEGRATION=1 时运行，避免污染默认 npm test（需要 Docker Neo4j） @@ -540,4 +543,70 @@ describe.skipIf(!ENABLED)("Neo4j integration (Docker)", () => { const hits = await searchNodes(driver, " ", 3); expect(hits.length).toBeLessThanOrEqual(3); }); + + it("重嵌入管线：clearAllEmbeddings + listNodeEmbeddingTargets 游标分页 + saveVector 回填", async () => { + const { node } = await upsertNode(driver, { + type: "SKILL", name: "Reembed Pipeline Skill", description: "re", content: "reembed me", + }, TEST_SID); + + // 先有向量 → 清空后节点必须重新出现在待嵌入列表 + const vec = new Array(1024).fill(0).map((_, i) => (i % 10) / 10); + await saveVector(driver, node.id, "reembed me", vec); + let targets = await listNodeEmbeddingTargets(driver, "", 10); + expect(targets.some(t => t.id === node.id)).toBe(false); + + const cleared = await clearAllEmbeddings(driver); + expect(cleared.nodes).toBeGreaterThanOrEqual(1); + + targets = await listNodeEmbeddingTargets(driver, "", 10); + const target = targets.find(t => t.id === node.id); + expect(target).toBeDefined(); + expect(target!.name).toBe("reembed-pipeline-skill"); // upsertNode 按规范化名入库 + + // 游标分页：以该节点 id 为游标，它不再出现在下一页 + const next = await listNodeEmbeddingTargets(driver, node.id, 10); + expect(next.some(t => t.id === node.id)).toBe(false); + + // 回填后再次从待嵌入列表消失（hash 一并恢复，syncEmbed 短路语义成立） + const text = buildNodeEmbeddingText(target!); + await saveVector(driver, node.id, text, vec); + const hash = await getVectorHash(driver, node.id); + expect(hash).toMatch(/^[a-f0-9]{32}$/); + targets = await listNodeEmbeddingTargets(driver, "", 10); + expect(targets.some(t => t.id === node.id)).toBe(false); + }); + + it("重嵌入管线：社区向量清空/回填 + getVectorIndexDimensions", async () => { + const session = getSession(driver); + try { + // 测试社区带 TEST_SID 以便 afterAll 清理扫到 + await session.run(` + CREATE (c:Community {id: $id, summary: $summary, nodeCount: 1, sourceSessions: [$sid]}) + `, { id: `c-reembed-${TEST_SID}`, summary: "reembed community summary", sid: TEST_SID }); + + const dimRes = await getVectorIndexDimensions(driver); + expect(Object.keys(dimRes)).toContain("gm_node_embedding"); + expect(Object.keys(dimRes)).toContain("gm_community_embedding"); + // 两个索引要么都报维度，要么都不存在（initSchema 成对创建） + if (dimRes.gm_node_embedding !== null && dimRes.gm_community_embedding !== null) { + expect(dimRes.gm_node_embedding).toBe(dimRes.gm_community_embedding); + } + + let targets = await listCommunityEmbeddingTargets(driver, "", 10); + const target = targets.find(t => t.id === `c-reembed-${TEST_SID}`); + expect(target).toBeDefined(); + expect(target!.summary).toBe("reembed community summary"); + + const commVec = new Array(dimRes.gm_community_embedding ?? 1024).fill(0.5); + await saveCommunityEmbedding(driver, target!.id, commVec); + targets = await listCommunityEmbeddingTargets(driver, "", 10); + expect(targets.some(t => t.id === `c-reembed-${TEST_SID}`)).toBe(false); + + await clearAllEmbeddings(driver); + targets = await listCommunityEmbeddingTargets(driver, "", 10); + expect(targets.some(t => t.id === `c-reembed-${TEST_SID}`)).toBe(true); + } finally { + await session.close(); + } + }); }); diff --git a/test/reembed.test.ts b/test/reembed.test.ts new file mode 100644 index 0000000..b9eb56c --- /dev/null +++ b/test/reembed.test.ts @@ -0,0 +1,105 @@ +/** + * graph-memory — 重嵌入管线纯逻辑单元测试 + * + * 覆盖 `graph-memory reembed` 的两个可纯测组件： + * - planReembed：模型输出维度 vs 向量索引维度对照后的行动决策 + * - parseBatchEmbeddingResponse：批量 /embeddings 响应 → 与输入对齐的向量数组 + * （OpenAI 按 index 归位 / 缺 index 按位置对齐 / MiniMax vector 字段 / 各类异常） + */ + +import { describe, it, expect } from "vitest"; +import { parseBatchEmbeddingResponse } from "../src/engine/embed.ts"; +import { planReembed } from "../src/cli-reembed.ts"; + +function vec(v: number): number[] { + return [v, v + 0.5, v + 1]; +} + +describe("planReembed 维度对照决策", () => { + it("模型维度与两个索引一致：直接运行", () => { + expect(planReembed({ probeDim: 1024, nodeIndexDim: 1024, communityIndexDim: 1024 })) + .toEqual({ action: "run" }); + }); + + it("索引不存在（全新库）：initSchema 已按配置建好，直接运行", () => { + expect(planReembed({ probeDim: 1536, nodeIndexDim: null, communityIndexDim: null })) + .toEqual({ action: "run" }); + }); + + it("维度不一致且无 --recreate-index：中止并给出可操作的指引", () => { + const plan = planReembed({ probeDim: 1536, nodeIndexDim: 1024, communityIndexDim: 1024 }); + expect(plan.action).toBe("abort"); + if (plan.action === "abort") { + expect(plan.reason).toContain("1024"); + expect(plan.reason).toContain("1536"); + expect(plan.reason).toContain("--recreate-index"); + } + }); + + it("维度不一致且带 --recreate-index：重建索引", () => { + const plan = planReembed({ probeDim: 1536, nodeIndexDim: 1024, communityIndexDim: 1024, recreateIndex: true }); + expect(plan).toMatchObject({ action: "recreate-index" }); + }); + + it("两个索引维度不一致时按异常集合报告（去重）", () => { + const plan = planReembed({ probeDim: 512, nodeIndexDim: 1024, communityIndexDim: 768 }); + expect(plan.action).toBe("abort"); + if (plan.action === "abort") { + expect(plan.reason).toContain("1024/768"); + } + }); + + it("只有一个索引存在且维度匹配：运行", () => { + expect(planReembed({ probeDim: 1024, nodeIndexDim: 1024, communityIndexDim: null })) + .toEqual({ action: "run" }); + }); +}); + +describe("parseBatchEmbeddingResponse 批量响应解析", () => { + it("OpenAI 风格：按 item.index 归位，响应乱序也能对齐输入", () => { + const data = [ + { index: 2, embedding: vec(3) }, + { index: 0, embedding: vec(1) }, + { index: 1, embedding: vec(2) }, + ]; + expect(parseBatchEmbeddingResponse(data, 3, false)).toEqual([vec(1), vec(2), vec(3)]); + }); + + it("缺 index 字段（部分本地端点）：按响应位置对齐", () => { + const data = [{ embedding: vec(1) }, { embedding: vec(2) }]; + expect(parseBatchEmbeddingResponse(data, 2, false)).toEqual([vec(1), vec(2)]); + }); + + it("MiniMax 风格：读 item.vector 字段", () => { + const data = [{ vector: vec(1) }, { vector: vec(2) }]; + expect(parseBatchEmbeddingResponse(data, 2, true)).toEqual([vec(1), vec(2)]); + }); + + it("data 不是数组：抛错", () => { + expect(() => parseBatchEmbeddingResponse(null, 2, false)).toThrow("missing data array"); + expect(() => parseBatchEmbeddingResponse({ embeddings: [] }, 2, false)).toThrow("missing data array"); + }); + + it("返回条数与输入不符：抛错（上层退化逐条兜底）", () => { + expect(() => parseBatchEmbeddingResponse([{ embedding: vec(1) }], 2, false)) + .toThrow("returned 1 vectors for 2 inputs"); + }); + + it("某项缺向量字段：抛错并指明位置", () => { + expect(() => parseBatchEmbeddingResponse([{ embedding: vec(1) }, { embedding: [] }], 2, false)) + .toThrow("item 1 has no vector"); + }); + + it("index 越界或重复：抛错", () => { + expect(() => parseBatchEmbeddingResponse( + [{ index: 0, embedding: vec(1) }, { index: 0, embedding: vec(2) }], 2, false, + )).toThrow("invalid index 0"); + expect(() => parseBatchEmbeddingResponse( + [{ index: 5, embedding: vec(1) }, { index: 6, embedding: vec(2) }], 2, false, + )).toThrow("invalid index 5"); + }); + + it("空输入：返回空数组（无需请求）", () => { + expect(parseBatchEmbeddingResponse([], 0, false)).toEqual([]); + }); +}); From 580a4e69ae2ca5a9e9b71a86bb8172f427e329c3 Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Thu, 3 Sep 2026 00:46:49 +0000 Subject: [PATCH 23/29] =?UTF-8?q?Feat:=20=E9=87=8D=E5=81=9ADeprecate+Delet?= =?UTF-8?q?e=E4=B8=9A=E5=8A=A1=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deprecation只保留三种： 1、节点不被调用根据时间流逝下调重要性直至被标记失效 2、手动断联并且标记过时 3、被合并（也就是变成类似占位符的东西仅维持图结构) delete只保留一种： 1、Deprecate后六十天内没有再被查出来用就删 --- README.md | 4 +- README_CN.md | 4 +- docs/decay.md | 58 +++++++-- index.ts | 47 +++---- openclaw.plugin.json | 9 +- src/graph/decay.ts | 63 ++++++++- src/graph/maintenance.ts | 22 +++- src/routes/crud.ts | 13 +- src/store/store.ts | 166 ++++++++++++++++++------ src/types.ts | 36 +++++- test/commit-turn.test.ts | 3 +- test/decay.test.ts | 66 ++++++++++ test/integration.graph.test.ts | 4 +- test/integration.neo4j.test.ts | 226 ++++++++++++++++++++++++++++----- test/register-guard.test.ts | 3 +- test/session-identity.test.ts | 2 +- test/update-node.test.ts | 27 +++- 17 files changed, 619 insertions(+), 134 deletions(-) diff --git a/README.md b/README.md index e939bb4..0295822 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ Anthropic direct (Claude) — drop `baseURL`, switch `provider`: ### Memory decay (forgetting curve) -Each maintenance cycle scores every active node with a three-factor weighted model (recency + frequency + intrinsic) and bidirectionally transitions nodes across three tiers: `core` / `working` / `peripheral`. Nodes never get `status=deprecated` from decay — only manual deprecate / merge does that. Decay only adjusts `tier`, so all active nodes remain searchable. +Each maintenance cycle scores every active node with a three-factor weighted model (recency + frequency + intrinsic) and bidirectionally transitions nodes across three tiers: `core` / `working` / `peripheral`. Two lifecycle stages sit on top of the forgetting curve (both on by default, configurable via `decay`): **auto-deprecation** — a `peripheral` node whose composite stays below `peripheralCompositeThreshold` and which has not been accessed for `autoDeprecateAfterDays` (30) days gets its edges severed and is marked `deprecated`; if the same knowledge is extracted or edited again, the node automatically revives to `active`. **purge** — any `deprecated` node (manual deprecation and merge losers included) is hard-deleted (`DETACH DELETE`, vectors included) after `purgeAfterDays` (60) days to reclaim storage. The full formula, field mapping from the reference implementation, default-value rationale, and tuning guide live in **[`docs/decay.md`](docs/decay.md)**. @@ -251,7 +251,7 @@ Inspect the graph with the bundled Cypher shell: | --- | --- | | `gm_search` | Recall graph knowledge for a query | | `gm_record` | Add a knowledge node manually | -| `gm_update` | Update, delete, or deprecate an existing node by exact name. `mode=update` (default) refines description/content; `mode=delete` hard-deletes the node and all its relationships; `mode=deprecate` marks `[DEPRECATED]` and removes all relationships while keeping the node (throws if not found) | +| `gm_update` | Update or deprecate an existing node by exact name. `mode=update` (default) refines description/content; `mode=deprecate` marks `[DEPRECATED]`, removes all relationships, and makes the node unreachable from recall (equivalent to deletion; physically purged by maintenance after `purgeAfterDays`). `mode=delete` was removed — deprecate replaces it (throws if not found) | | `gm_link` | Manually create or refine an edge between two existing nodes (validates type + direction against the whitelist; idempotent on from+to+type) | | `gm_unlink` | Remove edges between two nodes by name; optional `type` filter, otherwise all from→to edges | | `gm_merge` | Merge two same-type duplicate nodes: keep absorbs content/validatedCount/sessions + dedup-aware edge migration; merge is soft-deleted | diff --git a/README_CN.md b/README_CN.md index 75c495c..a54ebc2 100644 --- a/README_CN.md +++ b/README_CN.md @@ -94,7 +94,7 @@ Anthropic 直连（Claude）——去掉 `baseURL`，切换 `provider`： ### 记忆衰减（遗忘曲线） -每个维护周期对全部 active 节点做三因子加权评分（recency + frequency + intrinsic），并在三个 tier 之间双向转换：`core` / `working` / `peripheral`。衰减不会把节点置为 `status=deprecated`——只有手动弃用 / 合并才会。decay 只调整 `tier`，所有 active 节点始终保持可搜索。 +每个维护周期对全部 active 节点做三因子加权评分（recency + frequency + intrinsic），并在三个 tier 之间双向转换：`core` / `working` / `peripheral`。遗忘曲线之上还有两段生命周期（默认开启，可通过 `decay` 配置）：**自动弃用**——`peripheral` 层且 composite 持续低于 `peripheralCompositeThreshold`、超过 `autoDeprecateAfterDays`（30）天未访问的节点会被切断所有边并标记 `deprecated`；若同名知识之后被重新提取或编辑，节点自动复活回 `active`。**到期清理**——所有 `deprecated` 节点（含手动弃用与合并产物）自弃用起超过 `purgeAfterDays`（60）天后硬删（`DETACH DELETE`，向量随节点移除）以释放存储。 完整公式、字段映射、默认值依据与调参指南见 **[`docs/decay.md`](docs/decay.md)**。 @@ -228,7 +228,7 @@ openclaw gateway --verbose | --- | --- | | `gm_search` | 按查询召回图谱知识 | | `gm_record` | 手动记录知识节点 | -| `gm_update` | 按精确节点名称更新 / 删除 / 弃用已有节点（不存在则报错）。`mode=update`（默认）refine description/content；`mode=delete` 硬删除节点及其所有关系；`mode=deprecate` 标记 `[DEPRECATED]` 并删除所有关系（节点本身保留但被隔离） | +| `gm_update` | 按精确节点名称更新 / 弃用已有节点（不存在则报错）。`mode=update`（默认）refine description/content；`mode=deprecate` 标记 `[DEPRECATED]` 并切断所有关系，节点不再可被召回（等效删除；维护链在 `purgeAfterDays` 天后物理清理）。`mode=delete` 已移除，由 deprecate 取代 | | `gm_link` | 手动在两个已存在节点之间建立或细化关系边（按白名单校验类型+方向；from+to+type 已存在时仅更新 instruction） | | `gm_unlink` | 按名称删除两节点之间的关系边；可选 type 过滤，不传则删除 from→to 之间所有边 | | `gm_merge` | 合并两个同类型重复节点：keep 吸收 content/validatedCount/sessions + 去重边迁移；merge 节点被软删除（deprecated） | diff --git a/docs/decay.md b/docs/decay.md index ba57d5f..ab8dc9d 100644 --- a/docs/decay.md +++ b/docs/decay.md @@ -2,8 +2,8 @@ graph-memory-pro 的衰减机制采用**三因子加权评分 + tier 双向转换**，参考 [memory-lancedb-pro](https://github.com/CortexReach/memory-lancedb-pro) 的设计并映射到本仓库的图模型信号。 -- **decay 不动 `status`**——只调整 `tier`（`core` / `working` / `peripheral`）。`status=deprecated` 仅由手动弃用（`gm_update mode=deprecate` / merge）触发。 -- 每次 `gm_maintain` 或 `session_end` 维护的第 0 步执行：扫描所有 active 节点 → 评分 → tier 转换 → 写回 `decayScore` / `tier` / `decayComputedAt`。 +- **decay 评分只调整 `tier`**（`core` / `working` / `peripheral`）；tier 转换本身不改变 `status`。`status=deprecated` 有且仅有三个来源：手动弃用（`gm_update mode=deprecate` / REST DELETE / finalize invalidations，统一走断联+标记，见 §3.5）、merge 败者，以及遗忘曲线自动弃用（`autoDeprecate`，阶段一，见 §3.5）。 +- 每次 `gm_maintain` 或 `session_end` 维护的第 0 步执行：扫描所有 active 节点 → 评分 → tier 转换 → 写回 `decayScore` / `tier` / `decayComputedAt`（→ 自动弃用判定 → 过期硬删，见 §3.5）。 - 评分结果可通过 `gm_stats` / CRUD API 查看；外层搜索目前**不读 decayScore 排序**（已由 PageRank + tier 隐含分层）。 --- @@ -82,9 +82,35 @@ confidence = 1 − 1 / ( 1 + validatedCount ) # 饱和函数，收敛到 1 | **working → core** | `count >= coreAccessThreshold` **AND** `composite >= coreCompositeThreshold` **AND** `importance >= coreImportanceThreshold` | - 新节点默认 `tier = "working"`。 -- 节点保持 `status = active` 不变；tier 变化时仅更新 `updatedAt`，不改变搜索过滤行为。 +- tier 转换本身只写 `tier` / `updatedAt`，不改 `status`、不影响搜索过滤；被降至 `peripheral` 的节点仍可被搜索到（直到满足 §3.5 的自动弃用条件）。 - 不存在的"core→peripheral"和"peripheral→core"由两次相邻转换实现（经过 working）。 +### 3.5 节点生命周期：自动弃用与到期硬删（两阶段） + +tier 转换之上，`applyDecay` 还承担两阶段生命周期的**阶段一**；**阶段二**（硬删）由 `runMaintenance` 在 decay 步之后执行。 + +``` +active ──遗忘曲线──▶ peripheral（仍可搜索） + │ tier=peripheral + composite < peripheralCompositeThreshold + │ + 距 lastAccessedAt ≥ autoDeprecateAfterDays（阶段一） + ▼ + 自动弃用：切断所有边 + status='deprecated'（deprecatedBy='decay'， + 描述加 [DEPRECATED] 前缀）—— 全路径不可召回 + │ + ┌───────────────┴────────────────┐ + ▼ 重新提取/编辑命中（复活） ▼ deprecatedAt + purgeAfterDays（阶段二） + 恢复 active、剥前缀 DETACH DELETE 硬删（向量随节点移除， + 清 deprecatedAt/By 释放存储；适用于所有 deprecated 节点） +``` + +要点： + +- **阶段一判定**（`shouldAutoDeprecate`，三条件同时满足）：`tier=peripheral`（本轮 tier 转换后的层级，刚降到 peripheral 的老节点即刻参与）**AND** `composite < peripheralCompositeThreshold`（高 intrinsic 价值节点受保护）**AND** 距最近访问（`lastAccessedAt` → `updatedAt` → `createdAt` 回退链）≥ `autoDeprecateAfterDays`。`autoDeprecate: false` 或 `enabled: false` 时整体停用。 +- **手动弃用 = 一次性断联**：所有人工路径（`gm_update mode=deprecate`、finalize invalidations、REST `DELETE /nodes/:id`）统一走 `deprecateNodeAndDisconnectById`——切断所有边 + `[DEPRECATED]` 前缀 + `deprecatedBy='manual'`。由于所有召回路径都过滤 `status='active'` 且边已切断，手动弃用等效删除（不存在独立的 `gm_update mode=delete`，也没有 status-only 的轻量弃用路径）。 +- **复活**：`deprecatedBy='decay'` 的节点被同名知识重新提取（`upsertNode`）或手动编辑（`updateNode`）命中时，自动恢复 `active`、剥离 `[DEPRECATED]` 前缀、清除 `deprecatedAt`/`deprecatedBy`。手动弃用（`deprecatedBy='manual'`）与 merge 败者（`deprecatedBy='merge'`）**不复活**——人工/合并语义判定优先于遗忘曲线。 +- **阶段二硬删**：所有 `deprecated` 节点（含 manual/merge/存量数据），自 `deprecatedAt`（缺省回退 `updatedAt`）起超过 `purgeAfterDays` 天后 `DETACH DELETE`。`embedding`/`contentHash` 向量属性随节点一并移除，向量索引项同步消失。`purgeAfterDays: 0` 表示永不硬删。此步不受 `enabled` 总开关约束（手动弃用的节点也需要到期清理）。 +- **存量兼容**：升级前已 deprecated 的节点没有 `deprecatedBy`，一律按 `manual` 处理（不参与复活）；硬删只看 `coalesce(deprecatedAt, updatedAt)`，存量节点的弃用时间由 `updatedAt` 兜底。 + --- ## 4. 默认值与调参指南 @@ -109,7 +135,10 @@ confidence = 1 − 1 / ( 1 + validatedCount ) # 饱和函数，收敛到 1 "peripheralCompositeThreshold": 0.15, "peripheralAgeDays": 60, "workingAccessThreshold": 3, - "workingCompositeThreshold": 0.4 + "workingCompositeThreshold": 0.4, + "autoDeprecate": true, + "autoDeprecateAfterDays": 30, + "purgeAfterDays": 60 } } ``` @@ -124,6 +153,8 @@ confidence = 1 − 1 / ( 1 + validatedCount ) # 饱和函数，收敛到 1 | 7 个 tier 转换阈值 | — | lancedb-pro `tier-manager` 默认值 | | `recencyWeight / frequencyWeight / intrinsicWeight` | 0.4 / 0.3 / 0.3 | lancedb-pro 三因子权重，和为 1 | | `validatedCount` 分母 | 5 | lancedb-pro 的 `1 − exp(−count/5)` 基础频率项（未改） | +| `autoDeprecateAfterDays` | 30 | 遗忘终态缓冲：peripheral 降级后再经历一个半衰期量级的静默期才弃用，避免误伤短暂边缘化的重要知识 | +| `purgeAfterDays` | 60 | 弃用后两个月的"反悔窗口"，之后硬删释放存储；设 0 关闭硬删 | ### 4.3 常见调参场景 @@ -134,14 +165,16 @@ confidence = 1 − 1 / ( 1 + validatedCount ) # 饱和函数，收敛到 1 | 重要知识显著保得久 | 调高 `importanceModulation`（半衰期调制更强） | | 核心知识不易降级 | 调低 `betaCore`（更缓的尾部）或调高 `coreCompositeThreshold`（更难升 core，留在 working 也保得久） | | 单次曝光更易遗忘 | 调高 `workingAccessThreshold`（promote 到 working 需要更多确认） | -| 永久禁用衰减 | `"enabled": false` | +| 只降级、永不自动弃用 | `"autoDeprecate": false` | +| 弃用后保留更久再硬删 | 调高 `purgeAfterDays`（如 180），或设 0 永不硬删 | +| 永久禁用衰减（含自动弃用） | `"enabled": false` | ### 4.4 与原布尔阈值方案的对照（向后兼容） 旧版本（`maxAgeDays` + `minCalls`）的布尔规则已被这套柔性评分取代。原默认值 `maxAgeDays=30, minCalls=2` 在新模型下大致对应于： - 一个 `validatedCount=1`、`tier=working`、低 pagerank 的节点，约 30 天后 `recency` 跌破 0.15 → `composite` 跌破 `peripheralCompositeThreshold` → demote 到 `peripheral`。 -- 关键差别：新模型**不会 deprecate**，只是降到 `peripheral` tier，搜索过滤仍包含它（只是 decayScore 较低）。 +- 关键差别：新模型先降 tier 不直接弃用，节点仍可搜索（只是 decayScore 较低）；只有再经历 `autoDeprecateAfterDays` 静默期后才会按 §3.5 自动弃用（默认开启，可关闭），彻底断联则要到弃用时刻。 --- @@ -153,6 +186,8 @@ confidence = 1 − 1 / ( 1 + validatedCount ) # 饱和函数，收敛到 1 | `lastAccessedAt` | int (epoch ms) | `upsertNode`（重新提取时） | decay 评分的时间基准 | | `decayScore` | float (0~1) | `applyDecay` | 最近一次评分结果 | | `decayComputedAt` | int (epoch ms) | `applyDecay` | 评分时间戳 | +| `deprecatedAt` | int (epoch ms) | `deprecateNodeAndDisconnectById` / `mergeNodes` / `autoDeprecateNodes`（已弃用节点重复弃用时保留原值，不重置时钟） | 弃用时刻；阶段二硬删倒计时基准，缺失时回退 `updatedAt` | +| `deprecatedBy` | string | 同上 | `decay`（可复活）/ `manual` / `merge`（不复活）；缺失按 `manual` 处理 | 旧节点缺这些字段时： - `tier` 缺失 → 评分按 `working` 处理；首次 `applyDecay` 时自动写入 `working` @@ -167,9 +202,10 @@ confidence = 1 − 1 / ( 1 + validatedCount ) # 饱和函数，收敛到 1 | 文件 | 内容 | |---|---| -| `src/graph/decay.ts` | 评分函数 + tier 决策 + `applyDecay()` 批处理 | -| `src/types.ts` | `DecayConfig` 接口、`NodeTier` 类型、`GmNode` 新字段、`DEFAULT_CONFIG.decay` | -| `src/store/store.ts` | `toNode` 字段映射、`upsertNode` 初始化 `tier` / `lastAccessedAt` | -| `src/graph/maintenance.ts` | 调用入口（step 0） | -| `test/decay.test.ts` | 评分函数 + tier 决策纯函数单元测试 | +| `src/graph/decay.ts` | 评分函数 + tier 决策 + 自动弃用决策（`shouldAutoDeprecate`）+ `applyDecay()` 批处理 | +| `src/types.ts` | `DecayConfig` 接口、`NodeTier` / `DeprecatedBy` 类型、`GmNode` 新字段、`DEFAULT_CONFIG.decay` | +| `src/store/store.ts` | `toNode` 字段映射、`upsertNode` 初始化 `tier` / `lastAccessedAt`、复活逻辑（`upsertNode`/`updateNode`）、`autoDeprecateNodes()`、`purgeDeprecatedNodes()` | +| `src/graph/maintenance.ts` | 调用入口（step 0 评分/弃用 + step 0.5 硬删） | +| `test/decay.test.ts` | 评分函数 + tier 决策 + 自动弃用决策纯函数单元测试 | +| `test/integration.neo4j.test.ts` | 自动弃用 / 硬删 / 复活的集成测试（`NEO4J_INTEGRATION=1`） | | `openclaw.plugin.json` | 用户可见的配置 schema | diff --git a/index.ts b/index.ts index 4eee362..e0a208e 100755 --- a/index.ts +++ b/index.ts @@ -13,10 +13,10 @@ import { saveMessage, getUnextracted, getMaxTurnIndex, markExtracted, isTurnExtracted, commitTurnAdvance, upsertNode, upsertEdge, findByName, updateNode, - deleteNode, deprecateNodeAndDisconnect, + deprecateNodeAndDisconnect, deprecateNodeAndDisconnectById, getBySession, edgesTouching, deleteEdges, mergeNodes, - deprecate, getStats, + getStats, } from "./src/store/store.ts"; import { createCompleteFn, resolveProvider } from "./src/engine/llm.ts"; import { createEmbedFn } from "./src/engine/embed.ts"; @@ -717,6 +717,7 @@ const graphMemoryProPlugin = { `[graph-memory-pro] maintenance: ${result.durationMs}ms, ` + `dedup=${result.dedup.merged}, communities=${result.community.count}, ` + `summaries=${result.communitySummaries}, ` + + `autoDeprecate=${result.decay.autoDeprecated}, purged=${result.purged}, ` + (result.retention ? ("error" in result.retention ? `retention=failed: ${result.retention.error.slice(0, 120)}, ` @@ -1216,7 +1217,7 @@ const graphMemoryProPlugin = { }); } } - for (const id of fin.invalidations) await deprecate(driver, id); + for (const id of fin.invalidations) await deprecateNodeAndDisconnectById(driver, id); }); } @@ -1348,16 +1349,15 @@ const graphMemoryProPlugin = { label: "Update Graph Memory Node", description: "更新知识图谱中已存在的节点。必须提供精确的节点名称（不存在会报错）。" + - "三种模式：(1) 默认 update —— refine description/content；" + - "(2) delete —— 硬删除节点及其所有关系；" + - "(3) deprecate —— 标记 [DEPRECATED] 并切断所有关系（节点本身保留但被隔离）。", + "两种模式：(1) 默认 update —— refine description/content；" + + "(2) deprecate —— 标记 [DEPRECATED] 并切断所有关系（等效删除：deprecated 节点不可被召回，" + + "维护链会在 purgeAfterDays 天后物理清理）。", parameters: Type.Object({ name: Type.String({ description: "目标节点名称（必须精确匹配已有节点；名称会被标准化：全小写、空格/下划线转连字符）" }), mode: Type.Optional(Type.Union([ Type.Literal("update"), - Type.Literal("delete"), Type.Literal("deprecate"), - ], { description: "操作模式：update（默认，更新 description/content）、delete（硬删除节点+所有关系）、deprecate（标记 [DEPRECATED] 并删除所有关系，节点保留）" })), + ], { description: "操作模式：update（默认，更新 description/content）、deprecate（断联+标记弃用，等效删除）" })), description: Type.Optional( Type.String({ description: "新的一句话说明（one-line summary）。仅 update 模式生效，不传则保留原值" }), ), @@ -1369,7 +1369,7 @@ const graphMemoryProPlugin = { _toolCallId: string, p: { name: string; - mode?: "update" | "delete" | "deprecate"; + mode?: "update" | "deprecate"; description?: string; content?: string; }, @@ -1380,16 +1380,12 @@ const graphMemoryProPlugin = { `请检查节点名称是否精确（名称标准化规则：全小写、空格/下划线转连字符、移除非字母数字字符），` + `或使用 gm_record 创建新节点，也可用 gm_search 搜索已有节点。`; - if (mode === "delete") { - const deleted = await deleteNode(driver, p.name); - if (!deleted) throw new Error(notFoundHint); - return { - content: [{ - type: "text", - text: `已删除：${deleted.name} (${deleted.type}) —— 节点及其所有关系已从图谱中移除`, - }], - details: { mode, name: deleted.name, type: deleted.type, id: deleted.id }, - }; + // mode=delete 已移除（断联弃用等效删除）——为旧调用方保留明确报错而非静默降级为 update + if ((mode as string) === "delete") { + throw new Error( + "[graph-memory-pro] mode=delete 已移除：请改用 mode=deprecate（切断所有关系并标记 [DEPRECATED]，" + + "deprecated 节点不可被召回，等效删除；维护链将在 purgeAfterDays 天后自动物理清理）", + ); } if (mode === "deprecate") { @@ -1416,7 +1412,7 @@ const graphMemoryProPlugin = { if (p.description === undefined && p.content === undefined) { throw new Error( "[graph-memory-pro] gm_update mode=update 至少需要提供 description 或 content 中的一个" + - "（如需删除节点请用 mode=delete，如需弃用请用 mode=deprecate）", + "（如需移除节点请用 mode=deprecate —— 断联+标记弃用，等效删除）", ); } const updated = await updateNode(driver, p.name, { @@ -1666,8 +1662,15 @@ const graphMemoryProPlugin = { ? `衰减：扫描 ${result.decay.scanned} 个节点，tier 转换 ${totalTransitions} 次` + (totalTransitions > 0 ? `（core→working ${t.coreToWorking}，working→peripheral ${t.workingToPeripheral}，peripheral→working ${t.peripheralToWorking}，working→core ${t.workingToCore}）` + : "") + + (result.decay.autoDeprecated > 0 + ? `\n自动弃用：${result.decay.autoDeprecated} 个长期未用的 peripheral 节点已断联并标记 [DEPRECATED]` + + (result.decay.autoDeprecateError ? `（失败：${result.decay.autoDeprecateError.slice(0, 120)}）` : "") : "") : `衰减：已禁用`, + result.purged > 0 + ? `过期清理：硬删 ${result.purged} 个弃用超期的节点` + : "", `去重：${result.dedup.pairs.length} 对相似，合并 ${result.dedup.merged} 对`, ...(result.dedup.pairs.length > 0 ? result.dedup.pairs.slice(0, 5).map(p => ` "${p.nameA}" ≈ "${p.nameB}" (${(p.similarity * 100).toFixed(1)}%)`) @@ -1676,8 +1679,8 @@ const graphMemoryProPlugin = { `社区描述：${result.communitySummaries} 个`, `PageRank Top 5：`, ...result.pagerank.topK.slice(0, 5).map((n, i) => ` ${i + 1}. ${n.name} (${n.score.toFixed(4)})`), - ].join("\n"); - return { content: [{ type: "text", text }], details: { durationMs: result.durationMs, decayTransitions: totalTransitions, dedupMerged: result.dedup.merged, communities: result.community.count } }; + ].filter(Boolean).join("\n"); + return { content: [{ type: "text", text }], details: { durationMs: result.durationMs, decayTransitions: totalTransitions, autoDeprecated: result.decay.autoDeprecated, purged: result.purged, dedupMerged: result.dedup.merged, communities: result.community.count } }; }, }), { name: "gm_maintain" }, diff --git a/openclaw.plugin.json b/openclaw.plugin.json index e14582f..ff9700f 100755 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -31,9 +31,9 @@ "pagerankIterations": { "type": "number", "default": 20 }, "decay": { "type": "object", - "description": "柔性衰减：三因子加权评分（recency+frequency+intrinsic）+ tier 双向转换（core/working/peripheral）。完整公式与调参指南见 docs/decay.md。recencyWeight + frequencyWeight + intrinsicWeight 推荐和为 1（运行时会自动归一化）。", + "description": "柔性衰减：三因子加权评分（recency+frequency+intrinsic）+ tier 双向转换（core/working/peripheral），以及两阶段生命周期（长期未用的 peripheral 低分节点自动断联弃用；弃用超期的节点硬删释放存储）。完整公式与调参指南见 docs/decay.md。recencyWeight + frequencyWeight + intrinsicWeight 推荐和为 1（运行时会自动归一化）。", "properties": { - "enabled": { "type": "boolean", "default": true, "description": "是否启用自动衰减。关闭后 tier 永久保持初始 working 状态。" }, + "enabled": { "type": "boolean", "default": true, "description": "是否启用自动衰减。关闭后 tier 永久保持初始 working 状态，且自动弃用（autoDeprecate）一并停用。" }, "recencyHalfLifeDays": { "type": "number", "default": 30, "description": "Recency 半衰期（天）。effectiveHL = halfLife * exp(importanceModulation * importance)。" }, "recencyWeight": { "type": "number", "default": 0.4, "description": "Recency 在 composite 中的权重。三个权重推荐和为 1。" }, "importanceModulation": { "type": "number", "default": 1.5, "description": "半衰期调制系数；越大则高 importance 节点衰减越慢。" }, @@ -48,7 +48,10 @@ "peripheralCompositeThreshold": { "type": "number", "default": 0.15, "description": "composite 低于此值触发 demote（core→working 或 working→peripheral）。" }, "peripheralAgeDays": { "type": "number", "default": 60, "description": "working→peripheral 的年龄阈值（同时 validatedCount < workingAccessThreshold 才触发）。" }, "workingAccessThreshold": { "type": "number", "default": 3, "description": "demote（count 不足时）/ promote（count 充足时）的 access 次数分界。" }, - "workingCompositeThreshold": { "type": "number", "default": 0.4, "description": "peripheral→working 所需的最低 composite 分数。" } + "workingCompositeThreshold": { "type": "number", "default": 0.4, "description": "peripheral→working 所需的最低 composite 分数。" }, + "autoDeprecate": { "type": "boolean", "default": true, "description": "遗忘曲线自动弃用（两阶段生命周期·阶段一）：tier=peripheral 且 composite 低于 peripheralCompositeThreshold 且超过 autoDeprecateAfterDays 天未访问的节点，维护时自动切断所有边并标记 deprecated。受 enabled 总开关约束。被重新提取/编辑命中时自动复活。" }, + "autoDeprecateAfterDays": { "type": "number", "default": 30, "description": "自动弃用的未访问天数门槛（距 lastAccessedAt）。" }, + "purgeAfterDays": { "type": "number", "default": 60, "description": "两阶段生命周期·阶段二：deprecated 节点自弃用起超过该天数后硬删（DETACH DELETE，向量随节点移除）。0 = 永不硬删。适用于所有 deprecated 节点（含手动弃用与合并产物）。" } } }, "cron": { diff --git a/src/graph/decay.ts b/src/graph/decay.ts index 0f3b2dd..21f7066 100644 --- a/src/graph/decay.ts +++ b/src/graph/decay.ts @@ -5,13 +5,15 @@ * 评分 / tier 决策 / applyDecay 的入口均在本文件。 * * 调用时机：runMaintenance 的第 0 步（去重/PageRank/社区之前）。 - * decay 不动 status，只动 tier。 + * decay 调整 tier；另承担两阶段生命周期的阶段一（autoDeprecate，见 shouldAutoDeprecate）： + * 长期未被访问的 peripheral 低分节点被自动断联 + deprecated（deprecatedBy='decay'）， + * 阶段二（purge 到期硬删）在 maintenance.ts 里调用 store.purgeDeprecatedNodes。 */ import type { Driver } from "neo4j-driver"; import type { GmConfig, DecayConfig, GmNode, NodeTier } from "../types.ts"; import { getSession } from "../store/db.ts"; -import { allActiveNodes } from "../store/store.ts"; +import { allActiveNodes, autoDeprecateNodes } from "../store/store.ts"; const MS_PER_DAY = 86_400_000; @@ -33,6 +35,10 @@ export interface DecayResult { enabled: boolean; scanned: number; tierTransitions: TierTransition; + /** 本次维护中被遗忘曲线自动弃用（断联 + deprecated）的节点数。 */ + autoDeprecated: number; + /** 自动弃用批量写失败时的错误（fail-soft，不否定评分/tier 步骤）。 */ + autoDeprecateError?: string; durationMs: number; } @@ -187,6 +193,30 @@ export function decideTierTransition( return null; } +// ─── 遗忘曲线自动弃用决策（纯函数，两阶段生命周期·阶段一） ─── + +/** + * 判断节点是否应被自动弃用（断联 + status='deprecated'，deprecatedBy='decay'）。 + * 三个条件同时满足（复用遗忘曲线判定，tier 转换先行）： + * 1. 已降至 peripheral 层（遗忘曲线认为不再活跃）； + * 2. composite 仍低于 peripheralCompositeThreshold（高价值节点受 intrinsic 保护）； + * 3. 距最近访问（lastAccessedAt → updatedAt → createdAt 回退链）≥ autoDeprecateAfterDays。 + * autoDeprecate=false 时恒 false。硬删（阶段二）不在此判定，见 store.purgeDeprecatedNodes。 + */ +export function shouldAutoDeprecate( + node: Pick, + score: CompositeScore, + cfg: DecayConfig, + now: number = Date.now(), +): boolean { + if (!cfg.autoDeprecate) return false; + if ((node.tier ?? "working") !== "peripheral") return false; + if (score.composite >= cfg.peripheralCompositeThreshold) return false; + const lastActive = pickLastActive(node); + const daysSince = Math.max(0, (now - lastActive) / MS_PER_DAY); + return daysSince >= cfg.autoDeprecateAfterDays; +} + // ─── 应用层：扫描 + 评分 + 转换 ────────────────────────────── const EMPTY_TRANSITIONS: TierTransition = { @@ -204,25 +234,27 @@ function bumpTransition(transitions: TierTransition, from: NodeTier, to: NodeTie } /** - * 扫描所有 active 节点：评分 + tier 转换 + 写回 decayScore / tier。 - * 不动 status（status=deprecated 仅由手动弃用触发）。 + * 扫描所有 active 节点：评分 + tier 转换 + 写回 decayScore / tier； + * autoDeprecate 开启时，对同时满足 shouldAutoDeprecate 的节点执行阶段一自动弃用 + * （断联 + deprecated，deprecatedBy='decay'，可被重新提取/编辑复活）。 */ export async function applyDecay(driver: Driver, cfg: Pick): Promise { const start = Date.now(); const d = cfg.decay; if (!d?.enabled) { - return { enabled: false, scanned: 0, tierTransitions: { ...EMPTY_TRANSITIONS }, durationMs: 0 }; + return { enabled: false, scanned: 0, tierTransitions: { ...EMPTY_TRANSITIONS }, autoDeprecated: 0, durationMs: 0 }; } const nodes = await allActiveNodes(driver); if (nodes.length === 0) { - return { enabled: true, scanned: 0, tierTransitions: { ...EMPTY_TRANSITIONS }, durationMs: 0 }; + return { enabled: true, scanned: 0, tierTransitions: { ...EMPTY_TRANSITIONS }, autoDeprecated: 0, durationMs: 0 }; } // reduce 而非 Math.max(...map)：spread 在超大节点集（>10 万）会爆调用栈 const maxPagerank = nodes.reduce((m, n) => Math.max(m, n.pagerank), 0.0001); const updates: Array<{ id: string; tier: NodeTier; composite: number; tierChanged: boolean }> = []; + const autoDeprecateIds: string[] = []; const transitions: TierTransition = { ...EMPTY_TRANSITIONS }; for (const node of nodes) { @@ -235,6 +267,11 @@ export async function applyDecay(driver: Driver, cfg: Pick): if (tierChanged) bumpTransition(transitions, currentTier, finalTier); + // 用 tier 转换后的层级判定：本轮刚降到 peripheral 的老节点即刻参与阶段一 + if (shouldAutoDeprecate({ ...node, tier: finalTier }, score, d, start)) { + autoDeprecateIds.push(node.id); + } + updates.push({ id: node.id, tier: finalTier, @@ -260,10 +297,24 @@ export async function applyDecay(driver: Driver, cfg: Pick): } } + // 阶段一自动弃用：断联 + deprecated + [DEPRECATED] 前缀。 + // fail-soft：错误记入结果由调用方记日志，不阻塞维护链其余步骤，下一周期重试。 + let autoDeprecated = 0; + let autoDeprecateError: string | undefined; + if (autoDeprecateIds.length > 0) { + try { + autoDeprecated = await autoDeprecateNodes(driver, autoDeprecateIds, start); + } catch (err) { + autoDeprecateError = String(err); + } + } + return { enabled: true, scanned: nodes.length, tierTransitions: transitions, + autoDeprecated, + ...(autoDeprecateError ? { autoDeprecateError } : {}), durationMs: Date.now() - start, }; } diff --git a/src/graph/maintenance.ts b/src/graph/maintenance.ts index edbe6f2..25f7a0e 100755 --- a/src/graph/maintenance.ts +++ b/src/graph/maintenance.ts @@ -2,7 +2,7 @@ * graph-memory-pro — 图谱维护 * * 调用时机：session_end（finalize 之后） - * 执行顺序：衰减 → 去重 → 全局 PageRank → 社区检测 → 社区描述 → 消息保留（opt-in） + * 执行顺序：衰减（含自动弃用）→ 过期清理（硬删）→ 去重 → 全局 PageRank → 社区检测 → 社区描述 → 消息保留（opt-in） */ import type { Driver } from "neo4j-driver"; @@ -13,6 +13,7 @@ import { computeGlobalPageRank, type GlobalPageRankResult } from "./pagerank.ts" import { detectCommunities, summarizeCommunities, type CommunityResult } from "./community.ts"; import { dedup, type DedupResult } from "./dedup.ts"; import { applyDecay, type DecayResult } from "./decay.ts"; +import { purgeDeprecatedNodes } from "../store/store.ts"; import { normalizeMessageRetentionPolicy, runMessageRetention, type MessageRetentionResult, @@ -20,6 +21,10 @@ import { export interface MaintenanceResult { decay: DecayResult; + /** 两阶段生命周期·阶段二：本次硬删的过期 deprecated 节点数（purgeAfterDays=0 时恒 0）。 */ + purged: number; + /** 硬删步骤失败时的错误（fail-soft，不否定整轮维护；下一周期重试）。 */ + purgeError?: string; dedup: DedupResult; pagerank: GlobalPageRankResult; community: CommunityResult; @@ -35,9 +40,20 @@ export async function runMaintenance( ): Promise { const start = Date.now(); - // 0. 衰减（柔性评分 + tier 转换）—— 先于其他步骤，让后续基于最新 tier 集合运算 + // 0. 衰减（柔性评分 + tier 转换 + 遗忘曲线自动弃用）—— 先于其他步骤，让后续基于最新 tier 集合运算 const decayResult = await applyDecay(driver, cfg); + // 0.5 过期 deprecated 节点硬删（两阶段生命周期·阶段二，释放存储）。 + // 独立于 decay.enabled：手动弃用/merge 的节点同样到期清理，只看 purgeAfterDays。 + // fail-soft：错误记入结果由调用方记日志，不否定已完成的 decay 及后续步骤。 + let purged = 0; + let purgeError: string | undefined; + try { + purged = await purgeDeprecatedNodes(driver, (cfg.decay?.purgeAfterDays ?? 0) * 86_400_000, start); + } catch (err) { + purgeError = String(err); + } + // 1. 去重 const dedupResult = await dedup(driver, cfg); @@ -73,6 +89,8 @@ export async function runMaintenance( return { decay: decayResult, + purged, + ...(purgeError ? { purgeError } : {}), dedup: dedupResult, pagerank: pagerankResult, community: communityResult, diff --git a/src/routes/crud.ts b/src/routes/crud.ts index 9d4b961..4fe947c 100644 --- a/src/routes/crud.ts +++ b/src/routes/crud.ts @@ -19,7 +19,7 @@ import type { NodeType, EdgeType } from "../types.ts"; import { NODE_TYPE_TO_LABEL, isValidEdgeDirection, EDGE_DIRECTION_RULES, EDGE_TYPES } from "../types.ts"; import { upsertNode, findById, findByName, allActiveNodes, allEdges, - upsertEdge, edgesFrom, edgesTo, deprecate, mergeNodes, + upsertEdge, edgesFrom, edgesTo, deprecateNodeAndDisconnectById, mergeNodes, searchNodes, getStats, normalizeName, } from "../store/store.ts"; import { getSession } from "../store/db.ts"; @@ -347,7 +347,8 @@ async function handleUpdateNode( /** * DELETE /nodes?id=xxx - * 标记节点为 deprecated（软删除） + * 标记节点为 deprecated（断联弃用：切边 + [DEPRECATED] 前缀，等效删除； + * 维护链在 purgeAfterDays 天后物理清理） */ async function handleDeleteNode( res: ServerResponse, @@ -360,17 +361,13 @@ async function handleDeleteNode( return true; } - const existing = await findById(driver, id); + const existing = await deprecateNodeAndDisconnectById(driver, id); if (!existing) { json(res, 404, { error: `Node not found: ${id}` }); return true; } - await deprecate(driver, id); - - // 向量不需要删除 — deprecated 节点的向量搜索时会被 status='active' 过滤掉 - - json(res, 200, { success: true, id, name: existing.name }); + json(res, 200, { success: true, id, name: existing.name, status: existing.status }); return true; } diff --git a/src/store/store.ts b/src/store/store.ts index 1d30956..272fdea 100755 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -8,7 +8,7 @@ import type { Driver } from "neo4j-driver"; import neo4j from "neo4j-driver"; import { createHash, randomUUID } from "crypto"; -import type { GmNode, GmEdge, EdgeType, NodeType, NodeTier } from "../types.ts"; +import type { GmNode, GmEdge, EdgeType, NodeType, NodeTier, DeprecatedBy } from "../types.ts"; import { NODE_TYPE_TO_LABEL, isValidEdgeDirection, EDGE_TYPES } from "../types.ts"; import { getSession } from "./db.ts"; @@ -45,6 +45,8 @@ function toNode(r: any): GmNode { lastAccessedAt: toInt(n.lastAccessedAt ?? n.last_accessed_at ?? n.updatedAt ?? n.updated_at ?? n.createdAt ?? 0), decayScore: typeof n.decayScore === "number" ? n.decayScore : undefined, decayComputedAt: n.decayComputedAt ? toInt(n.decayComputedAt) : undefined, + deprecatedAt: n.deprecatedAt != null ? toInt(n.deprecatedAt) : undefined, + deprecatedBy: n.deprecatedBy != null ? (n.deprecatedBy as DeprecatedBy) : undefined, }; } @@ -158,10 +160,24 @@ export async function upsertNode( const session = getSession(driver); /** 按 name 更新已存在节点（find 命中与撞约束回退两条路径共用） */ const updateExisting = async (): Promise<{ node: GmNode; isNew: boolean }> => { + // revived：decay 自动弃用的节点被重新提取命中时自动复活（manual/merge 弃用不复活）。 + // revived 分支剥掉 [DEPRECATED] 前缀并忽略本次 description（语义与 stripDeprecateMarker 一致： + // 仅匹配 "[DEPRECATED] " 前缀或裸 "[DEPRECATED]" 整串，避免误伤恰以该子串开头的原文）。 + // REMOVE 只对 revived 行执行——manual/merge 节点必须保留 deprecatedAt/By， + // 否则既丢弃用溯源，purge 时钟又会回退到本查询刚写入的 updatedAt（无限续命）。 await session.run(` MATCH (n:Task|Skill|Event {name: $name}) + WITH n, (n.status = 'deprecated' AND coalesce(n.deprecatedBy, 'manual') = 'decay') AS revived SET n.content = CASE WHEN size($content) > size(n.content) THEN $content ELSE n.content END, - n.description = CASE WHEN size($description) > size(n.description) THEN $description ELSE n.description END, + n.description = CASE + WHEN revived AND n.description STARTS WITH '[DEPRECATED] ' + THEN substring(n.description, size('[DEPRECATED] ')) + WHEN revived AND n.description = '[DEPRECATED]' + THEN '' + WHEN size($description) > size(n.description) THEN $description + ELSE n.description + END, + n.status = CASE WHEN revived THEN 'active' ELSE n.status END, n.validatedCount = n.validatedCount + 1, n.sourceSessions = CASE WHEN NOT $sessionId IN n.sourceSessions @@ -170,6 +186,8 @@ export async function upsertNode( END, n.lastAccessedAt = $now, n.updatedAt = $now + FOREACH (_ IN CASE WHEN revived THEN [1] ELSE [] END + | REMOVE n.deprecatedAt, n.deprecatedBy) RETURN n `, { name, content: c.content, description: c.description, sessionId, now: Date.now() }); @@ -247,19 +265,32 @@ export async function updateNode( if (!ex) return null; const now = Date.now(); const { description, content } = applyNodePatch(ex, patch); + // decay 自动弃用的节点被手动编辑命中 → 顺手复活（manual/merge 弃用不复活）。 + // REMOVE 仅在复活时执行：非复活节点保留 deprecatedAt/By（溯源 + purge 时钟不被重置）。 + const revived = ex.status === "deprecated" && (ex.deprecatedBy ?? "manual") === "decay"; + const finalDescription = revived ? stripDeprecateMarker(description) : description; const session = getSession(driver); try { await session.run( `MATCH (n:Task|Skill|Event {id: $id}) SET n.description = $description, n.content = $content, - n.updatedAt = $now`, - { id: ex.id, description, content, now }, + n.status = $status, + n.updatedAt = $now + ${revived ? "REMOVE n.deprecatedAt, n.deprecatedBy" : ""}`, + { id: ex.id, description: finalDescription, content, status: revived ? "active" : ex.status, now }, ); } finally { await session.close(); } - return { ...ex, description, content, updatedAt: now }; + return { + ...ex, + description: finalDescription, + content, + updatedAt: now, + status: revived ? "active" : ex.status, + ...(revived ? { deprecatedAt: undefined, deprecatedBy: undefined } : {}), + }; } /** @@ -274,62 +305,121 @@ export function applyDeprecateMarker(description: string): string { } /** - * 按 name 硬删除节点：DETACH DELETE —— 节点 + 所有关系一并删除。 - * 找不到返回 null（调用方决定报错语义）。 + * 剥掉 [DEPRECATED] 前缀（applyDeprecateMarker 的逆操作）—— + * decay 自动弃用的节点复活时还原描述。无前缀则原样返回。 */ -export async function deleteNode(driver: Driver, name: string): Promise { - const ex = await findByName(driver, name); - if (!ex) return null; - const session = getSession(driver); - try { - await session.run( - "MATCH (n:Task|Skill|Event {id: $id}) DETACH DELETE n", - { id: ex.id }, - ); - } finally { - await session.close(); - } - return ex; +export function stripDeprecateMarker(description: string): string { + const prefix = "[DEPRECATED]"; + if (description === prefix) return ""; + if (description.startsWith(`${prefix} `)) return description.slice(prefix.length + 1); + return description; } /** - * 按 name deprecate 并切断：status='deprecated' + 描述加 [DEPRECATED] 前缀 + 删除所有边。 - * 节点本身保留（不硬删），但完全从知识图谱中隔离。 - * 找不到返回 null。 + * 手动弃用（一次性断联，按 id 定位）：status='deprecated' + deprecatedAt/deprecatedBy='manual' + * + 描述加 [DEPRECATED] 前缀 + 删除所有边。所有人工路径（gm_update mode=deprecate、 + * finalize invalidations、REST DELETE）统一走这里。 + * deprecated 节点对所有召回路径不可见且无边可走，效果等同删除；仅保留 purgeAfterDays + * （默认 60 天）反悔窗口，到期由 maintenance 硬删。找不到返回 null。 */ -export async function deprecateNodeAndDisconnect( +export async function deprecateNodeAndDisconnectById( driver: Driver, - name: string, + id: string, + now: number = Date.now(), ): Promise { - const ex = await findByName(driver, name); + const ex = await findById(driver, id); if (!ex) return null; - const now = Date.now(); + // 已弃用节点再次被弃用时保留原 deprecatedAt——否则每次重新提取触发的 + // finalize invalidations / REST 重试都会重置 60 天 purge 时钟（无限续命）。 + const deprecatedAt = ex.status === "deprecated" && ex.deprecatedAt ? ex.deprecatedAt : now; const description = applyDeprecateMarker(ex.description); const session = getSession(driver); try { await session.run( `MATCH (n:Task|Skill|Event {id: $id}) SET n.status = 'deprecated', + n.deprecatedAt = $deprecatedAt, + n.deprecatedBy = 'manual', n.description = $description, n.updatedAt = $now WITH n OPTIONAL MATCH (n)-[r]-() DELETE r`, - { id: ex.id, description, now }, + { id, deprecatedAt, description, now }, ); } finally { await session.close(); } - return { ...ex, status: "deprecated", description, updatedAt: now }; + return { ...ex, status: "deprecated", deprecatedAt, deprecatedBy: "manual", description, updatedAt: now }; } -export async function deprecate(driver: Driver, nodeId: string): Promise { +/** 按 name 弃用并切断（gm_update mode=deprecate 用）；找不到返回 null。 */ +export async function deprecateNodeAndDisconnect( + driver: Driver, + name: string, +): Promise { + const ex = await findByName(driver, name); + if (!ex) return null; + return deprecateNodeAndDisconnectById(driver, ex.id, Date.now()); +} + +/** + * 遗忘曲线自动弃用（两阶段生命周期·阶段一）：批量 status='deprecated' + deprecatedBy='decay' + * + 描述幂等加 [DEPRECATED] 前缀 + 切断所有边。与 deprecateNodeAndDisconnect 的区别： + * 按 id 批量、来源标记 decay（重新提取/编辑命中时可被 upsertNode/updateNode 复活）。 + * 返回实际弃用的节点数。 + */ +export async function autoDeprecateNodes( + driver: Driver, + ids: string[], + now: number = Date.now(), +): Promise { + if (!ids.length) return 0; const session = getSession(driver); try { - await session.run( - "MATCH (n:Task|Skill|Event {id: $id}) SET n.status = 'deprecated', n.updatedAt = $now", - { id: nodeId, now: Date.now() }, - ); + const result = await session.run(` + UNWIND $ids AS nid + MATCH (n:Task|Skill|Event {id: nid}) + SET n.status = 'deprecated', + n.deprecatedAt = $now, + n.deprecatedBy = 'decay', + n.description = CASE + WHEN n.description STARTS WITH '[DEPRECATED]' THEN n.description + WHEN n.description IS NULL OR size(n.description) = 0 THEN '[DEPRECATED]' + ELSE '[DEPRECATED] ' + n.description + END, + n.updatedAt = $now + WITH n + OPTIONAL MATCH (n)-[r]-() + DELETE r + RETURN count(DISTINCT n) AS c + `, { ids, now }); + return toInt(result.records[0]?.get("c") ?? 0); + } finally { + await session.close(); + } +} + +/** + * 两阶段生命周期·阶段二：硬删过期 deprecated 节点（DETACH DELETE，embedding/contentHash + * 向量属性随节点一并移除，释放存储）。时钟基准 deprecatedAt，存量节点缺省回退 updatedAt。 + * 适用于所有 deprecated 节点（decay/manual/merge）；purgeAfterMs<=0 时为 no-op。返回删除数。 + */ +export async function purgeDeprecatedNodes( + driver: Driver, + purgeAfterMs: number, + now: number = Date.now(), +): Promise { + if (!(purgeAfterMs > 0)) return 0; + const session = getSession(driver); + try { + const result = await session.run(` + MATCH (n:Task|Skill|Event {status: 'deprecated'}) + WHERE coalesce(n.deprecatedAt, n.updatedAt) < $cutoff + DETACH DELETE n + RETURN count(n) AS c + `, { cutoff: now - purgeAfterMs }); + return toInt(result.records[0]?.get("c") ?? 0); } finally { await session.close(); } @@ -404,9 +494,13 @@ export async function mergeNodes(driver: Driver, keepId: string, mergeId: string DELETE r `, { keepId }); - // 标记 deprecated + // 标记 deprecated（deprecatedBy='merge'：不参与 decay 复活；参与 purge 到期硬删） await tx.run( - "MATCH (n:Task|Skill|Event {id: $mergeId}) SET n.status = 'deprecated', n.updatedAt = $now", + `MATCH (n:Task|Skill|Event {id: $mergeId}) + SET n.status = 'deprecated', + n.deprecatedAt = $now, + n.deprecatedBy = 'merge', + n.updatedAt = $now`, { mergeId, now: Date.now() }, ); }); diff --git a/src/types.ts b/src/types.ts index d55a7b0..fc5842b 100755 --- a/src/types.ts +++ b/src/types.ts @@ -13,10 +13,22 @@ export type NodeStatus = "active" | "deprecated"; /** * 记忆分层 tier（与 NodeStatus 正交）。 * decay 评分模型据此双向转换：core↔working↔peripheral。 - * 节点仍保持 status=active，仅 tier 变化；status=deprecated 只由手动弃用触发。 + * 节点通常保持 status=active，仅 tier 变化；status=deprecated 由手动弃用、 + * merge 或 decay 自动弃用（autoDeprecate，见 DecayConfig）触发。 */ export type NodeTier = "core" | "working" | "peripheral"; +/** + * 弃用来源标记（节点属性 deprecatedBy）： + * - decay：遗忘曲线自动弃用（tier=peripheral + 低 composite + 超期未访问）； + * 重新提取/编辑命中时可自动复活回 active。 + * - manual：gm_update mode=deprecate / finalize invalidations / REST DELETE； + * 人工语义判定，不自动复活。 + * - merge：dedup 或手动合并的败者节点，不自动复活。 + * 缺省（存量数据）按 manual 处理——不复活；硬删判定只看 deprecatedAt/updatedAt。 + */ +export type DeprecatedBy = "decay" | "manual" | "merge"; + /** Neo4j label 映射：TASK->Task, SKILL->Skill, EVENT->Event */ export const NODE_TYPE_TO_LABEL: Record = { TASK: "Task", @@ -51,6 +63,10 @@ export interface GmNode { decayScore?: number; /** decayScore 的计算时间戳（epoch ms）。 */ decayComputedAt?: number; + /** 被标记 deprecated 的时刻（epoch ms）。硬删倒计时（purgeAfterDays）的基准。 */ + deprecatedAt?: number; + /** 弃用来源（见 DeprecatedBy）；缺省按 manual 处理。 */ + deprecatedBy?: DeprecatedBy; } // ─── 边 ─────────────────────────────────────────────────────── @@ -179,6 +195,21 @@ export interface DecayConfig { peripheralAgeDays: number; workingAccessThreshold: number; workingCompositeThreshold: number; + /** + * 遗忘曲线自动弃用开关（两阶段生命周期的第一阶段）： + * tier=peripheral 且 composite < peripheralCompositeThreshold 且 + * lastAccessedAt 距今 ≥ autoDeprecateAfterDays 的节点在维护时被断联 + deprecated。 + * 受 enabled 总开关约束（enabled=false 时整体跳过）。 + */ + autoDeprecate: boolean; + /** 自动弃用的未访问天数门槛（距 lastAccessedAt）。 */ + autoDeprecateAfterDays: number; + /** + * 两阶段生命周期的第二阶段：deprecated 节点自 deprecatedAt（缺省回退 updatedAt） + * 起超过该天数后硬删（DETACH DELETE，向量随节点一并移除）。 + * 0 = 永不硬删。适用于所有 deprecated 节点（含 manual/merge）。 + */ + purgeAfterDays: number; } // ─── cron 会话（定时任务）的图谱行为配置 ───────────────────── @@ -297,6 +328,9 @@ export const DEFAULT_CONFIG: GmConfig = { peripheralAgeDays: 60, workingAccessThreshold: 3, workingCompositeThreshold: 0.4, + autoDeprecate: true, + autoDeprecateAfterDays: 30, + purgeAfterDays: 60, }, cron: DEFAULT_CRON_CONFIG, }; diff --git a/test/commit-turn.test.ts b/test/commit-turn.test.ts index 422b267..d2218aa 100644 --- a/test/commit-turn.test.ts +++ b/test/commit-turn.test.ts @@ -21,13 +21,12 @@ vi.mock("../src/store/store.ts", () => ({ upsertEdge: async () => {}, findByName: async () => null, updateNode: async () => null, - deleteNode: async () => {}, deprecateNodeAndDisconnect: async () => {}, + deprecateNodeAndDisconnectById: async () => {}, getBySession: async () => [], edgesTouching: async () => [], deleteEdges: async () => {}, mergeNodes: async () => {}, - deprecate: async () => {}, getStats: async () => ({}), })); diff --git a/test/decay.test.ts b/test/decay.test.ts index c6ab7a6..2d02fc2 100644 --- a/test/decay.test.ts +++ b/test/decay.test.ts @@ -8,6 +8,7 @@ import { scoreIntrinsic, scoreNode, decideTierTransition, + shouldAutoDeprecate, } from "../src/graph/decay.ts"; import { DEFAULT_CONFIG, type DecayConfig, type GmNode } from "../src/types.ts"; @@ -276,3 +277,68 @@ describe("decideTierTransition", () => { expect(decideTierTransition(node, scoreLow, 0, cfg, NOW)).toBe("peripheral"); }); }); + +describe("shouldAutoDeprecate", () => { + // peripheralCompositeThreshold=0.15：低分 0.1 / 达标 0.5 + const scoreLow = { composite: 0.1, recency: 0, frequency: 0, intrinsic: 0 }; + const scoreHigh = { composite: 0.5, recency: 0.5, frequency: 0.5, intrinsic: 0.5 }; + + it("peripheral + 低分 + 超期未访问 → true（三条件齐备）", () => { + const node = makeNode({ tier: "peripheral", lastAccessedAt: NOW - 40 * MS_PER_DAY }); + expect(shouldAutoDeprecate(node, scoreLow, cfg, NOW)).toBe(true); + }); + + it("working 层 → false（遗忘曲线 tier 转换先行）", () => { + const node = makeNode({ tier: "working", lastAccessedAt: NOW - 40 * MS_PER_DAY }); + expect(shouldAutoDeprecate(node, scoreLow, cfg, NOW)).toBe(false); + }); + + it("core 层 → false", () => { + const node = makeNode({ tier: "core", lastAccessedAt: NOW - 40 * MS_PER_DAY }); + expect(shouldAutoDeprecate(node, scoreLow, cfg, NOW)).toBe(false); + }); + + it("composite 达标 → false（高 intrinsic 价值节点受保护）", () => { + const node = makeNode({ tier: "peripheral", lastAccessedAt: NOW - 40 * MS_PER_DAY }); + expect(shouldAutoDeprecate(node, scoreHigh, cfg, NOW)).toBe(false); + }); + + it("composite 恰等于阈值 → false（边界取保留）", () => { + const scoreAtThreshold = { composite: cfg.peripheralCompositeThreshold, recency: 0, frequency: 0, intrinsic: 0 }; + const node = makeNode({ tier: "peripheral", lastAccessedAt: NOW - 40 * MS_PER_DAY }); + expect(shouldAutoDeprecate(node, scoreAtThreshold, cfg, NOW)).toBe(false); + }); + + it("未超 autoDeprecateAfterDays → false", () => { + const node = makeNode({ tier: "peripheral", lastAccessedAt: NOW - 10 * MS_PER_DAY }); + expect(shouldAutoDeprecate(node, scoreLow, cfg, NOW)).toBe(false); + }); + + it("恰好达到天数门槛 → true（>= 含端点）", () => { + const node = makeNode({ tier: "peripheral", lastAccessedAt: NOW - cfg.autoDeprecateAfterDays * MS_PER_DAY }); + expect(shouldAutoDeprecate(node, scoreLow, cfg, NOW)).toBe(true); + }); + + it("autoDeprecate=false 恒 false（功能开关）", () => { + const offCfg: DecayConfig = { ...cfg, autoDeprecate: false }; + const node = makeNode({ tier: "peripheral", lastAccessedAt: NOW - 400 * MS_PER_DAY }); + expect(shouldAutoDeprecate(node, scoreLow, offCfg, NOW)).toBe(false); + }); + + it("lastAccessedAt 缺失时回退 updatedAt（与 decay 计时基准一致）", () => { + const viaFallback = makeNode({ + tier: "peripheral", + lastAccessedAt: 0, + updatedAt: NOW - (cfg.autoDeprecateAfterDays + 5) * MS_PER_DAY, + }); + expect(shouldAutoDeprecate(viaFallback, scoreLow, cfg, NOW)).toBe(true); + }); + + it("tier undefined 按 working 处理 → false", () => { + const node = makeNode({ + tier: undefined as unknown as GmNode["tier"], + lastAccessedAt: NOW - 400 * MS_PER_DAY, + }); + expect(shouldAutoDeprecate(node, scoreLow, cfg, NOW)).toBe(false); + }); +}); diff --git a/test/integration.graph.test.ts b/test/integration.graph.test.ts index 61cc0ca..4c957b4 100644 --- a/test/integration.graph.test.ts +++ b/test/integration.graph.test.ts @@ -12,7 +12,7 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import type { Driver } from "neo4j-driver"; import { getDriver, initSchema, closeDriver, getSession } from "../src/store/db.ts"; import { - upsertNode, upsertEdge, saveVector, findById, deprecate, getCommunitySummary, + upsertNode, upsertEdge, saveVector, findById, deprecateNodeAndDisconnectById, getCommunitySummary, } from "../src/store/store.ts"; import { personalizedPageRank, computeGlobalPageRank, @@ -146,7 +146,7 @@ describe.skipIf(!ENABLED)("graph layer integration (GDS, Docker)", () => { type: "SKILL", name: "Deprecated Pagerank Sentinel", description: "deprecated", content: "deprecated", }, TEST_SID); - await deprecate(driver, node.id); + await deprecateNodeAndDisconnectById(driver, node.id); const session = getSession(driver); try { await session.run( diff --git a/test/integration.neo4j.test.ts b/test/integration.neo4j.test.ts index b360b59..86c3ebe 100644 --- a/test/integration.neo4j.test.ts +++ b/test/integration.neo4j.test.ts @@ -7,14 +7,17 @@ import { upsertNode, findByName, findById, updateNode, upsertEdge, edgesFrom, edgesTo, graphWalk, saveMessage, getUnextracted, markExtracted, isTurnExtracted, - deprecate, getStats, mergeNodes, searchNodes, topNodes, + getStats, mergeNodes, searchNodes, topNodes, getBySession, saveVector, vectorSearchWithScore, getVectorHash, updateCommunities, - deleteNode, deprecateNodeAndDisconnect, + deprecateNodeAndDisconnect, deprecateNodeAndDisconnectById, deleteEdges, clearAllEmbeddings, listNodeEmbeddingTargets, listCommunityEmbeddingTargets, saveCommunityEmbedding, getVectorIndexDimensions, + autoDeprecateNodes, purgeDeprecatedNodes, } from "../src/store/store.ts"; +import { applyDecay } from "../src/graph/decay.ts"; +import { DEFAULT_CONFIG } from "../src/types.ts"; // 仅在 NEO4J_INTEGRATION=1 时运行，避免污染默认 npm test（需要 Docker Neo4j） const ENABLED = !!process.env.NEO4J_INTEGRATION; @@ -181,7 +184,7 @@ describe.skipIf(!ENABLED)("Neo4j integration (Docker)", () => { expect(edges.some(e => e.type === "USED_SKILL")).toBe(true); }); - it("graphWalk 不穿过 deprecated 中间节点连接两个 active 节点", async () => { + it("graphWalk 不穿过 deprecated 中间节点（弃用同时断联）", async () => { const { node: start } = await upsertNode(driver, { type: "SKILL", name: "Active Walk Start", description: "start", content: "start", }, TEST_SID); @@ -199,7 +202,10 @@ describe.skipIf(!ENABLED)("Neo4j integration (Docker)", () => { fromId: deprecatedBridge.id, toId: unreachable.id, type: "REQUIRES", instruction: "second hop", sessionId: TEST_SID, }); - await deprecate(driver, deprecatedBridge.id); + // 手动弃用（按 id）——同时切断两侧边 + await deprecateNodeAndDisconnectById(driver, deprecatedBridge.id); + expect(await edgesTo(driver, deprecatedBridge.id)).toHaveLength(0); + expect(await edgesFrom(driver, deprecatedBridge.id)).toHaveLength(0); const { nodes } = await graphWalk(driver, [start.id], 2); @@ -284,42 +290,26 @@ describe.skipIf(!ENABLED)("Neo4j integration (Docker)", () => { expect(stats.byType.SKILL).toBeGreaterThanOrEqual(1); }); - it("deprecate 软删除（status=deprecated，节点仍存在）", async () => { + it("deprecateNodeAndDisconnectById 手动弃用：断联 + 前缀 + manual 标记（finalize invalidations / REST DELETE 路径）", async () => { const { node } = await upsertNode(driver, { type: "EVENT", name: "Temp Event", description: "d", content: "c", }, TEST_SID); - await deprecate(driver, node.id); + const result = await deprecateNodeAndDisconnectById(driver, node.id); + expect(result).not.toBeNull(); + expect(result!.status).toBe("deprecated"); + expect(result!.deprecatedBy).toBe("manual"); + expect(result!.description).toBe("[DEPRECATED] d"); + const refetch = await findById(driver, node.id); expect(refetch).not.toBeNull(); expect(refetch!.status).toBe("deprecated"); - }); - - it("deleteNode 硬删除：节点 + 所有关系一并消失（gm_update mode=delete）", async () => { - const { node: task } = await upsertNode(driver, { - type: "TASK", name: "HardDelete Task", description: "victim", content: "victim", - }, TEST_SID); - const { node: skill } = await upsertNode(driver, { - type: "SKILL", name: "HardDelete Skill", description: "neighbor", content: "neighbor", - }, TEST_SID); - await upsertEdge(driver, { - fromId: task.id, toId: skill.id, type: "USED_SKILL", - instruction: "uses", sessionId: TEST_SID, - }); - - const deleted = await deleteNode(driver, task.name); - expect(deleted).not.toBeNull(); - expect(deleted!.id).toBe(task.id); + expect(await deprecateNodeAndDisconnectById(driver, "ghost-node-deprecate-id-xyz")).toBeNull(); - expect(await findById(driver, task.id)).toBeNull(); - - const remainingOut = await edgesFrom(driver, skill.id); - expect(remainingOut.filter(e => e.id === task.id)).toHaveLength(0); - const remainingIn = await edgesTo(driver, skill.id); - expect(remainingIn.filter(e => e.fromId === task.id)).toHaveLength(0); - }); - - it("deleteNode 未知 name 返回 null", async () => { - expect(await deleteNode(driver, "ghost-node-delete-xyz")).toBeNull(); + // 重复弃用保留原 deprecatedAt（不重置 purge 时钟） + const firstAt = refetch!.deprecatedAt!; + const again = await deprecateNodeAndDisconnectById(driver, node.id, firstAt + 86_400_000); + expect(again!.deprecatedAt).toBe(firstAt); + expect((await findById(driver, node.id))!.deprecatedAt).toBe(firstAt); }); it("deprecateNodeAndDisconnect 标记 [DEPRECATED] + 切边（节点保留，gm_update mode=deprecate）", async () => { @@ -609,4 +599,174 @@ describe.skipIf(!ENABLED)("Neo4j integration (Docker)", () => { await session.close(); } }); + + // ── 两阶段生命周期：遗忘曲线自动弃用 → 到期硬删（复活见 upsertNode/updateNode） ── + // 隔离性注意：purge 与 applyDecay 用例作用于整个数据库（不限于 TEST_SID—— + // purge 的 MATCH 不带 sourceSessions 过滤，applyDecay 扫描全部 active 节点）， + // 依赖 AGENTS.md 约定「NEO4J_INTEGRATION=1 指向一次性 Neo4j」保证数据正确性。 + + it("autoDeprecateNodes 批量标记 decay 弃用 + 切边 + 属性 + 前缀幂等", async () => { + const { node: victim } = await upsertNode(driver, { + type: "SKILL", name: "AutoDeprecate Target", description: "原描述", content: "content", + }, TEST_SID); + const { node: a } = await upsertNode(driver, { + type: "TASK", name: "AutoDeprecate Neighbor A", description: "n", content: "n", + }, TEST_SID); + const { node: b } = await upsertNode(driver, { + type: "SKILL", name: "AutoDeprecate Neighbor B", description: "n", content: "n", + }, TEST_SID); + await upsertEdge(driver, { + fromId: a.id, toId: victim.id, type: "USED_SKILL", + instruction: "uses", sessionId: TEST_SID, + }); + await upsertEdge(driver, { + fromId: victim.id, toId: b.id, type: "REQUIRES", + instruction: "needs", sessionId: TEST_SID, + }); + // 已带前缀的节点验证幂等（不双写） + const { node: prefixed } = await upsertNode(driver, { + type: "SKILL", name: "AutoDeprecate Prefixed", description: "[DEPRECATED] 已有前缀", content: "c", + }, TEST_SID); + + const now = Date.now(); + expect(await autoDeprecateNodes(driver, [], now)).toBe(0); + expect(await autoDeprecateNodes(driver, [victim.id, prefixed.id], now)).toBe(2); + + const v = await findById(driver, victim.id); + expect(v!.status).toBe("deprecated"); + expect(v!.deprecatedBy).toBe("decay"); + expect(v!.deprecatedAt).toBe(now); + expect(v!.description).toBe("[DEPRECATED] 原描述"); + expect(await edgesTo(driver, victim.id)).toHaveLength(0); + expect(await edgesFrom(driver, victim.id)).toHaveLength(0); + expect((await findById(driver, a.id))!.status).toBe("active"); + expect((await findById(driver, b.id))!.status).toBe("active"); + + const p = await findById(driver, prefixed.id); + expect(p!.deprecatedBy).toBe("decay"); + expect(p!.description).toBe("[DEPRECATED] 已有前缀"); + }); + + it("purgeDeprecatedNodes：过期 deprecated 硬删、未过期保留、active 不动、存量按 updatedAt 兜底", async () => { + const { node: fresh } = await upsertNode(driver, { + type: "SKILL", name: "Purge Fresh", description: "fresh", content: "c", + }, TEST_SID); + const { node: stale } = await upsertNode(driver, { + type: "SKILL", name: "Purge Stale", description: "stale", content: "c", + }, TEST_SID); + const { node: legacy } = await upsertNode(driver, { + type: "SKILL", name: "Purge Legacy", description: "legacy", content: "c", + }, TEST_SID); + const { node: keeper } = await upsertNode(driver, { + type: "SKILL", name: "Purge Active Keeper", description: "active", content: "c", + }, TEST_SID); + + const now = Date.now(); + const session = getSession(driver); + try { + // fresh：刚弃用（deprecatedAt = now），未到期 + await session.run( + "MATCH (n {id: $id}) SET n.status='deprecated', n.deprecatedAt=$now, n.deprecatedBy='decay'", + { id: fresh.id, now }, + ); + // stale：decay 弃用已超 60 天 + await session.run( + "MATCH (n {id: $id}) SET n.status='deprecated', n.deprecatedAt=$old, n.deprecatedBy='decay'", + { id: stale.id, old: now - 61 * 86_400_000 }, + ); + // legacy：无 deprecatedAt/deprecatedBy 的存量 deprecated —— 回退 updatedAt 基准，同样到期 + await session.run( + "MATCH (n {id: $id}) SET n.status='deprecated', n.deprecatedAt=null, n.updatedAt=$old REMOVE n.deprecatedBy", + { id: legacy.id, old: now - 61 * 86_400_000 }, + ); + } finally { + await session.close(); + } + + // purgeAfterMs=0 显式关闭 + expect(await purgeDeprecatedNodes(driver, 0, now)).toBe(0); + + // 共享 DB 上可能存在其他遗留的过期 deprecated 节点，断言下界 + expect(await purgeDeprecatedNodes(driver, 60 * 86_400_000, now)).toBeGreaterThanOrEqual(2); + expect(await findById(driver, stale.id)).toBeNull(); + expect(await findById(driver, legacy.id)).toBeNull(); + expect(await findById(driver, fresh.id)).not.toBeNull(); + expect((await findById(driver, keeper.id))!.status).toBe("active"); + }); + + it("复活：decay 弃用节点被 upsertNode/updateNode 命中后回 active；manual 弃用不复活", async () => { + // decay 弃用 → 重新提取 → 复活 + const { node: revivable } = await upsertNode(driver, { + type: "SKILL", name: "Revive Target", description: "原描述", content: "c", + }, TEST_SID); + await autoDeprecateNodes(driver, [revivable.id], Date.now()); + expect((await findById(driver, revivable.id))!.status).toBe("deprecated"); + + const revived = await upsertNode(driver, { + type: "SKILL", name: "Revive Target", description: "再次提取的描述", content: "更长的内容触发更新路径", + }, TEST_SID); + expect(revived.isNew).toBe(false); + expect(revived.node.status).toBe("active"); + expect(revived.node.description).toBe("原描述"); + expect(revived.node.deprecatedAt).toBeUndefined(); + expect(revived.node.deprecatedBy).toBeUndefined(); + + const refetched = await findById(driver, revivable.id); + expect(refetched!.status).toBe("active"); + expect(refetched!.description).toBe("原描述"); + expect(refetched!.deprecatedAt).toBeUndefined(); + + // decay 弃用 → 手动编辑 → 复活（updateNode 路径） + const { node: editable } = await upsertNode(driver, { + type: "SKILL", name: "Revive Edit Target", description: "编辑前", content: "c", + }, TEST_SID); + await autoDeprecateNodes(driver, [editable.id], Date.now()); + const edited = await updateNode(driver, "Revive Edit Target", { content: "编辑后内容" }); + expect(edited!.status).toBe("active"); + expect(edited!.description).toBe("编辑前"); + + // manual 弃用（deprecateNodeAndDisconnect）→ 重新提取不复活，前缀与弃用标记保留 + //（deprecatedAt/By 保留 = purge 时钟不被重新提取重置） + await deprecateNodeAndDisconnect(driver, "Revive Target"); + const afterManual = await upsertNode(driver, { + type: "SKILL", name: "Revive Target", description: "第三次提取", content: "更长的内容 2", + }, TEST_SID); + expect(afterManual.node.status).toBe("deprecated"); + expect(afterManual.node.description).toBe("[DEPRECATED] 原描述"); + expect(afterManual.node.deprecatedBy).toBe("manual"); + expect(afterManual.node.deprecatedAt).toBeGreaterThan(0); + }); + + it("applyDecay E2E：peripheral + 低分 + 超期未访问的节点被自动弃用", async () => { + const { node: target } = await upsertNode(driver, { + type: "SKILL", name: "Decay E2E Forgotten", description: "被遗忘的知识", content: "c", + }, TEST_SID); + // 构造遗忘终态：peripheral 层 + 400 天未访问 + 无 PageRank/访问加持 + const session = getSession(driver); + try { + await session.run( + `MATCH (n {id: $id}) + SET n.tier='peripheral', + n.createdAt=$old, n.updatedAt=$old, n.lastAccessedAt=$old, + n.pagerank=0.0, n.validatedCount=1`, + { id: target.id, old: Date.now() - 400 * 86_400_000 }, + ); + } finally { + await session.close(); + } + + const result = await applyDecay(driver, { decay: { ...DEFAULT_CONFIG.decay!, autoDeprecate: true } }); + expect(result.enabled).toBe(true); + expect(result.autoDeprecated).toBeGreaterThanOrEqual(1); + + const after = await findById(driver, target.id); + expect(after!.status).toBe("deprecated"); + expect(after!.deprecatedBy).toBe("decay"); + expect(after!.description).toBe("[DEPRECATED] 被遗忘的知识"); + + // 关闭开关后不再自动弃用（autoDeprecate=false 时 shouldAutoDeprecate 恒 false） + const off = await applyDecay(driver, { decay: { ...DEFAULT_CONFIG.decay!, autoDeprecate: false } }); + expect(off.enabled).toBe(true); + expect(off.autoDeprecated).toBe(0); + }); }); diff --git a/test/register-guard.test.ts b/test/register-guard.test.ts index f5c7809..b05e684 100644 --- a/test/register-guard.test.ts +++ b/test/register-guard.test.ts @@ -20,15 +20,14 @@ vi.mock("../src/store/store.ts", () => ({ upsertEdge: async () => {}, findByName: async () => null, updateNode: async () => null, - deleteNode: async () => {}, deprecateNodeAndDisconnect: async () => {}, + deprecateNodeAndDisconnectById: async () => {}, getBySession: async () => [], edgesFrom: async () => [], edgesTo: async () => [], edgesTouching: async () => [], deleteEdges: async () => {}, mergeNodes: async () => {}, - deprecate: async () => {}, getStats: async () => ({}), })); diff --git a/test/session-identity.test.ts b/test/session-identity.test.ts index 1f04900..36acde4 100644 --- a/test/session-identity.test.ts +++ b/test/session-identity.test.ts @@ -44,7 +44,7 @@ vi.mock("../src/store/store.ts", () => ({ edgesFrom: async () => [], edgesTo: async () => [], edgesTouching: async () => [], - deprecate: async () => {}, + deprecateNodeAndDisconnectById: async () => {}, getStats: async () => ({}), })); diff --git a/test/update-node.test.ts b/test/update-node.test.ts index 6276c6f..71819cf 100644 --- a/test/update-node.test.ts +++ b/test/update-node.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { applyNodePatch, applyDeprecateMarker } from "../src/store/store.ts"; +import { applyNodePatch, applyDeprecateMarker, stripDeprecateMarker } from "../src/store/store.ts"; import type { GmNode } from "../src/types.ts"; const baseNode: Pick = { @@ -64,3 +64,28 @@ describe("applyDeprecateMarker (gm_update mode=deprecate)", () => { expect(result.endsWith(long)).toBe(true); }); }); + +describe("stripDeprecateMarker (decay 自动弃用节点复活时还原描述)", () => { + it("剥掉 [DEPRECATED] 前缀，还原原文", () => { + expect(stripDeprecateMarker("[DEPRECATED] 处理 PDF 提取")).toBe("处理 PDF 提取"); + }); + + it("空描述的标记 [DEPRECATED]（无尾随空格）还原为空串", () => { + expect(stripDeprecateMarker("[DEPRECATED]")).toBe(""); + }); + + it("无前缀的描述原样返回", () => { + expect(stripDeprecateMarker("普通描述")).toBe("普通描述"); + expect(stripDeprecateMarker("")).toBe(""); + }); + + it("描述中间出现 [DEPRECATED] 子串时只剥前缀", () => { + expect(stripDeprecateMarker("[DEPRECATED] 讨论了 [DEPRECATED] 标记的用法")) + .toBe("讨论了 [DEPRECATED] 标记的用法"); + }); + + it("与 applyDeprecateMarker 互逆（非空描述）", () => { + const desc = "与遗忘曲线联动的生命周期"; + expect(stripDeprecateMarker(applyDeprecateMarker(desc))).toBe(desc); + }); +}); From be142d8aea3680a7200cc1c972dfdd9a8249bb1f Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Thu, 3 Sep 2026 01:13:07 +0000 Subject: [PATCH 24/29] Update integration.neo4j.test.ts --- test/integration.neo4j.test.ts | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/test/integration.neo4j.test.ts b/test/integration.neo4j.test.ts index 86c3ebe..e2e49ec 100644 --- a/test/integration.neo4j.test.ts +++ b/test/integration.neo4j.test.ts @@ -539,16 +539,29 @@ describe.skipIf(!ENABLED)("Neo4j integration (Docker)", () => { type: "SKILL", name: "Reembed Pipeline Skill", description: "re", content: "reembed me", }, TEST_SID); + // 前面用例在同库累积的活动节点已超过单页 limit——断言必须像 reembed CLI 一样 + // 按游标翻完整页（SKIP 分页在收缩集合上会跳号，游标翻页是实现的正确用法） + const collectAllTargets = async () => { + const all: Awaited> = []; + let cursor = ""; + for (let i = 0; i < 100; i++) { + const page = await listNodeEmbeddingTargets(driver, cursor, 10); + if (!page.length) break; + all.push(...page); + cursor = page[page.length - 1].id; + } + return all; + }; + // 先有向量 → 清空后节点必须重新出现在待嵌入列表 const vec = new Array(1024).fill(0).map((_, i) => (i % 10) / 10); await saveVector(driver, node.id, "reembed me", vec); - let targets = await listNodeEmbeddingTargets(driver, "", 10); - expect(targets.some(t => t.id === node.id)).toBe(false); + expect((await collectAllTargets()).some(t => t.id === node.id)).toBe(false); const cleared = await clearAllEmbeddings(driver); expect(cleared.nodes).toBeGreaterThanOrEqual(1); - targets = await listNodeEmbeddingTargets(driver, "", 10); + const targets = await collectAllTargets(); const target = targets.find(t => t.id === node.id); expect(target).toBeDefined(); expect(target!.name).toBe("reembed-pipeline-skill"); // upsertNode 按规范化名入库 @@ -562,8 +575,7 @@ describe.skipIf(!ENABLED)("Neo4j integration (Docker)", () => { await saveVector(driver, node.id, text, vec); const hash = await getVectorHash(driver, node.id); expect(hash).toMatch(/^[a-f0-9]{32}$/); - targets = await listNodeEmbeddingTargets(driver, "", 10); - expect(targets.some(t => t.id === node.id)).toBe(false); + expect((await collectAllTargets()).some(t => t.id === node.id)).toBe(false); }); it("重嵌入管线：社区向量清空/回填 + getVectorIndexDimensions", async () => { From 0c9b85aad4fd96fbae4bad373586d8efa6213cf6 Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:54:58 +0000 Subject: [PATCH 25/29] =?UTF-8?q?fix:=20deprecate=20lifecycle=20=E2=80=94?= =?UTF-8?q?=20autoDeprecateNodes=20status=20guard=20+=20legacy=20deprecate?= =?UTF-8?q?dAt=20backfill?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 两处评审修复： - autoDeprecateNodes MATCH 加 status:'active' 守卫：评分快照与批量写入的 时间窗内刚被手动弃用的节点不再被覆盖为 deprecatedBy='decay' + 新 deprecatedAt（既改变复活语义又重置 60 天 purge 时钟） - initSchema 启动时幂等补写：为缺 deprecatedAt 的存量 deprecated 节点 钉死 deprecatedAt = coalesce(updatedAt, createdAt)，否则 upsertNode 对 manual/merge 弃用节点的 updatedAt bump 会无限推迟 purge（永远删不掉） - 新增两个集成测试（状态守卫 + 存量补写/漂移回归/幂等），docs/decay.md 同步 --- docs/decay.md | 4 +-- src/store/db.ts | 13 +++++++ src/store/store.ts | 10 ++++-- test/integration.neo4j.test.ts | 62 ++++++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 5 deletions(-) diff --git a/docs/decay.md b/docs/decay.md index ab8dc9d..b928ab9 100644 --- a/docs/decay.md +++ b/docs/decay.md @@ -108,8 +108,8 @@ active ──遗忘曲线──▶ peripheral（仍可搜索） - **阶段一判定**（`shouldAutoDeprecate`，三条件同时满足）：`tier=peripheral`（本轮 tier 转换后的层级，刚降到 peripheral 的老节点即刻参与）**AND** `composite < peripheralCompositeThreshold`（高 intrinsic 价值节点受保护）**AND** 距最近访问（`lastAccessedAt` → `updatedAt` → `createdAt` 回退链）≥ `autoDeprecateAfterDays`。`autoDeprecate: false` 或 `enabled: false` 时整体停用。 - **手动弃用 = 一次性断联**：所有人工路径（`gm_update mode=deprecate`、finalize invalidations、REST `DELETE /nodes/:id`）统一走 `deprecateNodeAndDisconnectById`——切断所有边 + `[DEPRECATED]` 前缀 + `deprecatedBy='manual'`。由于所有召回路径都过滤 `status='active'` 且边已切断，手动弃用等效删除（不存在独立的 `gm_update mode=delete`，也没有 status-only 的轻量弃用路径）。 - **复活**：`deprecatedBy='decay'` 的节点被同名知识重新提取（`upsertNode`）或手动编辑（`updateNode`）命中时，自动恢复 `active`、剥离 `[DEPRECATED]` 前缀、清除 `deprecatedAt`/`deprecatedBy`。手动弃用（`deprecatedBy='manual'`）与 merge 败者（`deprecatedBy='merge'`）**不复活**——人工/合并语义判定优先于遗忘曲线。 -- **阶段二硬删**：所有 `deprecated` 节点（含 manual/merge/存量数据），自 `deprecatedAt`（缺省回退 `updatedAt`）起超过 `purgeAfterDays` 天后 `DETACH DELETE`。`embedding`/`contentHash` 向量属性随节点一并移除，向量索引项同步消失。`purgeAfterDays: 0` 表示永不硬删。此步不受 `enabled` 总开关约束（手动弃用的节点也需要到期清理）。 -- **存量兼容**：升级前已 deprecated 的节点没有 `deprecatedBy`，一律按 `manual` 处理（不参与复活）；硬删只看 `coalesce(deprecatedAt, updatedAt)`，存量节点的弃用时间由 `updatedAt` 兜底。 +- **阶段二硬删**：所有 `deprecated` 节点（含 manual/merge/存量数据），自 `deprecatedAt` 起超过 `purgeAfterDays` 天后 `DETACH DELETE`（`coalesce(n.deprecatedAt, n.updatedAt)` 中的 updatedAt 回退仅是防御性兜底）。`embedding`/`contentHash` 向量属性随节点一并移除，向量索引项同步消失。`purgeAfterDays: 0` 表示永不硬删。此步不受 `enabled` 总开关约束（手动弃用的节点也需要到期清理）。 +- **存量兼容**：升级前已 deprecated 的节点没有 `deprecatedBy`，一律按 `manual` 处理（不参与复活）。启动时（`initSchema`）幂等补写：为缺 `deprecatedAt` 的 deprecated 节点一次性钉死 `deprecatedAt = coalesce(updatedAt, createdAt)` 快照——否则 manual/merge 弃用节点被重新提取命中时 `upsertNode` 会 bump `updatedAt`（不复活但刷新时间戳），purge 期限被无限推后（永远删不掉）。 --- diff --git a/src/store/db.ts b/src/store/db.ts index f2fdc6a..bdbbe85 100755 --- a/src/store/db.ts +++ b/src/store/db.ts @@ -77,6 +77,19 @@ export async function initSchema(driver: Driver, embedding?: EmbeddingConfig): P // The search code queries one index across all knowledge labels. await session.run("MATCH (n:Task|Skill|Event) SET n:MemoryNode"); + + // 存量 deprecated 节点补写 deprecatedAt（幂等，等效一次性迁移）：purge 时钟基准是 + // deprecatedAt、缺失时回退 updatedAt——但 upsertNode 对 manual/merge 弃用节点只 bump + // updatedAt（不复活），存量无 deprecatedAt 的节点每次被重新提取命中都会把 purge 期限 + // 往后推（无限续命、永远删不掉）。启动时把时钟一次性钉死为当时的 + // coalesce(updatedAt, createdAt) 快照，此后 purge 不再受 updatedAt 漂移影响。 + // 全新弃用路径（manual/merge/decay）都已显式写 deprecatedAt，故首次运行后恒零命中。 + await session.run(` + MATCH (n:Task|Skill|Event {status: 'deprecated'}) + WHERE n.deprecatedAt IS NULL + SET n.deprecatedAt = coalesce(n.updatedAt, n.createdAt) + `); + await session.run(` CREATE VECTOR INDEX gm_node_embedding IF NOT EXISTS FOR (n:MemoryNode) ON (n.embedding) diff --git a/src/store/store.ts b/src/store/store.ts index 272fdea..f4dd5c0 100755 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -367,7 +367,9 @@ export async function deprecateNodeAndDisconnect( * 遗忘曲线自动弃用（两阶段生命周期·阶段一）：批量 status='deprecated' + deprecatedBy='decay' * + 描述幂等加 [DEPRECATED] 前缀 + 切断所有边。与 deprecateNodeAndDisconnect 的区别： * 按 id 批量、来源标记 decay（重新提取/编辑命中时可被 upsertNode/updateNode 复活）。 - * 返回实际弃用的节点数。 + * MATCH 带状态守卫：ids 来自 applyDecay 的 active 扫描快照，评分与批量写入之间存在 + * 时间窗——窗口内刚被手动弃用的节点不得被覆盖为 decay（既改变复活语义又重置 purge 时钟）， + * 因此只弃用仍为 active 的节点。返回实际弃用的节点数。 */ export async function autoDeprecateNodes( driver: Driver, @@ -379,7 +381,7 @@ export async function autoDeprecateNodes( try { const result = await session.run(` UNWIND $ids AS nid - MATCH (n:Task|Skill|Event {id: nid}) + MATCH (n:Task|Skill|Event {id: nid, status: 'active'}) SET n.status = 'deprecated', n.deprecatedAt = $now, n.deprecatedBy = 'decay', @@ -402,7 +404,9 @@ export async function autoDeprecateNodes( /** * 两阶段生命周期·阶段二：硬删过期 deprecated 节点（DETACH DELETE，embedding/contentHash - * 向量属性随节点一并移除，释放存储）。时钟基准 deprecatedAt，存量节点缺省回退 updatedAt。 + * 向量属性随节点一并移除，释放存储）。时钟基准 deprecatedAt；coalesce 回退 updatedAt 仅是 + * 防御性兜底——initSchema 启动时已为存量 deprecated 节点一次性补写 deprecatedAt（否则 + * upsertNode 对 manual/merge 弃用节点的 updatedAt bump 会无限推迟 purge）。 * 适用于所有 deprecated 节点（decay/manual/merge）；purgeAfterMs<=0 时为 no-op。返回删除数。 */ export async function purgeDeprecatedNodes( diff --git a/test/integration.neo4j.test.ts b/test/integration.neo4j.test.ts index e2e49ec..f8fae0b 100644 --- a/test/integration.neo4j.test.ts +++ b/test/integration.neo4j.test.ts @@ -659,6 +659,33 @@ describe.skipIf(!ENABLED)("Neo4j integration (Docker)", () => { expect(p!.description).toBe("[DEPRECATED] 已有前缀"); }); + it("autoDeprecateNodes 状态守卫：窗口内已被手动弃用的节点不被覆盖为 decay", async () => { + const { node: manual } = await upsertNode(driver, { + type: "SKILL", name: "AutoDeprecate Guard Manual", description: "手动弃用", content: "c", + }, TEST_SID); + const { node: active } = await upsertNode(driver, { + type: "SKILL", name: "AutoDeprecate Guard Active", description: "仍活跃", content: "c", + }, TEST_SID); + + // 模拟评分快照之后、批量写入之前落入窗口的手动弃用 + const manualAt = Date.now() - 1_000; + await deprecateNodeAndDisconnectById(driver, manual.id, manualAt); + + const now = Date.now(); + // 只统计仍为 active 的节点：manual 已弃用不计入 + expect(await autoDeprecateNodes(driver, [manual.id, active.id], now)).toBe(1); + + const m = await findById(driver, manual.id); + expect(m!.status).toBe("deprecated"); + expect(m!.deprecatedBy).toBe("manual"); + expect(m!.deprecatedAt).toBe(manualAt); // 溯源与 purge 时钟均不被 decay 覆盖 + + const a = await findById(driver, active.id); + expect(a!.status).toBe("deprecated"); + expect(a!.deprecatedBy).toBe("decay"); + expect(a!.deprecatedAt).toBe(now); + }); + it("purgeDeprecatedNodes：过期 deprecated 硬删、未过期保留、active 不动、存量按 updatedAt 兜底", async () => { const { node: fresh } = await upsertNode(driver, { type: "SKILL", name: "Purge Fresh", description: "fresh", content: "c", @@ -706,6 +733,41 @@ describe.skipIf(!ENABLED)("Neo4j integration (Docker)", () => { expect((await findById(driver, keeper.id))!.status).toBe("active"); }); + it("initSchema 存量迁移：为缺 deprecatedAt 的 deprecated 节点补写时钟（防 updatedAt 漂移无限续命）", async () => { + const { node: legacy } = await upsertNode(driver, { + type: "SKILL", name: "Backfill Legacy Deprecated", description: "存量弃用", content: "c", + }, TEST_SID); + const old = Date.now() - 90 * 86_400_000; + const session = getSession(driver); + try { + // 模拟存量数据：deprecated 但无 deprecatedAt（按 manual 处理，不复活） + await session.run( + "MATCH (n {id: $id}) SET n.status='deprecated', n.updatedAt=$old REMOVE n.deprecatedAt, n.deprecatedBy", + { id: legacy.id, old }, + ); + } finally { + await session.close(); + } + + await initSchema(driver); + expect((await findById(driver, legacy.id))!.deprecatedAt).toBe(old); + + // 补写后 manual 弃用节点被重新提取命中：upsertNode bump updatedAt（不复活）， + // 但 purge 时钟已钉死在 deprecatedAt，不再被推后（修复前：回退 updatedAt → 无限续命） + const hit = await upsertNode(driver, { + type: "SKILL", name: "Backfill Legacy Deprecated", description: "重新提取", content: "更长的内容触发更新路径", + }, TEST_SID); + expect(hit.isNew).toBe(false); + expect(hit.node.status).toBe("deprecated"); + const refetched = await findById(driver, legacy.id); + expect(refetched!.updatedAt!).toBeGreaterThan(old); + expect(refetched!.deprecatedAt).toBe(old); + + // 幂等：重复 initSchema 不改写已存在的 deprecatedAt + await initSchema(driver); + expect((await findById(driver, legacy.id))!.deprecatedAt).toBe(old); + }); + it("复活：decay 弃用节点被 upsertNode/updateNode 命中后回 active；manual 弃用不复活", async () => { // decay 弃用 → 重新提取 → 复活 const { node: revivable } = await upsertNode(driver, { From c548f2bccd6f6e06ab15af23c1261e07cd4da5ae Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:55:54 +0000 Subject: [PATCH 26/29] =?UTF-8?q?feat:=20=E6=8F=90=E5=8F=96/=E5=8F=AC?= =?UTF-8?q?=E5=9B=9E=20LLM=20=E4=B8=8E=20embedding=20=E6=88=90=E6=9C=AC?= =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=A5=97=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 提取模式 cfg.extract.mode：per-turn（默认）/ batched 攒批提取，LLM 调用 次数降为 ~1/N，session_end 冲洗尾批 - trivial 轮本地预筛（turn-filter）：清洗后为空/无意义词表/超短无技术词的 轮次直接 markExtracted(producedKnowledge=false)，省一次完整 completion - 查询向量 LRU 缓存（query-cache）：同会话重复召回复用 query embedding， 零 embedding API 调用（db 模式不入缓存） - 批量向量读写：getVectorHashes/saveVectors 一次 UNWIND 替代 N 次单节点 往返（syncEmbedBatch 用） - finalize 阶梯触发 + 社区摘要 top-k 稳定签名，削减 session_end LLM 开销 - 测试：turn-filter/query-cache/extract-cost-guards 单测 + 消融实验 + 真实服务 embedding E2E（EMBED_E2E 门控） --- index.ts | 234 +++++--- src/cli-extract.ts | 21 +- src/extractor/extract.ts | 11 + src/extractor/turn-filter.ts | 67 +++ src/graph/community.ts | 42 +- src/recaller/query-cache.ts | 43 ++ src/recaller/recall.ts | 85 ++- src/store/store.ts | 60 +++ src/types.ts | 27 + test/ablation.study.test.ts | 828 +++++++++++++++++++++++++++++ test/cli-extract.test.ts | 3 + test/commit-turn.test.ts | 1 + test/extract-cost-guards.test.ts | 82 +++ test/integration.embed-e2e.test.ts | 147 +++++ test/integration.recall.test.ts | 59 ++ test/query-cache.test.ts | 53 ++ test/register-guard.test.ts | 1 + test/session-identity.test.ts | 1 + test/turn-filter.test.ts | 102 ++++ 19 files changed, 1777 insertions(+), 90 deletions(-) create mode 100644 src/extractor/turn-filter.ts create mode 100644 src/recaller/query-cache.ts create mode 100644 test/ablation.study.test.ts create mode 100644 test/extract-cost-guards.test.ts create mode 100644 test/integration.embed-e2e.test.ts create mode 100644 test/query-cache.test.ts create mode 100644 test/turn-filter.test.ts diff --git a/index.ts b/index.ts index e0a208e..33c04b3 100755 --- a/index.ts +++ b/index.ts @@ -10,7 +10,7 @@ import { Type } from "@sinclair/typebox"; import { getDriver, initSchema, getSession } from "./src/store/db.ts"; import { Neo4jGate } from "./src/store/gate.ts"; import { - saveMessage, getUnextracted, getMaxTurnIndex, + saveMessage, getUnextracted, countUnextracted, getMaxTurnIndex, markExtracted, isTurnExtracted, commitTurnAdvance, upsertNode, upsertEdge, findByName, updateNode, deprecateNodeAndDisconnect, deprecateNodeAndDisconnectById, @@ -19,15 +19,16 @@ import { getStats, } from "./src/store/store.ts"; import { createCompleteFn, resolveProvider } from "./src/engine/llm.ts"; -import { createEmbedFn } from "./src/engine/embed.ts"; +import { createEmbedder } from "./src/engine/embed.ts"; import { estimateTokens } from "./src/tokens.ts"; import { Recaller, parseTimeRange } from "./src/recaller/recall.ts"; -import { Extractor } from "./src/extractor/extract.ts"; +import { Extractor, shouldRunFinalize } from "./src/extractor/extract.ts"; +import { shouldSkipTurnExtraction } from "./src/extractor/turn-filter.ts"; import { assembleContext } from "./src/format/assemble.ts"; import { sanitizeToolUseResultPairing } from "./src/format/transcript-repair.ts"; import { runMaintenance } from "./src/graph/maintenance.ts"; import { normalizeMessageRetentionPolicy } from "./src/store/retention.ts"; -import { DEFAULT_CONFIG, DEFAULT_CRON_CONFIG, isCronSessionKey, type GmConfig, type RecallResult, type EdgeType } from "./src/types.ts"; +import { DEFAULT_CONFIG, DEFAULT_CRON_CONFIG, isCronSessionKey, type GmConfig, type GmNode, type RecallResult, type EdgeType } from "./src/types.ts"; import { registerCrudRoutes } from "./src/routes/crud.ts"; import { createGraphMemoryCli } from "./src/cli.ts"; @@ -195,6 +196,17 @@ export function extractUserText(msg: any): string { return raw; } +/** + * 聚合一轮消息中 user 角色的（元数据剥离后）文本，供 trivial 预筛判定。 + * assistant/工具结果不参与：它们跟随用户意图，用户输入有意义时整轮照常提取。 + */ +export function turnUserText(messages: any[]): string { + return (messages ?? []) + .filter((m) => m && typeof m === "object" && m.role === "user") + .map((m) => extractUserText(m)) + .join(" "); +} + export function sliceLastTurn( messages: any[], keepTurns: number = KEEP_TURNS, @@ -331,6 +343,7 @@ const graphMemoryProPlugin = { if (raw.neo4j) cfg.neo4j = { ...DEFAULT_CONFIG.neo4j, ...raw.neo4j }; if (raw.decay) cfg.decay = { ...DEFAULT_CONFIG.decay, ...raw.decay }; if (raw.cron) cfg.cron = { ...DEFAULT_CONFIG.cron, ...raw.cron }; + if (raw.extract) cfg.extract = { ...DEFAULT_CONFIG.extract, ...raw.extract }; // 拼写兼容：接受小写 baseUrl（部分宿主/用户的配置习惯），统一归一到 baseURL。 // 显式 baseURL 优先；trim 后为空视为未配置。 @@ -360,6 +373,13 @@ const graphMemoryProPlugin = { } const cronCfg = cfg.cron ?? DEFAULT_CRON_CONFIG; + // 提取配置（LLM 成本控制）：per-turn/batched 模式 + trivial 轮本地预筛 + const extractCfg = cfg.extract ?? DEFAULT_CONFIG.extract!; + const trivialFilterOpts = { + maxChars: extractCfg.trivialMaxChars, + extraPrompts: extractCfg.trivialPrompts, + }; + const providerModel = readDefaultModel(api.config); // Model 解析链：cfg.llm.model（插件级显式配置） → agents.defaults.model（openclaw provider 级） @@ -428,10 +448,11 @@ const graphMemoryProPlugin = { let embedProbeInFlight = false; let lastEmbedProbeAt = 0; - createEmbedFn(cfg.embedding) - .then((fn) => { - if (fn) { - recaller.setEmbedFn(fn); + createEmbedder(cfg.embedding) + .then((embedder) => { + if (embedder) { + recaller.setEmbedFn(embedder.embed); + recaller.setEmbedBatchFn(embedder.embedBatch); api.logger.info("[graph-memory-pro] vector search ready"); } else { lastEmbedProbeAt = Date.now(); @@ -478,6 +499,15 @@ const graphMemoryProPlugin = { api.logger.info(`[graph-memory-pro] turn ${turnNum}: already extracted (compact), skipping`); return; } + + // LLM 成本控制：trivial 轮本地预筛（无意义词表 / ≤trivialMaxChars 纯文本）， + // 命中则零 LLM 直接标记（producedKnowledge=false，原始证据保留） + if (shouldSkipTurnExtraction(turnUserText(rawMessages), trivialFilterOpts)) { + await markExtracted(driver, sessionId, turnNum, false); + api.logger.info(`[graph-memory-pro] turn ${turnNum}: trivial prompt, extraction skipped (local pre-filter)`); + return; + } + const existing = (await getBySession(driver, sessionId)).map(n => n.name); const result = await extractor.extract({ messages: rawMessages, @@ -494,14 +524,17 @@ const graphMemoryProPlugin = { } const nameToId = new Map(); + const upsertedNodes: GmNode[] = []; for (const nc of result.nodes) { const { node } = await upsertNode(driver, { type: nc.type, name: nc.name, description: nc.description, content: nc.content, }, sessionId); nameToId.set(node.name, node.id); - recaller.syncEmbed(node).catch(() => {}); + upsertedNodes.push(node); } + // 批量向量同步：N 个节点一次 embedBatch + 一次 UNWIND 批读写（替代逐节点 N 次单发） + void recaller.syncEmbedBatch(upsertedNodes).catch(() => {}); for (const ec of result.edges) { const fromNode = await findByName(driver, ec.from); @@ -526,6 +559,95 @@ const graphMemoryProPlugin = { }); } + /** + * 批量提取共享体（compact / batched 模式攒批触发 / session_end 尾批冲洗）： + * 读未提取消息 → LLM → upsert + 批量向量同步 → markExtracted。 + * 调用方负责 cron/熔断门控；内部经 withExtractLock 与 per-turn 路径串行化。 + */ + async function extractUnextractedBatch(sessionId: string): Promise<{ + ok: boolean; compacted: boolean; reason?: string; summary?: string; + }> { + return withExtractLock(sessionId, async () => { + // 掉线恢复后的补提取路径：先把缓冲消息刷进 DB 再读未提取集 + if (messageBuffer.length) await flushMessageBuffer(); + const msgs = await getUnextracted(driver, sessionId, cfg.compactTurnCount * 3); + + if (!msgs.length) return { ok: true, compacted: false, reason: "no messages" }; + + try { + const existing = (await getBySession(driver, sessionId)).map(n => n.name); + const result = await extractor.extract({ messages: msgs, existingNames: existing }); + + const nameToId = new Map(); + const upsertedNodes: GmNode[] = []; + for (const nc of result.nodes) { + const { node } = await upsertNode(driver, { + type: nc.type, name: nc.name, + description: nc.description, content: nc.content, + }, sessionId); + nameToId.set(node.name, node.id); + upsertedNodes.push(node); + } + void recaller.syncEmbedBatch(upsertedNodes).catch(() => {}); + + for (const ec of result.edges) { + const fromNode = await findByName(driver, ec.from); + const toNode = await findByName(driver, ec.to); + const fromId = nameToId.get(ec.from) ?? fromNode?.id; + const toId = nameToId.get(ec.to) ?? toNode?.id; + if (fromId && toId) { + await upsertEdge(driver, { + fromId, toId, type: ec.type, + instruction: ec.instruction, condition: ec.condition, sessionId, + }); + } + } + + const maxTurn = Math.max(...msgs.map((m: any) => m.turn_index)); + await markExtracted( + driver, sessionId, maxTurn, + result.nodes.length > 0 || result.edges.length > 0, + ); + + return { + ok: true, compacted: true, + summary: `extracted ${result.nodes.length} nodes, ${result.edges.length} edges`, + }; + } catch (err) { + api.logger.error(`[graph-memory-pro] batch extraction failed: ${err}`); + return { ok: false, compacted: false, reason: String(err) }; + } + }); + } + + /** + * 轮提取统一入口（LLM 成本控制的模式分发）： + * - per-turn（默认）：每轮即时 LLM 提取（内含 trivial 本地预筛）； + * - batched：不在轮边界调 LLM；未提取消息累计到 compactTurnCount*3 条时 + * 批量提取一次（LLM 调用 ~1/N），session_end 冲洗尾批。 + * 注意 batched 不做逐轮 trivial 标记——markExtracted 是前缀语义， + * 会误吞留给攒批的更早轮次；trivial 消息随批进入 prompt，成本由批摊薄。 + */ + function scheduleTurnExtraction(sessionId: string, turnNum: number, rawMessages: any[]): void { + const run = extractCfg.mode === "batched" + ? handleBatchedTurn(sessionId) + : extractTurnKnowledge(sessionId, turnNum, rawMessages); + run.catch(err => api.logger.error(`[graph-memory-pro] extract failed: ${err}`)); + } + + async function handleBatchedTurn(sessionId: string): Promise { + // 熔断开启时跳过计数/提取：消息保持未标记，恢复后由攒批/冲洗/CLI 补提取 + if (!neo4jGate.isAvailable()) return; + try { + const pending = await countUnextracted(driver, sessionId); + if (pending >= cfg.compactTurnCount * 3) { + await extractUnextractedBatch(sessionId); + } + } catch (err) { + api.logger.error(`[graph-memory-pro] batched turn check failed: ${err}`); + } + } + // ── Session 运行时状态 ────────────────────────────────── const msgSeq = new Map(); const msgSeqLoaders = new Map>(); @@ -751,10 +873,11 @@ const graphMemoryProPlugin = { if (Date.now() - lastEmbedProbeAt < EMBED_REPROBE_INTERVAL_MS) return; embedProbeInFlight = true; lastEmbedProbeAt = Date.now(); - createEmbedFn(cfg.embedding) - .then(fn => { - if (fn) { - recaller.setEmbedFn(fn); + createEmbedder(cfg.embedding) + .then((embedder) => { + if (embedder) { + recaller.setEmbedFn(embedder.embed); + recaller.setEmbedBatchFn(embedder.embedBatch); api.logger.info("[graph-memory-pro] embedding re-probe succeeded — vector search re-enabled"); } }) @@ -932,58 +1055,17 @@ const graphMemoryProPlugin = { if (!neo4jGate.isAvailable()) { return { ok: true, compacted: false, reason: "neo4j unavailable (circuit open)" }; } - return withExtractLock(sessionId, async () => { - // compact 是掉线恢复后的补提取路径：先把缓冲消息刷进 DB 再读未提取集 - if (messageBuffer.length) await flushMessageBuffer(); - const msgs = await getUnextracted(driver, sessionId, cfg.compactTurnCount * 3); - - if (!msgs.length) return { ok: true, compacted: false, reason: "no messages" }; - - try { - const existing = (await getBySession(driver, sessionId)).map(n => n.name); - const result = await extractor.extract({ messages: msgs, existingNames: existing }); - - const nameToId = new Map(); - for (const nc of result.nodes) { - const { node } = await upsertNode(driver, { - type: nc.type, name: nc.name, - description: nc.description, content: nc.content, - }, sessionId); - nameToId.set(node.name, node.id); - recaller.syncEmbed(node).catch(() => {}); - } - - for (const ec of result.edges) { - const fromNode = await findByName(driver, ec.from); - const toNode = await findByName(driver, ec.to); - const fromId = nameToId.get(ec.from) ?? fromNode?.id; - const toId = nameToId.get(ec.to) ?? toNode?.id; - if (fromId && toId) { - await upsertEdge(driver, { - fromId, toId, type: ec.type, - instruction: ec.instruction, condition: ec.condition, sessionId, - }); - } - } - - const maxTurn = Math.max(...msgs.map((m: any) => m.turn_index)); - await markExtracted( - driver, sessionId, maxTurn, - result.nodes.length > 0 || result.edges.length > 0, - ); - - return { - ok: true, compacted: true, - result: { - summary: `extracted ${result.nodes.length} nodes, ${result.edges.length} edges`, - tokensBefore: currentTokenCount ?? 0, - }, - }; - } catch (err) { - api.logger.error(`[graph-memory-pro] compact failed: ${err}`); - return { ok: false, compacted: false, reason: String(err) }; - } - }); + const res = await extractUnextractedBatch(sessionId); + return { + ok: res.ok, compacted: res.compacted, + ...(res.reason ? { reason: res.reason } : {}), + ...(res.summary ? { + result: { + summary: res.summary, + tokensBefore: currentTokenCount ?? 0, + }, + } : {}), + }; }, async afterTurn({ sessionId, sessionKey, messages, prePromptMessageCount, isHeartbeat }: { @@ -1032,10 +1114,8 @@ const graphMemoryProPlugin = { return; } - // 直接用原始消息提取知识图谱（异步，不阻塞） - extractTurnKnowledge(sessionId, turnNum, newMessages).catch(err => { - api.logger.error(`[graph-memory-pro] extract failed: ${err}`); - }); + // 按模式分发：per-turn 即时 LLM 提取；batched 攒批（LLM 调用 ~1/N） + scheduleTurnExtraction(sessionId, turnNum, newMessages); }, /** @@ -1093,9 +1173,7 @@ const graphMemoryProPlugin = { // 只覆盖旧前缀 → 本轮消息保持未提取 → compact 重复提取 if (messageBuffer.length) await flushMessageBuffer(); const turnNum = msgSeq.get(sid) ?? 0; - extractTurnKnowledge(sid, turnNum, messages).catch(err => { - api.logger.error(`[graph-memory-pro] extract failed: ${err}`); - }); + scheduleTurnExtraction(sid, turnNum, messages); } } return { status: "committed" as const }; @@ -1167,6 +1245,11 @@ const graphMemoryProPlugin = { return; } + // batched 模式：会话结束冲洗残留未提取批（攒批未达阈值时保证知识不丢） + if (extractCfg.mode === "batched") { + await extractUnextractedBatch(sid); + } + let nodes: Awaited>; try { nodes = await getBySession(driver, sid); @@ -1177,7 +1260,7 @@ const graphMemoryProPlugin = { api.logger.error(`[graph-memory-pro] session_end error: ${err}`); return; } - if (nodes.length) { + if (nodes.length && shouldRunFinalize(nodes)) { // finalize 的 upsert 与 afterTurn/compact 的提取共用 per-session 互斥锁： // 最后一轮的 afterTurn 提取可能仍在途，不串行化会重复 upsert（validatedCount 双递增） await withExtractLock(sid, async () => { @@ -1219,6 +1302,11 @@ const graphMemoryProPlugin = { } for (const id of fin.invalidations) await deprecateNodeAndDisconnectById(driver, id); }); + } else if (nodes.length) { + // finalize 阶梯触发（LLM 成本控制）：小会话/无 EVENT 节点时跳过这次调用 + api.logger.info( + `[graph-memory-pro] session_end ${sid.slice(0, 12)}…: finalize skipped (small session / no EVENT nodes)`, + ); } // 图维护：后台单飞（A1）—— 衰减→去重→PR→社区→LLM 摘要可能耗时数分钟， diff --git a/src/cli-extract.ts b/src/cli-extract.ts index a34f3d7..1f61461 100644 --- a/src/cli-extract.ts +++ b/src/cli-extract.ts @@ -13,7 +13,7 @@ import readline from "node:readline/promises"; import { stdin as input, stdout as output } from "node:process"; import type { Driver } from "neo4j-driver"; -import type { GmConfig } from "./types.ts"; +import type { GmConfig, GmNode } from "./types.ts"; import { getDriver, initSchema, closeDriver } from "./store/db.ts"; import { listUnextractedSessions, @@ -26,7 +26,7 @@ import { type UnextractedSessionInfo, } from "./store/store.ts"; import { createCompleteFn, resolveProvider } from "./engine/llm.ts"; -import { createEmbedFn } from "./engine/embed.ts"; +import { createEmbedder } from "./engine/embed.ts"; import { Recaller } from "./recaller/recall.ts"; import { Extractor } from "./extractor/extract.ts"; @@ -127,10 +127,11 @@ export async function runBackfillExtraction( const llm = createCompleteFn(params.effectiveModel, cfg.llm); const extractor = new Extractor(llm); const recaller = new Recaller(driver, cfg); - const embedFn = await createEmbedFn(cfg.embedding); - if (embedFn) { - recaller.setEmbedFn(embedFn); - log("[graph-memory-pro] embedding 已就绪，新节点将同步向量。"); + const embedder = await createEmbedder(cfg.embedding); + if (embedder) { + recaller.setEmbedFn(embedder.embed); + recaller.setEmbedBatchFn(embedder.embedBatch); + log("[graph-memory-pro] embedding 已就绪，新节点将批量同步向量。"); } else { log("[graph-memory-pro] 未配置 embedding，跳过向量同步（dual-path recall 会降级为文本搜索）。"); } @@ -235,11 +236,11 @@ async function extractSessionLoop( const extraction = await extractor.extract({ messages: msgs, existingNames: existing }); const nameToId = new Map(); - // 批内 fire-and-forget 的 syncEmbed 收集到批边界统一 await： + // 批内节点收集到批边界统一批量嵌入（embedBatch + UNWIND 批读写）： // closeDriver 在 finally 里执行，若不等待，最后一批在途的 embedding // HTTP 请求会撞上已关闭的 driver 且错误被吞——向量丢失且不可自愈 // （markExtracted 已执行，重跑 extract 不会补）。 - const pendingEmbeds: Promise[] = []; + const batchNodes: GmNode[] = []; for (const nc of extraction.nodes) { const { node } = await upsertNode(driver, { type: nc.type, name: nc.name, @@ -247,7 +248,7 @@ async function extractSessionLoop( }, sessionId); nameToId.set(node.name, node.id); stats.nodes += 1; - pendingEmbeds.push(recaller.syncEmbed(node).catch(() => {})); + batchNodes.push(node); } for (const ec of extraction.edges) { @@ -264,7 +265,7 @@ async function extractSessionLoop( } } - await Promise.allSettled(pendingEmbeds); + await recaller.syncEmbedBatch(batchNodes).catch(() => {}); const maxTurn = msgs.reduce((m, msg) => Math.max(m, msg.turn_index ?? 0), 0); await markExtracted( diff --git a/src/extractor/extract.ts b/src/extractor/extract.ts index f0bc367..41276dc 100755 --- a/src/extractor/extract.ts +++ b/src/extractor/extract.ts @@ -7,6 +7,7 @@ import type { ExtractionResult, FinalizeResult } from "../types.ts"; import { EDGE_TYPES, isValidEdgeDirection } from "../types.ts"; +import type { GmNode } from "../types.ts"; import type { CompleteFn } from "../engine/llm.ts"; import { normalizeName } from "../store/store.ts"; @@ -205,6 +206,16 @@ export function correctEdgeType( // ─── Extractor ──────────────────────────────────────────────── +/** + * finalize 阶梯触发（LLM 成本控制）： + * finalize 的核心产出是 EVENT→SKILL 提升与跨会话建边/失效判定—— + * 会话规模 ≤ 2 或没有任何 EVENT 节点时，这次 LLM 调用几乎必然空转，直接跳过。 + */ +export function shouldRunFinalize(sessionNodes: Array>): boolean { + if (sessionNodes.length <= 2) return false; + return sessionNodes.some((n) => n.type === "EVENT"); +} + export class Extractor { constructor(private llm: CompleteFn) {} diff --git a/src/extractor/turn-filter.ts b/src/extractor/turn-filter.ts new file mode 100644 index 0000000..a619a3a --- /dev/null +++ b/src/extractor/turn-filter.ts @@ -0,0 +1,67 @@ +/** + * graph-memory-pro — trivial 轮次本地预筛（LLM 成本控制） + * + * 在 extractTurnKnowledge 进入 LLM 前判断该轮是否"不可能产出知识"： + * 命中则直接 markExtracted(producedKnowledge=false)，省掉一次完整 completion。 + * + * 判定（保守取向，宁可漏判 trivial 也不误杀知识轮）： + * 1. 用户输入清洗后为空； + * 2. 清洗后命中无意义词表（内置 + cfg.extract.trivialPrompts，精确匹配）； + * 3. 清洗后长度 ≤ trivialMaxChars（默认 5）且不含技术词 + * （连续 ≥3 位字母数字，如 pnpm/jwt/k8s——这类短输入仍走 LLM）。 + * + * 只看 user 角色文本；工具结果与 assistant 回复不参与（它们跟随用户意图， + * 用户输入有意义时整轮照常提取）。 + */ + +/** 清洗：去空白与中西文标点，转小写。用于词表精确匹配与长度计量。 */ +export function normalizeTrivialText(raw: string): string { + return raw + .toLowerCase() + .replace(/[\s\p{P}\p{S}]+/gu, "") + .trim(); +} + +/** + * 内置无意义词表：纯推进/确认/致谢类输入，语义上不可能携带可提取三元组。 + * 匹配前同样经过 normalizeTrivialText（"继续。" → "继续"，"OK!" → "ok"）。 + */ +export const BUILTIN_TRIVIAL_PROMPTS: readonly string[] = [ + // 推进类 + "继续", "请继续", "接着", "接着来", "往下", "继续吧", "goon", "goahead", + "continue", "resume", "next", "proceed", "keepgoing", + // 确认类 + "ok", "okay", "好", "好的", "好吧", "可以", "行", "嗯", + "嗯嗯", "对", "是的", "没问题", "明白", "知道了", "懂了", "收到", "gotit", + "understood", "yes", "yeah", "yep", "sure", + // 致谢类 + "谢谢", "多谢", "感谢", "辛苦了", "thanks", "thankyou", "thx", "tnx", +]; + +/** 连续 ≥3 位字母数字视为技术词（pnpm/jwt/k8s/csv…），短输入含技术词时不判 trivial。 */ +const TECH_TOKEN_RE = /[a-z0-9]{3,}/; + +export interface TrivialFilterOptions { + /** 长度阈值；默认 5。 */ + maxChars?: number; + /** 追加词表（与内置合并，匹配前统一 normalize）。 */ + extraPrompts?: string[]; +} + +/** 该轮是否应跳过 LLM 提取（本地零成本判定）。 */ +export function shouldSkipTurnExtraction(userText: string, opts?: TrivialFilterOptions): boolean { + const normalized = normalizeTrivialText(userText ?? ""); + if (!normalized) return true; + + const stoplist = new Set(BUILTIN_TRIVIAL_PROMPTS); + for (const p of opts?.extraPrompts ?? []) { + const n = normalizeTrivialText(p); + if (n) stoplist.add(n); + } + if (stoplist.has(normalized)) return true; + + const maxChars = opts?.maxChars ?? 5; + if (normalized.length <= maxChars && !TECH_TOKEN_RE.test(normalized)) return true; + + return false; +} diff --git a/src/graph/community.ts b/src/graph/community.ts index af76335..6238641 100755 --- a/src/graph/community.ts +++ b/src/graph/community.ts @@ -149,6 +149,28 @@ export function buildCommunityMemberSignature(memberIds: string[]): string { return createHash("sha1").update([...memberIds].sort().join(",")).digest("hex"); } +/** + * top-k 稳定签名（LLM 成本控制）：取 validatedCount 最高的 k 个成员计算签名， + * 而非全量成员集合。社区边界抖动（每次微增/减一个低频成员）不再触发 LLM 重摘要—— + * 摘要语义本就由高价值成员主导。成员数 ≤ k 时退化为全量签名（与旧格式一致）。 + * 并列的 validatedCount 用 id 字典序决胜负，保证确定性。 + */ +export const COMMUNITY_SIGNATURE_TOP_K = 8; + +export function buildTopKMemberSignature( + members: Array<{ id: string; validatedCount: number }>, + k: number = COMMUNITY_SIGNATURE_TOP_K, +): string { + const top = [...members] + .sort((a, b) => + b.validatedCount - a.validatedCount || + (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) + .slice(0, Math.max(1, k)) + .map((m) => m.id) + .sort(); + return createHash("sha1").update(top.join(",")).digest("hex"); +} + export async function summarizeCommunities( driver: Driver, communities: Map, @@ -160,7 +182,25 @@ export async function summarizeCommunities( for (const [communityId, memberIds] of communities) { if (memberIds.length === 0) continue; - const memberSignature = buildCommunityMemberSignature(memberIds); + // 轻量元数据查询（id + validatedCount）：top-k 签号的输入 + const metaSession = getSession(driver); + let memberMeta: Array<{ id: string; validatedCount: number }>; + try { + const metaResult = await metaSession.run(` + MATCH (n:Task|Skill|Event {status: 'active'}) + WHERE n.id IN $memberIds + RETURN n.id AS id, n.validatedCount AS vc + `, { memberIds }); + memberMeta = metaResult.records.map(r => ({ + id: r.get("id"), + validatedCount: typeof r.get("vc") === "number" ? r.get("vc") : (r.get("vc")?.toNumber?.() ?? 0), + })); + } finally { + await metaSession.close(); + } + if (memberMeta.length === 0) continue; + + const memberSignature = buildTopKMemberSignature(memberMeta); const current = await getCommunitySummary(driver, communityId); if (current?.memberSignature === memberSignature && current.summary.trim()) { diff --git a/src/recaller/query-cache.ts b/src/recaller/query-cache.ts new file mode 100644 index 0000000..adf3479 --- /dev/null +++ b/src/recaller/query-cache.ts @@ -0,0 +1,43 @@ +/** + * graph-memory-pro — 查询向量 LRU 缓存 + * + * recall 的 query embedding 是每次召回的固定开销（低成本但高频）： + * 同会话内重复/回退的相同查询（before_agent_start ↔ assemble ↔ gm_search） + * 直接复用向量，省一次 embedding API 调用。 + * + * 注意：db 模式向量不走缓存（节点文本入库前必算，且 MiniMax 的 db/query + * 走不同模型，向量不可互换）。embedding 端点/模型切换时调用方应 clear()。 + */ + +export class QueryVecCache { + private map = new Map(); + + constructor(private readonly capacity = 64) {} + + get(key: string): number[] | undefined { + const hit = this.map.get(key); + if (hit === undefined) return undefined; + // LRU 触碰：删掉重插，移到最新端 + this.map.delete(key); + this.map.set(key, hit); + return hit; + } + + set(key: string, vec: number[]): void { + if (this.map.has(key)) this.map.delete(key); + else if (this.map.size >= this.capacity) { + // Map 迭代序 = 插入序，最旧的是第一个 key + const oldest = this.map.keys().next().value; + if (oldest !== undefined) this.map.delete(oldest); + } + this.map.set(key, vec); + } + + clear(): void { + this.map.clear(); + } + + get size(): number { + return this.map.size; + } +} diff --git a/src/recaller/recall.ts b/src/recaller/recall.ts index 7625a60..a2ffc91 100755 --- a/src/recaller/recall.ts +++ b/src/recaller/recall.ts @@ -7,15 +7,19 @@ import type { Driver } from "neo4j-driver"; import { createHash } from "crypto"; import type { GmConfig, RecallResult, GmNode, GmEdge } from "../types.ts"; -import type { EmbedFn } from "../engine/embed.ts"; +import type { EmbedFn, EmbedBatchFn } from "../engine/embed.ts"; import { searchNodes, vectorSearchWithScore, graphWalk, communityRepresentatives, communityVectorSearch, nodesByCommunityIds, - saveVector, getVectorHash, + saveVector, getVectorHash, getVectorHashes, saveVectors, } from "../store/store.ts"; import { getCommunityPeers } from "../graph/community.ts"; import { personalizedPageRank } from "../graph/pagerank.ts"; +import { QueryVecCache } from "./query-cache.ts"; + +/** 批量嵌入分块大小：对齐 reembed 默认批（32），兼顾服务端批量上限与吞吐。 */ +const SYNC_EMBED_BATCH = 32; export function buildNodeEmbeddingText( node: Pick, @@ -83,10 +87,23 @@ export function matchTimeRange( export class Recaller { private embed: EmbedFn | null = null; + private embedBatch: EmbedBatchFn | null = null; + /** 查询向量 LRU：同文本重复召回省一次 embedding 调用（db 模式不入缓存）。 */ + private queryVecCache = new QueryVecCache(); constructor(private driver: Driver, private cfg: GmConfig) {} - setEmbedFn(fn: EmbedFn): void { this.embed = fn; } + setEmbedFn(fn: EmbedFn): void { + this.embed = fn; + // 端点可能刚从失败恢复/模型切换：旧向量与新端点不可混用 + this.queryVecCache.clear(); + } + + /** 注入批量 embedder（createEmbedder.embedBatch）；运行时批量同步向量用。 */ + setEmbedBatchFn(fn: EmbedBatchFn): void { + this.embedBatch = fn; + this.queryVecCache.clear(); + } /** 是否已接入 embedding（启动 probe 成功或会话级 re-probe 成功）。 */ hasEmbedFn(): boolean { return this.embed !== null; } @@ -99,10 +116,16 @@ export class Recaller { const timeRange = options ? parseTimeRange(options) : null; // query 向量只算一次，两条路径共享；失败统一落 null（各路径走文本兜底）。 + // LRU 命中时零 API 调用（同会话重复查询：before_agent_start ↔ assemble ↔ gm_search）。 // 双路径并行执行 —— 原串行 + 各自 embed 会把 2 次调用放大成 4 次 API 调用 // 与 4 次图遍历，全部压在调用方的预算窗口内。 + const cachedVec = this.queryVecCache.get(query); const embedPromise: Promise = this.embed - ? this.embed(query, "query").catch(() => null) + ? cachedVec + ? Promise.resolve(cachedVec) + : this.embed(query, "query") + .then((vec) => { this.queryVecCache.set(query, vec); return vec; }) + .catch(() => null) : Promise.resolve(null); const [precise, generalized] = await Promise.all([ @@ -264,14 +287,64 @@ export class Recaller { async syncEmbed(node: GmNode): Promise { - if (!this.embed) return; + if (!this.embed && !this.embedBatch) return; const text = buildNodeEmbeddingText(node); const hash = createHash("md5").update(text).digest("hex"); const existingHash = await getVectorHash(this.driver, node.id); if (existingHash === hash) return; try { - const vec = await this.embed(text, "db"); + const vec = this.embed + ? await this.embed(text, "db") + : (await this.embedBatch!([text], "db"))[0]; if (vec.length) await saveVector(this.driver, node.id, text, vec); } catch {} } + + /** + * 批量同步节点向量（embedding 成本控制）：一批提取产出 N 个节点时， + * N 次单发 API + N 次 hash 查询 → 一次批量 embed + 一次 UNWIND 批读写。 + * 与 syncEmbed 相同的 contentHash 短路语义（未变节点零调用）； + * 按 SYNC_EMBED_BATCH 分块（服务端批量上限保护），单块失败 fail-soft 跳过。 + * MiniMax 的 texts+type 批量格式由 embedder 的 embedBatch 内部处理。 + */ + async syncEmbedBatch(nodes: GmNode[]): Promise { + if (!nodes.length) return; + if (!this.embed && !this.embedBatch) return; + + const targets = nodes.map((node) => { + const text = buildNodeEmbeddingText(node); + return { node, text, hash: createHash("md5").update(text).digest("hex") }; + }); + + let existing: Map; + try { + existing = await getVectorHashes(this.driver, targets.map((t) => t.node.id)); + } catch { + return; // hash 批查失败：fail-soft，与单发路径的吞错策略一致 + } + const pending = targets.filter((t) => existing.get(t.node.id) !== t.hash); + if (!pending.length) return; + + if (!this.embedBatch) { + // 无批量能力（仅 setEmbedFn 的旧接线）：退回逐节点单发 + for (const t of pending) { + try { + const vec = await this.embed!(t.text, "db"); + if (vec.length) await saveVector(this.driver, t.node.id, t.text, vec); + } catch {} + } + return; + } + + for (let i = 0; i < pending.length; i += SYNC_EMBED_BATCH) { + const chunk = pending.slice(i, i + SYNC_EMBED_BATCH); + try { + const vecs = await this.embedBatch(chunk.map((t) => t.text), "db"); + if (vecs.length !== chunk.length) continue; // 数量校验失败：parse 层应已抛错，双保险 + await saveVectors(this.driver, chunk.map((t, j) => ({ + nodeId: t.node.id, content: t.text, vec: vecs[j], hash: t.hash, + }))); + } catch {} + } + } } diff --git a/src/store/store.ts b/src/store/store.ts index f4dd5c0..5e4b3bc 100755 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -825,6 +825,52 @@ export async function getVectorHash(driver: Driver, nodeId: string): Promise；不存在的节点映射为 null（视为需要嵌入）。 + */ +export async function getVectorHashes(driver: Driver, nodeIds: string[]): Promise> { + const out = new Map(); + if (!nodeIds.length) return out; + const session = getSession(driver); + try { + const result = await session.run( + `UNWIND $ids AS id + MATCH (n:Task|Skill|Event {id: id}) + RETURN n.id AS id, n.contentHash AS hash`, + { ids: nodeIds }, + ); + for (const r of result.records) out.set(r.get("id"), r.get("hash") ?? null); + } finally { + await session.close(); + } + for (const id of nodeIds) if (!out.has(id)) out.set(id, null); + return out; +} + +export interface BatchVectorEntry { + nodeId: string; + content: string; + vec: number[]; + hash: string; +} + +/** 批量写向量 + contentHash（syncEmbedBatch 用）：一次 UNWIND 写入替代 N 次 saveVector。 */ +export async function saveVectors(driver: Driver, entries: BatchVectorEntry[]): Promise { + if (!entries.length) return; + const session = getSession(driver); + try { + await session.run( + `UNWIND $entries AS e + MATCH (n:Task|Skill|Event {id: e.nodeId}) + SET n.embedding = e.vec, n.contentHash = e.hash`, + { entries: entries.map(e => ({ nodeId: e.nodeId, vec: e.vec, hash: e.hash })) }, + ); + } finally { + await session.close(); + } +} + // ─── 重嵌入（换 embedding 模型后的批量重建，graph-memory reembed） ─── export interface EmbeddingStats { @@ -1196,6 +1242,20 @@ export async function getUnextracted(driver: Driver, sid: string, limit: number) } } +/** 未提取消息计数（batched 提取模式的攒批触发判定）。 */ +export async function countUnextracted(driver: Driver, sid: string): Promise { + const session = getSession(driver); + try { + const result = await session.run( + "MATCH (m:GmMessage {sessionId: $sid, extracted: false}) RETURN count(m) AS c", + { sid }, + ); + return toInt(result.records[0].get("c")); + } finally { + await session.close(); + } +} + export interface UnextractedSessionInfo { sessionId: string; messageCount: number; diff --git a/src/types.ts b/src/types.ts index fc5842b..66f18e5 100755 --- a/src/types.ts +++ b/src/types.ts @@ -264,6 +264,27 @@ export interface MessageRetentionConfig { dryRun?: boolean; } +// ─── 知识提取配置（LLM 成本控制）───────────────────────────── + +export interface ExtractConfig { + /** + * 提取模式： + * - per-turn（默认）：每轮 afterTurn/commitTurn 即时 LLM 提取（知识实时入库）； + * - batched：攒批提取——trivial 轮本地标记跳过，未提取消息累计到 + * compactTurnCount*3 条时批量提取一次，session_end 冲洗尾批。 + * LLM 调用次数降为 per-turn 的 ~1/N，代价是会话中途召回不到本会话最新知识。 + */ + mode?: "per-turn" | "batched"; + /** + * 本地预筛阈值：用户输入清洗（去空白/标点）后长度 ≤ 该值且不含技术词 + * （连续 ≥3 位字母数字，如 pnpm/jwt）时，跳过 LLM 提取直接标记。 + * 保守默认 5（中文 5 字以内基本不可能承载可提取知识）。 + */ + trivialMaxChars?: number; + /** 额外无意义词表（与内置表合并，清洗后小写精确匹配，如 "继续"、"resume"）。 */ + trivialPrompts?: string[]; +} + // ─── 插件配置 ───────────────────────────────────────────────── export interface GmConfig { @@ -293,6 +314,8 @@ export interface GmConfig { pagerankIterations: number; /** 遗忘曲线衰减配置；未提供时使用 DEFAULT_CONFIG.decay。 */ decay?: DecayConfig; + /** 知识提取配置（模式 + trivial 预筛）；未提供时 per-turn + 默认预筛。 */ + extract?: ExtractConfig; cron?: CronConfig; /** 原始消息保留策略；未提供时等价 keep=all（永不删除）。 */ messageRetention?: MessageRetentionConfig; @@ -333,4 +356,8 @@ export const DEFAULT_CONFIG: GmConfig = { purgeAfterDays: 60, }, cron: DEFAULT_CRON_CONFIG, + extract: { + mode: "per-turn", + trivialMaxChars: 5, + }, }; diff --git a/test/ablation.study.test.ts b/test/ablation.study.test.ts new file mode 100644 index 0000000..71d351b --- /dev/null +++ b/test/ablation.study.test.ts @@ -0,0 +1,828 @@ +/** + * graph-memory-pro — 消融实验 (ablation study) + * + * 三组消融，量化各组件对系统质量的贡献： + * A. 召回管线：向量种子 / GDS PPR 排序 / 社区扩展 / 双路径（precise+generalized） + * B. 向量去重：dedup 对图规模收缩与召回多样性的贡献 + * C. 遗忘曲线：decay + autoDeprecate 对陈旧知识的遗忘（含 tier 转换） + * + * 安全门（生产库保护，最高优先级）： + * - 仅当 ABLATION_STUDY=1 时启用；普通 npm test / CI 永远跳过本文件 + * - NEO4J_TEST_URI 必须显式设置，否则整体跳过 —— 绝不回落 7687 默认值 + * + * 运行（一次性 Docker Neo4j，bolt 映射到 7688）： + * ABLATION_STUDY=1 NEO4J_TEST_URI=bolt://localhost:7688 \ + * npx vitest run test/ablation.study.test.ts + * + * 全程使用确定性 mock embedder（bag-of-words 符号哈希 → 1024 维 L2 归一化向量）， + * 不调用任何外部 LLM / embedding API。 + */ + +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import type { Driver } from "neo4j-driver"; +import { createHash } from "crypto"; +import { writeFile, mkdir } from "fs/promises"; +import { fileURLToPath } from "url"; +import path from "node:path"; +import { getDriver, initSchema, closeDriver, getSession } from "../src/store/db.ts"; +import { + upsertNode, upsertEdge, saveVector, saveCommunityEmbedding, + searchNodes, vectorSearchWithScore, graphWalk, + communityRepresentatives, communityVectorSearch, nodesByCommunityIds, +} from "../src/store/store.ts"; +import { + detectCommunities, getCommunityPeers, buildCommunityMemberSignature, +} from "../src/graph/community.ts"; +import { personalizedPageRank, computeGlobalPageRank } from "../src/graph/pagerank.ts"; +import { dedup, detectDuplicates } from "../src/graph/dedup.ts"; +import { applyDecay } from "../src/graph/decay.ts"; +import { Recaller, buildNodeEmbeddingText } from "../src/recaller/recall.ts"; +import { DEFAULT_CONFIG } from "../src/types.ts"; +import type { GmConfig, GmNode, RecallResult, EdgeType, NodeType } from "../src/types.ts"; +import type { EmbedFn } from "../src/engine/embed.ts"; + +// ─── 安全门 ───────────────────────────────────────────────── + +const ENABLED = !!process.env.ABLATION_STUDY && !!process.env.NEO4J_TEST_URI; +const NEO4J_URI = process.env.NEO4J_TEST_URI ?? ""; + +const CFG: GmConfig = { + ...DEFAULT_CONFIG, + neo4j: { uri: NEO4J_URI, user: "neo4j", password: "graphmemory" }, + recallMaxNodes: 6, + recallMaxDepth: 2, +}; +const K = CFG.recallMaxNodes; + +// ─── 确定性 embedder（bag-of-words 符号哈希）───────────────── + +const DIM = 1024; + +function embedText(text: string): number[] { + const vec = new Array(DIM).fill(0); + const words = text.toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length > 1); + for (const w of words) { + const digest = createHash("md5").update(w).digest(); + const code = digest.readUInt32BE(0); + const idx = code % DIM; + const sign = (digest[4] & 1) === 0 ? 1 : -1; + vec[idx] += sign; + } + let norm = 0; + for (const v of vec) norm += v * v; + norm = Math.sqrt(norm) || 1; + return vec.map((v) => v / norm); +} + +const mockEmbedFn: EmbedFn = async (text) => embedText(text); + +function meanVector(texts: string[]): number[] { + const acc = new Array(DIM).fill(0); + for (const t of texts) { + const v = embedText(t); + for (let i = 0; i < DIM; i++) acc[i] += v[i]; + } + let norm = 0; + for (const v of acc) norm += v * v; + norm = Math.sqrt(norm) || 1; + return acc.map((v) => v / norm); +} + +// ─── 指标 ─────────────────────────────────────────────────── + +interface Metrics { + recallAtK: number; + precisionAtK: number; + mrr: number; + nodesReturned: number; +} + +function scoreResult(result: RecallResult, relevant: Set): Metrics { + const retrieved = result.nodes.slice(0, K).map((n) => n.id); + const hits = retrieved.filter((id) => relevant.has(id)); + const firstHit = retrieved.findIndex((id) => relevant.has(id)); + return { + recallAtK: relevant.size ? hits.length / relevant.size : 0, + precisionAtK: retrieved.length ? hits.length / retrieved.length : 0, + mrr: firstHit >= 0 ? 1 / (firstHit + 1) : 0, + nodesReturned: result.nodes.length, + }; +} + +interface Row { + suite: string; + config: string; + recallAtK: string; + precisionAtK: string; + mrr: string; + nodes: string; + latencyMs: string; + note: string; +} + +const ROWS: Row[] = []; + +function addRow(suite: string, config: string, m: Metrics, latencyMs: number, note = ""): void { + ROWS.push({ + suite, + config, + recallAtK: m.recallAtK.toFixed(3), + precisionAtK: m.precisionAtK.toFixed(3), + mrr: m.mrr.toFixed(3), + nodes: `${m.nodesReturned}`, + latencyMs: latencyMs.toFixed(0), + note, + }); +} + +function averageMetrics(list: Metrics[]): Metrics { + const n = list.length || 1; + return { + recallAtK: list.reduce((s, m) => s + m.recallAtK, 0) / n, + precisionAtK: list.reduce((s, m) => s + m.precisionAtK, 0) / n, + mrr: list.reduce((s, m) => s + m.mrr, 0) / n, + nodesReturned: list.reduce((s, m) => s + m.nodesReturned, 0) / n, + }; +} + +// ─── 召回管线 harness（逐行镜像 src/recaller/recall.ts，组件可开关）─── + +interface Flags { + vector: boolean; + communityExpansion: boolean; + ppr: boolean; + precise: boolean; + generalized: boolean; +} + +const FULL: Flags = { + vector: true, communityExpansion: true, ppr: true, precise: true, generalized: true, +}; + +let driver: Driver; + +async function pipelinePrecise( + query: string, limit: number, embedPromise: Promise, flags: Flags, +): Promise { + let seeds: GmNode[] = []; + + const vec = await embedPromise; + if (vec) { + try { + const scored = await vectorSearchWithScore(driver, vec, Math.ceil(limit / 2)); + seeds = scored.map((s) => s.node); + if (seeds.length < 2) { + const fts = await searchNodes(driver, query, limit); + const seen = new Set(seeds.map((n) => n.id)); + seeds.push(...fts.filter((n) => !seen.has(n.id))); + } + } catch { + seeds = await searchNodes(driver, query, limit); + } + } else { + seeds = await searchNodes(driver, query, limit); + } + + if (!seeds.length) return { nodes: [], edges: [] }; + const seedIds = seeds.map((n) => n.id); + + const expandedIds = new Set(seedIds); + if (flags.communityExpansion) { + for (const seed of seeds) { + const peers = await getCommunityPeers(driver, seed.id, 2); + for (const peerId of peers) expandedIds.add(peerId); + } + } + + const { nodes, edges } = await graphWalk(driver, Array.from(expandedIds), CFG.recallMaxDepth); + if (!nodes.length) return { nodes: [], edges: [] }; + + const candidateIds = nodes.map((n) => n.id); + const { scores } = flags.ppr + ? await personalizedPageRank(driver, seedIds, candidateIds, CFG) + : { scores: new Map() }; + + const filtered = nodes + .sort((a, b) => + (scores.get(b.id) || 0) - (scores.get(a.id) || 0) || + b.validatedCount - a.validatedCount || + b.updatedAt - a.updatedAt) + .slice(0, limit); + + const ids = new Set(filtered.map((n) => n.id)); + return { nodes: filtered, edges: edges.filter((e) => ids.has(e.fromId) && ids.has(e.toId)) }; +} + +async function pipelineGeneralized( + limit: number, embedPromise: Promise, +): Promise { + let seeds: GmNode[] = []; + + const vec = await embedPromise; + if (vec) { + try { + const scoredCommunities = await communityVectorSearch(driver, vec); + if (scoredCommunities.length > 0) { + seeds = await nodesByCommunityIds(driver, scoredCommunities.map((c) => c.id), 3); + } + } catch { /* 与 recall.ts 一致：静默落 representatives 兜底 */ } + } + + if (!seeds.length) seeds = await communityRepresentatives(driver, 2); + if (!seeds.length) return { nodes: [], edges: [] }; + + const seedIds = seeds.map((n) => n.id); + const { nodes, edges } = await graphWalk(driver, seedIds, 1); + if (!nodes.length) return { nodes: [], edges: [] }; + + const candidateIds = nodes.map((n) => n.id); + const { scores } = await personalizedPageRank(driver, seedIds, candidateIds, CFG); + + const filtered = nodes + .sort((a, b) => + (scores.get(b.id) || 0) - (scores.get(a.id) || 0) || + b.updatedAt - a.updatedAt || + b.validatedCount - a.validatedCount) + .slice(0, limit); + + const ids = new Set(filtered.map((n) => n.id)); + return { nodes: filtered, edges: edges.filter((e) => ids.has(e.fromId) && ids.has(e.toId)) }; +} + +function mergeResults(a: RecallResult, b: RecallResult): RecallResult { + const nodeMap = new Map(); + const edgeMap = new Map(); + for (const n of a.nodes) nodeMap.set(n.id, n); + for (const e of a.edges) edgeMap.set(e.id, e); + for (const n of b.nodes) if (!nodeMap.has(n.id)) nodeMap.set(n.id, n); + const finalIds = new Set(nodeMap.keys()); + for (const e of b.edges) { + if (!edgeMap.has(e.id) && finalIds.has(e.fromId) && finalIds.has(e.toId)) edgeMap.set(e.id, e); + } + return { nodes: Array.from(nodeMap.values()), edges: Array.from(edgeMap.values()) }; +} + +async function recallPipeline(query: string, flags: Flags): Promise { + const limit = CFG.recallMaxNodes; + const embedPromise: Promise = flags.vector + ? mockEmbedFn(query).catch(() => null) + : Promise.resolve(null); + + const paths: Array> = []; + if (flags.precise) paths.push(pipelinePrecise(query, limit, embedPromise, flags)); + if (flags.generalized) paths.push(pipelineGeneralized(limit, embedPromise)); + + const results = await Promise.all(paths); + let merged: RecallResult = { nodes: [], edges: [] }; + for (const r of results) merged = mergeResults(merged, r); + return merged; +} + +// ─── 图 fixture ───────────────────────────────────────────── + +const RUN = Date.now().toString(36); +const SID_RECALL = `abl-${RUN}`; +const SID_DUP_A = `abl-dupA-${RUN}`; // 去重启用臂 +const SID_DUP_B = `abl-dupB-${RUN}`; // 去重消融臂 +const SID_DECAY = `abl-dec-${RUN}`; +const CLEAN_PREFIX = "abl-"; + +interface FixtureNode { + key: string; + type: NodeType; + name: string; + description: string; + content: string; + topic: string; + validatedCount?: number; +} + +interface FixtureEdge { from: string; to: string; type: EdgeType; instruction: string } + +const CLUSTERS: Array<{ topic: string; query: string; nodes: FixtureNode[] }> = [ + { + topic: "docker-deployment", + query: "deploy docker compose container stack registry", + nodes: [ + { key: "dep-task", type: "TASK", name: "abl-docker-stack-deployment", description: "docker compose container deployment orchestration", content: "deploy the multi-container stack with docker compose, pulling service images from the registry, orchestrating rollout and scaling for production workloads", topic: "docker-deployment", validatedCount: 3 }, + { key: "dep-compose", type: "SKILL", name: "abl-compose-service-orchestration", description: "docker compose service orchestration patterns", content: "compose files declare docker services, networks and volumes; compose up orchestrates container startup order, health gates and rolling image replacement for the deployment stack", topic: "docker-deployment", validatedCount: 7 }, + { key: "dep-registry", type: "SKILL", name: "abl-container-registry-lifecycle", description: "container image registry lifecycle", content: "version and push container images to the registry, scan layers, sign tags, promote build artifacts through staging channels before docker deployment", topic: "docker-deployment", validatedCount: 5 }, + { key: "dep-outage", type: "EVENT", name: "abl-registry-outage-postmortem", description: "registry outage during deployment", content: "the docker deployment stalled when the registry rate limited image pulls; container startup backoff cascaded until the compose stack pinned digest references", topic: "docker-deployment", validatedCount: 1 }, + ], + }, + { + topic: "vitest-testing", + query: "vitest unit test mock spy coverage", + nodes: [ + { key: "test-task", type: "TASK", name: "abl-unit-test-suite-setup", description: "vitest unit test suite setup", content: "scaffold a vitest workspace, configure unit test discovery, assertion styles and coverage collection for the repository test suite", topic: "vitest-testing", validatedCount: 2 }, + { key: "test-mocking", type: "SKILL", name: "abl-vitest-mocking-patterns", description: "vitest mocking and spy patterns", content: "vi.mock hoists module mocks, vi.spyOn wraps methods, mockReturnValue stubs responses, mock timers control asynchronous test flow in vitest", topic: "vitest-testing", validatedCount: 8 }, + { key: "test-coverage", type: "SKILL", name: "abl-coverage-threshold-gates", description: "coverage threshold gates", content: "v8 coverage provider maps uncovered branches, istanbul thresholds fail the test pipeline below eighty percent, coverage exclude lists keep fixtures out of the report", topic: "vitest-testing", validatedCount: 4 }, + { key: "test-flaky", type: "EVENT", name: "abl-flaky-spec-retrospective", description: "flaky spec retrospective", content: "a unit spec flickered until the spy leaked between tests; resetting mocks in beforeEach and stubbing timers made the vitest suite deterministic", topic: "vitest-testing", validatedCount: 1 }, + ], + }, + { + topic: "neo4j-schema", + query: "neo4j cypher index constraint migration", + nodes: [ + { key: "neo-task", type: "TASK", name: "abl-graph-schema-migration", description: "graph schema migration planning", content: "plan the schema migration in ordered cypher batches: create indexes and constraints first, backfill properties, then verify node counts before cutover", topic: "neo4j-schema", validatedCount: 3 }, + { key: "neo-cypher", type: "SKILL", name: "abl-cypher-query-optimization", description: "cypher query optimization", content: "profile query plans, replace cartesian products with anchored matches, hint range index scans, and batch apoc.periodic.iterate updates to keep the database responsive", topic: "neo4j-schema", validatedCount: 6 }, + { key: "neo-schema", type: "SKILL", name: "abl-neo4j-constraint-design", description: "neo4j constraint and index design", content: "uniqueness constraints guard identity keys, range indexes accelerate equality filters, fulltext indexes back keyword search, and constraint violations abort the migration transaction", topic: "neo4j-schema", validatedCount: 5 }, + { key: "neo-incident", type: "EVENT", name: "abl-slow-query-incident", description: "slow query incident review", content: "dashboard latency spiked when a migration dropped a covering index mid-deploy; cypher queries degraded to filters until the constraint and index pair was restored", topic: "neo4j-schema", validatedCount: 1 }, + ], + }, + { + topic: "typescript-build", + query: "typescript tsconfig module compilation strict", + nodes: [ + { key: "ts-task", type: "TASK", name: "abl-tsconfig-modernization", description: "tsconfig modernization initiative", content: "audit every tsconfig in the monorepo, unify module resolution, tighten strict flags, and gate compilation in ci so declaration emit stays consistent", topic: "typescript-build", validatedCount: 2 }, + { key: "ts-strict", type: "SKILL", name: "abl-strict-mode-migration", description: "strict mode migration playbook", content: "noImplicitAny and strictNullChecks surface latent defects; codemods annotate signatures, null guards narrow control flow, and exhaustive switch checks harden the compilation boundary", topic: "typescript-build", validatedCount: 6 }, + { key: "ts-resolution", type: "SKILL", name: "abl-module-resolution-deep-dive", description: "module resolution deep dive", content: "bundler resolution rewrites relative extensions, paths aliases map workspace packages, verbatimModuleSyntax separates types, and the emit target guides output module shape", topic: "typescript-build", validatedCount: 4 }, + { key: "ts-buildbreak", type: "EVENT", name: "abl-ci-build-break-postmortem", description: "ci compilation break postmortem", content: "the pipeline failed when a tsconfig override disabled declaration emit; incremental cache masked it locally until ci compiled from clean and exposed the missing types", topic: "typescript-build", validatedCount: 1 }, + ], + }, +]; + +const DISTRACTORS: FixtureNode[] = [ + { key: "d-pasta", type: "SKILL", name: "abl-pasta-carbonara-notes", description: "italian pasta cooking notes", content: "guanciale rendered crisp, pecorino and egg emulsion, tonnarelli tossed off heat, black pepper finish, a roman classic rehearsed by feel not measure", topic: "misc" }, + { key: "d-japan", type: "SKILL", name: "abl-kyoto-itinerary-draft", description: "japan travel itinerary draft", content: "shinkansen from tokyo to kyoto, early fushimi inari gates, gion at dusk, day trip to nara deer park, temple bookings before cherry blossom peak", topic: "misc" }, + { key: "d-guitar", type: "SKILL", name: "abl-acoustic-chord-drills", description: "acoustic guitar chord drills", content: "fingerpick travis pattern over c g am f shapes, metronome at seventy, barre chord squeeze routine, pentatonic ladder across six strings daily", topic: "misc" }, + { key: "d-gym", type: "SKILL", name: "abl-strength-block-plan", description: "gym strength block plan", content: "five by five back squat progression, paused bench press, romanian deadlift volume, sled finisher, deload every fourth week", topic: "misc" }, + { key: "d-novel", type: "SKILL", name: "abl-novel-outline-v3", description: "novel chapter outline v3", content: "lighthouse keeper hides the ledger, storm chapter mirrors act one betrayal, viewpoint alternates diary and present, climax reframes the inheritance", topic: "misc" }, + { key: "d-tomato", type: "SKILL", name: "abl-tomato-bed-log", description: "tomato garden bed log", content: "amended clay with compost, drip line on timer, pruned suckers weekly, staked indeterminates, mulched before the heatwave", topic: "misc" }, + { key: "d-car", type: "SKILL", name: "abl-hatchback-service-log", description: "hatchback maintenance log", content: "synthetic oil every ten thousand kilometres, tire rotation cross pattern, brake pad thickness measured, cabin filter swapped in spring", topic: "misc" }, + { key: "d-coffee", type: "SKILL", name: "abl-pourover-recipe-card", description: "pour over coffee recipe card", content: "twenty gram dose, three hundred forty milliliters water at ninety three celsius, forty five second bloom, single pour spiral, medium grind settles flat", topic: "misc" }, + { key: "d-photo", type: "SKILL", name: "abl-exposure-triangle-notes", description: "photography exposure notes", content: "aperture controls depth of field, shutter freezes or drags motion, iso lifts shadows with noise cost, meter for highlights and push later", topic: "misc" }, + { key: "d-chess", type: "SKILL", name: "abl-sicilian-defence-prep", description: "chess sicilian defence prep", content: "najdorf move orders, english attack with be3 f3 g4, rook lift to g5, poisoned pawn line memorized to move twenty", topic: "misc" }, + { key: "d-spanish", type: "SKILL", name: "abl-spanish-streak-deck", description: "spanish vocabulary deck", content: "subjunctive triggers on influence verbs, por versus para contrast cards, spaced repetition streak at two hundred days, kitchen nouns themed batch", topic: "misc" }, + { key: "d-budget", type: "SKILL", name: "abl-household-budget-sheet", description: "household budget spreadsheet", content: "fixed costs bucketed, sinking funds for insurance premiums, grocery category averaged quarterly, savings rate plotted against overtime", topic: "misc" }, +]; + +const RECALL_EDGES: FixtureEdge[] = [ + { from: "dep-task", to: "dep-compose", type: "USED_SKILL", instruction: "deploys with" }, + { from: "dep-task", to: "dep-registry", type: "USED_SKILL", instruction: "deploys with" }, + { from: "dep-compose", to: "dep-registry", type: "REQUIRES", instruction: "pulls images through" }, + { from: "dep-outage", to: "dep-registry", type: "SOLVED_BY", instruction: "mitigated by" }, + { from: "test-task", to: "test-mocking", type: "USED_SKILL", instruction: "tests with" }, + { from: "test-task", to: "test-coverage", type: "USED_SKILL", instruction: "tests with" }, + { from: "test-mocking", to: "test-coverage", type: "REQUIRES", instruction: "gated by" }, + { from: "test-flaky", to: "test-mocking", type: "SOLVED_BY", instruction: "mitigated by" }, + { from: "neo-task", to: "neo-cypher", type: "USED_SKILL", instruction: "migrates with" }, + { from: "neo-task", to: "neo-schema", type: "USED_SKILL", instruction: "migrates with" }, + { from: "neo-cypher", to: "neo-schema", type: "REQUIRES", instruction: "gated by" }, + { from: "neo-incident", to: "neo-cypher", type: "SOLVED_BY", instruction: "mitigated by" }, + { from: "ts-task", to: "ts-strict", type: "USED_SKILL", instruction: "migrates with" }, + { from: "ts-task", to: "ts-resolution", type: "USED_SKILL", instruction: "migrates with" }, + { from: "ts-strict", to: "ts-resolution", type: "REQUIRES", instruction: "gated by" }, + { from: "ts-buildbreak", to: "ts-strict", type: "SOLVED_BY", instruction: "mitigated by" }, + // 跨簇弱连接 ×2：检验社区检测/扩展在非完美分簇下的行为 + { from: "test-flaky", to: "ts-resolution", type: "SOLVED_BY", instruction: "mitigated by" }, + { from: "neo-cypher", to: "ts-resolution", type: "REQUIRES", instruction: "gated by" }, +]; + +/** key → 落库后的真实 GmNode */ +const nodeRegistry = new Map(); + +async function seedNodes(list: FixtureNode[], sid: string): Promise { + for (const f of list) { + const { node } = await upsertNode(driver, { + type: f.type, name: f.name, description: f.description, content: f.content, + }, sid); + nodeRegistry.set(f.key, node); + await saveVector(driver, node.id, buildNodeEmbeddingText(node), embedText(buildNodeEmbeddingText(node))); + if (f.validatedCount !== undefined) { + const session = getSession(driver); + try { + await session.run( + "MATCH (n:Task|Skill|Event {id: $id}) SET n.validatedCount = $vc", + { id: node.id, vc: f.validatedCount }, + ); + } finally { await session.close(); } + } + } +} + +async function seedEdges(list: FixtureEdge[], sid: string): Promise { + for (const e of list) { + const from = nodeRegistry.get(e.from); + const to = nodeRegistry.get(e.to); + if (!from || !to) throw new Error(`fixture edge references unknown key: ${e.from} -> ${e.to}`); + await upsertEdge(driver, { + fromId: from.id, toId: to.id, type: e.type, instruction: e.instruction, sessionId: sid, + }); + } +} + +/** 为 detectCommunities 的每个社区按生产形态补建 Community 节点（id/summary/nodeCount/embedding） */ +async function createCommunityNodes(): Promise { + const result = await detectCommunities(driver); + expect(result.count).toBeGreaterThanOrEqual(3); + for (const [cid, memberIds] of result.communities) { + const members = memberIds + .map((id) => Array.from(nodeRegistry.values()).find((n) => n.id === id)) + .filter((n): n is GmNode => !!n); + const topicCounts = new Map(); + for (const id of memberIds) { + const entry = Array.from(CLUSTERS.flatMap((c) => c.nodes)).find((f) => nodeRegistry.get(f.key)?.id === id); + if (entry) topicCounts.set(entry.topic, (topicCounts.get(entry.topic) ?? 0) + 1); + } + const topic = Array.from(topicCounts.entries()).sort((a, b) => b[1] - a[1])[0]?.[0] ?? "misc"; + const session = getSession(driver); + try { + await session.run( + `MERGE (c:Community {id: $cid}) + SET c.summary = $summary, c.nodeCount = $count, c.memberSignature = $sig`, + { cid, summary: `${topic} knowledge cluster`, count: memberIds.length, sig: buildCommunityMemberSignature(memberIds) }, + ); + } finally { await session.close(); } + await saveCommunityEmbedding(driver, cid, meanVector(members.map((n) => buildNodeEmbeddingText(n)))); + } + return result.count; +} + +// ─── 套件 ─────────────────────────────────────────────────── + +describe.skipIf(!ENABLED)("graph-memory-pro ablation study", () => { + + beforeAll(async () => { + console.log(`[ablation] NEO4J_TEST_URI = ${NEO4J_URI}（一次性容器，非生产库）`); + driver = getDriver(CFG.neo4j); + await initSchema(driver); + + // 环境自检：APOC + GDS 必须可用（否则 PPR/社区消融无意义） + const session = getSession(driver); + try { + const r = await session.run("RETURN apoc.version() AS apoc, gds.version() AS gds"); + const rec = r.records[0]; + console.log(`[ablation] APOC ${rec.get("apoc")} / GDS ${rec.get("gds")}`); + expect(rec.get("apoc")).toBeTruthy(); + expect(rec.get("gds")).toBeTruthy(); + } finally { await session.close(); } + + // 召回 cohort：4 主题簇 × 4 节点 + 12 干扰节点 + 簇内边 + 2 跨簇边 + await seedNodes([...CLUSTERS.flatMap((c) => c.nodes), ...DISTRACTORS], SID_RECALL); + await seedEdges(RECALL_EDGES, SID_RECALL); + // 全局 PageRank（searchNodes 排序依赖 n.pagerank） + await computeGlobalPageRank(driver, CFG); + // 社区检测 + Community 节点 + 社区向量（generalized 路径依赖） + const communities = await createCommunityNodes(); + console.log(`[ablation] 召回 cohort 就绪：${CLUSTERS.length * 4 + DISTRACTORS.length} 节点，${communities} 社区`); + }, 300_000); + + afterAll(async () => { + const session = getSession(driver); + try { + await session.run( + "MATCH (n) WHERE any(s IN n.sourceSessions WHERE s STARTS WITH $p) DETACH DELETE n", + { p: CLEAN_PREFIX }, + ); + await session.run("MATCH (c:Community) DETACH DELETE c"); + } finally { await session.close(); } + await closeDriver(); + }, 60_000); + + // ─── A. 召回管线消融 ───────────────────────────────────── + + describe("A. recall pipeline", () => { + // 惰性构造：describe 块体在收集阶段执行（skip 不阻止），此时 beforeAll 尚未跑 + const queries = () => CLUSTERS.map((c) => ({ + query: c.query, + relevant: new Set(c.nodes.map((n) => nodeRegistry.get(n.key)!.id)), + })); + + const ABLATIONS: Array<{ label: string; flags: Flags }> = [ + { label: "full（全部开启）", flags: FULL }, + { label: "-vector（去向量，FTS 兜底）", flags: { ...FULL, vector: false } }, + { label: "-ppr（去 PPR 排序）", flags: { ...FULL, ppr: false } }, + { label: "-communityExpansion（去社区扩展）", flags: { ...FULL, communityExpansion: false } }, + { label: "-generalized（仅精确路径）", flags: { ...FULL, generalized: false } }, + { label: "-precise（仅泛化路径）", flags: { ...FULL, precise: false } }, + ]; + + it("fidelity：harness full 管线与真实 Recaller.recall 输出一致", async () => { + const recaller = new Recaller(driver, CFG); + recaller.setEmbedFn(mockEmbedFn); + for (const { query, relevant } of queries()) { + const real = await recaller.recall(query); + const harness = await recallPipeline(query, FULL); + const realIds = new Set(real.nodes.map((n) => n.id)); + const harnessIds = new Set(harness.nodes.map((n) => n.id)); + let overlap = 0; + for (const id of harnessIds) if (realIds.has(id)) overlap++; + const ratio = harnessIds.size ? overlap / harnessIds.size : 1; + console.log(`[ablation:fidelity] q="${query}" real=${realIds.size} harness=${harnessIds.size} overlap=${ratio.toFixed(2)}`); + // 允许排序型微差，但集合重合度必须极高（harness 忠实性的门槛） + expect(ratio).toBeGreaterThanOrEqual(0.8); + expect(real.nodes.length).toBeGreaterThan(0); + expect(scoreResult(real, relevant).recallAtK).toBeGreaterThan(0); + } + }, 300_000); + + it("消融矩阵：逐组件关闭并测量 recall/precision/MRR/延迟", async () => { + for (const { label, flags } of ABLATIONS) { + const perQuery: Metrics[] = []; + let totalMs = 0; + for (const { query, relevant } of queries()) { + const t0 = performance.now(); + const result = await recallPipeline(query, flags); + totalMs += performance.now() - t0; + perQuery.push(scoreResult(result, relevant)); + } + addRow("A.recall", label, averageMetrics(perQuery), totalMs / queries().length); + } + }, 300_000); + + it("sanity：full 配置召回质量高于零假设", async () => { + // 单独重跑 full，给出可断言的聚合值（也验证矩阵运行后图未被污染） + const { query, relevant } = queries()[0]; + const result = await recallPipeline(query, FULL); + const m = scoreResult(result, relevant); + expect(m.recallAtK).toBeGreaterThanOrEqual(0.5); + addRow("A.recall", "full 复跑（sanity）", m, 0, "q=" + query); + }, 120_000); + }); + + // ─── B. 去重消融 ───────────────────────────────────────── + + describe("B. dedup", () => { + const DUP_CORE = "redis evicts least recently used keys first, ttl expiration sweeps lazy pass, maxmemory policy allkeys lru tunes cache hit rate under pressure"; + const NGINX_CORE = "nginx terminates tls at the edge, proxies upstream pools with least connections, buffers slow clients, and rewrites host headers for the internal mesh"; + const GRPC_CORE = "grpc clients balance across resolved endpoints, round robin picks per call, sticky streams pin to one backend, health checks drain unhealthy targets"; + + const dupNodes: FixtureNode[] = [ + { key: "redis-base", type: "SKILL", name: "abl-redis-cache-eviction", description: "redis cache eviction policy", content: DUP_CORE, topic: "dup", validatedCount: 5 }, + { key: "redis-v1", type: "SKILL", name: "abl-redis-eviction-policy-v2", description: "redis cache eviction policy notes", content: `${DUP_CORE} benchmarked eviction latency`, topic: "dup", validatedCount: 1 }, + { key: "redis-v2", type: "SKILL", name: "abl-redis-eviction-runbook", description: "redis cache eviction runbook", content: `${DUP_CORE} documented runbook fallback`, topic: "dup", validatedCount: 1 }, + { key: "nginx-base", type: "SKILL", name: "abl-nginx-reverse-proxy", description: "nginx reverse proxy configuration", content: NGINX_CORE, topic: "dup", validatedCount: 5 }, + { key: "nginx-v1", type: "SKILL", name: "abl-nginx-proxy-config-v2", description: "nginx reverse proxy notes", content: `${NGINX_CORE} benchmarked proxy latency`, topic: "dup", validatedCount: 1 }, + { key: "nginx-v2", type: "SKILL", name: "abl-nginx-proxy-tuning", description: "nginx reverse proxy tuning", content: `${NGINX_CORE} documented tuning baseline`, topic: "dup", validatedCount: 1 }, + { key: "grpc-base", type: "SKILL", name: "abl-grpc-load-balancing", description: "grpc client load balancing", content: GRPC_CORE, topic: "dup", validatedCount: 5 }, + { key: "grpc-v1", type: "SKILL", name: "abl-grpc-balancing-notes", description: "grpc client balancing notes", content: `${GRPC_CORE} benchmarked picker behavior`, topic: "dup", validatedCount: 1 }, + { key: "kafka-base", type: "SKILL", name: "abl-kafka-consumer-retry", description: "kafka consumer retry semantics", content: "kafka consumers retry poison pills on a dedicated topic, exponential backoff before dead letter, offsets committed after the handler succeeds", topic: "dup", validatedCount: 5 }, + ]; + + // 臂 B 副本：同内容不同名（name 唯一约束下这是产生"重复知识"的唯一方式， + // 等价于跨 session 重复提取到同一知识的场景） + const dupNodesB = dupNodes.map((f) => ({ + ...f, key: `${f.key}-b`, name: `${f.name}-b`, description: f.description, + })); + + const DUP_TOPICS = ["redis", "nginx", "grpc", "kafka"]; + const DIVERSITY_QUERY = "redis cache eviction nginx proxy grpc balancing kafka retry"; + const PRECISE_ONLY: Flags = { ...FULL, generalized: false }; + + /** 主题覆盖度量：top-K 覆盖的不同主题数 / 4 —— 近重复副本只计一次主题 */ + function topicMetrics(result: RecallResult): Metrics { + const retrieved = result.nodes.slice(0, K); + const covered: string[] = []; + for (const n of retrieved) { + const t = DUP_TOPICS.find((topic) => n.name.toLowerCase().includes(topic)); + if (t && !covered.includes(t)) covered.push(t); + } + const firstTopicIdx = retrieved.findIndex((n) => + DUP_TOPICS.some((topic) => n.name.toLowerCase().includes(topic))); + return { + recallAtK: covered.length / DUP_TOPICS.length, + precisionAtK: retrieved.length ? covered.length / retrieved.length : 0, + mrr: firstTopicIdx >= 0 ? 1 / (firstTopicIdx + 1) : 0, + nodesReturned: result.nodes.length, + }; + } + + beforeAll(async () => { + // 只播种臂 A：臂 B 必须在臂 A 去重之后再播种（detectDuplicates 是全局扫描） + await seedNodes(dupNodes, SID_DUP_A); + }, 120_000); + + it("相似度门槛可检出近重复对（fixture 有效性）", async () => { + const pairs = await detectDuplicates(driver, { ...CFG, dedupThreshold: 0.90 }); + const dupPairs = pairs.filter((p) => + p.nameA.startsWith("abl-redis") || p.nameA.startsWith("abl-nginx") || p.nameA.startsWith("abl-grpc")); + console.log(`[ablation:dedup] 检出近重复对 ${dupPairs.length}（全局 ${pairs.length}）：` + + dupPairs.slice(0, 8).map((p) => `${p.nameA}~${p.nameB}@${p.similarity.toFixed(3)}`).join(", ")); + expect(dupPairs.length).toBeGreaterThanOrEqual(6); // redis3 + nginx3 + grpc1 对 + for (const p of dupPairs) expect(p.similarity).toBeGreaterThanOrEqual(0.90); + }, 120_000); + + it("臂 A：dedup 合并近重复 → 召回主题多样性", async () => { + const armA = await dedup(driver, { ...CFG, dedupThreshold: 0.90 }); + console.log(`[ablation:dedup] 臂A 合并 ${armA.merged} 个节点（检出对 ${armA.pairs.length}）`); + expect(armA.merged).toBeGreaterThanOrEqual(5); + + const t0 = performance.now(); + const resA = await recallPipeline(DIVERSITY_QUERY, PRECISE_ONLY); + const msA = performance.now() - t0; + const mA = topicMetrics(resA); + console.log(`[ablation:dedup] 臂A 主题覆盖@${K}=${mA.recallAtK.toFixed(2)}（去重后变体已弃用，FTS 兜底可触达全部主题）`); + addRow("B.dedup", "dedup=on（臂A）", mA, msA, "主题覆盖@K；变体已合并"); + expect(mA.recallAtK).toBeGreaterThan(0.5); + }, 300_000); + + it("臂 B（消融）：同内容重复入库且不 dedup → 召回多样性受损", async () => { + // 臂 B 播种：同内容不同名 → 形成未去重的重复知识群 + await seedNodes(dupNodesB, SID_DUP_B); + + // 自检：redis 主题族当前活跃节点应 ≥ 4（臂A base + 臂B base/v1/v2） + const session = getSession(driver); + let redisActive = 0; + try { + const cnt = await session.run( + `MATCH (n:Skill {status: 'active'}) + WHERE n.name STARTS WITH 'abl-redis' RETURN count(n) AS c`, + ); + redisActive = cnt.records[0].get("c").toNumber(); + } finally { await session.close(); } + console.log(`[ablation:dedup] 臂B 播种后 redis 族活跃节点 = ${redisActive}（应 ≥ 4：未去重副本在场）`); + expect(redisActive).toBeGreaterThanOrEqual(4); + + const t0 = performance.now(); + const resB = await recallPipeline(DIVERSITY_QUERY, PRECISE_ONLY); + const msB = performance.now() - t0; + const mB = topicMetrics(resB); + console.log(`[ablation:dedup] 臂B 主题覆盖@${K}=${mB.recallAtK.toFixed(2)}（向量 top-3 被同主题副本占据 → 其他主题挤不出种子）`); + addRow("B.dedup", "dedup=off（臂B）", mB, msB, "主题覆盖@K；重复副本在场"); + expect(mB.recallAtK).toBeLessThan(1); + }, 300_000); + + it("结论断言：dedup 提升召回主题多样性", async () => { + // 从 ROWS 里取两臂的 recallAtK 对比（避免重复跑管线引入状态依赖） + const rowA = ROWS.find((r) => r.suite === "B.dedup" && r.config.includes("臂A")); + const rowB = ROWS.find((r) => r.suite === "B.dedup" && r.config.includes("臂B")); + expect(rowA).toBeDefined(); + expect(rowB).toBeDefined(); + expect(Number(rowA!.recallAtK)).toBeGreaterThan(Number(rowB!.recallAtK)); + }, 30_000); + }); + + // ─── C. 衰减消融 ───────────────────────────────────────── + + describe("C. decay", () => { + const DAY = 86_400_000; + const STALE_QUERY = "kubernetes helm chart deployment rollout"; + const FRESH_QUERY = "rust tokio async runtime channels"; + + const decayNodes: FixtureNode[] = [ + { key: "k8s-task", type: "TASK", name: "abl-k8s-rollout-automation", description: "kubernetes helm deployment automation", content: "helm charts template the kubernetes deployment, values overlays per cluster, helm upgrade rolls replicas with surge budget, rollback restores previous revision", topic: "stale" }, + { key: "k8s-helm", type: "SKILL", name: "abl-helm-chart-authoring", description: "helm chart authoring guide", content: "helm chart structure bundles templates, values and hooks; chart dependencies compose subcharts; kubeconform validates rendered manifests before the rollout applies", topic: "stale" }, + { key: "k8s-kubectl", type: "SKILL", name: "abl-kubectl-rollout-ops", description: "kubectl rollout operations", content: "kubectl rollout status watches deployment progress, rollout undo reverts the replica set, pod disruption budgets guard node drains during the rollout", topic: "stale" }, + { key: "k8s-event", type: "EVENT", name: "abl-helm-hook-incident", description: "helm hook incident notes", content: "a pre-upgrade helm hook deadlocked when jobs lacked ttl; the deployment rollout stalled until hook deletion policy and backoff limits were fixed", topic: "stale" }, + { key: "rust-task", type: "TASK", name: "abl-async-runtime-selection", description: "rust async runtime selection", content: "tokio drives the worker pool, multithreaded scheduler distributes tasks, io drivers poll sockets, rust futures compose selects and joins across the runtime", topic: "fresh" }, + { key: "rust-tokio", type: "SKILL", name: "abl-tokio-channel-patterns", description: "tokio channel patterns", content: "mpsc pipelines stages, broadcast fans out telemetry, oneshot awaits single replies, watch signals config reload across the async runtime", topic: "fresh" }, + { key: "rust-select", type: "SKILL", name: "abl-rust-select-timeouts", description: "rust select and timeouts", content: "tokio::select races futures, timeouts wrap slow branches, cancellation safety audits each arm, rust async blocks spawn detached from the parent scope", topic: "fresh" }, + { key: "rust-event", type: "EVENT", name: "abl-runtime-deadlock-retro", description: "async deadlock retrospective", content: "a mutex guard held across an await starved the tokio worker; restructuring critical sections and scoped channels unblocked the rust runtime", topic: "fresh" }, + ]; + + const staleRelevant = () => new Set(["k8s-task", "k8s-helm", "k8s-kubectl", "k8s-event"] + .map((k) => nodeRegistry.get(k)!.id)); + const freshRelevant = () => new Set(["rust-task", "rust-tokio", "rust-select", "rust-event"] + .map((k) => nodeRegistry.get(k)!.id)); + + async function setDecayProps(keys: string[], props: Record): Promise { + const session = getSession(driver); + try { + for (const k of keys) { + const node = nodeRegistry.get(k)!; + const assignments = Object.keys(props) + .map((p) => `n.${p} = $${p}`) + .join(", "); + await session.run( + `MATCH (n:Task|Skill|Event {id: $id}) SET ${assignments}`, + { id: node.id, ...props }, + ); + } + } finally { await session.close(); } + } + + beforeAll(async () => { + await seedNodes(decayNodes, SID_DECAY); + await seedEdges([ + { from: "k8s-task", to: "k8s-helm", type: "USED_SKILL", instruction: "rolls out with" }, + { from: "k8s-task", to: "k8s-kubectl", type: "USED_SKILL", instruction: "rolls out with" }, + { from: "k8s-helm", to: "k8s-kubectl", type: "REQUIRES", instruction: "gated by" }, + { from: "k8s-event", to: "k8s-helm", type: "SOLVED_BY", instruction: "mitigated by" }, + { from: "rust-task", to: "rust-tokio", type: "USED_SKILL", instruction: "runs on" }, + { from: "rust-task", to: "rust-select", type: "USED_SKILL", instruction: "runs on" }, + { from: "rust-tokio", to: "rust-select", type: "REQUIRES", instruction: "gated by" }, + { from: "rust-event", to: "rust-tokio", type: "SOLVED_BY", instruction: "mitigated by" }, + ], SID_DECAY); + + const now = Date.now(); + // 陈旧簇：120 天未访问、低频次 → 预期 peripheral + 自动弃用 + await setDecayProps(["k8s-task", "k8s-helm", "k8s-kubectl", "k8s-event"], { + lastAccessedAt: now - 120 * DAY, updatedAt: now - 120 * DAY, + createdAt: now - 200 * DAY, validatedCount: 1, + }); + // 新鲜簇：活跃高频 → 预期保持 working/core + await setDecayProps(["rust-task", "rust-tokio", "rust-select", "rust-event"], { + validatedCount: 8, + }); + + // 手工指派社区 + Community 节点（不重跑全局检测，避免扰动 A 组社区） + for (const [cid, keys, summary] of [ + ["abl-dec-stale", ["k8s-task", "k8s-helm", "k8s-kubectl", "k8s-event"], "kubernetes helm rollout knowledge"], + ["abl-dec-fresh", ["rust-task", "rust-tokio", "rust-select", "rust-event"], "rust tokio async knowledge"], + ] as Array<[string, string[], string]>) { + const session = getSession(driver); + try { + await session.run( + `MERGE (c:Community {id: $cid}) SET c.summary = $summary, c.nodeCount = $count`, + { cid, summary, count: keys.length }, + ); + for (const k of keys) { + await session.run( + "MATCH (n:Task|Skill|Event {id: $id}) SET n.communityId = $cid", + { id: nodeRegistry.get(k)!.id, cid }, + ); + } + } finally { await session.close(); } + await saveCommunityEmbedding(driver, cid, + meanVector(keys.map((k) => buildNodeEmbeddingText(nodeRegistry.get(k)!)))); + } + }, 120_000); + + it("arm0 基线：decay.enabled=false → 无扫描无弃用，陈旧知识仍可召回", async () => { + const r = await applyDecay(driver, { decay: { ...DEFAULT_CONFIG.decay!, enabled: false } }); + expect(r.enabled).toBe(false); + expect(r.autoDeprecated).toBe(0); + + const res = await recallPipeline(STALE_QUERY, FULL); + const m = scoreResult(res, staleRelevant()); + console.log(`[ablation:decay] arm0 基线 recall@${K}=${m.recallAtK.toFixed(2)}`); + expect(m.recallAtK).toBeGreaterThan(0); + addRow("C.decay", "decay=off（基线）", m, 0, STALE_QUERY); + }, 120_000); + + it("arm1 decay 开 + autoDeprecate 关：只降层不断联", async () => { + const r = await applyDecay(driver, { + decay: { ...DEFAULT_CONFIG.decay!, enabled: true, autoDeprecate: false }, + }); + console.log(`[ablation:decay] arm1 scanned=${r.scanned} transitions=${JSON.stringify(r.tierTransitions)} autoDeprecated=${r.autoDeprecated}`); + expect(r.enabled).toBe(true); + expect(r.autoDeprecated).toBe(0); + // 120 天未访问 + validatedCount=1 的陈旧簇应降入 peripheral + expect(r.tierTransitions.workingToPeripheral).toBeGreaterThanOrEqual(4); + + const res = await recallPipeline(STALE_QUERY, FULL); + const m = scoreResult(res, staleRelevant()); + console.log(`[ablation:decay] arm1 陈旧查询 recall@${K}=${m.recallAtK.toFixed(2)}（未弃用，仍可达）`); + expect(m.recallAtK).toBeGreaterThan(0); + addRow("C.decay", "decay=on, autoDeprecate=off", m, 0, "降层但不断联"); + }, 120_000); + + it("arm2 全量 decay+autoDeprecate：陈旧簇被遗忘且新鲜簇不受影响", async () => { + const r = await applyDecay(driver, { + decay: { ...DEFAULT_CONFIG.decay!, enabled: true, autoDeprecate: true }, + }); + console.log(`[ablation:decay] arm2 autoDeprecated=${r.autoDeprecated}`); + expect(r.autoDeprecated).toBeGreaterThanOrEqual(4); + + const session = getSession(driver); + let decayDeprecated = 0; + try { + const cnt = await session.run( + `MATCH (n:Task|Skill|Event) + WHERE any(s IN n.sourceSessions WHERE s STARTS WITH $p) + AND n.status = 'deprecated' AND n.deprecatedBy = 'decay' + RETURN count(n) AS c`, + { p: SID_DECAY }, + ); + decayDeprecated = cnt.records[0].get("c").toNumber(); + } finally { await session.close(); } + expect(decayDeprecated).toBeGreaterThanOrEqual(4); + + const staleRes = await recallPipeline(STALE_QUERY, FULL); + const staleM = scoreResult(staleRes, staleRelevant()); + console.log(`[ablation:decay] arm2 陈旧查询 recall@${K}=${staleM.recallAtK.toFixed(2)}（应为 0：已遗忘）`); + + const freshRes = await recallPipeline(FRESH_QUERY, FULL); + const freshM = scoreResult(freshRes, freshRelevant()); + console.log(`[ablation:decay] arm2 新鲜查询 recall@${K}=${freshM.recallAtK.toFixed(2)}（应不受影响）`); + + addRow("C.decay", "decay+autoDeprecate=on（陈旧查询）", staleM, 0, "应遗忘 → 0"); + addRow("C.decay", "decay+autoDeprecate=on（新鲜查询）", freshM, 0, "对照：不应受损"); + expect(staleM.recallAtK).toBe(0); + expect(freshM.recallAtK).toBeGreaterThan(0); + }, 120_000); + }); + + // ─── D. 汇总报告 ───────────────────────────────────────── + + describe("D. report", () => { + it("输出消融结果表 + JSON 工件", async () => { + console.log("\n========== ABLATION STUDY RESULTS =========="); + console.table(ROWS.map(({ suite, config, recallAtK, precisionAtK, mrr, nodes, latencyMs, note }) => ({ + suite, config, recallAt6: recallAtK, precisionAt6: precisionAtK, mrr, nodes, latencyMs, note, + }))); + + const outPath = fileURLToPath(new URL("../.zcode/ablation-results.json", import.meta.url)); + await mkdir(path.dirname(outPath), { recursive: true }); + await writeFile(outPath, JSON.stringify({ + generatedAt: new Date().toISOString(), + neo4jUri: NEO4J_URI, + k: K, + rows: ROWS, + }, null, 2), "utf-8"); + console.log(`[ablation] JSON 工件已写入 ${outPath}`); + expect(ROWS.length).toBeGreaterThan(0); + }, 30_000); + }); +}); diff --git a/test/cli-extract.test.ts b/test/cli-extract.test.ts index 7ba66db..5922b71 100644 --- a/test/cli-extract.test.ts +++ b/test/cli-extract.test.ts @@ -53,12 +53,15 @@ vi.mock("../src/engine/llm.ts", () => ({ vi.mock("../src/engine/embed.ts", () => ({ createEmbedFn: async () => null, + createEmbedder: async () => null, })); vi.mock("../src/recaller/recall.ts", () => ({ Recaller: class { setEmbedFn(): void {} + setEmbedBatchFn(): void {} async syncEmbed(): Promise {} + async syncEmbedBatch(): Promise {} }, })); diff --git a/test/commit-turn.test.ts b/test/commit-turn.test.ts index d2218aa..5417397 100644 --- a/test/commit-turn.test.ts +++ b/test/commit-turn.test.ts @@ -37,6 +37,7 @@ vi.mock("../src/engine/llm.ts", () => ({ vi.mock("../src/engine/embed.ts", () => ({ createEmbedFn: async () => null, + createEmbedder: async () => null, })); vi.mock("../src/recaller/recall.ts", () => ({ diff --git a/test/extract-cost-guards.test.ts b/test/extract-cost-guards.test.ts new file mode 100644 index 0000000..4f1ad6f --- /dev/null +++ b/test/extract-cost-guards.test.ts @@ -0,0 +1,82 @@ +/** + * finalize 阶梯触发 + 社区摘要 top-k 稳定签名 单测（LLM 成本控制） + */ + +import { describe, it, expect } from "vitest"; +import { shouldRunFinalize } from "../src/extractor/extract.ts"; +import { + buildCommunityMemberSignature, buildTopKMemberSignature, COMMUNITY_SIGNATURE_TOP_K, +} from "../src/graph/community.ts"; + +describe("shouldRunFinalize — finalize 阶梯触发", () => { + it("空会话 / ≤2 节点 → 跳过", () => { + expect(shouldRunFinalize([])).toBe(false); + expect(shouldRunFinalize([{ type: "TASK" }])).toBe(false); + expect(shouldRunFinalize([{ type: "TASK" }, { type: "SKILL" }])).toBe(false); + }); + + it("无 EVENT 节点 → 跳过（promotedSkills 无提升对象）", () => { + expect(shouldRunFinalize([ + { type: "TASK" }, { type: "SKILL" }, { type: "SKILL" }, + ])).toBe(false); + }); + + it("≥3 节点且含 EVENT → 触发", () => { + expect(shouldRunFinalize([ + { type: "TASK" }, { type: "SKILL" }, { type: "EVENT" }, + ])).toBe(true); + }); +}); + +describe("buildTopKMemberSignature — top-k 稳定签名", () => { + const members = [ + { id: "a", validatedCount: 10 }, + { id: "b", validatedCount: 8 }, + { id: "c", validatedCount: 7 }, + { id: "d", validatedCount: 2 }, // 低频成员 + { id: "e", validatedCount: 1 }, // 低频成员 + ]; + + it("低频成员进出不改变签名（社区边界抖动免疫）", () => { + // k=3：签名由 validatedCount 前三名（a/b/c）决定 + const base = buildTopKMemberSignature(members, 3); + const withNoise = buildTopKMemberSignature([ + ...members, + { id: "f", validatedCount: 1 }, + ], 3); + const withoutLow = buildTopKMemberSignature(members.slice(0, 3), 3); + expect(withNoise).toBe(base); + expect(withoutLow).toBe(base); + }); + + it("高价值成员变化 → 签名变化（语义主体变了要重摘要）", () => { + const base = buildTopKMemberSignature(members, 3); + const promoted = buildTopKMemberSignature([ + ...members, + { id: "g", validatedCount: 50 }, + ], 3); + expect(promoted).not.toBe(base); + }); + + it("成员数 ≤ k 时与全量签名一致（退化为旧语义）", () => { + const small = [ + { id: "x", validatedCount: 3 }, + { id: "y", validatedCount: 5 }, + ]; + const topk = buildTopKMemberSignature(small, COMMUNITY_SIGNATURE_TOP_K); + const full = buildCommunityMemberSignature(small.map((m) => m.id)); + expect(topk).toBe(full); + }); + + it("validatedCount 并列时按 id 字典序决胜负（确定性）", () => { + const s1 = buildTopKMemberSignature([ + { id: "a", validatedCount: 5 }, + { id: "b", validatedCount: 5 }, + ]); + const s2 = buildTopKMemberSignature([ + { id: "b", validatedCount: 5 }, + { id: "a", validatedCount: 5 }, + ]); + expect(s1).toBe(s2); + }); +}); diff --git a/test/integration.embed-e2e.test.ts b/test/integration.embed-e2e.test.ts new file mode 100644 index 0000000..643f6ca --- /dev/null +++ b/test/integration.embed-e2e.test.ts @@ -0,0 +1,147 @@ +/** + * 真实 embedding 服务端到端测试（syncEmbedBatch / 批量 API / 查询 LRU / 语义召回） + * + * 与其他集成测试的 mock 向量不同，本文件打真实 embedding 端点（默认 + * http://localhost:8000/v1，jina-embeddings-v5-text-small，1024 维）， + * 验证：探活、批量嵌入、contentHash 短路、查询向量 LRU、真实语义召回排序。 + * + * 安全门：需要显式 EMBED_E2E=1 + NEO4J_INTEGRATION=1 + NEO4J_TEST_URI（不回落 7687）。 + * + * 运行： + * EMBED_E2E=1 NEO4J_INTEGRATION=1 NEO4J_TEST_URI=bolt://localhost:7688 \ + * npx vitest run test/integration.embed-e2e.test.ts + */ + +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import type { Driver } from "neo4j-driver"; +import { getDriver, initSchema, closeDriver, getSession } from "../src/store/db.ts"; +import { upsertNode } from "../src/store/store.ts"; +import { createEmbedder } from "../src/engine/embed.ts"; +import { Recaller } from "../src/recaller/recall.ts"; +import { DEFAULT_CONFIG, type GmConfig } from "../src/types.ts"; + +const ENABLED = !!process.env.EMBED_E2E && !!process.env.NEO4J_INTEGRATION && !!process.env.NEO4J_TEST_URI; +const NEO4J_URI = process.env.NEO4J_TEST_URI ?? ""; +const EMBED_BASE = process.env.EMBED_E2E_BASE ?? "http://localhost:8000/v1"; +const EMBED_MODEL = process.env.EMBED_E2E_MODEL ?? "jina-embeddings-v5-text-small"; + +let driver: Driver; +const SID = `embed-e2e-${Date.now()}`; +const cfg: GmConfig = { ...DEFAULT_CONFIG, neo4j: { uri: NEO4J_URI, user: "neo4j", password: "graphmemory" } }; + +describe.skipIf(!ENABLED)("embedding E2E（真实服务）", () => { + let embedCalls = 0; + let batchCalls = 0; + let batchTexts = 0; + + beforeAll(async () => { + console.log(`[embed-e2e] neo4j=${NEO4J_URI} embed=${EMBED_BASE} model=${EMBED_MODEL}`); + driver = getDriver(cfg.neo4j); + await initSchema(driver, { baseURL: EMBED_BASE, model: EMBED_MODEL }); + }, 120_000); + + afterAll(async () => { + const session = getSession(driver); + try { + await session.run("MATCH (n) WHERE $sid IN n.sourceSessions DETACH DELETE n", { sid: SID }); + } finally { + await session.close(); + } + await closeDriver(); + }, 60_000); + + it("探活：createEmbedder probe 成功，双能力（embed/embedBatch）可用", async () => { + const embedder = await createEmbedder({ baseURL: EMBED_BASE, model: EMBED_MODEL }); + expect(embedder).not.toBeNull(); + const vec = await embedder!.embed("probe 维度", "query"); + expect(vec.length).toBeGreaterThan(0); + console.log(`[embed-e2e] 单发维度 = ${vec.length}`); + }, 60_000); + + it("syncEmbedBatch：真实批量 API 一次往返写回 N 个节点向量 + contentHash 短路", async () => { + const embedder = await createEmbedder({ baseURL: EMBED_BASE, model: EMBED_MODEL }); + expect(embedder).not.toBeNull(); + + // 包一层计数器，验证 LRU / 短路语义 + const countedEmbed = embedder!.embed; + const countedBatch = embedder!.embedBatch; + + const recaller = new Recaller(driver, cfg); + recaller.setEmbedFn(async (text, mode) => { + embedCalls++; + return countedEmbed(text, mode); + }); + recaller.setEmbedBatchFn(async (texts, mode) => { + batchCalls++; + batchTexts += texts.length; + return countedBatch(texts, mode); + }); + + const topics = [ + { name: "embed-e2e-docker-compose", description: "docker compose container deployment", content: "deploy multi-container applications with docker compose files and registry images" }, + { name: "embed-e2e-vitest-mocking", description: "vitest mock and spy patterns", content: "vi.mock hoists module mocks, vi.spyOn wraps methods, stub network responses in unit tests" }, + { name: "embed-e2e-neo4j-index", description: "neo4j index and constraint design", content: "uniqueness constraints guard identity keys, range indexes accelerate equality cypher filters" }, + ]; + const nodes = []; + for (const t of topics) { + const { node } = await upsertNode(driver, { type: "SKILL", ...t }, SID); + nodes.push(node); + } + + // 首次批量同步：一次 batch API 调用覆盖 3 个节点 + await recaller.syncEmbedBatch(nodes); + expect(batchCalls).toBe(1); + expect(batchTexts).toBe(3); + + // 落库校验：embedding 维度 + contentHash + const session = getSession(driver); + try { + const r = await session.run( + `MATCH (n:Skill) WHERE $sid IN n.sourceSessions + RETURN n.id AS id, size(n.embedding) AS dims, n.contentHash AS hash`, + { sid: SID }, + ); + expect(r.records.length).toBe(3); + for (const rec of r.records) { + // size() 返回 Neo4j Integer + expect((rec.get("dims")?.toNumber?.() ?? rec.get("dims"))).toBe(1024); + expect(rec.get("hash")).toBeTruthy(); + } + } finally { + await session.close(); + } + + // contentHash 短路：内容未变 → 零新 API 调用 + await recaller.syncEmbedBatch(nodes); + expect(batchCalls).toBe(1); + expect(batchTexts).toBe(3); + }, 120_000); + + it("真实语义召回 + 查询向量 LRU", async () => { + const embedder = await createEmbedder({ baseURL: EMBED_BASE, model: EMBED_MODEL }); + const countedEmbed = embedder!.embed; + let queryCalls = 0; + + const recaller = new Recaller(driver, cfg); + recaller.setEmbedFn(async (text, mode) => { + queryCalls++; + return countedEmbed(text, mode); + }); + + // 语义查询（非字面词重叠：orchestration ≠ compose/deployment） + const r1 = await recaller.recall("how to orchestrate multi container apps"); + expect(r1.nodes.length).toBeGreaterThan(0); + const firstCallCount = queryCalls; + expect(firstCallCount).toBe(1); + console.log(`[embed-e2e] 语义召回 top1 = ${r1.nodes[0]?.name}`); + + // 相同查询：LRU 命中，零新 embedding 调用 + const r2 = await recaller.recall("how to orchestrate multi container apps"); + expect(queryCalls).toBe(firstCallCount); + expect(r2.nodes.map((n) => n.id).sort()).toEqual(r1.nodes.map((n) => n.id).sort()); + + // 不同查询：新 embedding 调用 + await recaller.recall("unit testing with mocks"); + expect(queryCalls).toBe(firstCallCount + 1); + }, 120_000); +}); diff --git a/test/integration.recall.test.ts b/test/integration.recall.test.ts index 4eb1e0f..e5c15f6 100644 --- a/test/integration.recall.test.ts +++ b/test/integration.recall.test.ts @@ -174,4 +174,63 @@ describe.skipIf(!ENABLED)("Recaller integration", () => { await recaller.syncEmbed({ ...node, description: "new description" }); expect(embeddedText).toContain("new description"); }); + + it("syncEmbedBatch：批量一次往返 + contentHash 短路 + 分块 + 单发回退", async () => { + // 40 个节点 > SYNC_EMBED_BATCH(32) → 应拆 2 块 + const nodes = []; + for (let i = 0; i < 40; i++) { + const { node } = await upsertNode(driver, { + type: "SKILL", name: `syncembed-batch-target-${i}`, + description: `batch ${i}`, content: `content ${i}`, + }, TEST_SID); + nodes.push(node); + } + + let batchCalls = 0; + let singleCalls = 0; + let totalTexts = 0; + const recaller = new Recaller(driver, cfg); + recaller.setEmbedFn(async () => { + singleCalls++; + return new Array(1024).fill(0.1); + }); + recaller.setEmbedBatchFn(async (texts) => { + batchCalls++; + totalTexts += texts.length; + return texts.map(() => new Array(1024).fill(0.2)); + }); + + await recaller.syncEmbedBatch(nodes); + expect(batchCalls).toBe(2); // 32 + 8 两块 + expect(totalTexts).toBe(40); + expect(singleCalls).toBe(0); // 有批量能力时不走单发 + + // contentHash 短路：内容未变 → 零新调用 + await recaller.syncEmbedBatch(nodes); + expect(batchCalls).toBe(2); + expect(totalTexts).toBe(40); + + // 单节点内容变化 → 只有该节点重嵌入（1 块 1 条文本） + await recaller.syncEmbedBatch([{ ...nodes[0], content: "changed content after batch" }]); + expect(batchCalls).toBe(3); + expect(totalTexts).toBe(41); + + // 旧接线（仅 setEmbedFn）回退到逐节点单发 + const recaller2 = new Recaller(driver, cfg); + let single2 = 0; + recaller2.setEmbedFn(async () => { + single2++; + return new Array(1024).fill(0.3); + }); + const { node: fb } = await upsertNode(driver, { + type: "SKILL", name: "syncembed-batch-fallback", + description: "f", content: "fallback content", + }, TEST_SID); + await recaller2.syncEmbedBatch([fb]); + expect(single2).toBe(1); + + // 空输入 no-op + await recaller.syncEmbedBatch([]); + expect(batchCalls).toBe(3); + }); }); diff --git a/test/query-cache.test.ts b/test/query-cache.test.ts new file mode 100644 index 0000000..f9424e9 --- /dev/null +++ b/test/query-cache.test.ts @@ -0,0 +1,53 @@ +/** + * 查询向量 LRU 单测 — 低成本调用节约（重复查询零 embedding API 调用） + */ + +import { describe, it, expect } from "vitest"; +import { QueryVecCache } from "../src/recaller/query-cache.ts"; + +describe("QueryVecCache", () => { + it("基础 get/set/miss", () => { + const cache = new QueryVecCache(4); + expect(cache.get("q1")).toBeUndefined(); + cache.set("q1", [1, 2, 3]); + expect(cache.get("q1")).toEqual([1, 2, 3]); + }); + + it("容量满时淘汰最旧条目", () => { + const cache = new QueryVecCache(2); + cache.set("a", [1]); + cache.set("b", [2]); + cache.set("c", [3]); // 淘汰 a + expect(cache.get("a")).toBeUndefined(); + expect(cache.get("b")).toEqual([2]); + expect(cache.get("c")).toEqual([3]); + expect(cache.size).toBe(2); + }); + + it("命中会刷新 LRU 顺序（最近使用的不会被淘汰）", () => { + const cache = new QueryVecCache(2); + cache.set("a", [1]); + cache.set("b", [2]); + cache.get("a"); // a 变为最新 + cache.set("c", [3]); // 淘汰 b 而非 a + expect(cache.get("a")).toEqual([1]); + expect(cache.get("b")).toBeUndefined(); + }); + + it("重复 set 同 key 不占额外槽位", () => { + const cache = new QueryVecCache(2); + cache.set("a", [1]); + cache.set("a", [9]); + cache.set("b", [2]); + expect(cache.size).toBe(2); + expect(cache.get("a")).toEqual([9]); + }); + + it("clear 清空全部（embedding 端点切换时）", () => { + const cache = new QueryVecCache(4); + cache.set("a", [1]); + cache.clear(); + expect(cache.get("a")).toBeUndefined(); + expect(cache.size).toBe(0); + }); +}); diff --git a/test/register-guard.test.ts b/test/register-guard.test.ts index b05e684..f7a3f47 100644 --- a/test/register-guard.test.ts +++ b/test/register-guard.test.ts @@ -43,6 +43,7 @@ vi.mock("../src/engine/llm.ts", async (importActual) => { vi.mock("../src/engine/embed.ts", () => ({ createEmbedFn: async () => null, + createEmbedder: async () => null, })); vi.mock("../src/recaller/recall.ts", () => ({ diff --git a/test/session-identity.test.ts b/test/session-identity.test.ts index 36acde4..572ca93 100644 --- a/test/session-identity.test.ts +++ b/test/session-identity.test.ts @@ -55,6 +55,7 @@ vi.mock("../src/engine/llm.ts", () => ({ vi.mock("../src/engine/embed.ts", () => ({ createEmbedFn: async () => null, + createEmbedder: async () => null, })); vi.mock("../src/recaller/recall.ts", () => ({ diff --git a/test/turn-filter.test.ts b/test/turn-filter.test.ts new file mode 100644 index 0000000..0f037ba --- /dev/null +++ b/test/turn-filter.test.ts @@ -0,0 +1,102 @@ +/** + * trivial 轮次本地预筛单测 — LLM 成本控制的第一道闸 + * + * 判定语义（保守取向）： + * 1. 清洗后为空 → 跳过 + * 2. 命中无意义词表（内置 + extra，含标点/大小写容忍）→ 跳过 + * 3. 清洗后 ≤ maxChars（默认 5）且无技术词（连续 ≥3 位字母数字）→ 跳过 + * 4. 其余（正常提问 / 短但含技术词 / 长文本）→ 不跳过 + */ + +import { describe, it, expect } from "vitest"; +import { + normalizeTrivialText, shouldSkipTurnExtraction, BUILTIN_TRIVIAL_PROMPTS, +} from "../src/extractor/turn-filter.ts"; +import { turnUserText } from "../index.ts"; + +describe("normalizeTrivialText", () => { + it("去空白与中西文标点、转小写", () => { + expect(normalizeTrivialText(" 继续。！！ ")).toBe("继续"); + expect(normalizeTrivialText("OK!")).toBe("ok"); + expect(normalizeTrivialText("Resume, please")).toBe("resumeplease"); + }); +}); + +describe("shouldSkipTurnExtraction — 词表命中", () => { + it.each([ + "继续", "请继续", "继续。", "继续吧", "好的！", "ok", "OK", "resume", + "continue", "谢谢", "thanks", "got it", "知道了", + ])("%s → 跳过", (input) => { + expect(shouldSkipTurnExtraction(input)).toBe(true); + }); + + it("额外词表生效且同样容忍标点", () => { + expect(shouldSkipTurnExtraction("下一页。", { extraPrompts: ["下一页"] })).toBe(true); + expect(shouldSkipTurnExtraction("下一页", { extraPrompts: ["下一页"] })).toBe(true); + }); + + it("内置表可被 extra 扩展且不被覆盖", () => { + expect(BUILTIN_TRIVIAL_PROMPTS).toContain("继续"); + expect(shouldSkipTurnExtraction("继续")).toBe(true); + // 不在词表且超长阈值的正常词 → 不跳过 + expect(shouldSkipTurnExtraction("下一页部署 nginx")).toBe(false); + }); +}); + +describe("shouldSkipTurnExtraction — 短文本阈值", () => { + it("≤5 字纯文本（无技术词）→ 跳过", () => { + expect(shouldSkipTurnExtraction("是的呢")).toBe(true); + expect(shouldSkipTurnExtraction("嗯嗯嗯嗯嗯")).toBe(true); // 恰好 5 + expect(shouldSkipTurnExtraction("嗯嗯嗯嗯嗯嗯")).toBe(false); // 6 字，宁可多提取 + }); + + it("短但含技术词（连续 ≥3 位字母数字）→ 不跳过", () => { + expect(shouldSkipTurnExtraction("用pnpm")).toBe(false); + expect(shouldSkipTurnExtraction("试jwt")).toBe(false); + expect(shouldSkipTurnExtraction("k8s呢")).toBe(false); + }); + + it("空输入 → 跳过（无可提取内容）", () => { + expect(shouldSkipTurnExtraction("")).toBe(true); + expect(shouldSkipTurnExtraction(" 。！？ ")).toBe(true); + }); + + it("正常提问不跳过", () => { + expect(shouldSkipTurnExtraction("帮我把 neo4j 的索引重建一下")).toBe(false); + expect(shouldSkipTurnExtraction("这个报错怎么修")).toBe(false); + expect(shouldSkipTurnExtraction("why does the build fail")).toBe(false); + }); + + it("自定义阈值", () => { + expect(shouldSkipTurnExtraction("嗯嗯嗯嗯嗯嗯", { maxChars: 6 })).toBe(true); + expect(shouldSkipTurnExtraction("嗯嗯嗯", { maxChars: 2 })).toBe(false); + }); +}); + +describe("turnUserText — 轮级 user 文本聚合", () => { + it("只取 user 角色，剥离 OpenClaw 元数据", () => { + const messages = [ + { + role: "user", + content: [ + { type: "text", text: "```json\n{\"Sender\":\"meta\"}\n```\n继续" }, + ], + }, + { role: "assistant", content: [{ type: "text", text: "好的，我继续处理部署任务" }] }, + { role: "user", content: [{ type: "text", text: "" }] }, + ]; + const text = turnUserText(messages); + expect(text.trim()).toBe("继续"); + // 元数据剥离后应命中词表 → 该轮会被本地预筛跳过 + expect(shouldSkipTurnExtraction(text)).toBe(true); + }); + + it("string content 同样支持", () => { + expect(turnUserText([{ role: "user", content: "怎么修这个报错" }])).toBe("怎么修这个报错"); + }); + + it("非 user 消息不参与", () => { + expect(turnUserText([{ role: "assistant", content: "继续" }])).toBe(""); + expect(shouldSkipTurnExtraction(turnUserText([{ role: "assistant", content: "继续" }]))).toBe(true); + }); +}); From 5d2bc334a385186751119b76ae5dc19817b72ce4 Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:54:47 +0000 Subject: [PATCH 27/29] chore: Performed code quality review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 代码审查发现(专项扫描) 确认重复 13 处、疑似重叠 11 处，同时有 4 个既有约定(t tokens/http/normalizeName/uid)验证干净。 已实施的精修(12 文件，-374/+247 行) 修复真实缺陷:exchangeAuthorizationCode(OAuth 登录)原是裸 fetch 无超时，CLI 登录有卡死风险；现与 refreshOAuthSession 共用新提取的 postTokenRequest,统一走 http.ts 的 fetchWithTimeout,错误消息格式保持不变。 消灭最高危的三份复制：提取结果持久化体(upsertNode → syncEmbedBatch → upsertEdge)在 index.ts×2 + cli-extract.ts×1 逐字重复且已漂移，收敛为新的 src/extractor/persist.ts(awaitEmbedSync 开关保留运行时 fire-and-forget / CLI 必须等待的语义差异)。 边类型白名单 8 处 → 1 处：store.ts 5 条 Cypher、projection.ts、crud.ts、index.ts 的 TypeBox union 全部改为从 EDGE_TYPES 派生——以后加新边类型只改 types.ts 一处。 store.ts 边行投影 5 份 → EDGE_ROW_RETURN + mapEdgeRecords;REST 删边下沉为 deleteEdgeById/deleteEdges,路由层不再写 Cypher(响应新增 deleted 计数，additive)。 oauth/llm 错误样板 7 份 → throwForStatus(http.ts 新导出，消息格式逐字保留)；llm.ts 内联的 Responses output_text 遍历改为复用 oauth.ts 导出的解析函数(多段文本从无分隔拼接统一为 "\n" 连接)。 其他：pagerank.ts 三份 top-20 读取收敛为 readTopKByPagerank;index.ts 两处内联 top-pagerank Cypher 改用 store.topNodes;saveVector 接受预计算 hash 消灭 md5 双算；修复 commit-turn 测试 mock 里的幽灵字段；给 recall 双路径相反的 tiebreak 顺序补了说明注释(有意差异，消融 harness 镜像了它)。 --- index.ts | 96 +++++-------------------- src/cli-extract.ts | 46 +++--------- src/engine/embed.ts | 8 +-- src/engine/http.ts | 9 +++ src/engine/llm.ts | 27 ++----- src/engine/oauth.ts | 151 ++++++++++++++++++++------------------- src/extractor/persist.ts | 80 +++++++++++++++++++++ src/graph/pagerank.ts | 72 ++++++++----------- src/graph/projection.ts | 10 +-- src/recaller/recall.ts | 9 ++- src/routes/crud.ts | 50 +++++-------- src/store/store.ts | 141 +++++++++++++++++------------------- test/commit-turn.test.ts | 2 +- 13 files changed, 327 insertions(+), 374 deletions(-) create mode 100644 src/extractor/persist.ts diff --git a/index.ts b/index.ts index 33c04b3..00d961b 100755 --- a/index.ts +++ b/index.ts @@ -7,14 +7,14 @@ */ import type { OpenClawPluginApi } from "openclaw/plugin-sdk"; import { Type } from "@sinclair/typebox"; -import { getDriver, initSchema, getSession } from "./src/store/db.ts"; +import { getDriver, initSchema } from "./src/store/db.ts"; import { Neo4jGate } from "./src/store/gate.ts"; import { saveMessage, getUnextracted, countUnextracted, getMaxTurnIndex, markExtracted, isTurnExtracted, commitTurnAdvance, upsertNode, upsertEdge, findByName, updateNode, deprecateNodeAndDisconnect, deprecateNodeAndDisconnectById, - getBySession, edgesTouching, + getBySession, edgesTouching, topNodes, deleteEdges, mergeNodes, getStats, } from "./src/store/store.ts"; @@ -24,11 +24,12 @@ import { estimateTokens } from "./src/tokens.ts"; import { Recaller, parseTimeRange } from "./src/recaller/recall.ts"; import { Extractor, shouldRunFinalize } from "./src/extractor/extract.ts"; import { shouldSkipTurnExtraction } from "./src/extractor/turn-filter.ts"; +import { persistExtractionResult } from "./src/extractor/persist.ts"; import { assembleContext } from "./src/format/assemble.ts"; import { sanitizeToolUseResultPairing } from "./src/format/transcript-repair.ts"; import { runMaintenance } from "./src/graph/maintenance.ts"; import { normalizeMessageRetentionPolicy } from "./src/store/retention.ts"; -import { DEFAULT_CONFIG, DEFAULT_CRON_CONFIG, isCronSessionKey, type GmConfig, type GmNode, type RecallResult, type EdgeType } from "./src/types.ts"; +import { DEFAULT_CONFIG, DEFAULT_CRON_CONFIG, isCronSessionKey, EDGE_TYPES, type GmConfig, type RecallResult, type EdgeType } from "./src/types.ts"; import { registerCrudRoutes } from "./src/routes/crud.ts"; import { createGraphMemoryCli } from "./src/cli.ts"; @@ -523,31 +524,8 @@ const graphMemoryProPlugin = { return; } - const nameToId = new Map(); - const upsertedNodes: GmNode[] = []; - for (const nc of result.nodes) { - const { node } = await upsertNode(driver, { - type: nc.type, name: nc.name, - description: nc.description, content: nc.content, - }, sessionId); - nameToId.set(node.name, node.id); - upsertedNodes.push(node); - } - // 批量向量同步：N 个节点一次 embedBatch + 一次 UNWIND 批读写（替代逐节点 N 次单发） - void recaller.syncEmbedBatch(upsertedNodes).catch(() => {}); - - for (const ec of result.edges) { - const fromNode = await findByName(driver, ec.from); - const toNode = await findByName(driver, ec.to); - const fromId = nameToId.get(ec.from) ?? fromNode?.id; - const toId = nameToId.get(ec.to) ?? toNode?.id; - if (fromId && toId) { - await upsertEdge(driver, { - fromId, toId, type: ec.type, - instruction: ec.instruction, condition: ec.condition, sessionId, - }); - } - } + // upsert 节点 + 批量向量同步（fire-and-forget）+ 建边 —— 单一来源见 persist.ts + await persistExtractionResult(driver, recaller, result, { sessionId }); // 标记该轮消息已提取 await markExtracted(driver, sessionId, turnNum); @@ -578,30 +556,8 @@ const graphMemoryProPlugin = { const existing = (await getBySession(driver, sessionId)).map(n => n.name); const result = await extractor.extract({ messages: msgs, existingNames: existing }); - const nameToId = new Map(); - const upsertedNodes: GmNode[] = []; - for (const nc of result.nodes) { - const { node } = await upsertNode(driver, { - type: nc.type, name: nc.name, - description: nc.description, content: nc.content, - }, sessionId); - nameToId.set(node.name, node.id); - upsertedNodes.push(node); - } - void recaller.syncEmbedBatch(upsertedNodes).catch(() => {}); - - for (const ec of result.edges) { - const fromNode = await findByName(driver, ec.from); - const toNode = await findByName(driver, ec.to); - const fromId = nameToId.get(ec.from) ?? fromNode?.id; - const toId = nameToId.get(ec.to) ?? toNode?.id; - if (fromId && toId) { - await upsertEdge(driver, { - fromId, toId, type: ec.type, - instruction: ec.instruction, condition: ec.condition, sessionId, - }); - } - } + // upsert 节点 + 批量向量同步（fire-and-forget）+ 建边 —— 单一来源见 persist.ts + await persistExtractionResult(driver, recaller, result, { sessionId }); const maxTurn = Math.max(...msgs.map((m: any) => m.turn_index)); await markExtracted( @@ -1264,21 +1220,10 @@ const graphMemoryProPlugin = { // finalize 的 upsert 与 afterTurn/compact 的提取共用 per-session 互斥锁： // 最后一轮的 afterTurn 提取可能仍在途，不串行化会重复 upsert（validatedCount 双递增） await withExtractLock(sid, async () => { - // 获取图谱摘要 - const session = getSession(driver); - let summary = ""; - try { - const summaryResult = await session.run(` - MATCH (n:Task|Skill|Event {status: 'active'}) - RETURN n.name AS name, n.type AS type, n.validatedCount AS vc, n.pagerank AS pr - ORDER BY n.pagerank DESC LIMIT 20 - `); - summary = summaryResult.records - .map(r => `${r.get("type")}:${r.get("name")}(v${r.get("vc")},pr${(r.get("pr") ?? 0).toFixed?.(3) ?? "0"})`) - .join(", "); - } finally { - await session.close(); - } + // 图谱摘要：top-pagerank 节点走 store.topNodes（勿在路由/钩子里内联同义 Cypher） + const summary = (await topNodes(driver, 20)) + .map(n => `${n.type}:${n.name}(v${n.validatedCount},pr${n.pagerank.toFixed(3)})`) + .join(", "); const fin = await extractor.finalize({ sessionNodes: nodes, graphSummary: summary }); @@ -1532,11 +1477,9 @@ const graphMemoryProPlugin = { { name: "gm_update" }, ); - const EDGE_TYPE_LITERAL = (label: string) => Type.Literal(label); + // 边类型 union 从 EDGE_TYPES 派生（事实源 types.ts）—— 加新边类型只改一处 const edgeTypeUnion = (description: string) => Type.Union( - [EDGE_TYPE_LITERAL("USED_SKILL"), EDGE_TYPE_LITERAL("SOLVED_BY"), - EDGE_TYPE_LITERAL("REQUIRES"), EDGE_TYPE_LITERAL("PATCHES"), - EDGE_TYPE_LITERAL("CONFLICTS_WITH")], + EDGE_TYPES.map(t => Type.Literal(t)), { description }, ); @@ -1703,21 +1646,14 @@ const graphMemoryProPlugin = { parameters: Type.Object({}), async execute() { const stats = await getStats(driver); - const session = getSession(driver); - let topPr: any[] = []; - try { - const r = await session.run("MATCH (n:Task|Skill|Event {status:'active'}) RETURN n.name AS name, n.type AS type, n.pagerank AS pr ORDER BY n.pagerank DESC LIMIT 5"); - topPr = r.records.map(rec => ({ name: rec.get("name"), type: rec.get("type"), pr: rec.get("pr") ?? 0 })); - } finally { - await session.close(); - } + const topPr = await topNodes(driver, 5); const text = [ `📊 知识图谱统计（Neo4j）`, `节点：${stats.totalNodes} 个 (${Object.entries(stats.byType).map(([t, c]) => `${t}: ${c}`).join(", ")})`, `边：${stats.totalEdges} 条 (${Object.entries(stats.byEdgeType).map(([t, c]) => `${t}: ${c}`).join(", ")})`, `社区：${stats.communities} 个`, `PageRank Top 5：`, - ...topPr.map((n, i) => ` ${i + 1}. ${n.name} (${n.type}, pr=${(typeof n.pr === "number" ? n.pr : 0).toFixed(4)})`), + ...topPr.map((n, i) => ` ${i + 1}. ${n.name} (${n.type}, pr=${n.pagerank.toFixed(4)})`), ].join("\n"); return { content: [{ type: "text", text }], details: stats }; }, diff --git a/src/cli-extract.ts b/src/cli-extract.ts index 1f61461..efd8664 100644 --- a/src/cli-extract.ts +++ b/src/cli-extract.ts @@ -13,15 +13,12 @@ import readline from "node:readline/promises"; import { stdin as input, stdout as output } from "node:process"; import type { Driver } from "neo4j-driver"; -import type { GmConfig, GmNode } from "./types.ts"; +import type { GmConfig } from "./types.ts"; import { getDriver, initSchema, closeDriver } from "./store/db.ts"; import { listUnextractedSessions, getUnextracted, markExtracted, - upsertNode, - upsertEdge, - findByName, getBySession, type UnextractedSessionInfo, } from "./store/store.ts"; @@ -29,6 +26,7 @@ import { createCompleteFn, resolveProvider } from "./engine/llm.ts"; import { createEmbedder } from "./engine/embed.ts"; import { Recaller } from "./recaller/recall.ts"; import { Extractor } from "./extractor/extract.ts"; +import { persistExtractionResult } from "./extractor/persist.ts"; const AFFIRMATIVE = new Set(["y", "yes", "yeah", "yep", "ok", "okay", "true", "1", "confirm"]); @@ -235,37 +233,15 @@ async function extractSessionLoop( const existing = (await getBySession(driver, sessionId)).map(n => n.name); const extraction = await extractor.extract({ messages: msgs, existingNames: existing }); - const nameToId = new Map(); - // 批内节点收集到批边界统一批量嵌入（embedBatch + UNWIND 批读写）： - // closeDriver 在 finally 里执行，若不等待，最后一批在途的 embedding - // HTTP 请求会撞上已关闭的 driver 且错误被吞——向量丢失且不可自愈 - // （markExtracted 已执行，重跑 extract 不会补）。 - const batchNodes: GmNode[] = []; - for (const nc of extraction.nodes) { - const { node } = await upsertNode(driver, { - type: nc.type, name: nc.name, - description: nc.description, content: nc.content, - }, sessionId); - nameToId.set(node.name, node.id); - stats.nodes += 1; - batchNodes.push(node); - } - - for (const ec of extraction.edges) { - const fromNode = await findByName(driver, ec.from); - const toNode = await findByName(driver, ec.to); - const fromId = nameToId.get(ec.from) ?? fromNode?.id; - const toId = nameToId.get(ec.to) ?? toNode?.id; - if (fromId && toId) { - await upsertEdge(driver, { - fromId, toId, type: ec.type, - instruction: ec.instruction, condition: ec.condition, sessionId, - }); - stats.edges += 1; - } - } - - await recaller.syncEmbedBatch(batchNodes).catch(() => {}); + // awaitEmbedSync=true：closeDriver 在 finally 里执行，最后一批的 embedding + // 请求必须等完 —— 不等待会撞上已关闭的 driver 且错误被吞 + // （markExtracted 已执行，重跑 extract 不会补向量）。 + const outcome = await persistExtractionResult(driver, recaller, extraction, { + sessionId, + awaitEmbedSync: true, + onNodeUpserted: () => { stats.nodes += 1; }, + }); + stats.edges += outcome.edges; const maxTurn = msgs.reduce((m, msg) => Math.max(m, msg.turn_index ?? 0), 0); await markExtracted( diff --git a/src/engine/embed.ts b/src/engine/embed.ts index 4564282..ecaae53 100755 --- a/src/engine/embed.ts +++ b/src/engine/embed.ts @@ -15,7 +15,7 @@ */ import type { EmbeddingConfig } from "../types.ts"; -import { fetchRetry } from "./http.ts"; +import { fetchRetry, throwForStatus } from "./http.ts"; export type EmbedMode = "db" | "query"; export type EmbedFn = (text: string, mode?: EmbedMode) => Promise; @@ -138,8 +138,7 @@ export async function createEmbedder(cfg: EmbeddingConfig | undefined): Promise< const res = await postEmbedding(buildBody(input, mode), 10_000); if (!res.ok) { - const errText = await res.text().catch(() => ""); - throw new Error(`[graph-memory-pro] Embedding API ${res.status}: ${errText.slice(0, 200)}`); + await throwForStatus(res, "[graph-memory-pro] Embedding API"); } const data = await res.json() as any; @@ -157,8 +156,7 @@ export async function createEmbedder(cfg: EmbeddingConfig | undefined): Promise< const res = await postEmbedding(buildBody(texts, mode), Math.max(10_000, 2_000 * texts.length)); if (!res.ok) { - const errText = await res.text().catch(() => ""); - throw new Error(`[graph-memory-pro] Embedding API ${res.status}: ${errText.slice(0, 200)}`); + await throwForStatus(res, "[graph-memory-pro] Embedding API"); } const data = await res.json() as any; diff --git a/src/engine/http.ts b/src/engine/http.ts index 723b43d..fdee0b5 100644 --- a/src/engine/http.ts +++ b/src/engine/http.ts @@ -80,3 +80,12 @@ export async function fetchRetry( } throw new Error(`${label} request failed after retries`); } + +/** + * !res.ok 统一抛错（llm / embed 各调用方共用，消灭散落的 text().catch + slice 样板）。 + * 消息格式与既有调用方一致：`"${label} ${status}: ${errText.slice(0, sliceLen)}"`。 + */ +export async function throwForStatus(res: Response, label: string, sliceLen = 200): Promise { + const errText = await res.text().catch(() => ""); + throw new Error(`${label} ${res.status}: ${errText.slice(0, sliceLen)}`); +} diff --git a/src/engine/llm.ts b/src/engine/llm.ts index a6c67ac..976bcd6 100755 --- a/src/engine/llm.ts +++ b/src/engine/llm.ts @@ -33,9 +33,10 @@ import { normalizeOauthModel, buildOauthEndpoint, extractOutputTextFromSse, + extractOutputTextFromResponsePayload, } from "./oauth.ts"; import type { OAuthSession } from "./oauth.ts"; -import { fetchRetry } from "./http.ts"; +import { fetchRetry, throwForStatus } from "./http.ts"; export type LlmProvider = "openai" | "anthropic" | "oauth"; @@ -189,26 +190,14 @@ export function createCompleteFn( }, { retries: 3, timeoutMs, label: "[graph-memory] LLM" }); if (!res.ok) { - const errText = await res.text().catch(() => ""); - throw new Error(`[graph-memory] OAuth LLM API ${res.status}: ${errText.slice(0, 500)}`); + await throwForStatus(res, "[graph-memory] OAuth LLM API", 500); } const bodyText = await res.text(); let text: string | null = null; try { - const parsed = JSON.parse(bodyText) as Record; - const output = Array.isArray(parsed.output) ? parsed.output : []; - for (const item of output) { - if (!item || typeof item !== "object") continue; - const content = Array.isArray((item as Record).content) - ? (item as Record).content as Array> - : []; - for (const part of content) { - if (part?.type === "output_text" && typeof part.text === "string") { - text = (text ?? "") + part.text; - } - } - } + // Responses JSON → output_text 收集（与 oauth.ts 的 SSE 嵌套解析共用同一遍历） + text = extractOutputTextFromResponsePayload(JSON.parse(bodyText)); } catch { // 服务器忽略 stream:false 时回退到 SSE 解析 text = extractOutputTextFromSse(bodyText); @@ -242,8 +231,7 @@ export function createCompleteFn( }), }, { retries: 3, timeoutMs, label: "[graph-memory] LLM" }); if (!res.ok) { - const errText = await res.text().catch(() => ""); - throw new Error(`[graph-memory] Anthropic API ${res.status}: ${errText.slice(0, 200)}`); + await throwForStatus(res, "[graph-memory] Anthropic API"); } const data = await res.json() as any; // 遍历 content 找 text 块：只看 content[0] 时，thinking 块在前会误报 empty content @@ -287,8 +275,7 @@ export function createCompleteFn( }), }, { retries: 3, timeoutMs, label: "[graph-memory] LLM" }); if (!res.ok) { - const errText = await res.text().catch(() => ""); - throw new Error(`[graph-memory] LLM API ${res.status}: ${errText.slice(0, 200)}`); + await throwForStatus(res, "[graph-memory] LLM API"); } const data = await res.json() as any; const choice = data.choices?.[0]; diff --git a/src/engine/oauth.ts b/src/engine/oauth.ts index f0e1dfd..f8aaf4e 100644 --- a/src/engine/oauth.ts +++ b/src/engine/oauth.ts @@ -13,6 +13,7 @@ import { chmod, mkdir, readFile, writeFile } from "node:fs/promises"; import { dirname } from "node:path"; import { platform } from "node:os"; import { spawn } from "node:child_process"; +import { fetchWithTimeout } from "./http.ts"; // ─── Types ──────────────────────────────────────────────────── @@ -176,15 +177,44 @@ function pickTimestamp(container: Record, keys: string[]): numb return undefined; } -function createTimeoutSignal(timeoutMs?: number): { signal: AbortSignal; dispose: () => void } { +const DEFAULT_TOKEN_TIMEOUT_MS = 30_000; + +/** + * OAuth token 端点共用的 POST（refresh_token 与 authorization_code 交换同一端点、 + * 同一错误处理，此前是两份重复实现）。统一走 http.ts 的 fetchWithTimeout —— + * exchangeAuthorizationCode 曾是裸 fetch 无超时（CLI 登录流程卡死风险）， + * refresh 用手写 AbortController，现在两者语义一致且与全仓单一来源对齐。 + * 错误消息保持既有格式：`${errorLabel} (${status}): ${detail.slice(0, 500)}`。 + */ +async function postTokenRequest( + providerId: OAuthProviderId, + body: Record, + errorLabel: string, + timeoutMs?: number, +): Promise { const effectiveTimeoutMs = - typeof timeoutMs === "number" && Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 30_000; - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), effectiveTimeoutMs); - return { - signal: controller.signal, - dispose: () => clearTimeout(timer), - }; + typeof timeoutMs === "number" && Number.isFinite(timeoutMs) && timeoutMs > 0 + ? timeoutMs + : DEFAULT_TOKEN_TIMEOUT_MS; + const response = await fetchWithTimeout( + resolveOauthTokenUrl(undefined, providerId), + { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + body: new URLSearchParams(body), + }, + effectiveTimeoutMs, + "[graph-memory] OAuth", + ); + + if (!response.ok) { + const detail = await response.text().catch(() => ""); + throw new Error(`${errorLabel} (${response.status}): ${detail.slice(0, 500)}`); + } + + return await response.json() as TokenRefreshResponse; } // ─── Provider resolution ────────────────────────────────────── @@ -391,54 +421,36 @@ export async function refreshOAuthSession(session: OAuthSession, timeoutMs?: num ); } - const { signal, dispose } = createTimeoutSignal(timeoutMs); - try { - const response = await fetch(resolveOauthTokenUrl(undefined, session.providerId), { - method: "POST", - headers: { - "Content-Type": "application/x-www-form-urlencoded", - }, - body: new URLSearchParams({ - grant_type: "refresh_token", - refresh_token: session.refreshToken, - client_id: resolveOauthClientId(undefined, session.providerId), - }), - signal, - }); - - if (!response.ok) { - const detail = await response.text().catch(() => ""); - throw new Error(`OAuth refresh failed (${response.status}): ${detail.slice(0, 500)}`); - } - - const payload = await response.json() as TokenRefreshResponse; - if (!payload.access_token) { - throw new Error("OAuth refresh returned no access token"); - } + const payload = await postTokenRequest(session.providerId, { + grant_type: "refresh_token", + refresh_token: session.refreshToken, + client_id: resolveOauthClientId(undefined, session.providerId), + }, "OAuth refresh failed", timeoutMs); - const accessToken = payload.access_token; - const refreshToken = payload.refresh_token || session.refreshToken; - const expiresAt = - typeof payload.expires_in === "number" - ? Date.now() + payload.expires_in * 1000 - : getJwtExpiry(accessToken); - const accountId = getJwtAccountId(accessToken, session.providerId) || session.accountId; + if (!payload.access_token) { + throw new Error("OAuth refresh returned no access token"); + } - if (!accountId) { - throw new Error("OAuth refresh returned a token without a ChatGPT account id"); - } + const accessToken = payload.access_token; + const refreshToken = payload.refresh_token || session.refreshToken; + const expiresAt = + typeof payload.expires_in === "number" + ? Date.now() + payload.expires_in * 1000 + : getJwtExpiry(accessToken); + const accountId = getJwtAccountId(accessToken, session.providerId) || session.accountId; - return { - accessToken, - refreshToken, - expiresAt, - accountId, - providerId: session.providerId, - authPath: session.authPath, - }; - } finally { - dispose(); + if (!accountId) { + throw new Error("OAuth refresh returned a token without a ChatGPT account id"); } + + return { + accessToken, + refreshToken, + expiresAt, + accountId, + providerId: session.providerId, + authPath: session.authPath, + }; } export async function saveOAuthSession(authPath: string, session: OAuthSession): Promise { @@ -490,9 +502,14 @@ export function buildOauthEndpoint(baseURL?: string, providerId?: string): strin return `${root}/codex/responses`; } -// ─── SSE response parsing ───────────────────────────────────── +// ─── Responses payload 解析 ─────────────────────────────────── -function extractOutputTextFromResponsePayload(payload: unknown): string | null { +/** + * 从 Codex Responses API 的 JSON payload 收集 output_text（多段以 "\n" 连接）。 + * oauth.ts 的 SSE 嵌套回退与 llm.ts 的非流式主路径共用此遍历 —— 勿在调用方 + * 重新内联 output[].content[] 走树。 + */ +export function extractOutputTextFromResponsePayload(payload: unknown): string | null { if (!payload || typeof payload !== "object") return null; const response = payload as Record; @@ -565,26 +582,14 @@ export function extractOutputTextFromSse(bodyText: string): string | null { async function exchangeAuthorizationCode(code: string, verifier: string, providerId?: string): Promise { const resolvedProviderId = normalizeOAuthProviderId(providerId); - const response = await fetch(resolveOauthTokenUrl(undefined, resolvedProviderId), { - method: "POST", - headers: { - "Content-Type": "application/x-www-form-urlencoded", - }, - body: new URLSearchParams({ - grant_type: "authorization_code", - client_id: resolveOauthClientId(undefined, resolvedProviderId), - code, - code_verifier: verifier, - redirect_uri: resolveOauthRedirectUri(undefined, resolvedProviderId), - }), - }); - - if (!response.ok) { - const detail = await response.text().catch(() => ""); - throw new Error(`OAuth token exchange failed (${response.status}): ${detail.slice(0, 500)}`); - } + const payload = await postTokenRequest(resolvedProviderId, { + grant_type: "authorization_code", + client_id: resolveOauthClientId(undefined, resolvedProviderId), + code, + code_verifier: verifier, + redirect_uri: resolveOauthRedirectUri(undefined, resolvedProviderId), + }, "OAuth token exchange failed"); - const payload = await response.json() as TokenRefreshResponse; if (!payload.access_token) { throw new Error("OAuth token exchange returned no access token"); } diff --git a/src/extractor/persist.ts b/src/extractor/persist.ts new file mode 100644 index 0000000..d4d3b4a --- /dev/null +++ b/src/extractor/persist.ts @@ -0,0 +1,80 @@ +/** + * graph-memory-pro — 提取结果持久化（单一来源） + * + * "upsertNode → 批量向量同步 → 解析端点 → upsertEdge" 这段持久化体此前在 + * index.ts（per-turn 提取 / compact 攒批提取）与 cli-extract.ts（CLI 回填） + * 逐字复制三份，且已发生漂移（CLI await 了 syncEmbedBatch，运行时是 + * fire-and-forget）。收敛于此：改边解析/嵌入策略只需改这一处。 + */ + +import type { Driver } from "neo4j-driver"; +import type { ExtractionResult, GmNode } from "../types.ts"; +import { upsertNode, upsertEdge, findByName } from "../store/store.ts"; +import type { Recaller } from "../recaller/recall.ts"; + +export interface PersistExtractionOptions { + sessionId: string; + /** + * true = 等待批量向量同步完成再返回（CLI 路径：driver 将在 finally 里 + * closeDriver，不等待会让在途 embedding 请求撞上已关闭的 driver 且错误 + * 被吞——markExtracted 已执行，向量丢失不可自愈）；false = fire-and-forget + * （运行时路径：不让 embedding 延迟拖住回合）。 + */ + awaitEmbedSync?: boolean; + /** 每个成功 upsert 的节点回调（CLI 统计 nodesCreated 用）。 */ + onNodeUpserted?: (node: GmNode) => void; +} + +export interface PersistExtractionOutcome { + /** 实际 upsert 的节点 */ + nodes: GmNode[]; + /** 成功建立的边数（两端可解析且方向合法；upsertEdge 内部仍会按库中真实端点复核方向） */ + edges: number; +} + +/** + * 将一次 ExtractionResult 落库：upsert 全部节点 → 启动批量向量同步 + * （awaitEmbedSync 决定是否等待）→ 逐条解析并 upsert 边。 + * 边端点优先用本次 upsert 得到的 name→id 索引，未命中再查库（findByName）。 + * 不负责 markExtracted —— 各调用方的 upToTurn/producedKnowledge 语义不同。 + */ +export async function persistExtractionResult( + driver: Driver, + recaller: Pick, + result: ExtractionResult, + opts: PersistExtractionOptions, +): Promise { + const nameToId = new Map(); + const upserted: GmNode[] = []; + for (const nc of result.nodes) { + const { node } = await upsertNode(driver, { + type: nc.type, name: nc.name, + description: nc.description, content: nc.content, + }, opts.sessionId); + nameToId.set(node.name, node.id); + upserted.push(node); + opts.onNodeUpserted?.(node); + } + + // 批量向量同步：N 个节点一次 embedBatch + 一次 UNWIND 批读写（替代逐节点 N 次单发） + const embedSync = recaller.syncEmbedBatch(upserted).catch(() => {}); + if (opts.awaitEmbedSync) await embedSync; + else void embedSync; + + let edgeCount = 0; + for (const ec of result.edges) { + const fromNode = await findByName(driver, ec.from); + const toNode = await findByName(driver, ec.to); + const fromId = nameToId.get(ec.from) ?? fromNode?.id; + const toId = nameToId.get(ec.to) ?? toNode?.id; + if (fromId && toId) { + await upsertEdge(driver, { + fromId, toId, type: ec.type, + instruction: ec.instruction, condition: ec.condition, sessionId: opts.sessionId, + }); + edgeCount += 1; + } + } + + return { nodes: upserted, edges: edgeCount }; +} diff --git a/src/graph/pagerank.ts b/src/graph/pagerank.ts index a39edd6..5a37d86 100755 --- a/src/graph/pagerank.ts +++ b/src/graph/pagerank.ts @@ -5,11 +5,37 @@ * 所以先查有哪些关系类型，只投影存在的 */ -import type { Driver } from "neo4j-driver"; +import type { Driver, Session } from "neo4j-driver"; import type { GmConfig } from "../types.ts"; import { getSession } from "../store/db.ts"; import { getExistingActiveRelTypes, projectActiveGraph } from "./projection.ts"; +/** + * 读回 pagerank top-20 并构建 scores Map + topK 数组 —— global PR 的三条路径 + * （均匀分兜底 / GDS write 后读回 / 无 GDS fallback）共用的收尾读取。 + * tiebreakCreatedAt=true 时以 createdAt ASC 决胜（fallback 路径的稳定序）。 + */ +async function readTopKByPagerank(session: Session, tiebreakCreatedAt = false): Promise<{ + scores: Map; + topK: Array<{ id: string; name: string; score: number }>; +}> { + const topResult = await session.run(` + MATCH (n:Task|Skill|Event {status: 'active'}) + RETURN n.id AS id, n.name AS name, n.pagerank AS score + ORDER BY n.pagerank DESC ${tiebreakCreatedAt ? ", n.createdAt ASC" : ""} + LIMIT 20 + `); + const scores = new Map(); + const topK: Array<{ id: string; name: string; score: number }> = []; + for (const r of topResult.records) { + const rawScore = r.get("score"); + const score = typeof rawScore === "number" ? rawScore : (rawScore?.toNumber?.() ?? 0); + scores.set(r.get("id"), score); + topK.push({ id: r.get("id"), name: r.get("name"), score }); + } + return { scores, topK }; +} + // ─── 个性化 PageRank ───────────────────────────────────────── export interface PPRResult { @@ -108,16 +134,7 @@ export async function computeGlobalPageRank(driver: Driver, cfg: GmConfig): Prom // 没有关系，均匀分 const uniformScore = 1 / nodeCount; await session.run("MATCH (n:Task|Skill|Event {status: 'active'}) SET n.pagerank = $score", { score: uniformScore }); - const topResult = await session.run(` - MATCH (n:Task|Skill|Event {status: 'active'}) RETURN n.id AS id, n.name AS name, n.pagerank AS score - ORDER BY n.pagerank DESC LIMIT 20 - `); - const scores = new Map(); - const topK = topResult.records.map(r => { - scores.set(r.get("id"), uniformScore); - return { id: r.get("id"), name: r.get("name"), score: uniformScore }; - }); - return { scores, topK }; + return await readTopKByPagerank(session); } await projectActiveGraph(session, graphName, existingTypes); @@ -139,20 +156,7 @@ export async function computeGlobalPageRank(driver: Driver, cfg: GmConfig): Prom // write 已成功：pagerank 属性已是真值，后续读取失败只返回空排序， // 绝不回落外层 catch 的 fallback（那会覆盖全图正确分数） try { - const topResult = await session.run(` - MATCH (n:Task|Skill|Event {status: 'active'}) RETURN n.id AS id, n.name AS name, n.pagerank AS score - ORDER BY n.pagerank DESC LIMIT 20 - `); - - const scores = new Map(); - const topK: Array<{ id: string; name: string; score: number }> = []; - for (const r of topResult.records) { - const rawScore = r.get("score"); - const score = typeof rawScore === "number" ? rawScore : (rawScore?.toNumber?.() ?? 0); - scores.set(r.get("id"), score); - topK.push({ id: r.get("id"), name: r.get("name"), score }); - } - return { scores, topK }; + return await readTopKByPagerank(session); } catch { return { scores: new Map(), topK: [] }; } @@ -167,23 +171,7 @@ export async function computeGlobalPageRank(driver: Driver, cfg: GmConfig): Prom WITH nodes[idx] AS node, idx SET node.pagerank = 1.0 / toFloat(idx + 1) `); - const fallbackResult = await session.run(` - MATCH (n:Task|Skill|Event {status: 'active'}) - RETURN n.id AS id, n.name AS name, n.pagerank AS score - ORDER BY n.pagerank DESC, n.createdAt ASC - LIMIT 20 - `); - const scores = new Map(); - const topK: Array<{ id: string; name: string; score: number }> = []; - for (const r of fallbackResult.records) { - const rawScore = r.get("score"); - const score = typeof rawScore === "number" ? rawScore : (rawScore?.toNumber?.() ?? 0); - const id = r.get("id"); - const name = r.get("name"); - scores.set(id, score); - topK.push({ id, name, score }); - } - return { scores, topK }; + return await readTopKByPagerank(session, true); } finally { await session.close(); } diff --git a/src/graph/projection.ts b/src/graph/projection.ts index f35dc6b..fb78173 100644 --- a/src/graph/projection.ts +++ b/src/graph/projection.ts @@ -1,12 +1,8 @@ import type { Session } from "neo4j-driver"; +import { EDGE_TYPES } from "../types.ts"; -const KNOWLEDGE_REL_TYPES = [ - "USED_SKILL", - "SOLVED_BY", - "REQUIRES", - "PATCHES", - "CONFLICTS_WITH", -] as const; +/** 知识边类型白名单（事实源 types.ts 的 EDGE_TYPES，勿另维护字面量副本） */ +const KNOWLEDGE_REL_TYPES: readonly string[] = [...EDGE_TYPES]; export async function getExistingActiveRelTypes(session: Session): Promise { const result = await session.run(` diff --git a/src/recaller/recall.ts b/src/recaller/recall.ts index a2ffc91..869b93d 100755 --- a/src/recaller/recall.ts +++ b/src/recaller/recall.ts @@ -192,6 +192,10 @@ export class Recaller { this.driver, seedIds, candidateIds, this.cfg, ); + // PPR 排序后的过滤/截断尾块与 recallGeneralized 结构一致，但 tiebreak + // 顺序刻意相反：precise 以 validatedCount 决胜（精确命中的知识更受认可）， + // generalized 以 updatedAt 决胜（泛化探索偏向新鲜）。改动前先同步 + // test/ablation.study.test.ts 的镜像 harness。 const filtered = nodes .filter(n => !timeRange || matchTimeRange(n, timeRange)) .sort((a, b) => @@ -246,6 +250,7 @@ export class Recaller { this.driver, seedIds, candidateIds, this.cfg, ); + // tiebreak 与 recallPrecise 相反（updatedAt 决胜）——见该处的注释 const filtered = nodes .filter(n => !timeRange || matchTimeRange(n, timeRange)) .sort((a, b) => @@ -296,7 +301,7 @@ export class Recaller { const vec = this.embed ? await this.embed(text, "db") : (await this.embedBatch!([text], "db"))[0]; - if (vec.length) await saveVector(this.driver, node.id, text, vec); + if (vec.length) await saveVector(this.driver, node.id, text, vec, hash); } catch {} } @@ -330,7 +335,7 @@ export class Recaller { for (const t of pending) { try { const vec = await this.embed!(t.text, "db"); - if (vec.length) await saveVector(this.driver, t.node.id, t.text, vec); + if (vec.length) await saveVector(this.driver, t.node.id, t.text, vec, t.hash); } catch {} } return; diff --git a/src/routes/crud.ts b/src/routes/crud.ts index 4fe947c..7d6cc0f 100644 --- a/src/routes/crud.ts +++ b/src/routes/crud.ts @@ -20,7 +20,7 @@ import { NODE_TYPE_TO_LABEL, isValidEdgeDirection, EDGE_DIRECTION_RULES, EDGE_TY import { upsertNode, findById, findByName, allActiveNodes, allEdges, upsertEdge, edgesFrom, edgesTo, deprecateNodeAndDisconnectById, mergeNodes, - searchNodes, getStats, normalizeName, + searchNodes, getStats, normalizeName, deleteEdges, deleteEdgeById, } from "../store/store.ts"; import { getSession } from "../store/db.ts"; @@ -477,9 +477,8 @@ async function handleCreateEdge( return true; } - const validTypes = ["USED_SKILL", "SOLVED_BY", "REQUIRES", "PATCHES", "CONFLICTS_WITH"]; - if (!validTypes.includes(type)) { - json(res, 400, { error: `Invalid edge type: ${type}. Must be one of: ${validTypes.join(", ")}` }); + if (!EDGE_TYPES.includes(type)) { + json(res, 400, { error: `Invalid edge type: ${type}. Must be one of: ${EDGE_TYPES.join(", ")}` }); return true; } @@ -522,6 +521,7 @@ async function handleCreateEdge( /** * DELETE /edges?id=xxx * 或 DELETE /edges?fromId=xxx&toId=yyy&type=USED_SKILL + * 删除下沉到 store 层（deleteEdgeById / deleteEdges），路由层不写 Cypher。 */ async function handleDeleteEdge( res: ServerResponse, @@ -533,38 +533,20 @@ async function handleDeleteEdge( const toId = query.toId ?? query.to_id; const edgeType = query.type; - const session = getSession(driver); - try { - if (edgeId) { - // Delete by edge id - await session.run(` - MATCH ()-[r]->() - WHERE r.id = $edgeId - DELETE r - `, { edgeId }); - } else if (fromId && toId) { - // Delete by endpoints (+ optional type filter) - if (edgeType) { - await session.run(` - MATCH (a:Task|Skill|Event {id: $fromId})-[r]->(b:Task|Skill|Event {id: $toId}) - WHERE type(r) = $edgeType - DELETE r - `, { fromId, toId, edgeType: edgeType.toUpperCase() }); - } else { - await session.run(` - MATCH (a:Task|Skill|Event {id: $fromId})-[r]->(b:Task|Skill|Event {id: $toId}) - DELETE r - `, { fromId, toId }); - } - } else { - json(res, 400, { error: "Provide either id or fromId+toId" }); - return true; - } - } finally { - await session.close(); + let deleted: number; + if (edgeId) { + deleted = await deleteEdgeById(driver, edgeId); + } else if (fromId && toId) { + deleted = await deleteEdges( + driver, fromId, toId, + edgeType ? (edgeType.toUpperCase() as EdgeType) : undefined, + ); + } else { + json(res, 400, { error: "Provide either id or fromId+toId" }); + return true; } - json(res, 200, { success: true }); + json(res, 200, { success: true, deleted }); return true; } diff --git a/src/store/store.ts b/src/store/store.ts index 5e4b3bc..c9bfe4c 100755 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -76,6 +76,34 @@ function normalizeName(name: string): string { export { normalizeName }; +// ─── 边行读取（单一来源）───────────────────────────────────── +// 知识边类型白名单的 Cypher 片段与 RETURN 投影/记录映射只在此定义， +// allEdges/edgesFrom/edgesTo/edgesTouching/graphWalk/getStats 共用 —— +// 新增 EdgeType 时只改 types.ts 的 EDGE_TYPES，勿在 Cypher 里内联字面量。 + +const EDGE_TYPE_FILTER = `type(r) IN ${JSON.stringify([...EDGE_TYPES])}`; + +const EDGE_ROW_RETURN = ` + RETURN r.id AS id, a.id AS fromId, b.id AS toId, type(r) AS type, + r.instruction AS instruction, r.condition AS condition, + r.sessionId AS sessionId, r.createdAt AS createdAt`; + +type EdgeRowResult = { records: Array<{ get(key: string): any }> }; + +/** 边行记录 → GmEdge（与 EDGE_ROW_RETURN 投影一一对应） */ +function mapEdgeRecords(result: EdgeRowResult): GmEdge[] { + return result.records.map(r => ({ + id: r.get("id"), + fromId: r.get("fromId"), + toId: r.get("toId"), + type: r.get("type") as EdgeType, + instruction: r.get("instruction"), + condition: r.get("condition") ?? undefined, + sessionId: r.get("sessionId"), + createdAt: toInt(r.get("createdAt")), + })); +} + // ─── 节点 CRUD ─────────────────────────────────────────────── export async function findByName(driver: Driver, name: string): Promise { @@ -123,21 +151,9 @@ export async function allEdges(driver: Driver): Promise { try { const result = await session.run(` MATCH (a:Task|Skill|Event)-[r]->(b:Task|Skill|Event) - WHERE type(r) IN ['USED_SKILL','SOLVED_BY','REQUIRES','PATCHES','CONFLICTS_WITH'] - RETURN r.id AS id, a.id AS fromId, b.id AS toId, type(r) AS type, - r.instruction AS instruction, r.condition AS condition, - r.sessionId AS sessionId, r.createdAt AS createdAt + WHERE ${EDGE_TYPE_FILTER}${EDGE_ROW_RETURN} `); - return result.records.map(r => ({ - id: r.get("id"), - fromId: r.get("fromId"), - toId: r.get("toId"), - type: r.get("type") as EdgeType, - instruction: r.get("instruction"), - condition: r.get("condition") ?? undefined, - sessionId: r.get("sessionId"), - createdAt: toInt(r.get("createdAt")), - })); + return mapEdgeRecords(result); } finally { await session.close(); } @@ -593,21 +609,9 @@ export async function edgesFrom(driver: Driver, id: string): Promise { try { const result = await session.run(` MATCH (a:Task|Skill|Event {id: $id})-[r]->(b:Task|Skill|Event) - WHERE type(r) IN ['USED_SKILL','SOLVED_BY','REQUIRES','PATCHES','CONFLICTS_WITH'] - RETURN r.id AS id, a.id AS fromId, b.id AS toId, type(r) AS type, - r.instruction AS instruction, r.condition AS condition, - r.sessionId AS sessionId, r.createdAt AS createdAt + WHERE ${EDGE_TYPE_FILTER}${EDGE_ROW_RETURN} `, { id }); - return result.records.map(r => ({ - id: r.get("id"), - fromId: r.get("fromId"), - toId: r.get("toId"), - type: r.get("type") as EdgeType, - instruction: r.get("instruction"), - condition: r.get("condition") ?? undefined, - sessionId: r.get("sessionId"), - createdAt: toInt(r.get("createdAt")), - })); + return mapEdgeRecords(result); } finally { await session.close(); } @@ -618,21 +622,9 @@ export async function edgesTo(driver: Driver, id: string): Promise { try { const result = await session.run(` MATCH (a:Task|Skill|Event)-[r]->(b:Task|Skill|Event {id: $id}) - WHERE type(r) IN ['USED_SKILL','SOLVED_BY','REQUIRES','PATCHES','CONFLICTS_WITH'] - RETURN r.id AS id, a.id AS fromId, b.id AS toId, type(r) AS type, - r.instruction AS instruction, r.condition AS condition, - r.sessionId AS sessionId, r.createdAt AS createdAt + WHERE ${EDGE_TYPE_FILTER}${EDGE_ROW_RETURN} `, { id }); - return result.records.map(r => ({ - id: r.get("id"), - fromId: r.get("fromId"), - toId: r.get("toId"), - type: r.get("type") as EdgeType, - instruction: r.get("instruction"), - condition: r.get("condition") ?? undefined, - sessionId: r.get("sessionId"), - createdAt: toInt(r.get("createdAt")), - })); + return mapEdgeRecords(result); } finally { await session.close(); } @@ -646,21 +638,9 @@ export async function edgesTouching(driver: Driver, ids: string[]): Promise(b:Task|Skill|Event) WHERE (a.id IN $ids OR b.id IN $ids) - AND type(r) IN ${JSON.stringify([...EDGE_TYPES])} - RETURN r.id AS id, a.id AS fromId, b.id AS toId, type(r) AS type, - r.instruction AS instruction, r.condition AS condition, - r.sessionId AS sessionId, r.createdAt AS createdAt + AND ${EDGE_TYPE_FILTER}${EDGE_ROW_RETURN} `, { ids }); - return result.records.map(r => ({ - id: r.get("id"), - fromId: r.get("fromId"), - toId: r.get("toId"), - type: r.get("type") as EdgeType, - instruction: r.get("instruction"), - condition: r.get("condition") ?? undefined, - sessionId: r.get("sessionId"), - createdAt: toInt(r.get("createdAt")), - })); + return mapEdgeRecords(result); } finally { await session.close(); } @@ -695,6 +675,23 @@ export async function deleteEdges( } } +/** 按 edge id 删除单条边（REST DELETE /edges?id= 用）。返回删除条数（0/1）。 */ +export async function deleteEdgeById(driver: Driver, edgeId: string): Promise { + const session = getSession(driver); + try { + const result = await session.run( + `MATCH ()-[r]->() + WHERE r.id = $edgeId + DELETE r + RETURN count(r) AS deleted`, + { edgeId }, + ); + return toInt(result.records[0]?.get("deleted") ?? 0); + } finally { + await session.close(); + } +} + // ─── 搜索 ─────────────────────────────────────────────────── /** 全文搜索节点（CONTAINS 模糊匹配） */ @@ -799,14 +796,20 @@ export async function communityVectorSearch( // ─── 向量存储 ─────────────────────────────────────────────── -export async function saveVector(driver: Driver, nodeId: string, content: string, vec: number[]): Promise { - const hash = createHash("md5").update(content).digest("hex"); +/** + * 单发写向量 + contentHash。hash 可选：调用方已为短路检查预计算过 md5 时 + * 直接传入，避免对同一 content 重复哈希（缺省时内部补算）。 + */ +export async function saveVector( + driver: Driver, nodeId: string, content: string, vec: number[], hash?: string, +): Promise { + const contentHash = hash ?? createHash("md5").update(content).digest("hex"); const session = getSession(driver); try { await session.run(` MATCH (n:Task|Skill|Event {id: $nodeId}) - SET n.embedding = $vec, n.contentHash = $hash - `, { nodeId, vec, hash }); + SET n.embedding = $vec, n.contentHash = $contentHash + `, { nodeId, vec, contentHash }); } finally { await session.close(); } @@ -1083,22 +1086,10 @@ export async function graphWalk( const edgeResult = await session.run(` MATCH (a:Task|Skill|Event)-[r]->(b:Task|Skill|Event) WHERE a.id IN $nodeIds AND b.id IN $nodeIds - AND type(r) IN ['USED_SKILL','SOLVED_BY','REQUIRES','PATCHES','CONFLICTS_WITH'] - RETURN r.id AS id, a.id AS fromId, b.id AS toId, type(r) AS type, - r.instruction AS instruction, r.condition AS condition, - r.sessionId AS sessionId, r.createdAt AS createdAt + AND ${EDGE_TYPE_FILTER}${EDGE_ROW_RETURN} `, { nodeIds }); - const edges = edgeResult.records.map(r => ({ - id: r.get("id"), - fromId: r.get("fromId"), - toId: r.get("toId"), - type: r.get("type") as EdgeType, - instruction: r.get("instruction"), - condition: r.get("condition") ?? undefined, - sessionId: r.get("sessionId"), - createdAt: toInt(r.get("createdAt")), - })); + const edges = mapEdgeRecords(edgeResult); return { nodes, edges }; } finally { @@ -1389,7 +1380,7 @@ export async function getStats(driver: Driver): Promise<{ const edgeResult = await session.run(` MATCH ()-[r]->() - WHERE type(r) IN ['USED_SKILL','SOLVED_BY','REQUIRES','PATCHES','CONFLICTS_WITH'] + WHERE ${EDGE_TYPE_FILTER} RETURN type(r) AS type, count(r) AS c `); let totalEdges = 0; diff --git a/test/commit-turn.test.ts b/test/commit-turn.test.ts index 5417397..f11a902 100644 --- a/test/commit-turn.test.ts +++ b/test/commit-turn.test.ts @@ -32,7 +32,7 @@ vi.mock("../src/store/store.ts", () => ({ vi.mock("../src/engine/llm.ts", () => ({ createCompleteFn: () => async () => "", - resolveProvider: () => ({ provider: "anthropic", effectiveModel: "m", inferred: false }), + resolveProvider: () => ({ provider: "anthropic", inferred: false }), })); vi.mock("../src/engine/embed.ts", () => ({ From d38d972d6a1ebe7f81a5df018f8af98f99ad2f11 Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Thu, 3 Sep 2026 18:30:05 +0000 Subject: [PATCH 28/29] chore: local code audit results --- index.ts | 48 +++++++++-- src/extractor/persist.ts | 21 +++-- src/extractor/turn-filter.ts | 19 ++++- src/types.ts | 2 + test/persist.test.ts | 151 +++++++++++++++++++++++++++++++++++ test/turn-filter.test.ts | 40 +++++++++- 6 files changed, 263 insertions(+), 18 deletions(-) create mode 100644 test/persist.test.ts diff --git a/index.ts b/index.ts index 00d961b..9dbceb4 100755 --- a/index.ts +++ b/index.ts @@ -23,7 +23,7 @@ import { createEmbedder } from "./src/engine/embed.ts"; import { estimateTokens } from "./src/tokens.ts"; import { Recaller, parseTimeRange } from "./src/recaller/recall.ts"; import { Extractor, shouldRunFinalize } from "./src/extractor/extract.ts"; -import { shouldSkipTurnExtraction } from "./src/extractor/turn-filter.ts"; +import { shouldSkipTurnExtraction, turnHasToolWork } from "./src/extractor/turn-filter.ts"; import { persistExtractionResult } from "./src/extractor/persist.ts"; import { assembleContext } from "./src/format/assemble.ts"; import { sanitizeToolUseResultPairing } from "./src/format/transcript-repair.ts"; @@ -157,6 +157,9 @@ export function missingIngestMessages(messages: any[], ingestedCount: number): a const KEEP_TURNS = 5; +/** batched 模式 session_end 冲洗的最大批数（防积压失控；残余保持未提取，CLI extract 可回填） */ +const SESSION_END_FLUSH_ROUNDS = 5; + function estimateMsgTokens(msg: any): number { const text = typeof msg.content === "string" ? msg.content @@ -485,6 +488,19 @@ const graphMemoryProPlugin = { return result; } + /** + * 会话内"upsert 命中已有节点"（isNew=false）计数：finalize 阶梯的第二触发条件。 + * invalidations（纠错弃用）只有 finalize 一条产出路径，而纠错常发生在 + * 小会话/无 EVENT 会话——shouldRunFinalize 的规模/EVENT 双门恰好会漏掉它们； + * 会话触碰过既有知识 = 有纠错/跨会话建边价值，值得一次 finalize LLM 调用。 + */ + const sessionUpdatedHits = new Map(); + function trackUpdatedExisting(sessionId: string, count: number): void { + if (count > 0) { + sessionUpdatedHits.set(sessionId, (sessionUpdatedHits.get(sessionId) ?? 0) + count); + } + } + async function extractTurnKnowledge(sessionId: string, turnNum: number, rawMessages: any[]): Promise { // 熔断开启时跳过本轮提取：消息保持未标记，恢复后由 compact / extract 补提取 if (!neo4jGate.isAvailable()) { @@ -502,8 +518,11 @@ const graphMemoryProPlugin = { } // LLM 成本控制：trivial 轮本地预筛（无意义词表 / ≤trivialMaxChars 纯文本）， - // 命中则零 LLM 直接标记（producedKnowledge=false，原始证据保留） - if (shouldSkipTurnExtraction(turnUserText(rawMessages), trivialFilterOpts)) { + // 命中则零 LLM 直接标记（producedKnowledge=false，原始证据保留）。 + // 轮内含工具劳动（tool/toolResult）时不判 trivial："继续"触发的一轮真实 + // 修复劳动恰是图谱最该吸收的知识，且 per-turn 路径没有自动补提触发点。 + if (!turnHasToolWork(rawMessages) + && shouldSkipTurnExtraction(turnUserText(rawMessages), trivialFilterOpts)) { await markExtracted(driver, sessionId, turnNum, false); api.logger.info(`[graph-memory-pro] turn ${turnNum}: trivial prompt, extraction skipped (local pre-filter)`); return; @@ -525,7 +544,8 @@ const graphMemoryProPlugin = { } // upsert 节点 + 批量向量同步（fire-and-forget）+ 建边 —— 单一来源见 persist.ts - await persistExtractionResult(driver, recaller, result, { sessionId }); + const outcome = await persistExtractionResult(driver, recaller, result, { sessionId }); + trackUpdatedExisting(sessionId, outcome.updatedExisting); // 标记该轮消息已提取 await markExtracted(driver, sessionId, turnNum); @@ -557,7 +577,8 @@ const graphMemoryProPlugin = { const result = await extractor.extract({ messages: msgs, existingNames: existing }); // upsert 节点 + 批量向量同步（fire-and-forget）+ 建边 —— 单一来源见 persist.ts - await persistExtractionResult(driver, recaller, result, { sessionId }); + const outcome = await persistExtractionResult(driver, recaller, result, { sessionId }); + trackUpdatedExisting(sessionId, outcome.updatedExisting); const maxTurn = Math.max(...msgs.map((m: any) => m.turn_index)); await markExtracted( @@ -1153,6 +1174,7 @@ const graphMemoryProPlugin = { msgSeqLoaders.delete(childSessionId); extractLocks.delete(childSessionId); ingestedSinceTurn.delete(childSessionId); + sessionUpdatedHits.delete(childSessionId); } sessionIdsByKey.delete(childSessionKey); pendingSubagentRecall.delete(childSessionKey); @@ -1167,6 +1189,7 @@ const graphMemoryProPlugin = { sessionIdsByKey.clear(); pendingSubagentRecall.clear(); ingestedSinceTurn.clear(); + sessionUpdatedHits.clear(); // 仅当释放的是当前活跃引擎时清空标记 —— 真正的重载（dispose 后重新 // register）才会走完整初始化路径 if (activeEngine === engine) activeEngine = null; @@ -1201,9 +1224,15 @@ const graphMemoryProPlugin = { return; } - // batched 模式：会话结束冲洗残留未提取批（攒批未达阈值时保证知识不丢） + // batched 模式：会话结束冲洗残留未提取批（攒批未达阈值时保证知识不丢）。 + // drain 循环（上限 5）：中途 LLM 失败可能积压超过单批 limit（compactTurnCount*3）， + // 只冲一批会留尾部知识缺口；ok=false（本批失败）即停，避免同因反复打 LLM。 + // 仍超限的残余保持未提取——retention fail-closed 不删，可再跑 `graph-memory extract` 回填。 if (extractCfg.mode === "batched") { - await extractUnextractedBatch(sid); + for (let i = 0; i < SESSION_END_FLUSH_ROUNDS; i++) { + const flush = await extractUnextractedBatch(sid); + if (!flush.ok || !flush.compacted) break; + } } let nodes: Awaited>; @@ -1216,7 +1245,9 @@ const graphMemoryProPlugin = { api.logger.error(`[graph-memory-pro] session_end error: ${err}`); return; } - if (nodes.length && shouldRunFinalize(nodes)) { + // finalize 阶梯：规模/EVENT 门（shouldRunFinalize）之外，会话内 upsert 命中过 + // 已有节点也触发——纠错型 invalidations 只有 finalize 能产出，小会话纠错不可漏 + if (nodes.length && (shouldRunFinalize(nodes) || (sessionUpdatedHits.get(sid) ?? 0) > 0)) { // finalize 的 upsert 与 afterTurn/compact 的提取共用 per-session 互斥锁： // 最后一轮的 afterTurn 提取可能仍在途，不串行化会重复 upsert（validatedCount 双递增） await withExtractLock(sid, async () => { @@ -1266,6 +1297,7 @@ const graphMemoryProPlugin = { recalled.delete(sid); recalledPrompt.delete(sid); ingestedSinceTurn.delete(sid); + sessionUpdatedHits.delete(sid); if (sessionKey && sessionIdsByKey.get(sessionKey) === sid) { sessionIdsByKey.delete(sessionKey); pendingSubagentRecall.delete(sessionKey); diff --git a/src/extractor/persist.ts b/src/extractor/persist.ts index d4d3b4a..308fea9 100644 --- a/src/extractor/persist.ts +++ b/src/extractor/persist.ts @@ -30,12 +30,19 @@ export interface PersistExtractionOutcome { nodes: GmNode[]; /** 成功建立的边数（两端可解析且方向合法；upsertEdge 内部仍会按库中真实端点复核方向） */ edges: number; + /** + * 命中已有节点的 upsert 数（isNew=false）。index.ts 把它按会话累加为 + * finalize 阶梯的第二触发条件——纠错型 invalidations 只有 finalize 能产出， + * 而纠错常发生在小会话/无 EVENT 会话（shouldRunFinalize 的双门会漏掉）。 + */ + updatedExisting: number; } /** * 将一次 ExtractionResult 落库：upsert 全部节点 → 启动批量向量同步 * （awaitEmbedSync 决定是否等待）→ 逐条解析并 upsert 边。 - * 边端点优先用本次 upsert 得到的 name→id 索引，未命中再查库（findByName）。 + * 边端点优先查本次 upsert 得到的 name→id 索引（零往返；键是规范化名，LLM + * 原文未必规范化，未命中必须回源 findByName——它做 normalizeName）。 * 不负责 markExtracted —— 各调用方的 upToTurn/producedKnowledge 语义不同。 */ export async function persistExtractionResult( @@ -46,13 +53,15 @@ export async function persistExtractionResult( ): Promise { const nameToId = new Map(); const upserted: GmNode[] = []; + let updatedExisting = 0; for (const nc of result.nodes) { - const { node } = await upsertNode(driver, { + const { node, isNew } = await upsertNode(driver, { type: nc.type, name: nc.name, description: nc.description, content: nc.content, }, opts.sessionId); nameToId.set(node.name, node.id); upserted.push(node); + if (!isNew) updatedExisting += 1; opts.onNodeUpserted?.(node); } @@ -63,10 +72,8 @@ export async function persistExtractionResult( let edgeCount = 0; for (const ec of result.edges) { - const fromNode = await findByName(driver, ec.from); - const toNode = await findByName(driver, ec.to); - const fromId = nameToId.get(ec.from) ?? fromNode?.id; - const toId = nameToId.get(ec.to) ?? toNode?.id; + const fromId = nameToId.get(ec.from) ?? (await findByName(driver, ec.from))?.id; + const toId = nameToId.get(ec.to) ?? (await findByName(driver, ec.to))?.id; if (fromId && toId) { await upsertEdge(driver, { fromId, toId, type: ec.type, @@ -76,5 +83,5 @@ export async function persistExtractionResult( } } - return { nodes: upserted, edges: edgeCount }; + return { nodes: upserted, edges: edgeCount, updatedExisting }; } diff --git a/src/extractor/turn-filter.ts b/src/extractor/turn-filter.ts index a619a3a..f958640 100644 --- a/src/extractor/turn-filter.ts +++ b/src/extractor/turn-filter.ts @@ -10,8 +10,9 @@ * 3. 清洗后长度 ≤ trivialMaxChars（默认 5）且不含技术词 * （连续 ≥3 位字母数字，如 pnpm/jwt/k8s——这类短输入仍走 LLM）。 * - * 只看 user 角色文本；工具结果与 assistant 回复不参与（它们跟随用户意图， - * 用户输入有意义时整轮照常提取）。 + * 判定"文本"只取 user 角色（工具结果与 assistant 回复的内容不参与），但轮内只要 + * 存在工具劳动（tool/toolResult 角色，见 turnHasToolWork）就不判 trivial—— + * "继续"触发的一轮真实修复劳动恰是图谱最该吸收的知识。 */ /** 清洗：去空白与中西文标点，转小写。用于词表精确匹配与长度计量。 */ @@ -65,3 +66,17 @@ export function shouldSkipTurnExtraction(userText: string, opts?: TrivialFilterO return false; } + +/** + * 轮内是否含工具劳动（tool / toolResult 角色）。这类轮即使 user 文本命中 + * trivial 词表也不应跳过提取："继续"触发的一轮真实修复劳动恰是可提取知识， + * 且 per-turn 路径没有自动补提触发点——extracted 一旦标记，只有手动重置 + * + `graph-memory extract` 才能回挖。 + */ +export function turnHasToolWork(messages: readonly unknown[]): boolean { + return (messages ?? []).some((m) => { + if (!m || typeof m !== "object") return false; + const role = (m as { role?: unknown }).role; + return role === "tool" || role === "toolResult"; + }); +} diff --git a/src/types.ts b/src/types.ts index 66f18e5..c2d93df 100755 --- a/src/types.ts +++ b/src/types.ts @@ -279,6 +279,8 @@ export interface ExtractConfig { * 本地预筛阈值：用户输入清洗（去空白/标点）后长度 ≤ 该值且不含技术词 * （连续 ≥3 位字母数字，如 pnpm/jwt）时，跳过 LLM 提取直接标记。 * 保守默认 5（中文 5 字以内基本不可能承载可提取知识）。 + * 例外：轮内含工具劳动（tool/toolResult 角色）时不判 trivial——"继续"触发的 + * 一轮真实修复劳动恰是可提取知识，不误杀（见 turn-filter.ts 的 turnHasToolWork）。 */ trivialMaxChars?: number; /** 额外无意义词表（与内置表合并，清洗后小写精确匹配，如 "继续"、"resume"）。 */ diff --git a/test/persist.test.ts b/test/persist.test.ts new file mode 100644 index 0000000..9f4c276 --- /dev/null +++ b/test/persist.test.ts @@ -0,0 +1,151 @@ +/** + * persistExtractionResult 单测 — 提取结果持久化单一来源 + * + * 覆盖： + * - updatedExisting 统计（isNew=false 的 upsert 命中 → finalize 阶梯第二触发条件的信号源） + * - 边端点解析的 Map-first 顺序：nameToId 命中时零 findByName 往返 + * - nameToId 未命中时回源 findByName（键是规范化名，LLM 原文未必规范化） + * - awaitEmbedSync 开关（CLI 必须等待 vs 运行时 fire-and-forget） + */ + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + upsertNode: vi.fn(), + upsertEdge: vi.fn(async () => true), + // 返回类型放宽为 any：用例会分别模拟 null（未命中）与节点对象（回源命中） + findByName: vi.fn(async (_d: unknown, _name: unknown): Promise => null), +})); + +vi.mock("../src/store/store.ts", () => ({ + upsertNode: mocks.upsertNode, + upsertEdge: mocks.upsertEdge, + findByName: mocks.findByName, +})); + +import { persistExtractionResult } from "../src/extractor/persist.ts"; + +function mkNode(name: string) { + return { + id: `n-${name}`, + type: "SKILL" as const, + name, + description: "d", + content: "c", + status: "active" as const, + validatedCount: 1, + sourceSessions: [] as string[], + communityId: null, + pagerank: 0, + createdAt: 0, + updatedAt: 0, + }; +} + +const recaller = { syncEmbedBatch: async () => {} }; +const driver = {} as any; + +describe("persistExtractionResult", () => { + beforeEach(() => { + mocks.upsertNode.mockReset(); + mocks.upsertEdge.mockReset(); + mocks.findByName.mockReset(); + mocks.upsertEdge.mockResolvedValue(true); + mocks.findByName.mockResolvedValue(null); + }); + + it("统计 updatedExisting：isNew=false 的 upsert 计入，isNew=true 不计", async () => { + mocks.upsertNode + .mockResolvedValueOnce({ node: mkNode("fresh-node"), isNew: true }) + .mockResolvedValueOnce({ node: mkNode("existing-node"), isNew: false }); + + const outcome = await persistExtractionResult(driver, recaller, { + nodes: [ + { type: "TASK", name: "fresh-node", description: "d", content: "c" }, + { type: "SKILL", name: "existing-node", description: "d", content: "c" }, + ], + edges: [], + }, { sessionId: "s1" }); + + expect(outcome.nodes).toHaveLength(2); + expect(outcome.updatedExisting).toBe(1); + expect(outcome.edges).toBe(0); + }); + + it("边端点优先查 nameToId：命中时零 findByName 往返", async () => { + mocks.upsertNode.mockResolvedValue({ node: mkNode("deploy-task"), isNew: true }); + + const outcome = await persistExtractionResult(driver, recaller, { + nodes: [{ type: "TASK", name: "deploy-task", description: "d", content: "c" }], + edges: [ + { from: "deploy-task", to: "deploy-task", type: "USED_SKILL", instruction: "self" }, + { from: "deploy-task", to: "missing", type: "USED_SKILL", instruction: "unresolved" }, + ], + }, { sessionId: "s2" }); + + expect(outcome.edges).toBe(1); + // from 命中索引；to 未命中且 findByName 返回 null → 该边被丢弃 + expect(mocks.findByName).toHaveBeenCalledTimes(1); + expect(mocks.findByName).toHaveBeenCalledWith(driver, "missing"); + }); + + it("nameToId 未命中时回源 findByName（LLM 原文未必是规范化名）", async () => { + mocks.upsertNode.mockResolvedValue({ node: mkNode("skill-b"), isNew: true }); + mocks.findByName.mockResolvedValueOnce(mkNode("skill-b")); + + const outcome = await persistExtractionResult(driver, recaller, { + nodes: [{ type: "SKILL", name: "skill-b", description: "d", content: "c" }], + edges: [{ from: "Skill B", to: "skill-b", type: "REQUIRES", instruction: "i" }], + }, { sessionId: "s3" }); + + // from="Skill B" 未命中索引 → 回源 1 次；to="skill-b" 命中索引 → 零往返 + expect(mocks.findByName).toHaveBeenCalledTimes(1); + expect(mocks.findByName).toHaveBeenCalledWith(driver, "Skill B"); + expect(outcome.edges).toBe(1); + }); + + it("awaitEmbedSync=true 时等待向量同步完成才返回", async () => { + let release!: () => void; + const gate = new Promise((r) => { release = r; }); + const slowRecaller = { syncEmbedBatch: vi.fn(() => gate) }; + + mocks.upsertNode.mockResolvedValue({ node: mkNode("n"), isNew: true }); + + const pending = persistExtractionResult(driver, slowRecaller, { + nodes: [{ type: "SKILL", name: "n", description: "d", content: "c" }], + edges: [], + }, { sessionId: "s4", awaitEmbedSync: true }); + + // 排空微任务让 persist 走到 syncEmbedBatch 调用点（不刷宏任务，保持确定性） + for (let i = 0; i < 20; i++) await Promise.resolve(); + + // 同步已启动但未完成 → persist 仍挂起 + expect(slowRecaller.syncEmbedBatch).toHaveBeenCalledTimes(1); + let settled = false; + void pending.then(() => { settled = true; }); + await Promise.resolve(); + expect(settled).toBe(false); + + release(); + await pending; + }); + + it("awaitEmbedSync 缺省（运行时路径）时 fire-and-forget：不等待同步即返回", async () => { + let release!: () => void; + const gate = new Promise((r) => { release = r; }); + const slowRecaller = { syncEmbedBatch: vi.fn(() => gate) }; + + mocks.upsertNode.mockResolvedValue({ node: mkNode("n"), isNew: true }); + + // 同步 promise 永不 resolve，persist 仍应正常返回（fire-and-forget） + const outcome = await persistExtractionResult(driver, slowRecaller, { + nodes: [{ type: "SKILL", name: "n", description: "d", content: "c" }], + edges: [], + }, { sessionId: "s5" }); + + expect(slowRecaller.syncEmbedBatch).toHaveBeenCalledTimes(1); + expect(outcome.nodes).toHaveLength(1); + release(); + await slowRecaller.syncEmbedBatch.mock.results[0]!.value; + }); +}); diff --git a/test/turn-filter.test.ts b/test/turn-filter.test.ts index 0f037ba..6cdc338 100644 --- a/test/turn-filter.test.ts +++ b/test/turn-filter.test.ts @@ -10,7 +10,7 @@ import { describe, it, expect } from "vitest"; import { - normalizeTrivialText, shouldSkipTurnExtraction, BUILTIN_TRIVIAL_PROMPTS, + normalizeTrivialText, shouldSkipTurnExtraction, turnHasToolWork, BUILTIN_TRIVIAL_PROMPTS, } from "../src/extractor/turn-filter.ts"; import { turnUserText } from "../index.ts"; @@ -100,3 +100,41 @@ describe("turnUserText — 轮级 user 文本聚合", () => { expect(shouldSkipTurnExtraction(turnUserText([{ role: "assistant", content: "继续" }]))).toBe(true); }); }); + +describe("turnHasToolWork — 工具劳动守卫（trivial 预筛的误杀保险）", () => { + it("含 tool / toolResult 角色 → true", () => { + expect(turnHasToolWork([ + { role: "user", content: "继续" }, + { role: "assistant", content: [{ type: "toolUse", id: "t1", name: "fix" }] }, + { role: "tool", content: "patched" }, + ])).toBe(true); + expect(turnHasToolWork([ + { role: "user", content: "继续" }, + { role: "toolResult", content: [{ type: "text", text: "done" }] }, + ])).toBe(true); + }); + + it("纯 user/assistant 对话 → false", () => { + expect(turnHasToolWork([ + { role: "user", content: "继续" }, + { role: "assistant", content: "好的" }, + ])).toBe(false); + expect(turnHasToolWork([])).toBe(false); + }); + + it("畸形消息不炸", () => { + expect(turnHasToolWork([null, undefined, "str", 42, { role: 123 }, {}])).toBe(false); + }); + + it("组合语义：user 文本命中词表 + 轮内有工具劳动 → 整轮照常提取", () => { + const messages = [ + { role: "user", content: "继续" }, + { role: "assistant", content: [{ type: "toolUse", id: "t1", name: "bash" }] }, + { role: "toolResult", content: "bug fixed, tests green" }, + ]; + // 单独看 user 文本命中词表…… + expect(shouldSkipTurnExtraction(turnUserText(messages))).toBe(true); + // ……但轮内有工具劳动，预筛必须让路 + expect(turnHasToolWork(messages)).toBe(true); + }); +}); From 82cf3c5eba4e34cb5f5eb912389b62956e29ea08 Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Sun, 6 Sep 2026 00:04:04 +0000 Subject: [PATCH 29/29] =?UTF-8?q?chore:=20=E4=BB=A3=E7=A0=81=E8=B4=A8?= =?UTF-8?q?=E9=87=8F=E6=8F=90=E9=AB=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- index.ts | 74 ++++++++++++++++++++--------------- src/cli-extract.ts | 23 +---------- src/cli-io.ts | 26 ++++++++++++ src/cli-reembed.ts | 22 +---------- src/engine/embed.ts | 6 --- src/engine/llm.ts | 10 +++++ src/extractor/extract.ts | 7 ++-- src/graph/community.ts | 5 +-- src/store/db.ts | 4 +- src/store/store.ts | 1 - test/cli-extract.test.ts | 1 - test/commit-turn.test.ts | 1 - test/register-guard.test.ts | 1 - test/session-identity.test.ts | 1 - 14 files changed, 90 insertions(+), 92 deletions(-) create mode 100644 src/cli-io.ts diff --git a/index.ts b/index.ts index 9dbceb4..a5cd5c9 100755 --- a/index.ts +++ b/index.ts @@ -29,7 +29,7 @@ import { assembleContext } from "./src/format/assemble.ts"; import { sanitizeToolUseResultPairing } from "./src/format/transcript-repair.ts"; import { runMaintenance } from "./src/graph/maintenance.ts"; import { normalizeMessageRetentionPolicy } from "./src/store/retention.ts"; -import { DEFAULT_CONFIG, DEFAULT_CRON_CONFIG, isCronSessionKey, EDGE_TYPES, type GmConfig, type RecallResult, type EdgeType } from "./src/types.ts"; +import { DEFAULT_CONFIG, DEFAULT_CRON_CONFIG, isCronSessionKey, EDGE_TYPES, type GmConfig, type RecallResult, type EdgeType, type ExtractionResult } from "./src/types.ts"; import { registerCrudRoutes } from "./src/routes/crud.ts"; import { createGraphMemoryCli } from "./src/cli.ts"; @@ -90,16 +90,29 @@ export function readDefaultModel(apiConfig: unknown): string { return raw; } -function throwNodeNotFound(name: string): never { - throw new Error( +/** 节点未找到报错文案的单源（throwNodeNotFound / gm_update notFoundHint 共用）。 */ +function nodeNotFoundMessage( + name: string, + tail: string = "或使用 gm_search 搜索已有节点。", +): string { + return ( `[graph-memory-pro] 未找到名称为 "${name}" 的节点。` + `请检查节点名称是否精确（名称标准化规则：全小写、空格/下划线转连字符、移除非字母数字字符），` + - `或使用 gm_search 搜索已有节点。`, + tail ); } +function throwNodeNotFound(name: string, tail?: string): never { + throw new Error(nodeNotFoundMessage(name, tail)); +} + // ─── 清洗 OpenClaw metadata 包装 ───────────────────────────── +/** 剥离 OpenClaw 注入的命令前缀与时间戳标记（cleanPrompt / extractUserText 共用，逐字等价）。 */ +function stripCommandAndTimestampPrefix(s: string): string { + return s.replace(/^\/\w+\s+/, "").trim().replace(/^\[[\w\s\-:]+\]\s*/, "").trim(); +} + export function cleanPrompt(raw: string): string { let prompt = raw.trim(); if (prompt.includes("Sender (untrusted metadata)")) { @@ -113,9 +126,7 @@ export function cleanPrompt(raw: string): string { prompt = lines.join("\n").trim(); } } - prompt = prompt.replace(/^\/\w+\s+/, "").trim(); - prompt = prompt.replace(/^\[[\w\s\-:]+\]\s*/, "").trim(); - return prompt; + return stripCommandAndTimestampPrefix(prompt); } // ─── 规范化消息 content，防 OpenClaw content.filter() 崩溃 ──── @@ -195,9 +206,7 @@ export function extractUserText(msg: any): string { if (fenceEnd >= 0 && raw.includes("Sender")) { raw = raw.slice(fenceEnd + 3).trim(); } - raw = raw.replace(/^\/\w+\s+/, "").trim(); - raw = raw.replace(/^\[[\w\s\-:]+\]\s*/, "").trim(); - return raw; + return stripCommandAndTimestampPrefix(raw); } /** @@ -1258,24 +1267,25 @@ const graphMemoryProPlugin = { const fin = await extractor.finalize({ sessionNodes: nodes, graphSummary: summary }); - for (const nc of fin.promotedSkills) { - if (nc.name && nc.content) { - await upsertNode(driver, { - type: "SKILL", name: nc.name, - description: nc.description ?? "", content: nc.content, - }, sid); - } - } - for (const ec of fin.newEdges) { - const fromNode = await findByName(driver, ec.from); - const toNode = await findByName(driver, ec.to); - if (fromNode && toNode) { - await upsertEdge(driver, { - fromId: fromNode.id, toId: toNode.id, type: ec.type, - instruction: ec.instruction, sessionId: sid, - }); - } - } + // promotedSkills + newEdges 收敛进 persistExtractionResult（单一来源）： + // 旧内联循环是全库唯一不做向量同步的节点写路径 —— 晋升 SKILL 无 embedding， + // 精确召回不可见且无自动补向量机制；边端点解析也未复用 nameToId 零往返模式。 + // 收敛后 upsert → syncEmbedBatch → 建边与 per-turn/compact/CLI 路径完全同源。 + // persistExtractionResult 不做 markExtracted（finalize 无消息语义），无需补。 + const finResult: ExtractionResult = { + nodes: fin.promotedSkills + // parseFinalize 已过滤 name/content，这里保留双保险（与原内联循环一致） + .filter((nc) => nc.name && nc.content) + .map((nc) => ({ + type: "SKILL", + name: nc.name, + description: nc.description ?? "", + content: nc.content, + })), + edges: fin.newEdges, + }; + await persistExtractionResult(driver, recaller, finResult, { sessionId: sid }); + for (const id of fin.invalidations) await deprecateNodeAndDisconnectById(driver, id); }); } else if (nodes.length) { @@ -1440,10 +1450,10 @@ const graphMemoryProPlugin = { }, ) { const mode = p.mode ?? "update"; - const notFoundHint = - `[graph-memory-pro] 未找到名称为 "${p.name}" 的节点。` + - `请检查节点名称是否精确（名称标准化规则：全小写、空格/下划线转连字符、移除非字母数字字符），` + - `或使用 gm_record 创建新节点，也可用 gm_search 搜索已有节点。`; + const notFoundHint = nodeNotFoundMessage( + p.name, + "或使用 gm_record 创建新节点，也可用 gm_search 搜索已有节点。", + ); // mode=delete 已移除（断联弃用等效删除）——为旧调用方保留明确报错而非静默降级为 update if ((mode as string) === "delete") { diff --git a/src/cli-extract.ts b/src/cli-extract.ts index efd8664..f1b606f 100644 --- a/src/cli-extract.ts +++ b/src/cli-extract.ts @@ -9,9 +9,6 @@ * 完成 Neo4j driver / schema / LLM / embedder / Extractor / Recaller 的初始化。 */ -import readline from "node:readline/promises"; -import { stdin as input, stdout as output } from "node:process"; - import type { Driver } from "neo4j-driver"; import type { GmConfig } from "./types.ts"; import { getDriver, initSchema, closeDriver } from "./store/db.ts"; @@ -27,6 +24,7 @@ import { createEmbedder } from "./engine/embed.ts"; import { Recaller } from "./recaller/recall.ts"; import { Extractor } from "./extractor/extract.ts"; import { persistExtractionResult } from "./extractor/persist.ts"; +import { defaultLog, makeDefaultPrompt } from "./cli-io.ts"; const AFFIRMATIVE = new Set(["y", "yes", "yeah", "yep", "ok", "okay", "true", "1", "confirm"]); @@ -61,10 +59,6 @@ export interface BackfillExtractResult { const DEFAULT_BATCH_LIMIT_MULTIPLIER = 3; -function defaultLog(msg: string): void { - console.log(msg); -} - function formatSessionLine(info: UnextractedSessionInfo, index: number): string { const created = info.minCreatedAt > 0 ? new Date(info.minCreatedAt).toISOString().replace("T", " ").slice(0, 19) @@ -163,7 +157,7 @@ export async function runBackfillExtraction( } if (!opts.yes) { - const prompt = params.prompt ?? ((q: string) => defaultPrompt(q)); + const prompt = params.prompt ?? makeDefaultPrompt("GRAPH_MEMORY_EXTRACT_CONFIRM"); const answer = await prompt(`\n将对以上 ${sessions.length} 个会话发起 LLM 提取，继续？[y/N] `); if (!isAffirmative(answer)) { log("[graph-memory-pro] 已取消。"); @@ -260,16 +254,3 @@ async function extractSessionLoop( return stats; } - -async function defaultPrompt(question: string): Promise { - if (!process.stdin.isTTY && process.env.GRAPH_MEMORY_EXTRACT_CONFIRM === undefined) { - return ""; - } - const rl = readline.createInterface({ input, output }); - try { - const answer = await rl.question(question); - return answer; - } finally { - rl.close(); - } -} diff --git a/src/cli-io.ts b/src/cli-io.ts new file mode 100644 index 0000000..692a3eb --- /dev/null +++ b/src/cli-io.ts @@ -0,0 +1,26 @@ +/** + * graph-memory-pro CLI — 共享终端 IO（log / 确认提示）。 + * extract 与 reembed 子命令共用；确认提示以环境变量名参数化—— + * 非交互 stdin 且该环境变量未设置时返回空串（视为拒绝），与两命令原语义一致。 + */ + +import readline from "node:readline/promises"; +import { stdin as input, stdout as output } from "node:process"; + +export function defaultLog(msg: string): void { + console.log(msg); +} + +export function makeDefaultPrompt(confirmEnvVar: string): (question: string) => Promise { + return async (question: string): Promise => { + if (!process.stdin.isTTY && process.env[confirmEnvVar] === undefined) { + return ""; + } + const rl = readline.createInterface({ input, output }); + try { + return await rl.question(question); + } finally { + rl.close(); + } + }; +} diff --git a/src/cli-reembed.ts b/src/cli-reembed.ts index a4c158e..59ea2d0 100644 --- a/src/cli-reembed.ts +++ b/src/cli-reembed.ts @@ -14,9 +14,6 @@ * Neo4j driver / schema / embedder 的初始化，并在 finally 中 closeDriver。 */ -import readline from "node:readline/promises"; -import { stdin as input, stdout as output } from "node:process"; - import type { Driver } from "neo4j-driver"; import type { GmConfig } from "./types.ts"; import { getDriver, initSchema, closeDriver } from "./store/db.ts"; @@ -33,6 +30,7 @@ import { import { createEmbedder, type Embedder } from "./engine/embed.ts"; import { buildNodeEmbeddingText } from "./recaller/recall.ts"; import { isAffirmative } from "./cli-extract.ts"; +import { defaultLog, makeDefaultPrompt } from "./cli-io.ts"; export const DEFAULT_REEMBED_BATCH = 32; const MAX_REEMBED_BATCH = 256; @@ -108,22 +106,6 @@ function clampBatch(batch: number | undefined): number { return Math.min(Math.floor(batch), MAX_REEMBED_BATCH); } -function defaultLog(msg: string): void { - console.log(msg); -} - -async function defaultPrompt(question: string): Promise { - if (!process.stdin.isTTY && process.env.GRAPH_MEMORY_REEMBED_CONFIRM === undefined) { - return ""; - } - const rl = readline.createInterface({ input, output }); - try { - return await rl.question(question); - } finally { - rl.close(); - } -} - /** 单批嵌入：优先批量调用，失败退化为逐条请求（兼容批量响应结构未知的 provider） */ async function embedTexts( embedder: Embedder, @@ -231,7 +213,7 @@ export async function runReembed(params: ReembedParams): Promise } if (!opts.yes) { - const prompt = params.prompt ?? ((q: string) => defaultPrompt(q)); + const prompt = params.prompt ?? makeDefaultPrompt("GRAPH_MEMORY_REEMBED_CONFIRM"); const answer = await prompt( `\n将清除现有节点/社区向量并用当前模型重建（每批 ${batch} 条），继续？[y/N] `, ); diff --git a/src/engine/embed.ts b/src/engine/embed.ts index ecaae53..bfc603b 100755 --- a/src/engine/embed.ts +++ b/src/engine/embed.ts @@ -180,9 +180,3 @@ export async function createEmbedder(cfg: EmbeddingConfig | undefined): Promise< return null; } } - -/** 兼容包装：只需单发 embed 的调用方（index.ts / cli-extract.ts） */ -export async function createEmbedFn(cfg: EmbeddingConfig | undefined): Promise { - const embedder = await createEmbedder(cfg); - return embedder?.embed ?? null; -} diff --git a/src/engine/llm.ts b/src/engine/llm.ts index 976bcd6..36b7a98 100755 --- a/src/engine/llm.ts +++ b/src/engine/llm.ts @@ -64,6 +64,16 @@ interface LlmConfig { export type CompleteFn = (system: string, user: string) => Promise; +/** + * 剥离推理模型输出的 ... 思维链标签（兼容 MiniMax 等）， + * 含未闭合 兜底。LLM 输出清洗的单一来源——extractor 与社区摘要共用。 + */ +export function stripThinkTags(raw: string): string { + return raw + .replace(/[\s\S]*?<\/think>/gi, "") + .replace(/[\s\S]*/gi, ""); +} + const DEFAULT_LLM_TIMEOUT_MS = 60_000; const DEFAULT_LLM_MAX_TOKENS = 4_000; const ANTHROPIC_DEFAULT_BASE_URL = "https://api.anthropic.com"; diff --git a/src/extractor/extract.ts b/src/extractor/extract.ts index 41276dc..759dcc4 100755 --- a/src/extractor/extract.ts +++ b/src/extractor/extract.ts @@ -8,7 +8,7 @@ import type { ExtractionResult, FinalizeResult } from "../types.ts"; import { EDGE_TYPES, isValidEdgeDirection } from "../types.ts"; import type { GmNode } from "../types.ts"; -import type { CompleteFn } from "../engine/llm.ts"; +import { stripThinkTags, type CompleteFn } from "../engine/llm.ts"; import { normalizeName } from "../store/store.ts"; // ─── 节点/边合法值 ────────────────────────────────────────────── @@ -315,9 +315,8 @@ export class Extractor { function extractJson(raw: string): string { let s = raw.trim(); - // 清理 ... 思维链标签（兼容 MiniMax 等模型） - s = s.replace(/[\s\S]*?<\/think>/gi, ""); - s = s.replace(/[\s\S]*/gi, ""); // 未闭合的 + // 清理 ... 思维链标签（兼容 MiniMax 等模型）——单一来源 stripThinkTags + s = stripThinkTags(s); s = s.replace(/^```(?:json)?\s*\n?/i, "").replace(/\n?\s*```\s*$/i, ""); s = s.trim(); if (s.startsWith("{") && s.endsWith("}")) return s; diff --git a/src/graph/community.ts b/src/graph/community.ts index 6238641..805583c 100755 --- a/src/graph/community.ts +++ b/src/graph/community.ts @@ -18,6 +18,7 @@ import { pruneCommunitySummaries, } from "../store/store.ts"; import { getExistingActiveRelTypes, projectActiveGraph } from "./projection.ts"; +import { stripThinkTags } from "../engine/llm.ts"; export interface CommunityResult { labels: Map; @@ -251,9 +252,7 @@ export async function summarizeCommunities( `社区成员：\n${memberText}`, ); - const cleaned = summary.trim() - .replace(/[\s\S]*?<\/think>/gi, "") - .replace(/[\s\S]*/gi, "") + const cleaned = stripThinkTags(summary.trim()) .replace(/^["'「」]|["'「」]$/g, "") .replace(/\n/g, " ") .replace(/\s{2,}/g, " ") diff --git a/src/store/db.ts b/src/store/db.ts index bdbbe85..a5f84ab 100755 --- a/src/store/db.ts +++ b/src/store/db.ts @@ -76,7 +76,9 @@ export async function initSchema(driver: Driver, embedding?: EmbeddingConfig): P : 1024; // The search code queries one index across all knowledge labels. - await session.run("MATCH (n:Task|Skill|Event) SET n:MemoryNode"); + // 补标守卫：仅给缺失 MemoryNode 标签的节点补标 —— 常规启动退化为纯读扫描， + // 避免每次启动对全库做无谓写（属性/标签写会触发事务与日志）；首启/迁移兜底语义不变。 + await session.run("MATCH (n:Task|Skill|Event) WHERE NOT n:MemoryNode SET n:MemoryNode"); // 存量 deprecated 节点补写 deprecatedAt（幂等，等效一次性迁移）：purge 时钟基准是 // deprecatedAt、缺失时回退 updatedAt——但 upsertNode 对 manual/merge 弃用节点只 bump diff --git a/src/store/store.ts b/src/store/store.ts index c9bfe4c..f9a7176 100755 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -1224,7 +1224,6 @@ export async function getUnextracted(driver: Driver, sid: string, limit: number) return { role: m.role, content: JSON.parse(m.content), - turnIndex: toInt(m.turnIndex), turn_index: toInt(m.turnIndex), }; }); diff --git a/test/cli-extract.test.ts b/test/cli-extract.test.ts index 5922b71..9ea08ad 100644 --- a/test/cli-extract.test.ts +++ b/test/cli-extract.test.ts @@ -52,7 +52,6 @@ vi.mock("../src/engine/llm.ts", () => ({ })); vi.mock("../src/engine/embed.ts", () => ({ - createEmbedFn: async () => null, createEmbedder: async () => null, })); diff --git a/test/commit-turn.test.ts b/test/commit-turn.test.ts index f11a902..c115353 100644 --- a/test/commit-turn.test.ts +++ b/test/commit-turn.test.ts @@ -36,7 +36,6 @@ vi.mock("../src/engine/llm.ts", () => ({ })); vi.mock("../src/engine/embed.ts", () => ({ - createEmbedFn: async () => null, createEmbedder: async () => null, })); diff --git a/test/register-guard.test.ts b/test/register-guard.test.ts index f7a3f47..fbd672c 100644 --- a/test/register-guard.test.ts +++ b/test/register-guard.test.ts @@ -42,7 +42,6 @@ vi.mock("../src/engine/llm.ts", async (importActual) => { }); vi.mock("../src/engine/embed.ts", () => ({ - createEmbedFn: async () => null, createEmbedder: async () => null, })); diff --git a/test/session-identity.test.ts b/test/session-identity.test.ts index 572ca93..5da274c 100644 --- a/test/session-identity.test.ts +++ b/test/session-identity.test.ts @@ -54,7 +54,6 @@ vi.mock("../src/engine/llm.ts", () => ({ })); vi.mock("../src/engine/embed.ts", () => ({ - createEmbedFn: async () => null, createEmbedder: async () => null, }));