diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8207acb..6f04e6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,6 +91,12 @@ jobs: run: npm test env: NEO4J_INTEGRATION: "1" + # 集成测试（尤其删除类的 retention 测试）要求显式指定 URI； + # runner 上的 service container 是一次性实例，7687 安全 + NEO4J_TEST_URI: bolt://localhost:7687 + # 与 service 的 NEO4J_AUTH 保持同一来源，供可配置凭据的测试使用。 + NEO4J_TEST_USER: neo4j + NEO4J_TEST_PASSWORD: graphmemory CI: "true" shellcheck: diff --git a/README.md b/README.md index fbf1c23..0295822 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ The installer configures Neo4j to start at boot with a 3-tier no-sudo fallback: ## Manual Configuration -Install the local plugin, then make it the OpenClaw context engine: +Install the local plugin, then make it the OpenClaw context engine (**restart the gateway after changing config**: the plugin guards against duplicate `register()` — a second registration without `dispose()` reuses the active engine, so new config is never hot-reloaded): ```json { @@ -108,6 +108,99 @@ Anthropic direct (Claude) — drop `baseURL`, switch `provider`: `embedding` is optional. When present, `dimensions` must match the Neo4j vector index dimension. For a fresh database, the plugin creates matching indexes during startup. If you change dimensions later, recreate the vector indexes or the Neo4j database. +### Memory decay (forgetting curve) + +Each maintenance cycle scores every active node with a three-factor weighted model (recency + frequency + intrinsic) and bidirectionally transitions nodes across three tiers: `core` / `working` / `peripheral`. Two lifecycle stages sit on top of the forgetting curve (both on by default, configurable via `decay`): **auto-deprecation** — a `peripheral` node whose composite stays below `peripheralCompositeThreshold` and which has not been accessed for `autoDeprecateAfterDays` (30) days gets its edges severed and is marked `deprecated`; if the same knowledge is extracted or edited again, the node automatically revives to `active`. **purge** — any `deprecated` node (manual deprecation and merge losers included) is hard-deleted (`DETACH DELETE`, vectors included) after `purgeAfterDays` (60) days to reclaim storage. + +The full formula, field mapping from the reference implementation, default-value rationale, and tuning guide live in **[`docs/decay.md`](docs/decay.md)**. + +Minimal config (all fields optional, defaults shown): + +```json +"decay": { "enabled": true } +``` + +Common overrides — for fuller control see `docs/decay.md` §4: + +```json +"decay": { + "enabled": true, + "recencyHalfLifeDays": 30, + "peripheralCompositeThreshold": 0.15, + "workingAccessThreshold": 3 +} +``` + +### Cron sessions + +Sessions created by OpenClaw scheduled tasks can be configured independently of normal sessions. The host places the cron marker on the **sessionKey** (`sessionId` is a random UUID); real shapes are `cron:`, `agent::cron:`, or `agent::cron::run:`: + +```json +"cron": { + "enabled": true, + "extract": true, + "finalizeAndMaintain": true +} +``` + +| Option | Default | Description | +| --- | --- | --- | +| `enabled` | `true` | Enable graph functionality inside cron sessions (recall injection + message buffering). When `false`, cron sessions skip automatic recall and message persistence; the `gm_*` tools remain available for explicit calls (manual escape hatch). | +| `extract` | `true` | Trigger knowledge extraction (LLM triples) in cron sessions via `afterTurn` / `compact`. When `false`, messages are still buffered and can be backfilled later with `openclaw graph-memory extract`. | +| `finalizeAndMaintain` | `true` | Run finalize (EVENT→SKILL promotion) and graph maintenance (decay / PageRank / communities) when a cron session ends. Disable when frequent cron runs make end-of-session global maintenance too costly. | + +All three options default to **`true`**: cron sessions behave like normal sessions (recall, buffering, extraction, and end-of-session maintenance all enabled) unless explicitly disabled. `enabled: false` is the master switch — even with `extract` / `finalizeAndMaintain` set to `true`, nothing runs. Non-cron sessions are never affected by these options. + +All three sub-options are optional; omitted fields keep the default `true` (e.g. with `"cron": { "extract": false }` only extraction is disabled — recall, buffering, and end-of-session maintenance stay on). + +Caveat: when a cron job sets an explicit custom `sessionKey`, the host does not append the `cron` segment — such sessions cannot be detected and are treated as normal sessions. + +### Raw message retention (messageRetention, opt-in) + +Context compaction only changes what the model sees — it is **not authorization to delete persistent evidence**. The default `keep=all` never deletes any raw message (GmMessage) at zero overhead. To bound database growth, opt into bounded pruning; it runs at the tail of the graph maintenance chain and processes at most `batchSize` rows per cycle: + +```json +"messageRetention": { + "keep": "referenced", + "batchSize": 500, + "dryRun": false +} +``` + +| Option | Default | Description | +| --- | --- | --- | +| `keep` | `"all"` | `all` = keep everything (default, zero behavior change); `referenced` = delete only messages that were extracted **and** actually produced knowledge; `recent` = `referenced` plus a time-window guard (requires at least one window option). | +| `recentTurns` | `0` | `keep=recent`: keep the newest N real user turns per session (that turn and everything after it). Sessions with no user messages are fully protected. | +| `retentionDays` | `0` | `keep=recent`: keep messages ingested within the last N days. | +| `batchSize` | `500` | Maximum rows processed per maintenance cycle (1–10000), keeping the chain bounded. | +| `dryRun` | `false` | `true` reports the candidate set without deleting — **run one `dryRun` cycle to validate candidates before enabling pruning**. | + +Deletion semantics (fail-closed): + +- Only messages with `extracted=true` **and** `producedKnowledge=true` (the extraction actually produced nodes/edges) enter the candidate set. +- Turns where the LLM returned zero nodes and zero edges are marked `producedKnowledge=false` — the raw evidence stays in the database. Such turns are never re-extracted automatically; to re-mine them with a better prompt/model, manually reset their `extracted` flag to `false` and re-run `openclaw graph-memory extract`. +- Unextracted messages and legacy rows (extracted before this flag existed, no `producedKnowledge` property) are never deleted. +- A failure in the retention step itself (invalid policy fails closed with zero deletions / Neo4j error) never invalidates the other maintenance steps; it retries next cycle. + +### Re-embedding after switching embedding models + +Embedding vectors are model-specific — vectors produced by the old model are not comparable +(different dimensions break dedup and vector search outright), and vectors are write-once with +no automatic migration. After changing `embedding.model` / `embedding.dimensions`, run: + +```bash +openclaw graph-memory reembed --dry-run # report dimension match + vector coverage, no writes +openclaw graph-memory reembed # void all vectors and rebuild them in batches +``` + +The command voids every `MemoryNode.embedding` (and its `contentHash`, so the runtime +`syncEmbed` hash guard cannot short-circuit), then re-embeds all active nodes and community +summaries with the current model using batched requests (`--batch `, default 32; failed +batches automatically fall back to per-item requests). If the vector index dimensions no longer +match the configured model, the run aborts — add `--recreate-index` to drop and recreate +`gm_node_embedding` / `gm_community_embedding` with the new dimensions. Items that fail stay +vectorless and are picked up by a re-run. + ### OAuth login (experimental) ```bash @@ -127,7 +220,7 @@ conversation messages -> GmMessage nodes -> LLM triple extraction -> embeddings -> vector recall + community expansion + GDS PPR -> XML context injection -session end -> dedup -> global PageRank -> communities -> summaries +session end -> decay (forgetting curve) -> dedup -> global PageRank -> communities -> summaries ``` ## Verify @@ -158,18 +251,18 @@ Inspect the graph with the bundled Cypher shell: | --- | --- | | `gm_search` | Recall graph knowledge for a query | | `gm_record` | Add a knowledge node manually | -| `gm_update` | Update, delete, or deprecate an existing node by exact name. `mode=update` (default) refines description/content; `mode=delete` hard-deletes the node and all its relationships; `mode=deprecate` marks `[DEPRECATED]` and removes all relationships while keeping the node (throws if not found) | +| `gm_update` | Update or deprecate an existing node by exact name. `mode=update` (default) refines description/content; `mode=deprecate` marks `[DEPRECATED]`, removes all relationships, and makes the node unreachable from recall (equivalent to deletion; physically purged by maintenance after `purgeAfterDays`). `mode=delete` was removed — deprecate replaces it (throws if not found) | | `gm_link` | Manually create or refine an edge between two existing nodes (validates type + direction against the whitelist; idempotent on from+to+type) | | `gm_unlink` | Remove edges between two nodes by name; optional `type` filter, otherwise all from→to edges | | `gm_merge` | Merge two same-type duplicate nodes: keep absorbs content/validatedCount/sessions + dedup-aware edge migration; merge is soft-deleted | | `gm_stats` | Show node, relationship, community, and PageRank statistics | -| `gm_maintain` | Run deduplication, PageRank, and community maintenance | +| `gm_maintain` | Run decay scoring, deduplication, PageRank, and community maintenance | ## Development ```bash npm install -npm run build # tsc --noEmit +npm run build # compiles to dist/ npm test # unit tests only (no Neo4j required) ``` diff --git a/README_CN.md b/README_CN.md index eabc330..a54ebc2 100644 --- a/README_CN.md +++ b/README_CN.md @@ -44,7 +44,7 @@ bash setup-graph-memory-pro.sh --uninstall ## 手动配置 -安装插件后，在 `~/.openclaw/openclaw.json` 中配置： +安装插件后，在 `~/.openclaw/openclaw.json` 中配置（**修改配置后需重启网关生效**：插件内置防重复注册守卫，宿主未 dispose 的二次 `register()` 会复用现有引擎，新配置不会热加载）： ```json { @@ -62,6 +62,7 @@ bash setup-graph-memory-pro.sh --uninstall "password": "你的 Neo4j 密码" }, "llm": { + "provider": "openai", "apiKey": "你的 LLM API Key", "baseURL": "https://api.openai.com/v1", "model": "gpt-4o-mini" @@ -79,8 +80,106 @@ bash setup-graph-memory-pro.sh --uninstall } ``` +Anthropic 直连（Claude）——去掉 `baseURL`，切换 `provider`： + +```json +"llm": { + "provider": "anthropic", + "apiKey": "sk-ant-...", + "model": "claude-3-5-sonnet-20241022" +} +``` + `embedding` 可选。设置时，`dimensions` 必须与 Neo4j 向量索引维度一致。新数据库会在插件启动时按配置创建索引；更换维度后需要重建向量索引或 Neo4j 数据库。 +### 记忆衰减（遗忘曲线） + +每个维护周期对全部 active 节点做三因子加权评分（recency + frequency + intrinsic），并在三个 tier 之间双向转换：`core` / `working` / `peripheral`。遗忘曲线之上还有两段生命周期（默认开启，可通过 `decay` 配置）：**自动弃用**——`peripheral` 层且 composite 持续低于 `peripheralCompositeThreshold`、超过 `autoDeprecateAfterDays`（30）天未访问的节点会被切断所有边并标记 `deprecated`；若同名知识之后被重新提取或编辑，节点自动复活回 `active`。**到期清理**——所有 `deprecated` 节点（含手动弃用与合并产物）自弃用起超过 `purgeAfterDays`（60）天后硬删（`DETACH DELETE`，向量随节点移除）以释放存储。 + +完整公式、字段映射、默认值依据与调参指南见 **[`docs/decay.md`](docs/decay.md)**。 + +最小配置（所有字段可选，默认值如下）： + +```json +"decay": { "enabled": true } +``` + +常用覆盖——完整参数见 `docs/decay.md` §4： + +```json +"decay": { + "enabled": true, + "recencyHalfLifeDays": 30, + "peripheralCompositeThreshold": 0.15, + "workingAccessThreshold": 3 +} +``` + +### cron 会话行为控制 + +OpenClaw 定时任务创建的会话可以独立配置图谱行为。host 把 cron 标记放在 **sessionKey** 上（`sessionId` 是随机 UUID），实际形状为 `cron:`、`agent::cron:` 或 `agent::cron::run:`： + +```json +"cron": { + "enabled": true, + "extract": true, + "finalizeAndMaintain": true +} +``` + +| 选项 | 默认 | 说明 | +| --- | --- | --- | +| `enabled` | `true` | 是否在 cron 会话内启用图谱功能（召回注入 + 消息入库）。关闭后 cron 会话不自动召回、不自动入库；`gm_*` 工具仍可手动调用（作为显式逃生通道）。 | +| `extract` | `true` | 是否在 cron 会话内触发知识提取（afterTurn / compact 的 LLM 三元组提取）。关闭后消息仍入库缓冲，之后可用 `openclaw graph-memory extract` 手动回填。 | +| `finalizeAndMaintain` | `true` | cron 会话结束时是否执行 finalize（EVENT→SKILL 晋升）和图维护（decay / PageRank / 社区检测）。定时任务频繁时可关闭，避免每次会话结束都跑全局维护。 | + +三个选项**默认全部开启**：cron 会话默认使用图谱，需按需显式关闭。`enabled=false` 是总开关：即使 `extract`/`finalizeAndMaintain` 设为 `true` 也不生效。非 cron 会话不受这些选项影响。三个子项均可省略，未写的字段取默认值 `true`。 + +注意：若 cron 任务显式设置了自定义 `sessionKey`，host 不再附加 `cron` 段，此类会话无法被识别，将按普通会话处理。 + +### 原始消息保留（messageRetention，opt-in） + +上下文压缩只改变模型可见面，**不构成删除持久证据的授权**——默认 `keep=all` 永不删除任何原始消息（GmMessage），零开销。需要控制库体积时可显式开启有界清理，挂在图维护链尾部，每个维护周期最多处理 `batchSize` 行： + +```json +"messageRetention": { + "keep": "referenced", + "batchSize": 500, + "dryRun": false +} +``` + +| 选项 | 默认 | 说明 | +| --- | --- | --- | +| `keep` | `"all"` | `all`=全部保留（默认，零行为变化）；`referenced`=只删"已提取完成且实际产出知识"的消息；`recent`=在 `referenced` 基础上叠加时间窗保护（需至少配一个窗口参数）。 | +| `recentTurns` | `0` | `keep=recent`：每 session 保留最近 N 轮真实用户发言（该轮及其后的全部消息保留）。无 user 消息的 session 完全保护。 | +| `retentionDays` | `0` | `keep=recent`：保留最近 N 天内入库的消息。 | +| `batchSize` | `500` | 单个维护周期最多处理的行数（1~10000），保证维护链工作有界。 | +| `dryRun` | `false` | `true` 时只报告候选集不删除——**启用清理前建议先跑一轮 `dryRun` 验证候选集**。 | + +删除语义（fail-closed）： + +- 只有 `extracted=true` **且** `producedKnowledge=true`（该轮 LLM 提取实际产出节点/边）的消息才会进入候选。 +- LLM 空提取（成功返回零节点零边）的轮次标记 `producedKnowledge=false`，原始证据保留在库中。此类轮次不会自动重提；如需用更好的 prompt/模型重挖，需手动将这些行的 `extracted` 重置为 `false`，再运行 `openclaw graph-memory extract` 回填。 +- 未提取消息、以及标记机制上线前的遗留行（无 `producedKnowledge` 属性）一律不删。 +- 保留步骤自身失败（非法策略 fail-closed 不删 / Neo4j 故障）不影响维护链的其他步骤，下一周期自动重试。 + +### 更换 embedding 模型后的重嵌入 + +embedding 向量与模型绑定——旧模型产出的向量与新模型不可比（维度不同时去重与向量搜索直接失效）， +且向量只写不删、没有自动迁移。更改 `embedding.model` / `embedding.dimensions` 后请执行： + +```bash +openclaw graph-memory reembed --dry-run # 只报告维度对照与向量覆盖情况，不写入 +openclaw graph-memory reembed # 清空全部向量并按批重建 +``` + +该命令清空所有 `MemoryNode.embedding`（连同 `contentHash`，避免运行时 `syncEmbed` +被旧文本 hash 短路），然后用当前模型按批重建全部活跃节点与社区摘要向量（`--batch `， +默认 32；批失败自动退化为逐条请求）。若向量索引维度与当前模型不符，命令会中止——加 +`--recreate-index` 删除并按新维度重建 `gm_node_embedding` / `gm_community_embedding`。 +失败条目保持无向量，修复端点后重跑本命令即可增量补齐。 + ### OAuth 登录（实验性） ```bash @@ -100,7 +199,7 @@ openclaw graph-memory auth login -> embedding -> 向量召回 + 社区扩展 + GDS PPR -> XML 上下文注入 -会话结束 -> 去重 -> 全局 PageRank -> 社区 -> 社区摘要 +会话结束 -> 衰减（遗忘曲线）-> 去重 -> 全局 PageRank -> 社区 -> 社区摘要 ``` ## 验证 @@ -129,12 +228,12 @@ openclaw gateway --verbose | --- | --- | | `gm_search` | 按查询召回图谱知识 | | `gm_record` | 手动记录知识节点 | -| `gm_update` | 按精确节点名称更新 / 删除 / 弃用已有节点（不存在则报错）。`mode=update`（默认）refine description/content；`mode=delete` 硬删除节点及其所有关系；`mode=deprecate` 标记 `[DEPRECATED]` 并删除所有关系（节点本身保留但被隔离） | +| `gm_update` | 按精确节点名称更新 / 弃用已有节点（不存在则报错）。`mode=update`（默认）refine description/content；`mode=deprecate` 标记 `[DEPRECATED]` 并切断所有关系，节点不再可被召回（等效删除；维护链在 `purgeAfterDays` 天后物理清理）。`mode=delete` 已移除，由 deprecate 取代 | | `gm_link` | 手动在两个已存在节点之间建立或细化关系边（按白名单校验类型+方向；from+to+type 已存在时仅更新 instruction） | | `gm_unlink` | 按名称删除两节点之间的关系边；可选 type 过滤，不传则删除 from→to 之间所有边 | | `gm_merge` | 合并两个同类型重复节点：keep 吸收 content/validatedCount/sessions + 去重边迁移；merge 节点被软删除（deprecated） | | `gm_stats` | 查看节点、关系、社区和 PageRank 统计 | -| `gm_maintain` | 执行去重、PageRank 和社区维护 | +| `gm_maintain` | 执行衰减评分、去重、PageRank 和社区维护 | ## 开发 diff --git a/docs/decay.md b/docs/decay.md new file mode 100644 index 0000000..b928ab9 --- /dev/null +++ b/docs/decay.md @@ -0,0 +1,211 @@ +# Memory Decay — 柔性评分模型 + +graph-memory-pro 的衰减机制采用**三因子加权评分 + tier 双向转换**，参考 [memory-lancedb-pro](https://github.com/CortexReach/memory-lancedb-pro) 的设计并映射到本仓库的图模型信号。 + +- **decay 评分只调整 `tier`**（`core` / `working` / `peripheral`）；tier 转换本身不改变 `status`。`status=deprecated` 有且仅有三个来源：手动弃用（`gm_update mode=deprecate` / REST DELETE / finalize invalidations，统一走断联+标记，见 §3.5）、merge 败者，以及遗忘曲线自动弃用（`autoDeprecate`，阶段一，见 §3.5）。 +- 每次 `gm_maintain` 或 `session_end` 维护的第 0 步执行：扫描所有 active 节点 → 评分 → tier 转换 → 写回 `decayScore` / `tier` / `decayComputedAt`（→ 自动弃用判定 → 过期硬删，见 §3.5）。 +- 评分结果可通过 `gm_stats` / CRUD API 查看；外层搜索目前**不读 decayScore 排序**（已由 PageRank + tier 隐含分层）。 + +--- + +## 1. 评分公式 + +``` +composite = wR · recency + wF · frequency + wI · intrinsic +``` + +三个权重默认 `0.4 / 0.3 / 0.3`，**推荐**和为 1。运行时若和≠1 会自动按比例归一化（`wR' = wR / (wR+wF+wI)`），保证 `composite ∈ [0,1]`，避免用户覆盖单个权重导致评分越界。归一化在 `scoreNode()` 内进行，原始 `cfg.*Weight` 值不被修改。 + +### 1.1 Recency（时间衰减，权重 0.4） + +Weibull 拉伸指数： + +``` +recency = exp( −λ · daysSinceLastAccess^β ) + +λ = ln(2) / effectiveHL +effectiveHL = recencyHalfLifeDays · exp( importanceModulation · importance ) +``` + +- **半衰期调制**：重要记忆（高 `importance`）的 `effectiveHL` 更大 → 衰减更慢。对应艾宾浩斯曲线"重要事件保留更久"。 +- **tier-β**：曲线形状随 tier 变化，反馈式调整衰减速度： + + | tier | β | 效果 | + |---|---|---| + | `core` | 0.8 | 尾部衰减缓（核心知识保得久） | + | `working` | 1.0 | 标准指数衰减 | + | `peripheral` | 1.3 | 加速衰减（边缘知识更快被遗忘） | + +### 1.2 Frequency（访问频率，权重 0.3） + +``` +frequency = base · ( 0.5 + 0.5 · recentnessBonus ) + +base = 1 − exp( −validatedCount / 5 ) +recentnessBonus = exp( −avgAccessGapDays / 30 ) # 仅当 validatedCount > 1 +avgAccessGapDays = ( lastAccessedAt − createdAt ) / ( validatedCount − 1 ) +``` + +- 用 `validatedCount`（LLM 重新提取的次数）替代 lancedb-pro 的 `accessCount`（manual recall 触发的次数）。前者是更强的"重新确认"信号。 +- `validatedCount ≤ 1` 时跳过 `recentnessBonus`，只返回 `base`（无法算平均间隔）。 + +### 1.3 Intrinsic（内在价值，权重 0.3） + +``` +intrinsic = importance · confidence + +importance = pagerank / maxPagerank # 每次扫描时按当前批次归一化到 [0,1] +confidence = 1 − 1 / ( 1 + validatedCount ) # 饱和函数，收敛到 1 +``` + +--- + +## 2. 字段映射（lancedb-pro → graph-memory-pro） + +| lancedb-pro 字段 | 本仓库替代 | 说明 | +|---|---|---| +| `accessCount` | `validatedCount` | LLM 重新提取次数（强信号，原为 manual recall 触发） | +| `lastAccessedAt` | `lastAccessedAt` | 由 `upsertNode` 在任意写入路径刷新（重新提取、`gm_record`、`gm_update`、CRUD POST）。`mergeNodes` 故意不刷新（合并 ≠ 用户重新激活） | +| `importance` | `pagerank / maxPagerank` | 图结构重要性，每次扫描归一化 | +| `confidence` | `1 − 1/(1+validatedCount)` | 饱和置信度 | +| `tier` | `tier`（新增字段） | 与 `status` 正交 | + +--- + +## 3. Tier 双向转换 + +| 转换 | 条件 | +|---|---| +| **core → working** | `composite < peripheralCompositeThreshold` **AND** `count < workingAccessThreshold` | +| **working → peripheral** | `composite < peripheralCompositeThreshold` **OR**（`ageDays > peripheralAgeDays` **AND** `count < workingAccessThreshold`） | +| **peripheral → working** | `count >= workingAccessThreshold` **AND** `composite >= workingCompositeThreshold` | +| **working → core** | `count >= coreAccessThreshold` **AND** `composite >= coreCompositeThreshold` **AND** `importance >= coreImportanceThreshold` | + +- 新节点默认 `tier = "working"`。 +- tier 转换本身只写 `tier` / `updatedAt`，不改 `status`、不影响搜索过滤；被降至 `peripheral` 的节点仍可被搜索到（直到满足 §3.5 的自动弃用条件）。 +- 不存在的"core→peripheral"和"peripheral→core"由两次相邻转换实现（经过 working）。 + +### 3.5 节点生命周期：自动弃用与到期硬删（两阶段） + +tier 转换之上，`applyDecay` 还承担两阶段生命周期的**阶段一**；**阶段二**（硬删）由 `runMaintenance` 在 decay 步之后执行。 + +``` +active ──遗忘曲线──▶ peripheral（仍可搜索） + │ tier=peripheral + composite < peripheralCompositeThreshold + │ + 距 lastAccessedAt ≥ autoDeprecateAfterDays（阶段一） + ▼ + 自动弃用：切断所有边 + status='deprecated'（deprecatedBy='decay'， + 描述加 [DEPRECATED] 前缀）—— 全路径不可召回 + │ + ┌───────────────┴────────────────┐ + ▼ 重新提取/编辑命中（复活） ▼ deprecatedAt + purgeAfterDays（阶段二） + 恢复 active、剥前缀 DETACH DELETE 硬删（向量随节点移除， + 清 deprecatedAt/By 释放存储；适用于所有 deprecated 节点） +``` + +要点： + +- **阶段一判定**（`shouldAutoDeprecate`，三条件同时满足）：`tier=peripheral`（本轮 tier 转换后的层级，刚降到 peripheral 的老节点即刻参与）**AND** `composite < peripheralCompositeThreshold`（高 intrinsic 价值节点受保护）**AND** 距最近访问（`lastAccessedAt` → `updatedAt` → `createdAt` 回退链）≥ `autoDeprecateAfterDays`。`autoDeprecate: false` 或 `enabled: false` 时整体停用。 +- **手动弃用 = 一次性断联**：所有人工路径（`gm_update mode=deprecate`、finalize invalidations、REST `DELETE /nodes/:id`）统一走 `deprecateNodeAndDisconnectById`——切断所有边 + `[DEPRECATED]` 前缀 + `deprecatedBy='manual'`。由于所有召回路径都过滤 `status='active'` 且边已切断，手动弃用等效删除（不存在独立的 `gm_update mode=delete`，也没有 status-only 的轻量弃用路径）。 +- **复活**：`deprecatedBy='decay'` 的节点被同名知识重新提取（`upsertNode`）或手动编辑（`updateNode`）命中时，自动恢复 `active`、剥离 `[DEPRECATED]` 前缀、清除 `deprecatedAt`/`deprecatedBy`。手动弃用（`deprecatedBy='manual'`）与 merge 败者（`deprecatedBy='merge'`）**不复活**——人工/合并语义判定优先于遗忘曲线。 +- **阶段二硬删**：所有 `deprecated` 节点（含 manual/merge/存量数据），自 `deprecatedAt` 起超过 `purgeAfterDays` 天后 `DETACH DELETE`（`coalesce(n.deprecatedAt, n.updatedAt)` 中的 updatedAt 回退仅是防御性兜底）。`embedding`/`contentHash` 向量属性随节点一并移除，向量索引项同步消失。`purgeAfterDays: 0` 表示永不硬删。此步不受 `enabled` 总开关约束（手动弃用的节点也需要到期清理）。 +- **存量兼容**：升级前已 deprecated 的节点没有 `deprecatedBy`，一律按 `manual` 处理（不参与复活）。启动时（`initSchema`）幂等补写：为缺 `deprecatedAt` 的 deprecated 节点一次性钉死 `deprecatedAt = coalesce(updatedAt, createdAt)` 快照——否则 manual/merge 弃用节点被重新提取命中时 `upsertNode` 会 bump `updatedAt`（不复活但刷新时间戳），purge 期限被无限推后（永远删不掉）。 + +--- + +## 4. 默认值与调参指南 + +### 4.1 默认配置 + +```json +{ + "decay": { + "enabled": true, + "recencyHalfLifeDays": 30, + "recencyWeight": 0.4, + "importanceModulation": 1.5, + "frequencyWeight": 0.3, + "intrinsicWeight": 0.3, + "betaCore": 0.8, + "betaWorking": 1.0, + "betaPeripheral": 1.3, + "coreAccessThreshold": 10, + "coreCompositeThreshold": 0.7, + "coreImportanceThreshold": 0.8, + "peripheralCompositeThreshold": 0.15, + "peripheralAgeDays": 60, + "workingAccessThreshold": 3, + "workingCompositeThreshold": 0.4, + "autoDeprecate": true, + "autoDeprecateAfterDays": 30, + "purgeAfterDays": 60 + } +} +``` + +### 4.2 数值来源 + +| 参数 | 默认值 | 来源 | +|---|---|---| +| `recencyHalfLifeDays` | 30 | 艾宾浩斯曲线 ~25% 保留率拐点；同时与 lancedb-pro 的 `recencyHalfLifeDays` + `ACCESS_DECAY_HALF_LIFE_DAYS` 一致 | +| `importanceModulation` | 1.5 | lancedb-pro：`effectiveHL = 30 · exp(1.5 · importance)`，importance=1 时半衰期延长到 ~134 天 | +| `betaCore/Working/Peripheral` | 0.8 / 1.0 / 1.3 | lancedb-pro Weibull 形状参数 | +| 7 个 tier 转换阈值 | — | lancedb-pro `tier-manager` 默认值 | +| `recencyWeight / frequencyWeight / intrinsicWeight` | 0.4 / 0.3 / 0.3 | lancedb-pro 三因子权重，和为 1 | +| `validatedCount` 分母 | 5 | lancedb-pro 的 `1 − exp(−count/5)` 基础频率项（未改） | +| `autoDeprecateAfterDays` | 30 | 遗忘终态缓冲：peripheral 降级后再经历一个半衰期量级的静默期才弃用，避免误伤短暂边缘化的重要知识 | +| `purgeAfterDays` | 60 | 弃用后两个月的"反悔窗口"，之后硬删释放存储；设 0 关闭硬删 | + +### 4.3 常见调参场景 + +| 想要的效果 | 调整方向 | +|---|---| +| 记忆整体保留更久 | 调高 `recencyHalfLifeDays`（如 60）或调低 `peripheralCompositeThreshold`（更难降级） | +| 更激进遗忘 | 调低 `recencyHalfLifeDays`（如 14）或调高 `peripheralCompositeThreshold` | +| 重要知识显著保得久 | 调高 `importanceModulation`（半衰期调制更强） | +| 核心知识不易降级 | 调低 `betaCore`（更缓的尾部）或调高 `coreCompositeThreshold`（更难升 core，留在 working 也保得久） | +| 单次曝光更易遗忘 | 调高 `workingAccessThreshold`（promote 到 working 需要更多确认） | +| 只降级、永不自动弃用 | `"autoDeprecate": false` | +| 弃用后保留更久再硬删 | 调高 `purgeAfterDays`（如 180），或设 0 永不硬删 | +| 永久禁用衰减（含自动弃用） | `"enabled": false` | + +### 4.4 与原布尔阈值方案的对照（向后兼容） + +旧版本（`maxAgeDays` + `minCalls`）的布尔规则已被这套柔性评分取代。原默认值 `maxAgeDays=30, minCalls=2` 在新模型下大致对应于： + +- 一个 `validatedCount=1`、`tier=working`、低 pagerank 的节点，约 30 天后 `recency` 跌破 0.15 → `composite` 跌破 `peripheralCompositeThreshold` → demote 到 `peripheral`。 +- 关键差别：新模型先降 tier 不直接弃用，节点仍可搜索（只是 decayScore 较低）；只有再经历 `autoDeprecateAfterDays` 静默期后才会按 §3.5 自动弃用（默认开启，可关闭），彻底断联则要到弃用时刻。 + +--- + +## 5. 数据库字段 + +| 字段 | 类型 | 写入者 | 说明 | +|---|---|---|---| +| `tier` | string | `applyDecay` / `upsertNode`（创建时初始化为 `working`） | `core` / `working` / `peripheral` | +| `lastAccessedAt` | int (epoch ms) | `upsertNode`（重新提取时） | decay 评分的时间基准 | +| `decayScore` | float (0~1) | `applyDecay` | 最近一次评分结果 | +| `decayComputedAt` | int (epoch ms) | `applyDecay` | 评分时间戳 | +| `deprecatedAt` | int (epoch ms) | `deprecateNodeAndDisconnectById` / `mergeNodes` / `autoDeprecateNodes`（已弃用节点重复弃用时保留原值，不重置时钟） | 弃用时刻；阶段二硬删倒计时基准，缺失时回退 `updatedAt` | +| `deprecatedBy` | string | 同上 | `decay`（可复活）/ `manual` / `merge`（不复活）；缺失按 `manual` 处理 | + +旧节点缺这些字段时： +- `tier` 缺失 → 评分按 `working` 处理；首次 `applyDecay` 时自动写入 `working` +- `lastAccessedAt` 缺失 → 回退到 `updatedAt` / `createdAt` +- `decayScore` / `decayComputedAt` 缺失 → 在首次 `applyDecay` 前为 undefined，不影响评分 + +**Backfill 时机**：新字段在第一次 `applyDecay` 运行时为每个 active 节点批量写入。如果部署初始用 `decay.enabled=false`，字段会一直缺失直到切换为 `true` 后的第一次维护周期。在切换前的窗口期，对 raw DB 直接做 `tier` 过滤查询会返回 null/missing 而非 `"working"`——目前搜索路径不读 `tier`，但自定义查询需要留意。 + +--- + +## 6. 实现位置 + +| 文件 | 内容 | +|---|---| +| `src/graph/decay.ts` | 评分函数 + tier 决策 + 自动弃用决策（`shouldAutoDeprecate`）+ `applyDecay()` 批处理 | +| `src/types.ts` | `DecayConfig` 接口、`NodeTier` / `DeprecatedBy` 类型、`GmNode` 新字段、`DEFAULT_CONFIG.decay` | +| `src/store/store.ts` | `toNode` 字段映射、`upsertNode` 初始化 `tier` / `lastAccessedAt`、复活逻辑（`upsertNode`/`updateNode`）、`autoDeprecateNodes()`、`purgeDeprecatedNodes()` | +| `src/graph/maintenance.ts` | 调用入口（step 0 评分/弃用 + step 0.5 硬删） | +| `test/decay.test.ts` | 评分函数 + tier 决策 + 自动弃用决策纯函数单元测试 | +| `test/integration.neo4j.test.ts` | 自动弃用 / 硬删 / 复活的集成测试（`NEO4J_INTEGRATION=1`） | +| `openclaw.plugin.json` | 用户可见的配置 schema | diff --git a/index.ts b/index.ts index a5f3c9b..a5cd5c9 100755 --- a/index.ts +++ b/index.ts @@ -7,24 +7,29 @@ */ import type { OpenClawPluginApi } from "openclaw/plugin-sdk"; import { Type } from "@sinclair/typebox"; -import { getDriver, initSchema, getSession } from "./src/store/db.ts"; +import { getDriver, initSchema } from "./src/store/db.ts"; +import { Neo4jGate } from "./src/store/gate.ts"; import { - saveMessage, getUnextracted, - markExtracted, isTurnExtracted, + saveMessage, getUnextracted, countUnextracted, getMaxTurnIndex, + markExtracted, isTurnExtracted, commitTurnAdvance, upsertNode, upsertEdge, findByName, updateNode, - deleteNode, deprecateNodeAndDisconnect, - getBySession, edgesFrom, edgesTo, + deprecateNodeAndDisconnect, deprecateNodeAndDisconnectById, + getBySession, edgesTouching, topNodes, deleteEdges, mergeNodes, - deprecate, getStats, + getStats, } from "./src/store/store.ts"; import { createCompleteFn, resolveProvider } from "./src/engine/llm.ts"; -import { createEmbedFn } from "./src/engine/embed.ts"; +import { createEmbedder } from "./src/engine/embed.ts"; +import { estimateTokens } from "./src/tokens.ts"; import { Recaller, parseTimeRange } from "./src/recaller/recall.ts"; -import { Extractor } from "./src/extractor/extract.ts"; +import { Extractor, shouldRunFinalize } from "./src/extractor/extract.ts"; +import { shouldSkipTurnExtraction, turnHasToolWork } from "./src/extractor/turn-filter.ts"; +import { persistExtractionResult } from "./src/extractor/persist.ts"; import { assembleContext } from "./src/format/assemble.ts"; import { sanitizeToolUseResultPairing } from "./src/format/transcript-repair.ts"; import { runMaintenance } from "./src/graph/maintenance.ts"; -import { DEFAULT_CONFIG, type GmConfig, type RecallResult, type EdgeType } from "./src/types.ts"; +import { normalizeMessageRetentionPolicy } from "./src/store/retention.ts"; +import { DEFAULT_CONFIG, DEFAULT_CRON_CONFIG, isCronSessionKey, EDGE_TYPES, type GmConfig, type RecallResult, type EdgeType, type ExtractionResult } from "./src/types.ts"; import { registerCrudRoutes } from "./src/routes/crud.ts"; import { createGraphMemoryCli } from "./src/cli.ts"; @@ -37,6 +42,28 @@ export function isGraphMemoryCliInvocation(argv: readonly string[] = process.arg return argv.slice(2).includes("graph-memory"); } +/** + * Host registration modes that never serve logical turns ("readOnlyDiscovery" + * lifecycle in the host registry — turn resolution degrades such entries to + * legacy by design, and runtime entries are adopted from the composition root + * instead of being replaced). Running the full runtime init (second Neo4j + * driver, embed probe, schema DDL) in these loads is pure waste, and their + * registerContextEngine calls are no-ops against a runtime registry entry. + * Modes: cli-metadata (CLI discovery), discovery / tool-discovery (scoped + * loads, e.g. config hot reload inspection), setup-only (setup contract with + * empty pluginConfig). Undefined mode (OpenClaw < 2026.7) counts as full. + */ +const METADATA_ONLY_REGISTRATION_MODES = new Set([ + "cli-metadata", + "discovery", + "tool-discovery", + "setup-only", +]); + +export function isMetadataOnlyRegistration(mode: unknown): boolean { + return typeof mode === "string" && METADATA_ONLY_REGISTRATION_MODES.has(mode); +} + // ─── 从 OpenClaw config 读默认 model 名 ────────────────────── /** @@ -63,16 +90,29 @@ export function readDefaultModel(apiConfig: unknown): string { return raw; } -function throwNodeNotFound(name: string): never { - throw new Error( +/** 节点未找到报错文案的单源（throwNodeNotFound / gm_update notFoundHint 共用）。 */ +function nodeNotFoundMessage( + name: string, + tail: string = "或使用 gm_search 搜索已有节点。", +): string { + return ( `[graph-memory-pro] 未找到名称为 "${name}" 的节点。` + `请检查节点名称是否精确（名称标准化规则：全小写、空格/下划线转连字符、移除非字母数字字符），` + - `或使用 gm_search 搜索已有节点。`, + tail ); } +function throwNodeNotFound(name: string, tail?: string): never { + throw new Error(nodeNotFoundMessage(name, tail)); +} + // ─── 清洗 OpenClaw metadata 包装 ───────────────────────────── +/** 剥离 OpenClaw 注入的命令前缀与时间戳标记（cleanPrompt / extractUserText 共用，逐字等价）。 */ +function stripCommandAndTimestampPrefix(s: string): string { + return s.replace(/^\/\w+\s+/, "").trim().replace(/^\[[\w\s\-:]+\]\s*/, "").trim(); +} + export function cleanPrompt(raw: string): string { let prompt = raw.trim(); if (prompt.includes("Sender (untrusted metadata)")) { @@ -86,9 +126,7 @@ export function cleanPrompt(raw: string): string { prompt = lines.join("\n").trim(); } } - prompt = prompt.replace(/^\/\w+\s+/, "").trim(); - prompt = prompt.replace(/^\[[\w\s\-:]+\]\s*/, "").trim(); - return prompt; + return stripCommandAndTimestampPrefix(prompt); } // ─── 规范化消息 content，防 OpenClaw content.filter() 崩溃 ──── @@ -130,21 +168,28 @@ export function missingIngestMessages(messages: any[], ingestedCount: number): a const KEEP_TURNS = 5; +/** batched 模式 session_end 冲洗的最大批数（防积压失控；残余保持未提取，CLI extract 可回填） */ +const SESSION_END_FLUSH_ROUNDS = 5; + function estimateMsgTokens(msg: any): number { const text = typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content ?? ""); - return Math.ceil(text.length / 3); + return estimateTokens(text.length); +} + +/** 从 content blocks 数组抽取纯文本（text block 拼接） */ +function textFromBlocks(blocks: any[]): string { + return blocks + .filter((b: any) => b && typeof b === "object" && b.type === "text" && typeof b.text === "string") + .map((b: any) => b.text) + .join("\n"); } export function extractAssistantText(msg: any): string { if (typeof msg.content === "string") return msg.content; if (!Array.isArray(msg.content)) return ""; - return msg.content - .filter((b: any) => b && typeof b === "object" && b.type === "text" && typeof b.text === "string") - .map((b: any) => b.text) - .join("\n") - .trim(); + return textFromBlocks(msg.content).trim(); } export function extractUserText(msg: any): string { @@ -154,35 +199,41 @@ export function extractUserText(msg: any): string { } else if (!Array.isArray(msg.content)) { raw = String(msg.content ?? ""); } else { - raw = msg.content - .filter((b: any) => b && typeof b === "object" && b.type === "text" && typeof b.text === "string") - .map((b: any) => b.text) - .join("\n") - .trim(); + raw = textFromBlocks(msg.content).trim(); } // 去掉 OpenClaw metadata（Sender JSON block、命令前缀、时间戳） const fenceEnd = raw.lastIndexOf("```"); if (fenceEnd >= 0 && raw.includes("Sender")) { raw = raw.slice(fenceEnd + 3).trim(); } - raw = raw.replace(/^\/\w+\s+/, "").trim(); - raw = raw.replace(/^\[[\w\s\-:]+\]\s*/, "").trim(); - return raw; + return stripCommandAndTimestampPrefix(raw); +} + +/** + * 聚合一轮消息中 user 角色的（元数据剥离后）文本，供 trivial 预筛判定。 + * assistant/工具结果不参与：它们跟随用户意图，用户输入有意义时整轮照常提取。 + */ +export function turnUserText(messages: any[]): string { + return (messages ?? []) + .filter((m) => m && typeof m === "object" && m.role === "user") + .map((m) => extractUserText(m)) + .join(" "); } export function sliceLastTurn( messages: any[], + keepTurns: number = KEEP_TURNS, ): { messages: any[]; tokens: number; dropped: number } { if (!messages.length) { return { messages: [], tokens: 0, dropped: 0 }; } - // 找到最近 N 个 user 消息的位置 + // 找到最近 N 个 user 消息的位置（N = keepTurns，由 cfg.freshTailCount 注入） const userIndices: number[] = []; for (let i = messages.length - 1; i >= 0; i--) { if (messages[i].role === "user") { userIndices.push(i); - if (userIndices.length >= KEEP_TURNS) break; + if (userIndices.length >= keepTurns) break; } } if (!userIndices.length) { @@ -231,8 +282,9 @@ export function sliceLastTurn( /** 图谱为空时也必须执行相同的裁剪、工具配对修复和 content 规范化。 */ export function prepareAssemblyMessages( messages: any[], + keepTurns: number = KEEP_TURNS, ): { messages: any[]; tokens: number; dropped: number } { - const sliced = sliceLastTurn(messages); + const sliced = sliceLastTurn(messages, keepTurns); return { messages: normalizeMessageContent(sanitizeToolUseResultPairing(sliced.messages)), tokens: sliced.tokens, @@ -242,6 +294,15 @@ export function prepareAssemblyMessages( // ─── 插件对象 ───────────────────────────────────────────────── +/** + * 当前活跃引擎（防重复注册守卫）。 + * 某些宿主版本会在未调用 dispose 的情况下重复调用 register()（热重载、配置变更）： + * 重新注册全套 hook/工具/路由会让每轮工作翻倍，且新引擎实例持有自己那份 + * per-session 提取锁，与旧实例并行时同一 session 的提取互斥被打破。 + * 只有 dispose() 清空标记（真正的重载）后，下一次 register() 才走完整初始化。 + */ +let activeEngine: { dispose: () => Promise | void } | null = null; + const graphMemoryProPlugin = { id: "graph-memory-pro", name: "Graph Memory Pro", @@ -264,19 +325,73 @@ const graphMemoryProPlugin = { pluginId: "graph-memory-pro", pluginConfig: raw as Record | undefined, resolveConfigPath: (p: string) => api.resolvePath?.(p) ?? p, + defaultModel: readDefaultModel(api.config), }), { commands: ["graph-memory"] }, ); } if ( - api.registrationMode === "cli-metadata" || + isMetadataOnlyRegistration(api.registrationMode) || isGraphMemoryCliInvocation() ) { return; } + // 防重复注册：CLI 元数据仍可重复注册（幂等），但运行时只允许一份。 + // 复用现有引擎，只重绑 ContextEngine 工厂（见 activeEngine 上的说明）。 + if (activeEngine) { + // 工厂必须捕获引擎对象本身，绝不能写成 `() => activeEngine`： + // host 逐逻辑 turn 惰性调用工厂，dispose() 清空 activeEngine 之后 + // 该闭包会返回 null —— host 按契约判定 "factory returned null" 并 + // 逐回合降级 legacy，直到 gateway 重启（2026-09-01 事故）。返回已 + // dispose 的引擎对象是安全的：方法仍满足 ContextEngine 契约，且 + // per-session 状态自愈（msgSeq 经 getMaxTurnIndex 从 DB 恢复）。 + const reusableEngine = activeEngine; + api.registerContextEngine("graph-memory-pro", () => reusableEngine); + api.logger.warn("[graph-memory-pro] duplicate register() ignored; reusing active engine"); + return; + } + const cfg: GmConfig = { ...DEFAULT_CONFIG, ...raw }; if (raw.neo4j) cfg.neo4j = { ...DEFAULT_CONFIG.neo4j, ...raw.neo4j }; + if (raw.decay) cfg.decay = { ...DEFAULT_CONFIG.decay, ...raw.decay }; + if (raw.cron) cfg.cron = { ...DEFAULT_CONFIG.cron, ...raw.cron }; + if (raw.extract) cfg.extract = { ...DEFAULT_CONFIG.extract, ...raw.extract }; + + // 拼写兼容：接受小写 baseUrl（部分宿主/用户的配置习惯），统一归一到 baseURL。 + // 显式 baseURL 优先；trim 后为空视为未配置。 + const rawLlm = (raw.llm ?? {}) as Record; + if (cfg.llm && !cfg.llm.baseURL && typeof rawLlm.baseUrl === "string" && rawLlm.baseUrl.trim()) { + cfg.llm = { ...cfg.llm, baseURL: rawLlm.baseUrl.trim() }; + } + const rawEmbedding = (raw.embedding ?? {}) as Record; + if ( + cfg.embedding && !cfg.embedding.baseURL && + typeof rawEmbedding.baseUrl === "string" && rawEmbedding.baseUrl.trim() + ) { + cfg.embedding = { ...cfg.embedding, baseURL: rawEmbedding.baseUrl.trim() }; + } + + // messageRetention 配置预校验：非法策略在启动时报错并回退 keep=all（fail closed）， + // 而不是等到 session_end 维护链里每次抛错 + if (cfg.messageRetention) { + try { + normalizeMessageRetentionPolicy(cfg.messageRetention); + } catch (err) { + api.logger.error( + `[graph-memory-pro] messageRetention 配置非法，保留策略回退为 keep=all（不删除任何消息）：${err}`, + ); + delete cfg.messageRetention; + } + } + const cronCfg = cfg.cron ?? DEFAULT_CRON_CONFIG; + + // 提取配置（LLM 成本控制）：per-turn/batched 模式 + trivial 轮本地预筛 + const extractCfg = cfg.extract ?? DEFAULT_CONFIG.extract!; + const trivialFilterOpts = { + maxChars: extractCfg.trivialMaxChars, + extraPrompts: extractCfg.trivialPrompts, + }; const providerModel = readDefaultModel(api.config); @@ -324,39 +439,104 @@ const graphMemoryProPlugin = { // ── 初始化 Neo4j ──────────────────────────────────────── const driver = getDriver(cfg.neo4j); + // Neo4j 熔断门控：掉线时快速降级（跳图谱注入 / 缓冲消息），避免每轮吃满 driver 超时 + const neo4jGate = new Neo4jGate(); + // Schema 初始化（异步，不阻塞启动） initSchema(driver, cfg.embedding) .then(() => api.logger.info("[graph-memory-pro] Neo4j schema initialized")) - .catch(err => api.logger.error(`[graph-memory-pro] schema init failed: ${err}`)); + .catch(err => { + neo4jGate.recordFailure(); + api.logger.error(`[graph-memory-pro] schema init failed: ${err}`); + }); const llm = createCompleteFn(effectiveModel, cfg.llm); const recaller = new Recaller(driver, cfg); const extractor = new Extractor(llm); // ── 初始化 embedding ──────────────────────────────────── - createEmbedFn(cfg.embedding) - .then((fn) => { - if (fn) { - recaller.setEmbedFn(fn); + // re-probe 状态提前声明：启动 probe 失败时记录时间戳，bootstrap 的 + // 会话级 re-probe 据此退避（端点宕机时不逐会话重试刷日志/打 API） + const embeddingConfigured = !!(cfg.embedding && (cfg.embedding.apiKey || cfg.embedding.baseURL)); + let embedProbeInFlight = false; + let lastEmbedProbeAt = 0; + + createEmbedder(cfg.embedding) + .then((embedder) => { + if (embedder) { + recaller.setEmbedFn(embedder.embed); + recaller.setEmbedBatchFn(embedder.embedBatch); api.logger.info("[graph-memory-pro] vector search ready"); } else { + lastEmbedProbeAt = Date.now(); api.logger.info("[graph-memory-pro] text search mode (配置 embedding 可启用语义搜索)"); } }) .catch(() => { + lastEmbedProbeAt = Date.now(); api.logger.info("[graph-memory-pro] text search mode"); }); /** * 每轮结束后直接从原始消息提取知识图谱 * 一轮 = 用户发一条消息 → agent 不管调了多少工具 → 最终回复用户 + * + * compact() 与本函数对同一 session 存在 TOCTOU 竞争：两条路径都先 + * isTurnExtracted/getUnextracted → 调 LLM → 最后 markExtracted，中间窗口 + * 允许另一条路径重复提取同一批消息（重复 LLM 调用 + validatedCount 双递增）。 + * 用 per-session async 互斥锁串行化两条路径的提取体。 */ + const extractLocks = new Map>(); + function withExtractLock(sessionId: string, fn: () => Promise): Promise { + const prev = extractLocks.get(sessionId) ?? Promise.resolve(); + const chain = prev.catch(() => {}); + const result = chain.then(() => fn()); + // 链上只保留"上一轮是否结束"的状态，丢弃返回值并吞掉错误， + // 否则一次失败会永久污染链 → 后续 acquire 直接 reject。 + extractLocks.set(sessionId, result.then(() => undefined, () => undefined)); + return result; + } + + /** + * 会话内"upsert 命中已有节点"（isNew=false）计数：finalize 阶梯的第二触发条件。 + * invalidations（纠错弃用）只有 finalize 一条产出路径，而纠错常发生在 + * 小会话/无 EVENT 会话——shouldRunFinalize 的规模/EVENT 双门恰好会漏掉它们； + * 会话触碰过既有知识 = 有纠错/跨会话建边价值，值得一次 finalize LLM 调用。 + */ + const sessionUpdatedHits = new Map(); + function trackUpdatedExisting(sessionId: string, count: number): void { + if (count > 0) { + sessionUpdatedHits.set(sessionId, (sessionUpdatedHits.get(sessionId) ?? 0) + count); + } + } + async function extractTurnKnowledge(sessionId: string, turnNum: number, rawMessages: any[]): Promise { - try { - if (await isTurnExtracted(driver, sessionId, turnNum)) { - api.logger.info(`[graph-memory-pro] turn ${turnNum}: already extracted (compact), skipping`); - return; - } + // 熔断开启时跳过本轮提取：消息保持未标记，恢复后由 compact / extract 补提取 + if (!neo4jGate.isAvailable()) { + api.logger.info(`[graph-memory-pro] turn ${turnNum}: extraction skipped (neo4j circuit open)`); + return; + } + return withExtractLock(sessionId, async () => { + try { + // 先等掉线期间缓冲的消息落库，再判断/标记 extracted——否则行落库晚于 + // markExtracted 时会以 extracted=false 重现，被下一轮 compact 重复提取 + if (messageBuffer.length) await flushMessageBuffer(); + if (await isTurnExtracted(driver, sessionId, turnNum)) { + api.logger.info(`[graph-memory-pro] turn ${turnNum}: already extracted (compact), skipping`); + return; + } + + // LLM 成本控制：trivial 轮本地预筛（无意义词表 / ≤trivialMaxChars 纯文本）， + // 命中则零 LLM 直接标记（producedKnowledge=false，原始证据保留）。 + // 轮内含工具劳动（tool/toolResult）时不判 trivial："继续"触发的一轮真实 + // 修复劳动恰是图谱最该吸收的知识，且 per-turn 路径没有自动补提触发点。 + if (!turnHasToolWork(rawMessages) + && shouldSkipTurnExtraction(turnUserText(rawMessages), trivialFilterOpts)) { + await markExtracted(driver, sessionId, turnNum, false); + api.logger.info(`[graph-memory-pro] turn ${turnNum}: trivial prompt, extraction skipped (local pre-filter)`); + return; + } + const existing = (await getBySession(driver, sessionId)).map(n => n.name); const result = await extractor.extract({ messages: rawMessages, @@ -364,33 +544,17 @@ const graphMemoryProPlugin = { }); if (!result.nodes.length && !result.edges.length) { - await markExtracted(driver, sessionId, turnNum); - api.logger.info(`[graph-memory-pro] turn ${turnNum}: no knowledge extracted (marked extracted)`); + // 空提取也要标记 extracted（防止重复提取），但 producedKnowledge=false： + // 该轮没有知识固化进图谱，原始证据保留（retention 不删； + // 重挖需手动重置 extracted 后跑 graph-memory extract） + await markExtracted(driver, sessionId, turnNum, false); + api.logger.info(`[graph-memory-pro] turn ${turnNum}: no knowledge extracted (marked extracted, evidence retained)`); return; } - const nameToId = new Map(); - for (const nc of result.nodes) { - const { node } = await upsertNode(driver, { - type: nc.type, name: nc.name, - description: nc.description, content: nc.content, - }, sessionId); - nameToId.set(node.name, node.id); - recaller.syncEmbed(node).catch(() => {}); - } - - for (const ec of result.edges) { - const fromNode = await findByName(driver, ec.from); - const toNode = await findByName(driver, ec.to); - const fromId = nameToId.get(ec.from) ?? fromNode?.id; - const toId = nameToId.get(ec.to) ?? toNode?.id; - if (fromId && toId) { - await upsertEdge(driver, { - fromId, toId, type: ec.type, - instruction: ec.instruction, condition: ec.condition, sessionId, - }); - } - } + // upsert 节点 + 批量向量同步（fire-and-forget）+ 建边 —— 单一来源见 persist.ts + const outcome = await persistExtractionResult(driver, recaller, result, { sessionId }); + trackUpdatedExisting(sessionId, outcome.updatedExisting); // 标记该轮消息已提取 await markExtracted(driver, sessionId, turnNum); @@ -399,11 +563,83 @@ const graphMemoryProPlugin = { } catch (err) { api.logger.error(`[graph-memory-pro] turn ${turnNum} extract failed: ${err}`); } + }); + } + + /** + * 批量提取共享体（compact / batched 模式攒批触发 / session_end 尾批冲洗）： + * 读未提取消息 → LLM → upsert + 批量向量同步 → markExtracted。 + * 调用方负责 cron/熔断门控；内部经 withExtractLock 与 per-turn 路径串行化。 + */ + async function extractUnextractedBatch(sessionId: string): Promise<{ + ok: boolean; compacted: boolean; reason?: string; summary?: string; + }> { + return withExtractLock(sessionId, async () => { + // 掉线恢复后的补提取路径：先把缓冲消息刷进 DB 再读未提取集 + if (messageBuffer.length) await flushMessageBuffer(); + const msgs = await getUnextracted(driver, sessionId, cfg.compactTurnCount * 3); + + if (!msgs.length) return { ok: true, compacted: false, reason: "no messages" }; + + try { + const existing = (await getBySession(driver, sessionId)).map(n => n.name); + const result = await extractor.extract({ messages: msgs, existingNames: existing }); + + // upsert 节点 + 批量向量同步（fire-and-forget）+ 建边 —— 单一来源见 persist.ts + const outcome = await persistExtractionResult(driver, recaller, result, { sessionId }); + trackUpdatedExisting(sessionId, outcome.updatedExisting); + + const maxTurn = Math.max(...msgs.map((m: any) => m.turn_index)); + await markExtracted( + driver, sessionId, maxTurn, + result.nodes.length > 0 || result.edges.length > 0, + ); + + return { + ok: true, compacted: true, + summary: `extracted ${result.nodes.length} nodes, ${result.edges.length} edges`, + }; + } catch (err) { + api.logger.error(`[graph-memory-pro] batch extraction failed: ${err}`); + return { ok: false, compacted: false, reason: String(err) }; + } + }); + } + + /** + * 轮提取统一入口（LLM 成本控制的模式分发）： + * - per-turn（默认）：每轮即时 LLM 提取（内含 trivial 本地预筛）； + * - batched：不在轮边界调 LLM；未提取消息累计到 compactTurnCount*3 条时 + * 批量提取一次（LLM 调用 ~1/N），session_end 冲洗尾批。 + * 注意 batched 不做逐轮 trivial 标记——markExtracted 是前缀语义， + * 会误吞留给攒批的更早轮次；trivial 消息随批进入 prompt，成本由批摊薄。 + */ + function scheduleTurnExtraction(sessionId: string, turnNum: number, rawMessages: any[]): void { + const run = extractCfg.mode === "batched" + ? handleBatchedTurn(sessionId) + : extractTurnKnowledge(sessionId, turnNum, rawMessages); + run.catch(err => api.logger.error(`[graph-memory-pro] extract failed: ${err}`)); + } + + async function handleBatchedTurn(sessionId: string): Promise { + // 熔断开启时跳过计数/提取：消息保持未标记，恢复后由攒批/冲洗/CLI 补提取 + if (!neo4jGate.isAvailable()) return; + try { + const pending = await countUnextracted(driver, sessionId); + if (pending >= cfg.compactTurnCount * 3) { + await extractUnextractedBatch(sessionId); + } + } catch (err) { + api.logger.error(`[graph-memory-pro] batched turn check failed: ${err}`); + } } // ── Session 运行时状态 ────────────────────────────────── const msgSeq = new Map(); + const msgSeqLoaders = new Map>(); const recalled = new Map(); + // recalled 结果对应的 recall prompt（assemble 复用缓存判定用；继承路径不写，查不到则照常新鲜召回） + const recalledPrompt = new Map(); const sessionIdsByKey = new Map(); const pendingSubagentRecall = new Map(); const ingestedSinceTurn = new Map(); @@ -419,33 +655,253 @@ const graphMemoryProPlugin = { } async function ingestMessage(sessionId: string, message: any): Promise { + if (!msgSeq.has(sessionId)) { + // 插件重启后内存 Map 会丢，必须从 DB 恢复 MAX(turnIndex)，否则下一条消息 + // turnIndex=1 → MERGE 命中旧行 → ON CREATE 被跳过 → 新消息静默丢失。 + // in-flight Promise 去重，避免并发 ingest 同时查询 + 互相覆盖 seq。 + let loader = msgSeqLoaders.get(sessionId); + if (!loader) { + loader = getMaxTurnIndex(driver, sessionId).then(max => { + msgSeq.set(sessionId, max); + msgSeqLoaders.delete(sessionId); + return max; + }).catch(err => { + msgSeqLoaders.delete(sessionId); + throw err; + }); + msgSeqLoaders.set(sessionId, loader); + } + await loader; + } const seq = (msgSeq.get(sessionId) ?? 0) + 1; msgSeq.set(sessionId, seq); await saveMessage(driver, sessionId, seq, message.role ?? "unknown", message); } + // ── 消息持久化：门控 + 内存缓冲（Neo4j 掉线时兜底） ──── + + interface BufferedMessage { sessionId: string; message: any } + const messageBuffer: BufferedMessage[] = []; + const MESSAGE_BUFFER_CAP = 2000; + let flushRun: Promise | null = null; + + /** + * 缓冲一条消息。不在缓冲时分配 seq：内存 msgSeq 在 session_end 清理 / + * DB 故障时与 DB 脱节，预分配的 seq 会与已有行撞号，saveMessage 的 + * ON MATCH SET 会静默覆盖旧行内容。seq 统一在 flush 时由 ingestMessage + * 分配（那时 DB 可达，getMaxTurnIndex 恢复能正确兜底）。 + */ + function bufferMessage(sessionId: string, message: any): void { + // 不可序列化的消息（循环引用 / BigInt）永远写不进 DB——当场丢弃， + // 否则它会永久堵在 flush 队列头并反复重跳熔断 + try { JSON.stringify(message); } catch (err) { + api.logger.warn(`[graph-memory-pro] message not serializable, dropped from outage buffer: ${err}`); + return; + } + if (messageBuffer.length >= MESSAGE_BUFFER_CAP) { + messageBuffer.shift(); + api.logger.warn("[graph-memory-pro] message buffer full, dropping oldest buffered message"); + } + messageBuffer.push({ sessionId, message }); + } + + /** + * 恢复后把缓冲消息刷回 Neo4j。single-flight：返回同一个 in-flight + * promise，让 extract / compact 路径能真正等它完成再继续。 + */ + function flushMessageBuffer(): Promise { + if (flushRun) return flushRun; + if (!messageBuffer.length || !neo4jGate.isAvailable()) return Promise.resolve(); + flushRun = (async () => { + let flushed = 0; + try { + while (messageBuffer.length) { + const next = messageBuffer[0]; + try { + await ingestMessage(next.sessionId, next.message); + messageBuffer.shift(); + flushed += 1; + } catch (err) { + neo4jGate.recordFailure(); + api.logger.warn(`[graph-memory-pro] buffered message flush failed, will retry later: ${err}`); + break; + } + } + if (flushed > 0) { + api.logger.info(`[graph-memory-pro] flushed ${flushed} buffered message(s) to neo4j`); + // 补偿掉线期间被熔断跳过的维护：缓冲消息已补录，趁 gate 可用重排一轮 + // （scheduleMaintenance 自带单飞 + gate 检查，无会话时它是安全的 no-op 调用） + scheduleMaintenance(); + } + } finally { + flushRun = null; + } + })(); + return flushRun; + } + + /** + * ingest / afterTurn 共用的落库入口： + * 可用 → 直接写；不可用或写失败 → 缓冲并吞掉错误（不向 host 抛）， + * 恢复后由 flushMessageBuffer 补写。返回的 ingested=true 语义为"引擎已接管该消息"。 + */ + async function persistMessage(sessionId: string, message: any): Promise { + if (!neo4jGate.isAvailable()) { + bufferMessage(sessionId, message); + return; + } + try { + await ingestMessage(sessionId, message); + neo4jGate.recordSuccess(); + void flushMessageBuffer(); + } catch (err) { + neo4jGate.recordFailure(); + bufferMessage(sessionId, message); + api.logger.warn(`[graph-memory-pro] neo4j write failed, message buffered (${messageBuffer.length} pending): ${err}`); + } + } + + // ── recall 超时预算：慢查询不拖回合，回退缓存/降级 ────── + + const RECALL_BUDGET_MS = 5_000; + + // 超时退避：withBudget 只放弃等待、不取消底层查询，反复超时会在后台堆积 + // 占连接的 Neo4j 查询链；冷却窗口内跳过新 recall，直接走缓存/降级。 + const RECALL_BACKOFF_MS = 30_000; + let recallBackoffUntil = 0; + function markRecallBackoffOnTimeout(err: unknown): void { + if (String(err).includes("timed out")) recallBackoffUntil = Date.now() + RECALL_BACKOFF_MS; + } + + /** + * 给 Promise 加等待上限。不取消底层操作（Neo4j 查询会在后台自然完成、 + * 连接归还连接池），只是放弃等待 —— 慢 != 死。 + */ + function withBudget(p: Promise, ms: number, label: string): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms); + p.then( + v => { clearTimeout(timer); resolve(v); }, + e => { clearTimeout(timer); reject(e); }, + ); + }); + } + + // ── 图维护：后台单飞 + trailing rerun（A1） ───────────── + // session_end 不再 await 维护链（衰减→去重→PR→社区→LLM 摘要可能耗时数分钟）； + // 全局单飞修掉多会话并发跑维护的竞态；运行期间的再次请求只标记 rerun， + // 当前一轮结束后最多补跑一次（覆盖"最后一个结束的会话"）。 + + let maintenanceRun: Promise> | { skipped: string } | { failed: string }> | null = null; + let maintenanceRerunRequested = false; + + /** + * 唯一的维护入口（session_end 与 gm_maintain 共用）： + * - 在跑 → 标记 rerun（保留 session_end 的 trailing 补跑语义）并 join + * 同一个 in-flight promise（gm_maintain 据此拿到结果而非并发裸跑） + * - gate 打开（熔断）→ 返回 skipped 标记，不触碰数据库 + * - 空闲 → 自己成为那一轮 + * 并发跑两条维护链会导致 dedup 双计 validatedCount、communityId 互相覆盖。 + */ + function scheduleMaintenance(): Promise> | { skipped: string } | { failed: string }> { + if (!neo4jGate.isAvailable()) { + api.logger.info("[graph-memory-pro] maintenance skipped: neo4j unavailable (circuit open)"); + return Promise.resolve({ skipped: "neo4j unavailable (circuit open)" }); + } + if (maintenanceRun) { + maintenanceRerunRequested = true; + api.logger.info("[graph-memory-pro] maintenance already running, rerun queued + joining in-flight run"); + return maintenanceRun; + } + maintenanceRun = (async () => { + try { + let result: Awaited>; + do { + maintenanceRerunRequested = false; + const embedFn = recaller.embedFn ?? undefined; + result = await runMaintenance(driver, cfg, llm, embedFn); + neo4jGate.recordSuccess(); + api.logger.info( + `[graph-memory-pro] maintenance: ${result.durationMs}ms, ` + + `dedup=${result.dedup.merged}, communities=${result.community.count}, ` + + `summaries=${result.communitySummaries}, ` + + `autoDeprecate=${result.decay.autoDeprecated}, purged=${result.purged}, ` + + (result.retention + ? ("error" in result.retention + ? `retention=failed: ${result.retention.error.slice(0, 120)}, ` + : `retention=${result.retention.dryRun ? "dryRun:" : ""}` + + `${result.retention.deletedRows}/${result.retention.selectedRows} msgs, `) + : "") + + `top_pr=${result.pagerank.topK.slice(0, 3).map(n => `${n.name}(${n.score.toFixed(3)})`).join(",")}`, + ); + } while (maintenanceRerunRequested && neo4jGate.isAvailable()); + return result; + } catch (err) { + neo4jGate.recordFailure(); + api.logger.error(`[graph-memory-pro] maintenance failed: ${err}`); + return { failed: String(err) }; + } finally { + maintenanceRun = null; + maintenanceRerunRequested = false; + } + })(); + return maintenanceRun; + } + + // ── embedding 会话级 re-probe ────────────────────────── + // 启动 probe 失败会让插件停在文本搜索模式直到重启；这里在每个会话开始时 + // 重试（single-flight + 5 分钟退避），临时性故障恢复后自动回到向量召回。 + + const EMBED_REPROBE_INTERVAL_MS = 300_000; + + function ensureEmbeddingReady(): void { + if (!embeddingConfigured || recaller.hasEmbedFn() || embedProbeInFlight) return; + if (Date.now() - lastEmbedProbeAt < EMBED_REPROBE_INTERVAL_MS) return; + embedProbeInFlight = true; + lastEmbedProbeAt = Date.now(); + createEmbedder(cfg.embedding) + .then((embedder) => { + if (embedder) { + recaller.setEmbedFn(embedder.embed); + recaller.setEmbedBatchFn(embedder.embedBatch); + api.logger.info("[graph-memory-pro] embedding re-probe succeeded — vector search re-enabled"); + } + }) + .catch(() => {}) + .finally(() => { embedProbeInFlight = false; }); + } + // ── before_agent_start：召回 ──────────────────────────── api.on("before_agent_start", async (event: any, ctx: any) => { try { + // cron session 关闭图谱功能时不召回（cron 标记在 sessionKey 上，sessionId 是随机 UUID） + if (isCronSessionKey(typeof ctx?.sessionKey === "string" ? ctx.sessionKey : null) && !cronCfg.enabled) return; + const rawPrompt = typeof event?.prompt === "string" ? event.prompt : ""; const prompt = cleanPrompt(rawPrompt); if (!prompt) return; if (prompt.includes("/new or /reset") || prompt.includes("new session was started")) return; + // 熔断开启时跳过召回 —— assemble 也会走降级路径（仅转录文本） + if (!neo4jGate.isAvailable()) return; + // 超时冷却窗口内跳过（后台可能仍有在途查询，不再叠加） + if (Date.now() < recallBackoffUntil) return; api.logger.info(`[graph-memory-pro] recall query: "${prompt.slice(0, 80)}"`); - const res = await recaller.recall(prompt); + const res = await withBudget(recaller.recall(prompt), RECALL_BUDGET_MS, "[graph-memory-pro] recall"); if (res.nodes.length) { const sessionId = typeof ctx?.sessionId === "string" ? ctx.sessionId : undefined; const sessionKey = typeof ctx?.sessionKey === "string" ? ctx.sessionKey : undefined; if (sessionId) { bindSessionIdentity(sessionId, sessionKey); recalled.set(sessionId, res); + recalledPrompt.set(sessionId, prompt); } api.logger.info(`[graph-memory-pro] recalled ${res.nodes.length} nodes, ${res.edges.length} edges`); } } catch (err) { + markRecallBackoffOnTimeout(err); api.logger.warn(`[graph-memory-pro] recall failed: ${err}`); } }); @@ -457,17 +913,31 @@ const graphMemoryProPlugin = { id: "graph-memory-pro", name: "Graph Memory Pro", ownsCompaction: true, + // OpenClaw 2026.3.7+ transcript fencing 契约。未声明时 host 会把引擎 + // 逐回合降级到 legacy（"current-turn transcript fencing is not declared"）。 + // 两项承诺：轮前读取只看到 admitted user entry 之前的精确前缀； + // 轮推进经 commitTurn 以 advancementKey 原子幂等落盘。 + transcriptSemantics: { + currentTurnFence: "before-current-turn-entry-v1", + turnAdvancementIdempotency: "atomic-idempotent-v1", + }, }, async bootstrap({ sessionId, sessionKey }: { sessionId: string; sessionKey?: string }) { bindSessionIdentity(sessionId, sessionKey); + // 每个会话开始时尝试恢复 embedding（启动 probe 失败后的会话级 re-probe） + ensureEmbeddingReady(); return { bootstrapped: true }; }, async ingest({ sessionId, sessionKey, message, isHeartbeat }: { sessionId: string; sessionKey?: string; message: any; isHeartbeat?: boolean }) { if (isHeartbeat) return { ingested: false }; bindSessionIdentity(sessionId, sessionKey); - await ingestMessage(sessionId, message); + // cron session 关闭图谱功能：消息不入库 + if (isCronSessionKey(sessionKey) && !cronCfg.enabled) { + return { ingested: false }; + } + await persistMessage(sessionId, message); ingestedSinceTurn.set(sessionId, (ingestedSinceTurn.get(sessionId) ?? 0) + 1); return { ingested: true }; }, @@ -478,37 +948,13 @@ const graphMemoryProPlugin = { bindSessionIdentity(sessionId, sessionKey); const budget = tokenBudget ?? 128_000; - const activeNodes = await getBySession(driver, sessionId); - const activeEdges: any[] = []; - for (const n of activeNodes) { - activeEdges.push(...await edgesFrom(driver, n.id)); - activeEdges.push(...await edgesTo(driver, n.id)); - } - - // prompt-aware recall：优先用当前 prompt 做新鲜召回，回退到 before_agent_start 缓存 - let rec = recalled.get(sessionId) ?? { nodes: [], edges: [] }; - if (prompt) { - const cleaned = cleanPrompt(prompt); - if (cleaned) { - try { - const freshRec = await recaller.recall(cleaned); - if (freshRec.nodes.length) { - rec = freshRec; - recalled.set(sessionId, freshRec); - } - } catch (err) { - api.logger.warn(`[graph-memory-pro] assemble recall failed: ${err}`); - } - } - } - const totalGmNodes = activeNodes.length + rec.nodes.length; - const prepared = prepareAssemblyMessages(messages); - - if (totalGmNodes === 0) { + // cron session 关闭图谱功能：仅做消息裁剪与配对修复，不注入图谱上下文 + if (isCronSessionKey(sessionKey) && !cronCfg.enabled) { + const prepared = prepareAssemblyMessages(messages, cfg.freshTailCount); if (prepared.dropped > 0) { api.logger.info( `[graph-memory-pro] assemble: ${prepared.messages.length} msgs (~${prepared.tokens} tok), ` + - `dropped ${prepared.dropped} older msgs, graph ~0 tok`, + `dropped ${prepared.dropped} older msgs, graph skipped (cron session)`, ); } return { @@ -517,80 +963,95 @@ const graphMemoryProPlugin = { }; } - const { xml, systemPrompt, tokens: gmTokens } = await assembleContext(driver, { - tokenBudget: budget, - activeNodes, - activeEdges, - recalledNodes: rec.nodes, - recalledEdges: rec.edges, - }); + // prompt-aware recall：clean 后的 prompt 与缓存命中同一查询时直接复用 + // before_agent_start 的结果，只有变化才发起第二次召回 + let rec = recalled.get(sessionId) ?? { nodes: [], edges: [] }; + const cachedPrompt = recalledPrompt.get(sessionId); + const cleanedPrompt = prompt ? cleanPrompt(prompt) : ""; + if (cleanedPrompt && neo4jGate.isAvailable() && Date.now() >= recallBackoffUntil && cleanedPrompt !== cachedPrompt) { + try { + const freshRec = await withBudget(recaller.recall(cleanedPrompt), RECALL_BUDGET_MS, "[graph-memory-pro] assemble recall"); + if (freshRec.nodes.length) { + rec = freshRec; + recalled.set(sessionId, freshRec); + recalledPrompt.set(sessionId, cleanedPrompt); + } + } catch (err) { + markRecallBackoffOnTimeout(err); + api.logger.warn(`[graph-memory-pro] assemble recall failed: ${err}`); + } + } + const prepared = prepareAssemblyMessages(messages, cfg.freshTailCount); + + // 图谱段：门控 + 降级 —— Neo4j 掉线/超时时只返回裁剪后的转录， + // 不让错误抛回 host（原实现无 catch，getBySession 失败会炸掉 assemble） + let graphTokens = 0; + let systemPromptAddition: string | undefined; + if (neo4jGate.isAvailable()) { + try { + const activeNodes = await getBySession(driver, sessionId); + // 单次批量查询替代逐节点 edgesFrom+edgesTo 的 2N 次串行往返 + const activeEdges = await edgesTouching(driver, activeNodes.map(n => n.id)); + + if (activeNodes.length + rec.nodes.length > 0) { + const { xml, systemPrompt, tokens } = await assembleContext(driver, { + tokenBudget: budget, + activeNodes, + activeEdges, + recalledNodes: rec.nodes, + recalledEdges: rec.edges, + }); + graphTokens = tokens; + if (xml) { + systemPromptAddition = systemPrompt ? `${systemPrompt}\n\n${xml}` : xml; + } + } + neo4jGate.recordSuccess(); + void flushMessageBuffer(); + } catch (err) { + neo4jGate.recordFailure(); + api.logger.warn(`[graph-memory-pro] assemble: graph context unavailable, transcript-only: ${err}`); + } + } if (prepared.dropped > 0) { api.logger.info( `[graph-memory-pro] assemble: ${prepared.messages.length} msgs (~${prepared.tokens} tok), ` + - `dropped ${prepared.dropped} older msgs, graph ~${gmTokens} tok`, + `dropped ${prepared.dropped} older msgs, graph ~${graphTokens} tok`, ); } - let systemPromptAddition: string | undefined; - if (xml) { - systemPromptAddition = systemPrompt ? `${systemPrompt}\n\n${xml}` : xml; - } - return { messages: prepared.messages, - estimatedTokens: gmTokens + prepared.tokens, + estimatedTokens: graphTokens + prepared.tokens, ...(systemPromptAddition ? { systemPromptAddition } : {}), }; }, async compact({ sessionId, sessionKey, currentTokenCount }: { sessionId: string; sessionKey?: string; sessionFile: string; tokenBudget?: number; force?: boolean; currentTokenCount?: number }) { bindSessionIdentity(sessionId, sessionKey); - const msgs = await getUnextracted(driver, sessionId, cfg.compactTurnCount * 3); - - if (!msgs.length) return { ok: true, compacted: false, reason: "no messages" }; - - try { - const existing = (await getBySession(driver, sessionId)).map(n => n.name); - const result = await extractor.extract({ messages: msgs, existingNames: existing }); - - const nameToId = new Map(); - for (const nc of result.nodes) { - const { node } = await upsertNode(driver, { - type: nc.type, name: nc.name, - description: nc.description, content: nc.content, - }, sessionId); - nameToId.set(node.name, node.id); - recaller.syncEmbed(node).catch(() => {}); - } - - for (const ec of result.edges) { - const fromNode = await findByName(driver, ec.from); - const toNode = await findByName(driver, ec.to); - const fromId = nameToId.get(ec.from) ?? fromNode?.id; - const toId = nameToId.get(ec.to) ?? toNode?.id; - if (fromId && toId) { - await upsertEdge(driver, { - fromId, toId, type: ec.type, - instruction: ec.instruction, condition: ec.condition, sessionId, - }); - } - } - - const maxTurn = Math.max(...msgs.map((m: any) => m.turn_index)); - await markExtracted(driver, sessionId, maxTurn); - + // cron session 关闭图谱功能或知识提取：不触发 LLM 提取 + if (isCronSessionKey(sessionKey) && !(cronCfg.enabled && cronCfg.extract)) { return { - ok: true, compacted: true, + ok: true, compacted: false, + reason: cronCfg.enabled ? "cron session extraction disabled" : "cron session graph disabled", + }; + } + // 熔断开启时跳过提取：未提取消息保留，恢复后下一次 compact / extract 补上 + if (!neo4jGate.isAvailable()) { + return { ok: true, compacted: false, reason: "neo4j unavailable (circuit open)" }; + } + const res = await extractUnextractedBatch(sessionId); + return { + ok: res.ok, compacted: res.compacted, + ...(res.reason ? { reason: res.reason } : {}), + ...(res.summary ? { result: { - summary: `extracted ${result.nodes.length} nodes, ${result.edges.length} edges`, + summary: res.summary, tokensBefore: currentTokenCount ?? 0, }, - }; - } catch (err) { - api.logger.error(`[graph-memory-pro] compact failed: ${err}`); - return { ok: false, compacted: false, reason: String(err) }; - } + } : {}), + }; }, async afterTurn({ sessionId, sessionKey, messages, prePromptMessageCount, isHeartbeat }: { @@ -607,6 +1068,12 @@ const graphMemoryProPlugin = { return; } + // cron session 关闭图谱功能：跳过入库回填与知识提取 + if (isCronSessionKey(sessionKey) && !cronCfg.enabled) { + ingestedSinceTurn.delete(sessionId); + return; + } + // Official OpenClaw delivers ingest() and afterTurn() as separate // lifecycle phases. Older downstream builds incorrectly call only // afterTurn(). Persist just the missing suffix so neither host loses @@ -614,7 +1081,7 @@ const graphMemoryProPlugin = { const ingestedCount = ingestedSinceTurn.get(sessionId) ?? 0; const missingMessages = missingIngestMessages(newMessages, ingestedCount); for (const message of missingMessages) { - await ingestMessage(sessionId, message); + await persistMessage(sessionId, message); } if (missingMessages.length > 0) { api.logger.warn( @@ -627,10 +1094,75 @@ const graphMemoryProPlugin = { api.logger.info(`[graph-memory-pro] afterTurn sid=${sessionId.slice(0, 8)} turn=${turnNum} rawMsgs=${newMessages.length}`); - // 直接用原始消息提取知识图谱（异步，不阻塞） - extractTurnKnowledge(sessionId, turnNum, newMessages).catch(err => { - api.logger.error(`[graph-memory-pro] extract failed: ${err}`); - }); + // cron session 关闭知识提取：消息仅入库缓冲，可稍后用 `graph-memory extract` 手动回填 + if (isCronSessionKey(sessionKey) && !cronCfg.extract) { + api.logger.info("[graph-memory-pro] cron session: extraction skipped (cron.extract=false)"); + return; + } + + // 按模式分发：per-turn 即时 LLM 提取；batched 攒批（LLM 调用 ~1/N） + scheduleTurnExtraction(sessionId, turnNum, newMessages); + }, + + /** + * OpenClaw 2026.3.7+ transcript fencing 契约的轮推进点（见 info.transcriptSemantics）。 + * host 只对成功接受的轮次调用；重试携带同一 advancementKey。 + * 义务 = 一次以 advancementKey 为键的原子幂等写（GmTurnCommit 唯一约束 + CREATE）： + * 首次 → committed；重试撞约束 → duplicate，副作用不再重放。 + * + * 消息持久化仍归 ingest/afterTurn（契约保证 fenced 路径下 ingest 照常逐条触发； + * afterTurn 的 backfill 依赖 ingestedSinceTurn 计数器，commitTurn 若也回填， + * 两生命周期点并存时会以新 seq 重写整轮消息 → GmMessage 重复行）。 + * 此处只做：标记落盘 + 触发本轮提取 —— 提取与 afterTurn 共用同一条幂等管线 + * （withExtractLock + isTurnExtracted 双守卫），两点并存时后进入者自动空转。 + */ + async commitTurn({ sessionId, sessionKey, advancementKey, messages, isHeartbeat }: { + sessionId?: string; sessionKey?: string; advancementKey?: string; messages?: any[]; isHeartbeat?: boolean; + }) { + if (isHeartbeat || !advancementKey) return { status: "committed" as const }; + if (sessionId) bindSessionIdentity(sessionId, sessionKey); + const sid = sessionId ?? (sessionKey ? sessionIdsByKey.get(sessionKey) : undefined); + + // 宿主只传 advancementKey 且 sessionKey 无历史绑定（缺 bootstrap/ingest） + // 时无法归属会话：标记与提取都不可用。明确告警而非静默 no-op —— + // 返回 committed 只是"不阻塞回合"的降级，不代表已落盘。 + if (!sid) { + api.logger.warn(`[graph-memory-pro] commitTurn: cannot resolve session for advancementKey=${advancementKey.slice(0, 12)}…, marker + extraction skipped`); + return { status: "committed" as const }; + } + + // cron session 关闭图谱功能：整个提交按 no-op 处理（不写标记、不提取） + if (isCronSessionKey(sessionKey) && !cronCfg.enabled) return { status: "committed" as const }; + + let advance: "committed" | "duplicate" = "committed"; + if (neo4jGate.isAvailable()) { + try { + advance = await commitTurnAdvance(driver, sid, advancementKey, messages?.length ?? 0); + neo4jGate.recordSuccess(); + } catch (err) { + // 标记写失败不向 host 抛错：提取副作用自带幂等守卫，host 重试 + // 最多多跑一次空检查；沿用 ingest/assemble 的"降级不炸回合"哲学 + neo4jGate.recordFailure(); + api.logger.warn(`[graph-memory-pro] commitTurn marker write failed (side effects remain idempotent): ${err}`); + } + } + // duplicate 短路：若首次提交的提取本身失败过（LLM 错误只记日志）， + // 这里也不会重放 —— 恢复路径是 compact() / `graph-memory extract` 回填 + if (advance === "duplicate") return { status: "duplicate" as const }; + + if (messages?.length) { + if (isCronSessionKey(sessionKey) && !cronCfg.extract) { + api.logger.info("[graph-memory-pro] cron session: extraction skipped (cron.extract=false)"); + } else { + // 读 turn 编号前先等掉线缓冲落库（与 afterTurn 的 backfill-then-read + // 对齐）：flush 会给缓冲消息分配新 seq，预读的旧值会让 markExtracted + // 只覆盖旧前缀 → 本轮消息保持未提取 → compact 重复提取 + if (messageBuffer.length) await flushMessageBuffer(); + const turnNum = msgSeq.get(sid) ?? 0; + scheduleTurnExtraction(sid, turnNum, messages); + } + } + return { status: "committed" as const }; }, async prepareSubagentSpawn({ parentSessionKey, childSessionKey, parentSessionId }: { @@ -646,8 +1178,12 @@ const graphMemoryProPlugin = { const childSessionId = sessionIdsByKey.get(childSessionKey); if (childSessionId) { recalled.delete(childSessionId); + recalledPrompt.delete(childSessionId); msgSeq.delete(childSessionId); + msgSeqLoaders.delete(childSessionId); + extractLocks.delete(childSessionId); ingestedSinceTurn.delete(childSessionId); + sessionUpdatedHits.delete(childSessionId); } sessionIdsByKey.delete(childSessionKey); pendingSubagentRecall.delete(childSessionKey); @@ -655,14 +1191,22 @@ const graphMemoryProPlugin = { async dispose() { msgSeq.clear(); + msgSeqLoaders.clear(); + extractLocks.clear(); recalled.clear(); + recalledPrompt.clear(); sessionIdsByKey.clear(); pendingSubagentRecall.clear(); ingestedSinceTurn.clear(); + sessionUpdatedHits.clear(); + // 仅当释放的是当前活跃引擎时清空标记 —— 真正的重载（dispose 后重新 + // register）才会走完整初始化路径 + if (activeEngine === engine) activeEngine = null; // 不关闭 Neo4j driver — 连接池自管理生命周期，进程退出时由 OS 回收 }, }; + activeEngine = engine; api.registerContextEngine("graph-memory-pro", () => engine); // ── session_end：finalize + 图维护 ────────────────────── @@ -677,62 +1221,93 @@ const graphMemoryProPlugin = { : typeof ctx?.sessionKey === "string" ? ctx.sessionKey : undefined; try { - const nodes = await getBySession(driver, sid); - if (nodes.length) { - // 获取图谱摘要 - const session = getSession(driver); - let summary = ""; - try { - const summaryResult = await session.run(` - MATCH (n:Task|Skill|Event {status: 'active'}) - RETURN n.name AS name, n.type AS type, n.validatedCount AS vc, n.pagerank AS pr - ORDER BY n.pagerank DESC LIMIT 20 - `); - summary = summaryResult.records - .map(r => `${r.get("type")}:${r.get("name")}(v${r.get("vc")},pr${(r.get("pr") ?? 0).toFixed?.(3) ?? "0"})`) - .join(", "); - } finally { - await session.close(); - } + // cron session：图谱功能关闭或明确禁用时，跳过 finalize 与图维护（finally 清理仍执行） + if (isCronSessionKey(sessionKey) && !(cronCfg.enabled && cronCfg.finalizeAndMaintain)) { + api.logger.info(`[graph-memory-pro] cron session ${sid.slice(0, 12)}…: finalize + maintenance skipped (cron config)`); + return; + } - const fin = await extractor.finalize({ sessionNodes: nodes, graphSummary: summary }); + // 熔断开启时跳过 finalize（全是 Neo4j 写）—— 消息已缓冲，恢复后补齐 + if (!neo4jGate.isAvailable()) { + api.logger.warn(`[graph-memory-pro] session_end ${sid.slice(0, 12)}…: neo4j unavailable (circuit open), finalize + maintenance skipped`); + return; + } - for (const nc of fin.promotedSkills) { - if (nc.name && nc.content) { - await upsertNode(driver, { - type: "SKILL", name: nc.name, - description: nc.description ?? "", content: nc.content, - }, sid); - } - } - for (const ec of fin.newEdges) { - const fromNode = await findByName(driver, ec.from); - const toNode = await findByName(driver, ec.to); - if (fromNode && toNode) { - await upsertEdge(driver, { - fromId: fromNode.id, toId: toNode.id, type: ec.type, - instruction: ec.instruction, sessionId: sid, - }); - } + // batched 模式：会话结束冲洗残留未提取批（攒批未达阈值时保证知识不丢）。 + // drain 循环（上限 5）：中途 LLM 失败可能积压超过单批 limit（compactTurnCount*3）， + // 只冲一批会留尾部知识缺口；ok=false（本批失败）即停，避免同因反复打 LLM。 + // 仍超限的残余保持未提取——retention fail-closed 不删，可再跑 `graph-memory extract` 回填。 + if (extractCfg.mode === "batched") { + for (let i = 0; i < SESSION_END_FLUSH_ROUNDS; i++) { + const flush = await extractUnextractedBatch(sid); + if (!flush.ok || !flush.compacted) break; } - for (const id of fin.invalidations) await deprecate(driver, id); } - // 图维护 - const embedFn = (recaller as any).embed ?? undefined; - const result = await runMaintenance(driver, cfg, llm, embedFn); - api.logger.info( - `[graph-memory-pro] maintenance: ${result.durationMs}ms, ` + - `dedup=${result.dedup.merged}, communities=${result.community.count}, ` + - `summaries=${result.communitySummaries}, ` + - `top_pr=${result.pagerank.topK.slice(0, 3).map(n => `${n.name}(${n.score.toFixed(3)})`).join(",")}`, - ); + let nodes: Awaited>; + try { + nodes = await getBySession(driver, sid); + neo4jGate.recordSuccess(); + void flushMessageBuffer(); + } catch (err) { + neo4jGate.recordFailure(); + api.logger.error(`[graph-memory-pro] session_end error: ${err}`); + return; + } + // finalize 阶梯：规模/EVENT 门（shouldRunFinalize）之外，会话内 upsert 命中过 + // 已有节点也触发——纠错型 invalidations 只有 finalize 能产出，小会话纠错不可漏 + if (nodes.length && (shouldRunFinalize(nodes) || (sessionUpdatedHits.get(sid) ?? 0) > 0)) { + // finalize 的 upsert 与 afterTurn/compact 的提取共用 per-session 互斥锁： + // 最后一轮的 afterTurn 提取可能仍在途，不串行化会重复 upsert（validatedCount 双递增） + await withExtractLock(sid, async () => { + // 图谱摘要：top-pagerank 节点走 store.topNodes（勿在路由/钩子里内联同义 Cypher） + const summary = (await topNodes(driver, 20)) + .map(n => `${n.type}:${n.name}(v${n.validatedCount},pr${n.pagerank.toFixed(3)})`) + .join(", "); + + const fin = await extractor.finalize({ sessionNodes: nodes, graphSummary: summary }); + + // promotedSkills + newEdges 收敛进 persistExtractionResult（单一来源）： + // 旧内联循环是全库唯一不做向量同步的节点写路径 —— 晋升 SKILL 无 embedding， + // 精确召回不可见且无自动补向量机制；边端点解析也未复用 nameToId 零往返模式。 + // 收敛后 upsert → syncEmbedBatch → 建边与 per-turn/compact/CLI 路径完全同源。 + // persistExtractionResult 不做 markExtracted（finalize 无消息语义），无需补。 + const finResult: ExtractionResult = { + nodes: fin.promotedSkills + // parseFinalize 已过滤 name/content，这里保留双保险（与原内联循环一致） + .filter((nc) => nc.name && nc.content) + .map((nc) => ({ + type: "SKILL", + name: nc.name, + description: nc.description ?? "", + content: nc.content, + })), + edges: fin.newEdges, + }; + await persistExtractionResult(driver, recaller, finResult, { sessionId: sid }); + + for (const id of fin.invalidations) await deprecateNodeAndDisconnectById(driver, id); + }); + } else if (nodes.length) { + // finalize 阶梯触发（LLM 成本控制）：小会话/无 EVENT 节点时跳过这次调用 + api.logger.info( + `[graph-memory-pro] session_end ${sid.slice(0, 12)}…: finalize skipped (small session / no EVENT nodes)`, + ); + } + + // 图维护：后台单飞（A1）—— 衰减→去重→PR→社区→LLM 摘要可能耗时数分钟， + // 不再阻塞 session_end；结果只进日志，host 对其零依赖 + scheduleMaintenance(); } catch (err) { api.logger.error(`[graph-memory-pro] session_end error: ${err}`); } finally { msgSeq.delete(sid); + msgSeqLoaders.delete(sid); + extractLocks.delete(sid); recalled.delete(sid); + recalledPrompt.delete(sid); ingestedSinceTurn.delete(sid); + sessionUpdatedHits.delete(sid); if (sessionKey && sessionIdsByKey.get(sessionKey) === sid) { sessionIdsByKey.delete(sessionKey); pendingSubagentRecall.delete(sessionKey); @@ -823,7 +1398,8 @@ const graphMemoryProPlugin = { relatedSkill: Type.Optional(Type.String({ description: "关联的已有技能名" })), }), async execute(_toolCallId: string, p: any) { - const sid = ctx?.sessionKey ?? ctx?.sessionId ?? "manual"; + // 溯源统一用 sessionId（与 getBySession 的会话视图对齐）；无会话上下文才落 "manual" + const sid = ctx?.sessionId ?? "manual"; if (!["TASK", "SKILL", "EVENT"].includes(p.type)) { throw new Error(`[graph-memory-pro] 无效节点类型：${String(p.type)}`); } @@ -848,16 +1424,15 @@ const graphMemoryProPlugin = { label: "Update Graph Memory Node", description: "更新知识图谱中已存在的节点。必须提供精确的节点名称（不存在会报错）。" + - "三种模式：(1) 默认 update —— refine description/content；" + - "(2) delete —— 硬删除节点及其所有关系；" + - "(3) deprecate —— 标记 [DEPRECATED] 并切断所有关系（节点本身保留但被隔离）。", + "两种模式：(1) 默认 update —— refine description/content；" + + "(2) deprecate —— 标记 [DEPRECATED] 并切断所有关系（等效删除：deprecated 节点不可被召回，" + + "维护链会在 purgeAfterDays 天后物理清理）。", parameters: Type.Object({ name: Type.String({ description: "目标节点名称（必须精确匹配已有节点；名称会被标准化：全小写、空格/下划线转连字符）" }), mode: Type.Optional(Type.Union([ Type.Literal("update"), - Type.Literal("delete"), Type.Literal("deprecate"), - ], { description: "操作模式：update（默认，更新 description/content）、delete（硬删除节点+所有关系）、deprecate（标记 [DEPRECATED] 并删除所有关系，节点保留）" })), + ], { description: "操作模式：update（默认，更新 description/content）、deprecate（断联+标记弃用，等效删除）" })), description: Type.Optional( Type.String({ description: "新的一句话说明（one-line summary）。仅 update 模式生效，不传则保留原值" }), ), @@ -869,27 +1444,23 @@ const graphMemoryProPlugin = { _toolCallId: string, p: { name: string; - mode?: "update" | "delete" | "deprecate"; + mode?: "update" | "deprecate"; description?: string; content?: string; }, ) { const mode = p.mode ?? "update"; - const notFoundHint = - `[graph-memory-pro] 未找到名称为 "${p.name}" 的节点。` + - `请检查节点名称是否精确（名称标准化规则：全小写、空格/下划线转连字符、移除非字母数字字符），` + - `或使用 gm_record 创建新节点，也可用 gm_search 搜索已有节点。`; - - if (mode === "delete") { - const deleted = await deleteNode(driver, p.name); - if (!deleted) throw new Error(notFoundHint); - return { - content: [{ - type: "text", - text: `已删除：${deleted.name} (${deleted.type}) —— 节点及其所有关系已从图谱中移除`, - }], - details: { mode, name: deleted.name, type: deleted.type, id: deleted.id }, - }; + const notFoundHint = nodeNotFoundMessage( + p.name, + "或使用 gm_record 创建新节点，也可用 gm_search 搜索已有节点。", + ); + + // mode=delete 已移除（断联弃用等效删除）——为旧调用方保留明确报错而非静默降级为 update + if ((mode as string) === "delete") { + throw new Error( + "[graph-memory-pro] mode=delete 已移除：请改用 mode=deprecate（切断所有关系并标记 [DEPRECATED]，" + + "deprecated 节点不可被召回，等效删除；维护链将在 purgeAfterDays 天后自动物理清理）", + ); } if (mode === "deprecate") { @@ -916,7 +1487,7 @@ const graphMemoryProPlugin = { if (p.description === undefined && p.content === undefined) { throw new Error( "[graph-memory-pro] gm_update mode=update 至少需要提供 description 或 content 中的一个" + - "（如需删除节点请用 mode=delete，如需弃用请用 mode=deprecate）", + "（如需移除节点请用 mode=deprecate —— 断联+标记弃用，等效删除）", ); } const updated = await updateNode(driver, p.name, { @@ -948,16 +1519,14 @@ const graphMemoryProPlugin = { { name: "gm_update" }, ); - const EDGE_TYPE_LITERAL = (label: string) => Type.Literal(label); + // 边类型 union 从 EDGE_TYPES 派生（事实源 types.ts）—— 加新边类型只改一处 const edgeTypeUnion = (description: string) => Type.Union( - [EDGE_TYPE_LITERAL("USED_SKILL"), EDGE_TYPE_LITERAL("SOLVED_BY"), - EDGE_TYPE_LITERAL("REQUIRES"), EDGE_TYPE_LITERAL("PATCHES"), - EDGE_TYPE_LITERAL("CONFLICTS_WITH")], + EDGE_TYPES.map(t => Type.Literal(t)), { description }, ); api.registerTool( - (_ctx: any) => ({ + (ctx: any) => ({ name: "gm_link", label: "Link Graph Memory Nodes", description: @@ -982,7 +1551,7 @@ const graphMemoryProPlugin = { const stored = await upsertEdge(driver, { fromId: fromNode.id, toId: toNode.id, type: p.type, - instruction: p.instruction, condition: p.condition, sessionId: "manual", + instruction: p.instruction, condition: p.condition, sessionId: ctx?.sessionId ?? "manual", }); if (!stored) { throw new Error( @@ -1119,21 +1688,14 @@ const graphMemoryProPlugin = { parameters: Type.Object({}), async execute() { const stats = await getStats(driver); - const session = getSession(driver); - let topPr: any[] = []; - try { - const r = await session.run("MATCH (n:Task|Skill|Event {status:'active'}) RETURN n.name AS name, n.type AS type, n.pagerank AS pr ORDER BY n.pagerank DESC LIMIT 5"); - topPr = r.records.map(rec => ({ name: rec.get("name"), type: rec.get("type"), pr: rec.get("pr") ?? 0 })); - } finally { - await session.close(); - } + const topPr = await topNodes(driver, 5); const text = [ `📊 知识图谱统计（Neo4j）`, `节点：${stats.totalNodes} 个 (${Object.entries(stats.byType).map(([t, c]) => `${t}: ${c}`).join(", ")})`, `边：${stats.totalEdges} 条 (${Object.entries(stats.byEdgeType).map(([t, c]) => `${t}: ${c}`).join(", ")})`, `社区：${stats.communities} 个`, `PageRank Top 5：`, - ...topPr.map((n, i) => ` ${i + 1}. ${n.name} (${n.type}, pr=${(typeof n.pr === "number" ? n.pr : 0).toFixed(4)})`), + ...topPr.map((n, i) => ` ${i + 1}. ${n.name} (${n.type}, pr=${n.pagerank.toFixed(4)})`), ].join("\n"); return { content: [{ type: "text", text }], details: stats }; }, @@ -1145,13 +1707,36 @@ const graphMemoryProPlugin = { (_ctx: any) => ({ name: "gm_maintain", label: "Graph Memory Maintenance", - description: "手动触发图维护：去重、PageRank、社区检测。", + description: "手动触发图维护：衰减评分 + tier 转换、去重、PageRank、社区检测。", parameters: Type.Object({}), async execute() { - const embedFn = (recaller as any).embed ?? undefined; - const result = await runMaintenance(driver, cfg, llm, embedFn); + // 走 scheduleMaintenance 单飞入口：后台维护在跑时 join 而非并发裸跑 + // （并发会导致 dedup 双计 validatedCount、communityId 互相覆盖） + const result = await scheduleMaintenance(); + if (!("decay" in result)) { + const reason = "skipped" in result ? result.skipped : result.failed; + return { + content: [{ type: "text", text: `⚠️ 图维护未完成：${reason}` }], + details: result, + }; + } + const t = result.decay.tierTransitions; + const totalTransitions = t.coreToWorking + t.workingToPeripheral + t.peripheralToWorking + t.workingToCore; const text = [ `🔧 图维护完成（${result.durationMs}ms）`, + result.decay.enabled + ? `衰减：扫描 ${result.decay.scanned} 个节点，tier 转换 ${totalTransitions} 次` + + (totalTransitions > 0 + ? `（core→working ${t.coreToWorking}，working→peripheral ${t.workingToPeripheral}，peripheral→working ${t.peripheralToWorking}，working→core ${t.workingToCore}）` + : "") + + (result.decay.autoDeprecated > 0 + ? `\n自动弃用：${result.decay.autoDeprecated} 个长期未用的 peripheral 节点已断联并标记 [DEPRECATED]` + + (result.decay.autoDeprecateError ? `（失败：${result.decay.autoDeprecateError.slice(0, 120)}）` : "") + : "") + : `衰减：已禁用`, + result.purged > 0 + ? `过期清理：硬删 ${result.purged} 个弃用超期的节点` + : "", `去重：${result.dedup.pairs.length} 对相似，合并 ${result.dedup.merged} 对`, ...(result.dedup.pairs.length > 0 ? result.dedup.pairs.slice(0, 5).map(p => ` "${p.nameA}" ≈ "${p.nameB}" (${(p.similarity * 100).toFixed(1)}%)`) @@ -1160,8 +1745,8 @@ const graphMemoryProPlugin = { `社区描述：${result.communitySummaries} 个`, `PageRank Top 5：`, ...result.pagerank.topK.slice(0, 5).map((n, i) => ` ${i + 1}. ${n.name} (${n.score.toFixed(4)})`), - ].join("\n"); - return { content: [{ type: "text", text }], details: { durationMs: result.durationMs, dedupMerged: result.dedup.merged, communities: result.community.count } }; + ].filter(Boolean).join("\n"); + return { content: [{ type: "text", text }], details: { durationMs: result.durationMs, decayTransitions: totalTransitions, autoDeprecated: result.decay.autoDeprecated, purged: result.purged, dedupMerged: result.dedup.merged, communities: result.community.count } }; }, }), { name: "gm_maintain" }, diff --git a/openclaw.plugin.json b/openclaw.plugin.json index f54c9d0..ff9700f 100755 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -23,12 +23,62 @@ } }, "compactTurnCount": { "type": "number", "default": 6 }, - "recallMaxNodes": { "type": "number", "default": 3 }, + "recallMaxNodes": { "type": "number", "default": 6 }, "recallMaxDepth": { "type": "number", "default": 2 }, - "freshTailCount": { "type": "number", "default": 10 }, + "freshTailCount": { "type": "number", "default": 5, "description": "assemble 转录保留的最近轮数（裁剪窗口）" }, "dedupThreshold": { "type": "number", "default": 0.90 }, "pagerankDamping": { "type": "number", "default": 0.85 }, "pagerankIterations": { "type": "number", "default": 20 }, + "decay": { + "type": "object", + "description": "柔性衰减：三因子加权评分（recency+frequency+intrinsic）+ tier 双向转换（core/working/peripheral），以及两阶段生命周期（长期未用的 peripheral 低分节点自动断联弃用；弃用超期的节点硬删释放存储）。完整公式与调参指南见 docs/decay.md。recencyWeight + frequencyWeight + intrinsicWeight 推荐和为 1（运行时会自动归一化）。", + "properties": { + "enabled": { "type": "boolean", "default": true, "description": "是否启用自动衰减。关闭后 tier 永久保持初始 working 状态，且自动弃用（autoDeprecate）一并停用。" }, + "recencyHalfLifeDays": { "type": "number", "default": 30, "description": "Recency 半衰期（天）。effectiveHL = halfLife * exp(importanceModulation * importance)。" }, + "recencyWeight": { "type": "number", "default": 0.4, "description": "Recency 在 composite 中的权重。三个权重推荐和为 1。" }, + "importanceModulation": { "type": "number", "default": 1.5, "description": "半衰期调制系数；越大则高 importance 节点衰减越慢。" }, + "frequencyWeight": { "type": "number", "default": 0.3, "description": "Frequency 在 composite 中的权重。三个权重推荐和为 1。" }, + "intrinsicWeight": { "type": "number", "default": 0.3, "description": "Intrinsic（importance × confidence）在 composite 中的权重。三个权重推荐和为 1。" }, + "betaCore": { "type": "number", "default": 0.8, "description": "core tier 的 Weibull 形状参数；<1 = 缓衰。" }, + "betaWorking": { "type": "number", "default": 1.0, "description": "working tier 的 Weibull 形状参数；=1 = 标准指数衰减。" }, + "betaPeripheral": { "type": "number", "default": 1.3, "description": "peripheral tier 的 Weibull 形状参数；>1 = 加速衰减。" }, + "coreAccessThreshold": { "type": "number", "default": 10, "description": "working→core 所需的最低 validatedCount。" }, + "coreCompositeThreshold": { "type": "number", "default": 0.7, "description": "working→core 所需的最低 composite 分数。" }, + "coreImportanceThreshold": { "type": "number", "default": 0.8, "description": "working→core 所需的最低归一化 importance。" }, + "peripheralCompositeThreshold": { "type": "number", "default": 0.15, "description": "composite 低于此值触发 demote（core→working 或 working→peripheral）。" }, + "peripheralAgeDays": { "type": "number", "default": 60, "description": "working→peripheral 的年龄阈值（同时 validatedCount < workingAccessThreshold 才触发）。" }, + "workingAccessThreshold": { "type": "number", "default": 3, "description": "demote（count 不足时）/ promote（count 充足时）的 access 次数分界。" }, + "workingCompositeThreshold": { "type": "number", "default": 0.4, "description": "peripheral→working 所需的最低 composite 分数。" }, + "autoDeprecate": { "type": "boolean", "default": true, "description": "遗忘曲线自动弃用（两阶段生命周期·阶段一）：tier=peripheral 且 composite 低于 peripheralCompositeThreshold 且超过 autoDeprecateAfterDays 天未访问的节点，维护时自动切断所有边并标记 deprecated。受 enabled 总开关约束。被重新提取/编辑命中时自动复活。" }, + "autoDeprecateAfterDays": { "type": "number", "default": 30, "description": "自动弃用的未访问天数门槛（距 lastAccessedAt）。" }, + "purgeAfterDays": { "type": "number", "default": 60, "description": "两阶段生命周期·阶段二：deprecated 节点自弃用起超过该天数后硬删（DETACH DELETE，向量随节点移除）。0 = 永不硬删。适用于所有 deprecated 节点（含手动弃用与合并产物）。" } + } + }, + "cron": { + "type": "object", + "description": "cron 定时会话的图谱行为开关（host 将 cron 标记放在 sessionKey 上，形如 agent::cron:）。默认全部启用，与普通会话一致。注意：cron 任务若显式设置自定义 sessionKey，则无法被识别为 cron 会话。", + "properties": { + "enabled": { "type": "boolean", "default": true, "description": "是否在 cron session 内启用图谱功能（召回注入 + 消息入库）。关闭后 cron 会话不自动召回、不自动入库；gm_* 工具仍可手动调用。" }, + "extract": { "type": "boolean", "default": true, "description": "是否在 cron session 内触发知识提取（afterTurn / compact 的 LLM 三元组提取）。关闭后消息仍入库缓冲，可用 openclaw graph-memory extract 手动回填。" }, + "finalizeAndMaintain": { "type": "boolean", "default": true, "description": "cron session 结束时是否执行 finalize（EVENT→SKILL 晋升）和图维护（decay/PageRank/社区检测）。频繁的 cron 任务可关闭以避免每次结束都跑全局维护。" } + } + }, + "messageRetention": { + "type": "object", + "description": "原始消息（GmMessage）有界保留策略。默认 keep=all 永不删除；上下文压缩不构成删除持久证据的授权。挂在图维护链尾部，每个维护周期最多处理 batchSize 行。建议先配 dryRun=true 观察候选集再实际启用。", + "properties": { + "keep": { + "type": "string", + "enum": ["all", "referenced", "recent"], + "default": "all", + "description": "all=全部保留（默认）；referenced=只删已提取完成且实际产出知识的消息（producedKnowledge=true，知识已固化进图谱；LLM 空提取的轮次保留原始证据，重挖需手动重置 extracted 后跑 graph-memory extract）；recent=在 referenced 基础上按时间窗保护最近内容（需配 recentTurns 或 retentionDays）" + }, + "recentTurns": { "type": "number", "default": 0, "description": "keep=recent：每 session 保留最近 N 轮真实用户发言（该轮及其后的全部消息保留）。" }, + "retentionDays": { "type": "number", "default": 0, "description": "keep=recent：保留最近 N 天内入库的消息。" }, + "batchSize": { "type": "number", "default": 500, "description": "单个维护周期最多处理的行数（1~10000），保证维护链工作有界。" }, + "dryRun": { "type": "boolean", "default": false, "description": "true 时只报告候选集不删除 —— 启用前先跑一轮 dryRun 验证。" } + } + }, "llm": { "type": "object", "properties": { diff --git a/package.json b/package.json index 4ea628f..2c0112d 100755 --- a/package.json +++ b/package.json @@ -24,8 +24,9 @@ "test:watch": "vitest --passWithNoTests" }, "dependencies": { + "@sinclair/typebox": "^0.34.48", "neo4j-driver": "^5.27.0", - "@sinclair/typebox": "^0.34.48" + "opencode-ai": "^1.18.16" }, "devDependencies": { "@types/node": "^20.0.0", diff --git a/setup-graph-memory-pro.sh b/setup-graph-memory-pro.sh index 6307a09..1e9baf8 100644 --- a/setup-graph-memory-pro.sh +++ b/setup-graph-memory-pro.sh @@ -79,6 +79,7 @@ NEO4J_USER="neo4j" NEO4J_URI="" # 留空 → 根据是否自建 Neo4j 自动决定 PLUGIN_REF="" INTERACTIVE=true +PC="" # 嵌入式 provider 选择（1-7）；交互模式由 read 赋值，非交互留空 AUTOSTART_METHODS=() # configure_autostart 写入；卸载与完成提示读取 while [[ $# -gt 0 ]]; do case "$1" in diff --git a/src/cli-extract.ts b/src/cli-extract.ts new file mode 100644 index 0000000..f1b606f --- /dev/null +++ b/src/cli-extract.ts @@ -0,0 +1,256 @@ +/** + * graph-memory-pro CLI — `openclaw graph-memory extract` + * + * 对未被提取的会话消息做批量图谱提取，补齐因 compact 未触发、提取失败或 + * 进程退出而残留的 GmMessage。流程镜像 index.ts 的 compact() 路径： + * getUnextracted → extractor.extract → upsertNode + syncEmbed → upsertEdge → markExtracted + * + * 命令在 cli-metadata 模式下运行（register() 早早 return），所以这里必须自行 + * 完成 Neo4j driver / schema / LLM / embedder / Extractor / Recaller 的初始化。 + */ + +import type { Driver } from "neo4j-driver"; +import type { GmConfig } from "./types.ts"; +import { getDriver, initSchema, closeDriver } from "./store/db.ts"; +import { + listUnextractedSessions, + getUnextracted, + markExtracted, + getBySession, + type UnextractedSessionInfo, +} from "./store/store.ts"; +import { createCompleteFn, resolveProvider } from "./engine/llm.ts"; +import { createEmbedder } from "./engine/embed.ts"; +import { Recaller } from "./recaller/recall.ts"; +import { Extractor } from "./extractor/extract.ts"; +import { persistExtractionResult } from "./extractor/persist.ts"; +import { defaultLog, makeDefaultPrompt } from "./cli-io.ts"; + +const AFFIRMATIVE = new Set(["y", "yes", "yeah", "yep", "ok", "okay", "true", "1", "confirm"]); + +export function isAffirmative(answer: string): boolean { + return AFFIRMATIVE.has(answer.trim().toLowerCase()); +} + +export interface BackfillExtractOptions { + yes?: boolean; + limit?: number; + session?: string; + dryRun?: boolean; +} + +export interface BackfillExtractParams { + cfg: GmConfig; + effectiveModel: string; + options: BackfillExtractOptions; + log?: (msg: string) => void; + prompt?: (question: string) => Promise; +} + +export interface BackfillExtractResult { + sessionsTotal: number; + sessionsProcessed: number; + sessionsSkipped: number; + nodesCreated: number; + edgesCreated: number; + batches: number; + durationMs: number; +} + +const DEFAULT_BATCH_LIMIT_MULTIPLIER = 3; + +function formatSessionLine(info: UnextractedSessionInfo, index: number): string { + const created = info.minCreatedAt > 0 + ? new Date(info.minCreatedAt).toISOString().replace("T", " ").slice(0, 19) + : "?"; + return ` ${String(index + 1).padStart(3, " ")}. sid=${info.sessionId.slice(0, 12)}… msgs=${info.messageCount} maxTurn=${info.maxTurn} since=${created}`; +} + +export async function runBackfillExtraction( + params: BackfillExtractParams, +): Promise { + const start = Date.now(); + const log = params.log ?? defaultLog; + const opts = params.options; + const cfg = params.cfg; + + const result: BackfillExtractResult = { + sessionsTotal: 0, + sessionsProcessed: 0, + sessionsSkipped: 0, + nodesCreated: 0, + edgesCreated: 0, + batches: 0, + durationMs: 0, + }; + + if (!cfg.neo4j?.uri) { + throw new Error( + "[graph-memory-pro] extract 需要 neo4j.uri 配置。请在 graph-memory-pro 插件配置中设置 neo4j.uri / neo4j.user / neo4j.password。", + ); + } + + if (!params.effectiveModel) { + throw new Error( + "[graph-memory-pro] extract 需要一个 LLM model。请在 config.llm.model 或 agents.defaults.model 中设置。", + ); + } + + const providerInfo = resolveProvider(cfg.llm); + if (providerInfo.provider === "anthropic" && !cfg.llm?.apiKey) { + throw new Error("[graph-memory-pro] llm.provider=anthropic 但未配 llm.apiKey，无法提取。"); + } + if (providerInfo.provider === "openai" && (!cfg.llm?.apiKey || !cfg.llm?.baseURL)) { + throw new Error("[graph-memory-pro] llm.provider=openai 需要 llm.apiKey + llm.baseURL，无法提取。"); + } + if (providerInfo.provider === "oauth" && !cfg.llm?.oauthPath) { + throw new Error( + "[graph-memory-pro] llm.provider=oauth 但未配 llm.oauthPath。请先运行 `openclaw graph-memory auth login`。", + ); + } + + const driver: Driver = getDriver(cfg.neo4j); + + try { + log("[graph-memory-pro] 正在初始化 Neo4j schema..."); + await initSchema(driver, cfg.embedding); + + log("[graph-memory-pro] 正在初始化 LLM 与 embedder..."); + const llm = createCompleteFn(params.effectiveModel, cfg.llm); + const extractor = new Extractor(llm); + const recaller = new Recaller(driver, cfg); + const embedder = await createEmbedder(cfg.embedding); + if (embedder) { + recaller.setEmbedFn(embedder.embed); + recaller.setEmbedBatchFn(embedder.embedBatch); + log("[graph-memory-pro] embedding 已就绪，新节点将批量同步向量。"); + } else { + log("[graph-memory-pro] 未配置 embedding，跳过向量同步（dual-path recall 会降级为文本搜索）。"); + } + + let sessions = await listUnextractedSessions(driver); + if (opts.session) { + sessions = sessions.filter(s => s.sessionId === opts.session); + if (!sessions.length) { + log(`[graph-memory-pro] --session=${opts.session} 没有匹配到含未提取消息的会话。`); + result.durationMs = Date.now() - start; + return result; + } + } + result.sessionsTotal = sessions.length; + + if (sessions.length === 0) { + log("[graph-memory-pro] 没有需要提取的会话。"); + result.durationMs = Date.now() - start; + return result; + } + + const totalMessages = sessions.reduce((s, info) => s + info.messageCount, 0); + log(`[graph-memory-pro] 发现 ${sessions.length} 个会话共 ${totalMessages} 条未提取消息：`); + sessions.forEach((info, i) => log(formatSessionLine(info, i))); + + if (opts.dryRun) { + log("[graph-memory-pro] --dry-run 模式，未执行提取。"); + result.sessionsSkipped = sessions.length; + result.durationMs = Date.now() - start; + return result; + } + + if (!opts.yes) { + const prompt = params.prompt ?? makeDefaultPrompt("GRAPH_MEMORY_EXTRACT_CONFIRM"); + const answer = await prompt(`\n将对以上 ${sessions.length} 个会话发起 LLM 提取，继续？[y/N] `); + if (!isAffirmative(answer)) { + log("[graph-memory-pro] 已取消。"); + result.sessionsSkipped = sessions.length; + result.durationMs = Date.now() - start; + return result; + } + } + + const batchLimit = opts.limit && opts.limit > 0 + ? opts.limit + : Math.max(1, cfg.compactTurnCount) * DEFAULT_BATCH_LIMIT_MULTIPLIER; + + log(`\n[graph-memory-pro] 开始提取（每批最多 ${batchLimit} 条消息）...`); + + for (const info of sessions) { + log(`\n[graph-memory-pro] 会话 ${info.sessionId.slice(0, 12)}… (${info.messageCount} 条消息)`); + try { + const processed = await extractSessionLoop(driver, extractor, recaller, info.sessionId, batchLimit, log); + result.nodesCreated += processed.nodes; + result.edgesCreated += processed.edges; + result.batches += processed.batches; + result.sessionsProcessed += 1; + log(` -> 完成：${processed.nodes} 节点 / ${processed.edges} 边 / ${processed.batches} 批`); + } catch (err) { + result.sessionsSkipped += 1; + log(` -> 失败：${err instanceof Error ? err.message : String(err)}`); + } + } + + result.durationMs = Date.now() - start; + + log( + `\n[graph-memory-pro] 提取完成：${result.sessionsProcessed}/${result.sessionsTotal} 会话，` + + `${result.nodesCreated} 节点，${result.edgesCreated} 边，${result.batches} 批，` + + `用时 ${(result.durationMs / 1000).toFixed(1)}s`, + ); + return result; + } finally { + await closeDriver(); + } +} + +interface SessionExtractStats { + nodes: number; + edges: number; + batches: number; +} + +async function extractSessionLoop( + driver: Driver, + extractor: Extractor, + recaller: Recaller, + sessionId: string, + batchLimit: number, + log: (msg: string) => void, +): Promise { + const stats: SessionExtractStats = { nodes: 0, edges: 0, batches: 0 }; + const hardBatchCeiling = 50; + let exhausted = false; + + for (let i = 0; i < hardBatchCeiling; i++) { + const msgs = await getUnextracted(driver, sessionId, batchLimit); + if (!msgs.length) break; + + stats.batches += 1; + const existing = (await getBySession(driver, sessionId)).map(n => n.name); + const extraction = await extractor.extract({ messages: msgs, existingNames: existing }); + + // awaitEmbedSync=true：closeDriver 在 finally 里执行，最后一批的 embedding + // 请求必须等完 —— 不等待会撞上已关闭的 driver 且错误被吞 + // （markExtracted 已执行，重跑 extract 不会补向量）。 + const outcome = await persistExtractionResult(driver, recaller, extraction, { + sessionId, + awaitEmbedSync: true, + onNodeUpserted: () => { stats.nodes += 1; }, + }); + stats.edges += outcome.edges; + + const maxTurn = msgs.reduce((m, msg) => Math.max(m, msg.turn_index ?? 0), 0); + await markExtracted( + driver, sessionId, maxTurn, + extraction.nodes.length > 0 || extraction.edges.length > 0, + ); + log(` batch ${stats.batches}: ${msgs.length} 消息 -> ${extraction.nodes.length} 节点 / ${extraction.edges.length} 边（累计 ${stats.nodes}/${stats.edges}）`); + + if (msgs.length < batchLimit) break; + if (i === hardBatchCeiling - 1) exhausted = true; + } + + if (exhausted) { + log(` 警告：达到批数上限 ${hardBatchCeiling}，会话 ${sessionId.slice(0, 12)}… 仍有未提取消息，请再次运行。`); + } + + return stats; +} diff --git a/src/cli-io.ts b/src/cli-io.ts new file mode 100644 index 0000000..692a3eb --- /dev/null +++ b/src/cli-io.ts @@ -0,0 +1,26 @@ +/** + * graph-memory-pro CLI — 共享终端 IO（log / 确认提示）。 + * extract 与 reembed 子命令共用；确认提示以环境变量名参数化—— + * 非交互 stdin 且该环境变量未设置时返回空串（视为拒绝），与两命令原语义一致。 + */ + +import readline from "node:readline/promises"; +import { stdin as input, stdout as output } from "node:process"; + +export function defaultLog(msg: string): void { + console.log(msg); +} + +export function makeDefaultPrompt(confirmEnvVar: string): (question: string) => Promise { + return async (question: string): Promise => { + if (!process.stdin.isTTY && process.env[confirmEnvVar] === undefined) { + return ""; + } + const rl = readline.createInterface({ input, output }); + try { + return await rl.question(question); + } finally { + rl.close(); + } + }; +} diff --git a/src/cli-reembed.ts b/src/cli-reembed.ts new file mode 100644 index 0000000..59ea2d0 --- /dev/null +++ b/src/cli-reembed.ts @@ -0,0 +1,313 @@ +/** + * graph-memory-pro CLI — `openclaw graph-memory reembed` + * + * 换 embedding 模型后的一次性重建管线。旧模型的向量与新模型不可比（维度甚至 + * 不同），而向量只写不删 —— recall/dedup 的 `WHERE embedding IS NOT NULL` 防御 + * 会让旧向量静默失效（dedup 维度不符时整个向量查询直接报错）。流程： + * 1. 探测当前模型输出维度，对照 gm_node_embedding / gm_community_embedding 索引维度 + * （不符时默认中止；--recreate-index 删除索引并按配置重建） + * 2. 清空（void）所有 MemoryNode.embedding + contentHash、Community.embedding + * 3. 用当前 embedding 模型按批重建（标准端点走批量 input[]；批失败退化为逐条） + * 文本格式复用 buildNodeEmbeddingText，hash 语义与运行时 syncEmbed 一致。 + * + * 命令在 cli-metadata 模式下运行（register() 早早 return），所以这里自行完成 + * Neo4j driver / schema / embedder 的初始化，并在 finally 中 closeDriver。 + */ + +import type { Driver } from "neo4j-driver"; +import type { GmConfig } from "./types.ts"; +import { getDriver, initSchema, closeDriver } from "./store/db.ts"; +import { + getEmbeddingStats, + clearAllEmbeddings, + listNodeEmbeddingTargets, + listCommunityEmbeddingTargets, + saveVector, + saveCommunityEmbedding, + getVectorIndexDimensions, + dropVectorIndexes, +} from "./store/store.ts"; +import { createEmbedder, type Embedder } from "./engine/embed.ts"; +import { buildNodeEmbeddingText } from "./recaller/recall.ts"; +import { isAffirmative } from "./cli-extract.ts"; +import { defaultLog, makeDefaultPrompt } from "./cli-io.ts"; + +export const DEFAULT_REEMBED_BATCH = 32; +const MAX_REEMBED_BATCH = 256; + +export interface ReembedOptions { + yes?: boolean; + dryRun?: boolean; + batch?: number; + recreateIndex?: boolean; +} + +export interface ReembedParams { + cfg: GmConfig; + options: ReembedOptions; + log?: (msg: string) => void; + prompt?: (question: string) => Promise; +} + +export interface ReembedResult { + clearedNodes: number; + clearedCommunities: number; + nodesEmbedded: number; + nodesFailed: number; + communitiesEmbedded: number; + communitiesFailed: number; + batches: number; + recreatedIndex: boolean; + durationMs: number; +} + +// ─── 维度决策（纯函数，可测） ───────────────────────────────── + +export type ReembedPlan = + | { action: "run" } + | { action: "recreate-index"; reason: string } + | { action: "abort"; reason: string }; + +/** + * 对照"当前模型输出维度"与"两个向量索引的实际维度"决定行动。 + * 索引不存在（全新库 / 索引被删）→ run：initSchema 已按当前配置重建。 + */ +export function planReembed(params: { + probeDim: number; + nodeIndexDim: number | null; + communityIndexDim: number | null; + recreateIndex?: boolean; +}): ReembedPlan { + const { probeDim, nodeIndexDim, communityIndexDim, recreateIndex } = params; + const dims = [...new Set([nodeIndexDim, communityIndexDim].filter( + (d): d is number => typeof d === "number", + ))]; + if (!dims.length) return { action: "run" }; + if (!dims.some(d => d !== probeDim)) return { action: "run" }; + + const dimText = dims.join("/"); + if (recreateIndex) { + return { + action: "recreate-index", + reason: `向量索引维度（${dimText}）与当前模型输出（${probeDim}）不符，将删除索引并按配置重建`, + }; + } + return { + action: "abort", + reason: + `向量索引维度（${dimText}）与当前 embedding 模型输出维度（${probeDim}）不一致。` + + `换模型后必须重建索引：加 --recreate-index 让本命令删除并按新维度重建（旧向量会一并作废），` + + `或把 embedding.dimensions 改回 ${dimText}。`, + }; +} + +function clampBatch(batch: number | undefined): number { + if (!batch || !Number.isFinite(batch) || batch < 1) return DEFAULT_REEMBED_BATCH; + return Math.min(Math.floor(batch), MAX_REEMBED_BATCH); +} + +/** 单批嵌入：优先批量调用，失败退化为逐条请求（兼容批量响应结构未知的 provider） */ +async function embedTexts( + embedder: Embedder, + texts: string[], + log: (msg: string) => void, +): Promise<{ vecs: (number[] | null)[]; batchCalls: number }> { + try { + const vecs = await embedder.embedBatch(texts, "db"); + return { vecs, batchCalls: 1 }; + } catch (err) { + if (texts.length === 1) throw err; + log(` 批量调用失败（${err instanceof Error ? err.message : String(err)}），本批退化为逐条请求`); + } + const vecs: (number[] | null)[] = []; + for (const text of texts) { + try { + vecs.push(await embedder.embed(text, "db")); + } catch { + vecs.push(null); + } + } + return { vecs, batchCalls: texts.length }; +} + +export async function runReembed(params: ReembedParams): Promise { + const start = Date.now(); + const log = params.log ?? defaultLog; + const opts = params.options; + const cfg = params.cfg; + const batch = clampBatch(opts.batch); + + const result: ReembedResult = { + clearedNodes: 0, + clearedCommunities: 0, + nodesEmbedded: 0, + nodesFailed: 0, + communitiesEmbedded: 0, + communitiesFailed: 0, + batches: 0, + recreatedIndex: false, + durationMs: 0, + }; + + if (!cfg.neo4j?.uri) { + throw new Error( + "[graph-memory-pro] reembed 需要 neo4j.uri 配置。请在 graph-memory-pro 插件配置中设置 neo4j.uri / neo4j.user / neo4j.password。", + ); + } + if (!cfg.embedding || (!cfg.embedding.apiKey && !cfg.embedding.baseURL)) { + throw new Error( + "[graph-memory-pro] reembed 需要 embedding 配置（embedding.apiKey 或 embedding.baseURL）。" + + "请先在插件配置中指向新的 embedding 模型。", + ); + } + + const driver: Driver = getDriver(cfg.neo4j); + + try { + log("[graph-memory-pro] 正在初始化 Neo4j schema..."); + await initSchema(driver, cfg.embedding); + + log("[graph-memory-pro] 正在探测 embedding 端点..."); + const embedder = await createEmbedder(cfg.embedding); + if (!embedder) { + throw new Error("[graph-memory-pro] embedding 端点探测失败，无法重嵌入。请检查 embedding.apiKey / baseURL / model。"); + } + const probeDim = (await embedder.embed("ping", "query")).length; + + // ── 维度对照 ── + const indexDims = await getVectorIndexDimensions(driver); + const plan = planReembed({ + probeDim, + nodeIndexDim: indexDims.gm_node_embedding, + communityIndexDim: indexDims.gm_community_embedding, + recreateIndex: opts.recreateIndex, + }); + + const stats = await getEmbeddingStats(driver); + log(`\n[graph-memory-pro] 当前模型输出维度: ${probeDim}`); + log(`[graph-memory-pro] 向量索引维度: node=${indexDims.gm_node_embedding ?? "(未建)"} community=${indexDims.gm_community_embedding ?? "(未建)"}`); + log(`[graph-memory-pro] 活跃节点: ${stats.nodesTotal}（已有向量 ${stats.nodesEmbedded}，待重建 ${stats.nodesTotal - stats.nodesEmbedded}）`); + log(`[graph-memory-pro] 社区: ${stats.communitiesTotal}（已有向量 ${stats.communitiesEmbedded}）`); + + if (plan.action === "abort") { + throw new Error(`[graph-memory-pro] ${plan.reason}`); + } + if (plan.action === "recreate-index") { + log(`[graph-memory-pro] ${plan.reason}`); + if (opts.dryRun) { + log("[graph-memory-pro] --dry-run 模式，未执行任何更改。"); + result.durationMs = Date.now() - start; + return result; + } + await dropVectorIndexes(driver); + await initSchema(driver, cfg.embedding); + result.recreatedIndex = true; + log("[graph-memory-pro] 向量索引已按当前配置重建。"); + } + + if (opts.dryRun) { + const pending = stats.nodesTotal - stats.nodesEmbedded + stats.communitiesTotal - stats.communitiesEmbedded; + log(`\n[graph-memory-pro] --dry-run：将清除全部现有向量并按每批 ${batch} 条重建约 ${pending} 条 embedding，未执行任何更改。`); + result.durationMs = Date.now() - start; + return result; + } + + if (!opts.yes) { + const prompt = params.prompt ?? makeDefaultPrompt("GRAPH_MEMORY_REEMBED_CONFIRM"); + const answer = await prompt( + `\n将清除现有节点/社区向量并用当前模型重建（每批 ${batch} 条），继续？[y/N] `, + ); + if (!isAffirmative(answer)) { + log("[graph-memory-pro] 已取消。"); + result.durationMs = Date.now() - start; + return result; + } + } + + // ── 清空旧向量 ── + const cleared = await clearAllEmbeddings(driver); + result.clearedNodes = cleared.nodes; + result.clearedCommunities = cleared.communities; + log(`\n[graph-memory-pro] 已清除 ${cleared.nodes} 个节点 / ${cleared.communities} 个社区的旧向量。`); + + // ── 节点重建（游标分页：重建会让 NULL 集合收缩，SKIP 分页会跳号） ── + log(`[graph-memory-pro] 开始重建节点向量（每批 ${batch} 条）...`); + let nodeCursor = ""; + for (;;) { + const targets = await listNodeEmbeddingTargets(driver, nodeCursor, batch); + if (!targets.length) break; + nodeCursor = targets[targets.length - 1].id; + + const texts = targets.map(t => buildNodeEmbeddingText(t)); + const embeddable = targets + .map((t, i) => ({ t, text: texts[i] })) + .filter(x => x.text.trim().length > 0); + + if (embeddable.length < targets.length) { + const skipped = targets.length - embeddable.length; + result.nodesFailed += skipped; + log(` 跳过 ${skipped} 个文本为空的节点（保持无向量，不影响文本搜索）`); + } + + if (embeddable.length) { + const { vecs, batchCalls } = await embedTexts( + embedder, + embeddable.map(x => x.text), + log, + ); + result.batches += batchCalls; + for (let i = 0; i < embeddable.length; i++) { + const vec = vecs[i]; + if (!vec) { + result.nodesFailed += 1; + continue; + } + await saveVector(driver, embeddable[i].t.id, embeddable[i].text, vec); + result.nodesEmbedded += 1; + } + } + + log(` 进度: 已重建 ${result.nodesEmbedded}，失败/跳过 ${result.nodesFailed}（游标 ${nodeCursor.slice(0, 12)}…）`); + } + + // ── 社区重建 ── + log("[graph-memory-pro] 开始重建社区向量..."); + let communityCursor = ""; + for (;;) { + const targets = await listCommunityEmbeddingTargets(driver, communityCursor, batch); + if (!targets.length) break; + communityCursor = targets[targets.length - 1].id; + + const { vecs, batchCalls } = await embedTexts( + embedder, + targets.map(t => t.summary), + log, + ); + result.batches += batchCalls; + for (let i = 0; i < targets.length; i++) { + const vec = vecs[i]; + if (!vec) { + result.communitiesFailed += 1; + continue; + } + await saveCommunityEmbedding(driver, targets[i].id, vec); + result.communitiesEmbedded += 1; + } + log(` 进度: 已重建 ${result.communitiesEmbedded}，失败 ${result.communitiesFailed}`); + } + + result.durationMs = Date.now() - start; + log( + `\n[graph-memory-pro] 重嵌入完成：节点 ${result.nodesEmbedded}（失败/跳过 ${result.nodesFailed}），` + + `社区 ${result.communitiesEmbedded}（失败 ${result.communitiesFailed}），` + + `${result.batches} 次 embedding 调用${result.recreatedIndex ? "，索引已重建" : ""}，` + + `用时 ${(result.durationMs / 1000).toFixed(1)}s`, + ); + if (result.nodesFailed > 0 || result.communitiesFailed > 0) { + log("[graph-memory-pro] 提示：失败条目保持无向量，修复端点后再次运行本命令即可增量补齐。"); + } + return result; + } finally { + await closeDriver(); + } +} diff --git a/src/cli.ts b/src/cli.ts index f4c5a25..14eef92 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -25,6 +25,9 @@ import { type OAuthProviderId, } from "./engine/oauth.ts"; import type { ReasoningEffort } from "./engine/llm.ts"; +import { runBackfillExtraction } from "./cli-extract.ts"; +import { runReembed } from "./cli-reembed.ts"; +import { DEFAULT_CONFIG, type GmConfig } from "./types.ts"; // ─── 最小 Commander 鸭子类型（避免引入 commander 依赖） ─────────── // host 运行时注入真正的 commander.Command 实例，结构兼容此接口即可。 @@ -45,6 +48,7 @@ export interface GraphMemoryCliDeps { pluginId?: string; pluginConfig?: Record | undefined; resolveConfigPath?: (input: string) => string; + defaultModel?: string; oauthTestHooks?: { openUrl?: (url: string) => void | Promise; authorizeUrl?: (url: string) => void | Promise; @@ -375,5 +379,108 @@ export function createGraphMemoryCli(deps: GraphMemoryCliDeps) { throw new Error(`[graph-memory-pro] OAuth login failed: ${message}`); } }); + + root + .command("extract") + .description( + "扫描 Neo4j 中未提取的会话消息，按 compact 流程批量补提知识图谱，并同步节点 embedding", + ) + .option("--yes", "跳过确认提示，直接执行提取", false) + .option("--dry-run", "只列出待提取会话，不调用 LLM", false) + .option("--limit ", "每个会话每批最多提取的消息条数（默认 compactTurnCount * 3）", undefined) + .option("--session ", "仅提取指定 sessionId（默认全部含未提取消息的会话）", undefined) + .option("--model ", "本次提取使用的 LLM 模型（覆盖配置中的 llm.model / agents.defaults.model）", undefined) + .action(async (options: Record) => { + try { + const rawCfg = isPlainObject(deps.pluginConfig) + ? (deps.pluginConfig as Record) + : {}; + const cfg: GmConfig = { + ...DEFAULT_CONFIG, + ...(rawCfg as Partial), + }; + if (isPlainObject(rawCfg.neo4j)) { + cfg.neo4j = { ...DEFAULT_CONFIG.neo4j, ...(rawCfg.neo4j as any) }; + } + + const cfgLlm = isPlainObject(rawCfg.llm) ? (rawCfg.llm as any) : undefined; + const flagModel = typeof options.model === "string" && options.model.trim() + ? options.model.trim() + : undefined; + const effectiveModel = flagModel ?? cfgLlm?.model ?? deps.defaultModel ?? ""; + + const limitFlag = typeof options.limit === "string" + ? Number.parseInt(options.limit, 10) + : (typeof options.limit === "number" ? options.limit : undefined); + + await runBackfillExtraction({ + cfg, + effectiveModel, + options: { + yes: options.yes === true, + dryRun: options.dryRun === true, + session: typeof options.session === "string" ? options.session : undefined, + limit: limitFlag !== undefined && Number.isFinite(limitFlag) && limitFlag > 0 + ? Math.floor(limitFlag) + : undefined, + }, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error("[graph-memory-pro] extract 失败：", message); + throw new Error(`[graph-memory-pro] extract failed: ${message}`); + } + }); + + root + .command("reembed") + .description( + "清除现有向量并用当前 embedding 模型批量重建（换 embedding 模型后必须执行，否则旧向量静默失效）", + ) + .option("--yes", "跳过确认提示，直接执行", false) + .option("--dry-run", "只报告向量覆盖情况与维度对照，不写入", false) + .option("--batch ", "每次 embedding 请求携带的文本条数（默认 32，上限 256）", undefined) + .option( + "--recreate-index", + "向量索引维度与当前模型输出不符时，删除索引并按 embedding.dimensions 重建", + false, + ) + .action(async (options: Record) => { + try { + const rawCfg = isPlainObject(deps.pluginConfig) + ? (deps.pluginConfig as Record) + : {}; + const cfg: GmConfig = { + ...DEFAULT_CONFIG, + ...(rawCfg as Partial), + }; + if (isPlainObject(rawCfg.neo4j)) { + cfg.neo4j = { ...DEFAULT_CONFIG.neo4j, ...(rawCfg.neo4j as any) }; + } + if (isPlainObject(rawCfg.embedding)) { + cfg.embedding = { ...(rawCfg.embedding as any) }; + } + + const batchFlag = typeof options.batch === "string" + ? Number.parseInt(options.batch, 10) + : (typeof options.batch === "number" ? options.batch : undefined); + + await runReembed({ + cfg, + options: { + yes: options.yes === true, + dryRun: options.dryRun === true, + recreateIndex: options.recreateIndex === true, + batch: batchFlag !== undefined && Number.isFinite(batchFlag) && batchFlag > 0 + ? Math.floor(batchFlag) + : undefined, + }, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error("[graph-memory-pro] reembed 失败：", message); + throw new Error(`[graph-memory-pro] reembed failed: ${message}`); + } + }); }; } diff --git a/src/engine/embed.ts b/src/engine/embed.ts index fa98dc7..bfc603b 100755 --- a/src/engine/embed.ts +++ b/src/engine/embed.ts @@ -15,30 +15,15 @@ */ import type { EmbeddingConfig } from "../types.ts"; +import { fetchRetry, throwForStatus } from "./http.ts"; export type EmbedMode = "db" | "query"; export type EmbedFn = (text: string, mode?: EmbedMode) => Promise; +export type EmbedBatchFn = (texts: string[], mode?: EmbedMode) => Promise; -// ─── 带重试 + 超时的 fetch ───────────────────────────────────── - -const RETRYABLE = new Set([429, 500, 502, 503, 529]); - -async function fetchRetry(url: string, init: RequestInit, retries = 3, timeoutMs = 10_000): Promise { - for (let i = 0; i <= retries; i++) { - const ctrl = new AbortController(); - const t = setTimeout(() => ctrl.abort(), timeoutMs); - try { - const res = await fetch(url, { ...init, signal: ctrl.signal }); - clearTimeout(t); - if (res.ok || i >= retries || !RETRYABLE.has(res.status)) return res; - await new Promise(r => setTimeout(r, 1000 * Math.pow(2, i))); - } catch (err: any) { - clearTimeout(t); - if (i >= retries) throw err; - await new Promise(r => setTimeout(r, 1000 * (i + 1))); - } - } - throw new Error("[graph-memory-pro] embed fetch failed after retries"); +export interface Embedder { + embed: EmbedFn; + embedBatch: EmbedBatchFn; } // ─── Provider 识别 ─────────────────────────────────────────── @@ -64,9 +49,49 @@ export function isMinimaxEndpoint(baseURL: string): boolean { ); } +// ─── 批量响应解析（纯函数，可测） ───────────────────────────── + +/** + * 从 /embeddings 响应的 data 数组重建与输入等长、顺序一致的向量数组。 + * OpenAI 兼容端点按 item.index 归位；缺 index 时按响应位置对齐。 + * MiniMax 用 item.vector 字段。条数不符 / 缺向量 / index 非法都抛错—— + * 上层（CLI 重嵌入）会退化为逐条请求兜底。 + */ +export function parseBatchEmbeddingResponse( + data: unknown, + expectedCount: number, + minimax: boolean, +): number[][] { + if (!Array.isArray(data)) { + throw new Error("[graph-memory-pro] Embedding batch response missing data array"); + } + if (data.length !== expectedCount) { + throw new Error( + `[graph-memory-pro] Embedding batch returned ${data.length} vectors for ${expectedCount} inputs`, + ); + } + const out: number[][] = new Array(expectedCount); + data.forEach((item: any, position: number) => { + const vec = minimax ? item?.vector : item?.embedding; + if (!Array.isArray(vec) || !vec.length) { + throw new Error(`[graph-memory-pro] Embedding batch response item ${position} has no vector`); + } + const index = typeof item?.index === "number" ? item.index : position; + if (index < 0 || index >= expectedCount || out[index]) { + throw new Error(`[graph-memory-pro] Embedding batch response has invalid index ${index}`); + } + out[index] = vec; + }); + return out; +} + // ─── EmbedFn 工厂 ─────────────────────────────────────────── -export async function createEmbedFn(cfg: EmbeddingConfig | undefined): Promise { +/** + * 构造单发 + 批量两个 embed 函数（共享 probe 与配置解析）。 + * 运行时路径只用单发 embed；`graph-memory reembed` 额外消费批量接口。 + */ +export async function createEmbedder(cfg: EmbeddingConfig | undefined): Promise { // Local OpenAI-compatible servers commonly do not require a key. A key by // itself still selects the default OpenAI endpoint; a URL by itself selects // an unauthenticated local/custom endpoint. @@ -83,12 +108,13 @@ export async function createEmbedFn(cfg: EmbeddingConfig | undefined): Promise { + function buildBody(input: string | string[], mode: EmbedMode): Record { if (minimax) { return { model, - texts: [input], + texts: Array.isArray(input) ? input : [input], type: mode, }; } @@ -97,19 +123,22 @@ export async function createEmbedFn(cfg: EmbeddingConfig | undefined): Promise { - const res = await fetchRetry(`${baseURL}/embeddings`, { + async function postEmbedding(body: Record, timeoutMs: number): Promise { + return fetchRetry(`${baseURL}/embeddings`, { method: "POST", headers: { "Content-Type": "application/json", ...(apiKey ? { "Authorization": `Bearer ${apiKey}` } : {}), }, - body: JSON.stringify(buildBody(input, mode)), - }); + body: JSON.stringify(body), + }, { timeoutMs, label: "[graph-memory-pro] Embedding", retryOnTimeout: true }); + } + + async function callEmbedding(input: string, mode: EmbedMode): Promise { + const res = await postEmbedding(buildBody(input, mode), 10_000); if (!res.ok) { - const errText = await res.text().catch(() => ""); - throw new Error(`[graph-memory-pro] Embedding API ${res.status}: ${errText.slice(0, 200)}`); + await throwForStatus(res, "[graph-memory-pro] Embedding API"); } const data = await res.json() as any; @@ -121,15 +150,33 @@ export async function createEmbedFn(cfg: EmbeddingConfig | undefined): Promise { + if (!texts.length) return []; + // 批量请求服务端耗时随条数线性增长：按 2s/条 推算超时，下限沿用单发的 10s + const res = await postEmbedding(buildBody(texts, mode), Math.max(10_000, 2_000 * texts.length)); + + if (!res.ok) { + await throwForStatus(res, "[graph-memory-pro] Embedding API"); + } + + const data = await res.json() as any; + return parseBatchEmbeddingResponse(data?.data, texts.length, minimax); + } + try { const probe = await callEmbedding("ping", "query"); if (!probe.length) return null; - return async (text: string, mode: EmbedMode = "db"): Promise => { - return callEmbedding(text.slice(0, 8000), mode); + return { + embed: async (text: string, mode: EmbedMode = "db"): Promise => { + return callEmbedding(text.slice(0, 8000), mode); + }, + embedBatch: async (texts: string[], mode: EmbedMode = "db"): Promise => { + return callEmbeddingBatch(texts.map(t => t.slice(0, 8000)), mode); + }, }; - } catch (err) { - console.error(`[graph-memory-pro] embedding probe failed:`, err); + } catch { + // probe 失败返回 null（调用方日志已有 "text search mode" 降级提示），不在库代码里写 stdout return null; } } diff --git a/src/engine/http.ts b/src/engine/http.ts new file mode 100644 index 0000000..fdee0b5 --- /dev/null +++ b/src/engine/http.ts @@ -0,0 +1,91 @@ +/** + * graph-memory-pro — 共享 HTTP 客户端工具 + * + * 统一 LLM / Embedding 调用的超时与重试语义（此前 llm.ts / embed.ts 各有一份 + * fetchRetry，且已发生行为分叉：llm 侧在重构中丢失了网络异常重试分支）。 + * + * 语义： + * - 可重试 HTTP 状态码（429/5xx）：指数退避重试 + * - 网络级异常（连接失败、连接被重置等）：同样重试 + * - 超时：默认立即抛出 HttpTimeoutError；仅当 retryOnTimeout=true 时计入重试 + * （LLM 调用耗时长，超时重试会成倍拉长最坏阻塞时间；embedding 调用短，重试无害） + */ + +const RETRYABLE_STATUS = new Set([429, 500, 502, 503, 529]); + +/** 请求超时（AbortError 的友好化包装；instanceof 可与网络异常区分） */ +export class HttpTimeoutError extends Error { + constructor(message: string) { + super(message); + this.name = "HttpTimeoutError"; + } +} + +export interface FetchRetryOptions { + /** 最大重试次数（不含首次），默认 3 */ + retries?: number; + /** 单次请求超时，默认 30_000ms */ + timeoutMs?: number; + /** 错误信息前缀（如 "[graph-memory] LLM"），保持各调用方原报错格式 */ + label?: string; + /** 超时是否计入重试；默认 false（立即抛出） */ + retryOnTimeout?: boolean; + /** 重试退避函数，attempt 从 0 开始；默认指数退避 1s/2s/4s…（测试可注入 0） */ + backoffMs?: (attempt: number) => number; +} + +export async function fetchWithTimeout( + url: string, + init: RequestInit, + timeoutMs: number, + label = "[graph-memory]", +): Promise { + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), timeoutMs); + try { + return await fetch(url, { ...init, signal: ctrl.signal }); + } catch (err: any) { + if (err?.name === "AbortError") { + throw new HttpTimeoutError(`${label} request timed out after ${timeoutMs}ms`); + } + throw err; + } finally { + clearTimeout(timer); + } +} + +export async function fetchRetry( + url: string, + init: RequestInit, + opts: FetchRetryOptions = {}, +): Promise { + const { + retries = 3, + timeoutMs = 30_000, + label = "[graph-memory]", + retryOnTimeout = false, + backoffMs = (attempt: number) => 1000 * Math.pow(2, attempt), + } = opts; + + for (let i = 0; i <= retries; i++) { + try { + const res = await fetchWithTimeout(url, init, timeoutMs, label); + if (res.ok || i >= retries || !RETRYABLE_STATUS.has(res.status)) return res; + await new Promise((r) => setTimeout(r, backoffMs(i))); + } catch (err) { + if (err instanceof HttpTimeoutError && !retryOnTimeout) throw err; + if (i >= retries) throw err; + await new Promise((r) => setTimeout(r, backoffMs(i))); + } + } + throw new Error(`${label} request failed after retries`); +} + +/** + * !res.ok 统一抛错（llm / embed 各调用方共用，消灭散落的 text().catch + slice 样板）。 + * 消息格式与既有调用方一致：`"${label} ${status}: ${errText.slice(0, sliceLen)}"`。 + */ +export async function throwForStatus(res: Response, label: string, sliceLen = 200): Promise { + const errText = await res.text().catch(() => ""); + throw new Error(`${label} ${res.status}: ${errText.slice(0, sliceLen)}`); +} diff --git a/src/engine/llm-guard.ts b/src/engine/llm-guard.ts new file mode 100644 index 0000000..b6d8386 --- /dev/null +++ b/src/engine/llm-guard.ts @@ -0,0 +1,85 @@ +/** + * graph-memory-pro — LLM 失败冷却守卫 + * + * 与 Neo4jGate（DB 熔断）对偶：保护 LLM 依赖。 + * 仅对"持久性配置错误"触发冷却 —— 401/403/404（凭证失效、无权限、模型名/端点错误）， + * 这类错误不会自愈，逐轮重试只会浪费每次完整的请求 + 超时等待。 + * + * 不触发的情况： + * - 429/5xx：瞬时故障，fetchRetry 已内部重试，冷却反而放大抖动 + * - 400/422：可能只是单条坏 prompt（超长/格式问题），不能殃及后续正常调用 + * - 无状态码的错误（超时、空返回、缺配置）：交给各自的正常失败路径 + * + * 例外：OAuth token 端点的 400 = invalid_grant（refresh token 被吊销/过期）， + * 与聊天端点的 400（可能只是单条坏 prompt）语义不同 —— 属于持久性凭证故障， + * 同样触发冷却；恢复路径是重新 auth login（llm.ts 会按会话文件 mtime 自动解除）。 + */ + +const RETRYABLE_STATUSES = new Set([429, 500, 502, 503, 529]); +const PAUSING_STATUSES = new Set([401, 403, 404]); +const OAUTH_REFRESH_PAUSING_STATUSES = new Set([400, 401, 403]); + +/** + * 从错误消息中提取 HTTP 状态码。 + * 覆盖三条 provider 路径的报错格式： + * "[graph-memory] LLM API 401: …" （openai 兼容） + * "[graph-memory] Anthropic API 403: …" （anthropic） + * "[graph-memory] OAuth LLM API 401: …" （oauth，含 "LLM API 401" 子串） + */ +export function extractLlmStatus(error: unknown): number | null { + const text = String(error ?? ""); + const match = text.match(/\b(?:LLM|Anthropic) API (\d{3})\b/); + if (!match) return null; + return Number(match[1]); +} + +/** + * 识别 oauth.ts refreshOAuthSession 的报错格式："OAuth refresh failed (400): …"。 + * 独立于 extractLlmStatus —— token 端点与聊天端点对同一状态码的语义不同。 + */ +export function extractOAuthRefreshStatus(error: unknown): number | null { + const match = String(error ?? "").match(/\bOAuth refresh failed \((\d{3})\)/); + if (!match) return null; + return Number(match[1]); +} + +export class LlmFailureGuard { + private pausedUntil = 0; + + constructor( + private readonly cooldownMs = 10 * 60_000, + private readonly now: () => number = () => Date.now(), + ) {} + + canRun(): boolean { + return this.now() >= this.pausedUntil; + } + + remainingMs(): number { + return Math.max(0, this.pausedUntil - this.now()); + } + + /** 成功调用后清除冷却，下一次失败重新计时。 */ + reset(): void { + this.pausedUntil = 0; + } + + /** + * 按错误类型决定是否触发冷却。 + * 返回是否触发（调用方仅用于日志/测试；冷却本身幂等，重复触发取更晚到期时间）。 + */ + tripIfNeeded(error: unknown): boolean { + const refreshStatus = extractOAuthRefreshStatus(error); + if (refreshStatus != null) { + if (!OAUTH_REFRESH_PAUSING_STATUSES.has(refreshStatus)) return false; + this.pausedUntil = Math.max(this.pausedUntil, this.now() + this.cooldownMs); + return true; + } + const status = extractLlmStatus(error); + if (status == null || RETRYABLE_STATUSES.has(status) || !PAUSING_STATUSES.has(status)) { + return false; + } + this.pausedUntil = Math.max(this.pausedUntil, this.now() + this.cooldownMs); + return true; + } +} diff --git a/src/engine/llm.ts b/src/engine/llm.ts index c474dc7..36b7a98 100755 --- a/src/engine/llm.ts +++ b/src/engine/llm.ts @@ -23,6 +23,8 @@ * 超时：AbortController 强制；默认 60s，cfg.llm.timeoutMs 可调（慢速 API 用户可调大）。 */ +import { stat } from "node:fs/promises"; +import { LlmFailureGuard } from "./llm-guard.ts"; import { loadOAuthSession, needsRefresh, @@ -31,8 +33,10 @@ import { normalizeOauthModel, buildOauthEndpoint, extractOutputTextFromSse, + extractOutputTextFromResponsePayload, } from "./oauth.ts"; import type { OAuthSession } from "./oauth.ts"; +import { fetchRetry, throwForStatus } from "./http.ts"; export type LlmProvider = "openai" | "anthropic" | "oauth"; @@ -41,7 +45,7 @@ export type ReasoningEffort = "low" | "medium" | "high"; const DEFAULT_REASONING_EFFORT: ReasoningEffort = "medium"; -export interface LlmConfig { +interface LlmConfig { /** 显式 provider 切换。未设时按 baseURL 是否存在推断（向后兼容，仅产生 openai/anthropic）。 */ provider?: LlmProvider; apiKey?: string; @@ -60,6 +64,16 @@ export interface LlmConfig { export type CompleteFn = (system: string, user: string) => Promise; +/** + * 剥离推理模型输出的 ... 思维链标签（兼容 MiniMax 等）， + * 含未闭合 兜底。LLM 输出清洗的单一来源——extractor 与社区摘要共用。 + */ +export function stripThinkTags(raw: string): string { + return raw + .replace(/[\s\S]*?<\/think>/gi, "") + .replace(/[\s\S]*/gi, ""); +} + const DEFAULT_LLM_TIMEOUT_MS = 60_000; const DEFAULT_LLM_MAX_TOKENS = 4_000; const ANTHROPIC_DEFAULT_BASE_URL = "https://api.anthropic.com"; @@ -82,41 +96,6 @@ export function resolveProvider(cfg: LlmConfig | undefined): { return { provider: inferred, inferred: true }; } -async function fetchWithTimeout( - url: string, - init: RequestInit, - timeoutMs: number, -): Promise { - const ctrl = new AbortController(); - const timer = setTimeout(() => ctrl.abort(), timeoutMs); - try { - return await fetch(url, { ...init, signal: ctrl.signal }); - } catch (err: any) { - if (err?.name === "AbortError") { - throw new Error(`[graph-memory] LLM request timed out after ${timeoutMs}ms`); - } - throw err; - } finally { - clearTimeout(timer); - } -} - -const RETRYABLE = new Set([429, 500, 502, 503, 529]); - -async function fetchRetry( - url: string, - init: RequestInit, - retries: number, - timeoutMs: number, -): Promise { - for (let i = 0; i <= retries; i++) { - const res = await fetchWithTimeout(url, init, timeoutMs); - if (res.ok || i >= retries || !RETRYABLE.has(res.status)) return res; - await new Promise((r) => setTimeout(r, 1000 * Math.pow(2, i))); - } - throw new Error("[graph-memory] fetch failed after retries"); -} - /** * 构造 LLM CompleteFn。 * @@ -144,13 +123,19 @@ export function createCompleteFn( // ── OAuth 会话缓存：单飞刷新，避免并发请求同时触发 refresh ── const oauthPath = provider === "oauth" ? llmConfig?.oauthPath : undefined; let cachedSessionPromise: Promise | null = null; + let cachedSessionMtimeMs: number | null = null; let refreshPromise: Promise | null = null; async function getOAuthSession(): Promise { if (!oauthPath) { throw new Error("[graph-memory] provider=oauth 需要 llm.oauthPath"); } - if (!cachedSessionPromise) { + // oauthPath 可能被运行中的其他进程重写（CLI auth login / CLI extract 刷新 token）。 + // 进程内缓存按 mtime 失效：文件更新后下一次调用即重载，无需重启网关。 + let mtimeMs: number | null = null; + try { mtimeMs = (await stat(oauthPath)).mtimeMs; } catch { /* 文件暂不可达：沿用缓存 */ } + if (!cachedSessionPromise || (mtimeMs !== null && mtimeMs !== cachedSessionMtimeMs)) { + cachedSessionMtimeMs = mtimeMs; cachedSessionPromise = loadOAuthSession(oauthPath).catch((error) => { cachedSessionPromise = null; throw error; @@ -163,6 +148,8 @@ export function createCompleteFn( .then(async (s) => { await saveOAuthSession(oauthPath, s); cachedSessionPromise = Promise.resolve(s); + // 同步 mtime 标记，避免下次调用因文件刚写入而多余重载一次 + try { cachedSessionMtimeMs = (await stat(oauthPath)).mtimeMs; } catch {} refreshPromise = null; return s; }) @@ -176,7 +163,7 @@ export function createCompleteFn( return session; } - return async (system, user) => { + const complete = async (system: string, user: string): Promise => { // ── 路径 C：OAuth Codex Responses API ── if (provider === "oauth") { if (!oauthPath) { @@ -210,29 +197,17 @@ export function createCompleteFn( stream: false, text: { format: { type: "text" } }, }), - }, 3, timeoutMs); + }, { retries: 3, timeoutMs, label: "[graph-memory] LLM" }); if (!res.ok) { - const errText = await res.text().catch(() => ""); - throw new Error(`[graph-memory] OAuth LLM API ${res.status}: ${errText.slice(0, 500)}`); + await throwForStatus(res, "[graph-memory] OAuth LLM API", 500); } const bodyText = await res.text(); let text: string | null = null; try { - const parsed = JSON.parse(bodyText) as Record; - const output = Array.isArray(parsed.output) ? parsed.output : []; - for (const item of output) { - if (!item || typeof item !== "object") continue; - const content = Array.isArray((item as Record).content) - ? (item as Record).content as Array> - : []; - for (const part of content) { - if (part?.type === "output_text" && typeof part.text === "string") { - text = (text ?? "") + part.text; - } - } - } + // Responses JSON → output_text 收集（与 oauth.ts 的 SSE 嵌套解析共用同一遍历） + text = extractOutputTextFromResponsePayload(JSON.parse(bodyText)); } catch { // 服务器忽略 stream:false 时回退到 SSE 解析 text = extractOutputTextFromSse(bodyText); @@ -251,7 +226,7 @@ export function createCompleteFn( ); } const baseURL = (llmConfig?.baseURL ?? ANTHROPIC_DEFAULT_BASE_URL).replace(/\/+$/, ""); - const res = await fetchWithTimeout(`${baseURL}/v1/messages`, { + const res = await fetchRetry(`${baseURL}/v1/messages`, { method: "POST", headers: { "Content-Type": "application/json", @@ -264,13 +239,18 @@ export function createCompleteFn( system, messages: [{ role: "user", content: user }], }), - }, timeoutMs); + }, { retries: 3, timeoutMs, label: "[graph-memory] LLM" }); if (!res.ok) { - const errText = await res.text().catch(() => ""); - throw new Error(`[graph-memory] Anthropic API ${res.status}: ${errText.slice(0, 200)}`); + await throwForStatus(res, "[graph-memory] Anthropic API"); } const data = await res.json() as any; - const text = data.content?.[0]?.text; + // 遍历 content 找 text 块：只看 content[0] 时，thinking 块在前会误报 empty content + const text = Array.isArray(data.content) + ? data.content + .filter((b: any) => b?.type === "text" && typeof b.text === "string") + .map((b: any) => b.text) + .join("") + : ""; if (text) return text; const stop = data.choices?.[0]?.finish_reason ?? data.stop_reason; throw new Error( @@ -288,7 +268,7 @@ export function createCompleteFn( ); } const url = `${baseURL.replace(/\/+$/, "")}/chat/completions`; - const res = await fetchWithTimeout(url, { + const res = await fetchRetry(url, { method: "POST", headers: { "Content-Type": "application/json", @@ -303,10 +283,9 @@ export function createCompleteFn( max_tokens: maxTokens, temperature: 0.1, }), - }, timeoutMs); + }, { retries: 3, timeoutMs, label: "[graph-memory] LLM" }); if (!res.ok) { - const errText = await res.text().catch(() => ""); - throw new Error(`[graph-memory] LLM API ${res.status}: ${errText.slice(0, 200)}`); + await throwForStatus(res, "[graph-memory] LLM API"); } const data = await res.json() as any; const choice = data.choices?.[0]; @@ -320,4 +299,45 @@ export function createCompleteFn( `. Raise llm.maxTokens if recurring.`, ); }; + + // ── 失败冷却守卫：持久性配置错误（401/403/404）后冷却 10 分钟，快速失败 ── + // 避免凭证失效/模型名错误时每轮照付一次完整请求 + 超时等待。成功调用即清除。 + // OAuth 例外自愈：oauthPath 可能被外部进程重写（CLI auth login / CLI extract + // 刷新 token）。冷却触发时记录会话文件 mtime，后续调用发现文件已变化 = + // 凭证已被修复 → 立即解除冷却重试（401 的常见诱因是时钟偏移导致缓存的 + // access token 提前过期，重登即可恢复，不应被迫等满 10 分钟）。 + const guard = new LlmFailureGuard(); + let oauthTripMtimeMs: number | null | undefined; // undefined = 冷却非 oauth 路径触发 + return async (system: string, user: string): Promise => { + if (!guard.canRun()) { + if (provider === "oauth" && oauthPath && oauthTripMtimeMs !== undefined) { + let currentMtimeMs: number | null = null; + try { currentMtimeMs = (await stat(oauthPath)).mtimeMs; } catch { /* 文件暂不可达：维持冷却 */ } + // 仅在"确实读到不同的 mtime"或"trip 时读不到、现在读得到"时解除； + // 瞬时 stat 失败（null）不解除 —— 避免 AV/EBUSY 类抖动白白放行一次必败请求 + const fileReplaced = oauthTripMtimeMs === null + ? currentMtimeMs !== null + : currentMtimeMs !== null && currentMtimeMs !== oauthTripMtimeMs; + if (fileReplaced) guard.reset(); + } + if (!guard.canRun()) { + const seconds = Math.max(1, Math.ceil(guard.remainingMs() / 1000)); + throw new Error( + `[graph-memory] LLM paused for ${seconds}s after a previous permanent API error` + + (provider === "oauth" ? " — 重新 auth login 或等待 token 文件刷新后自动解除" : ""), + ); + } + } + try { + const text = await complete(system, user); + guard.reset(); + return text; + } catch (err) { + if (guard.tripIfNeeded(err) && provider === "oauth" && oauthPath) { + try { oauthTripMtimeMs = (await stat(oauthPath)).mtimeMs; } + catch { oauthTripMtimeMs = null; } + } + throw err; + } + }; } diff --git a/src/engine/oauth.ts b/src/engine/oauth.ts index f0e1dfd..f8aaf4e 100644 --- a/src/engine/oauth.ts +++ b/src/engine/oauth.ts @@ -13,6 +13,7 @@ import { chmod, mkdir, readFile, writeFile } from "node:fs/promises"; import { dirname } from "node:path"; import { platform } from "node:os"; import { spawn } from "node:child_process"; +import { fetchWithTimeout } from "./http.ts"; // ─── Types ──────────────────────────────────────────────────── @@ -176,15 +177,44 @@ function pickTimestamp(container: Record, keys: string[]): numb return undefined; } -function createTimeoutSignal(timeoutMs?: number): { signal: AbortSignal; dispose: () => void } { +const DEFAULT_TOKEN_TIMEOUT_MS = 30_000; + +/** + * OAuth token 端点共用的 POST（refresh_token 与 authorization_code 交换同一端点、 + * 同一错误处理，此前是两份重复实现）。统一走 http.ts 的 fetchWithTimeout —— + * exchangeAuthorizationCode 曾是裸 fetch 无超时（CLI 登录流程卡死风险）， + * refresh 用手写 AbortController，现在两者语义一致且与全仓单一来源对齐。 + * 错误消息保持既有格式：`${errorLabel} (${status}): ${detail.slice(0, 500)}`。 + */ +async function postTokenRequest( + providerId: OAuthProviderId, + body: Record, + errorLabel: string, + timeoutMs?: number, +): Promise { const effectiveTimeoutMs = - typeof timeoutMs === "number" && Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 30_000; - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), effectiveTimeoutMs); - return { - signal: controller.signal, - dispose: () => clearTimeout(timer), - }; + typeof timeoutMs === "number" && Number.isFinite(timeoutMs) && timeoutMs > 0 + ? timeoutMs + : DEFAULT_TOKEN_TIMEOUT_MS; + const response = await fetchWithTimeout( + resolveOauthTokenUrl(undefined, providerId), + { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + body: new URLSearchParams(body), + }, + effectiveTimeoutMs, + "[graph-memory] OAuth", + ); + + if (!response.ok) { + const detail = await response.text().catch(() => ""); + throw new Error(`${errorLabel} (${response.status}): ${detail.slice(0, 500)}`); + } + + return await response.json() as TokenRefreshResponse; } // ─── Provider resolution ────────────────────────────────────── @@ -391,54 +421,36 @@ export async function refreshOAuthSession(session: OAuthSession, timeoutMs?: num ); } - const { signal, dispose } = createTimeoutSignal(timeoutMs); - try { - const response = await fetch(resolveOauthTokenUrl(undefined, session.providerId), { - method: "POST", - headers: { - "Content-Type": "application/x-www-form-urlencoded", - }, - body: new URLSearchParams({ - grant_type: "refresh_token", - refresh_token: session.refreshToken, - client_id: resolveOauthClientId(undefined, session.providerId), - }), - signal, - }); - - if (!response.ok) { - const detail = await response.text().catch(() => ""); - throw new Error(`OAuth refresh failed (${response.status}): ${detail.slice(0, 500)}`); - } - - const payload = await response.json() as TokenRefreshResponse; - if (!payload.access_token) { - throw new Error("OAuth refresh returned no access token"); - } + const payload = await postTokenRequest(session.providerId, { + grant_type: "refresh_token", + refresh_token: session.refreshToken, + client_id: resolveOauthClientId(undefined, session.providerId), + }, "OAuth refresh failed", timeoutMs); - const accessToken = payload.access_token; - const refreshToken = payload.refresh_token || session.refreshToken; - const expiresAt = - typeof payload.expires_in === "number" - ? Date.now() + payload.expires_in * 1000 - : getJwtExpiry(accessToken); - const accountId = getJwtAccountId(accessToken, session.providerId) || session.accountId; + if (!payload.access_token) { + throw new Error("OAuth refresh returned no access token"); + } - if (!accountId) { - throw new Error("OAuth refresh returned a token without a ChatGPT account id"); - } + const accessToken = payload.access_token; + const refreshToken = payload.refresh_token || session.refreshToken; + const expiresAt = + typeof payload.expires_in === "number" + ? Date.now() + payload.expires_in * 1000 + : getJwtExpiry(accessToken); + const accountId = getJwtAccountId(accessToken, session.providerId) || session.accountId; - return { - accessToken, - refreshToken, - expiresAt, - accountId, - providerId: session.providerId, - authPath: session.authPath, - }; - } finally { - dispose(); + if (!accountId) { + throw new Error("OAuth refresh returned a token without a ChatGPT account id"); } + + return { + accessToken, + refreshToken, + expiresAt, + accountId, + providerId: session.providerId, + authPath: session.authPath, + }; } export async function saveOAuthSession(authPath: string, session: OAuthSession): Promise { @@ -490,9 +502,14 @@ export function buildOauthEndpoint(baseURL?: string, providerId?: string): strin return `${root}/codex/responses`; } -// ─── SSE response parsing ───────────────────────────────────── +// ─── Responses payload 解析 ─────────────────────────────────── -function extractOutputTextFromResponsePayload(payload: unknown): string | null { +/** + * 从 Codex Responses API 的 JSON payload 收集 output_text（多段以 "\n" 连接）。 + * oauth.ts 的 SSE 嵌套回退与 llm.ts 的非流式主路径共用此遍历 —— 勿在调用方 + * 重新内联 output[].content[] 走树。 + */ +export function extractOutputTextFromResponsePayload(payload: unknown): string | null { if (!payload || typeof payload !== "object") return null; const response = payload as Record; @@ -565,26 +582,14 @@ export function extractOutputTextFromSse(bodyText: string): string | null { async function exchangeAuthorizationCode(code: string, verifier: string, providerId?: string): Promise { const resolvedProviderId = normalizeOAuthProviderId(providerId); - const response = await fetch(resolveOauthTokenUrl(undefined, resolvedProviderId), { - method: "POST", - headers: { - "Content-Type": "application/x-www-form-urlencoded", - }, - body: new URLSearchParams({ - grant_type: "authorization_code", - client_id: resolveOauthClientId(undefined, resolvedProviderId), - code, - code_verifier: verifier, - redirect_uri: resolveOauthRedirectUri(undefined, resolvedProviderId), - }), - }); - - if (!response.ok) { - const detail = await response.text().catch(() => ""); - throw new Error(`OAuth token exchange failed (${response.status}): ${detail.slice(0, 500)}`); - } + const payload = await postTokenRequest(resolvedProviderId, { + grant_type: "authorization_code", + client_id: resolveOauthClientId(undefined, resolvedProviderId), + code, + code_verifier: verifier, + redirect_uri: resolveOauthRedirectUri(undefined, resolvedProviderId), + }, "OAuth token exchange failed"); - const payload = await response.json() as TokenRefreshResponse; if (!payload.access_token) { throw new Error("OAuth token exchange returned no access token"); } diff --git a/src/extractor/extract.ts b/src/extractor/extract.ts index 8e4f95d..759dcc4 100755 --- a/src/extractor/extract.ts +++ b/src/extractor/extract.ts @@ -7,7 +7,9 @@ import type { ExtractionResult, FinalizeResult } from "../types.ts"; import { EDGE_TYPES, isValidEdgeDirection } from "../types.ts"; -import type { CompleteFn } from "../engine/llm.ts"; +import type { GmNode } from "../types.ts"; +import { stripThinkTags, type CompleteFn } from "../engine/llm.ts"; +import { normalizeName } from "../store/store.ts"; // ─── 节点/边合法值 ────────────────────────────────────────────── @@ -159,16 +161,6 @@ ${JSON.stringify(nodes.map(n => ({ ${summary}`; -// ─── 名称标准化（与 store.ts 一致）──────────────────────────── - -export function normalizeName(name: string): string { - return name.trim().toLowerCase() - .replace(/[\s_]+/g, "-") - .replace(/[^a-z0-9\u4e00-\u9fff\-]/g, "") - .replace(/-{2,}/g, "-") - .replace(/^-|-$/g, ""); -} - // ─── 边类型自动修正 ───────────────────────────────────────────── /** @@ -214,6 +206,16 @@ export function correctEdgeType( // ─── Extractor ──────────────────────────────────────────────── +/** + * finalize 阶梯触发（LLM 成本控制）： + * finalize 的核心产出是 EVENT→SKILL 提升与跨会话建边/失效判定—— + * 会话规模 ≤ 2 或没有任何 EVENT 节点时，这次 LLM 调用几乎必然空转，直接跳过。 + */ +export function shouldRunFinalize(sessionNodes: Array>): boolean { + if (sessionNodes.length <= 2) return false; + return sessionNodes.some((n) => n.type === "EVENT"); +} + export class Extractor { constructor(private llm: CompleteFn) {} @@ -313,9 +315,8 @@ export class Extractor { function extractJson(raw: string): string { let s = raw.trim(); - // 清理 ... 思维链标签（兼容 MiniMax 等模型） - s = s.replace(/[\s\S]*?<\/think>/gi, ""); - s = s.replace(/[\s\S]*/gi, ""); // 未闭合的 + // 清理 ... 思维链标签（兼容 MiniMax 等模型）——单一来源 stripThinkTags + s = stripThinkTags(s); s = s.replace(/^```(?:json)?\s*\n?/i, "").replace(/\n?\s*```\s*$/i, ""); s = s.trim(); if (s.startsWith("{") && s.endsWith("}")) return s; diff --git a/src/extractor/persist.ts b/src/extractor/persist.ts new file mode 100644 index 0000000..308fea9 --- /dev/null +++ b/src/extractor/persist.ts @@ -0,0 +1,87 @@ +/** + * graph-memory-pro — 提取结果持久化（单一来源） + * + * "upsertNode → 批量向量同步 → 解析端点 → upsertEdge" 这段持久化体此前在 + * index.ts（per-turn 提取 / compact 攒批提取）与 cli-extract.ts（CLI 回填） + * 逐字复制三份，且已发生漂移（CLI await 了 syncEmbedBatch，运行时是 + * fire-and-forget）。收敛于此：改边解析/嵌入策略只需改这一处。 + */ + +import type { Driver } from "neo4j-driver"; +import type { ExtractionResult, GmNode } from "../types.ts"; +import { upsertNode, upsertEdge, findByName } from "../store/store.ts"; +import type { Recaller } from "../recaller/recall.ts"; + +export interface PersistExtractionOptions { + sessionId: string; + /** + * true = 等待批量向量同步完成再返回（CLI 路径：driver 将在 finally 里 + * closeDriver，不等待会让在途 embedding 请求撞上已关闭的 driver 且错误 + * 被吞——markExtracted 已执行，向量丢失不可自愈）；false = fire-and-forget + * （运行时路径：不让 embedding 延迟拖住回合）。 + */ + awaitEmbedSync?: boolean; + /** 每个成功 upsert 的节点回调（CLI 统计 nodesCreated 用）。 */ + onNodeUpserted?: (node: GmNode) => void; +} + +export interface PersistExtractionOutcome { + /** 实际 upsert 的节点 */ + nodes: GmNode[]; + /** 成功建立的边数（两端可解析且方向合法；upsertEdge 内部仍会按库中真实端点复核方向） */ + edges: number; + /** + * 命中已有节点的 upsert 数（isNew=false）。index.ts 把它按会话累加为 + * finalize 阶梯的第二触发条件——纠错型 invalidations 只有 finalize 能产出， + * 而纠错常发生在小会话/无 EVENT 会话（shouldRunFinalize 的双门会漏掉）。 + */ + updatedExisting: number; +} + +/** + * 将一次 ExtractionResult 落库：upsert 全部节点 → 启动批量向量同步 + * （awaitEmbedSync 决定是否等待）→ 逐条解析并 upsert 边。 + * 边端点优先查本次 upsert 得到的 name→id 索引（零往返；键是规范化名，LLM + * 原文未必规范化，未命中必须回源 findByName——它做 normalizeName）。 + * 不负责 markExtracted —— 各调用方的 upToTurn/producedKnowledge 语义不同。 + */ +export async function persistExtractionResult( + driver: Driver, + recaller: Pick, + result: ExtractionResult, + opts: PersistExtractionOptions, +): Promise { + const nameToId = new Map(); + const upserted: GmNode[] = []; + let updatedExisting = 0; + for (const nc of result.nodes) { + const { node, isNew } = await upsertNode(driver, { + type: nc.type, name: nc.name, + description: nc.description, content: nc.content, + }, opts.sessionId); + nameToId.set(node.name, node.id); + upserted.push(node); + if (!isNew) updatedExisting += 1; + opts.onNodeUpserted?.(node); + } + + // 批量向量同步：N 个节点一次 embedBatch + 一次 UNWIND 批读写（替代逐节点 N 次单发） + const embedSync = recaller.syncEmbedBatch(upserted).catch(() => {}); + if (opts.awaitEmbedSync) await embedSync; + else void embedSync; + + let edgeCount = 0; + for (const ec of result.edges) { + const fromId = nameToId.get(ec.from) ?? (await findByName(driver, ec.from))?.id; + const toId = nameToId.get(ec.to) ?? (await findByName(driver, ec.to))?.id; + if (fromId && toId) { + await upsertEdge(driver, { + fromId, toId, type: ec.type, + instruction: ec.instruction, condition: ec.condition, sessionId: opts.sessionId, + }); + edgeCount += 1; + } + } + + return { nodes: upserted, edges: edgeCount, updatedExisting }; +} diff --git a/src/extractor/turn-filter.ts b/src/extractor/turn-filter.ts new file mode 100644 index 0000000..f958640 --- /dev/null +++ b/src/extractor/turn-filter.ts @@ -0,0 +1,82 @@ +/** + * graph-memory-pro — trivial 轮次本地预筛（LLM 成本控制） + * + * 在 extractTurnKnowledge 进入 LLM 前判断该轮是否"不可能产出知识"： + * 命中则直接 markExtracted(producedKnowledge=false)，省掉一次完整 completion。 + * + * 判定（保守取向，宁可漏判 trivial 也不误杀知识轮）： + * 1. 用户输入清洗后为空； + * 2. 清洗后命中无意义词表（内置 + cfg.extract.trivialPrompts，精确匹配）； + * 3. 清洗后长度 ≤ trivialMaxChars（默认 5）且不含技术词 + * （连续 ≥3 位字母数字，如 pnpm/jwt/k8s——这类短输入仍走 LLM）。 + * + * 判定"文本"只取 user 角色（工具结果与 assistant 回复的内容不参与），但轮内只要 + * 存在工具劳动（tool/toolResult 角色，见 turnHasToolWork）就不判 trivial—— + * "继续"触发的一轮真实修复劳动恰是图谱最该吸收的知识。 + */ + +/** 清洗：去空白与中西文标点，转小写。用于词表精确匹配与长度计量。 */ +export function normalizeTrivialText(raw: string): string { + return raw + .toLowerCase() + .replace(/[\s\p{P}\p{S}]+/gu, "") + .trim(); +} + +/** + * 内置无意义词表：纯推进/确认/致谢类输入，语义上不可能携带可提取三元组。 + * 匹配前同样经过 normalizeTrivialText（"继续。" → "继续"，"OK!" → "ok"）。 + */ +export const BUILTIN_TRIVIAL_PROMPTS: readonly string[] = [ + // 推进类 + "继续", "请继续", "接着", "接着来", "往下", "继续吧", "goon", "goahead", + "continue", "resume", "next", "proceed", "keepgoing", + // 确认类 + "ok", "okay", "好", "好的", "好吧", "可以", "行", "嗯", + "嗯嗯", "对", "是的", "没问题", "明白", "知道了", "懂了", "收到", "gotit", + "understood", "yes", "yeah", "yep", "sure", + // 致谢类 + "谢谢", "多谢", "感谢", "辛苦了", "thanks", "thankyou", "thx", "tnx", +]; + +/** 连续 ≥3 位字母数字视为技术词（pnpm/jwt/k8s/csv…），短输入含技术词时不判 trivial。 */ +const TECH_TOKEN_RE = /[a-z0-9]{3,}/; + +export interface TrivialFilterOptions { + /** 长度阈值；默认 5。 */ + maxChars?: number; + /** 追加词表（与内置合并，匹配前统一 normalize）。 */ + extraPrompts?: string[]; +} + +/** 该轮是否应跳过 LLM 提取（本地零成本判定）。 */ +export function shouldSkipTurnExtraction(userText: string, opts?: TrivialFilterOptions): boolean { + const normalized = normalizeTrivialText(userText ?? ""); + if (!normalized) return true; + + const stoplist = new Set(BUILTIN_TRIVIAL_PROMPTS); + for (const p of opts?.extraPrompts ?? []) { + const n = normalizeTrivialText(p); + if (n) stoplist.add(n); + } + if (stoplist.has(normalized)) return true; + + const maxChars = opts?.maxChars ?? 5; + if (normalized.length <= maxChars && !TECH_TOKEN_RE.test(normalized)) return true; + + return false; +} + +/** + * 轮内是否含工具劳动（tool / toolResult 角色）。这类轮即使 user 文本命中 + * trivial 词表也不应跳过提取："继续"触发的一轮真实修复劳动恰是可提取知识， + * 且 per-turn 路径没有自动补提触发点——extracted 一旦标记，只有手动重置 + * + `graph-memory extract` 才能回挖。 + */ +export function turnHasToolWork(messages: readonly unknown[]): boolean { + return (messages ?? []).some((m) => { + if (!m || typeof m !== "object") return false; + const role = (m as { role?: unknown }).role; + return role === "tool" || role === "toolResult"; + }); +} diff --git a/src/format/assemble.ts b/src/format/assemble.ts index 00c516d..37050ce 100755 --- a/src/format/assemble.ts +++ b/src/format/assemble.ts @@ -8,8 +8,7 @@ import type { Driver } from "neo4j-driver"; import type { GmNode, GmEdge } from "../types.ts"; import { getCommunitySummary, type CommunitySummary } from "../store/store.ts"; - -const CHARS_PER_TOKEN = 3; +import { CHARS_PER_TOKEN, estimateTokens } from "../tokens.ts"; export function buildSystemPromptAddition(params: { selectedNodes: Array<{ type: string; src: "active" | "recalled" }>; @@ -117,13 +116,15 @@ export async function assembleContext( selectedIds.has(e.fromId) && selectedIds.has(e.toId) && !seen.has(e.id) && seen.add(e.id) ); - // 预加载所有需要的社区摘要 - const communityIds = new Set(selected.map(n => n.communityId).filter(Boolean) as string[]); + // 预加载所有需要的社区摘要（并发拉取，避免逐个 await 的串行往返） + const communityIds = Array.from(new Set(selected.map(n => n.communityId).filter(Boolean) as string[])); + const summaries = await Promise.all( + communityIds.map(cid => getCommunitySummary(driver, cid)), + ); const communitySummaries = new Map(); - for (const cid of communityIds) { - const summary = await getCommunitySummary(driver, cid); - if (summary) communitySummaries.set(cid, summary); - } + communityIds.forEach((cid, i) => { + if (summaries[i]) communitySummaries.set(cid, summaries[i]!); + }); // 按社区分组 const byCommunity = new Map(); @@ -178,7 +179,7 @@ export async function assembleContext( }); const fullContent = systemPrompt + "\n\n" + xml; - return { xml, systemPrompt, tokens: Math.ceil(fullContent.length / CHARS_PER_TOKEN) }; + return { xml, systemPrompt, tokens: estimateTokens(fullContent.length) }; } function escapeXml(s: string): string { diff --git a/src/graph/community.ts b/src/graph/community.ts index d5f5dd5..805583c 100755 --- a/src/graph/community.ts +++ b/src/graph/community.ts @@ -6,15 +6,19 @@ * 保留 summarizeCommunities()（需要 LLM） */ +import { createHash } from "node:crypto"; import type { Driver } from "neo4j-driver"; import { getSession } from "../store/db.ts"; import { clearCommunities, updateCommunities, upsertCommunitySummary, + getCommunitySummary, + getCommunitySummaryBySignature, pruneCommunitySummaries, } from "../store/store.ts"; import { getExistingActiveRelTypes, projectActiveGraph } from "./projection.ts"; +import { stripThinkTags } from "../engine/llm.ts"; export interface CommunityResult { labels: Map; @@ -142,18 +146,81 @@ const COMMUNITY_SUMMARY_SYS = `你是知识图谱社区摘要引擎。根据社 - 不要使用"社区"这个词 - 不要加引号或标点以外的格式`; +export function buildCommunityMemberSignature(memberIds: string[]): string { + return createHash("sha1").update([...memberIds].sort().join(",")).digest("hex"); +} + +/** + * top-k 稳定签名（LLM 成本控制）：取 validatedCount 最高的 k 个成员计算签名， + * 而非全量成员集合。社区边界抖动（每次微增/减一个低频成员）不再触发 LLM 重摘要—— + * 摘要语义本就由高价值成员主导。成员数 ≤ k 时退化为全量签名（与旧格式一致）。 + * 并列的 validatedCount 用 id 字典序决胜负，保证确定性。 + */ +export const COMMUNITY_SIGNATURE_TOP_K = 8; + +export function buildTopKMemberSignature( + members: Array<{ id: string; validatedCount: number }>, + k: number = COMMUNITY_SIGNATURE_TOP_K, +): string { + const top = [...members] + .sort((a, b) => + b.validatedCount - a.validatedCount || + (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) + .slice(0, Math.max(1, k)) + .map((m) => m.id) + .sort(); + return createHash("sha1").update(top.join(",")).digest("hex"); +} + export async function summarizeCommunities( driver: Driver, communities: Map, llm: CompleteFn, embedFn?: EmbedFn, ): Promise { - await pruneCommunitySummaries(driver); let generated = 0; for (const [communityId, memberIds] of communities) { if (memberIds.length === 0) continue; + // 轻量元数据查询（id + validatedCount）：top-k 签号的输入 + const metaSession = getSession(driver); + let memberMeta: Array<{ id: string; validatedCount: number }>; + try { + const metaResult = await metaSession.run(` + MATCH (n:Task|Skill|Event {status: 'active'}) + WHERE n.id IN $memberIds + RETURN n.id AS id, n.validatedCount AS vc + `, { memberIds }); + memberMeta = metaResult.records.map(r => ({ + id: r.get("id"), + validatedCount: typeof r.get("vc") === "number" ? r.get("vc") : (r.get("vc")?.toNumber?.() ?? 0), + })); + } finally { + await metaSession.close(); + } + if (memberMeta.length === 0) continue; + + const memberSignature = buildTopKMemberSignature(memberMeta); + + const current = await getCommunitySummary(driver, communityId); + if (current?.memberSignature === memberSignature && current.summary.trim()) { + continue; + } + + const reusable = await getCommunitySummaryBySignature(driver, memberSignature); + if (reusable?.summary.trim()) { + await upsertCommunitySummary( + driver, + communityId, + reusable.summary, + memberIds.length, + reusable.embedding, + memberSignature, + ); + continue; + } + const session = getSession(driver); let members: any[]; try { @@ -185,9 +252,7 @@ export async function summarizeCommunities( `社区成员：\n${memberText}`, ); - const cleaned = summary.trim() - .replace(/[\s\S]*?<\/think>/gi, "") - .replace(/[\s\S]*/gi, "") + const cleaned = stripThinkTags(summary.trim()) .replace(/^["'「」]|["'「」]$/g, "") .replace(/\n/g, " ") .replace(/\s{2,}/g, " ") @@ -204,12 +269,17 @@ export async function summarizeCommunities( } catch {} } - await upsertCommunitySummary(driver, communityId, cleaned, memberIds.length, embedding); + await upsertCommunitySummary(driver, communityId, cleaned, memberIds.length, embedding, memberSignature); generated++; - } catch (err) { - console.log(` [WARN] community summary failed for ${communityId}: ${err}`); + } catch { + // 单社区摘要失败静默跳过（与 syncEmbed 的吞错策略一致）——库代码不直接写 stdout } } + // prune 必须在复用查找之后：detectCommunities 每轮按成员数重编号 c-1..c-N， + // 旧 id 社区（summary/memberSignature/embedding 的持有者）在新编号下"无人引用"， + // 先 prune 会把捐赠者删掉，签名复用永远不生效 → 每轮维护全量重算 LLM 摘要。 + await pruneCommunitySummaries(driver); + return generated; } diff --git a/src/graph/decay.ts b/src/graph/decay.ts new file mode 100644 index 0000000..21f7066 --- /dev/null +++ b/src/graph/decay.ts @@ -0,0 +1,320 @@ +/** + * graph-memory-pro — 柔性衰减（三因子加权评分 + tier 双向转换） + * + * 完整公式、字段映射、默认值来源、调参指南见 docs/decay.md。 + * 评分 / tier 决策 / applyDecay 的入口均在本文件。 + * + * 调用时机：runMaintenance 的第 0 步（去重/PageRank/社区之前）。 + * decay 调整 tier；另承担两阶段生命周期的阶段一（autoDeprecate，见 shouldAutoDeprecate）： + * 长期未被访问的 peripheral 低分节点被自动断联 + deprecated（deprecatedBy='decay'）， + * 阶段二（purge 到期硬删）在 maintenance.ts 里调用 store.purgeDeprecatedNodes。 + */ + +import type { Driver } from "neo4j-driver"; +import type { GmConfig, DecayConfig, GmNode, NodeTier } from "../types.ts"; +import { getSession } from "../store/db.ts"; +import { allActiveNodes, autoDeprecateNodes } from "../store/store.ts"; + +const MS_PER_DAY = 86_400_000; + +export interface CompositeScore { + composite: number; + recency: number; + frequency: number; + intrinsic: number; +} + +export interface TierTransition { + coreToWorking: number; + workingToPeripheral: number; + peripheralToWorking: number; + workingToCore: number; +} + +export interface DecayResult { + enabled: boolean; + scanned: number; + tierTransitions: TierTransition; + /** 本次维护中被遗忘曲线自动弃用（断联 + deprecated）的节点数。 */ + autoDeprecated: number; + /** 自动弃用批量写失败时的错误（fail-soft，不否定评分/tier 步骤）。 */ + autoDeprecateError?: string; + durationMs: number; +} + +// ─── 归一化辅助（纯函数，便于单元测试） ────────────────────── + +/** importance ∈ [0,1]：当前批次的 pagerank 归一化值。 */ +export function normalizeImportance(pagerank: number, maxPagerank: number): number { + if (maxPagerank <= 0) return 0; + return Math.min(1, Math.max(0, pagerank / maxPagerank)); +} + +/** confidence ∈ [0,1)：validatedCount 越高越可信，饱和收敛到 1。 */ +export function computeConfidence(validatedCount: number): number { + const c = Math.max(0, validatedCount); + return 1 - 1 / (1 + c); +} + +// ─── 三因子评分（纯函数） ──────────────────────────────────── + +/** 选 lastAccessedAt → updatedAt → createdAt 中第一个 > 0 的，用于回退旧节点缺字段。 */ +function pickLastActive(node: Pick): number { + const la = node.lastAccessedAt ?? 0; + const up = node.updatedAt ?? 0; + if (la > 0) return la; + if (up > 0) return up; + return node.createdAt ?? 0; +} + +/** β 随 tier 变化：core 缓衰、peripheral 促衰。 */ +export function computeBeta(tier: NodeTier, cfg: DecayConfig): number { + switch (tier) { + case "core": return cfg.betaCore; + case "working": return cfg.betaWorking; + case "peripheral": return cfg.betaPeripheral; + } +} + +/** + * Recency 分量：Weibull 拉伸指数衰减。 + * tier 决定 β；importance 调制半衰期（高重要性 → 慢衰减）。 + */ +export function scoreRecency( + node: Pick, + importance: number, + now: number, + cfg: DecayConfig, +): number { + const lastActive = pickLastActive(node); + const daysSince = Math.max(0, (now - lastActive) / MS_PER_DAY); + + const effectiveHL = cfg.recencyHalfLifeDays * Math.exp(cfg.importanceModulation * importance); + const lambda = Math.LN2 / effectiveHL; + const beta = computeBeta(node.tier ?? "working", cfg); + + return Math.exp(-lambda * Math.pow(daysSince, beta)); +} + +/** + * Frequency 分量：基础饱和项 × 平均访问间隔新鲜度。 + * validatedCount ≤ 1 时只返回基础项（无法算平均间隔）。 + */ +export function scoreFrequency( + node: Pick, +): number { + const count = Math.max(0, node.validatedCount); + const base = 1 - Math.exp(-count / 5); + if (count <= 1) return base; + + const lastActive = pickLastActive(node); + const accessSpanDays = Math.max(1, (lastActive - node.createdAt) / MS_PER_DAY); + const avgGapDays = accessSpanDays / Math.max(count - 1, 1); + const recentnessBonus = Math.exp(-avgGapDays / 30); + + return base * (0.5 + 0.5 * recentnessBonus); +} + +/** Intrinsic 分量：importance × confidence。 */ +export function scoreIntrinsic(importance: number, confidence: number): number { + return importance * confidence; +} + +/** 三因子加权汇总。权重和在运行时归一化到 1，避免用户配置偏差导致 composite > 1。 */ +export function scoreNode( + node: Pick, + maxPagerank: number, + now: number, + cfg: DecayConfig, +): CompositeScore { + const importance = normalizeImportance(node.pagerank, maxPagerank); + const confidence = computeConfidence(node.validatedCount); + const recency = scoreRecency(node, importance, now, cfg); + const frequency = scoreFrequency(node); + const intrinsic = scoreIntrinsic(importance, confidence); + + const wSum = cfg.recencyWeight + cfg.frequencyWeight + cfg.intrinsicWeight; + const safeSum = wSum > 0 ? wSum : 1; + const wR = cfg.recencyWeight / safeSum; + const wF = cfg.frequencyWeight / safeSum; + const wI = cfg.intrinsicWeight / safeSum; + + const composite = wR * recency + wF * frequency + wI * intrinsic; + + return { composite, recency, frequency, intrinsic }; +} + +// ─── Tier 转换决策（纯函数） ───────────────────────────────── + +/** + * 决定节点的下一个 tier。返回 null 表示保持不变。 + * importance 已归一化（调用方须先 normalizeImportance）。 + */ +export function decideTierTransition( + node: Pick, + score: CompositeScore, + importance: number, + cfg: DecayConfig, + now: number = Date.now(), +): NodeTier | null { + const current = node.tier ?? "working"; + const count = node.validatedCount; + const ageDays = Math.max(0, (now - node.createdAt) / MS_PER_DAY); + const composite = score.composite; + + if (current === "core" + && composite < cfg.peripheralCompositeThreshold + && count < cfg.workingAccessThreshold) { + return "working"; + } + + if (current === "working") { + if (composite < cfg.peripheralCompositeThreshold) return "peripheral"; + if (ageDays > cfg.peripheralAgeDays && count < cfg.workingAccessThreshold) { + return "peripheral"; + } + } + + if (current === "peripheral" + && count >= cfg.workingAccessThreshold + && composite >= cfg.workingCompositeThreshold) { + return "working"; + } + + if (current === "working" + && count >= cfg.coreAccessThreshold + && composite >= cfg.coreCompositeThreshold + && importance >= cfg.coreImportanceThreshold) { + return "core"; + } + + return null; +} + +// ─── 遗忘曲线自动弃用决策（纯函数，两阶段生命周期·阶段一） ─── + +/** + * 判断节点是否应被自动弃用（断联 + status='deprecated'，deprecatedBy='decay'）。 + * 三个条件同时满足（复用遗忘曲线判定，tier 转换先行）： + * 1. 已降至 peripheral 层（遗忘曲线认为不再活跃）； + * 2. composite 仍低于 peripheralCompositeThreshold（高价值节点受 intrinsic 保护）； + * 3. 距最近访问（lastAccessedAt → updatedAt → createdAt 回退链）≥ autoDeprecateAfterDays。 + * autoDeprecate=false 时恒 false。硬删（阶段二）不在此判定，见 store.purgeDeprecatedNodes。 + */ +export function shouldAutoDeprecate( + node: Pick, + score: CompositeScore, + cfg: DecayConfig, + now: number = Date.now(), +): boolean { + if (!cfg.autoDeprecate) return false; + if ((node.tier ?? "working") !== "peripheral") return false; + if (score.composite >= cfg.peripheralCompositeThreshold) return false; + const lastActive = pickLastActive(node); + const daysSince = Math.max(0, (now - lastActive) / MS_PER_DAY); + return daysSince >= cfg.autoDeprecateAfterDays; +} + +// ─── 应用层：扫描 + 评分 + 转换 ────────────────────────────── + +const EMPTY_TRANSITIONS: TierTransition = { + coreToWorking: 0, + workingToPeripheral: 0, + peripheralToWorking: 0, + workingToCore: 0, +}; + +function bumpTransition(transitions: TierTransition, from: NodeTier, to: NodeTier): void { + if (from === "core" && to === "working") transitions.coreToWorking++; + else if (from === "working" && to === "peripheral") transitions.workingToPeripheral++; + else if (from === "peripheral" && to === "working") transitions.peripheralToWorking++; + else if (from === "working" && to === "core") transitions.workingToCore++; +} + +/** + * 扫描所有 active 节点：评分 + tier 转换 + 写回 decayScore / tier； + * autoDeprecate 开启时，对同时满足 shouldAutoDeprecate 的节点执行阶段一自动弃用 + * （断联 + deprecated，deprecatedBy='decay'，可被重新提取/编辑复活）。 + */ +export async function applyDecay(driver: Driver, cfg: Pick): Promise { + const start = Date.now(); + const d = cfg.decay; + if (!d?.enabled) { + return { enabled: false, scanned: 0, tierTransitions: { ...EMPTY_TRANSITIONS }, autoDeprecated: 0, durationMs: 0 }; + } + + const nodes = await allActiveNodes(driver); + if (nodes.length === 0) { + return { enabled: true, scanned: 0, tierTransitions: { ...EMPTY_TRANSITIONS }, autoDeprecated: 0, durationMs: 0 }; + } + + // reduce 而非 Math.max(...map)：spread 在超大节点集（>10 万）会爆调用栈 + const maxPagerank = nodes.reduce((m, n) => Math.max(m, n.pagerank), 0.0001); + + const updates: Array<{ id: string; tier: NodeTier; composite: number; tierChanged: boolean }> = []; + const autoDeprecateIds: string[] = []; + const transitions: TierTransition = { ...EMPTY_TRANSITIONS }; + + for (const node of nodes) { + const score = scoreNode(node, maxPagerank, start, d); + const importance = normalizeImportance(node.pagerank, maxPagerank); + const currentTier = node.tier ?? "working"; + const nextTier = decideTierTransition(node, score, importance, d, start); + const finalTier = nextTier ?? currentTier; + const tierChanged = nextTier !== null; + + if (tierChanged) bumpTransition(transitions, currentTier, finalTier); + + // 用 tier 转换后的层级判定：本轮刚降到 peripheral 的老节点即刻参与阶段一 + if (shouldAutoDeprecate({ ...node, tier: finalTier }, score, d, start)) { + autoDeprecateIds.push(node.id); + } + + updates.push({ + id: node.id, + tier: finalTier, + composite: score.composite, + tierChanged, + }); + } + + if (updates.length > 0) { + const session = getSession(driver); + try { + await session.run( + `UNWIND $updates AS u + MATCH (n:Task|Skill|Event {id: u.id}) + SET n.tier = u.tier, + n.decayScore = u.composite, + n.decayComputedAt = $now, + n.updatedAt = CASE WHEN u.tierChanged THEN $now ELSE n.updatedAt END`, + { updates, now: start }, + ); + } finally { + await session.close(); + } + } + + // 阶段一自动弃用：断联 + deprecated + [DEPRECATED] 前缀。 + // fail-soft：错误记入结果由调用方记日志，不阻塞维护链其余步骤，下一周期重试。 + let autoDeprecated = 0; + let autoDeprecateError: string | undefined; + if (autoDeprecateIds.length > 0) { + try { + autoDeprecated = await autoDeprecateNodes(driver, autoDeprecateIds, start); + } catch (err) { + autoDeprecateError = String(err); + } + } + + return { + enabled: true, + scanned: nodes.length, + tierTransitions: transitions, + autoDeprecated, + ...(autoDeprecateError ? { autoDeprecateError } : {}), + durationMs: Date.now() - start, + }; +} diff --git a/src/graph/dedup.ts b/src/graph/dedup.ts index 9fac7b5..94e1f4b 100755 --- a/src/graph/dedup.ts +++ b/src/graph/dedup.ts @@ -39,36 +39,37 @@ export async function detectDuplicates(driver: Driver, cfg: GmConfig): Promise ({ + id: r.get("id"), + name: r.get("name"), + embedding: r.get("embedding"), + })); + const searchResult = await session.run(` + UNWIND $nodes AS n + CALL db.index.vector.queryNodes('gm_node_embedding', 5, n.embedding) YIELD node, score + WHERE node.id <> n.id AND node.status = 'active' AND score >= $threshold + RETURN n.id AS nodeA, n.name AS nameA, node.id AS nodeB, node.name AS nameB, score AS similarity + `, { nodes, threshold: cfg.dedupThreshold }); + const pairs: DuplicatePair[] = []; const seenPairs = new Set(); - // 对每个节点做向量搜索 - for (const record of nodesResult.records) { - const nodeId = record.get("id"); - const nodeName = record.get("name"); - const embedding = record.get("embedding"); - - const searchResult = await session.run(` - CALL db.index.vector.queryNodes('gm_node_embedding', 5, $vec) - YIELD node, score - WHERE node.id <> $nodeId AND node.status = 'active' AND score >= $threshold - RETURN node.id AS id, node.name AS name, score - `, { vec: embedding, nodeId, threshold: cfg.dedupThreshold }); - - for (const sr of searchResult.records) { - const otherId = sr.get("id"); - const pairKey = [nodeId, otherId].sort().join("|"); - if (seenPairs.has(pairKey)) continue; - seenPairs.add(pairKey); - - pairs.push({ - nodeA: nodeId, - nodeB: otherId, - nameA: nodeName, - nameB: sr.get("name"), - similarity: sr.get("score"), - }); - } + for (const sr of searchResult.records) { + const nodeId = sr.get("nodeA"); + const otherId = sr.get("nodeB"); + const pairKey = [nodeId, otherId].sort().join("|"); + if (seenPairs.has(pairKey)) continue; + seenPairs.add(pairKey); + + pairs.push({ + nodeA: nodeId, + nodeB: otherId, + nameA: sr.get("nameA"), + nameB: sr.get("nameB"), + similarity: sr.get("similarity"), + }); } return pairs.sort((a, b) => b.similarity - a.similarity); diff --git a/src/graph/maintenance.ts b/src/graph/maintenance.ts index 64cd4fa..25f7a0e 100755 --- a/src/graph/maintenance.ts +++ b/src/graph/maintenance.ts @@ -2,7 +2,7 @@ * graph-memory-pro — 图谱维护 * * 调用时机：session_end（finalize 之后） - * 执行顺序：去重 → 全局 PageRank → 社区检测 → 社区描述 + * 执行顺序：衰减（含自动弃用）→ 过期清理（硬删）→ 去重 → 全局 PageRank → 社区检测 → 社区描述 → 消息保留（opt-in） */ import type { Driver } from "neo4j-driver"; @@ -12,12 +12,26 @@ import type { EmbedFn } from "../engine/embed.ts"; import { computeGlobalPageRank, type GlobalPageRankResult } from "./pagerank.ts"; import { detectCommunities, summarizeCommunities, type CommunityResult } from "./community.ts"; import { dedup, type DedupResult } from "./dedup.ts"; +import { applyDecay, type DecayResult } from "./decay.ts"; +import { purgeDeprecatedNodes } from "../store/store.ts"; +import { + normalizeMessageRetentionPolicy, runMessageRetention, + type MessageRetentionResult, +} from "../store/retention.ts"; export interface MaintenanceResult { + decay: DecayResult; + /** 两阶段生命周期·阶段二：本次硬删的过期 deprecated 节点数（purgeAfterDays=0 时恒 0）。 */ + purged: number; + /** 硬删步骤失败时的错误（fail-soft，不否定整轮维护；下一周期重试）。 */ + purgeError?: string; dedup: DedupResult; pagerank: GlobalPageRankResult; community: CommunityResult; communitySummaries: number; + /** 原始消息保留（opt-in）；未配置 messageRetention 或 keep=all 时为 undefined， + * 执行失败时为 { error }（fail-soft，不否定整轮维护）。 */ + retention?: MessageRetentionResult | { error: string }; durationMs: number; } @@ -26,6 +40,20 @@ export async function runMaintenance( ): Promise { const start = Date.now(); + // 0. 衰减（柔性评分 + tier 转换 + 遗忘曲线自动弃用）—— 先于其他步骤，让后续基于最新 tier 集合运算 + const decayResult = await applyDecay(driver, cfg); + + // 0.5 过期 deprecated 节点硬删（两阶段生命周期·阶段二，释放存储）。 + // 独立于 decay.enabled：手动弃用/merge 的节点同样到期清理，只看 purgeAfterDays。 + // fail-soft：错误记入结果由调用方记日志，不否定已完成的 decay 及后续步骤。 + let purged = 0; + let purgeError: string | undefined; + try { + purged = await purgeDeprecatedNodes(driver, (cfg.decay?.purgeAfterDays ?? 0) * 86_400_000, start); + } catch (err) { + purgeError = String(err); + } + // 1. 去重 const dedupResult = await dedup(driver, cfg); @@ -43,11 +71,31 @@ export async function runMaintenance( } catch {} } + // 5. 原始消息保留（opt-in；keep=all 零开销直返）。 + // fail-soft：链尾清理失败（非法策略 fail-closed 不删 / Neo4j 故障）不应否定 + // 已完成的 decay/dedup/PageRank/社区步骤 —— 前面步骤成功说明连接健康， + // 也不计入 Neo4j 熔断。错误记入结果，由调用方记日志，下一周期重试。 + let retention: MessageRetentionResult | { error: string } | undefined; + if (cfg.messageRetention) { + try { + const policy = normalizeMessageRetentionPolicy(cfg.messageRetention); + if (policy.keep !== "all") { + retention = await runMessageRetention(driver, policy); + } + } catch (err) { + retention = { error: String(err) }; + } + } + return { + decay: decayResult, + purged, + ...(purgeError ? { purgeError } : {}), dedup: dedupResult, pagerank: pagerankResult, community: communityResult, communitySummaries, + ...(retention ? { retention } : {}), durationMs: Date.now() - start, }; } diff --git a/src/graph/pagerank.ts b/src/graph/pagerank.ts index 2cb7030..5a37d86 100755 --- a/src/graph/pagerank.ts +++ b/src/graph/pagerank.ts @@ -5,11 +5,37 @@ * 所以先查有哪些关系类型，只投影存在的 */ -import type { Driver } from "neo4j-driver"; +import type { Driver, Session } from "neo4j-driver"; import type { GmConfig } from "../types.ts"; import { getSession } from "../store/db.ts"; import { getExistingActiveRelTypes, projectActiveGraph } from "./projection.ts"; +/** + * 读回 pagerank top-20 并构建 scores Map + topK 数组 —— global PR 的三条路径 + * （均匀分兜底 / GDS write 后读回 / 无 GDS fallback）共用的收尾读取。 + * tiebreakCreatedAt=true 时以 createdAt ASC 决胜（fallback 路径的稳定序）。 + */ +async function readTopKByPagerank(session: Session, tiebreakCreatedAt = false): Promise<{ + scores: Map; + topK: Array<{ id: string; name: string; score: number }>; +}> { + const topResult = await session.run(` + MATCH (n:Task|Skill|Event {status: 'active'}) + RETURN n.id AS id, n.name AS name, n.pagerank AS score + ORDER BY n.pagerank DESC ${tiebreakCreatedAt ? ", n.createdAt ASC" : ""} + LIMIT 20 + `); + const scores = new Map(); + const topK: Array<{ id: string; name: string; score: number }> = []; + for (const r of topResult.records) { + const rawScore = r.get("score"); + const score = typeof rawScore === "number" ? rawScore : (rawScore?.toNumber?.() ?? 0); + scores.set(r.get("id"), score); + topK.push({ id: r.get("id"), name: r.get("name"), score }); + } + return { scores, topK }; +} + // ─── 个性化 PageRank ───────────────────────────────────────── export interface PPRResult { @@ -108,44 +134,32 @@ export async function computeGlobalPageRank(driver: Driver, cfg: GmConfig): Prom // 没有关系，均匀分 const uniformScore = 1 / nodeCount; await session.run("MATCH (n:Task|Skill|Event {status: 'active'}) SET n.pagerank = $score", { score: uniformScore }); - const topResult = await session.run(` - MATCH (n:Task|Skill|Event {status: 'active'}) RETURN n.id AS id, n.name AS name, n.pagerank AS score - ORDER BY n.pagerank DESC LIMIT 20 - `); - const scores = new Map(); - const topK = topResult.records.map(r => { - scores.set(r.get("id"), uniformScore); - return { id: r.get("id"), name: r.get("name"), score: uniformScore }; - }); - return { scores, topK }; + return await readTopKByPagerank(session); } await projectActiveGraph(session, graphName, existingTypes); - await session.run(` - CALL gds.pageRank.write('${graphName}', { - writeProperty: 'pagerank', - dampingFactor: $damping, - maxIterations: toInteger($iterations) - }) - `, { damping: cfg.pagerankDamping, iterations: cfg.pagerankIterations }); - - await session.run(`CALL gds.graph.drop('${graphName}')`); - - const topResult = await session.run(` - MATCH (n:Task|Skill|Event {status: 'active'}) RETURN n.id AS id, n.name AS name, n.pagerank AS score - ORDER BY n.pagerank DESC LIMIT 20 - `); + try { + await session.run(` + CALL gds.pageRank.write('${graphName}', { + writeProperty: 'pagerank', + dampingFactor: $damping, + maxIterations: toInteger($iterations) + }) + `, { damping: cfg.pagerankDamping, iterations: cfg.pagerankIterations }); + } finally { + // drop 失败只能吞掉（宁泄漏一个临时投影）：此分支若抛错落入外层 catch， + // fallback 会用 1/(i+1) 覆盖刚 write 成功的真实 PageRank —— 数据损坏远重于投影泄漏 + try { await session.run(`CALL gds.graph.drop('${graphName}')`); } catch {} + } - const scores = new Map(); - const topK: Array<{ id: string; name: string; score: number }> = []; - for (const r of topResult.records) { - const rawScore = r.get("score"); - const score = typeof rawScore === "number" ? rawScore : (rawScore?.toNumber?.() ?? 0); - scores.set(r.get("id"), score); - topK.push({ id: r.get("id"), name: r.get("name"), score }); + // write 已成功：pagerank 属性已是真值，后续读取失败只返回空排序， + // 绝不回落外层 catch 的 fallback（那会覆盖全图正确分数） + try { + return await readTopKByPagerank(session); + } catch { + return { scores: new Map(), topK: [] }; } - return { scores, topK }; } catch { try { await session.run(`CALL gds.graph.drop('${graphName}')`); } catch {} // GDS 不可用时降级为确定性 fallback（与 PPR 一致：按稳定排序赋 1/(i+1)） @@ -157,23 +171,7 @@ export async function computeGlobalPageRank(driver: Driver, cfg: GmConfig): Prom WITH nodes[idx] AS node, idx SET node.pagerank = 1.0 / toFloat(idx + 1) `); - const fallbackResult = await session.run(` - MATCH (n:Task|Skill|Event {status: 'active'}) - RETURN n.id AS id, n.name AS name, n.pagerank AS score - ORDER BY n.pagerank DESC, n.createdAt ASC - LIMIT 20 - `); - const scores = new Map(); - const topK: Array<{ id: string; name: string; score: number }> = []; - for (const r of fallbackResult.records) { - const rawScore = r.get("score"); - const score = typeof rawScore === "number" ? rawScore : (rawScore?.toNumber?.() ?? 0); - const id = r.get("id"); - const name = r.get("name"); - scores.set(id, score); - topK.push({ id, name, score }); - } - return { scores, topK }; + return await readTopKByPagerank(session, true); } finally { await session.close(); } diff --git a/src/graph/projection.ts b/src/graph/projection.ts index f35dc6b..fb78173 100644 --- a/src/graph/projection.ts +++ b/src/graph/projection.ts @@ -1,12 +1,8 @@ import type { Session } from "neo4j-driver"; +import { EDGE_TYPES } from "../types.ts"; -const KNOWLEDGE_REL_TYPES = [ - "USED_SKILL", - "SOLVED_BY", - "REQUIRES", - "PATCHES", - "CONFLICTS_WITH", -] as const; +/** 知识边类型白名单（事实源 types.ts 的 EDGE_TYPES，勿另维护字面量副本） */ +const KNOWLEDGE_REL_TYPES: readonly string[] = [...EDGE_TYPES]; export async function getExistingActiveRelTypes(session: Session): Promise { const result = await session.run(` diff --git a/src/recaller/query-cache.ts b/src/recaller/query-cache.ts new file mode 100644 index 0000000..adf3479 --- /dev/null +++ b/src/recaller/query-cache.ts @@ -0,0 +1,43 @@ +/** + * graph-memory-pro — 查询向量 LRU 缓存 + * + * recall 的 query embedding 是每次召回的固定开销（低成本但高频）： + * 同会话内重复/回退的相同查询（before_agent_start ↔ assemble ↔ gm_search） + * 直接复用向量，省一次 embedding API 调用。 + * + * 注意：db 模式向量不走缓存（节点文本入库前必算，且 MiniMax 的 db/query + * 走不同模型，向量不可互换）。embedding 端点/模型切换时调用方应 clear()。 + */ + +export class QueryVecCache { + private map = new Map(); + + constructor(private readonly capacity = 64) {} + + get(key: string): number[] | undefined { + const hit = this.map.get(key); + if (hit === undefined) return undefined; + // LRU 触碰：删掉重插，移到最新端 + this.map.delete(key); + this.map.set(key, hit); + return hit; + } + + set(key: string, vec: number[]): void { + if (this.map.has(key)) this.map.delete(key); + else if (this.map.size >= this.capacity) { + // Map 迭代序 = 插入序，最旧的是第一个 key + const oldest = this.map.keys().next().value; + if (oldest !== undefined) this.map.delete(oldest); + } + this.map.set(key, vec); + } + + clear(): void { + this.map.clear(); + } + + get size(): number { + return this.map.size; + } +} diff --git a/src/recaller/recall.ts b/src/recaller/recall.ts index 87ec456..869b93d 100755 --- a/src/recaller/recall.ts +++ b/src/recaller/recall.ts @@ -7,15 +7,19 @@ import type { Driver } from "neo4j-driver"; import { createHash } from "crypto"; import type { GmConfig, RecallResult, GmNode, GmEdge } from "../types.ts"; -import type { EmbedFn } from "../engine/embed.ts"; +import type { EmbedFn, EmbedBatchFn } from "../engine/embed.ts"; import { searchNodes, vectorSearchWithScore, graphWalk, communityRepresentatives, communityVectorSearch, nodesByCommunityIds, - saveVector, getVectorHash, + saveVector, getVectorHash, getVectorHashes, saveVectors, } from "../store/store.ts"; import { getCommunityPeers } from "../graph/community.ts"; import { personalizedPageRank } from "../graph/pagerank.ts"; +import { QueryVecCache } from "./query-cache.ts"; + +/** 批量嵌入分块大小：对齐 reembed 默认批（32），兼顾服务端批量上限与吞吐。 */ +const SYNC_EMBED_BATCH = 32; export function buildNodeEmbeddingText( node: Pick, @@ -25,9 +29,9 @@ export function buildNodeEmbeddingText( // ─── 时间筛选 ─────────────────────────────────────────────── -export type RecallTimeField = "createdAt" | "updatedAt"; +type RecallTimeField = "createdAt" | "updatedAt"; -export interface RecallOptions { +interface RecallOptions { /** ISO 8601 字符串；只返回 timeField 对应时刻 >= after 的节点 */ after?: string; /** ISO 8601 字符串；只返回 timeField 对应时刻 <= before 的节点 */ @@ -83,17 +87,51 @@ export function matchTimeRange( export class Recaller { private embed: EmbedFn | null = null; + private embedBatch: EmbedBatchFn | null = null; + /** 查询向量 LRU：同文本重复召回省一次 embedding 调用（db 模式不入缓存）。 */ + private queryVecCache = new QueryVecCache(); constructor(private driver: Driver, private cfg: GmConfig) {} - setEmbedFn(fn: EmbedFn): void { this.embed = fn; } + setEmbedFn(fn: EmbedFn): void { + this.embed = fn; + // 端点可能刚从失败恢复/模型切换：旧向量与新端点不可混用 + this.queryVecCache.clear(); + } + + /** 注入批量 embedder（createEmbedder.embedBatch）；运行时批量同步向量用。 */ + setEmbedBatchFn(fn: EmbedBatchFn): void { + this.embedBatch = fn; + this.queryVecCache.clear(); + } + + /** 是否已接入 embedding（启动 probe 成功或会话级 re-probe 成功）。 */ + hasEmbedFn(): boolean { return this.embed !== null; } + + /** 只读暴露 embedFn（maintenance / gm_maintain 需要），替代 (recaller as any).embed。 */ + get embedFn(): EmbedFn | null { return this.embed; } async recall(query: string, options?: RecallOptions): Promise { const limit = this.cfg.recallMaxNodes; const timeRange = options ? parseTimeRange(options) : null; - const precise = await this.recallPrecise(query, limit, timeRange); - const generalized = await this.recallGeneralized(query, limit, timeRange); + // query 向量只算一次，两条路径共享；失败统一落 null（各路径走文本兜底）。 + // LRU 命中时零 API 调用（同会话重复查询：before_agent_start ↔ assemble ↔ gm_search）。 + // 双路径并行执行 —— 原串行 + 各自 embed 会把 2 次调用放大成 4 次 API 调用 + // 与 4 次图遍历，全部压在调用方的预算窗口内。 + const cachedVec = this.queryVecCache.get(query); + const embedPromise: Promise = this.embed + ? cachedVec + ? Promise.resolve(cachedVec) + : this.embed(query, "query") + .then((vec) => { this.queryVecCache.set(query, vec); return vec; }) + .catch(() => null) + : Promise.resolve(null); + + const [precise, generalized] = await Promise.all([ + this.recallPrecise(query, limit, timeRange, embedPromise), + this.recallGeneralized(limit, timeRange, embedPromise), + ]); const merged = this.mergeResults(precise, generalized); return merged; @@ -106,12 +144,13 @@ export class Recaller { query: string, limit: number, timeRange: ParsedTimeRange | null, + embedPromise: Promise, ): Promise { let seeds: GmNode[] = []; - if (this.embed) { + const vec = await embedPromise; + if (vec) { try { - const vec = await this.embed(query, "query"); const scored = await vectorSearchWithScore(this.driver, vec, Math.ceil(limit / 2)); seeds = scored.map(s => s.node); @@ -127,7 +166,7 @@ export class Recaller { seeds = await searchNodes(this.driver, query, limit); } - if (!seeds.length) return { nodes: [], edges: [], tokenEstimate: 0 }; + if (!seeds.length) return { nodes: [], edges: [] }; const seedIds = seeds.map(n => n.id); @@ -145,7 +184,7 @@ export class Recaller { this.cfg.recallMaxDepth, ); - if (!nodes.length) return { nodes: [], edges: [], tokenEstimate: 0 }; + if (!nodes.length) return { nodes: [], edges: [] }; // PPR 排序 const candidateIds = nodes.map(n => n.id); @@ -153,6 +192,10 @@ export class Recaller { this.driver, seedIds, candidateIds, this.cfg, ); + // PPR 排序后的过滤/截断尾块与 recallGeneralized 结构一致，但 tiebreak + // 顺序刻意相反：precise 以 validatedCount 决胜（精确命中的知识更受认可）， + // generalized 以 updatedAt 决胜（泛化探索偏向新鲜）。改动前先同步 + // test/ablation.study.test.ts 的镜像 harness。 const filtered = nodes .filter(n => !timeRange || matchTimeRange(n, timeRange)) .sort((a, b) => @@ -166,7 +209,6 @@ export class Recaller { return { nodes: filtered, edges: edges.filter(e => ids.has(e.fromId) && ids.has(e.toId)), - tokenEstimate: this.estimateTokens(filtered), }; } @@ -174,15 +216,15 @@ export class Recaller { * 泛化召回：社区向量搜索 → 图遍历 → PPR 排序 */ private async recallGeneralized( - query: string, limit: number, timeRange: ParsedTimeRange | null, + embedPromise: Promise, ): Promise { let seeds: GmNode[] = []; - if (this.embed) { + const vec = await embedPromise; + if (vec) { try { - const vec = await this.embed(query, "query"); const scoredCommunities = await communityVectorSearch(this.driver, vec); if (scoredCommunities.length > 0) { @@ -197,17 +239,18 @@ export class Recaller { seeds = await communityRepresentatives(this.driver, 2); } - if (!seeds.length) return { nodes: [], edges: [], tokenEstimate: 0 }; + if (!seeds.length) return { nodes: [], edges: [] }; const seedIds = seeds.map(n => n.id); const { nodes, edges } = await graphWalk(this.driver, seedIds, 1); - if (!nodes.length) return { nodes: [], edges: [], tokenEstimate: 0 }; + if (!nodes.length) return { nodes: [], edges: [] }; const candidateIds = nodes.map(n => n.id); const { scores: pprScores } = await personalizedPageRank( this.driver, seedIds, candidateIds, this.cfg, ); + // tiebreak 与 recallPrecise 相反（updatedAt 决胜）——见该处的注释 const filtered = nodes .filter(n => !timeRange || matchTimeRange(n, timeRange)) .sort((a, b) => @@ -221,7 +264,6 @@ export class Recaller { return { nodes: filtered, edges: edges.filter(e => ids.has(e.fromId) && ids.has(e.toId)), - tokenEstimate: this.estimateTokens(filtered), }; } @@ -245,22 +287,69 @@ export class Recaller { const nodes = Array.from(nodeMap.values()); const edges = Array.from(edgeMap.values()); - return { nodes, edges, tokenEstimate: this.estimateTokens(nodes) }; + return { nodes, edges }; } - private estimateTokens(nodes: GmNode[]): number { - return Math.ceil(nodes.reduce((s, n) => s + n.content.length + n.description.length, 0) / 3); - } async syncEmbed(node: GmNode): Promise { - if (!this.embed) return; + if (!this.embed && !this.embedBatch) return; const text = buildNodeEmbeddingText(node); const hash = createHash("md5").update(text).digest("hex"); const existingHash = await getVectorHash(this.driver, node.id); if (existingHash === hash) return; try { - const vec = await this.embed(text, "db"); - if (vec.length) await saveVector(this.driver, node.id, text, vec); + const vec = this.embed + ? await this.embed(text, "db") + : (await this.embedBatch!([text], "db"))[0]; + if (vec.length) await saveVector(this.driver, node.id, text, vec, hash); } catch {} } + + /** + * 批量同步节点向量（embedding 成本控制）：一批提取产出 N 个节点时， + * N 次单发 API + N 次 hash 查询 → 一次批量 embed + 一次 UNWIND 批读写。 + * 与 syncEmbed 相同的 contentHash 短路语义（未变节点零调用）； + * 按 SYNC_EMBED_BATCH 分块（服务端批量上限保护），单块失败 fail-soft 跳过。 + * MiniMax 的 texts+type 批量格式由 embedder 的 embedBatch 内部处理。 + */ + async syncEmbedBatch(nodes: GmNode[]): Promise { + if (!nodes.length) return; + if (!this.embed && !this.embedBatch) return; + + const targets = nodes.map((node) => { + const text = buildNodeEmbeddingText(node); + return { node, text, hash: createHash("md5").update(text).digest("hex") }; + }); + + let existing: Map; + try { + existing = await getVectorHashes(this.driver, targets.map((t) => t.node.id)); + } catch { + return; // hash 批查失败：fail-soft，与单发路径的吞错策略一致 + } + const pending = targets.filter((t) => existing.get(t.node.id) !== t.hash); + if (!pending.length) return; + + if (!this.embedBatch) { + // 无批量能力（仅 setEmbedFn 的旧接线）：退回逐节点单发 + for (const t of pending) { + try { + const vec = await this.embed!(t.text, "db"); + if (vec.length) await saveVector(this.driver, t.node.id, t.text, vec, t.hash); + } catch {} + } + return; + } + + for (let i = 0; i < pending.length; i += SYNC_EMBED_BATCH) { + const chunk = pending.slice(i, i + SYNC_EMBED_BATCH); + try { + const vecs = await this.embedBatch(chunk.map((t) => t.text), "db"); + if (vecs.length !== chunk.length) continue; // 数量校验失败：parse 层应已抛错，双保险 + await saveVectors(this.driver, chunk.map((t, j) => ({ + nodeId: t.node.id, content: t.text, vec: vecs[j], hash: t.hash, + }))); + } catch {} + } + } } diff --git a/src/routes/crud.ts b/src/routes/crud.ts index c837ea3..7d6cc0f 100644 --- a/src/routes/crud.ts +++ b/src/routes/crud.ts @@ -16,11 +16,11 @@ import type { Driver } from "neo4j-driver"; import type { OpenClawPluginApi } from "openclaw/plugin-sdk"; import type { Recaller } from "../recaller/recall.ts"; import type { NodeType, EdgeType } from "../types.ts"; -import { NODE_TYPE_TO_LABEL, isValidEdgeDirection } from "../types.ts"; +import { NODE_TYPE_TO_LABEL, isValidEdgeDirection, EDGE_DIRECTION_RULES, EDGE_TYPES } from "../types.ts"; import { - upsertNode, findById, allActiveNodes, allEdges, - upsertEdge, edgesFrom, edgesTo, deprecate, mergeNodes, - searchNodes, getStats, + upsertNode, findById, findByName, allActiveNodes, allEdges, + upsertEdge, edgesFrom, edgesTo, deprecateNodeAndDisconnectById, mergeNodes, + searchNodes, getStats, normalizeName, deleteEdges, deleteEdgeById, } from "../store/store.ts"; import { getSession } from "../store/db.ts"; @@ -262,12 +262,20 @@ async function handleUpdateNode( params.content = body.content as string; } if (body.name !== undefined) { - // Name change — normalize - const newName = (body.name as string).trim().toLowerCase() - .replace(/[\s_]+/g, "-") - .replace(/[^a-z0-9\u4e00-\u9fff\-]/g, "") - .replace(/-{2,}/g, "-") - .replace(/^-|-$/g, ""); + // 与 store/extractor 的 normalizeName 同源（受 normalize-name.test.ts 跨文件一致性保护） + const newName = normalizeName(body.name as string); + if (!newName) { + json(res, 400, { error: "name normalizes to empty string" }); + return true; + } + if (newName !== existing.name) { + // 重名预检：撞 *_name 唯一约束会让裸 session.run 抛成 500，提前回 409 + const conflict = await findByName(driver, newName); + if (conflict && conflict.id !== existing.id) { + json(res, 409, { error: `Node name already exists: ${newName}` }); + return true; + } + } updates.push("n.name = $newName"); params.newName = newName; } @@ -287,12 +295,40 @@ async function handleUpdateNode( if (updates.length > 1) { // always has updatedAt const session = getSession(driver); try { - await session.run( - `MATCH (n:Task|Skill|Event {id: $id}) - SET ${updates.join(", ")} - ${newType ? `REMOVE n:Task, n:Skill, n:Event SET n:${NODE_TYPE_TO_LABEL[newType]}` : ""}`, - params, - ); + // 单事务：改 type 与非法边清理必须原子 —— 若 DELETE 瞬时失败而 SET 已提交， + // 节点会停留在"新 type + 违反白名单的存量边"状态（正是本端点要修复的不变量） + await session.executeWrite(async tx => { + await tx.run( + `MATCH (n:Task|Skill|Event {id: $id}) + SET ${updates.join(", ")} + ${newType ? `REMOVE n:Task, n:Skill, n:Event SET n:${NODE_TYPE_TO_LABEL[newType]}` : ""}`, + params, + ); + + if (newType) { + // 清理方向白名单外的存量边（gm_* 工具链不允许改 type，仅此端点允许—— + // 必须自己恢复图谱不变量）。合法谓词从 EDGE_DIRECTION_RULES 生成（与 + // isValidEdgeDirection 同一事实源），出/入边各一条定向查询——无向匹配无法区分 + // source/target，from/to 集合不对称时会误删合法边。 + const legal = EDGE_TYPES + .map(t => { + const rule = EDGE_DIRECTION_RULES[t]; + return `(type(r) = '${t}' AND source.type IN ${JSON.stringify(rule.from)} AND target.type IN ${JSON.stringify(rule.to)})`; + }) + .join(" OR "); + const types = JSON.stringify([...EDGE_TYPES]); + await tx.run(` + MATCH (source:Task|Skill|Event {id: $id})-[r]->(target:Task|Skill|Event) + WHERE type(r) IN ${types} AND NOT (${legal}) + DELETE r + `, { id }); + await tx.run(` + MATCH (source:Task|Skill|Event)-[r]->(target:Task|Skill|Event {id: $id}) + WHERE type(r) IN ${types} AND NOT (${legal}) + DELETE r + `, { id }); + } + }); } finally { await session.close(); } @@ -311,7 +347,8 @@ async function handleUpdateNode( /** * DELETE /nodes?id=xxx - * 标记节点为 deprecated（软删除） + * 标记节点为 deprecated（断联弃用：切边 + [DEPRECATED] 前缀，等效删除； + * 维护链在 purgeAfterDays 天后物理清理） */ async function handleDeleteNode( res: ServerResponse, @@ -324,17 +361,13 @@ async function handleDeleteNode( return true; } - const existing = await findById(driver, id); + const existing = await deprecateNodeAndDisconnectById(driver, id); if (!existing) { json(res, 404, { error: `Node not found: ${id}` }); return true; } - await deprecate(driver, id); - - // 向量不需要删除 — deprecated 节点的向量搜索时会被 status='active' 过滤掉 - - json(res, 200, { success: true, id, name: existing.name }); + json(res, 200, { success: true, id, name: existing.name, status: existing.status }); return true; } @@ -444,9 +477,8 @@ async function handleCreateEdge( return true; } - const validTypes = ["USED_SKILL", "SOLVED_BY", "REQUIRES", "PATCHES", "CONFLICTS_WITH"]; - if (!validTypes.includes(type)) { - json(res, 400, { error: `Invalid edge type: ${type}. Must be one of: ${validTypes.join(", ")}` }); + if (!EDGE_TYPES.includes(type)) { + json(res, 400, { error: `Invalid edge type: ${type}. Must be one of: ${EDGE_TYPES.join(", ")}` }); return true; } @@ -489,6 +521,7 @@ async function handleCreateEdge( /** * DELETE /edges?id=xxx * 或 DELETE /edges?fromId=xxx&toId=yyy&type=USED_SKILL + * 删除下沉到 store 层（deleteEdgeById / deleteEdges），路由层不写 Cypher。 */ async function handleDeleteEdge( res: ServerResponse, @@ -500,38 +533,20 @@ async function handleDeleteEdge( const toId = query.toId ?? query.to_id; const edgeType = query.type; - const session = getSession(driver); - try { - if (edgeId) { - // Delete by edge id - await session.run(` - MATCH ()-[r]->() - WHERE r.id = $edgeId - DELETE r - `, { edgeId }); - } else if (fromId && toId) { - // Delete by endpoints (+ optional type filter) - if (edgeType) { - await session.run(` - MATCH (a:Task|Skill|Event {id: $fromId})-[r]->(b:Task|Skill|Event {id: $toId}) - WHERE type(r) = $edgeType - DELETE r - `, { fromId, toId, edgeType: edgeType.toUpperCase() }); - } else { - await session.run(` - MATCH (a:Task|Skill|Event {id: $fromId})-[r]->(b:Task|Skill|Event {id: $toId}) - DELETE r - `, { fromId, toId }); - } - } else { - json(res, 400, { error: "Provide either id or fromId+toId" }); - return true; - } - } finally { - await session.close(); + let deleted: number; + if (edgeId) { + deleted = await deleteEdgeById(driver, edgeId); + } else if (fromId && toId) { + deleted = await deleteEdges( + driver, fromId, toId, + edgeType ? (edgeType.toUpperCase() as EdgeType) : undefined, + ); + } else { + json(res, 400, { error: "Provide either id or fromId+toId" }); + return true; } - json(res, 200, { success: true }); + json(res, 200, { success: true, deleted }); return true; } diff --git a/src/store/db.ts b/src/store/db.ts index 620441c..a5f84ab 100755 --- a/src/store/db.ts +++ b/src/store/db.ts @@ -1,52 +1,41 @@ /** * graph-memory-pro — Neo4j 连接管理（加固版） * - * 解决 "Pool is closed" 问题： * - driver 是长生命周期单例，不在 dispose 时关闭 - * - getSession 在 driver 被意外关闭时自动重建 + * - getSession 永远优先模块级单例（见函数注释） */ import neo4j, { type Driver, type Session } from "neo4j-driver"; import type { EmbeddingConfig, Neo4jConfig } from "../types.ts"; let _driver: Driver | null = null; -let _cfg: Neo4jConfig | null = null; /** * 获取 Neo4j Driver 单例 - * 保存配置，支持自动重连 */ export function getDriver(cfg: Neo4jConfig): Driver { - _cfg = cfg; if (_driver) return _driver; _driver = neo4j.driver(cfg.uri, neo4j.auth.basic(cfg.user, cfg.password), { maxConnectionPoolSize: 50, - connectionAcquisitionTimeout: 60000, - maxTransactionRetryTime: 30000, + // 快速失败配合 gate.ts 熔断：掉线时 ~10s 内报错跳闸，而不是每次卡 30-60s + connectionAcquisitionTimeout: 15_000, + maxTransactionRetryTime: 10_000, }); return _driver; } /** * 获取一个 Session（用完必须 close） - * 如果 driver 被关闭了，自动用保存的配置重建 + * + * 永远优先模块级 _driver 单例：调用方（register() 启动时捕获一次并四处传递） + * 持有的旧引用在单例重建后会指向已关闭的池。入参仅作 getDriver 未初始化时 + * 的回退兼容。 + * 注：neo4j-driver 5.x 的 driver.session() 构造阶段不抛错，"Pool is closed" + * 在 session.run() 才报——掉线恢复由 gate 熔断 + 驱动自身连接池重连负责， + * 这里不做（也做不了）session 级重连。 */ -export function getSession(driver: Driver): Session { - try { - return driver.session({ database: "neo4j" }); - } catch (err) { - // Pool is closed — 尝试重建 driver - if (_cfg && String(err).includes("closed")) { - console.log("[graph-memory-pro] reconnecting Neo4j driver..."); - _driver = neo4j.driver(_cfg.uri, neo4j.auth.basic(_cfg.user, _cfg.password), { - maxConnectionPoolSize: 50, - connectionAcquisitionTimeout: 60000, - maxTransactionRetryTime: 30000, - }); - return _driver.session({ database: "neo4j" }); - } - throw err; - } +export function getSession(passedDriver: Driver): Session { + return (_driver ?? passedDriver).session({ database: "neo4j" }); } /** @@ -75,6 +64,7 @@ export async function initSchema(driver: Driver, embedding?: EmbeddingConfig): P // Community await session.run("CREATE CONSTRAINT community_id IF NOT EXISTS FOR (c:Community) REQUIRE c.id IS UNIQUE"); + await session.run("CREATE INDEX community_member_signature IF NOT EXISTS FOR (c:Community) ON (c.memberSignature)"); // Message (temporary extraction buffer) await session.run("CREATE CONSTRAINT gm_msg_id IF NOT EXISTS FOR (m:GmMessage) REQUIRE m.id IS UNIQUE"); @@ -86,7 +76,22 @@ export async function initSchema(driver: Driver, embedding?: EmbeddingConfig): P : 1024; // The search code queries one index across all knowledge labels. - await session.run("MATCH (n:Task|Skill|Event) SET n:MemoryNode"); + // 补标守卫：仅给缺失 MemoryNode 标签的节点补标 —— 常规启动退化为纯读扫描， + // 避免每次启动对全库做无谓写（属性/标签写会触发事务与日志）；首启/迁移兜底语义不变。 + await session.run("MATCH (n:Task|Skill|Event) WHERE NOT n:MemoryNode SET n:MemoryNode"); + + // 存量 deprecated 节点补写 deprecatedAt（幂等，等效一次性迁移）：purge 时钟基准是 + // deprecatedAt、缺失时回退 updatedAt——但 upsertNode 对 manual/merge 弃用节点只 bump + // updatedAt（不复活），存量无 deprecatedAt 的节点每次被重新提取命中都会把 purge 期限 + // 往后推（无限续命、永远删不掉）。启动时把时钟一次性钉死为当时的 + // coalesce(updatedAt, createdAt) 快照，此后 purge 不再受 updatedAt 漂移影响。 + // 全新弃用路径（manual/merge/decay）都已显式写 deprecatedAt，故首次运行后恒零命中。 + await session.run(` + MATCH (n:Task|Skill|Event {status: 'deprecated'}) + WHERE n.deprecatedAt IS NULL + SET n.deprecatedAt = coalesce(n.updatedAt, n.createdAt) + `); + await session.run(` CREATE VECTOR INDEX gm_node_embedding IF NOT EXISTS FOR (n:MemoryNode) ON (n.embedding) @@ -97,6 +102,20 @@ export async function initSchema(driver: Driver, embedding?: EmbeddingConfig): P FOR (c:Community) ON (c.embedding) OPTIONS {indexConfig: {\`vector.dimensions\`: ${dimensions}, \`vector.similarity_function\`: 'cosine'}} `); + + // Turn commit marker (OpenClaw transcript fencing contract). + // 放在 DDL 链末尾：升级后首启若约束尚未建好，并发重试的 CREATE 可能 + // 已写入重复 advancementKey —— 带冲突数据的 CREATE CONSTRAINT 会失败， + // 先清重复行再建约束；即使这里失败也不能波及上面的向量索引。 + // 唯一约束是幂等提交的原子性来源：重试的 CREATE 撞约束报 + // ConstraintValidationFailed，commitTurnAdvance 据此区分 committed/duplicate。 + await session.run(` + MATCH (t:GmTurnCommit) + WITH t.advancementKey AS key, collect(t) AS marks + WHERE size(marks) > 1 + FOREACH (n IN marks[1..] | DELETE n) + `); + await session.run("CREATE CONSTRAINT gm_turn_commit_key IF NOT EXISTS FOR (t:GmTurnCommit) REQUIRE t.advancementKey IS UNIQUE"); } finally { await session.close(); } diff --git a/src/store/gate.ts b/src/store/gate.ts new file mode 100644 index 0000000..7a8e9c3 --- /dev/null +++ b/src/store/gate.ts @@ -0,0 +1,60 @@ +/** + * graph-memory-pro — Neo4j 熔断门控（circuit breaker） + * + * 解决的问题：Neo4j 掉线时，每次 ingest / assemble / recall 都要吃满 + * driver 的重试与连接获取超时（最坏数十秒），对话每轮都被拖住， + * 体验上等同于"卡死"。 + * + * 语义： + * - closed（正常）：isAvailable() === true，所有操作放行。 + * - open（跳闸）：连续失败 >= failureThreshold 次后进入；冷却期内 + * isAvailable() === false，调用方应立即降级（跳过图谱、缓冲消息）， + * 而不是等 driver 超时。 + * - half-open（半开探测）：冷却期结束后 isAvailable() 恢复 true， + * 下一个真实操作充当探测 —— 成功则复位 closed，失败则重新计时冷却。 + * + * 注意：失败计数只应从"纯 Neo4j 调用点"记录（saveMessage / getBySession + * 等）。混合了 LLM / embedding 的调用点（recall、compact）不要记录， + * 否则 LLM 超时会误跳闸。 + */ + +export class Neo4jGate { + private consecutiveFailures = 0; + private open = false; + private openedAt = 0; + + constructor( + /** 连续失败多少次后跳闸 */ + private readonly failureThreshold: number = 2, + /** 跳闸后的冷却时长（ms），到期进入半开 */ + private readonly cooldownMs: number = 120_000, + ) {} + + /** 操作成功：复位计数并闭合熔断。 */ + recordSuccess(): void { + this.consecutiveFailures = 0; + this.open = false; + } + + /** + * 操作失败：累计计数；达到阈值跳闸。 + * 已处于 open 时再次失败（半开探测失败 / 在途请求迟到失败）会 + * 重置冷却计时 —— 但被门控的调用方在 open 期间不会发起操作， + * 所以不会出现"永远无法恢复"的抖动。 + */ + recordFailure(): void { + this.consecutiveFailures += 1; + if (!this.open && this.consecutiveFailures >= this.failureThreshold) { + this.open = true; + } + if (this.open) { + this.openedAt = Date.now(); + } + } + + /** 当前是否放行操作（closed 或 冷却到期的 half-open）。 */ + isAvailable(): boolean { + if (!this.open) return true; + return Date.now() - this.openedAt >= this.cooldownMs; + } +} diff --git a/src/store/retention.ts b/src/store/retention.ts new file mode 100644 index 0000000..8415c6a --- /dev/null +++ b/src/store/retention.ts @@ -0,0 +1,247 @@ +/** + * graph-memory-pro — 原始消息有界保留（opt-in） + * + * 移植自上游 #96（SQLite → Neo4j Cypher）。设计原则不变： + * "上下文压缩改变的是模型可见面，不构成删除持久证据的授权。" + * + * - 默认 keep=all：不做任何删除，本模块在维护链中零开销直返。 + * - referenced：只删"已提取完成且实际产出知识"的消息（extracted=true 且 + * producedKnowledge=true）—— 知识已固化进图谱节点/边，原始文本退役。 + * LLM 空提取（零节点零边）的轮次标记 producedKnowledge=false，原始证据 + * 保留；重挖需手动重置 extracted 后跑 graph-memory extract。 + * - recent：referenced 之上叠加时间窗保护（每 session 最近 N 轮真实用户发言 + * 及其后消息、最近 N 天内入库的消息），验证时要求至少配置一个窗口参数。 + * + * 与上游的差异：v2.0 schema 没有消息级出处边（节点仅记 sourceSessions， + * 粒度为 session），上游 "无 gm_node_sources 引用" 的前置条件在这里等价于 + * extracted=true AND producedKnowledge=true。遗留行（producedKnowledge 属性 + * 缺失，标记机制上线前已提取）fail-closed 不删。DELETE 前按同条件重新校验， + * 防止候选查询与删除语义未来漂移 —— 候选集不能成为删除的授权。 + */ + +import { createHash } from "node:crypto"; +import { int } from "neo4j-driver"; +import type { Driver } from "neo4j-driver"; +import { getSession } from "./db.ts"; +import type { MessageRetentionConfig } from "../types.ts"; + +export type NormalizedMessageRetentionPolicy = Required; + +export interface MessageRetentionResult { + policy: string; + policyRevision: string; + dryRun: boolean; + selectedRows: number; + selectedBytes: number; + deletedRows: number; + deletedBytes: number; + selectedSessions: number; + byRole: Record; + oldestCreatedAt: number | null; + newestCreatedAt: number | null; + hasMore: boolean; + cutoffAt: number; + durationMs: number; +} + +interface CandidateRow { + id: string; + sessionId: string; + role: string; + createdAt: number; + contentBytes: number; +} + +function boundedInteger( + value: unknown, + name: string, + fallback: number, + minimum: number, + maximum: number, +): number { + if (value === undefined) return fallback; + if (!Number.isInteger(value) || Number(value) < minimum || Number(value) > maximum) { + throw new TypeError( + `[graph-memory] messageRetention.${name} must be an integer between ${minimum} and ${maximum}, received ${String(value)}`, + ); + } + return Number(value); +} + +/** 配置校验。非法策略 fail closed（抛错 → 不删任何东西）；register 时提前验证以给出友好报错。 */ +export function normalizeMessageRetentionPolicy( + input: MessageRetentionConfig | undefined, +): NormalizedMessageRetentionPolicy { + if (input === undefined) return { keep: "all", recentTurns: 0, retentionDays: 0, batchSize: 500, dryRun: false }; + if (!input || typeof input !== "object" || Array.isArray(input)) { + throw new TypeError("[graph-memory] messageRetention must be an object"); + } + + const keep = input.keep ?? "all"; + if (keep !== "all" && keep !== "referenced" && keep !== "recent") { + throw new TypeError( + `[graph-memory] messageRetention.keep must be all, referenced, or recent, received ${String(keep)}`, + ); + } + const recentTurns = boundedInteger(input.recentTurns, "recentTurns", 0, 0, 100_000); + const retentionDays = boundedInteger(input.retentionDays, "retentionDays", 0, 0, 36_500); + const batchSize = boundedInteger(input.batchSize, "batchSize", 500, 1, 10_000); + const dryRun = input.dryRun ?? false; + if (typeof dryRun !== "boolean") { + throw new TypeError( + `[graph-memory] messageRetention.dryRun must be a boolean, received ${String(dryRun)}`, + ); + } + if (keep === "recent" && recentTurns === 0 && retentionDays === 0) { + throw new TypeError( + "[graph-memory] messageRetention.keep=recent requires recentTurns or retentionDays", + ); + } + + return { keep, recentTurns, retentionDays, batchSize, dryRun }; +} + +export function messageRetentionPolicyRevision( + policy: NormalizedMessageRetentionPolicy, +): string { + return createHash("sha256") + .update(JSON.stringify(policy)) + .digest("hex") + .slice(0, 12); +} + +function emptyResult(policy: NormalizedMessageRetentionPolicy, cutoffAt: number, start: number): MessageRetentionResult { + return { + policy: policy.keep, + policyRevision: messageRetentionPolicyRevision(policy), + dryRun: policy.dryRun, + selectedRows: 0, + selectedBytes: 0, + deletedRows: 0, + deletedBytes: 0, + selectedSessions: 0, + byRole: {}, + oldestCreatedAt: null, + newestCreatedAt: null, + hasMore: false, + cutoffAt, + durationMs: Date.now() - start, + }; +} + +/** + * 构造候选查询。 + * + * recentTurns 保护：以 session 内最近 N 条 user 消息里最旧一条的 turnIndex 为界， + * 只候选更早的消息（该轮的 assistant/tool 消息随所属轮次保留）； + * 没有任何 user 消息的 session 完全保护 —— 与上游 LEFT JOIN 的 NULL 语义一致。 + * createdAt 缺失/为零/为负/超前一律 fail closed（不进候选）。 + */ +function buildCandidateQuery( + policy: NormalizedMessageRetentionPolicy, + now: number, +): { cypher: string; params: Record } { + const params: Record = { + // JS number 会以 Float 发送 —— LIMIT 与列表下标要求整数，必须用 neo4j int() + recentTurns: int(policy.recentTurns), + batchLimit: int(policy.batchSize + 1), + }; + + let match = "MATCH (m:GmMessage)\nWHERE m.extracted = true AND m.producedKnowledge = true\n"; + if (policy.recentTurns > 0) { + match = + "MATCH (u:GmMessage {role: 'user'})\n" + + "WITH u.sessionId AS sid, u ORDER BY sid, u.turnIndex DESC\n" + + "WITH sid, collect(u.turnIndex) AS turns\n" + + "WITH sid, CASE WHEN size(turns) >= $recentTurns THEN turns[$recentTurns - 1] ELSE -1 END AS cutoffTurn\n" + + "MATCH (m:GmMessage {sessionId: sid})\n" + + "WHERE m.extracted = true AND m.producedKnowledge = true\n" + + " AND m.turnIndex < cutoffTurn\n"; + } + if (policy.retentionDays > 0) { + params.ageCutoff = now - policy.retentionDays * 86_400_000; + match += " AND m.createdAt > 0 AND m.createdAt < $ageCutoff\n"; + } + + const cypher = + match + + "WITH m ORDER BY m.createdAt, m.sessionId, m.turnIndex\n" + + "LIMIT $batchLimit\n" + + "RETURN m.id AS id, m.sessionId AS sessionId, m.role AS role, " + + "m.createdAt AS createdAt, size(m.content) AS contentBytes"; + return { cypher, params }; +} + +/** + * 跑一个有界批次。候选选择与删除放在同一个写事务里冻结边界； + * 删除前按 extracted + producedKnowledge 重新校验（候选查询不是删除的授权）。 + * size(m.content) 返回字符数（近似体积），与上游 BLOB 字节数略有差异，仅用于统计。 + */ +export async function runMessageRetention( + driver: Driver, + policy: NormalizedMessageRetentionPolicy, + now: number = Date.now(), +): Promise { + const start = Date.now(); + if (policy.keep === "all") return emptyResult(policy, now, start); + + const { cypher, params } = buildCandidateQuery(policy, now); + + const session = getSession(driver); + try { + const { rows, deletedRows, hasMore } = await session.executeWrite(async (tx) => { + const candRes = await tx.run(cypher, params); + const rows: CandidateRow[] = candRes.records.map((r) => ({ + id: r.get("id"), + sessionId: r.get("sessionId"), + role: r.get("role"), + createdAt: Number(r.get("createdAt")), + contentBytes: Number(r.get("contentBytes")), + })); + + let hasMore = false; + if (rows.length > policy.batchSize) { + hasMore = true; + rows.pop(); + } + + let deletedRows = 0; + if (!policy.dryRun && rows.length) { + const delRes = await tx.run( + "UNWIND $ids AS mid " + + "MATCH (m:GmMessage {id: mid}) " + + "WHERE m.extracted = true AND m.producedKnowledge = true " + + "DETACH DELETE m " + + "RETURN count(m) AS deleted", + { ids: rows.map((r) => r.id) }, + ); + deletedRows = delRes.records[0]?.get("deleted").toNumber() ?? 0; + } + return { rows, deletedRows, hasMore }; + }); + + const selectedBytes = rows.reduce((total, r) => total + r.contentBytes, 0); + const byRole: Record = {}; + for (const r of rows) byRole[r.role] = (byRole[r.role] ?? 0) + 1; + const created = rows.map((r) => r.createdAt).filter(Number.isFinite); + + return { + policy: policy.keep, + policyRevision: messageRetentionPolicyRevision(policy), + dryRun: policy.dryRun, + selectedRows: rows.length, + selectedBytes, + deletedRows, + deletedBytes: deletedRows === rows.length ? selectedBytes : 0, + selectedSessions: new Set(rows.map((r) => r.sessionId)).size, + byRole, + oldestCreatedAt: created.length ? Math.min(...created) : null, + newestCreatedAt: created.length ? Math.max(...created) : null, + hasMore, + cutoffAt: now, + durationMs: Date.now() - start, + }; + } finally { + await session.close(); + } +} diff --git a/src/store/store.ts b/src/store/store.ts index e08787c..f9a7176 100755 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -7,9 +7,9 @@ import type { Driver } from "neo4j-driver"; import neo4j from "neo4j-driver"; -import { createHash } from "crypto"; -import type { GmNode, GmEdge, EdgeType, NodeType } from "../types.ts"; -import { NODE_TYPE_TO_LABEL, isValidEdgeDirection } from "../types.ts"; +import { createHash, randomUUID } from "crypto"; +import type { GmNode, GmEdge, EdgeType, NodeType, NodeTier, DeprecatedBy } from "../types.ts"; +import { NODE_TYPE_TO_LABEL, isValidEdgeDirection, EDGE_TYPES } from "../types.ts"; import { getSession } from "./db.ts"; /** Neo4j LIMIT/索引参数必须是 Integer */ @@ -20,7 +20,7 @@ function nint(v: number): any { // ─── 工具 ───────────────────────────────────────────────────── function uid(p: string): string { - return `${p}-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`; + return `${p}-${randomUUID()}`; } function toNode(r: any): GmNode { @@ -32,6 +32,8 @@ function toNode(r: any): GmNode { description: n.description ?? "", content: n.content, status: n.status, + tier: (n.tier === "core" || n.tier === "working" || n.tier === "peripheral" + ? n.tier : "working") as NodeTier, validatedCount: toInt(n.validatedCount ?? n.validated_count ?? 1), sourceSessions: typeof n.sourceSessions === "string" ? JSON.parse(n.sourceSessions) @@ -40,6 +42,11 @@ function toNode(r: any): GmNode { pagerank: toFloat(n.pagerank ?? 0), createdAt: toInt(n.createdAt ?? n.created_at ?? 0), updatedAt: toInt(n.updatedAt ?? n.updated_at ?? 0), + lastAccessedAt: toInt(n.lastAccessedAt ?? n.last_accessed_at ?? n.updatedAt ?? n.updated_at ?? n.createdAt ?? 0), + decayScore: typeof n.decayScore === "number" ? n.decayScore : undefined, + decayComputedAt: n.decayComputedAt ? toInt(n.decayComputedAt) : undefined, + deprecatedAt: n.deprecatedAt != null ? toInt(n.deprecatedAt) : undefined, + deprecatedBy: n.deprecatedBy != null ? (n.deprecatedBy as DeprecatedBy) : undefined, }; } @@ -69,6 +76,34 @@ function normalizeName(name: string): string { export { normalizeName }; +// ─── 边行读取（单一来源）───────────────────────────────────── +// 知识边类型白名单的 Cypher 片段与 RETURN 投影/记录映射只在此定义， +// allEdges/edgesFrom/edgesTo/edgesTouching/graphWalk/getStats 共用 —— +// 新增 EdgeType 时只改 types.ts 的 EDGE_TYPES，勿在 Cypher 里内联字面量。 + +const EDGE_TYPE_FILTER = `type(r) IN ${JSON.stringify([...EDGE_TYPES])}`; + +const EDGE_ROW_RETURN = ` + RETURN r.id AS id, a.id AS fromId, b.id AS toId, type(r) AS type, + r.instruction AS instruction, r.condition AS condition, + r.sessionId AS sessionId, r.createdAt AS createdAt`; + +type EdgeRowResult = { records: Array<{ get(key: string): any }> }; + +/** 边行记录 → GmEdge（与 EDGE_ROW_RETURN 投影一一对应） */ +function mapEdgeRecords(result: EdgeRowResult): GmEdge[] { + return result.records.map(r => ({ + id: r.get("id"), + fromId: r.get("fromId"), + toId: r.get("toId"), + type: r.get("type") as EdgeType, + instruction: r.get("instruction"), + condition: r.get("condition") ?? undefined, + sessionId: r.get("sessionId"), + createdAt: toInt(r.get("createdAt")), + })); +} + // ─── 节点 CRUD ─────────────────────────────────────────────── export async function findByName(driver: Driver, name: string): Promise { @@ -116,26 +151,20 @@ export async function allEdges(driver: Driver): Promise { try { const result = await session.run(` MATCH (a:Task|Skill|Event)-[r]->(b:Task|Skill|Event) - WHERE type(r) IN ['USED_SKILL','SOLVED_BY','REQUIRES','PATCHES','CONFLICTS_WITH'] - RETURN r.id AS id, a.id AS fromId, b.id AS toId, type(r) AS type, - r.instruction AS instruction, r.condition AS condition, - r.sessionId AS sessionId, r.createdAt AS createdAt + WHERE ${EDGE_TYPE_FILTER}${EDGE_ROW_RETURN} `); - return result.records.map(r => ({ - id: r.get("id"), - fromId: r.get("fromId"), - toId: r.get("toId"), - type: r.get("type") as EdgeType, - instruction: r.get("instruction"), - condition: r.get("condition") ?? undefined, - sessionId: r.get("sessionId"), - createdAt: toInt(r.get("createdAt")), - })); + return mapEdgeRecords(result); } finally { await session.close(); } } +/** 判断错误是否为 *_name 唯一约束冲突（CREATE 撞上并发创建时用于幂等回退） */ +function isNameConstraintViolation(err: unknown): boolean { + const s = String(err); + return s.includes("ConstraintValidationFailed") || s.includes("already exists with label"); +} + export async function upsertNode( driver: Driver, c: { type: NodeType; name: string; description: string; content: string }, @@ -145,6 +174,51 @@ export async function upsertNode( const label = NODE_TYPE_TO_LABEL[c.type as NodeType]; if (!label) throw new Error(`[graph-memory-pro] Invalid node type: ${String(c.type)}`); const session = getSession(driver); + /** 按 name 更新已存在节点（find 命中与撞约束回退两条路径共用） */ + const updateExisting = async (): Promise<{ node: GmNode; isNew: boolean }> => { + // revived：decay 自动弃用的节点被重新提取命中时自动复活（manual/merge 弃用不复活）。 + // revived 分支剥掉 [DEPRECATED] 前缀并忽略本次 description（语义与 stripDeprecateMarker 一致： + // 仅匹配 "[DEPRECATED] " 前缀或裸 "[DEPRECATED]" 整串，避免误伤恰以该子串开头的原文）。 + // REMOVE 只对 revived 行执行——manual/merge 节点必须保留 deprecatedAt/By， + // 否则既丢弃用溯源，purge 时钟又会回退到本查询刚写入的 updatedAt（无限续命）。 + await session.run(` + MATCH (n:Task|Skill|Event {name: $name}) + WITH n, (n.status = 'deprecated' AND coalesce(n.deprecatedBy, 'manual') = 'decay') AS revived + SET n.content = CASE WHEN size($content) > size(n.content) THEN $content ELSE n.content END, + n.description = CASE + WHEN revived AND n.description STARTS WITH '[DEPRECATED] ' + THEN substring(n.description, size('[DEPRECATED] ')) + WHEN revived AND n.description = '[DEPRECATED]' + THEN '' + WHEN size($description) > size(n.description) THEN $description + ELSE n.description + END, + n.status = CASE WHEN revived THEN 'active' ELSE n.status END, + n.validatedCount = n.validatedCount + 1, + n.sourceSessions = CASE + WHEN NOT $sessionId IN n.sourceSessions + THEN n.sourceSessions + $sessionId + ELSE n.sourceSessions + END, + n.lastAccessedAt = $now, + n.updatedAt = $now + FOREACH (_ IN CASE WHEN revived THEN [1] ELSE [] END + | REMOVE n.deprecatedAt, n.deprecatedBy) + RETURN n + `, { name, content: c.content, description: c.description, sessionId, now: Date.now() }); + + const updated = await session.run( + "MATCH (n:Task|Skill|Event {name: $name}) RETURN n", + { name }, + ); + // 撞约束回退路径存在窄窗口：并发创建的同名节点可能在 MATCH 前被删除 + //（如 maintenance mergeNodes）——守卫让单个节点失败而不是 TypeError 炸整批 + const record = updated.records[0]?.get("n"); + if (!record) { + throw new Error(`[graph-memory-pro] upsertNode: node "${name}" disappeared during update`); + } + return { node: toNode(record), isNew: false }; + }; try { // Try to find existing node with this name across all knowledge labels const existing = await session.run( @@ -153,36 +227,21 @@ export async function upsertNode( ); if (existing.records.length > 0) { - // Update existing node - await session.run(` - MATCH (n:Task|Skill|Event {name: $name}) - SET n.content = CASE WHEN size($content) > size(n.content) THEN $content ELSE n.content END, - n.description = CASE WHEN size($description) > size(n.description) THEN $description ELSE n.description END, - n.validatedCount = n.validatedCount + 1, - n.sourceSessions = CASE - WHEN NOT $sessionId IN n.sourceSessions - THEN n.sourceSessions + $sessionId - ELSE n.sourceSessions - END, - n.updatedAt = $now - RETURN n - `, { name, content: c.content, description: c.description, sessionId, now: Date.now() }); - - const updated = await session.run( - "MATCH (n:Task|Skill|Event {name: $name}) RETURN n", - { name }, - ); - return { node: toNode(updated.records[0].get("n")), isNew: false }; - } else { - // Create new node with specific label + // 必须 return await：否则 finally 的 session.close() 会与闭包内的 + // 第二次 session.run 竞态（closed session 错误） + return await updateExisting(); + } + // Create new node with specific label + try { const now = Date.now(); const result = await session.run(` CREATE (n:MemoryNode:${label} { id: $id, name: $name, type: $type, description: $description, content: $content, - status: 'active', validatedCount: 1, + status: 'active', tier: 'working', validatedCount: 1, sourceSessions: $sessions, communityId: null, - pagerank: 0.0, createdAt: $now, updatedAt: $now + pagerank: 0.0, createdAt: $now, updatedAt: $now, + lastAccessedAt: $now }) RETURN n `, { @@ -191,6 +250,11 @@ export async function upsertNode( sessions: [sessionId], now, }); return { node: toNode(result.records[0].get("n")), isNew: true }; + } catch (err) { + // find-then-create 窗口内并发路径抢先创建了同名节点（撞 *_name 唯一约束） + // → 退回更新路径保持幂等，而不是让整轮提取失败重试 + if (isNameConstraintViolation(err)) return await updateExisting(); + throw err; } } finally { await session.close(); @@ -217,19 +281,32 @@ export async function updateNode( if (!ex) return null; const now = Date.now(); const { description, content } = applyNodePatch(ex, patch); + // decay 自动弃用的节点被手动编辑命中 → 顺手复活（manual/merge 弃用不复活）。 + // REMOVE 仅在复活时执行：非复活节点保留 deprecatedAt/By（溯源 + purge 时钟不被重置）。 + const revived = ex.status === "deprecated" && (ex.deprecatedBy ?? "manual") === "decay"; + const finalDescription = revived ? stripDeprecateMarker(description) : description; const session = getSession(driver); try { await session.run( `MATCH (n:Task|Skill|Event {id: $id}) SET n.description = $description, n.content = $content, - n.updatedAt = $now`, - { id: ex.id, description, content, now }, + n.status = $status, + n.updatedAt = $now + ${revived ? "REMOVE n.deprecatedAt, n.deprecatedBy" : ""}`, + { id: ex.id, description: finalDescription, content, status: revived ? "active" : ex.status, now }, ); } finally { await session.close(); } - return { ...ex, description, content, updatedAt: now }; + return { + ...ex, + description: finalDescription, + content, + updatedAt: now, + status: revived ? "active" : ex.status, + ...(revived ? { deprecatedAt: undefined, deprecatedBy: undefined } : {}), + }; } /** @@ -244,62 +321,125 @@ export function applyDeprecateMarker(description: string): string { } /** - * 按 name 硬删除节点：DETACH DELETE —— 节点 + 所有关系一并删除。 - * 找不到返回 null（调用方决定报错语义）。 + * 剥掉 [DEPRECATED] 前缀（applyDeprecateMarker 的逆操作）—— + * decay 自动弃用的节点复活时还原描述。无前缀则原样返回。 */ -export async function deleteNode(driver: Driver, name: string): Promise { - const ex = await findByName(driver, name); - if (!ex) return null; - const session = getSession(driver); - try { - await session.run( - "MATCH (n:Task|Skill|Event {id: $id}) DETACH DELETE n", - { id: ex.id }, - ); - } finally { - await session.close(); - } - return ex; +export function stripDeprecateMarker(description: string): string { + const prefix = "[DEPRECATED]"; + if (description === prefix) return ""; + if (description.startsWith(`${prefix} `)) return description.slice(prefix.length + 1); + return description; } /** - * 按 name deprecate 并切断：status='deprecated' + 描述加 [DEPRECATED] 前缀 + 删除所有边。 - * 节点本身保留（不硬删），但完全从知识图谱中隔离。 - * 找不到返回 null。 + * 手动弃用（一次性断联，按 id 定位）：status='deprecated' + deprecatedAt/deprecatedBy='manual' + * + 描述加 [DEPRECATED] 前缀 + 删除所有边。所有人工路径（gm_update mode=deprecate、 + * finalize invalidations、REST DELETE）统一走这里。 + * deprecated 节点对所有召回路径不可见且无边可走，效果等同删除；仅保留 purgeAfterDays + * （默认 60 天）反悔窗口，到期由 maintenance 硬删。找不到返回 null。 */ -export async function deprecateNodeAndDisconnect( +export async function deprecateNodeAndDisconnectById( driver: Driver, - name: string, + id: string, + now: number = Date.now(), ): Promise { - const ex = await findByName(driver, name); + const ex = await findById(driver, id); if (!ex) return null; - const now = Date.now(); + // 已弃用节点再次被弃用时保留原 deprecatedAt——否则每次重新提取触发的 + // finalize invalidations / REST 重试都会重置 60 天 purge 时钟（无限续命）。 + const deprecatedAt = ex.status === "deprecated" && ex.deprecatedAt ? ex.deprecatedAt : now; const description = applyDeprecateMarker(ex.description); const session = getSession(driver); try { await session.run( `MATCH (n:Task|Skill|Event {id: $id}) SET n.status = 'deprecated', + n.deprecatedAt = $deprecatedAt, + n.deprecatedBy = 'manual', n.description = $description, n.updatedAt = $now WITH n OPTIONAL MATCH (n)-[r]-() DELETE r`, - { id: ex.id, description, now }, + { id, deprecatedAt, description, now }, ); } finally { await session.close(); } - return { ...ex, status: "deprecated", description, updatedAt: now }; + return { ...ex, status: "deprecated", deprecatedAt, deprecatedBy: "manual", description, updatedAt: now }; } -export async function deprecate(driver: Driver, nodeId: string): Promise { +/** 按 name 弃用并切断（gm_update mode=deprecate 用）；找不到返回 null。 */ +export async function deprecateNodeAndDisconnect( + driver: Driver, + name: string, +): Promise { + const ex = await findByName(driver, name); + if (!ex) return null; + return deprecateNodeAndDisconnectById(driver, ex.id, Date.now()); +} + +/** + * 遗忘曲线自动弃用（两阶段生命周期·阶段一）：批量 status='deprecated' + deprecatedBy='decay' + * + 描述幂等加 [DEPRECATED] 前缀 + 切断所有边。与 deprecateNodeAndDisconnect 的区别： + * 按 id 批量、来源标记 decay（重新提取/编辑命中时可被 upsertNode/updateNode 复活）。 + * MATCH 带状态守卫：ids 来自 applyDecay 的 active 扫描快照，评分与批量写入之间存在 + * 时间窗——窗口内刚被手动弃用的节点不得被覆盖为 decay（既改变复活语义又重置 purge 时钟）， + * 因此只弃用仍为 active 的节点。返回实际弃用的节点数。 + */ +export async function autoDeprecateNodes( + driver: Driver, + ids: string[], + now: number = Date.now(), +): Promise { + if (!ids.length) return 0; const session = getSession(driver); try { - await session.run( - "MATCH (n:Task|Skill|Event {id: $id}) SET n.status = 'deprecated', n.updatedAt = $now", - { id: nodeId, now: Date.now() }, - ); + const result = await session.run(` + UNWIND $ids AS nid + MATCH (n:Task|Skill|Event {id: nid, status: 'active'}) + SET n.status = 'deprecated', + n.deprecatedAt = $now, + n.deprecatedBy = 'decay', + n.description = CASE + WHEN n.description STARTS WITH '[DEPRECATED]' THEN n.description + WHEN n.description IS NULL OR size(n.description) = 0 THEN '[DEPRECATED]' + ELSE '[DEPRECATED] ' + n.description + END, + n.updatedAt = $now + WITH n + OPTIONAL MATCH (n)-[r]-() + DELETE r + RETURN count(DISTINCT n) AS c + `, { ids, now }); + return toInt(result.records[0]?.get("c") ?? 0); + } finally { + await session.close(); + } +} + +/** + * 两阶段生命周期·阶段二：硬删过期 deprecated 节点（DETACH DELETE，embedding/contentHash + * 向量属性随节点一并移除，释放存储）。时钟基准 deprecatedAt；coalesce 回退 updatedAt 仅是 + * 防御性兜底——initSchema 启动时已为存量 deprecated 节点一次性补写 deprecatedAt（否则 + * upsertNode 对 manual/merge 弃用节点的 updatedAt bump 会无限推迟 purge）。 + * 适用于所有 deprecated 节点（decay/manual/merge）；purgeAfterMs<=0 时为 no-op。返回删除数。 + */ +export async function purgeDeprecatedNodes( + driver: Driver, + purgeAfterMs: number, + now: number = Date.now(), +): Promise { + if (!(purgeAfterMs > 0)) return 0; + const session = getSession(driver); + try { + const result = await session.run(` + MATCH (n:Task|Skill|Event {status: 'deprecated'}) + WHERE coalesce(n.deprecatedAt, n.updatedAt) < $cutoff + DETACH DELETE n + RETURN count(n) AS c + `, { cutoff: now - purgeAfterMs }); + return toInt(result.records[0]?.get("c") ?? 0); } finally { await session.close(); } @@ -374,9 +514,13 @@ export async function mergeNodes(driver: Driver, keepId: string, mergeId: string DELETE r `, { keepId }); - // 标记 deprecated + // 标记 deprecated（deprecatedBy='merge'：不参与 decay 复活；参与 purge 到期硬删） await tx.run( - "MATCH (n:Task|Skill|Event {id: $mergeId}) SET n.status = 'deprecated', n.updatedAt = $now", + `MATCH (n:Task|Skill|Event {id: $mergeId}) + SET n.status = 'deprecated', + n.deprecatedAt = $now, + n.deprecatedBy = 'merge', + n.updatedAt = $now`, { mergeId, now: Date.now() }, ); }); @@ -385,22 +529,6 @@ export async function mergeNodes(driver: Driver, keepId: string, mergeId: string } } -/** 批量更新 PageRank 分数 */ -export async function updatePageranks(driver: Driver, scores: Map): Promise { - if (scores.size === 0) return; - const session = getSession(driver); - try { - const entries = Array.from(scores.entries()).map(([id, score]) => ({ id, score })); - await session.run(` - UNWIND $entries AS entry - MATCH (n:Task|Skill|Event {id: entry.id}) - SET n.pagerank = entry.score - `, { entries }); - } finally { - await session.close(); - } -} - /** 批量更新社区 ID */ export async function updateCommunities(driver: Driver, labels: Map): Promise { if (labels.size === 0) return; @@ -451,42 +579,25 @@ export async function upsertEdge( const toType = endpoints.records[0].get("toType"); if (!isValidEdgeDirection(e.type, fromType, toType)) return false; - // 检查是否已存在同 from+to+type 的边 - const existing = await session.run(` - MATCH (a:Task|Skill|Event {id: $fromId})-[r]->(b:Task|Skill|Event {id: $toId}) - WHERE type(r) = $type - RETURN r - `, { fromId: e.fromId, toId: e.toId, type: e.type }); - - if (existing.records.length > 0) { - await session.run(` - MATCH (a:Task|Skill|Event {id: $fromId})-[r]->(b:Task|Skill|Event {id: $toId}) - WHERE type(r) = $type - SET r.instruction = $instruction - `, { fromId: e.fromId, toId: e.toId, type: e.type, instruction: e.instruction }); - } else { - // 用 APOC 动态创建关系（type 是变量） - await session.run(` - MATCH (a:Task|Skill|Event {id: $fromId}), (b:Task|Skill|Event {id: $toId}) - CALL apoc.create.relationship(a, $type, { - id: $id, - instruction: $instruction, - condition: $condition, - sessionId: $sessionId, - createdAt: $now - }, b) YIELD rel - RETURN rel - `, { - fromId: e.fromId, - toId: e.toId, - type: e.type, - id: uid("e"), - instruction: e.instruction, - condition: e.condition ?? null, - sessionId: e.sessionId, - now: Date.now(), - }); - } + // MERGE 语义：查重 + 创建/更新合并为单条原子语句，消除并发下绕过查重产生重复边的窗口。 + // onCreate 写入全部属性；onMatch 仅刷新 instruction（与原查重-更新分支行为一致）。 + await session.run(` + MATCH (a:Task|Skill|Event {id: $fromId}), (b:Task|Skill|Event {id: $toId}) + CALL apoc.merge.relationship(a, $type, {}, { + id: $id, instruction: $instruction, condition: $condition, + sessionId: $sessionId, createdAt: $now + }, b, { instruction: $instruction }) YIELD rel + RETURN rel + `, { + fromId: e.fromId, + toId: e.toId, + type: e.type, + id: uid("e"), + instruction: e.instruction, + condition: e.condition ?? null, + sessionId: e.sessionId, + now: Date.now(), + }); return true; } finally { await session.close(); @@ -498,21 +609,9 @@ export async function edgesFrom(driver: Driver, id: string): Promise { try { const result = await session.run(` MATCH (a:Task|Skill|Event {id: $id})-[r]->(b:Task|Skill|Event) - WHERE type(r) IN ['USED_SKILL','SOLVED_BY','REQUIRES','PATCHES','CONFLICTS_WITH'] - RETURN r.id AS id, a.id AS fromId, b.id AS toId, type(r) AS type, - r.instruction AS instruction, r.condition AS condition, - r.sessionId AS sessionId, r.createdAt AS createdAt + WHERE ${EDGE_TYPE_FILTER}${EDGE_ROW_RETURN} `, { id }); - return result.records.map(r => ({ - id: r.get("id"), - fromId: r.get("fromId"), - toId: r.get("toId"), - type: r.get("type") as EdgeType, - instruction: r.get("instruction"), - condition: r.get("condition") ?? undefined, - sessionId: r.get("sessionId"), - createdAt: toInt(r.get("createdAt")), - })); + return mapEdgeRecords(result); } finally { await session.close(); } @@ -523,21 +622,25 @@ export async function edgesTo(driver: Driver, id: string): Promise { try { const result = await session.run(` MATCH (a:Task|Skill|Event)-[r]->(b:Task|Skill|Event {id: $id}) - WHERE type(r) IN ['USED_SKILL','SOLVED_BY','REQUIRES','PATCHES','CONFLICTS_WITH'] - RETURN r.id AS id, a.id AS fromId, b.id AS toId, type(r) AS type, - r.instruction AS instruction, r.condition AS condition, - r.sessionId AS sessionId, r.createdAt AS createdAt + WHERE ${EDGE_TYPE_FILTER}${EDGE_ROW_RETURN} `, { id }); - return result.records.map(r => ({ - id: r.get("id"), - fromId: r.get("fromId"), - toId: r.get("toId"), - type: r.get("type") as EdgeType, - instruction: r.get("instruction"), - condition: r.get("condition") ?? undefined, - sessionId: r.get("sessionId"), - createdAt: toInt(r.get("createdAt")), - })); + return mapEdgeRecords(result); + } finally { + await session.close(); + } +} + +/** 批量查询至少一端在 ids 内的知识边 —— 一次往返替代逐节点 edgesFrom+edgesTo 的 2N 次往返。 */ +export async function edgesTouching(driver: Driver, ids: string[]): Promise { + if (!ids.length) return []; + const session = getSession(driver); + try { + const result = await session.run(` + MATCH (a:Task|Skill|Event)-[r]->(b:Task|Skill|Event) + WHERE (a.id IN $ids OR b.id IN $ids) + AND ${EDGE_TYPE_FILTER}${EDGE_ROW_RETURN} + `, { ids }); + return mapEdgeRecords(result); } finally { await session.close(); } @@ -572,6 +675,23 @@ export async function deleteEdges( } } +/** 按 edge id 删除单条边（REST DELETE /edges?id= 用）。返回删除条数（0/1）。 */ +export async function deleteEdgeById(driver: Driver, edgeId: string): Promise { + const session = getSession(driver); + try { + const result = await session.run( + `MATCH ()-[r]->() + WHERE r.id = $edgeId + DELETE r + RETURN count(r) AS deleted`, + { edgeId }, + ); + return toInt(result.records[0]?.get("deleted") ?? 0); + } finally { + await session.close(); + } +} + // ─── 搜索 ─────────────────────────────────────────────────── /** 全文搜索节点（CONTAINS 模糊匹配） */ @@ -623,7 +743,7 @@ export async function topNodes(driver: Driver, limit = 6): Promise { // ─── 向量搜索 ─────────────────────────────────────────────── -export type ScoredNode = { node: GmNode; score: number }; +type ScoredNode = { node: GmNode; score: number }; export async function vectorSearchWithScore( driver: Driver, queryVec: number[], limit: number, minScore = 0.35, @@ -647,15 +767,8 @@ export async function vectorSearchWithScore( } } -export async function vectorSearch( - driver: Driver, queryVec: number[], limit: number, minScore = 0.35, -): Promise { - const scored = await vectorSearchWithScore(driver, queryVec, limit, minScore); - return scored.map(s => s.node); -} - /** 社区向量搜索 */ -export type ScoredCommunity = { id: string; summary: string; score: number; nodeCount: number }; +type ScoredCommunity = { id: string; summary: string; score: number; nodeCount: number }; export async function communityVectorSearch( driver: Driver, queryVec: number[], minScore = 0.15, @@ -683,14 +796,20 @@ export async function communityVectorSearch( // ─── 向量存储 ─────────────────────────────────────────────── -export async function saveVector(driver: Driver, nodeId: string, content: string, vec: number[]): Promise { - const hash = createHash("md5").update(content).digest("hex"); +/** + * 单发写向量 + contentHash。hash 可选：调用方已为短路检查预计算过 md5 时 + * 直接传入，避免对同一 content 重复哈希（缺省时内部补算）。 + */ +export async function saveVector( + driver: Driver, nodeId: string, content: string, vec: number[], hash?: string, +): Promise { + const contentHash = hash ?? createHash("md5").update(content).digest("hex"); const session = getSession(driver); try { await session.run(` MATCH (n:Task|Skill|Event {id: $nodeId}) - SET n.embedding = $vec, n.contentHash = $hash - `, { nodeId, vec, hash }); + SET n.embedding = $vec, n.contentHash = $contentHash + `, { nodeId, vec, contentHash }); } finally { await session.close(); } @@ -709,24 +828,228 @@ export async function getVectorHash(driver: Driver, nodeId: string): Promise> { +/** + * 批量读取 contentHash（syncEmbedBatch 用）：一次往返替代逐节点 getVectorHash 的 N 次。 + * 返回 Map；不存在的节点映射为 null（视为需要嵌入）。 + */ +export async function getVectorHashes(driver: Driver, nodeIds: string[]): Promise> { + const out = new Map(); + if (!nodeIds.length) return out; const session = getSession(driver); try { - const result = await session.run(` - MATCH (n:Task|Skill|Event {status: 'active'}) - WHERE n.embedding IS NOT NULL - RETURN n.id AS nodeId, n.embedding AS embedding + const result = await session.run( + `UNWIND $ids AS id + MATCH (n:Task|Skill|Event {id: id}) + RETURN n.id AS id, n.contentHash AS hash`, + { ids: nodeIds }, + ); + for (const r of result.records) out.set(r.get("id"), r.get("hash") ?? null); + } finally { + await session.close(); + } + for (const id of nodeIds) if (!out.has(id)) out.set(id, null); + return out; +} + +export interface BatchVectorEntry { + nodeId: string; + content: string; + vec: number[]; + hash: string; +} + +/** 批量写向量 + contentHash（syncEmbedBatch 用）：一次 UNWIND 写入替代 N 次 saveVector。 */ +export async function saveVectors(driver: Driver, entries: BatchVectorEntry[]): Promise { + if (!entries.length) return; + const session = getSession(driver); + try { + await session.run( + `UNWIND $entries AS e + MATCH (n:Task|Skill|Event {id: e.nodeId}) + SET n.embedding = e.vec, n.contentHash = e.hash`, + { entries: entries.map(e => ({ nodeId: e.nodeId, vec: e.vec, hash: e.hash })) }, + ); + } finally { + await session.close(); + } +} + +// ─── 重嵌入（换 embedding 模型后的批量重建，graph-memory reembed） ─── + +export interface EmbeddingStats { + nodesTotal: number; + nodesEmbedded: number; + communitiesTotal: number; + communitiesEmbedded: number; +} + +/** 向量覆盖情况统计（reembed 报告 + dry-run 用） */ +export async function getEmbeddingStats(driver: Driver): Promise { + const session = getSession(driver); + try { + const nodeRes = await session.run(` + MATCH (n:MemoryNode {status: 'active'}) + RETURN count(n) AS total, count(n.embedding) AS embedded `); + const commRes = await session.run(` + MATCH (c:Community) + RETURN count(c) AS total, count(c.embedding) AS embedded + `); + const nodeRec = nodeRes.records[0]; + const commRec = commRes.records[0]; + return { + nodesTotal: nodeRec ? toInt(nodeRec.get("total")) : 0, + nodesEmbedded: nodeRec ? toInt(nodeRec.get("embedded")) : 0, + communitiesTotal: commRec ? toInt(commRec.get("total")) : 0, + communitiesEmbedded: commRec ? toInt(commRec.get("embedded")) : 0, + }; + } finally { + await session.close(); + } +} + +/** + * 清空（void）全部向量。contentHash 必须与 embedding 一同清除： + * syncEmbed 以"文本 hash 未变"短路，只清 embedding 会让重嵌入被旧 hash 跳过。 + */ +export async function clearAllEmbeddings( + driver: Driver, +): Promise<{ nodes: number; communities: number }> { + const session = getSession(driver); + try { + const nodeRes = await session.run(` + MATCH (n:MemoryNode) + WHERE n.embedding IS NOT NULL OR n.contentHash IS NOT NULL + REMOVE n.embedding, n.contentHash + RETURN count(n) AS cleared + `); + const commRes = await session.run(` + MATCH (c:Community) + WHERE c.embedding IS NOT NULL + REMOVE c.embedding + RETURN count(c) AS cleared + `); + return { + nodes: nodeRes.records[0] ? toInt(nodeRes.records[0].get("cleared")) : 0, + communities: commRes.records[0] ? toInt(commRes.records[0].get("cleared")) : 0, + }; + } finally { + await session.close(); + } +} + +/** 待重嵌入节点（游标分页 —— 重建过程会让集合收缩，SKIP 分页会跳号） */ +export interface NodeEmbeddingTarget { + id: string; + name: string; + description: string; + content: string; +} + +export async function listNodeEmbeddingTargets( + driver: Driver, + afterId: string, + limit: number, +): Promise { + const session = getSession(driver); + try { + const result = await session.run(` + MATCH (n:MemoryNode {status: 'active'}) + WHERE n.embedding IS NULL AND n.id > $afterId + RETURN n.id AS id, n.name AS name, n.description AS description, n.content AS content + ORDER BY n.id + LIMIT toInteger($limit) + `, { afterId, limit: nint(limit) }); + return result.records.map(r => ({ + id: r.get("id") ?? "", + name: r.get("name") ?? "", + description: r.get("description") ?? "", + content: r.get("content") ?? "", + })); + } finally { + await session.close(); + } +} + +/** 待重嵌入社区（摘要非空才可嵌入） */ +export interface CommunityEmbeddingTarget { + id: string; + summary: string; +} + +export async function listCommunityEmbeddingTargets( + driver: Driver, + afterId: string, + limit: number, +): Promise { + const session = getSession(driver); + try { + const result = await session.run(` + MATCH (c:Community) + WHERE c.embedding IS NULL AND c.summary IS NOT NULL AND trim(c.summary) <> '' + RETURN c.id AS id, c.summary AS summary + ORDER BY c.id + LIMIT toInteger($limit) + `, { afterId, limit: nint(limit) }); return result.records.map(r => ({ - nodeId: r.get("nodeId"), - embedding: r.get("embedding"), + id: r.get("id") ?? "", + summary: r.get("summary") ?? "", })); } finally { await session.close(); } } +export async function saveCommunityEmbedding(driver: Driver, id: string, vec: number[]): Promise { + const session = getSession(driver); + try { + await session.run( + "MATCH (c:Community {id: $id}) SET c.embedding = $vec", + { id, vec }, + ); + } finally { + await session.close(); + } +} + +/** 读取两个向量索引的维度（索引不存在时为 null —— 全新库场景） */ +export async function getVectorIndexDimensions(driver: Driver): Promise> { + const session = getSession(driver); + try { + const result = await session.run(` + SHOW INDEXES YIELD name, options + WHERE name IN ['gm_node_embedding', 'gm_community_embedding'] + RETURN name, options.indexConfig AS indexConfig + `); + const out: Record = { + gm_node_embedding: null, + gm_community_embedding: null, + }; + for (const r of result.records) { + const name = r.get("name"); + const indexConfig = r.get("indexConfig") as any; + const dim = indexConfig?.["vector.dimensions"]; + if (typeof name === "string" && name in out && typeof dim === "number") { + out[name] = dim; + } + } + return out; + } finally { + await session.close(); + } +} + +/** 删除两个向量索引（--recreate-index 重建路径；initSchema 会按配置重新创建） */ +export async function dropVectorIndexes(driver: Driver): Promise { + const session = getSession(driver); + try { + await session.run("DROP INDEX gm_node_embedding IF EXISTS"); + await session.run("DROP INDEX gm_community_embedding IF EXISTS"); + } finally { + await session.close(); + } +} + // ─── 图遍历 ──────────────────────────────────────────────── export async function graphWalk( @@ -736,6 +1059,10 @@ export async function graphWalk( ): Promise<{ nodes: GmNode[]; edges: GmEdge[] }> { if (!seedIds.length) return { nodes: [], edges: [] }; + // maxDepth 来自配置且直接内插进 Cypher —— clamp 到 [1,4]，非法值只会得到安全深度而非语法错误 + const parsedDepth = Number(maxDepth); + const depth = Math.max(1, Math.min(4, Number.isFinite(parsedDepth) ? Math.floor(parsedDepth) : 2)); + const session = getSession(driver); try { // 用 Neo4j 的变长路径匹配做图遍历 @@ -744,7 +1071,7 @@ export async function graphWalk( WHERE seed.id IN $seedIds AND seed.status = 'active' CALL { WITH seed - MATCH path = (seed)-[*0..${maxDepth}]-(neighbor:Task|Skill|Event {status: 'active'}) + MATCH path = (seed)-[*0..${depth}]-(neighbor:Task|Skill|Event {status: 'active'}) WHERE all(node IN nodes(path) WHERE node.status = 'active') RETURN DISTINCT neighbor } @@ -759,22 +1086,10 @@ export async function graphWalk( const edgeResult = await session.run(` MATCH (a:Task|Skill|Event)-[r]->(b:Task|Skill|Event) WHERE a.id IN $nodeIds AND b.id IN $nodeIds - AND type(r) IN ['USED_SKILL','SOLVED_BY','REQUIRES','PATCHES','CONFLICTS_WITH'] - RETURN r.id AS id, a.id AS fromId, b.id AS toId, type(r) AS type, - r.instruction AS instruction, r.condition AS condition, - r.sessionId AS sessionId, r.createdAt AS createdAt + AND ${EDGE_TYPE_FILTER}${EDGE_ROW_RETURN} `, { nodeIds }); - const edges = edgeResult.records.map(r => ({ - id: r.get("id"), - fromId: r.get("fromId"), - toId: r.get("toId"), - type: r.get("type") as EdgeType, - instruction: r.get("instruction"), - condition: r.get("condition") ?? undefined, - sessionId: r.get("sessionId"), - createdAt: toInt(r.get("createdAt")), - })); + const edges = mapEdgeRecords(edgeResult); return { nodes, edges }; } finally { @@ -800,7 +1115,16 @@ export async function getBySession(driver: Driver, sessionId: string): Promise { +/** + * 每社区取最近更新的 perCommunity 个代表节点。 + * totalLimit 封顶总返回数（按社区规模降序截断）—— recall 兜底路径用它做 + * graphWalk 种子，社区很多时无上限种子会把遍历放大成全图扫描。 + */ +export async function communityRepresentatives( + driver: Driver, + perCommunity = 2, + totalLimit = 20, +): Promise { const session = getSession(driver); try { const result = await session.run(` @@ -809,9 +1133,11 @@ export async function communityRepresentatives(driver: Driver, perCommunity = 2) WITH n.communityId AS cid, n ORDER BY n.updatedAt DESC WITH cid, collect(n) AS members + ORDER BY size(members) DESC UNWIND members[0..toInteger($perCommunity)] AS m RETURN m AS n - `, { perCommunity }); + LIMIT toInteger($totalLimit) + `, { perCommunity, totalLimit }); return result.records.map(r => toNode(r.get("n"))); } finally { await session.close(); @@ -852,6 +1178,9 @@ export async function saveMessage( m.content = $content, m.extracted = false, m.createdAt = $now + ON MATCH SET + m.role = $role, + m.content = $content `, { id: uid("m"), sid, @@ -865,6 +1194,22 @@ export async function saveMessage( } } +/** 该会话当前最大 turnIndex（无消息返回 0）。用于插件重启后恢复内存 msgSeq； + * 否则 turnIndex 从 1 重计 → MERGE 命中旧行 → ON CREATE 被跳过 → 新消息被静默丢弃。 */ +export async function getMaxTurnIndex(driver: Driver, sid: string): Promise { + const session = getSession(driver); + try { + const result = await session.run( + `MATCH (m:GmMessage {sessionId: $sid}) + RETURN coalesce(max(m.turnIndex), 0) AS maxTurn`, + { sid }, + ); + return toInt(result.records[0].get("maxTurn")); + } finally { + await session.close(); + } +} + export async function getUnextracted(driver: Driver, sid: string, limit: number): Promise { const session = getSession(driver); try { @@ -879,7 +1224,6 @@ export async function getUnextracted(driver: Driver, sid: string, limit: number) return { role: m.role, content: JSON.parse(m.content), - turnIndex: toInt(m.turnIndex), turn_index: toInt(m.turnIndex), }; }); @@ -888,14 +1232,74 @@ export async function getUnextracted(driver: Driver, sid: string, limit: number) } } -export async function markExtracted(driver: Driver, sid: string, upToTurn: number): Promise { +/** 未提取消息计数（batched 提取模式的攒批触发判定）。 */ +export async function countUnextracted(driver: Driver, sid: string): Promise { + const session = getSession(driver); + try { + const result = await session.run( + "MATCH (m:GmMessage {sessionId: $sid, extracted: false}) RETURN count(m) AS c", + { sid }, + ); + return toInt(result.records[0].get("c")); + } finally { + await session.close(); + } +} + +export interface UnextractedSessionInfo { + sessionId: string; + messageCount: number; + maxTurn: number; + minCreatedAt: number; +} + +export async function listUnextractedSessions(driver: Driver): Promise { + const session = getSession(driver); + try { + const result = await session.run(` + MATCH (m:GmMessage {extracted: false}) + WITH m.sessionId AS sid, + count(*) AS msgCount, + max(m.turnIndex) AS maxTurn, + min(coalesce(m.createdAt, 0)) AS minCreated + WHERE sid IS NOT NULL + RETURN sid, msgCount, maxTurn, minCreated + ORDER BY minCreated ASC, sid ASC + `); + return result.records.map(r => ({ + sessionId: r.get("sid"), + messageCount: toInt(r.get("msgCount")), + maxTurn: toInt(r.get("maxTurn")), + minCreatedAt: toInt(r.get("minCreated")), + })); + } finally { + await session.close(); + } +} + +/** + * 标记 sid 会话内 turnIndex ≤ upToTurn 的消息为已提取。 + * + * producedKnowledge：该次提取是否实际产出节点/边。LLM 成功返回零节点零边时 + * 传 false —— 原始证据保留（retention 只删 producedKnowledge=true 的行）。 + * 注意空提取轮次不会自动重提：重挖需手动将其 extracted 重置为 false 再跑 + * `openclaw graph-memory extract`。 + * + * 只作用于尚未提取的行（extracted=false）：producedKnowledge 在提取转换时刻 + * 一次性写入，不会被后续更大范围的批量标记覆盖（compact 的范围标记可能横跨 + * 早期已标记的轮次）。遗留行（无 producedKnowledge 属性）保持 null， + * retention 对其 fail-closed 不删。 + */ +export async function markExtracted( + driver: Driver, sid: string, upToTurn: number, producedKnowledge = true, +): Promise { const session = getSession(driver); try { await session.run(` MATCH (m:GmMessage {sessionId: $sid}) - WHERE m.turnIndex <= $upToTurn - SET m.extracted = true - `, { sid, upToTurn }); + WHERE m.turnIndex <= $upToTurn AND m.extracted = false + SET m.extracted = true, m.producedKnowledge = $pk + `, { sid, upToTurn, pk: producedKnowledge }); } finally { await session.close(); } @@ -915,6 +1319,39 @@ export async function isTurnExtracted(driver: Driver, sid: string, turn: number) } } +// ─── 轮次提交标记（OpenClaw transcript fencing 契约） ───────── + +/** neo4j-driver 约束冲突错误的 code 形如 Neo.ClientError.Schema.ConstraintValidationFailed */ +function isConstraintViolation(err: unknown): boolean { + const code = (err as { code?: unknown } | null | undefined)?.code; + if (typeof code === "string" && code.includes("ConstraintValidation")) return true; + return String(err).includes("ConstraintValidation"); +} + +/** + * 幂等提交一个 logical turn（host 只对成功接受的轮次调用，重试携带同一 advancementKey）。 + * 唯一约束 + CREATE 保证原子幂等：首次 → "committed"；重试撞约束 → "duplicate"。 + * 标记节点本身即"一次原子幂等写"的全部载荷 —— 消息已由 ingest/afterTurn 逐条落库， + * 不在标记里重复存（否则与 GmMessage 行双写、outage 缓冲路径无法对齐）。 + */ +export async function commitTurnAdvance( + driver: Driver, sid: string, advancementKey: string, messageCount: number, +): Promise<"committed" | "duplicate"> { + const session = getSession(driver); + try { + await session.run( + `CREATE (t:GmTurnCommit {advancementKey: $key, sessionId: $sid, messageCount: $count, createdAt: $now})`, + { key: advancementKey, sid, count: messageCount, now: Date.now() }, + ); + return "committed"; + } catch (err) { + if (isConstraintViolation(err)) return "duplicate"; + throw err; + } finally { + await session.close(); + } +} + // ─── 统计 ──────────────────────────────────────────────────── export async function getStats(driver: Driver): Promise<{ @@ -942,7 +1379,7 @@ export async function getStats(driver: Driver): Promise<{ const edgeResult = await session.run(` MATCH ()-[r]->() - WHERE type(r) IN ['USED_SKILL','SOLVED_BY','REQUIRES','PATCHES','CONFLICTS_WITH'] + WHERE ${EDGE_TYPE_FILTER} RETURN type(r) AS type, count(r) AS c `); let totalEdges = 0; @@ -972,12 +1409,15 @@ export interface CommunitySummary { id: string; summary: string; nodeCount: number; + /** 成员 ID 排序后的 sha1 — 用于识别"成员构成未变"的社区（复用摘要） */ + memberSignature: string | null; createdAt: number; updatedAt: number; } export async function upsertCommunitySummary( - driver: Driver, id: string, summary: string, nodeCount: number, embedding?: number[], + driver: Driver, id: string, summary: string, nodeCount: number, + embedding?: number[], memberSignature?: string, ): Promise { const session = getSession(driver); try { @@ -987,18 +1427,21 @@ export async function upsertCommunitySummary( c.summary = $summary, c.nodeCount = $nodeCount, c.embedding = $embedding, + c.memberSignature = $memberSignature, c.createdAt = $now, c.updatedAt = $now ON MATCH SET c.summary = $summary, c.nodeCount = $nodeCount, c.embedding = CASE WHEN $embedding IS NOT NULL THEN $embedding ELSE c.embedding END, + c.memberSignature = CASE WHEN $memberSignature IS NOT NULL THEN $memberSignature ELSE c.memberSignature END, c.updatedAt = $now `, { id, summary, nodeCount, embedding: embedding ?? null, + memberSignature: memberSignature ?? null, now: Date.now(), }); } finally { @@ -1019,6 +1462,7 @@ export async function getCommunitySummary(driver: Driver, id: string): Promise { +export async function getCommunitySummaryBySignature( + driver: Driver, memberSignature: string, +): Promise<(CommunitySummary & { embedding?: number[] }) | null> { const session = getSession(driver); try { const result = await session.run( - "MATCH (c:Community) RETURN c ORDER BY c.nodeCount DESC" + "MATCH (c:Community {memberSignature: $memberSignature}) RETURN c ORDER BY c.updatedAt DESC LIMIT 1", + { memberSignature }, ); - return result.records.map(r => { - const c = r.get("c").properties; - return { - id: c.id, - summary: c.summary, - nodeCount: toInt(c.nodeCount), - createdAt: toInt(c.createdAt), - updatedAt: toInt(c.updatedAt), - }; - }); + if (result.records.length === 0) return null; + const c = result.records[0].get("c").properties; + return { + id: c.id, + summary: c.summary, + nodeCount: toInt(c.nodeCount), + memberSignature: c.memberSignature ?? null, + embedding: Array.isArray(c.embedding) ? (c.embedding as number[]) : undefined, + createdAt: toInt(c.createdAt), + updatedAt: toInt(c.updatedAt), + }; } finally { await session.close(); } diff --git a/src/tokens.ts b/src/tokens.ts new file mode 100644 index 0000000..2938d9b --- /dev/null +++ b/src/tokens.ts @@ -0,0 +1,13 @@ +/** + * graph-memory-pro — token 估算 + * + * 统一的字符→token 粗估换算（约 3 字符 = 1 token，中英混合文本的经验值）。 + * 全仓库所有 token 估算必须经由本模块，避免系数多处漂移。 + */ + +export const CHARS_PER_TOKEN = 3; + +/** 按字符数粗估 token 数（向上取整） */ +export function estimateTokens(chars: number): number { + return Math.ceil(chars / CHARS_PER_TOKEN); +} diff --git a/src/types.ts b/src/types.ts index 4ae637d..c2d93df 100755 --- a/src/types.ts +++ b/src/types.ts @@ -10,6 +10,25 @@ export type NodeType = "TASK" | "SKILL" | "EVENT"; export type NodeStatus = "active" | "deprecated"; +/** + * 记忆分层 tier（与 NodeStatus 正交）。 + * decay 评分模型据此双向转换：core↔working↔peripheral。 + * 节点通常保持 status=active，仅 tier 变化；status=deprecated 由手动弃用、 + * merge 或 decay 自动弃用（autoDeprecate，见 DecayConfig）触发。 + */ +export type NodeTier = "core" | "working" | "peripheral"; + +/** + * 弃用来源标记（节点属性 deprecatedBy）： + * - decay：遗忘曲线自动弃用（tier=peripheral + 低 composite + 超期未访问）； + * 重新提取/编辑命中时可自动复活回 active。 + * - manual：gm_update mode=deprecate / finalize invalidations / REST DELETE； + * 人工语义判定，不自动复活。 + * - merge：dedup 或手动合并的败者节点，不自动复活。 + * 缺省（存量数据）按 manual 处理——不复活；硬删判定只看 deprecatedAt/updatedAt。 + */ +export type DeprecatedBy = "decay" | "manual" | "merge"; + /** Neo4j label 映射：TASK->Task, SKILL->Skill, EVENT->Event */ export const NODE_TYPE_TO_LABEL: Record = { TASK: "Task", @@ -24,12 +43,30 @@ export interface GmNode { description: string; content: string; status: NodeStatus; + /** 与 NodeStatus 正交的衰减分层；旧节点/新节点缺省时按 working 处理。 */ + tier?: NodeTier; validatedCount: number; sourceSessions: string[]; communityId: string | null; pagerank: number; createdAt: number; updatedAt: number; + /** + * 最近一次"相关性活动"时间戳（epoch ms），由 upsertNode 在任意写入路径刷新 + * （重新提取、gm_record、gm_update、CRUD POST）。是衰减判定的基准。 + * 与 updatedAt 的区别：updatedAt 在 deprecate/merge 时也会变，不能代表相关性； + * 而 mergeNodes 故意不更新 lastAccessedAt（合并 ≠ 用户重新激活）。 + * 缺省时回退到 updatedAt / createdAt。 + */ + lastAccessedAt?: number; + /** 最近一次 decay 评分（0~1，越大越鲜活/重要）。仅 applyDecay 写入。 */ + decayScore?: number; + /** decayScore 的计算时间戳（epoch ms）。 */ + decayComputedAt?: number; + /** 被标记 deprecated 的时刻（epoch ms）。硬删倒计时（purgeAfterDays）的基准。 */ + deprecatedAt?: number; + /** 弃用来源（见 DeprecatedBy）；缺省按 manual 处理。 */ + deprecatedBy?: DeprecatedBy; } // ─── 边 ─────────────────────────────────────────────────────── @@ -44,7 +81,7 @@ export const EDGE_TYPES = [ export type EdgeType = (typeof EDGE_TYPES)[number]; -const EDGE_DIRECTION_RULES: Record = { @@ -117,7 +154,6 @@ export interface FinalizeResult { export interface RecallResult { nodes: GmNode[]; edges: GmEdge[]; - tokenEstimate: number; } // ─── Embedding 配置 ────────────────────────────────────────── @@ -137,6 +173,120 @@ export interface Neo4jConfig { password: string; } +// ─── 衰减（柔性评分模型）配置 ───────────────────────────────── +// +// 完整公式、字段映射、默认值来源、调参指南见 docs/decay.md。 +// 评分和 tier 转换逻辑实现在 src/graph/decay.ts。 + +export interface DecayConfig { + enabled: boolean; + recencyHalfLifeDays: number; + recencyWeight: number; + importanceModulation: number; + frequencyWeight: number; + intrinsicWeight: number; + betaCore: number; + betaWorking: number; + betaPeripheral: number; + coreAccessThreshold: number; + coreCompositeThreshold: number; + coreImportanceThreshold: number; + peripheralCompositeThreshold: number; + peripheralAgeDays: number; + workingAccessThreshold: number; + workingCompositeThreshold: number; + /** + * 遗忘曲线自动弃用开关（两阶段生命周期的第一阶段）： + * tier=peripheral 且 composite < peripheralCompositeThreshold 且 + * lastAccessedAt 距今 ≥ autoDeprecateAfterDays 的节点在维护时被断联 + deprecated。 + * 受 enabled 总开关约束（enabled=false 时整体跳过）。 + */ + autoDeprecate: boolean; + /** 自动弃用的未访问天数门槛（距 lastAccessedAt）。 */ + autoDeprecateAfterDays: number; + /** + * 两阶段生命周期的第二阶段：deprecated 节点自 deprecatedAt（缺省回退 updatedAt） + * 起超过该天数后硬删（DETACH DELETE，向量随节点一并移除）。 + * 0 = 永不硬删。适用于所有 deprecated 节点（含 manual/merge）。 + */ + purgeAfterDays: number; +} + +// ─── cron 会话（定时任务）的图谱行为配置 ───────────────────── + +/** + * 判断是否为 cron 定时会话。host 把 cron 标记放在 sessionKey 上（sessionId 是随机 UUID）， + * 实际形状：cron: / agent::cron: / agent::cron::run:。 + * 按段匹配（split(":") 后包含 "cron"），避免误匹配 "cron-daily" 这类自定义段。 + * 注意：cron 任务若显式设置了自定义 sessionKey，host 不再附加 cron 段，此类会话无法识别（见 README）。 + * cron session 的图谱行为（召回/消息入库、知识提取、结束维护）可由 `cron` 配置独立开关；非 cron session 不受影响。 + */ +export function isCronSessionKey(sessionKey: string | undefined | null): boolean { + return typeof sessionKey === "string" && sessionKey.split(":").includes("cron"); +} + +export interface CronConfig { + enabled: boolean; + extract: boolean; + finalizeAndMaintain: boolean; +} + +export const DEFAULT_CRON_CONFIG: CronConfig = { + enabled: true, + extract: true, + finalizeAndMaintain: true, +}; + +// ─── 原始消息保留（opt-in 有界清理）────────────────────────── +// +// 设计原则（移植自上游 #96）："上下文压缩改变的是模型可见面， +// 不构成删除持久证据的授权。" 默认 keep=all 零行为变化； +// 清理逻辑实现在 src/store/retention.ts，挂在 runMaintenance 尾部。 + +export type MessageRetentionMode = "all" | "referenced" | "recent"; + +export interface MessageRetentionConfig { + /** + * all（默认）保留全部原始消息，零开销； + * referenced 只删"已提取完成且实际产出知识"的消息（producedKnowledge=true， + * 知识已固化进图谱，原始文本退役；LLM 空提取的轮次保留原始证据， + * 重挖需手动重置 extracted 后跑 graph-memory extract）； + * recent 在 referenced 的基础上按时间窗保护最近内容。 + */ + keep?: MessageRetentionMode; + /** keep=recent：每个 session 保留最近 N 轮真实用户发言（该轮及其后全部保留）。 */ + recentTurns?: number; + /** keep=recent：保留最近 N 天内入库的消息。 */ + retentionDays?: number; + /** 单个维护周期最多处理的行数（保证维护链工作有界）。默认 500。 */ + batchSize?: number; + /** true 时只报告候选集不删除 —— 启用前先跑一轮 dryRun 验证。 */ + dryRun?: boolean; +} + +// ─── 知识提取配置（LLM 成本控制）───────────────────────────── + +export interface ExtractConfig { + /** + * 提取模式： + * - per-turn（默认）：每轮 afterTurn/commitTurn 即时 LLM 提取（知识实时入库）； + * - batched：攒批提取——trivial 轮本地标记跳过，未提取消息累计到 + * compactTurnCount*3 条时批量提取一次，session_end 冲洗尾批。 + * LLM 调用次数降为 per-turn 的 ~1/N，代价是会话中途召回不到本会话最新知识。 + */ + mode?: "per-turn" | "batched"; + /** + * 本地预筛阈值：用户输入清洗（去空白/标点）后长度 ≤ 该值且不含技术词 + * （连续 ≥3 位字母数字，如 pnpm/jwt）时，跳过 LLM 提取直接标记。 + * 保守默认 5（中文 5 字以内基本不可能承载可提取知识）。 + * 例外：轮内含工具劳动（tool/toolResult 角色）时不判 trivial——"继续"触发的 + * 一轮真实修复劳动恰是可提取知识，不误杀（见 turn-filter.ts 的 turnHasToolWork）。 + */ + trivialMaxChars?: number; + /** 额外无意义词表（与内置表合并，清洗后小写精确匹配，如 "继续"、"resume"）。 */ + trivialPrompts?: string[]; +} + // ─── 插件配置 ───────────────────────────────────────────────── export interface GmConfig { @@ -144,6 +294,7 @@ export interface GmConfig { compactTurnCount: number; recallMaxNodes: number; recallMaxDepth: number; + /** assemble 保留的最近轮数（裁剪窗口）；默认 5，与 sliceLastTurn 的回退值一致。 */ freshTailCount: number; embedding?: EmbeddingConfig; llm?: { @@ -163,6 +314,13 @@ export interface GmConfig { dedupThreshold: number; pagerankDamping: number; pagerankIterations: number; + /** 遗忘曲线衰减配置；未提供时使用 DEFAULT_CONFIG.decay。 */ + decay?: DecayConfig; + /** 知识提取配置（模式 + trivial 预筛）；未提供时 per-turn + 默认预筛。 */ + extract?: ExtractConfig; + cron?: CronConfig; + /** 原始消息保留策略；未提供时等价 keep=all（永不删除）。 */ + messageRetention?: MessageRetentionConfig; } export const DEFAULT_CONFIG: GmConfig = { @@ -174,8 +332,34 @@ export const DEFAULT_CONFIG: GmConfig = { compactTurnCount: 6, recallMaxNodes: 6, recallMaxDepth: 2, - freshTailCount: 10, + freshTailCount: 5, dedupThreshold: 0.90, pagerankDamping: 0.85, pagerankIterations: 20, + decay: { + enabled: true, + recencyHalfLifeDays: 30, + recencyWeight: 0.4, + importanceModulation: 1.5, + frequencyWeight: 0.3, + intrinsicWeight: 0.3, + betaCore: 0.8, + betaWorking: 1.0, + betaPeripheral: 1.3, + coreAccessThreshold: 10, + coreCompositeThreshold: 0.7, + coreImportanceThreshold: 0.8, + peripheralCompositeThreshold: 0.15, + peripheralAgeDays: 60, + workingAccessThreshold: 3, + workingCompositeThreshold: 0.4, + autoDeprecate: true, + autoDeprecateAfterDays: 30, + purgeAfterDays: 60, + }, + cron: DEFAULT_CRON_CONFIG, + extract: { + mode: "per-turn", + trivialMaxChars: 5, + }, }; diff --git a/test/ablation.study.test.ts b/test/ablation.study.test.ts new file mode 100644 index 0000000..71d351b --- /dev/null +++ b/test/ablation.study.test.ts @@ -0,0 +1,828 @@ +/** + * graph-memory-pro — 消融实验 (ablation study) + * + * 三组消融，量化各组件对系统质量的贡献： + * A. 召回管线：向量种子 / GDS PPR 排序 / 社区扩展 / 双路径（precise+generalized） + * B. 向量去重：dedup 对图规模收缩与召回多样性的贡献 + * C. 遗忘曲线：decay + autoDeprecate 对陈旧知识的遗忘（含 tier 转换） + * + * 安全门（生产库保护，最高优先级）： + * - 仅当 ABLATION_STUDY=1 时启用；普通 npm test / CI 永远跳过本文件 + * - NEO4J_TEST_URI 必须显式设置，否则整体跳过 —— 绝不回落 7687 默认值 + * + * 运行（一次性 Docker Neo4j，bolt 映射到 7688）： + * ABLATION_STUDY=1 NEO4J_TEST_URI=bolt://localhost:7688 \ + * npx vitest run test/ablation.study.test.ts + * + * 全程使用确定性 mock embedder（bag-of-words 符号哈希 → 1024 维 L2 归一化向量）， + * 不调用任何外部 LLM / embedding API。 + */ + +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import type { Driver } from "neo4j-driver"; +import { createHash } from "crypto"; +import { writeFile, mkdir } from "fs/promises"; +import { fileURLToPath } from "url"; +import path from "node:path"; +import { getDriver, initSchema, closeDriver, getSession } from "../src/store/db.ts"; +import { + upsertNode, upsertEdge, saveVector, saveCommunityEmbedding, + searchNodes, vectorSearchWithScore, graphWalk, + communityRepresentatives, communityVectorSearch, nodesByCommunityIds, +} from "../src/store/store.ts"; +import { + detectCommunities, getCommunityPeers, buildCommunityMemberSignature, +} from "../src/graph/community.ts"; +import { personalizedPageRank, computeGlobalPageRank } from "../src/graph/pagerank.ts"; +import { dedup, detectDuplicates } from "../src/graph/dedup.ts"; +import { applyDecay } from "../src/graph/decay.ts"; +import { Recaller, buildNodeEmbeddingText } from "../src/recaller/recall.ts"; +import { DEFAULT_CONFIG } from "../src/types.ts"; +import type { GmConfig, GmNode, RecallResult, EdgeType, NodeType } from "../src/types.ts"; +import type { EmbedFn } from "../src/engine/embed.ts"; + +// ─── 安全门 ───────────────────────────────────────────────── + +const ENABLED = !!process.env.ABLATION_STUDY && !!process.env.NEO4J_TEST_URI; +const NEO4J_URI = process.env.NEO4J_TEST_URI ?? ""; + +const CFG: GmConfig = { + ...DEFAULT_CONFIG, + neo4j: { uri: NEO4J_URI, user: "neo4j", password: "graphmemory" }, + recallMaxNodes: 6, + recallMaxDepth: 2, +}; +const K = CFG.recallMaxNodes; + +// ─── 确定性 embedder（bag-of-words 符号哈希）───────────────── + +const DIM = 1024; + +function embedText(text: string): number[] { + const vec = new Array(DIM).fill(0); + const words = text.toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length > 1); + for (const w of words) { + const digest = createHash("md5").update(w).digest(); + const code = digest.readUInt32BE(0); + const idx = code % DIM; + const sign = (digest[4] & 1) === 0 ? 1 : -1; + vec[idx] += sign; + } + let norm = 0; + for (const v of vec) norm += v * v; + norm = Math.sqrt(norm) || 1; + return vec.map((v) => v / norm); +} + +const mockEmbedFn: EmbedFn = async (text) => embedText(text); + +function meanVector(texts: string[]): number[] { + const acc = new Array(DIM).fill(0); + for (const t of texts) { + const v = embedText(t); + for (let i = 0; i < DIM; i++) acc[i] += v[i]; + } + let norm = 0; + for (const v of acc) norm += v * v; + norm = Math.sqrt(norm) || 1; + return acc.map((v) => v / norm); +} + +// ─── 指标 ─────────────────────────────────────────────────── + +interface Metrics { + recallAtK: number; + precisionAtK: number; + mrr: number; + nodesReturned: number; +} + +function scoreResult(result: RecallResult, relevant: Set): Metrics { + const retrieved = result.nodes.slice(0, K).map((n) => n.id); + const hits = retrieved.filter((id) => relevant.has(id)); + const firstHit = retrieved.findIndex((id) => relevant.has(id)); + return { + recallAtK: relevant.size ? hits.length / relevant.size : 0, + precisionAtK: retrieved.length ? hits.length / retrieved.length : 0, + mrr: firstHit >= 0 ? 1 / (firstHit + 1) : 0, + nodesReturned: result.nodes.length, + }; +} + +interface Row { + suite: string; + config: string; + recallAtK: string; + precisionAtK: string; + mrr: string; + nodes: string; + latencyMs: string; + note: string; +} + +const ROWS: Row[] = []; + +function addRow(suite: string, config: string, m: Metrics, latencyMs: number, note = ""): void { + ROWS.push({ + suite, + config, + recallAtK: m.recallAtK.toFixed(3), + precisionAtK: m.precisionAtK.toFixed(3), + mrr: m.mrr.toFixed(3), + nodes: `${m.nodesReturned}`, + latencyMs: latencyMs.toFixed(0), + note, + }); +} + +function averageMetrics(list: Metrics[]): Metrics { + const n = list.length || 1; + return { + recallAtK: list.reduce((s, m) => s + m.recallAtK, 0) / n, + precisionAtK: list.reduce((s, m) => s + m.precisionAtK, 0) / n, + mrr: list.reduce((s, m) => s + m.mrr, 0) / n, + nodesReturned: list.reduce((s, m) => s + m.nodesReturned, 0) / n, + }; +} + +// ─── 召回管线 harness（逐行镜像 src/recaller/recall.ts，组件可开关）─── + +interface Flags { + vector: boolean; + communityExpansion: boolean; + ppr: boolean; + precise: boolean; + generalized: boolean; +} + +const FULL: Flags = { + vector: true, communityExpansion: true, ppr: true, precise: true, generalized: true, +}; + +let driver: Driver; + +async function pipelinePrecise( + query: string, limit: number, embedPromise: Promise, flags: Flags, +): Promise { + let seeds: GmNode[] = []; + + const vec = await embedPromise; + if (vec) { + try { + const scored = await vectorSearchWithScore(driver, vec, Math.ceil(limit / 2)); + seeds = scored.map((s) => s.node); + if (seeds.length < 2) { + const fts = await searchNodes(driver, query, limit); + const seen = new Set(seeds.map((n) => n.id)); + seeds.push(...fts.filter((n) => !seen.has(n.id))); + } + } catch { + seeds = await searchNodes(driver, query, limit); + } + } else { + seeds = await searchNodes(driver, query, limit); + } + + if (!seeds.length) return { nodes: [], edges: [] }; + const seedIds = seeds.map((n) => n.id); + + const expandedIds = new Set(seedIds); + if (flags.communityExpansion) { + for (const seed of seeds) { + const peers = await getCommunityPeers(driver, seed.id, 2); + for (const peerId of peers) expandedIds.add(peerId); + } + } + + const { nodes, edges } = await graphWalk(driver, Array.from(expandedIds), CFG.recallMaxDepth); + if (!nodes.length) return { nodes: [], edges: [] }; + + const candidateIds = nodes.map((n) => n.id); + const { scores } = flags.ppr + ? await personalizedPageRank(driver, seedIds, candidateIds, CFG) + : { scores: new Map() }; + + const filtered = nodes + .sort((a, b) => + (scores.get(b.id) || 0) - (scores.get(a.id) || 0) || + b.validatedCount - a.validatedCount || + b.updatedAt - a.updatedAt) + .slice(0, limit); + + const ids = new Set(filtered.map((n) => n.id)); + return { nodes: filtered, edges: edges.filter((e) => ids.has(e.fromId) && ids.has(e.toId)) }; +} + +async function pipelineGeneralized( + limit: number, embedPromise: Promise, +): Promise { + let seeds: GmNode[] = []; + + const vec = await embedPromise; + if (vec) { + try { + const scoredCommunities = await communityVectorSearch(driver, vec); + if (scoredCommunities.length > 0) { + seeds = await nodesByCommunityIds(driver, scoredCommunities.map((c) => c.id), 3); + } + } catch { /* 与 recall.ts 一致：静默落 representatives 兜底 */ } + } + + if (!seeds.length) seeds = await communityRepresentatives(driver, 2); + if (!seeds.length) return { nodes: [], edges: [] }; + + const seedIds = seeds.map((n) => n.id); + const { nodes, edges } = await graphWalk(driver, seedIds, 1); + if (!nodes.length) return { nodes: [], edges: [] }; + + const candidateIds = nodes.map((n) => n.id); + const { scores } = await personalizedPageRank(driver, seedIds, candidateIds, CFG); + + const filtered = nodes + .sort((a, b) => + (scores.get(b.id) || 0) - (scores.get(a.id) || 0) || + b.updatedAt - a.updatedAt || + b.validatedCount - a.validatedCount) + .slice(0, limit); + + const ids = new Set(filtered.map((n) => n.id)); + return { nodes: filtered, edges: edges.filter((e) => ids.has(e.fromId) && ids.has(e.toId)) }; +} + +function mergeResults(a: RecallResult, b: RecallResult): RecallResult { + const nodeMap = new Map(); + const edgeMap = new Map(); + for (const n of a.nodes) nodeMap.set(n.id, n); + for (const e of a.edges) edgeMap.set(e.id, e); + for (const n of b.nodes) if (!nodeMap.has(n.id)) nodeMap.set(n.id, n); + const finalIds = new Set(nodeMap.keys()); + for (const e of b.edges) { + if (!edgeMap.has(e.id) && finalIds.has(e.fromId) && finalIds.has(e.toId)) edgeMap.set(e.id, e); + } + return { nodes: Array.from(nodeMap.values()), edges: Array.from(edgeMap.values()) }; +} + +async function recallPipeline(query: string, flags: Flags): Promise { + const limit = CFG.recallMaxNodes; + const embedPromise: Promise = flags.vector + ? mockEmbedFn(query).catch(() => null) + : Promise.resolve(null); + + const paths: Array> = []; + if (flags.precise) paths.push(pipelinePrecise(query, limit, embedPromise, flags)); + if (flags.generalized) paths.push(pipelineGeneralized(limit, embedPromise)); + + const results = await Promise.all(paths); + let merged: RecallResult = { nodes: [], edges: [] }; + for (const r of results) merged = mergeResults(merged, r); + return merged; +} + +// ─── 图 fixture ───────────────────────────────────────────── + +const RUN = Date.now().toString(36); +const SID_RECALL = `abl-${RUN}`; +const SID_DUP_A = `abl-dupA-${RUN}`; // 去重启用臂 +const SID_DUP_B = `abl-dupB-${RUN}`; // 去重消融臂 +const SID_DECAY = `abl-dec-${RUN}`; +const CLEAN_PREFIX = "abl-"; + +interface FixtureNode { + key: string; + type: NodeType; + name: string; + description: string; + content: string; + topic: string; + validatedCount?: number; +} + +interface FixtureEdge { from: string; to: string; type: EdgeType; instruction: string } + +const CLUSTERS: Array<{ topic: string; query: string; nodes: FixtureNode[] }> = [ + { + topic: "docker-deployment", + query: "deploy docker compose container stack registry", + nodes: [ + { key: "dep-task", type: "TASK", name: "abl-docker-stack-deployment", description: "docker compose container deployment orchestration", content: "deploy the multi-container stack with docker compose, pulling service images from the registry, orchestrating rollout and scaling for production workloads", topic: "docker-deployment", validatedCount: 3 }, + { key: "dep-compose", type: "SKILL", name: "abl-compose-service-orchestration", description: "docker compose service orchestration patterns", content: "compose files declare docker services, networks and volumes; compose up orchestrates container startup order, health gates and rolling image replacement for the deployment stack", topic: "docker-deployment", validatedCount: 7 }, + { key: "dep-registry", type: "SKILL", name: "abl-container-registry-lifecycle", description: "container image registry lifecycle", content: "version and push container images to the registry, scan layers, sign tags, promote build artifacts through staging channels before docker deployment", topic: "docker-deployment", validatedCount: 5 }, + { key: "dep-outage", type: "EVENT", name: "abl-registry-outage-postmortem", description: "registry outage during deployment", content: "the docker deployment stalled when the registry rate limited image pulls; container startup backoff cascaded until the compose stack pinned digest references", topic: "docker-deployment", validatedCount: 1 }, + ], + }, + { + topic: "vitest-testing", + query: "vitest unit test mock spy coverage", + nodes: [ + { key: "test-task", type: "TASK", name: "abl-unit-test-suite-setup", description: "vitest unit test suite setup", content: "scaffold a vitest workspace, configure unit test discovery, assertion styles and coverage collection for the repository test suite", topic: "vitest-testing", validatedCount: 2 }, + { key: "test-mocking", type: "SKILL", name: "abl-vitest-mocking-patterns", description: "vitest mocking and spy patterns", content: "vi.mock hoists module mocks, vi.spyOn wraps methods, mockReturnValue stubs responses, mock timers control asynchronous test flow in vitest", topic: "vitest-testing", validatedCount: 8 }, + { key: "test-coverage", type: "SKILL", name: "abl-coverage-threshold-gates", description: "coverage threshold gates", content: "v8 coverage provider maps uncovered branches, istanbul thresholds fail the test pipeline below eighty percent, coverage exclude lists keep fixtures out of the report", topic: "vitest-testing", validatedCount: 4 }, + { key: "test-flaky", type: "EVENT", name: "abl-flaky-spec-retrospective", description: "flaky spec retrospective", content: "a unit spec flickered until the spy leaked between tests; resetting mocks in beforeEach and stubbing timers made the vitest suite deterministic", topic: "vitest-testing", validatedCount: 1 }, + ], + }, + { + topic: "neo4j-schema", + query: "neo4j cypher index constraint migration", + nodes: [ + { key: "neo-task", type: "TASK", name: "abl-graph-schema-migration", description: "graph schema migration planning", content: "plan the schema migration in ordered cypher batches: create indexes and constraints first, backfill properties, then verify node counts before cutover", topic: "neo4j-schema", validatedCount: 3 }, + { key: "neo-cypher", type: "SKILL", name: "abl-cypher-query-optimization", description: "cypher query optimization", content: "profile query plans, replace cartesian products with anchored matches, hint range index scans, and batch apoc.periodic.iterate updates to keep the database responsive", topic: "neo4j-schema", validatedCount: 6 }, + { key: "neo-schema", type: "SKILL", name: "abl-neo4j-constraint-design", description: "neo4j constraint and index design", content: "uniqueness constraints guard identity keys, range indexes accelerate equality filters, fulltext indexes back keyword search, and constraint violations abort the migration transaction", topic: "neo4j-schema", validatedCount: 5 }, + { key: "neo-incident", type: "EVENT", name: "abl-slow-query-incident", description: "slow query incident review", content: "dashboard latency spiked when a migration dropped a covering index mid-deploy; cypher queries degraded to filters until the constraint and index pair was restored", topic: "neo4j-schema", validatedCount: 1 }, + ], + }, + { + topic: "typescript-build", + query: "typescript tsconfig module compilation strict", + nodes: [ + { key: "ts-task", type: "TASK", name: "abl-tsconfig-modernization", description: "tsconfig modernization initiative", content: "audit every tsconfig in the monorepo, unify module resolution, tighten strict flags, and gate compilation in ci so declaration emit stays consistent", topic: "typescript-build", validatedCount: 2 }, + { key: "ts-strict", type: "SKILL", name: "abl-strict-mode-migration", description: "strict mode migration playbook", content: "noImplicitAny and strictNullChecks surface latent defects; codemods annotate signatures, null guards narrow control flow, and exhaustive switch checks harden the compilation boundary", topic: "typescript-build", validatedCount: 6 }, + { key: "ts-resolution", type: "SKILL", name: "abl-module-resolution-deep-dive", description: "module resolution deep dive", content: "bundler resolution rewrites relative extensions, paths aliases map workspace packages, verbatimModuleSyntax separates types, and the emit target guides output module shape", topic: "typescript-build", validatedCount: 4 }, + { key: "ts-buildbreak", type: "EVENT", name: "abl-ci-build-break-postmortem", description: "ci compilation break postmortem", content: "the pipeline failed when a tsconfig override disabled declaration emit; incremental cache masked it locally until ci compiled from clean and exposed the missing types", topic: "typescript-build", validatedCount: 1 }, + ], + }, +]; + +const DISTRACTORS: FixtureNode[] = [ + { key: "d-pasta", type: "SKILL", name: "abl-pasta-carbonara-notes", description: "italian pasta cooking notes", content: "guanciale rendered crisp, pecorino and egg emulsion, tonnarelli tossed off heat, black pepper finish, a roman classic rehearsed by feel not measure", topic: "misc" }, + { key: "d-japan", type: "SKILL", name: "abl-kyoto-itinerary-draft", description: "japan travel itinerary draft", content: "shinkansen from tokyo to kyoto, early fushimi inari gates, gion at dusk, day trip to nara deer park, temple bookings before cherry blossom peak", topic: "misc" }, + { key: "d-guitar", type: "SKILL", name: "abl-acoustic-chord-drills", description: "acoustic guitar chord drills", content: "fingerpick travis pattern over c g am f shapes, metronome at seventy, barre chord squeeze routine, pentatonic ladder across six strings daily", topic: "misc" }, + { key: "d-gym", type: "SKILL", name: "abl-strength-block-plan", description: "gym strength block plan", content: "five by five back squat progression, paused bench press, romanian deadlift volume, sled finisher, deload every fourth week", topic: "misc" }, + { key: "d-novel", type: "SKILL", name: "abl-novel-outline-v3", description: "novel chapter outline v3", content: "lighthouse keeper hides the ledger, storm chapter mirrors act one betrayal, viewpoint alternates diary and present, climax reframes the inheritance", topic: "misc" }, + { key: "d-tomato", type: "SKILL", name: "abl-tomato-bed-log", description: "tomato garden bed log", content: "amended clay with compost, drip line on timer, pruned suckers weekly, staked indeterminates, mulched before the heatwave", topic: "misc" }, + { key: "d-car", type: "SKILL", name: "abl-hatchback-service-log", description: "hatchback maintenance log", content: "synthetic oil every ten thousand kilometres, tire rotation cross pattern, brake pad thickness measured, cabin filter swapped in spring", topic: "misc" }, + { key: "d-coffee", type: "SKILL", name: "abl-pourover-recipe-card", description: "pour over coffee recipe card", content: "twenty gram dose, three hundred forty milliliters water at ninety three celsius, forty five second bloom, single pour spiral, medium grind settles flat", topic: "misc" }, + { key: "d-photo", type: "SKILL", name: "abl-exposure-triangle-notes", description: "photography exposure notes", content: "aperture controls depth of field, shutter freezes or drags motion, iso lifts shadows with noise cost, meter for highlights and push later", topic: "misc" }, + { key: "d-chess", type: "SKILL", name: "abl-sicilian-defence-prep", description: "chess sicilian defence prep", content: "najdorf move orders, english attack with be3 f3 g4, rook lift to g5, poisoned pawn line memorized to move twenty", topic: "misc" }, + { key: "d-spanish", type: "SKILL", name: "abl-spanish-streak-deck", description: "spanish vocabulary deck", content: "subjunctive triggers on influence verbs, por versus para contrast cards, spaced repetition streak at two hundred days, kitchen nouns themed batch", topic: "misc" }, + { key: "d-budget", type: "SKILL", name: "abl-household-budget-sheet", description: "household budget spreadsheet", content: "fixed costs bucketed, sinking funds for insurance premiums, grocery category averaged quarterly, savings rate plotted against overtime", topic: "misc" }, +]; + +const RECALL_EDGES: FixtureEdge[] = [ + { from: "dep-task", to: "dep-compose", type: "USED_SKILL", instruction: "deploys with" }, + { from: "dep-task", to: "dep-registry", type: "USED_SKILL", instruction: "deploys with" }, + { from: "dep-compose", to: "dep-registry", type: "REQUIRES", instruction: "pulls images through" }, + { from: "dep-outage", to: "dep-registry", type: "SOLVED_BY", instruction: "mitigated by" }, + { from: "test-task", to: "test-mocking", type: "USED_SKILL", instruction: "tests with" }, + { from: "test-task", to: "test-coverage", type: "USED_SKILL", instruction: "tests with" }, + { from: "test-mocking", to: "test-coverage", type: "REQUIRES", instruction: "gated by" }, + { from: "test-flaky", to: "test-mocking", type: "SOLVED_BY", instruction: "mitigated by" }, + { from: "neo-task", to: "neo-cypher", type: "USED_SKILL", instruction: "migrates with" }, + { from: "neo-task", to: "neo-schema", type: "USED_SKILL", instruction: "migrates with" }, + { from: "neo-cypher", to: "neo-schema", type: "REQUIRES", instruction: "gated by" }, + { from: "neo-incident", to: "neo-cypher", type: "SOLVED_BY", instruction: "mitigated by" }, + { from: "ts-task", to: "ts-strict", type: "USED_SKILL", instruction: "migrates with" }, + { from: "ts-task", to: "ts-resolution", type: "USED_SKILL", instruction: "migrates with" }, + { from: "ts-strict", to: "ts-resolution", type: "REQUIRES", instruction: "gated by" }, + { from: "ts-buildbreak", to: "ts-strict", type: "SOLVED_BY", instruction: "mitigated by" }, + // 跨簇弱连接 ×2：检验社区检测/扩展在非完美分簇下的行为 + { from: "test-flaky", to: "ts-resolution", type: "SOLVED_BY", instruction: "mitigated by" }, + { from: "neo-cypher", to: "ts-resolution", type: "REQUIRES", instruction: "gated by" }, +]; + +/** key → 落库后的真实 GmNode */ +const nodeRegistry = new Map(); + +async function seedNodes(list: FixtureNode[], sid: string): Promise { + for (const f of list) { + const { node } = await upsertNode(driver, { + type: f.type, name: f.name, description: f.description, content: f.content, + }, sid); + nodeRegistry.set(f.key, node); + await saveVector(driver, node.id, buildNodeEmbeddingText(node), embedText(buildNodeEmbeddingText(node))); + if (f.validatedCount !== undefined) { + const session = getSession(driver); + try { + await session.run( + "MATCH (n:Task|Skill|Event {id: $id}) SET n.validatedCount = $vc", + { id: node.id, vc: f.validatedCount }, + ); + } finally { await session.close(); } + } + } +} + +async function seedEdges(list: FixtureEdge[], sid: string): Promise { + for (const e of list) { + const from = nodeRegistry.get(e.from); + const to = nodeRegistry.get(e.to); + if (!from || !to) throw new Error(`fixture edge references unknown key: ${e.from} -> ${e.to}`); + await upsertEdge(driver, { + fromId: from.id, toId: to.id, type: e.type, instruction: e.instruction, sessionId: sid, + }); + } +} + +/** 为 detectCommunities 的每个社区按生产形态补建 Community 节点（id/summary/nodeCount/embedding） */ +async function createCommunityNodes(): Promise { + const result = await detectCommunities(driver); + expect(result.count).toBeGreaterThanOrEqual(3); + for (const [cid, memberIds] of result.communities) { + const members = memberIds + .map((id) => Array.from(nodeRegistry.values()).find((n) => n.id === id)) + .filter((n): n is GmNode => !!n); + const topicCounts = new Map(); + for (const id of memberIds) { + const entry = Array.from(CLUSTERS.flatMap((c) => c.nodes)).find((f) => nodeRegistry.get(f.key)?.id === id); + if (entry) topicCounts.set(entry.topic, (topicCounts.get(entry.topic) ?? 0) + 1); + } + const topic = Array.from(topicCounts.entries()).sort((a, b) => b[1] - a[1])[0]?.[0] ?? "misc"; + const session = getSession(driver); + try { + await session.run( + `MERGE (c:Community {id: $cid}) + SET c.summary = $summary, c.nodeCount = $count, c.memberSignature = $sig`, + { cid, summary: `${topic} knowledge cluster`, count: memberIds.length, sig: buildCommunityMemberSignature(memberIds) }, + ); + } finally { await session.close(); } + await saveCommunityEmbedding(driver, cid, meanVector(members.map((n) => buildNodeEmbeddingText(n)))); + } + return result.count; +} + +// ─── 套件 ─────────────────────────────────────────────────── + +describe.skipIf(!ENABLED)("graph-memory-pro ablation study", () => { + + beforeAll(async () => { + console.log(`[ablation] NEO4J_TEST_URI = ${NEO4J_URI}（一次性容器，非生产库）`); + driver = getDriver(CFG.neo4j); + await initSchema(driver); + + // 环境自检：APOC + GDS 必须可用（否则 PPR/社区消融无意义） + const session = getSession(driver); + try { + const r = await session.run("RETURN apoc.version() AS apoc, gds.version() AS gds"); + const rec = r.records[0]; + console.log(`[ablation] APOC ${rec.get("apoc")} / GDS ${rec.get("gds")}`); + expect(rec.get("apoc")).toBeTruthy(); + expect(rec.get("gds")).toBeTruthy(); + } finally { await session.close(); } + + // 召回 cohort：4 主题簇 × 4 节点 + 12 干扰节点 + 簇内边 + 2 跨簇边 + await seedNodes([...CLUSTERS.flatMap((c) => c.nodes), ...DISTRACTORS], SID_RECALL); + await seedEdges(RECALL_EDGES, SID_RECALL); + // 全局 PageRank（searchNodes 排序依赖 n.pagerank） + await computeGlobalPageRank(driver, CFG); + // 社区检测 + Community 节点 + 社区向量（generalized 路径依赖） + const communities = await createCommunityNodes(); + console.log(`[ablation] 召回 cohort 就绪：${CLUSTERS.length * 4 + DISTRACTORS.length} 节点，${communities} 社区`); + }, 300_000); + + afterAll(async () => { + const session = getSession(driver); + try { + await session.run( + "MATCH (n) WHERE any(s IN n.sourceSessions WHERE s STARTS WITH $p) DETACH DELETE n", + { p: CLEAN_PREFIX }, + ); + await session.run("MATCH (c:Community) DETACH DELETE c"); + } finally { await session.close(); } + await closeDriver(); + }, 60_000); + + // ─── A. 召回管线消融 ───────────────────────────────────── + + describe("A. recall pipeline", () => { + // 惰性构造：describe 块体在收集阶段执行（skip 不阻止），此时 beforeAll 尚未跑 + const queries = () => CLUSTERS.map((c) => ({ + query: c.query, + relevant: new Set(c.nodes.map((n) => nodeRegistry.get(n.key)!.id)), + })); + + const ABLATIONS: Array<{ label: string; flags: Flags }> = [ + { label: "full（全部开启）", flags: FULL }, + { label: "-vector（去向量，FTS 兜底）", flags: { ...FULL, vector: false } }, + { label: "-ppr（去 PPR 排序）", flags: { ...FULL, ppr: false } }, + { label: "-communityExpansion（去社区扩展）", flags: { ...FULL, communityExpansion: false } }, + { label: "-generalized（仅精确路径）", flags: { ...FULL, generalized: false } }, + { label: "-precise（仅泛化路径）", flags: { ...FULL, precise: false } }, + ]; + + it("fidelity：harness full 管线与真实 Recaller.recall 输出一致", async () => { + const recaller = new Recaller(driver, CFG); + recaller.setEmbedFn(mockEmbedFn); + for (const { query, relevant } of queries()) { + const real = await recaller.recall(query); + const harness = await recallPipeline(query, FULL); + const realIds = new Set(real.nodes.map((n) => n.id)); + const harnessIds = new Set(harness.nodes.map((n) => n.id)); + let overlap = 0; + for (const id of harnessIds) if (realIds.has(id)) overlap++; + const ratio = harnessIds.size ? overlap / harnessIds.size : 1; + console.log(`[ablation:fidelity] q="${query}" real=${realIds.size} harness=${harnessIds.size} overlap=${ratio.toFixed(2)}`); + // 允许排序型微差，但集合重合度必须极高（harness 忠实性的门槛） + expect(ratio).toBeGreaterThanOrEqual(0.8); + expect(real.nodes.length).toBeGreaterThan(0); + expect(scoreResult(real, relevant).recallAtK).toBeGreaterThan(0); + } + }, 300_000); + + it("消融矩阵：逐组件关闭并测量 recall/precision/MRR/延迟", async () => { + for (const { label, flags } of ABLATIONS) { + const perQuery: Metrics[] = []; + let totalMs = 0; + for (const { query, relevant } of queries()) { + const t0 = performance.now(); + const result = await recallPipeline(query, flags); + totalMs += performance.now() - t0; + perQuery.push(scoreResult(result, relevant)); + } + addRow("A.recall", label, averageMetrics(perQuery), totalMs / queries().length); + } + }, 300_000); + + it("sanity：full 配置召回质量高于零假设", async () => { + // 单独重跑 full，给出可断言的聚合值（也验证矩阵运行后图未被污染） + const { query, relevant } = queries()[0]; + const result = await recallPipeline(query, FULL); + const m = scoreResult(result, relevant); + expect(m.recallAtK).toBeGreaterThanOrEqual(0.5); + addRow("A.recall", "full 复跑（sanity）", m, 0, "q=" + query); + }, 120_000); + }); + + // ─── B. 去重消融 ───────────────────────────────────────── + + describe("B. dedup", () => { + const DUP_CORE = "redis evicts least recently used keys first, ttl expiration sweeps lazy pass, maxmemory policy allkeys lru tunes cache hit rate under pressure"; + const NGINX_CORE = "nginx terminates tls at the edge, proxies upstream pools with least connections, buffers slow clients, and rewrites host headers for the internal mesh"; + const GRPC_CORE = "grpc clients balance across resolved endpoints, round robin picks per call, sticky streams pin to one backend, health checks drain unhealthy targets"; + + const dupNodes: FixtureNode[] = [ + { key: "redis-base", type: "SKILL", name: "abl-redis-cache-eviction", description: "redis cache eviction policy", content: DUP_CORE, topic: "dup", validatedCount: 5 }, + { key: "redis-v1", type: "SKILL", name: "abl-redis-eviction-policy-v2", description: "redis cache eviction policy notes", content: `${DUP_CORE} benchmarked eviction latency`, topic: "dup", validatedCount: 1 }, + { key: "redis-v2", type: "SKILL", name: "abl-redis-eviction-runbook", description: "redis cache eviction runbook", content: `${DUP_CORE} documented runbook fallback`, topic: "dup", validatedCount: 1 }, + { key: "nginx-base", type: "SKILL", name: "abl-nginx-reverse-proxy", description: "nginx reverse proxy configuration", content: NGINX_CORE, topic: "dup", validatedCount: 5 }, + { key: "nginx-v1", type: "SKILL", name: "abl-nginx-proxy-config-v2", description: "nginx reverse proxy notes", content: `${NGINX_CORE} benchmarked proxy latency`, topic: "dup", validatedCount: 1 }, + { key: "nginx-v2", type: "SKILL", name: "abl-nginx-proxy-tuning", description: "nginx reverse proxy tuning", content: `${NGINX_CORE} documented tuning baseline`, topic: "dup", validatedCount: 1 }, + { key: "grpc-base", type: "SKILL", name: "abl-grpc-load-balancing", description: "grpc client load balancing", content: GRPC_CORE, topic: "dup", validatedCount: 5 }, + { key: "grpc-v1", type: "SKILL", name: "abl-grpc-balancing-notes", description: "grpc client balancing notes", content: `${GRPC_CORE} benchmarked picker behavior`, topic: "dup", validatedCount: 1 }, + { key: "kafka-base", type: "SKILL", name: "abl-kafka-consumer-retry", description: "kafka consumer retry semantics", content: "kafka consumers retry poison pills on a dedicated topic, exponential backoff before dead letter, offsets committed after the handler succeeds", topic: "dup", validatedCount: 5 }, + ]; + + // 臂 B 副本：同内容不同名（name 唯一约束下这是产生"重复知识"的唯一方式， + // 等价于跨 session 重复提取到同一知识的场景） + const dupNodesB = dupNodes.map((f) => ({ + ...f, key: `${f.key}-b`, name: `${f.name}-b`, description: f.description, + })); + + const DUP_TOPICS = ["redis", "nginx", "grpc", "kafka"]; + const DIVERSITY_QUERY = "redis cache eviction nginx proxy grpc balancing kafka retry"; + const PRECISE_ONLY: Flags = { ...FULL, generalized: false }; + + /** 主题覆盖度量：top-K 覆盖的不同主题数 / 4 —— 近重复副本只计一次主题 */ + function topicMetrics(result: RecallResult): Metrics { + const retrieved = result.nodes.slice(0, K); + const covered: string[] = []; + for (const n of retrieved) { + const t = DUP_TOPICS.find((topic) => n.name.toLowerCase().includes(topic)); + if (t && !covered.includes(t)) covered.push(t); + } + const firstTopicIdx = retrieved.findIndex((n) => + DUP_TOPICS.some((topic) => n.name.toLowerCase().includes(topic))); + return { + recallAtK: covered.length / DUP_TOPICS.length, + precisionAtK: retrieved.length ? covered.length / retrieved.length : 0, + mrr: firstTopicIdx >= 0 ? 1 / (firstTopicIdx + 1) : 0, + nodesReturned: result.nodes.length, + }; + } + + beforeAll(async () => { + // 只播种臂 A：臂 B 必须在臂 A 去重之后再播种（detectDuplicates 是全局扫描） + await seedNodes(dupNodes, SID_DUP_A); + }, 120_000); + + it("相似度门槛可检出近重复对（fixture 有效性）", async () => { + const pairs = await detectDuplicates(driver, { ...CFG, dedupThreshold: 0.90 }); + const dupPairs = pairs.filter((p) => + p.nameA.startsWith("abl-redis") || p.nameA.startsWith("abl-nginx") || p.nameA.startsWith("abl-grpc")); + console.log(`[ablation:dedup] 检出近重复对 ${dupPairs.length}（全局 ${pairs.length}）：` + + dupPairs.slice(0, 8).map((p) => `${p.nameA}~${p.nameB}@${p.similarity.toFixed(3)}`).join(", ")); + expect(dupPairs.length).toBeGreaterThanOrEqual(6); // redis3 + nginx3 + grpc1 对 + for (const p of dupPairs) expect(p.similarity).toBeGreaterThanOrEqual(0.90); + }, 120_000); + + it("臂 A：dedup 合并近重复 → 召回主题多样性", async () => { + const armA = await dedup(driver, { ...CFG, dedupThreshold: 0.90 }); + console.log(`[ablation:dedup] 臂A 合并 ${armA.merged} 个节点（检出对 ${armA.pairs.length}）`); + expect(armA.merged).toBeGreaterThanOrEqual(5); + + const t0 = performance.now(); + const resA = await recallPipeline(DIVERSITY_QUERY, PRECISE_ONLY); + const msA = performance.now() - t0; + const mA = topicMetrics(resA); + console.log(`[ablation:dedup] 臂A 主题覆盖@${K}=${mA.recallAtK.toFixed(2)}（去重后变体已弃用，FTS 兜底可触达全部主题）`); + addRow("B.dedup", "dedup=on（臂A）", mA, msA, "主题覆盖@K；变体已合并"); + expect(mA.recallAtK).toBeGreaterThan(0.5); + }, 300_000); + + it("臂 B（消融）：同内容重复入库且不 dedup → 召回多样性受损", async () => { + // 臂 B 播种：同内容不同名 → 形成未去重的重复知识群 + await seedNodes(dupNodesB, SID_DUP_B); + + // 自检：redis 主题族当前活跃节点应 ≥ 4（臂A base + 臂B base/v1/v2） + const session = getSession(driver); + let redisActive = 0; + try { + const cnt = await session.run( + `MATCH (n:Skill {status: 'active'}) + WHERE n.name STARTS WITH 'abl-redis' RETURN count(n) AS c`, + ); + redisActive = cnt.records[0].get("c").toNumber(); + } finally { await session.close(); } + console.log(`[ablation:dedup] 臂B 播种后 redis 族活跃节点 = ${redisActive}（应 ≥ 4：未去重副本在场）`); + expect(redisActive).toBeGreaterThanOrEqual(4); + + const t0 = performance.now(); + const resB = await recallPipeline(DIVERSITY_QUERY, PRECISE_ONLY); + const msB = performance.now() - t0; + const mB = topicMetrics(resB); + console.log(`[ablation:dedup] 臂B 主题覆盖@${K}=${mB.recallAtK.toFixed(2)}（向量 top-3 被同主题副本占据 → 其他主题挤不出种子）`); + addRow("B.dedup", "dedup=off（臂B）", mB, msB, "主题覆盖@K；重复副本在场"); + expect(mB.recallAtK).toBeLessThan(1); + }, 300_000); + + it("结论断言：dedup 提升召回主题多样性", async () => { + // 从 ROWS 里取两臂的 recallAtK 对比（避免重复跑管线引入状态依赖） + const rowA = ROWS.find((r) => r.suite === "B.dedup" && r.config.includes("臂A")); + const rowB = ROWS.find((r) => r.suite === "B.dedup" && r.config.includes("臂B")); + expect(rowA).toBeDefined(); + expect(rowB).toBeDefined(); + expect(Number(rowA!.recallAtK)).toBeGreaterThan(Number(rowB!.recallAtK)); + }, 30_000); + }); + + // ─── C. 衰减消融 ───────────────────────────────────────── + + describe("C. decay", () => { + const DAY = 86_400_000; + const STALE_QUERY = "kubernetes helm chart deployment rollout"; + const FRESH_QUERY = "rust tokio async runtime channels"; + + const decayNodes: FixtureNode[] = [ + { key: "k8s-task", type: "TASK", name: "abl-k8s-rollout-automation", description: "kubernetes helm deployment automation", content: "helm charts template the kubernetes deployment, values overlays per cluster, helm upgrade rolls replicas with surge budget, rollback restores previous revision", topic: "stale" }, + { key: "k8s-helm", type: "SKILL", name: "abl-helm-chart-authoring", description: "helm chart authoring guide", content: "helm chart structure bundles templates, values and hooks; chart dependencies compose subcharts; kubeconform validates rendered manifests before the rollout applies", topic: "stale" }, + { key: "k8s-kubectl", type: "SKILL", name: "abl-kubectl-rollout-ops", description: "kubectl rollout operations", content: "kubectl rollout status watches deployment progress, rollout undo reverts the replica set, pod disruption budgets guard node drains during the rollout", topic: "stale" }, + { key: "k8s-event", type: "EVENT", name: "abl-helm-hook-incident", description: "helm hook incident notes", content: "a pre-upgrade helm hook deadlocked when jobs lacked ttl; the deployment rollout stalled until hook deletion policy and backoff limits were fixed", topic: "stale" }, + { key: "rust-task", type: "TASK", name: "abl-async-runtime-selection", description: "rust async runtime selection", content: "tokio drives the worker pool, multithreaded scheduler distributes tasks, io drivers poll sockets, rust futures compose selects and joins across the runtime", topic: "fresh" }, + { key: "rust-tokio", type: "SKILL", name: "abl-tokio-channel-patterns", description: "tokio channel patterns", content: "mpsc pipelines stages, broadcast fans out telemetry, oneshot awaits single replies, watch signals config reload across the async runtime", topic: "fresh" }, + { key: "rust-select", type: "SKILL", name: "abl-rust-select-timeouts", description: "rust select and timeouts", content: "tokio::select races futures, timeouts wrap slow branches, cancellation safety audits each arm, rust async blocks spawn detached from the parent scope", topic: "fresh" }, + { key: "rust-event", type: "EVENT", name: "abl-runtime-deadlock-retro", description: "async deadlock retrospective", content: "a mutex guard held across an await starved the tokio worker; restructuring critical sections and scoped channels unblocked the rust runtime", topic: "fresh" }, + ]; + + const staleRelevant = () => new Set(["k8s-task", "k8s-helm", "k8s-kubectl", "k8s-event"] + .map((k) => nodeRegistry.get(k)!.id)); + const freshRelevant = () => new Set(["rust-task", "rust-tokio", "rust-select", "rust-event"] + .map((k) => nodeRegistry.get(k)!.id)); + + async function setDecayProps(keys: string[], props: Record): Promise { + const session = getSession(driver); + try { + for (const k of keys) { + const node = nodeRegistry.get(k)!; + const assignments = Object.keys(props) + .map((p) => `n.${p} = $${p}`) + .join(", "); + await session.run( + `MATCH (n:Task|Skill|Event {id: $id}) SET ${assignments}`, + { id: node.id, ...props }, + ); + } + } finally { await session.close(); } + } + + beforeAll(async () => { + await seedNodes(decayNodes, SID_DECAY); + await seedEdges([ + { from: "k8s-task", to: "k8s-helm", type: "USED_SKILL", instruction: "rolls out with" }, + { from: "k8s-task", to: "k8s-kubectl", type: "USED_SKILL", instruction: "rolls out with" }, + { from: "k8s-helm", to: "k8s-kubectl", type: "REQUIRES", instruction: "gated by" }, + { from: "k8s-event", to: "k8s-helm", type: "SOLVED_BY", instruction: "mitigated by" }, + { from: "rust-task", to: "rust-tokio", type: "USED_SKILL", instruction: "runs on" }, + { from: "rust-task", to: "rust-select", type: "USED_SKILL", instruction: "runs on" }, + { from: "rust-tokio", to: "rust-select", type: "REQUIRES", instruction: "gated by" }, + { from: "rust-event", to: "rust-tokio", type: "SOLVED_BY", instruction: "mitigated by" }, + ], SID_DECAY); + + const now = Date.now(); + // 陈旧簇：120 天未访问、低频次 → 预期 peripheral + 自动弃用 + await setDecayProps(["k8s-task", "k8s-helm", "k8s-kubectl", "k8s-event"], { + lastAccessedAt: now - 120 * DAY, updatedAt: now - 120 * DAY, + createdAt: now - 200 * DAY, validatedCount: 1, + }); + // 新鲜簇：活跃高频 → 预期保持 working/core + await setDecayProps(["rust-task", "rust-tokio", "rust-select", "rust-event"], { + validatedCount: 8, + }); + + // 手工指派社区 + Community 节点（不重跑全局检测，避免扰动 A 组社区） + for (const [cid, keys, summary] of [ + ["abl-dec-stale", ["k8s-task", "k8s-helm", "k8s-kubectl", "k8s-event"], "kubernetes helm rollout knowledge"], + ["abl-dec-fresh", ["rust-task", "rust-tokio", "rust-select", "rust-event"], "rust tokio async knowledge"], + ] as Array<[string, string[], string]>) { + const session = getSession(driver); + try { + await session.run( + `MERGE (c:Community {id: $cid}) SET c.summary = $summary, c.nodeCount = $count`, + { cid, summary, count: keys.length }, + ); + for (const k of keys) { + await session.run( + "MATCH (n:Task|Skill|Event {id: $id}) SET n.communityId = $cid", + { id: nodeRegistry.get(k)!.id, cid }, + ); + } + } finally { await session.close(); } + await saveCommunityEmbedding(driver, cid, + meanVector(keys.map((k) => buildNodeEmbeddingText(nodeRegistry.get(k)!)))); + } + }, 120_000); + + it("arm0 基线：decay.enabled=false → 无扫描无弃用，陈旧知识仍可召回", async () => { + const r = await applyDecay(driver, { decay: { ...DEFAULT_CONFIG.decay!, enabled: false } }); + expect(r.enabled).toBe(false); + expect(r.autoDeprecated).toBe(0); + + const res = await recallPipeline(STALE_QUERY, FULL); + const m = scoreResult(res, staleRelevant()); + console.log(`[ablation:decay] arm0 基线 recall@${K}=${m.recallAtK.toFixed(2)}`); + expect(m.recallAtK).toBeGreaterThan(0); + addRow("C.decay", "decay=off（基线）", m, 0, STALE_QUERY); + }, 120_000); + + it("arm1 decay 开 + autoDeprecate 关：只降层不断联", async () => { + const r = await applyDecay(driver, { + decay: { ...DEFAULT_CONFIG.decay!, enabled: true, autoDeprecate: false }, + }); + console.log(`[ablation:decay] arm1 scanned=${r.scanned} transitions=${JSON.stringify(r.tierTransitions)} autoDeprecated=${r.autoDeprecated}`); + expect(r.enabled).toBe(true); + expect(r.autoDeprecated).toBe(0); + // 120 天未访问 + validatedCount=1 的陈旧簇应降入 peripheral + expect(r.tierTransitions.workingToPeripheral).toBeGreaterThanOrEqual(4); + + const res = await recallPipeline(STALE_QUERY, FULL); + const m = scoreResult(res, staleRelevant()); + console.log(`[ablation:decay] arm1 陈旧查询 recall@${K}=${m.recallAtK.toFixed(2)}（未弃用，仍可达）`); + expect(m.recallAtK).toBeGreaterThan(0); + addRow("C.decay", "decay=on, autoDeprecate=off", m, 0, "降层但不断联"); + }, 120_000); + + it("arm2 全量 decay+autoDeprecate：陈旧簇被遗忘且新鲜簇不受影响", async () => { + const r = await applyDecay(driver, { + decay: { ...DEFAULT_CONFIG.decay!, enabled: true, autoDeprecate: true }, + }); + console.log(`[ablation:decay] arm2 autoDeprecated=${r.autoDeprecated}`); + expect(r.autoDeprecated).toBeGreaterThanOrEqual(4); + + const session = getSession(driver); + let decayDeprecated = 0; + try { + const cnt = await session.run( + `MATCH (n:Task|Skill|Event) + WHERE any(s IN n.sourceSessions WHERE s STARTS WITH $p) + AND n.status = 'deprecated' AND n.deprecatedBy = 'decay' + RETURN count(n) AS c`, + { p: SID_DECAY }, + ); + decayDeprecated = cnt.records[0].get("c").toNumber(); + } finally { await session.close(); } + expect(decayDeprecated).toBeGreaterThanOrEqual(4); + + const staleRes = await recallPipeline(STALE_QUERY, FULL); + const staleM = scoreResult(staleRes, staleRelevant()); + console.log(`[ablation:decay] arm2 陈旧查询 recall@${K}=${staleM.recallAtK.toFixed(2)}（应为 0：已遗忘）`); + + const freshRes = await recallPipeline(FRESH_QUERY, FULL); + const freshM = scoreResult(freshRes, freshRelevant()); + console.log(`[ablation:decay] arm2 新鲜查询 recall@${K}=${freshM.recallAtK.toFixed(2)}（应不受影响）`); + + addRow("C.decay", "decay+autoDeprecate=on（陈旧查询）", staleM, 0, "应遗忘 → 0"); + addRow("C.decay", "decay+autoDeprecate=on（新鲜查询）", freshM, 0, "对照：不应受损"); + expect(staleM.recallAtK).toBe(0); + expect(freshM.recallAtK).toBeGreaterThan(0); + }, 120_000); + }); + + // ─── D. 汇总报告 ───────────────────────────────────────── + + describe("D. report", () => { + it("输出消融结果表 + JSON 工件", async () => { + console.log("\n========== ABLATION STUDY RESULTS =========="); + console.table(ROWS.map(({ suite, config, recallAtK, precisionAtK, mrr, nodes, latencyMs, note }) => ({ + suite, config, recallAt6: recallAtK, precisionAt6: precisionAtK, mrr, nodes, latencyMs, note, + }))); + + const outPath = fileURLToPath(new URL("../.zcode/ablation-results.json", import.meta.url)); + await mkdir(path.dirname(outPath), { recursive: true }); + await writeFile(outPath, JSON.stringify({ + generatedAt: new Date().toISOString(), + neo4jUri: NEO4J_URI, + k: K, + rows: ROWS, + }, null, 2), "utf-8"); + console.log(`[ablation] JSON 工件已写入 ${outPath}`); + expect(ROWS.length).toBeGreaterThan(0); + }, 30_000); + }); +}); diff --git a/test/assemble-context.test.ts b/test/assemble-context.test.ts index e025e90..0c2340c 100644 --- a/test/assemble-context.test.ts +++ b/test/assemble-context.test.ts @@ -11,12 +11,14 @@ function makeNode(overrides: Partial): GmNode { description: "description", content: "content", status: "active", + tier: "working", validatedCount: 1, sourceSessions: ["test"], communityId: null, pagerank: 0, createdAt: now, updatedAt: now, + lastAccessedAt: now, ...overrides, }; } diff --git a/test/cli-extract.test.ts b/test/cli-extract.test.ts new file mode 100644 index 0000000..9ea08ad --- /dev/null +++ b/test/cli-extract.test.ts @@ -0,0 +1,441 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + listUnextractedSessions: vi.fn(async () => [] as any[]), + getUnextracted: vi.fn(async (_d: any, _sid: any, _limit: any) => [] as any[]), + markExtracted: vi.fn(async () => {}), + upsertNode: vi.fn(async (_driver: any, c: any) => ({ + node: { + id: `n-${c.name}`, + type: c.type, + name: c.name, + description: c.description ?? "", + content: c.content, + status: "active", + validatedCount: 1, + sourceSessions: [], + communityId: null, + pagerank: 0, + createdAt: 0, + updatedAt: 0, + }, + isNew: true, + })), + upsertEdge: vi.fn(async () => {}), + findByName: vi.fn(async () => null), + getBySession: vi.fn(async () => [] as any[]), + extract: vi.fn(async () => ({ nodes: [] as any[], edges: [] as any[] })), + initSchema: vi.fn(async () => {}), + closeDriver: vi.fn(async () => {}), +})); + +vi.mock("../src/store/db.ts", () => ({ + getDriver: () => ({}), + initSchema: mocks.initSchema, + getSession: () => ({ close: async () => {} }), + closeDriver: mocks.closeDriver, +})); + +vi.mock("../src/store/store.ts", () => ({ + listUnextractedSessions: mocks.listUnextractedSessions, + getUnextracted: mocks.getUnextracted, + markExtracted: mocks.markExtracted, + upsertNode: mocks.upsertNode, + upsertEdge: mocks.upsertEdge, + findByName: mocks.findByName, + getBySession: mocks.getBySession, +})); + +vi.mock("../src/engine/llm.ts", () => ({ + createCompleteFn: () => async () => "", + resolveProvider: () => ({ provider: "openai", inferred: false }), +})); + +vi.mock("../src/engine/embed.ts", () => ({ + createEmbedder: async () => null, +})); + +vi.mock("../src/recaller/recall.ts", () => ({ + Recaller: class { + setEmbedFn(): void {} + setEmbedBatchFn(): void {} + async syncEmbed(): Promise {} + async syncEmbedBatch(): Promise {} + }, +})); + +vi.mock("../src/extractor/extract.ts", () => ({ + Extractor: class { + async extract() { + return mocks.extract(); + } + }, +})); + +import { isAffirmative, runBackfillExtraction } from "../src/cli-extract.ts"; +import { DEFAULT_CONFIG } from "../src/types.ts"; + +function makeCfg(overrides: Record = {}) { + return { + ...DEFAULT_CONFIG, + neo4j: { uri: "bolt://localhost:7687", user: "neo4j", password: "x" }, + llm: { provider: "openai", apiKey: "k", baseURL: "https://api.openai.com/v1", model: "gpt-test" }, + ...overrides, + } as any; +} + +const SAMPLE_SESSION = { + sessionId: "sid-abc-1234567890", + messageCount: 5, + maxTurn: 5, + minCreatedAt: 1700000000000, +}; + +describe("isAffirmative", () => { + it.each([ + ["y", true], + ["Y", true], + ["yes", true], + ["YES", true], + [" yes ", true], + ["yeah", true], + ["ok", true], + ["confirm", true], + ["1", true], + ["true", true], + ["n", false], + ["no", false], + ["", false], + ["maybe", false], + ["nope", false], + ["0", false], + ])("isAffirmative(%j) -> %s", (input, expected) => { + expect(isAffirmative(input)).toBe(expected); + }); +}); + +describe("runBackfillExtraction", () => { + beforeEach(() => { + mocks.listUnextractedSessions.mockReset(); + mocks.getUnextracted.mockReset(); + mocks.markExtracted.mockReset(); + mocks.upsertNode.mockReset(); + mocks.upsertEdge.mockReset(); + mocks.findByName.mockReset(); + mocks.getBySession.mockReset(); + mocks.extract.mockReset(); + mocks.initSchema.mockReset(); + mocks.closeDriver.mockReset(); + + mocks.initSchema.mockResolvedValue(undefined); + mocks.closeDriver.mockResolvedValue(undefined); + mocks.getBySession.mockResolvedValue([]); + mocks.extract.mockResolvedValue({ nodes: [], edges: [] }); + mocks.upsertNode.mockImplementation(async (_d: any, c: any) => ({ + node: { + id: `n-${c.name}`, + type: c.type, + name: c.name, + description: c.description ?? "", + content: c.content, + status: "active", + validatedCount: 1, + sourceSessions: [], + communityId: null, + pagerank: 0, + createdAt: 0, + updatedAt: 0, + }, + isNew: true, + })); + mocks.upsertEdge.mockResolvedValue(undefined); + mocks.findByName.mockResolvedValue(null); + mocks.markExtracted.mockResolvedValue(undefined); + }); + + it("returns sessionsTotal=0 and skips everything when no unextracted sessions", async () => { + mocks.listUnextractedSessions.mockResolvedValue([]); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: {}, + log, + }); + + expect(result.sessionsTotal).toBe(0); + expect(result.sessionsProcessed).toBe(0); + expect(mocks.extract).not.toHaveBeenCalled(); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + expect(log).toHaveBeenCalledWith(expect.stringContaining("没有需要提取的会话")); + }); + + it("requires an LLM model and throws a clear error when missing", async () => { + mocks.listUnextractedSessions.mockResolvedValue([]); + await expect( + runBackfillExtraction({ cfg: makeCfg(), effectiveModel: "", options: {}, log: vi.fn() }), + ).rejects.toThrow(/LLM model/); + expect(mocks.closeDriver).not.toHaveBeenCalled(); + }); + + it("requires neo4j.uri and throws a clear error when missing", async () => { + mocks.listUnextractedSessions.mockResolvedValue([]); + await expect( + runBackfillExtraction({ + cfg: { ...makeCfg(), neo4j: { uri: "", user: "", password: "" } } as any, + effectiveModel: "gpt-test", + options: {}, + log: vi.fn(), + }), + ).rejects.toThrow(/neo4j\.uri/); + }); + + it("aborts when the user declines the confirmation prompt", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + const prompt = vi.fn().mockResolvedValue("n"); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: {}, + log, + prompt, + }); + + expect(prompt).toHaveBeenCalledTimes(1); + expect(result.sessionsProcessed).toBe(0); + expect(result.sessionsSkipped).toBe(1); + expect(mocks.extract).not.toHaveBeenCalled(); + expect(mocks.markExtracted).not.toHaveBeenCalled(); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("does not prompt when --yes is set and runs extraction", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + mocks.getUnextracted.mockResolvedValueOnce([ + { role: "user", content: "hello", turn_index: 1 }, + { role: "assistant", content: "hi", turn_index: 2 }, + ]).mockResolvedValueOnce([]); + mocks.extract.mockResolvedValueOnce({ + nodes: [ + { type: "TASK", name: "t1", description: "d", content: "c" }, + { type: "SKILL", name: "s1", description: "d", content: "c" }, + ], + edges: [ + { from: "t1", to: "s1", type: "USED_SKILL", instruction: "i" }, + ], + }); + const prompt = vi.fn(); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { yes: true }, + log, + prompt, + }); + + expect(prompt).not.toHaveBeenCalled(); + expect(result.sessionsProcessed).toBe(1); + expect(result.nodesCreated).toBe(2); + expect(result.edgesCreated).toBe(1); + expect(result.batches).toBe(1); + expect(mocks.extract).toHaveBeenCalledTimes(1); + // 提取产出 2 节点 → producedKnowledge=true（空提取时应为 false，证据保留） + expect(mocks.markExtracted).toHaveBeenCalledWith(expect.anything(), "sid-abc-1234567890", 2, true); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("marks empty extractions as producedKnowledge=false (evidence retained)", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + mocks.getUnextracted.mockResolvedValueOnce([ + { role: "user", content: "hello", turn_index: 1 }, + ]).mockResolvedValueOnce([]); + mocks.extract.mockResolvedValueOnce({ nodes: [], edges: [] }); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { yes: true }, + log, + prompt: vi.fn(), + }); + + expect(result.sessionsProcessed).toBe(1); + expect(result.nodesCreated).toBe(0); + expect(mocks.markExtracted).toHaveBeenCalledWith(expect.anything(), "sid-abc-1234567890", 1, false); + }); + + it("filters sessions to the one specified by --session", async () => { + mocks.listUnextractedSessions.mockResolvedValue([ + { ...SAMPLE_SESSION, sessionId: "aaa" }, + { ...SAMPLE_SESSION, sessionId: "bbb" }, + ]); + mocks.getUnextracted.mockResolvedValue([]); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { yes: true, session: "bbb" }, + log, + }); + + expect(result.sessionsTotal).toBe(1); + expect(result.sessionsProcessed).toBe(1); + expect(mocks.getUnextracted).toHaveBeenCalledWith(expect.anything(), "bbb", expect.any(Number)); + }); + + it("exits cleanly when --session matches no sessions", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { session: "does-not-exist" }, + log, + }); + + expect(result.sessionsTotal).toBe(0); + expect(mocks.extract).not.toHaveBeenCalled(); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("lists sessions but does not extract under --dry-run", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + const prompt = vi.fn(); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { dryRun: true }, + log, + prompt, + }); + + expect(prompt).not.toHaveBeenCalled(); + expect(result.sessionsSkipped).toBe(1); + expect(result.sessionsProcessed).toBe(0); + expect(mocks.extract).not.toHaveBeenCalled(); + expect(log).toHaveBeenCalledWith(expect.stringContaining("--dry-run")); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("loops multiple batches until getUnextracted returns empty", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + mocks.getUnextracted + .mockResolvedValueOnce([ + { role: "user", content: "m1", turn_index: 1 }, + ]) + .mockResolvedValueOnce([ + { role: "user", content: "m2", turn_index: 2 }, + ]) + .mockResolvedValueOnce([]); + mocks.extract.mockResolvedValue({ nodes: [], edges: [] }); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { yes: true, limit: 1 }, + log, + }); + + expect(result.batches).toBe(2); + expect(mocks.markExtracted).toHaveBeenCalledTimes(2); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("records a session as skipped when getUnextracted throws", async () => { + mocks.listUnextractedSessions.mockResolvedValue([ + { ...SAMPLE_SESSION, sessionId: "good" }, + { ...SAMPLE_SESSION, sessionId: "bad" }, + ]); + const callCount = new Map(); + mocks.getUnextracted.mockImplementation(async (_d: any, sid: string) => { + if (sid === "bad") throw new Error("boom"); + const n = (callCount.get(sid) ?? 0) + 1; + callCount.set(sid, n); + if (n === 1) return [{ role: "user", content: "x", turn_index: 1 }]; + return []; + }); + mocks.extract.mockResolvedValue({ nodes: [], edges: [] }); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { yes: true }, + log, + }); + + expect(result.sessionsProcessed).toBe(1); + expect(result.sessionsSkipped).toBe(1); + expect(result.sessionsTotal).toBe(2); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("calls closeDriver even when listUnextractedSessions throws (try/finally)", async () => { + mocks.listUnextractedSessions.mockRejectedValue(new Error("neo4j down")); + const log = vi.fn(); + + await expect( + runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { yes: true }, + log, + }), + ).rejects.toThrow("neo4j down"); + + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("does not warn about batch ceiling when session completes before the ceiling", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + const fullBatches = 3; + mocks.getUnextracted.mockImplementation(async () => { + const call = mocks.getUnextracted.mock.calls.length; + if (call < fullBatches) return Array.from({ length: 5 }, (_, i) => ({ role: "user", content: `m${i}`, turn_index: call * 5 + i })); + return [{ role: "user", content: "last", turn_index: fullBatches * 5 }]; + }); + mocks.extract.mockResolvedValue({ nodes: [], edges: [] }); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg({ compactTurnCount: 1 }), + effectiveModel: "gpt-test", + options: { yes: true, limit: 5 }, + log, + }); + + expect(result.batches).toBe(fullBatches); + expect(result.sessionsProcessed).toBe(1); + const warningCalls = log.mock.calls.filter(c => typeof c[0] === "string" && c[0].includes("达到批数上限")); + expect(warningCalls).toHaveLength(0); + }); + + it("warns about batch ceiling when the session genuinely has more messages than the ceiling allows", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + mocks.getUnextracted.mockResolvedValue(Array.from({ length: 5 }, (_, i) => ({ role: "user", content: `m${i}`, turn_index: i }))); + mocks.extract.mockResolvedValue({ nodes: [], edges: [] }); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg({ compactTurnCount: 1 }), + effectiveModel: "gpt-test", + options: { yes: true, limit: 5 }, + log, + }); + + expect(result.batches).toBe(50); + const warningCalls = log.mock.calls.filter(c => typeof c[0] === "string" && c[0].includes("达到批数上限")); + expect(warningCalls).toHaveLength(1); + }); +}); diff --git a/test/commit-turn.test.ts b/test/commit-turn.test.ts new file mode 100644 index 0000000..c115353 --- /dev/null +++ b/test/commit-turn.test.ts @@ -0,0 +1,239 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +// ── 模块隔离：register() 的完整初始化路径绝不能触碰真实 Neo4j ── +// 不 mock 的话 getDriver() 会连 bolt://localhost:7687，initSchema 还会执行 +// 真实 DDL —— 任何 7687 端口有真实库的机器上裸跑 `npm test` 都有写风险。 +vi.mock("../src/store/db.ts", () => ({ + getDriver: () => ({}), + initSchema: async () => {}, + getSession: () => ({ close: async () => {} }), + closeDriver: async () => {}, +})); + +vi.mock("../src/store/store.ts", () => ({ + saveMessage: vi.fn(async () => {}), + getUnextracted: async () => [], + getMaxTurnIndex: async () => 0, + markExtracted: vi.fn(async () => {}), + isTurnExtracted: vi.fn(async () => false), + commitTurnAdvance: vi.fn(async () => "committed" as const), + upsertNode: async () => ({ node: {}, isNew: false }), + upsertEdge: async () => {}, + findByName: async () => null, + updateNode: async () => null, + deprecateNodeAndDisconnect: async () => {}, + deprecateNodeAndDisconnectById: async () => {}, + getBySession: async () => [], + edgesTouching: async () => [], + deleteEdges: async () => {}, + mergeNodes: async () => {}, + getStats: async () => ({}), +})); + +vi.mock("../src/engine/llm.ts", () => ({ + createCompleteFn: () => async () => "", + resolveProvider: () => ({ provider: "anthropic", inferred: false }), +})); + +vi.mock("../src/engine/embed.ts", () => ({ + createEmbedder: async () => null, +})); + +vi.mock("../src/recaller/recall.ts", () => ({ + Recaller: class { + setEmbedFn(): void {} + hasEmbedFn(): boolean { return false; } + get embedFn() { return null; } + async recall() { return { nodes: [], edges: [] }; } + async syncEmbed(): Promise {} + }, + parseTimeRange: () => null, +})); + +vi.mock("../src/extractor/extract.ts", () => ({ + Extractor: class { + async extract() { return { nodes: [], edges: [] }; } + async finalize() { return { promotedSkills: [], newEdges: [], invalidations: [] }; } + }, +})); + +vi.mock("../src/format/assemble.ts", () => ({ + assembleContext: async () => ({ xml: "", systemPrompt: "", tokens: 0 }), +})); + +vi.mock("../src/graph/maintenance.ts", () => ({ + runMaintenance: async () => ({ durationMs: 0 }), +})); + +vi.mock("../src/routes/crud.ts", () => ({ + registerCrudRoutes: () => {}, +})); + +import graphMemoryProPlugin from "../index.ts"; +import { closeDriver } from "../src/store/db.ts"; +import { commitTurnAdvance, isTurnExtracted, markExtracted, saveMessage } from "../src/store/store.ts"; + +const commitMock = vi.mocked(commitTurnAdvance); +const isTurnExtractedMock = vi.mocked(isTurnExtracted); +const markExtractedMock = vi.mocked(markExtracted); +const saveMessageMock = vi.mocked(saveMessage); + +function fullApi(pluginConfig: Record = {}) { + return { + pluginConfig, + config: {}, + resolvePath: (v: string) => v, + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + registerCli: vi.fn(), + registerTool: vi.fn(), + registerContextEngine: vi.fn(), + registerHttpRoute: vi.fn(), + on: vi.fn(), + } as any; +} + +function buildEngine(pluginConfig: Record = {}) { + const api = fullApi(pluginConfig); + graphMemoryProPlugin.register(api); + const engine = api.registerContextEngine.mock.calls[0][1]() as any; + return { api, engine }; +} + +/** 轮询等待 fire-and-forget 的提取管线真正跑起来。 */ +async function until(cond: () => boolean, ms = 2000): Promise { + const start = Date.now(); + while (!cond()) { + if (Date.now() - start > ms) throw new Error("condition not met in time"); + await new Promise((r) => setTimeout(r, 10)); + } +} + +describe("commitTurn (OpenClaw transcript fencing contract)", () => { + let engine: any; + let api: any; + + afterEach(async () => { + if (engine) { + await engine.dispose(); + engine = null; + } + await closeDriver(); + commitMock.mockReset(); + commitMock.mockResolvedValue("committed"); + isTurnExtractedMock.mockClear(); + markExtractedMock.mockClear(); + saveMessageMock.mockClear(); + }); + + it("declares current-turn transcript fencing semantics in engine.info", () => { + ({ api, engine } = buildEngine()); + expect(engine.info.transcriptSemantics).toEqual({ + currentTurnFence: "before-current-turn-entry-v1", + turnAdvancementIdempotency: "atomic-idempotent-v1", + }); + }); + + it("returns committed on first write and schedules turn extraction", async () => { + ({ api, engine } = buildEngine()); + const res = await engine.commitTurn({ + sessionId: "s-commit-1", + sessionKey: "agent:t:session-1", + advancementKey: "adv-key-1", + messages: [{ role: "user", content: "hi" }, { role: "assistant", content: "yo" }], + }); + expect(res).toEqual({ status: "committed" }); + expect(commitMock).toHaveBeenCalledTimes(1); + expect(commitMock.mock.calls[0][2]).toBe("adv-key-1"); + expect(commitMock.mock.calls[0][3]).toBe(2); + + // 提取是 fire-and-forget，但最终必须进入 isTurnExtracted 幂等检查 + await until(() => isTurnExtractedMock.mock.calls.length > 0); + }); + + it("returns duplicate on host retry without replaying side effects", async () => { + ({ api, engine } = buildEngine()); + commitMock.mockResolvedValueOnce("committed"); + commitMock.mockResolvedValueOnce("duplicate"); + + const first = await engine.commitTurn({ + sessionId: "s-commit-2", advancementKey: "adv-key-2", messages: [{ role: "user", content: "q" }], + }); + const retry = await engine.commitTurn({ + sessionId: "s-commit-2", advancementKey: "adv-key-2", messages: [{ role: "user", content: "q" }], + }); + expect(first).toEqual({ status: "committed" }); + expect(retry).toEqual({ status: "duplicate" }); + expect(commitMock).toHaveBeenCalledTimes(2); + + // duplicate 短路后不得再触发提取：先等首次提交的提取管线完整落地 + // （markExtracted 是管线最后一步），再确认 retry 没有安排新的提取 + await until(() => markExtractedMock.mock.calls.length > 0); + const settledCalls = markExtractedMock.mock.calls.length; + isTurnExtractedMock.mockClear(); + await new Promise((r) => setTimeout(r, 100)); + expect(isTurnExtractedMock).not.toHaveBeenCalled(); + expect(markExtractedMock.mock.calls.length).toBe(settledCalls); + }); + + it("degrades to committed when the marker write fails (non-constraint error)", async () => { + ({ api, engine } = buildEngine()); + commitMock.mockRejectedValueOnce(new Error("pool closed")); + const res = await engine.commitTurn({ + sessionId: "s-commit-3", advancementKey: "adv-key-3", messages: [], + }); + expect(res).toEqual({ status: "committed" }); + expect(api.logger.warn).toHaveBeenCalledWith( + expect.stringContaining("commitTurn marker write failed"), + ); + }); + + it("no-ops for heartbeats and missing advancementKey", async () => { + ({ api, engine } = buildEngine()); + expect(await engine.commitTurn({ sessionId: "s", advancementKey: "k", isHeartbeat: true })) + .toEqual({ status: "committed" }); + expect(await engine.commitTurn({ sessionId: "s", messages: [] })) + .toEqual({ status: "committed" }); + expect(commitMock).not.toHaveBeenCalled(); + }); + + it("skips the marker entirely for cron sessions with cron.enabled=false", async () => { + ({ api, engine } = buildEngine({ cron: { enabled: false } })); + const res = await engine.commitTurn({ + sessionId: "s-cron-1", + sessionKey: "agent:main:cron:job-1", + advancementKey: "adv-cron-1", + messages: [{ role: "user", content: "tick" }], + }); + expect(res).toEqual({ status: "committed" }); + expect(commitMock).not.toHaveBeenCalled(); + expect(isTurnExtractedMock).not.toHaveBeenCalled(); + }); + + it("never persists messages itself — persistence stays with ingest/afterTurn", async () => { + // 设计决策的回归钉子：commitTurn 若也开始落消息，与 afterTurn 并存时 + // 会以新 seq 重写整轮 → GmMessage 重复行 + ({ api, engine } = buildEngine()); + await engine.commitTurn({ + sessionId: "s-nopersist", + advancementKey: "adv-nopersist", + messages: [{ role: "user", content: "hello" }], + }); + await until(() => isTurnExtractedMock.mock.calls.length > 0); + await new Promise((r) => setTimeout(r, 50)); + expect(saveMessageMock).not.toHaveBeenCalled(); + }); + + it("warns and skips marker + extraction when the session cannot be resolved", async () => { + ({ api, engine } = buildEngine()); + const res = await engine.commitTurn({ + advancementKey: "adv-no-session", + messages: [{ role: "user", content: "orphan" }], + }); + expect(res).toEqual({ status: "committed" }); + expect(commitMock).not.toHaveBeenCalled(); + expect(isTurnExtractedMock).not.toHaveBeenCalled(); + expect(api.logger.warn).toHaveBeenCalledWith( + expect.stringContaining("cannot resolve session"), + ); + }); +}); diff --git a/test/community-signature.test.ts b/test/community-signature.test.ts new file mode 100644 index 0000000..d7c8979 --- /dev/null +++ b/test/community-signature.test.ts @@ -0,0 +1,36 @@ +/** + * buildCommunityMemberSignature — 社区成员签名纯逻辑 + * 移植自上游 v1.x "reuse unchanged community summaries"（commit 1fdec04） + */ + +import { describe, it, expect } from "vitest"; +import { buildCommunityMemberSignature } from "../src/graph/community.ts"; + +describe("buildCommunityMemberSignature", () => { + it("成员顺序不影响签名（排序后哈希）", () => { + expect(buildCommunityMemberSignature(["a", "b", "c"])) + .toBe(buildCommunityMemberSignature(["c", "a", "b"])); + }); + + it("相同成员恒生成相同签名", () => { + expect(buildCommunityMemberSignature(["x", "y"])) + .toBe(buildCommunityMemberSignature(["x", "y"])); + }); + + it("成员构成不同则签名不同", () => { + expect(buildCommunityMemberSignature(["a", "b"])) + .not.toBe(buildCommunityMemberSignature(["a", "c"])); + expect(buildCommunityMemberSignature(["a", "b"])) + .not.toBe(buildCommunityMemberSignature(["a", "b", "c"])); + }); + + it("输出为 40 位小写 hex（sha1）", () => { + expect(buildCommunityMemberSignature(["a"])).toMatch(/^[0-9a-f]{40}$/); + }); + + it("不修改入参数组", () => { + const input = ["b", "a"]; + buildCommunityMemberSignature(input); + expect(input).toEqual(["b", "a"]); + }); +}); diff --git a/test/decay.test.ts b/test/decay.test.ts new file mode 100644 index 0000000..2d02fc2 --- /dev/null +++ b/test/decay.test.ts @@ -0,0 +1,344 @@ +import { describe, it, expect } from "vitest"; +import { + normalizeImportance, + computeConfidence, + computeBeta, + scoreRecency, + scoreFrequency, + scoreIntrinsic, + scoreNode, + decideTierTransition, + shouldAutoDeprecate, +} from "../src/graph/decay.ts"; +import { DEFAULT_CONFIG, type DecayConfig, type GmNode } from "../src/types.ts"; + +const cfg: DecayConfig = { ...DEFAULT_CONFIG.decay! }; +const NOW = Date.UTC(2026, 0, 15, 0, 0, 0); +const MS_PER_DAY = 86_400_000; + +function makeNode(overrides: Partial = {}): GmNode { + return { + id: "test-id", + type: "SKILL", + name: "test", + description: "", + content: "", + status: "active", + tier: "working", + validatedCount: 1, + sourceSessions: [], + communityId: null, + pagerank: 0, + createdAt: NOW - 10 * MS_PER_DAY, + updatedAt: NOW - 10 * MS_PER_DAY, + lastAccessedAt: NOW - 10 * MS_PER_DAY, + ...overrides, + }; +} + +describe("normalizeImportance", () => { + it("pagerank=0 时返回 0（即使 maxPagerank>0）", () => { + expect(normalizeImportance(0, 1.0)).toBe(0); + }); + + it("maxPagerank≤0 时返回 0（避免除零）", () => { + expect(normalizeImportance(5, 0)).toBe(0); + expect(normalizeImportance(5, -1)).toBe(0); + }); + + it("pagerank = maxPagerank 时返回 1", () => { + expect(normalizeImportance(0.5, 0.5)).toBe(1); + }); + + it("截断到 [0,1]", () => { + expect(normalizeImportance(2.0, 1.0)).toBe(1); + expect(normalizeImportance(-1, 1.0)).toBe(0); + }); +}); + +describe("computeConfidence", () => { + it("count=0 时 confidence=0", () => { + expect(computeConfidence(0)).toBe(0); + }); + + it("count=1 时 confidence=0.5", () => { + expect(computeConfidence(1)).toBeCloseTo(0.5, 6); + }); + + it("count 增大时饱和收敛到 1（永不达到）", () => { + expect(computeConfidence(10)).toBeLessThan(1); + expect(computeConfidence(100)).toBeLessThan(1); + expect(computeConfidence(100)).toBeGreaterThan(computeConfidence(10)); + }); + + it("负数按 0 处理", () => { + expect(computeConfidence(-5)).toBe(0); + }); +}); + +describe("computeBeta", () => { + it("core < working < peripheral（缓衰 → 促衰）", () => { + expect(computeBeta("core", cfg)).toBe(0.8); + expect(computeBeta("working", cfg)).toBe(1.0); + expect(computeBeta("peripheral", cfg)).toBe(1.3); + }); +}); + +describe("scoreRecency", () => { + it("刚刚访问（daysSince=0）→ 1.0", () => { + const node = makeNode({ lastAccessedAt: NOW }); + expect(scoreRecency(node, 0, NOW, cfg)).toBeCloseTo(1, 6); + }); + + it("importance=0 + working tier + 30 天 → recency ≈ 0.5（半衰期）", () => { + const node = makeNode({ tier: "working", lastAccessedAt: NOW - 30 * MS_PER_DAY }); + expect(scoreRecency(node, 0, NOW, cfg)).toBeCloseTo(0.5, 2); + }); + + it("高 importance 拉长 effectiveHL（衰减更慢）", () => { + const node = makeNode({ tier: "working", lastAccessedAt: NOW - 30 * MS_PER_DAY }); + const highImp = scoreRecency(node, 1.0, NOW, cfg); + const zeroImp = scoreRecency(node, 0, NOW, cfg); + expect(highImp).toBeGreaterThan(zeroImp); + expect(highImp).toBeGreaterThan(0.5); + }); + + it("tier=peripheral 比 tier=working 衰减更快", () => { + const days = 10; + const w = scoreRecency(makeNode({ tier: "working", lastAccessedAt: NOW - days * MS_PER_DAY }), 0, NOW, cfg); + const p = scoreRecency(makeNode({ tier: "peripheral", lastAccessedAt: NOW - days * MS_PER_DAY }), 0, NOW, cfg); + expect(p).toBeLessThan(w); + }); + + it("lastAccessedAt 缺失时回退到 updatedAt", () => { + const viaFallback = makeNode({ lastAccessedAt: 0, updatedAt: NOW - 5 * MS_PER_DAY }); + const direct = makeNode({ lastAccessedAt: NOW - 5 * MS_PER_DAY }); + expect(scoreRecency(viaFallback, 0, NOW, cfg)) + .toBeCloseTo(scoreRecency(direct, 0, NOW, cfg), 6); + }); +}); + +describe("scoreFrequency", () => { + it("count=0 时 base=0", () => { + expect(scoreFrequency(makeNode({ validatedCount: 0 }))).toBe(0); + }); + + it("count=1 时只返回 base（无 recentnessBonus）", () => { + const expected = 1 - Math.exp(-1 / 5); + expect(scoreFrequency(makeNode({ validatedCount: 1 }))).toBeCloseTo(expected, 6); + }); + + it("count > 1 时 base × (0.5 + 0.5*recentnessBonus)，结果 ≤ base", () => { + const node = makeNode({ + validatedCount: 3, + createdAt: NOW - 30 * MS_PER_DAY, + lastAccessedAt: NOW, + }); + const base = 1 - Math.exp(-3 / 5); + const score = scoreFrequency(node); + expect(score).toBeLessThanOrEqual(base); + expect(score).toBeGreaterThan(0); + }); + + it("访问越紧凑（avgGapDays 越小）recentnessBonus 越大", () => { + const tight = makeNode({ + validatedCount: 5, + createdAt: NOW - 4 * MS_PER_DAY, + lastAccessedAt: NOW, + }); + const sparse = makeNode({ + validatedCount: 5, + createdAt: NOW - 100 * MS_PER_DAY, + lastAccessedAt: NOW, + }); + expect(scoreFrequency(tight)).toBeGreaterThan(scoreFrequency(sparse)); + }); +}); + +describe("scoreIntrinsic", () => { + it("= importance × confidence", () => { + expect(scoreIntrinsic(0.5, 0.5)).toBeCloseTo(0.25, 6); + expect(scoreIntrinsic(1, 1)).toBe(1); + expect(scoreIntrinsic(0, 0.5)).toBe(0); + }); +}); + +describe("scoreNode", () => { + it("权重和为 1 时 composite 落在 [0,1]", () => { + const node = makeNode({ pagerank: 0.5, validatedCount: 5, lastAccessedAt: NOW }); + const r = scoreNode(node, 1.0, NOW, cfg); + expect(r.composite).toBeGreaterThanOrEqual(0); + expect(r.composite).toBeLessThanOrEqual(1); + }); + + it("新鲜高 PR 节点 composite 显著高于陈旧低 PR 节点", () => { + const fresh = makeNode({ pagerank: 1.0, validatedCount: 1, lastAccessedAt: NOW }); + const stale = makeNode({ + pagerank: 0, + validatedCount: 1, + lastAccessedAt: NOW - 90 * MS_PER_DAY, + }); + expect(scoreNode(fresh, 1.0, NOW, cfg).composite) + .toBeGreaterThan(scoreNode(stale, 1.0, NOW, cfg).composite); + }); + + it("权重和≠1 时自动归一化，composite 仍落在 [0,1]", () => { + const skewedCfg: DecayConfig = { + ...cfg, + recencyWeight: 0.5, + frequencyWeight: 0.5, + intrinsicWeight: 0.5, // 和=1.5 + }; + const node = makeNode({ + pagerank: 1.0, + validatedCount: 10, + lastAccessedAt: NOW, + updatedAt: NOW, + createdAt: NOW, + }); + const r = scoreNode(node, 1.0, NOW, skewedCfg); + expect(r.composite).toBeLessThanOrEqual(1); + expect(r.composite).toBeGreaterThanOrEqual(0); + }); + + it("权重和为 0 时回退到等权重，不抛错", () => { + const zeroCfg: DecayConfig = { + ...cfg, + recencyWeight: 0, + frequencyWeight: 0, + intrinsicWeight: 0, + }; + const node = makeNode({ pagerank: 0.5, validatedCount: 1, lastAccessedAt: NOW }); + const r = scoreNode(node, 1.0, NOW, zeroCfg); + expect(Number.isFinite(r.composite)).toBe(true); + }); +}); + +describe("decideTierTransition", () => { + const scoreLow = { composite: 0.1, recency: 0, frequency: 0, intrinsic: 0 }; + const scoreHigh = { composite: 0.9, recency: 0.9, frequency: 0.9, intrinsic: 0.9 }; + const scoreMid = { composite: 0.5, recency: 0.5, frequency: 0.5, intrinsic: 0 }; + + it("core + composite 低 + count 低 → working", () => { + const node = makeNode({ tier: "core", validatedCount: 1 }); + expect(decideTierTransition(node, scoreLow, 0, cfg, NOW)).toBe("working"); + }); + + it("core + composite 高 → 保持 core", () => { + const node = makeNode({ tier: "core", validatedCount: 20 }); + expect(decideTierTransition(node, scoreHigh, 0.9, cfg, NOW)).toBeNull(); + }); + + it("working + composite < pct → peripheral", () => { + const node = makeNode({ tier: "working", validatedCount: 1 }); + expect(decideTierTransition(node, scoreLow, 0, cfg, NOW)).toBe("peripheral"); + }); + + it("working + 陈旧（age > peripheralAgeDays）+ count 低 → peripheral", () => { + const node = makeNode({ + tier: "working", + validatedCount: 1, + createdAt: NOW - (cfg.peripheralAgeDays + 1) * MS_PER_DAY, + }); + expect(decideTierTransition(node, scoreMid, 0, cfg, NOW)).toBe("peripheral"); + }); + + it("working + 陈旧但 count 充足 → 保持 working", () => { + const node = makeNode({ + tier: "working", + validatedCount: 5, + createdAt: NOW - (cfg.peripheralAgeDays + 1) * MS_PER_DAY, + }); + expect(decideTierTransition(node, scoreMid, 0, cfg, NOW)).toBeNull(); + }); + + it("peripheral + count 充足 + composite 高 → working", () => { + const node = makeNode({ tier: "peripheral", validatedCount: 5 }); + expect(decideTierTransition(node, scoreMid, 0, cfg, NOW)).toBe("working"); + }); + + it("peripheral + count 不足 → 保持 peripheral", () => { + const node = makeNode({ tier: "peripheral", validatedCount: 1 }); + expect(decideTierTransition(node, scoreHigh, 0, cfg, NOW)).toBeNull(); + }); + + it("working + count + composite + importance 都高 → core", () => { + const node = makeNode({ tier: "working", validatedCount: 15 }); + expect(decideTierTransition(node, scoreHigh, 0.9, cfg, NOW)).toBe("core"); + }); + + it("working + count + composite 高但 importance 不足 → 保持 working", () => { + const node = makeNode({ tier: "working", validatedCount: 15 }); + expect(decideTierTransition(node, scoreHigh, 0.5, cfg, NOW)).toBeNull(); + }); + + it("tier undefined 按 working 处理", () => { + const node = makeNode({ tier: undefined as unknown as GmNode["tier"], validatedCount: 1 }); + expect(decideTierTransition(node, scoreLow, 0, cfg, NOW)).toBe("peripheral"); + }); +}); + +describe("shouldAutoDeprecate", () => { + // peripheralCompositeThreshold=0.15：低分 0.1 / 达标 0.5 + const scoreLow = { composite: 0.1, recency: 0, frequency: 0, intrinsic: 0 }; + const scoreHigh = { composite: 0.5, recency: 0.5, frequency: 0.5, intrinsic: 0.5 }; + + it("peripheral + 低分 + 超期未访问 → true（三条件齐备）", () => { + const node = makeNode({ tier: "peripheral", lastAccessedAt: NOW - 40 * MS_PER_DAY }); + expect(shouldAutoDeprecate(node, scoreLow, cfg, NOW)).toBe(true); + }); + + it("working 层 → false（遗忘曲线 tier 转换先行）", () => { + const node = makeNode({ tier: "working", lastAccessedAt: NOW - 40 * MS_PER_DAY }); + expect(shouldAutoDeprecate(node, scoreLow, cfg, NOW)).toBe(false); + }); + + it("core 层 → false", () => { + const node = makeNode({ tier: "core", lastAccessedAt: NOW - 40 * MS_PER_DAY }); + expect(shouldAutoDeprecate(node, scoreLow, cfg, NOW)).toBe(false); + }); + + it("composite 达标 → false（高 intrinsic 价值节点受保护）", () => { + const node = makeNode({ tier: "peripheral", lastAccessedAt: NOW - 40 * MS_PER_DAY }); + expect(shouldAutoDeprecate(node, scoreHigh, cfg, NOW)).toBe(false); + }); + + it("composite 恰等于阈值 → false（边界取保留）", () => { + const scoreAtThreshold = { composite: cfg.peripheralCompositeThreshold, recency: 0, frequency: 0, intrinsic: 0 }; + const node = makeNode({ tier: "peripheral", lastAccessedAt: NOW - 40 * MS_PER_DAY }); + expect(shouldAutoDeprecate(node, scoreAtThreshold, cfg, NOW)).toBe(false); + }); + + it("未超 autoDeprecateAfterDays → false", () => { + const node = makeNode({ tier: "peripheral", lastAccessedAt: NOW - 10 * MS_PER_DAY }); + expect(shouldAutoDeprecate(node, scoreLow, cfg, NOW)).toBe(false); + }); + + it("恰好达到天数门槛 → true（>= 含端点）", () => { + const node = makeNode({ tier: "peripheral", lastAccessedAt: NOW - cfg.autoDeprecateAfterDays * MS_PER_DAY }); + expect(shouldAutoDeprecate(node, scoreLow, cfg, NOW)).toBe(true); + }); + + it("autoDeprecate=false 恒 false（功能开关）", () => { + const offCfg: DecayConfig = { ...cfg, autoDeprecate: false }; + const node = makeNode({ tier: "peripheral", lastAccessedAt: NOW - 400 * MS_PER_DAY }); + expect(shouldAutoDeprecate(node, scoreLow, offCfg, NOW)).toBe(false); + }); + + it("lastAccessedAt 缺失时回退 updatedAt（与 decay 计时基准一致）", () => { + const viaFallback = makeNode({ + tier: "peripheral", + lastAccessedAt: 0, + updatedAt: NOW - (cfg.autoDeprecateAfterDays + 5) * MS_PER_DAY, + }); + expect(shouldAutoDeprecate(viaFallback, scoreLow, cfg, NOW)).toBe(true); + }); + + it("tier undefined 按 working 处理 → false", () => { + const node = makeNode({ + tier: undefined as unknown as GmNode["tier"], + lastAccessedAt: NOW - 400 * MS_PER_DAY, + }); + expect(shouldAutoDeprecate(node, scoreLow, cfg, NOW)).toBe(false); + }); +}); diff --git a/test/extract-cost-guards.test.ts b/test/extract-cost-guards.test.ts new file mode 100644 index 0000000..4f1ad6f --- /dev/null +++ b/test/extract-cost-guards.test.ts @@ -0,0 +1,82 @@ +/** + * finalize 阶梯触发 + 社区摘要 top-k 稳定签名 单测（LLM 成本控制） + */ + +import { describe, it, expect } from "vitest"; +import { shouldRunFinalize } from "../src/extractor/extract.ts"; +import { + buildCommunityMemberSignature, buildTopKMemberSignature, COMMUNITY_SIGNATURE_TOP_K, +} from "../src/graph/community.ts"; + +describe("shouldRunFinalize — finalize 阶梯触发", () => { + it("空会话 / ≤2 节点 → 跳过", () => { + expect(shouldRunFinalize([])).toBe(false); + expect(shouldRunFinalize([{ type: "TASK" }])).toBe(false); + expect(shouldRunFinalize([{ type: "TASK" }, { type: "SKILL" }])).toBe(false); + }); + + it("无 EVENT 节点 → 跳过（promotedSkills 无提升对象）", () => { + expect(shouldRunFinalize([ + { type: "TASK" }, { type: "SKILL" }, { type: "SKILL" }, + ])).toBe(false); + }); + + it("≥3 节点且含 EVENT → 触发", () => { + expect(shouldRunFinalize([ + { type: "TASK" }, { type: "SKILL" }, { type: "EVENT" }, + ])).toBe(true); + }); +}); + +describe("buildTopKMemberSignature — top-k 稳定签名", () => { + const members = [ + { id: "a", validatedCount: 10 }, + { id: "b", validatedCount: 8 }, + { id: "c", validatedCount: 7 }, + { id: "d", validatedCount: 2 }, // 低频成员 + { id: "e", validatedCount: 1 }, // 低频成员 + ]; + + it("低频成员进出不改变签名（社区边界抖动免疫）", () => { + // k=3：签名由 validatedCount 前三名（a/b/c）决定 + const base = buildTopKMemberSignature(members, 3); + const withNoise = buildTopKMemberSignature([ + ...members, + { id: "f", validatedCount: 1 }, + ], 3); + const withoutLow = buildTopKMemberSignature(members.slice(0, 3), 3); + expect(withNoise).toBe(base); + expect(withoutLow).toBe(base); + }); + + it("高价值成员变化 → 签名变化（语义主体变了要重摘要）", () => { + const base = buildTopKMemberSignature(members, 3); + const promoted = buildTopKMemberSignature([ + ...members, + { id: "g", validatedCount: 50 }, + ], 3); + expect(promoted).not.toBe(base); + }); + + it("成员数 ≤ k 时与全量签名一致（退化为旧语义）", () => { + const small = [ + { id: "x", validatedCount: 3 }, + { id: "y", validatedCount: 5 }, + ]; + const topk = buildTopKMemberSignature(small, COMMUNITY_SIGNATURE_TOP_K); + const full = buildCommunityMemberSignature(small.map((m) => m.id)); + expect(topk).toBe(full); + }); + + it("validatedCount 并列时按 id 字典序决胜负（确定性）", () => { + const s1 = buildTopKMemberSignature([ + { id: "a", validatedCount: 5 }, + { id: "b", validatedCount: 5 }, + ]); + const s2 = buildTopKMemberSignature([ + { id: "b", validatedCount: 5 }, + { id: "a", validatedCount: 5 }, + ]); + expect(s1).toBe(s2); + }); +}); diff --git a/test/http-retry.test.ts b/test/http-retry.test.ts new file mode 100644 index 0000000..1f9ecfb --- /dev/null +++ b/test/http-retry.test.ts @@ -0,0 +1,143 @@ +/** + * graph-memory — src/engine/http.ts 单元测试 + * + * 覆盖统一的超时/重试语义： + * - 可重试状态码（429/5xx）指数退避重试 + * - 网络异常重试（llm.ts 侧曾在此丢失，回归保护） + * - 超时默认立即抛出（HttpTimeoutError），retryOnTimeout=true 时计入重试 + * - 不可重试状态码（401 等）立即返回响应 + */ + +import { describe, it, expect, vi, afterEach } from "vitest"; +import { fetchRetry, HttpTimeoutError } from "../src/engine/http.ts"; + +const NO_BACKOFF = { backoffMs: () => 0 }; + +function jsonResponse(status: number): Response { + return new Response(JSON.stringify({ ok: false }), { status }); +} + +/** 永远挂起、仅在 abort 信号触发时以 AbortError 拒绝的 mock fetch（模拟超时） */ +function hangingFetch() { + return vi.fn((_url: string, init?: RequestInit) => + new Promise((_, reject) => { + init?.signal?.addEventListener("abort", () => { + const err = new Error("The operation was aborted"); + err.name = "AbortError"; + reject(err); + }); + }), + ); +} + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("fetchRetry 状态码重试", () => { + it("429 两次后 200：重试并成功，共请求 3 次", async () => { + const mock = vi.fn() + .mockResolvedValueOnce(jsonResponse(429)) + .mockResolvedValueOnce(jsonResponse(429)) + .mockResolvedValueOnce(new Response("{}", { status: 200 })); + vi.stubGlobal("fetch", mock); + + const res = await fetchRetry("https://x.test/v1", { method: "POST" }, NO_BACKOFF); + + expect(res.ok).toBe(true); + expect(mock).toHaveBeenCalledTimes(3); + }); + + it("持续 500：重试耗尽后返回最后一次响应（由调用方产出带状态码的报错）", async () => { + const mock = vi.fn().mockResolvedValue(jsonResponse(500)); + vi.stubGlobal("fetch", mock); + + const res = await fetchRetry("https://x.test/v1", { method: "POST" }, NO_BACKOFF); + + expect(res.status).toBe(500); + expect(mock).toHaveBeenCalledTimes(4); // 首次 + 3 次重试 + }); + + it("401 不可重试：立即返回，仅请求 1 次", async () => { + const mock = vi.fn().mockResolvedValue(jsonResponse(401)); + vi.stubGlobal("fetch", mock); + + const res = await fetchRetry("https://x.test/v1", { method: "POST" }, NO_BACKOFF); + + expect(res.status).toBe(401); + expect(mock).toHaveBeenCalledTimes(1); + }); +}); + +describe("fetchRetry 网络异常重试（llm.ts 侧曾丢失的行为）", () => { + it("网络错误后成功：重试生效", async () => { + const err = new Error("socket hang up"); + const mock = vi.fn() + .mockRejectedValueOnce(err) + .mockResolvedValueOnce(new Response("{}", { status: 200 })); + vi.stubGlobal("fetch", mock); + + const res = await fetchRetry("https://x.test/v1", { method: "POST" }, NO_BACKOFF); + + expect(res.ok).toBe(true); + expect(mock).toHaveBeenCalledTimes(2); + }); + + it("持续网络错误：重试耗尽后抛出最后一次错误", async () => { + const mock = vi.fn().mockRejectedValue(new Error("ECONNRESET")); + vi.stubGlobal("fetch", mock); + + await expect( + fetchRetry("https://x.test/v1", { method: "POST" }, NO_BACKOFF), + ).rejects.toThrow("ECONNRESET"); + expect(mock).toHaveBeenCalledTimes(4); + }); +}); + +describe("fetchRetry 超时语义", () => { + it("默认超时不重试：立即抛 HttpTimeoutError，仅请求 1 次", async () => { + const mock = hangingFetch(); + vi.stubGlobal("fetch", mock); + + await expect( + fetchRetry("https://x.test/v1", { method: "POST" }, { timeoutMs: 30, ...NO_BACKOFF }), + ).rejects.toBeInstanceOf(HttpTimeoutError); + expect(mock).toHaveBeenCalledTimes(1); + }); + + it("retryOnTimeout=true：超时计入重试，第 2 次成功", async () => { + const mock = vi.fn() + .mockImplementationOnce((_url: string, init?: RequestInit) => + new Promise((_, reject) => { + init?.signal?.addEventListener("abort", () => { + const err = new Error("aborted"); + err.name = "AbortError"; + reject(err); + }); + }), + ) + .mockResolvedValueOnce(new Response("{}", { status: 200 })); + vi.stubGlobal("fetch", mock); + + const res = await fetchRetry( + "https://x.test/v1", + { method: "POST" }, + { timeoutMs: 30, retryOnTimeout: true, ...NO_BACKOFF }, + ); + + expect(res.ok).toBe(true); + expect(mock).toHaveBeenCalledTimes(2); + }); + + it("超时错误信息包含 label 与时长", async () => { + vi.stubGlobal("fetch", hangingFetch()); + + await expect( + fetchRetry( + "https://x.test/v1", + { method: "POST" }, + { timeoutMs: 25, label: "[graph-memory] LLM", ...NO_BACKOFF }, + ), + ).rejects.toThrow("[graph-memory] LLM request timed out after 25ms"); + }); +}); diff --git a/test/integration.assemble.test.ts b/test/integration.assemble.test.ts index 594458e..705985b 100644 --- a/test/integration.assemble.test.ts +++ b/test/integration.assemble.test.ts @@ -24,12 +24,14 @@ function makeNode(over: Partial): GmNode { description: over.description ?? "desc", content: over.content ?? "content body", status: over.status ?? "active", + tier: over.tier ?? "working", validatedCount: over.validatedCount ?? 1, sourceSessions: over.sourceSessions ?? ["s1"], communityId: over.communityId ?? null, pagerank: over.pagerank ?? 0, createdAt: over.createdAt ?? Date.now(), updatedAt: over.updatedAt ?? Date.now(), + lastAccessedAt: over.lastAccessedAt ?? Date.now(), }; } diff --git a/test/integration.embed-e2e.test.ts b/test/integration.embed-e2e.test.ts new file mode 100644 index 0000000..643f6ca --- /dev/null +++ b/test/integration.embed-e2e.test.ts @@ -0,0 +1,147 @@ +/** + * 真实 embedding 服务端到端测试（syncEmbedBatch / 批量 API / 查询 LRU / 语义召回） + * + * 与其他集成测试的 mock 向量不同，本文件打真实 embedding 端点（默认 + * http://localhost:8000/v1，jina-embeddings-v5-text-small，1024 维）， + * 验证：探活、批量嵌入、contentHash 短路、查询向量 LRU、真实语义召回排序。 + * + * 安全门：需要显式 EMBED_E2E=1 + NEO4J_INTEGRATION=1 + NEO4J_TEST_URI（不回落 7687）。 + * + * 运行： + * EMBED_E2E=1 NEO4J_INTEGRATION=1 NEO4J_TEST_URI=bolt://localhost:7688 \ + * npx vitest run test/integration.embed-e2e.test.ts + */ + +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import type { Driver } from "neo4j-driver"; +import { getDriver, initSchema, closeDriver, getSession } from "../src/store/db.ts"; +import { upsertNode } from "../src/store/store.ts"; +import { createEmbedder } from "../src/engine/embed.ts"; +import { Recaller } from "../src/recaller/recall.ts"; +import { DEFAULT_CONFIG, type GmConfig } from "../src/types.ts"; + +const ENABLED = !!process.env.EMBED_E2E && !!process.env.NEO4J_INTEGRATION && !!process.env.NEO4J_TEST_URI; +const NEO4J_URI = process.env.NEO4J_TEST_URI ?? ""; +const EMBED_BASE = process.env.EMBED_E2E_BASE ?? "http://localhost:8000/v1"; +const EMBED_MODEL = process.env.EMBED_E2E_MODEL ?? "jina-embeddings-v5-text-small"; + +let driver: Driver; +const SID = `embed-e2e-${Date.now()}`; +const cfg: GmConfig = { ...DEFAULT_CONFIG, neo4j: { uri: NEO4J_URI, user: "neo4j", password: "graphmemory" } }; + +describe.skipIf(!ENABLED)("embedding E2E（真实服务）", () => { + let embedCalls = 0; + let batchCalls = 0; + let batchTexts = 0; + + beforeAll(async () => { + console.log(`[embed-e2e] neo4j=${NEO4J_URI} embed=${EMBED_BASE} model=${EMBED_MODEL}`); + driver = getDriver(cfg.neo4j); + await initSchema(driver, { baseURL: EMBED_BASE, model: EMBED_MODEL }); + }, 120_000); + + afterAll(async () => { + const session = getSession(driver); + try { + await session.run("MATCH (n) WHERE $sid IN n.sourceSessions DETACH DELETE n", { sid: SID }); + } finally { + await session.close(); + } + await closeDriver(); + }, 60_000); + + it("探活：createEmbedder probe 成功，双能力（embed/embedBatch）可用", async () => { + const embedder = await createEmbedder({ baseURL: EMBED_BASE, model: EMBED_MODEL }); + expect(embedder).not.toBeNull(); + const vec = await embedder!.embed("probe 维度", "query"); + expect(vec.length).toBeGreaterThan(0); + console.log(`[embed-e2e] 单发维度 = ${vec.length}`); + }, 60_000); + + it("syncEmbedBatch：真实批量 API 一次往返写回 N 个节点向量 + contentHash 短路", async () => { + const embedder = await createEmbedder({ baseURL: EMBED_BASE, model: EMBED_MODEL }); + expect(embedder).not.toBeNull(); + + // 包一层计数器，验证 LRU / 短路语义 + const countedEmbed = embedder!.embed; + const countedBatch = embedder!.embedBatch; + + const recaller = new Recaller(driver, cfg); + recaller.setEmbedFn(async (text, mode) => { + embedCalls++; + return countedEmbed(text, mode); + }); + recaller.setEmbedBatchFn(async (texts, mode) => { + batchCalls++; + batchTexts += texts.length; + return countedBatch(texts, mode); + }); + + const topics = [ + { name: "embed-e2e-docker-compose", description: "docker compose container deployment", content: "deploy multi-container applications with docker compose files and registry images" }, + { name: "embed-e2e-vitest-mocking", description: "vitest mock and spy patterns", content: "vi.mock hoists module mocks, vi.spyOn wraps methods, stub network responses in unit tests" }, + { name: "embed-e2e-neo4j-index", description: "neo4j index and constraint design", content: "uniqueness constraints guard identity keys, range indexes accelerate equality cypher filters" }, + ]; + const nodes = []; + for (const t of topics) { + const { node } = await upsertNode(driver, { type: "SKILL", ...t }, SID); + nodes.push(node); + } + + // 首次批量同步：一次 batch API 调用覆盖 3 个节点 + await recaller.syncEmbedBatch(nodes); + expect(batchCalls).toBe(1); + expect(batchTexts).toBe(3); + + // 落库校验：embedding 维度 + contentHash + const session = getSession(driver); + try { + const r = await session.run( + `MATCH (n:Skill) WHERE $sid IN n.sourceSessions + RETURN n.id AS id, size(n.embedding) AS dims, n.contentHash AS hash`, + { sid: SID }, + ); + expect(r.records.length).toBe(3); + for (const rec of r.records) { + // size() 返回 Neo4j Integer + expect((rec.get("dims")?.toNumber?.() ?? rec.get("dims"))).toBe(1024); + expect(rec.get("hash")).toBeTruthy(); + } + } finally { + await session.close(); + } + + // contentHash 短路：内容未变 → 零新 API 调用 + await recaller.syncEmbedBatch(nodes); + expect(batchCalls).toBe(1); + expect(batchTexts).toBe(3); + }, 120_000); + + it("真实语义召回 + 查询向量 LRU", async () => { + const embedder = await createEmbedder({ baseURL: EMBED_BASE, model: EMBED_MODEL }); + const countedEmbed = embedder!.embed; + let queryCalls = 0; + + const recaller = new Recaller(driver, cfg); + recaller.setEmbedFn(async (text, mode) => { + queryCalls++; + return countedEmbed(text, mode); + }); + + // 语义查询（非字面词重叠：orchestration ≠ compose/deployment） + const r1 = await recaller.recall("how to orchestrate multi container apps"); + expect(r1.nodes.length).toBeGreaterThan(0); + const firstCallCount = queryCalls; + expect(firstCallCount).toBe(1); + console.log(`[embed-e2e] 语义召回 top1 = ${r1.nodes[0]?.name}`); + + // 相同查询：LRU 命中，零新 embedding 调用 + const r2 = await recaller.recall("how to orchestrate multi container apps"); + expect(queryCalls).toBe(firstCallCount); + expect(r2.nodes.map((n) => n.id).sort()).toEqual(r1.nodes.map((n) => n.id).sort()); + + // 不同查询：新 embedding 调用 + await recaller.recall("unit testing with mocks"); + expect(queryCalls).toBe(firstCallCount + 1); + }, 120_000); +}); diff --git a/test/integration.graph.test.ts b/test/integration.graph.test.ts index 9dc2e90..4c957b4 100644 --- a/test/integration.graph.test.ts +++ b/test/integration.graph.test.ts @@ -12,12 +12,14 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import type { Driver } from "neo4j-driver"; import { getDriver, initSchema, closeDriver, getSession } from "../src/store/db.ts"; import { - upsertNode, upsertEdge, saveVector, findById, deprecate, + upsertNode, upsertEdge, saveVector, findById, deprecateNodeAndDisconnectById, getCommunitySummary, } from "../src/store/store.ts"; import { personalizedPageRank, computeGlobalPageRank, } from "../src/graph/pagerank.ts"; -import { detectCommunities, getCommunityPeers } from "../src/graph/community.ts"; +import { + detectCommunities, getCommunityPeers, summarizeCommunities, buildCommunityMemberSignature, +} from "../src/graph/community.ts"; import { detectDuplicates, dedup } from "../src/graph/dedup.ts"; import { runMaintenance } from "../src/graph/maintenance.ts"; import { DEFAULT_CONFIG, type GmConfig } from "../src/types.ts"; @@ -144,7 +146,7 @@ describe.skipIf(!ENABLED)("graph layer integration (GDS, Docker)", () => { type: "SKILL", name: "Deprecated Pagerank Sentinel", description: "deprecated", content: "deprecated", }, TEST_SID); - await deprecate(driver, node.id); + await deprecateNodeAndDisconnectById(driver, node.id); const session = getSession(driver); try { await session.run( @@ -187,6 +189,74 @@ describe.skipIf(!ENABLED)("graph layer integration (GDS, Docker)", () => { } }); + it("summarizeCommunities：社区成员未变时复用摘要，不重调 LLM", async () => { + const memberIds = [nodeIds["gmpsrc-deploy"], nodeIds["gmpsrc-compose"]]; + + // 生产不变量：detectCommunities 会先给成员节点写入 communityId， + // pruneCommunitySummaries 只保留仍被 active 成员引用的社区 — 不先 SET 会被 prune 删掉 + const prepare = getSession(driver); + try { + await prepare.run( + "MATCH (n:MemoryNode) WHERE n.id IN $ids SET n.communityId = $cid", + { ids: memberIds, cid: "c-reuse-test" }, + ); + } finally { + await prepare.close(); + } + + let llmCalls = 0; + const llm = async () => { + llmCalls += 1; + return "容器部署与编排技能"; + }; + + const first = await summarizeCommunities(driver, new Map([["c-reuse-test", memberIds]]), llm); + const second = await summarizeCommunities(driver, new Map([["c-reuse-test", memberIds]]), llm); + + expect(first).toBe(1); + expect(second).toBe(0); + expect(llmCalls).toBe(1); + + const summary = await getCommunitySummary(driver, "c-reuse-test"); + expect(summary?.summary).toBe("容器部署与编排技能"); + expect(summary?.memberSignature).toBe(buildCommunityMemberSignature(memberIds)); + + // detectCommunities 每轮按成员数重编号（c-1..c-N），ID 变但成员相同 → 按签名跨社区复用。 + // 生产链路里 updateCommunities 会先把成员 communityId 改写到新 id 再进 summarize —— + // 这里同样 SET 成员指向新 id（保持生产不变量），旧 id 成为"无人引用"的捐赠者， + // 复用查找发生在 prune 之前，捐赠者复制完摘要后才被 prune 清理。 + const renumber = getSession(driver); + try { + await renumber.run( + "MATCH (n:MemoryNode) WHERE n.id IN $ids SET n.communityId = $cid", + { ids: memberIds, cid: "c-reuse-renumbered" }, + ); + } finally { + await renumber.close(); + } + const third = await summarizeCommunities( + driver, new Map([["c-reuse-renumbered", memberIds]]), llm, + ); + expect(third).toBe(0); + expect(llmCalls).toBe(1); + const renumbered = await getCommunitySummary(driver, "c-reuse-renumbered"); + expect(renumbered?.summary).toBe("容器部署与编排技能"); + expect(renumbered?.memberSignature).toBe(buildCommunityMemberSignature(memberIds)); + + const cleanup = getSession(driver); + try { + await cleanup.run( + "MATCH (c:Community) WHERE c.id IN ['c-reuse-test', 'c-reuse-renumbered'] DELETE c", + ); + await cleanup.run( + "MATCH (n:MemoryNode) WHERE n.id IN $ids SET n.communityId = null", + { ids: memberIds }, + ); + } finally { + await cleanup.close(); + } + }); + it("detectDuplicates：gmpsrc-* 无 embedding，函数不抛错", async () => { let passed = false; await expectDimSafe(async () => { diff --git a/test/integration.neo4j.test.ts b/test/integration.neo4j.test.ts index 3b12b2c..f8fae0b 100644 --- a/test/integration.neo4j.test.ts +++ b/test/integration.neo4j.test.ts @@ -2,16 +2,22 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import type { Driver } from "neo4j-driver"; import graphMemoryProPlugin from "../index.ts"; import { getDriver, initSchema, closeDriver, getSession } from "../src/store/db.ts"; +import { buildNodeEmbeddingText } from "../src/recaller/recall.ts"; import { upsertNode, findByName, findById, updateNode, upsertEdge, edgesFrom, edgesTo, graphWalk, saveMessage, getUnextracted, markExtracted, isTurnExtracted, - deprecate, getStats, mergeNodes, searchNodes, topNodes, + getStats, mergeNodes, searchNodes, topNodes, getBySession, saveVector, vectorSearchWithScore, getVectorHash, - updateCommunities, updatePageranks, - deleteNode, deprecateNodeAndDisconnect, + updateCommunities, + deprecateNodeAndDisconnect, deprecateNodeAndDisconnectById, deleteEdges, + clearAllEmbeddings, listNodeEmbeddingTargets, listCommunityEmbeddingTargets, + saveCommunityEmbedding, getVectorIndexDimensions, + autoDeprecateNodes, purgeDeprecatedNodes, } from "../src/store/store.ts"; +import { applyDecay } from "../src/graph/decay.ts"; +import { DEFAULT_CONFIG } from "../src/types.ts"; // 仅在 NEO4J_INTEGRATION=1 时运行，避免污染默认 npm test（需要 Docker Neo4j） const ENABLED = !!process.env.NEO4J_INTEGRATION; @@ -178,7 +184,7 @@ describe.skipIf(!ENABLED)("Neo4j integration (Docker)", () => { expect(edges.some(e => e.type === "USED_SKILL")).toBe(true); }); - it("graphWalk 不穿过 deprecated 中间节点连接两个 active 节点", async () => { + it("graphWalk 不穿过 deprecated 中间节点（弃用同时断联）", async () => { const { node: start } = await upsertNode(driver, { type: "SKILL", name: "Active Walk Start", description: "start", content: "start", }, TEST_SID); @@ -196,7 +202,10 @@ describe.skipIf(!ENABLED)("Neo4j integration (Docker)", () => { fromId: deprecatedBridge.id, toId: unreachable.id, type: "REQUIRES", instruction: "second hop", sessionId: TEST_SID, }); - await deprecate(driver, deprecatedBridge.id); + // 手动弃用（按 id）——同时切断两侧边 + await deprecateNodeAndDisconnectById(driver, deprecatedBridge.id); + expect(await edgesTo(driver, deprecatedBridge.id)).toHaveLength(0); + expect(await edgesFrom(driver, deprecatedBridge.id)).toHaveLength(0); const { nodes } = await graphWalk(driver, [start.id], 2); @@ -281,42 +290,26 @@ describe.skipIf(!ENABLED)("Neo4j integration (Docker)", () => { expect(stats.byType.SKILL).toBeGreaterThanOrEqual(1); }); - it("deprecate 软删除（status=deprecated，节点仍存在）", async () => { + it("deprecateNodeAndDisconnectById 手动弃用：断联 + 前缀 + manual 标记（finalize invalidations / REST DELETE 路径）", async () => { const { node } = await upsertNode(driver, { type: "EVENT", name: "Temp Event", description: "d", content: "c", }, TEST_SID); - await deprecate(driver, node.id); + const result = await deprecateNodeAndDisconnectById(driver, node.id); + expect(result).not.toBeNull(); + expect(result!.status).toBe("deprecated"); + expect(result!.deprecatedBy).toBe("manual"); + expect(result!.description).toBe("[DEPRECATED] d"); + const refetch = await findById(driver, node.id); expect(refetch).not.toBeNull(); expect(refetch!.status).toBe("deprecated"); - }); - - it("deleteNode 硬删除：节点 + 所有关系一并消失（gm_update mode=delete）", async () => { - const { node: task } = await upsertNode(driver, { - type: "TASK", name: "HardDelete Task", description: "victim", content: "victim", - }, TEST_SID); - const { node: skill } = await upsertNode(driver, { - type: "SKILL", name: "HardDelete Skill", description: "neighbor", content: "neighbor", - }, TEST_SID); - await upsertEdge(driver, { - fromId: task.id, toId: skill.id, type: "USED_SKILL", - instruction: "uses", sessionId: TEST_SID, - }); - - const deleted = await deleteNode(driver, task.name); - expect(deleted).not.toBeNull(); - expect(deleted!.id).toBe(task.id); - - expect(await findById(driver, task.id)).toBeNull(); + expect(await deprecateNodeAndDisconnectById(driver, "ghost-node-deprecate-id-xyz")).toBeNull(); - const remainingOut = await edgesFrom(driver, skill.id); - expect(remainingOut.filter(e => e.id === task.id)).toHaveLength(0); - const remainingIn = await edgesTo(driver, skill.id); - expect(remainingIn.filter(e => e.fromId === task.id)).toHaveLength(0); - }); - - it("deleteNode 未知 name 返回 null", async () => { - expect(await deleteNode(driver, "ghost-node-delete-xyz")).toBeNull(); + // 重复弃用保留原 deprecatedAt（不重置 purge 时钟） + const firstAt = refetch!.deprecatedAt!; + const again = await deprecateNodeAndDisconnectById(driver, node.id, firstAt + 86_400_000); + expect(again!.deprecatedAt).toBe(firstAt); + expect((await findById(driver, node.id))!.deprecatedAt).toBe(firstAt); }); it("deprecateNodeAndDisconnect 标记 [DEPRECATED] + 切边（节点保留，gm_update mode=deprecate）", async () => { @@ -520,11 +513,16 @@ describe.skipIf(!ENABLED)("Neo4j integration (Docker)", () => { expect(hits.length).toBeGreaterThanOrEqual(2); expect(hits.every(n => n.name.includes("kubernetes") || n.description.includes("k8s") || n.content.includes("kubectl"))).toBe(true); - // topNodes：先 updatePageranks，再查 top + // topNodes：直接用 Cypher 写 pagerank（生产由 gds.pageRank.write 写入），再查 top const { node: top } = await upsertNode(driver, { type: "TASK", name: "Top Ranked Task", description: "high", content: "important", }, TEST_SID); - await updatePageranks(driver, new Map([[top.id, 999]])); + const w = driver.session(); + try { + await w.run("MATCH (n:Task|Skill|Event {id: $id}) SET n.pagerank = 999", { id: top.id }); + } finally { + await w.close(); + } const topHits = await topNodes(driver, 3); expect(topHits.length).toBeGreaterThanOrEqual(1); expect(topHits[0].id).toBe(top.id); @@ -535,4 +533,314 @@ describe.skipIf(!ENABLED)("Neo4j integration (Docker)", () => { const hits = await searchNodes(driver, " ", 3); expect(hits.length).toBeLessThanOrEqual(3); }); + + it("重嵌入管线：clearAllEmbeddings + listNodeEmbeddingTargets 游标分页 + saveVector 回填", async () => { + const { node } = await upsertNode(driver, { + type: "SKILL", name: "Reembed Pipeline Skill", description: "re", content: "reembed me", + }, TEST_SID); + + // 前面用例在同库累积的活动节点已超过单页 limit——断言必须像 reembed CLI 一样 + // 按游标翻完整页（SKIP 分页在收缩集合上会跳号，游标翻页是实现的正确用法） + const collectAllTargets = async () => { + const all: Awaited> = []; + let cursor = ""; + for (let i = 0; i < 100; i++) { + const page = await listNodeEmbeddingTargets(driver, cursor, 10); + if (!page.length) break; + all.push(...page); + cursor = page[page.length - 1].id; + } + return all; + }; + + // 先有向量 → 清空后节点必须重新出现在待嵌入列表 + const vec = new Array(1024).fill(0).map((_, i) => (i % 10) / 10); + await saveVector(driver, node.id, "reembed me", vec); + expect((await collectAllTargets()).some(t => t.id === node.id)).toBe(false); + + const cleared = await clearAllEmbeddings(driver); + expect(cleared.nodes).toBeGreaterThanOrEqual(1); + + const targets = await collectAllTargets(); + const target = targets.find(t => t.id === node.id); + expect(target).toBeDefined(); + expect(target!.name).toBe("reembed-pipeline-skill"); // upsertNode 按规范化名入库 + + // 游标分页：以该节点 id 为游标，它不再出现在下一页 + const next = await listNodeEmbeddingTargets(driver, node.id, 10); + expect(next.some(t => t.id === node.id)).toBe(false); + + // 回填后再次从待嵌入列表消失（hash 一并恢复，syncEmbed 短路语义成立） + const text = buildNodeEmbeddingText(target!); + await saveVector(driver, node.id, text, vec); + const hash = await getVectorHash(driver, node.id); + expect(hash).toMatch(/^[a-f0-9]{32}$/); + expect((await collectAllTargets()).some(t => t.id === node.id)).toBe(false); + }); + + it("重嵌入管线：社区向量清空/回填 + getVectorIndexDimensions", async () => { + const session = getSession(driver); + try { + // 测试社区带 TEST_SID 以便 afterAll 清理扫到 + await session.run(` + CREATE (c:Community {id: $id, summary: $summary, nodeCount: 1, sourceSessions: [$sid]}) + `, { id: `c-reembed-${TEST_SID}`, summary: "reembed community summary", sid: TEST_SID }); + + const dimRes = await getVectorIndexDimensions(driver); + expect(Object.keys(dimRes)).toContain("gm_node_embedding"); + expect(Object.keys(dimRes)).toContain("gm_community_embedding"); + // 两个索引要么都报维度，要么都不存在（initSchema 成对创建） + if (dimRes.gm_node_embedding !== null && dimRes.gm_community_embedding !== null) { + expect(dimRes.gm_node_embedding).toBe(dimRes.gm_community_embedding); + } + + let targets = await listCommunityEmbeddingTargets(driver, "", 10); + const target = targets.find(t => t.id === `c-reembed-${TEST_SID}`); + expect(target).toBeDefined(); + expect(target!.summary).toBe("reembed community summary"); + + const commVec = new Array(dimRes.gm_community_embedding ?? 1024).fill(0.5); + await saveCommunityEmbedding(driver, target!.id, commVec); + targets = await listCommunityEmbeddingTargets(driver, "", 10); + expect(targets.some(t => t.id === `c-reembed-${TEST_SID}`)).toBe(false); + + await clearAllEmbeddings(driver); + targets = await listCommunityEmbeddingTargets(driver, "", 10); + expect(targets.some(t => t.id === `c-reembed-${TEST_SID}`)).toBe(true); + } finally { + await session.close(); + } + }); + + // ── 两阶段生命周期：遗忘曲线自动弃用 → 到期硬删（复活见 upsertNode/updateNode） ── + // 隔离性注意：purge 与 applyDecay 用例作用于整个数据库（不限于 TEST_SID—— + // purge 的 MATCH 不带 sourceSessions 过滤，applyDecay 扫描全部 active 节点）， + // 依赖 AGENTS.md 约定「NEO4J_INTEGRATION=1 指向一次性 Neo4j」保证数据正确性。 + + it("autoDeprecateNodes 批量标记 decay 弃用 + 切边 + 属性 + 前缀幂等", async () => { + const { node: victim } = await upsertNode(driver, { + type: "SKILL", name: "AutoDeprecate Target", description: "原描述", content: "content", + }, TEST_SID); + const { node: a } = await upsertNode(driver, { + type: "TASK", name: "AutoDeprecate Neighbor A", description: "n", content: "n", + }, TEST_SID); + const { node: b } = await upsertNode(driver, { + type: "SKILL", name: "AutoDeprecate Neighbor B", description: "n", content: "n", + }, TEST_SID); + await upsertEdge(driver, { + fromId: a.id, toId: victim.id, type: "USED_SKILL", + instruction: "uses", sessionId: TEST_SID, + }); + await upsertEdge(driver, { + fromId: victim.id, toId: b.id, type: "REQUIRES", + instruction: "needs", sessionId: TEST_SID, + }); + // 已带前缀的节点验证幂等（不双写） + const { node: prefixed } = await upsertNode(driver, { + type: "SKILL", name: "AutoDeprecate Prefixed", description: "[DEPRECATED] 已有前缀", content: "c", + }, TEST_SID); + + const now = Date.now(); + expect(await autoDeprecateNodes(driver, [], now)).toBe(0); + expect(await autoDeprecateNodes(driver, [victim.id, prefixed.id], now)).toBe(2); + + const v = await findById(driver, victim.id); + expect(v!.status).toBe("deprecated"); + expect(v!.deprecatedBy).toBe("decay"); + expect(v!.deprecatedAt).toBe(now); + expect(v!.description).toBe("[DEPRECATED] 原描述"); + expect(await edgesTo(driver, victim.id)).toHaveLength(0); + expect(await edgesFrom(driver, victim.id)).toHaveLength(0); + expect((await findById(driver, a.id))!.status).toBe("active"); + expect((await findById(driver, b.id))!.status).toBe("active"); + + const p = await findById(driver, prefixed.id); + expect(p!.deprecatedBy).toBe("decay"); + expect(p!.description).toBe("[DEPRECATED] 已有前缀"); + }); + + it("autoDeprecateNodes 状态守卫：窗口内已被手动弃用的节点不被覆盖为 decay", async () => { + const { node: manual } = await upsertNode(driver, { + type: "SKILL", name: "AutoDeprecate Guard Manual", description: "手动弃用", content: "c", + }, TEST_SID); + const { node: active } = await upsertNode(driver, { + type: "SKILL", name: "AutoDeprecate Guard Active", description: "仍活跃", content: "c", + }, TEST_SID); + + // 模拟评分快照之后、批量写入之前落入窗口的手动弃用 + const manualAt = Date.now() - 1_000; + await deprecateNodeAndDisconnectById(driver, manual.id, manualAt); + + const now = Date.now(); + // 只统计仍为 active 的节点：manual 已弃用不计入 + expect(await autoDeprecateNodes(driver, [manual.id, active.id], now)).toBe(1); + + const m = await findById(driver, manual.id); + expect(m!.status).toBe("deprecated"); + expect(m!.deprecatedBy).toBe("manual"); + expect(m!.deprecatedAt).toBe(manualAt); // 溯源与 purge 时钟均不被 decay 覆盖 + + const a = await findById(driver, active.id); + expect(a!.status).toBe("deprecated"); + expect(a!.deprecatedBy).toBe("decay"); + expect(a!.deprecatedAt).toBe(now); + }); + + it("purgeDeprecatedNodes：过期 deprecated 硬删、未过期保留、active 不动、存量按 updatedAt 兜底", async () => { + const { node: fresh } = await upsertNode(driver, { + type: "SKILL", name: "Purge Fresh", description: "fresh", content: "c", + }, TEST_SID); + const { node: stale } = await upsertNode(driver, { + type: "SKILL", name: "Purge Stale", description: "stale", content: "c", + }, TEST_SID); + const { node: legacy } = await upsertNode(driver, { + type: "SKILL", name: "Purge Legacy", description: "legacy", content: "c", + }, TEST_SID); + const { node: keeper } = await upsertNode(driver, { + type: "SKILL", name: "Purge Active Keeper", description: "active", content: "c", + }, TEST_SID); + + const now = Date.now(); + const session = getSession(driver); + try { + // fresh：刚弃用（deprecatedAt = now），未到期 + await session.run( + "MATCH (n {id: $id}) SET n.status='deprecated', n.deprecatedAt=$now, n.deprecatedBy='decay'", + { id: fresh.id, now }, + ); + // stale：decay 弃用已超 60 天 + await session.run( + "MATCH (n {id: $id}) SET n.status='deprecated', n.deprecatedAt=$old, n.deprecatedBy='decay'", + { id: stale.id, old: now - 61 * 86_400_000 }, + ); + // legacy：无 deprecatedAt/deprecatedBy 的存量 deprecated —— 回退 updatedAt 基准，同样到期 + await session.run( + "MATCH (n {id: $id}) SET n.status='deprecated', n.deprecatedAt=null, n.updatedAt=$old REMOVE n.deprecatedBy", + { id: legacy.id, old: now - 61 * 86_400_000 }, + ); + } finally { + await session.close(); + } + + // purgeAfterMs=0 显式关闭 + expect(await purgeDeprecatedNodes(driver, 0, now)).toBe(0); + + // 共享 DB 上可能存在其他遗留的过期 deprecated 节点，断言下界 + expect(await purgeDeprecatedNodes(driver, 60 * 86_400_000, now)).toBeGreaterThanOrEqual(2); + expect(await findById(driver, stale.id)).toBeNull(); + expect(await findById(driver, legacy.id)).toBeNull(); + expect(await findById(driver, fresh.id)).not.toBeNull(); + expect((await findById(driver, keeper.id))!.status).toBe("active"); + }); + + it("initSchema 存量迁移：为缺 deprecatedAt 的 deprecated 节点补写时钟（防 updatedAt 漂移无限续命）", async () => { + const { node: legacy } = await upsertNode(driver, { + type: "SKILL", name: "Backfill Legacy Deprecated", description: "存量弃用", content: "c", + }, TEST_SID); + const old = Date.now() - 90 * 86_400_000; + const session = getSession(driver); + try { + // 模拟存量数据：deprecated 但无 deprecatedAt（按 manual 处理，不复活） + await session.run( + "MATCH (n {id: $id}) SET n.status='deprecated', n.updatedAt=$old REMOVE n.deprecatedAt, n.deprecatedBy", + { id: legacy.id, old }, + ); + } finally { + await session.close(); + } + + await initSchema(driver); + expect((await findById(driver, legacy.id))!.deprecatedAt).toBe(old); + + // 补写后 manual 弃用节点被重新提取命中：upsertNode bump updatedAt（不复活）， + // 但 purge 时钟已钉死在 deprecatedAt，不再被推后（修复前：回退 updatedAt → 无限续命） + const hit = await upsertNode(driver, { + type: "SKILL", name: "Backfill Legacy Deprecated", description: "重新提取", content: "更长的内容触发更新路径", + }, TEST_SID); + expect(hit.isNew).toBe(false); + expect(hit.node.status).toBe("deprecated"); + const refetched = await findById(driver, legacy.id); + expect(refetched!.updatedAt!).toBeGreaterThan(old); + expect(refetched!.deprecatedAt).toBe(old); + + // 幂等：重复 initSchema 不改写已存在的 deprecatedAt + await initSchema(driver); + expect((await findById(driver, legacy.id))!.deprecatedAt).toBe(old); + }); + + it("复活：decay 弃用节点被 upsertNode/updateNode 命中后回 active；manual 弃用不复活", async () => { + // decay 弃用 → 重新提取 → 复活 + const { node: revivable } = await upsertNode(driver, { + type: "SKILL", name: "Revive Target", description: "原描述", content: "c", + }, TEST_SID); + await autoDeprecateNodes(driver, [revivable.id], Date.now()); + expect((await findById(driver, revivable.id))!.status).toBe("deprecated"); + + const revived = await upsertNode(driver, { + type: "SKILL", name: "Revive Target", description: "再次提取的描述", content: "更长的内容触发更新路径", + }, TEST_SID); + expect(revived.isNew).toBe(false); + expect(revived.node.status).toBe("active"); + expect(revived.node.description).toBe("原描述"); + expect(revived.node.deprecatedAt).toBeUndefined(); + expect(revived.node.deprecatedBy).toBeUndefined(); + + const refetched = await findById(driver, revivable.id); + expect(refetched!.status).toBe("active"); + expect(refetched!.description).toBe("原描述"); + expect(refetched!.deprecatedAt).toBeUndefined(); + + // decay 弃用 → 手动编辑 → 复活（updateNode 路径） + const { node: editable } = await upsertNode(driver, { + type: "SKILL", name: "Revive Edit Target", description: "编辑前", content: "c", + }, TEST_SID); + await autoDeprecateNodes(driver, [editable.id], Date.now()); + const edited = await updateNode(driver, "Revive Edit Target", { content: "编辑后内容" }); + expect(edited!.status).toBe("active"); + expect(edited!.description).toBe("编辑前"); + + // manual 弃用（deprecateNodeAndDisconnect）→ 重新提取不复活，前缀与弃用标记保留 + //（deprecatedAt/By 保留 = purge 时钟不被重新提取重置） + await deprecateNodeAndDisconnect(driver, "Revive Target"); + const afterManual = await upsertNode(driver, { + type: "SKILL", name: "Revive Target", description: "第三次提取", content: "更长的内容 2", + }, TEST_SID); + expect(afterManual.node.status).toBe("deprecated"); + expect(afterManual.node.description).toBe("[DEPRECATED] 原描述"); + expect(afterManual.node.deprecatedBy).toBe("manual"); + expect(afterManual.node.deprecatedAt).toBeGreaterThan(0); + }); + + it("applyDecay E2E：peripheral + 低分 + 超期未访问的节点被自动弃用", async () => { + const { node: target } = await upsertNode(driver, { + type: "SKILL", name: "Decay E2E Forgotten", description: "被遗忘的知识", content: "c", + }, TEST_SID); + // 构造遗忘终态：peripheral 层 + 400 天未访问 + 无 PageRank/访问加持 + const session = getSession(driver); + try { + await session.run( + `MATCH (n {id: $id}) + SET n.tier='peripheral', + n.createdAt=$old, n.updatedAt=$old, n.lastAccessedAt=$old, + n.pagerank=0.0, n.validatedCount=1`, + { id: target.id, old: Date.now() - 400 * 86_400_000 }, + ); + } finally { + await session.close(); + } + + const result = await applyDecay(driver, { decay: { ...DEFAULT_CONFIG.decay!, autoDeprecate: true } }); + expect(result.enabled).toBe(true); + expect(result.autoDeprecated).toBeGreaterThanOrEqual(1); + + const after = await findById(driver, target.id); + expect(after!.status).toBe("deprecated"); + expect(after!.deprecatedBy).toBe("decay"); + expect(after!.description).toBe("[DEPRECATED] 被遗忘的知识"); + + // 关闭开关后不再自动弃用（autoDeprecate=false 时 shouldAutoDeprecate 恒 false） + const off = await applyDecay(driver, { decay: { ...DEFAULT_CONFIG.decay!, autoDeprecate: false } }); + expect(off.enabled).toBe(true); + expect(off.autoDeprecated).toBe(0); + }); }); diff --git a/test/integration.recall.test.ts b/test/integration.recall.test.ts index 1cf5390..e5c15f6 100644 --- a/test/integration.recall.test.ts +++ b/test/integration.recall.test.ts @@ -17,6 +17,7 @@ import { Recaller, buildNodeEmbeddingText } from "../src/recaller/recall.ts"; import { DEFAULT_CONFIG, type GmConfig } from "../src/types.ts"; const ENABLED = !!process.env.NEO4J_INTEGRATION; +const NEO4J_URI = process.env.NEO4J_TEST_URI ?? "bolt://localhost:7687"; let driver: Driver; const TEST_SID = `recall-${Date.now()}`; @@ -24,7 +25,7 @@ const cfg: GmConfig = { ...DEFAULT_CONFIG, recallMaxNodes: 5, recallMaxDepth: 2 describe.skipIf(!ENABLED)("Recaller integration", () => { beforeAll(async () => { - driver = getDriver({ uri: "bolt://localhost:7687", user: "neo4j", password: "graphmemory" }); + driver = getDriver({ uri: NEO4J_URI, user: "neo4j", password: "graphmemory" }); await initSchema(driver); // 构造可被关键词召回的图 @@ -62,13 +63,8 @@ describe.skipIf(!ENABLED)("Recaller integration", () => { // 结构验证（不依赖具体节点返回 —— PPR 排序受共享 Neo4j 现有数据影响） expect(result).toHaveProperty("nodes"); expect(result).toHaveProperty("edges"); - expect(result).toHaveProperty("tokenEstimate"); expect(Array.isArray(result.nodes)).toBe(true); expect(Array.isArray(result.edges)).toBe(true); - // 如果有节点返回，tokenEstimate 应 > 0 - if (result.nodes.length > 0) { - expect(result.tokenEstimate).toBeGreaterThan(0); - } }); it("recall 带 mock embedFn：走向量搜索路径，不抛错", async () => { @@ -85,7 +81,6 @@ describe.skipIf(!ENABLED)("Recaller integration", () => { const result = await recaller.recall("docker"); expect(result).toHaveProperty("nodes"); expect(result).toHaveProperty("edges"); - expect(result).toHaveProperty("tokenEstimate"); }); it("recall 空查询：降级到 topNodes，返回合法结构", async () => { @@ -93,7 +88,6 @@ describe.skipIf(!ENABLED)("Recaller integration", () => { const result = await recaller.recall(" "); expect(result).toHaveProperty("nodes"); expect(result).toHaveProperty("edges"); - expect(result).toHaveProperty("tokenEstimate"); expect(result.nodes.length).toBeGreaterThanOrEqual(0); }); @@ -180,4 +174,63 @@ describe.skipIf(!ENABLED)("Recaller integration", () => { await recaller.syncEmbed({ ...node, description: "new description" }); expect(embeddedText).toContain("new description"); }); + + it("syncEmbedBatch：批量一次往返 + contentHash 短路 + 分块 + 单发回退", async () => { + // 40 个节点 > SYNC_EMBED_BATCH(32) → 应拆 2 块 + const nodes = []; + for (let i = 0; i < 40; i++) { + const { node } = await upsertNode(driver, { + type: "SKILL", name: `syncembed-batch-target-${i}`, + description: `batch ${i}`, content: `content ${i}`, + }, TEST_SID); + nodes.push(node); + } + + let batchCalls = 0; + let singleCalls = 0; + let totalTexts = 0; + const recaller = new Recaller(driver, cfg); + recaller.setEmbedFn(async () => { + singleCalls++; + return new Array(1024).fill(0.1); + }); + recaller.setEmbedBatchFn(async (texts) => { + batchCalls++; + totalTexts += texts.length; + return texts.map(() => new Array(1024).fill(0.2)); + }); + + await recaller.syncEmbedBatch(nodes); + expect(batchCalls).toBe(2); // 32 + 8 两块 + expect(totalTexts).toBe(40); + expect(singleCalls).toBe(0); // 有批量能力时不走单发 + + // contentHash 短路：内容未变 → 零新调用 + await recaller.syncEmbedBatch(nodes); + expect(batchCalls).toBe(2); + expect(totalTexts).toBe(40); + + // 单节点内容变化 → 只有该节点重嵌入（1 块 1 条文本） + await recaller.syncEmbedBatch([{ ...nodes[0], content: "changed content after batch" }]); + expect(batchCalls).toBe(3); + expect(totalTexts).toBe(41); + + // 旧接线（仅 setEmbedFn）回退到逐节点单发 + const recaller2 = new Recaller(driver, cfg); + let single2 = 0; + recaller2.setEmbedFn(async () => { + single2++; + return new Array(1024).fill(0.3); + }); + const { node: fb } = await upsertNode(driver, { + type: "SKILL", name: "syncembed-batch-fallback", + description: "f", content: "fallback content", + }, TEST_SID); + await recaller2.syncEmbedBatch([fb]); + expect(single2).toBe(1); + + // 空输入 no-op + await recaller.syncEmbedBatch([]); + expect(batchCalls).toBe(3); + }); }); diff --git a/test/integration.retention.test.ts b/test/integration.retention.test.ts new file mode 100644 index 0000000..c9a6c60 --- /dev/null +++ b/test/integration.retention.test.ts @@ -0,0 +1,234 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import type { Driver } from "neo4j-driver"; +import { getDriver, initSchema, closeDriver, getSession } from "../src/store/db.ts"; +import { saveMessage, markExtracted } from "../src/store/store.ts"; +import { normalizeMessageRetentionPolicy, runMessageRetention } from "../src/store/retention.ts"; + +// 仅在显式提供 NEO4J_TEST_URI 时运行（需要 Docker Neo4j，CI 执行）。 +// 刻意不默认 bolt://localhost:7687：本测试会删除数据，绝不允许 +// 因缺省值静默落到可能是真实数据库的端口上。 +const NEO4J_URI = process.env.NEO4J_TEST_URI; +const ENABLED = !!process.env.NEO4J_INTEGRATION && !!NEO4J_URI; +const TEST_PREFIX = "retention-"; + +let driver: Driver; +const SID_A = `retention-a-${Date.now()}`; // referenced 场景：前半提取、后半未提取 +const SID_B = `retention-b-${Date.now()}`; // recentTurns 场景：全部提取 +const SID_C = `retention-c-${Date.now()}`; // retentionDays 场景：无 user 消息（recentTurns 下全保护） +const SID_D = `retention-d-${Date.now()}`; // 无 user 消息的 session +const SID_E = `retention-e-${Date.now()}`; // 空提取：extracted=true 但 producedKnowledge=false +const SID_F = `retention-f-${Date.now()}`; // 遗留行：直接 SET extracted=true，无 producedKnowledge 属性 +const DAY = 86_400_000; + +async function seedSession(sid: string, turns: number, roles: (t: number) => string): Promise { + for (let t = 0; t < turns; t++) { + await saveMessage(driver, sid, t, roles(t), { text: `msg-${sid}-${t}` }); + } +} + +async function survivingTurns(sid: string): Promise { + const session = getSession(driver); + try { + const res = await session.run( + "MATCH (m:GmMessage {sessionId: $sid}) RETURN m.turnIndex AS t ORDER BY t", + { sid }, + ); + return res.records.map((r) => Number(r.get("t"))); + } finally { + await session.close(); + } +} + +async function setCreatedAt(sid: string, turn: number, ts: number): Promise { + const session = getSession(driver); + try { + await session.run( + "MATCH (m:GmMessage {sessionId: $sid, turnIndex: $turn}) SET m.createdAt = $ts", + { sid, turn, ts }, + ); + } finally { + await session.close(); + } +} + +/** 模拟 producedKnowledge 标记机制上线前的遗留行：直接 SET extracted=true。 */ +async function setExtractedLegacy(sid: string): Promise { + const session = getSession(driver); + try { + await session.run( + "MATCH (m:GmMessage {sessionId: $sid}) SET m.extracted = true", + { sid }, + ); + } finally { + await session.close(); + } +} + +describe.skipIf(!ENABLED)("GmMessage retention (Docker)", () => { + beforeAll(async () => { + driver = getDriver({ uri: NEO4J_URI!, user: "neo4j", password: "graphmemory" }); + await initSchema(driver); + + // 隔离守卫：库内存在任何非本测试前缀的 GmMessage 即中止 —— + // 这是删除类测试，绝不能在有真实数据的库上执行 + const guard = getSession(driver); + try { + const res = await guard.run( + "MATCH (m:GmMessage) WHERE NOT m.sessionId STARTS WITH $prefix RETURN count(m) AS n", + { prefix: TEST_PREFIX }, + ); + const foreign = res.records[0].get("n").toNumber(); + if (foreign > 0) { + throw new Error( + `[retention-test] 目标数据库含 ${foreign} 条非测试前缀消息，疑似真实数据库 —— 拒绝运行。` + + `请确认 NEO4J_TEST_URI 指向一次性测试实例。`, + ); + } + // 清理历史残留（前次中断的运行），保证候选集可预测 + await guard.run( + "MATCH (m:GmMessage) WHERE m.sessionId STARTS WITH $prefix DETACH DELETE m", + { prefix: TEST_PREFIX }, + ); + } finally { + await guard.close(); + } + + // A：10 条（偶数轮 user / 奇数轮 assistant），turns 0-5 已提取，6-9 未提取 + await seedSession(SID_A, 10, (t) => (t % 2 === 0 ? "user" : "assistant")); + await markExtracted(driver, SID_A, 5); + // B：10 条全部提取，角色同 A（user 轮 0/2/4/6/8） + await seedSession(SID_B, 10, (t) => (t % 2 === 0 ? "user" : "assistant")); + await markExtracted(driver, SID_B, 9); + // C：4 条全部 assistant（无 user 消息）、全部提取；turns 0/1 一百天前，2/3 昨天 + await seedSession(SID_C, 4, () => "assistant"); + await markExtracted(driver, SID_C, 3); + await setCreatedAt(SID_C, 0, Date.now() - 100 * DAY); + await setCreatedAt(SID_C, 1, Date.now() - 100 * DAY); + await setCreatedAt(SID_C, 2, Date.now() - 1 * DAY); + await setCreatedAt(SID_C, 3, Date.now() - 1 * DAY); + // D：2 条 assistant、全部提取、无 user 消息 + await seedSession(SID_D, 2, () => "assistant"); + await markExtracted(driver, SID_D, 1); + // E：4 条空提取 —— LLM 成功返回零节点零边：extracted=true, producedKnowledge=false + await seedSession(SID_E, 4, () => "assistant"); + await markExtracted(driver, SID_E, 3, false); + // F：遗留行（标记机制上线前已提取）：无 producedKnowledge 属性 + await seedSession(SID_F, 2, () => "assistant"); + await setExtractedLegacy(SID_F); + }); + + afterAll(async () => { + const s = getSession(driver); + try { + await s.run( + "MATCH (m:GmMessage) WHERE m.sessionId STARTS WITH $prefix DETACH DELETE m", + { prefix: TEST_PREFIX }, + ); + } finally { + await s.close(); + } + await closeDriver(); + }); + + it("keep=all is a no-op that never touches the database", async () => { + const policy = normalizeMessageRetentionPolicy({ keep: "all" }); + const result = await runMessageRetention(driver, policy); + expect(result.selectedRows).toBe(0); + expect(result.deletedRows).toBe(0); + expect(await survivingTurns(SID_A)).toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + }); + + it("retentionDays window only removes extracted messages older than the cutoff", async () => { + const policy = normalizeMessageRetentionPolicy({ keep: "recent", retentionDays: 30 }); + const result = await runMessageRetention(driver, policy); + + expect(result.policy).toBe("recent"); + expect(result.deletedRows).toBeGreaterThanOrEqual(2); // 至少 C 的 turns 0/1 + // C：只有一百天前的 turns 0/1 被删，昨天的 2/3 保留 + expect(await survivingTurns(SID_C)).toEqual([2, 3]); + // A/B/D 全部保留（createdAt 都是新写入的） + expect(await survivingTurns(SID_A)).toHaveLength(10); + expect(await survivingTurns(SID_B)).toHaveLength(10); + expect(await survivingTurns(SID_D)).toHaveLength(2); + }); + + it("recentTurns window protects the newest user turns and unextracted messages", async () => { + const policy = normalizeMessageRetentionPolicy({ keep: "recent", recentTurns: 2 }); + const result = await runMessageRetention(driver, policy); + + // cutoff = 第 2 新 user 轮的 turnIndex（user 轮 0/2/4/6/8 → cutoff=6），该轮及其后保留 + expect(result.deletedRows).toBeGreaterThanOrEqual(12); // A/B 各删 turns 0-5 + // A：turns 0-5（已提取、窗口外）删；6-9 留（6/8 在窗口内，7/9 未提取） + expect(await survivingTurns(SID_A)).toEqual([6, 7, 8, 9]); + // B：同结构全提取，turns 0-5 删、6-9 留 + expect(await survivingTurns(SID_B)).toEqual([6, 7, 8, 9]); + // C/D 没有 user 消息 → 完全保护（与上游 NULL cutoff 语义一致） + expect(await survivingTurns(SID_C)).toEqual([2, 3]); + expect(await survivingTurns(SID_D)).toEqual([0, 1]); + }); + + it("dryRun reports candidates without deleting", async () => { + const policy = normalizeMessageRetentionPolicy({ keep: "referenced", dryRun: true }); + const result = await runMessageRetention(driver, policy); + + expect(result.dryRun).toBe(true); + expect(result.deletedRows).toBe(0); + // 候选 = 剩余全部 producedKnowledge=true：B(6-9) + C(2,3) + D(0,1) = 8 + // （E 空提取 pk=false、F 遗留行无属性 —— 均不进候选） + expect(result.selectedRows).toBe(8); + expect(await survivingTurns(SID_A)).toEqual([6, 7, 8, 9]); + expect(await survivingTurns(SID_B)).toEqual([6, 7, 8, 9]); + }); + + it("keep=referenced removes remaining extracted messages and never unextracted ones", async () => { + const policy = normalizeMessageRetentionPolicy({ keep: "referenced" }); + const result = await runMessageRetention(driver, policy); + + expect(result.deletedRows).toBe(8); + // A 只剩未提取的 turns 6-9 —— 未提取消息永远不进候选 + expect(await survivingTurns(SID_A)).toEqual([6, 7, 8, 9]); + expect(await survivingTurns(SID_B)).toEqual([]); + expect(await survivingTurns(SID_C)).toEqual([]); + expect(await survivingTurns(SID_D)).toEqual([]); + }); + + it("keeps empty-extraction turns and legacy rows (producedKnowledge gate)", async () => { + const policy = normalizeMessageRetentionPolicy({ keep: "referenced" }); + const result = await runMessageRetention(driver, policy); + + // E：空提取（extracted=true, producedKnowledge=false）—— 原始证据保留， + // 日后换更好的 prompt/模型仍可回捞 + expect(await survivingTurns(SID_E)).toEqual([0, 1, 2, 3]); + // F：遗留行（无 producedKnowledge 属性）—— fail-closed 不删 + expect(await survivingTurns(SID_F)).toEqual([0, 1]); + // 上一测试已清空全部 producedKnowledge=true 候选，本轮零删除 + expect(result.selectedRows).toBe(0); + expect(result.deletedRows).toBe(0); + }); + + it("reports per-role stats and session spread", async () => { + // 上一测试后 A 仅剩 turns 6-9（user 6/8, assistant 7/9）——再跑一轮 referenced 空转 + const policy = normalizeMessageRetentionPolicy({ keep: "referenced" }); + const result = await runMessageRetention(driver, policy); + expect(result.selectedRows).toBeGreaterThanOrEqual(0); + expect(result.cutoffAt).toBeGreaterThan(0); + expect(result.durationMs).toBeGreaterThanOrEqual(0); + expect(result.policyRevision).toMatch(/^[0-9a-f]{12}$/); + }); + + it("batches deletion and reports hasMore when candidates exceed batchSize", async () => { + // 专用 session：6 条全部提取、无 user 消息（referenced 下全部可候选） + const sid = `retention-batch-${Date.now()}`; + await seedSession(sid, 6, () => "assistant"); + await markExtracted(driver, sid, 5); + + // batchSize=2：一个维护周期只处理最旧的 2 条，hasMore=true 提示还有剩余 + const policy = normalizeMessageRetentionPolicy({ keep: "referenced", batchSize: 2 }); + const result = await runMessageRetention(driver, policy); + + expect(result.selectedRows).toBe(2); + expect(result.deletedRows).toBe(2); + expect(result.hasMore).toBe(true); + expect(await survivingTurns(sid)).toEqual([2, 3, 4, 5]); + }); +}); diff --git a/test/integration.routes.test.ts b/test/integration.routes.test.ts index 9337726..5077ad1 100644 --- a/test/integration.routes.test.ts +++ b/test/integration.routes.test.ts @@ -6,6 +6,7 @@ import { closeDriver, getDriver, getSession, initSchema } from "../src/store/db. import { findById, upsertNode } from "../src/store/store.ts"; const ENABLED = !!process.env.NEO4J_INTEGRATION; +const NEO4J_URI = process.env.NEO4J_TEST_URI ?? "bolt://localhost:7687"; const TEST_SID = `routes-${Date.now()}`; let driver: Driver; @@ -30,7 +31,7 @@ async function request(method: string, path: string, body?: Record { beforeAll(async () => { - driver = getDriver({ uri: "bolt://localhost:7687", user: "neo4j", password: "graphmemory" }); + driver = getDriver({ uri: NEO4J_URI, user: "neo4j", password: "graphmemory" }); await initSchema(driver); const api = { @@ -109,4 +110,50 @@ describe.skipIf(!ENABLED)("CRUD route integration", () => { }); expect(response.status).toBe(400); }); + + it("drops direction-violating edges when the node type changes (TASK→EVENT)", async () => { + const { node: skill } = await upsertNode(driver, { + type: "SKILL", name: "route-typechange-skill", description: "d", content: "c", + }, TEST_SID); + const { node: task } = await upsertNode(driver, { + type: "TASK", name: "route-typechange-task", description: "d", content: "c", + }, TEST_SID); + + // TASK→SKILL 的 USED_SKILL 合法；节点改成 EVENT 后 USED_SKILL 出边违反白名单 + const created = await request("POST", "edges", { + fromId: task.id, + toId: skill.id, + type: "USED_SKILL", + instruction: "legal before type change", + }); + expect(created.status).toBe(201); + + const changed = await request("PUT", `nodes?id=${task.id}`, { type: "EVENT" }); + expect(changed.status).toBe(200); + + const session = getSession(driver); + try { + const outEdges = await session.run( + "MATCH (n {id: $id})-[r]->() RETURN type(r) AS type", { id: task.id }, + ); + expect(outEdges.records).toHaveLength(0); + } finally { + await session.close(); + } + }); + + it("rejects renaming a node to an existing name with 409", async () => { + const { node: keeper } = await upsertNode(driver, { + type: "SKILL", name: "route-name-keeper", description: "d", content: "c", + }, TEST_SID); + const { node: victim } = await upsertNode(driver, { + type: "TASK", name: "route-name-victim", description: "d", content: "c", + }, TEST_SID); + + const response = await request("PUT", `nodes?id=${victim.id}`, { name: keeper.name }); + expect(response.status).toBe(409); + // 自身同名（标准化后未变）不算冲突 + const selfRename = await request("PUT", `nodes?id=${victim.id}`, { name: "route-name-victim" }); + expect(selfRename.status).toBe(200); + }); }); diff --git a/test/integration.turn-commit.test.ts b/test/integration.turn-commit.test.ts new file mode 100644 index 0000000..8a61751 --- /dev/null +++ b/test/integration.turn-commit.test.ts @@ -0,0 +1,80 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import type { Driver } from "neo4j-driver"; +import { getDriver, initSchema, closeDriver, getSession } from "../src/store/db.ts"; +import { commitTurnAdvance } from "../src/store/store.ts"; + +// 仅在显式提供 NEO4J_TEST_URI 时运行（需要 Docker Neo4j，CI 执行）。 +// 刻意不默认 bolt://localhost:7687：本测试会写/删数据，绝不允许 +// 因缺省值静默落到可能是真实数据库的端口上。 +const NEO4J_URI = process.env.NEO4J_TEST_URI; +const ENABLED = !!process.env.NEO4J_INTEGRATION && !!NEO4J_URI; +const SID = `turncommit-${Date.now()}`; + +let driver: Driver; + +async function markerCount(advancementKey: string): Promise { + const session = getSession(driver); + try { + const res = await session.run( + "MATCH (t:GmTurnCommit {advancementKey: $key}) RETURN count(t) AS c", + { key: advancementKey }, + ); + return Number(res.records[0].get("c")); + } finally { + await session.close(); + } +} + +describe.skipIf(!ENABLED)("GmTurnCommit marker (Docker)", () => { + beforeAll(async () => { + driver = getDriver({ + uri: NEO4J_URI!, + user: process.env.NEO4J_TEST_USER ?? "neo4j", + // Keep the local fallback aligned with the disposable Neo4j instance in CI. + password: process.env.NEO4J_TEST_PASSWORD ?? "graphmemory", + }); + await initSchema(driver); + }); + + afterAll(async () => { + // 只清理本测试写入的标记，不动其他数据 + const session = getSession(driver); + try { + await session.run( + "MATCH (t:GmTurnCommit {sessionId: $sid}) DETACH DELETE t", + { sid: SID }, + ); + } finally { + await session.close(); + } + await closeDriver(); + }); + + it("initSchema creates the uniqueness constraint backing atomic idempotent commits", async () => { + const session = getSession(driver); + try { + const res = await session.run( + "SHOW CONSTRAINTS YIELD type, labelsOrTypes " + + "WHERE type = 'UNIQUENESS' AND 'GmTurnCommit' IN labelsOrTypes " + + "RETURN type", + ); + expect(res.records.length).toBeGreaterThanOrEqual(1); + } finally { + await session.close(); + } + }); + + it("first write commits, retry with the same advancementKey reports duplicate", async () => { + const key = `${SID}-retry`; + await expect(commitTurnAdvance(driver, SID, key, 3)).resolves.toBe("committed"); + await expect(commitTurnAdvance(driver, SID, key, 3)).resolves.toBe("duplicate"); + expect(await markerCount(key)).toBe(1); + }); + + it("distinct advancementKeys commit independently", async () => { + await expect(commitTurnAdvance(driver, SID, `${SID}-a`, 1)).resolves.toBe("committed"); + await expect(commitTurnAdvance(driver, SID, `${SID}-b`, 2)).resolves.toBe("committed"); + expect(await markerCount(`${SID}-a`)).toBe(1); + expect(await markerCount(`${SID}-b`)).toBe(1); + }); +}); diff --git a/test/llm-cooldown-recovery.test.ts b/test/llm-cooldown-recovery.test.ts new file mode 100644 index 0000000..c8669a5 --- /dev/null +++ b/test/llm-cooldown-recovery.test.ts @@ -0,0 +1,98 @@ +import { mkdtemp, rm, utimes, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { createCompleteFn } from "../src/engine/llm.ts"; + +/** + * P2 修复回归：LLM 冷却期间，oauthPath 会话文件被外部进程重写 + * （`openclaw graph-memory auth login` / CLI extract 刷新 token）必须 + * 立即解除冷却并重试，而不是等满 10 分钟。 + * fetch 全程 mock，绝不触网；会话文件用临时目录 + utimes 精确控制 mtime。 + */ + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + vi.unstubAllGlobals(); + await Promise.all(temporaryDirectories.splice(0).map((dir) => rm(dir, { + recursive: true, + force: true, + }))); +}); + +/** 无 expiresAt → needsRefresh=false，跳过刷新直接进入请求（fetch mock 拦截）。 */ +async function writeOAuthFile(authPath: string, accessToken: string, mtime: Date): Promise { + await writeFile(authPath, JSON.stringify({ + accessToken, + accountId: "test-account", + providerId: "openai-codex", + })); + await utimes(authPath, mtime, mtime); +} + +function codexSuccessResponse(): Response { + return new Response( + JSON.stringify({ output: [{ type: "message", content: [{ type: "output_text", text: "ok" }] }] }), + { status: 200, headers: { "content-type": "application/json" } }, + ); +} + +describe("LLM cooldown recovers when the OAuth session file is rewritten", () => { + it("fast-fails during cooldown, then auto-lifts after the file changes", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "graph-memory-cooldown-")); + temporaryDirectories.push(directory); + const oauthPath = path.join(directory, "oauth.json"); + + const fetchMock = vi.fn() + .mockResolvedValueOnce(new Response("unauthorized", { status: 401 })) + .mockResolvedValueOnce(codexSuccessResponse()); + vi.stubGlobal("fetch", fetchMock); + + const complete = createCompleteFn("gpt-5.6-luna", { provider: "oauth", oauthPath }); + + // 1) 首次调用：401（持久性凭证错误）→ 触发冷却 + await writeOAuthFile(oauthPath, "stale-token", new Date(1_700_000_000_000)); + await expect(complete("sys", "user")).rejects.toThrow(/OAuth LLM API 401/); + expect(fetchMock).toHaveBeenCalledTimes(1); + + // 2) 文件未变：冷却期快速失败，不再发请求 + await expect(complete("sys", "user")).rejects.toThrow(/LLM paused for \d+s/); + expect(fetchMock).toHaveBeenCalledTimes(1); + + // 3) 外部进程重写会话文件（mtime 变化）→ 冷却自动解除并重试 + await writeOAuthFile(oauthPath, "fresh-token", new Date(1_700_000_060_000)); + await expect(complete("sys", "user")).resolves.toBe("ok"); + expect(fetchMock).toHaveBeenCalledTimes(2); + // 新 token 确实被使用（第 2 次请求的 Authorization 头） + const authHeader = fetchMock.mock.calls[1][1].headers["Authorization"]; + expect(authHeader).toBe("Bearer fresh-token"); + }); + + it("keeps the cooldown while the file stays untouched", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "graph-memory-cooldown-")); + temporaryDirectories.push(directory); + const oauthPath = path.join(directory, "oauth.json"); + + const fetchMock = vi.fn() + .mockResolvedValueOnce(new Response("unauthorized", { status: 401 })) + .mockResolvedValueOnce(codexSuccessResponse()); + vi.stubGlobal("fetch", fetchMock); + + const complete = createCompleteFn("gpt-5.6-luna", { provider: "oauth", oauthPath }); + + await writeOAuthFile(oauthPath, "stale-token", new Date(1_700_000_000_000)); + await expect(complete("sys", "user")).rejects.toThrow(/OAuth LLM API 401/); + + // 相同 mtime 重写内容（异常场景）：mtime 未变 → 维持冷却 + await writeFile(oauthPath, JSON.stringify({ + accessToken: "same-mtime-token", + accountId: "test-account", + providerId: "openai-codex", + })); + await utimes(oauthPath, new Date(1_700_000_000_000), new Date(1_700_000_000_000)); + await expect(complete("sys", "user")).rejects.toThrow(/LLM paused for \d+s/); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/test/llm-guard.test.ts b/test/llm-guard.test.ts new file mode 100644 index 0000000..935f426 --- /dev/null +++ b/test/llm-guard.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from "vitest"; + +import { extractLlmStatus, LlmFailureGuard } from "../src/engine/llm-guard.ts"; + +describe("LlmFailureGuard", () => { + it("pauses after permanent 4xx API errors and recovers after cooldown", () => { + let now = 1_000; + const guard = new LlmFailureGuard(60_000, () => now); + + expect(guard.canRun()).toBe(true); + expect( + guard.tripIfNeeded(new Error('[graph-memory] LLM API 403: {"error":"User not found or inactive"}')), + ).toBe(true); + expect(guard.canRun()).toBe(false); + expect(guard.remainingMs()).toBe(60_000); + + now += 59_000; + expect(guard.canRun()).toBe(false); + + now += 2_000; + expect(guard.canRun()).toBe(true); + }); + + it("ignores retryable errors (fetchRetry already retried them)", () => { + const guard = new LlmFailureGuard(60_000, () => 1_000); + + expect(guard.tripIfNeeded(new Error("[graph-memory] LLM API 429: rate limited"))).toBe(false); + expect(guard.tripIfNeeded(new Error("[graph-memory] LLM API 503: upstream down"))).toBe(false); + expect(guard.canRun()).toBe(true); + }); + + it("does not pause for request-specific 400 and 422 errors", () => { + const guard = new LlmFailureGuard(60_000, () => 1_000); + + expect(guard.tripIfNeeded(new Error("[graph-memory] LLM API 400: prompt too long"))).toBe(false); + expect(guard.tripIfNeeded(new Error("[graph-memory] LLM API 422: invalid message"))).toBe(false); + expect(guard.canRun()).toBe(true); + }); + + it("does not pause for errors without an HTTP status (timeout, empty content, config)", () => { + const guard = new LlmFailureGuard(60_000, () => 1_000); + + expect(guard.tripIfNeeded(new Error("[graph-memory] LLM request timed out after 60000ms"))).toBe(false); + expect(guard.tripIfNeeded(new Error("[graph-memory] LLM returned empty content"))).toBe(false); + expect(guard.tripIfNeeded(new Error("[graph-memory] llm.provider=anthropic 但未配 llm.apiKey"))).toBe(false); + expect(guard.canRun()).toBe(true); + }); + + it("recognizes all three provider error formats (openai / anthropic / oauth)", () => { + expect(extractLlmStatus(new Error("[graph-memory] LLM API 401: invalid key"))).toBe(401); + expect(extractLlmStatus(new Error("[graph-memory] Anthropic API 403: forbidden"))).toBe(403); + expect(extractLlmStatus(new Error("[graph-memory] OAuth LLM API 404: model not found"))).toBe(404); + }); + + it("pauses when the OAuth refresh token is rejected (persistent credential failure)", () => { + // token 端点的 400 = invalid_grant（refresh token 被吊销/过期）—— 持久性故障 + const guard = new LlmFailureGuard(60_000, () => 1_000); + expect(guard.tripIfNeeded(new Error("OAuth refresh failed (400): invalid_grant"))).toBe(true); + expect(guard.tripIfNeeded(new Error("OAuth refresh failed (401): unauthorized"))).toBe(true); + expect(guard.canRun()).toBe(false); + }); + + it("does not pause for transient OAuth refresh failures", () => { + const guard = new LlmFailureGuard(60_000, () => 1_000); + expect(guard.tripIfNeeded(new Error("OAuth refresh failed (429): slow down"))).toBe(false); + expect(guard.tripIfNeeded(new Error("OAuth refresh failed (500): upstream error"))).toBe(false); + expect(guard.tripIfNeeded(new Error("OAuth refresh returned no access token"))).toBe(false); + expect(guard.tripIfNeeded(new Error("OAuth session from /x is expired and has no refresh token"))).toBe(false); + expect(guard.canRun()).toBe(true); + }); + + it("pauses Anthropic authentication failures", () => { + const guard = new LlmFailureGuard(60_000, () => 1_000); + + expect(guard.tripIfNeeded(new Error("[graph-memory] Anthropic API 401: invalid x-api-key"))).toBe(true); + expect(guard.canRun()).toBe(false); + }); + + it("repeated trips extend the pause instead of shortening it", () => { + let now = 1_000; + const guard = new LlmFailureGuard(60_000, () => now); + + guard.tripIfNeeded(new Error("[graph-memory] LLM API 401: expired")); + now += 30_000; + guard.tripIfNeeded(new Error("[graph-memory] LLM API 403: revoked")); + expect(guard.remainingMs()).toBe(60_000); + }); + + it("reset() clears the pause after a successful call", () => { + const guard = new LlmFailureGuard(60_000, () => 1_000); + + guard.tripIfNeeded(new Error("[graph-memory] LLM API 401: expired")); + expect(guard.canRun()).toBe(false); + guard.reset(); + expect(guard.canRun()).toBe(true); + }); +}); diff --git a/test/neo4j-gate.test.ts b/test/neo4j-gate.test.ts new file mode 100644 index 0000000..4a430d0 --- /dev/null +++ b/test/neo4j-gate.test.ts @@ -0,0 +1,68 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { Neo4jGate } from "../src/store/gate.ts"; + +describe("Neo4j 熔断门控 (Neo4jGate)", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("closed 状态放行所有操作，成功复位失败计数", () => { + const gate = new Neo4jGate(2, 120_000); + expect(gate.isAvailable()).toBe(true); + + gate.recordFailure(); + expect(gate.isAvailable()).toBe(true); + + gate.recordSuccess(); + gate.recordFailure(); + expect(gate.isAvailable()).toBe(true); + }); + + it("连续失败达到阈值后跳闸，冷却期内不可用", () => { + vi.useFakeTimers(); + const gate = new Neo4jGate(2, 120_000); + + gate.recordFailure(); + gate.recordFailure(); + expect(gate.isAvailable()).toBe(false); + + vi.advanceTimersByTime(119_999); + expect(gate.isAvailable()).toBe(false); + + vi.advanceTimersByTime(1); + expect(gate.isAvailable()).toBe(true); + }); + + it("半开后一次成功即完全恢复（计数归零）", () => { + vi.useFakeTimers(); + const gate = new Neo4jGate(2, 120_000); + + gate.recordFailure(); + gate.recordFailure(); + vi.advanceTimersByTime(120_000); + expect(gate.isAvailable()).toBe(true); + + gate.recordSuccess(); + gate.recordFailure(); + expect(gate.isAvailable()).toBe(true); + }); + + it("半开探测失败重新进入冷却", () => { + vi.useFakeTimers(); + const gate = new Neo4jGate(2, 120_000); + + gate.recordFailure(); + gate.recordFailure(); + vi.advanceTimersByTime(120_000); + + gate.recordFailure(); + expect(gate.isAvailable()).toBe(false); + + vi.advanceTimersByTime(119_999); + expect(gate.isAvailable()).toBe(false); + + vi.advanceTimersByTime(1); + expect(gate.isAvailable()).toBe(true); + }); +}); diff --git a/test/normalize-name.test.ts b/test/normalize-name.test.ts index 45c7358..d8cc705 100644 --- a/test/normalize-name.test.ts +++ b/test/normalize-name.test.ts @@ -1,6 +1,5 @@ import { describe, it, expect } from "vitest"; import { normalizeName as normalize } from "../src/store/store.ts"; -import { normalizeName as normalizeNameExtract } from "../src/extractor/extract.ts"; describe("normalizeName", () => { it("小写化", () => { @@ -53,23 +52,3 @@ describe("normalizeName", () => { expect(normalize("")).toBe(""); }); }); - -describe("normalizeName 跨文件一致性（store.ts 与 extract.ts 必须相同）", () => { - const corpus = [ - "Docker Build", - "API_KEY", - "React 18!", - "数据库迁移", - " mixed_Case Name! ", - "a---b__c d", - "", - "已经-标准化", - "Neovis 3D 可视化", - ]; - - for (const input of corpus) { - it(`相同输入相同输出: ${JSON.stringify(input)}`, () => { - expect(normalizeNameExtract(input)).toBe(normalize(input)); - }); - } -}); diff --git a/test/persist.test.ts b/test/persist.test.ts new file mode 100644 index 0000000..9f4c276 --- /dev/null +++ b/test/persist.test.ts @@ -0,0 +1,151 @@ +/** + * persistExtractionResult 单测 — 提取结果持久化单一来源 + * + * 覆盖： + * - updatedExisting 统计（isNew=false 的 upsert 命中 → finalize 阶梯第二触发条件的信号源） + * - 边端点解析的 Map-first 顺序：nameToId 命中时零 findByName 往返 + * - nameToId 未命中时回源 findByName（键是规范化名，LLM 原文未必规范化） + * - awaitEmbedSync 开关（CLI 必须等待 vs 运行时 fire-and-forget） + */ + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + upsertNode: vi.fn(), + upsertEdge: vi.fn(async () => true), + // 返回类型放宽为 any：用例会分别模拟 null（未命中）与节点对象（回源命中） + findByName: vi.fn(async (_d: unknown, _name: unknown): Promise => null), +})); + +vi.mock("../src/store/store.ts", () => ({ + upsertNode: mocks.upsertNode, + upsertEdge: mocks.upsertEdge, + findByName: mocks.findByName, +})); + +import { persistExtractionResult } from "../src/extractor/persist.ts"; + +function mkNode(name: string) { + return { + id: `n-${name}`, + type: "SKILL" as const, + name, + description: "d", + content: "c", + status: "active" as const, + validatedCount: 1, + sourceSessions: [] as string[], + communityId: null, + pagerank: 0, + createdAt: 0, + updatedAt: 0, + }; +} + +const recaller = { syncEmbedBatch: async () => {} }; +const driver = {} as any; + +describe("persistExtractionResult", () => { + beforeEach(() => { + mocks.upsertNode.mockReset(); + mocks.upsertEdge.mockReset(); + mocks.findByName.mockReset(); + mocks.upsertEdge.mockResolvedValue(true); + mocks.findByName.mockResolvedValue(null); + }); + + it("统计 updatedExisting：isNew=false 的 upsert 计入，isNew=true 不计", async () => { + mocks.upsertNode + .mockResolvedValueOnce({ node: mkNode("fresh-node"), isNew: true }) + .mockResolvedValueOnce({ node: mkNode("existing-node"), isNew: false }); + + const outcome = await persistExtractionResult(driver, recaller, { + nodes: [ + { type: "TASK", name: "fresh-node", description: "d", content: "c" }, + { type: "SKILL", name: "existing-node", description: "d", content: "c" }, + ], + edges: [], + }, { sessionId: "s1" }); + + expect(outcome.nodes).toHaveLength(2); + expect(outcome.updatedExisting).toBe(1); + expect(outcome.edges).toBe(0); + }); + + it("边端点优先查 nameToId：命中时零 findByName 往返", async () => { + mocks.upsertNode.mockResolvedValue({ node: mkNode("deploy-task"), isNew: true }); + + const outcome = await persistExtractionResult(driver, recaller, { + nodes: [{ type: "TASK", name: "deploy-task", description: "d", content: "c" }], + edges: [ + { from: "deploy-task", to: "deploy-task", type: "USED_SKILL", instruction: "self" }, + { from: "deploy-task", to: "missing", type: "USED_SKILL", instruction: "unresolved" }, + ], + }, { sessionId: "s2" }); + + expect(outcome.edges).toBe(1); + // from 命中索引；to 未命中且 findByName 返回 null → 该边被丢弃 + expect(mocks.findByName).toHaveBeenCalledTimes(1); + expect(mocks.findByName).toHaveBeenCalledWith(driver, "missing"); + }); + + it("nameToId 未命中时回源 findByName（LLM 原文未必是规范化名）", async () => { + mocks.upsertNode.mockResolvedValue({ node: mkNode("skill-b"), isNew: true }); + mocks.findByName.mockResolvedValueOnce(mkNode("skill-b")); + + const outcome = await persistExtractionResult(driver, recaller, { + nodes: [{ type: "SKILL", name: "skill-b", description: "d", content: "c" }], + edges: [{ from: "Skill B", to: "skill-b", type: "REQUIRES", instruction: "i" }], + }, { sessionId: "s3" }); + + // from="Skill B" 未命中索引 → 回源 1 次；to="skill-b" 命中索引 → 零往返 + expect(mocks.findByName).toHaveBeenCalledTimes(1); + expect(mocks.findByName).toHaveBeenCalledWith(driver, "Skill B"); + expect(outcome.edges).toBe(1); + }); + + it("awaitEmbedSync=true 时等待向量同步完成才返回", async () => { + let release!: () => void; + const gate = new Promise((r) => { release = r; }); + const slowRecaller = { syncEmbedBatch: vi.fn(() => gate) }; + + mocks.upsertNode.mockResolvedValue({ node: mkNode("n"), isNew: true }); + + const pending = persistExtractionResult(driver, slowRecaller, { + nodes: [{ type: "SKILL", name: "n", description: "d", content: "c" }], + edges: [], + }, { sessionId: "s4", awaitEmbedSync: true }); + + // 排空微任务让 persist 走到 syncEmbedBatch 调用点（不刷宏任务，保持确定性） + for (let i = 0; i < 20; i++) await Promise.resolve(); + + // 同步已启动但未完成 → persist 仍挂起 + expect(slowRecaller.syncEmbedBatch).toHaveBeenCalledTimes(1); + let settled = false; + void pending.then(() => { settled = true; }); + await Promise.resolve(); + expect(settled).toBe(false); + + release(); + await pending; + }); + + it("awaitEmbedSync 缺省（运行时路径）时 fire-and-forget：不等待同步即返回", async () => { + let release!: () => void; + const gate = new Promise((r) => { release = r; }); + const slowRecaller = { syncEmbedBatch: vi.fn(() => gate) }; + + mocks.upsertNode.mockResolvedValue({ node: mkNode("n"), isNew: true }); + + // 同步 promise 永不 resolve，persist 仍应正常返回（fire-and-forget） + const outcome = await persistExtractionResult(driver, slowRecaller, { + nodes: [{ type: "SKILL", name: "n", description: "d", content: "c" }], + edges: [], + }, { sessionId: "s5" }); + + expect(slowRecaller.syncEmbedBatch).toHaveBeenCalledTimes(1); + expect(outcome.nodes).toHaveLength(1); + release(); + await slowRecaller.syncEmbedBatch.mock.results[0]!.value; + }); +}); diff --git a/test/plugin-cli-lifecycle.test.ts b/test/plugin-cli-lifecycle.test.ts index 22de604..b0aa0bf 100644 --- a/test/plugin-cli-lifecycle.test.ts +++ b/test/plugin-cli-lifecycle.test.ts @@ -84,6 +84,18 @@ describe("plugin CLI lifecycle", () => { expect(api.registerHttpRoute).not.toHaveBeenCalled(); }); + it("registers only CLI metadata in discovery/tool-discovery/setup-only modes", () => { + for (const mode of ["discovery", "tool-discovery", "setup-only"]) { + const api = metadataApi(mode); + graphMemoryProPlugin.register(api as any); + expect(api.registerCli).toHaveBeenCalledOnce(); + expect(api.registerTool).not.toHaveBeenCalled(); + expect(api.registerContextEngine).not.toHaveBeenCalled(); + expect(api.registerHttpRoute).not.toHaveBeenCalled(); + expect(api.on).not.toHaveBeenCalled(); + } + }); + it("recognizes the plugin command for legacy OpenClaw discovery", () => { expect(isGraphMemoryCliInvocation(["node", "openclaw", "graph-memory", "auth", "login"])).toBe(true); expect(isGraphMemoryCliInvocation(["node", "openclaw", "gateway"])).toBe(false); diff --git a/test/query-cache.test.ts b/test/query-cache.test.ts new file mode 100644 index 0000000..f9424e9 --- /dev/null +++ b/test/query-cache.test.ts @@ -0,0 +1,53 @@ +/** + * 查询向量 LRU 单测 — 低成本调用节约（重复查询零 embedding API 调用） + */ + +import { describe, it, expect } from "vitest"; +import { QueryVecCache } from "../src/recaller/query-cache.ts"; + +describe("QueryVecCache", () => { + it("基础 get/set/miss", () => { + const cache = new QueryVecCache(4); + expect(cache.get("q1")).toBeUndefined(); + cache.set("q1", [1, 2, 3]); + expect(cache.get("q1")).toEqual([1, 2, 3]); + }); + + it("容量满时淘汰最旧条目", () => { + const cache = new QueryVecCache(2); + cache.set("a", [1]); + cache.set("b", [2]); + cache.set("c", [3]); // 淘汰 a + expect(cache.get("a")).toBeUndefined(); + expect(cache.get("b")).toEqual([2]); + expect(cache.get("c")).toEqual([3]); + expect(cache.size).toBe(2); + }); + + it("命中会刷新 LRU 顺序（最近使用的不会被淘汰）", () => { + const cache = new QueryVecCache(2); + cache.set("a", [1]); + cache.set("b", [2]); + cache.get("a"); // a 变为最新 + cache.set("c", [3]); // 淘汰 b 而非 a + expect(cache.get("a")).toEqual([1]); + expect(cache.get("b")).toBeUndefined(); + }); + + it("重复 set 同 key 不占额外槽位", () => { + const cache = new QueryVecCache(2); + cache.set("a", [1]); + cache.set("a", [9]); + cache.set("b", [2]); + expect(cache.size).toBe(2); + expect(cache.get("a")).toEqual([9]); + }); + + it("clear 清空全部（embedding 端点切换时）", () => { + const cache = new QueryVecCache(4); + cache.set("a", [1]); + cache.clear(); + expect(cache.get("a")).toBeUndefined(); + expect(cache.size).toBe(0); + }); +}); diff --git a/test/reembed.test.ts b/test/reembed.test.ts new file mode 100644 index 0000000..b9eb56c --- /dev/null +++ b/test/reembed.test.ts @@ -0,0 +1,105 @@ +/** + * graph-memory — 重嵌入管线纯逻辑单元测试 + * + * 覆盖 `graph-memory reembed` 的两个可纯测组件： + * - planReembed：模型输出维度 vs 向量索引维度对照后的行动决策 + * - parseBatchEmbeddingResponse：批量 /embeddings 响应 → 与输入对齐的向量数组 + * （OpenAI 按 index 归位 / 缺 index 按位置对齐 / MiniMax vector 字段 / 各类异常） + */ + +import { describe, it, expect } from "vitest"; +import { parseBatchEmbeddingResponse } from "../src/engine/embed.ts"; +import { planReembed } from "../src/cli-reembed.ts"; + +function vec(v: number): number[] { + return [v, v + 0.5, v + 1]; +} + +describe("planReembed 维度对照决策", () => { + it("模型维度与两个索引一致：直接运行", () => { + expect(planReembed({ probeDim: 1024, nodeIndexDim: 1024, communityIndexDim: 1024 })) + .toEqual({ action: "run" }); + }); + + it("索引不存在（全新库）：initSchema 已按配置建好，直接运行", () => { + expect(planReembed({ probeDim: 1536, nodeIndexDim: null, communityIndexDim: null })) + .toEqual({ action: "run" }); + }); + + it("维度不一致且无 --recreate-index：中止并给出可操作的指引", () => { + const plan = planReembed({ probeDim: 1536, nodeIndexDim: 1024, communityIndexDim: 1024 }); + expect(plan.action).toBe("abort"); + if (plan.action === "abort") { + expect(plan.reason).toContain("1024"); + expect(plan.reason).toContain("1536"); + expect(plan.reason).toContain("--recreate-index"); + } + }); + + it("维度不一致且带 --recreate-index：重建索引", () => { + const plan = planReembed({ probeDim: 1536, nodeIndexDim: 1024, communityIndexDim: 1024, recreateIndex: true }); + expect(plan).toMatchObject({ action: "recreate-index" }); + }); + + it("两个索引维度不一致时按异常集合报告（去重）", () => { + const plan = planReembed({ probeDim: 512, nodeIndexDim: 1024, communityIndexDim: 768 }); + expect(plan.action).toBe("abort"); + if (plan.action === "abort") { + expect(plan.reason).toContain("1024/768"); + } + }); + + it("只有一个索引存在且维度匹配：运行", () => { + expect(planReembed({ probeDim: 1024, nodeIndexDim: 1024, communityIndexDim: null })) + .toEqual({ action: "run" }); + }); +}); + +describe("parseBatchEmbeddingResponse 批量响应解析", () => { + it("OpenAI 风格：按 item.index 归位，响应乱序也能对齐输入", () => { + const data = [ + { index: 2, embedding: vec(3) }, + { index: 0, embedding: vec(1) }, + { index: 1, embedding: vec(2) }, + ]; + expect(parseBatchEmbeddingResponse(data, 3, false)).toEqual([vec(1), vec(2), vec(3)]); + }); + + it("缺 index 字段（部分本地端点）：按响应位置对齐", () => { + const data = [{ embedding: vec(1) }, { embedding: vec(2) }]; + expect(parseBatchEmbeddingResponse(data, 2, false)).toEqual([vec(1), vec(2)]); + }); + + it("MiniMax 风格：读 item.vector 字段", () => { + const data = [{ vector: vec(1) }, { vector: vec(2) }]; + expect(parseBatchEmbeddingResponse(data, 2, true)).toEqual([vec(1), vec(2)]); + }); + + it("data 不是数组：抛错", () => { + expect(() => parseBatchEmbeddingResponse(null, 2, false)).toThrow("missing data array"); + expect(() => parseBatchEmbeddingResponse({ embeddings: [] }, 2, false)).toThrow("missing data array"); + }); + + it("返回条数与输入不符：抛错（上层退化逐条兜底）", () => { + expect(() => parseBatchEmbeddingResponse([{ embedding: vec(1) }], 2, false)) + .toThrow("returned 1 vectors for 2 inputs"); + }); + + it("某项缺向量字段：抛错并指明位置", () => { + expect(() => parseBatchEmbeddingResponse([{ embedding: vec(1) }, { embedding: [] }], 2, false)) + .toThrow("item 1 has no vector"); + }); + + it("index 越界或重复：抛错", () => { + expect(() => parseBatchEmbeddingResponse( + [{ index: 0, embedding: vec(1) }, { index: 0, embedding: vec(2) }], 2, false, + )).toThrow("invalid index 0"); + expect(() => parseBatchEmbeddingResponse( + [{ index: 5, embedding: vec(1) }, { index: 6, embedding: vec(2) }], 2, false, + )).toThrow("invalid index 5"); + }); + + it("空输入：返回空数组（无需请求）", () => { + expect(parseBatchEmbeddingResponse([], 0, false)).toEqual([]); + }); +}); diff --git a/test/register-guard.test.ts b/test/register-guard.test.ts new file mode 100644 index 0000000..fbd672c --- /dev/null +++ b/test/register-guard.test.ts @@ -0,0 +1,205 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +// ── 模块隔离：register() 的完整初始化路径绝不能触碰真实 Neo4j ── +// 不 mock 的话 getDriver() 会连 bolt://localhost:7687，initSchema 还会执行 +// 真实 DDL —— 任何 7687 端口有真实库的机器上裸跑 `npm test` 都有写风险。 +vi.mock("../src/store/db.ts", () => ({ + getDriver: () => ({}), + initSchema: async () => {}, + getSession: () => ({ close: async () => {} }), + closeDriver: async () => {}, +})); + +vi.mock("../src/store/store.ts", () => ({ + saveMessage: async () => {}, + getUnextracted: async () => [], + getMaxTurnIndex: async () => 0, + markExtracted: async () => {}, + isTurnExtracted: async () => false, + upsertNode: async () => ({ node: {}, isNew: false }), + upsertEdge: async () => {}, + findByName: async () => null, + updateNode: async () => null, + deprecateNodeAndDisconnect: async () => {}, + deprecateNodeAndDisconnectById: async () => {}, + getBySession: async () => [], + edgesFrom: async () => [], + edgesTo: async () => [], + edgesTouching: async () => [], + deleteEdges: async () => {}, + mergeNodes: async () => {}, + getStats: async () => ({}), +})); + +// resolveProvider 必须是真实实现：baseUrl 归一化的用例断言的就是它的推断告警。 +// 只替换 createCompleteFn（真实实现会发起 HTTP / 触发 OAuth 文件读取）。 +vi.mock("../src/engine/llm.ts", async (importActual) => { + const actual = await importActual(); + return { + ...actual, + createCompleteFn: () => async () => "", + }; +}); + +vi.mock("../src/engine/embed.ts", () => ({ + createEmbedder: async () => null, +})); + +vi.mock("../src/recaller/recall.ts", () => ({ + Recaller: class { + setEmbedFn(): void {} + hasEmbedFn(): boolean { return false; } + get embedFn() { return null; } + async recall() { return { nodes: [], edges: [] }; } + async syncEmbed(): Promise {} + }, + parseTimeRange: () => null, +})); + +vi.mock("../src/extractor/extract.ts", () => ({ + Extractor: class { + async extract() { return { nodes: [], edges: [] }; } + async finalize() { return { promotedSkills: [], newEdges: [], invalidations: [] }; } + }, +})); + +vi.mock("../src/format/assemble.ts", () => ({ + assembleContext: async () => ({ xml: "", systemPrompt: "", tokens: 0 }), +})); + +vi.mock("../src/graph/maintenance.ts", () => ({ + runMaintenance: async () => ({ durationMs: 0 }), +})); + +vi.mock("../src/routes/crud.ts", () => ({ + registerCrudRoutes: () => {}, +})); + +import graphMemoryProPlugin from "../index.ts"; +import { closeDriver } from "../src/store/db.ts"; + +/** 完整运行时模式的 fake api（覆盖 register() 用到的全部方法）。 */ +function fullApi(pluginConfig: Record = {}) { + return { + pluginConfig, + config: {}, + resolvePath: (v: string) => v, + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + registerCli: vi.fn(), + registerTool: vi.fn(), + registerContextEngine: vi.fn(), + registerHttpRoute: vi.fn(), + on: vi.fn(), + } as any; +} + +describe("duplicate register() guard", () => { + let createdEngine: { dispose: () => Promise } | null = null; + + afterEach(async () => { + if (createdEngine) { + await createdEngine.dispose(); + createdEngine = null; + } + await closeDriver(); + }); + + it("reuses the active engine when the host registers twice without dispose", () => { + const api1 = fullApi(); + graphMemoryProPlugin.register(api1); + expect(api1.registerContextEngine).toHaveBeenCalledTimes(1); + createdEngine = api1.registerContextEngine.mock.calls[0][1](); + + // 第二次 register：只重绑引擎工厂，不再注册工具/路由/hook + const api2 = fullApi(); + graphMemoryProPlugin.register(api2); + expect(api2.registerContextEngine).toHaveBeenCalledTimes(1); + expect(api2.registerTool).not.toHaveBeenCalled(); + expect(api2.registerHttpRoute).not.toHaveBeenCalled(); + expect(api2.on).not.toHaveBeenCalled(); + expect(api2.registerContextEngine.mock.calls[0][1]()).toBe(createdEngine); + expect(api2.logger.warn).toHaveBeenCalledWith( + expect.stringContaining("duplicate register() ignored"), + ); + }); + + // 2026-09-01 事故回归：host 逐逻辑 turn 惰性调用工厂。守卫曾把工厂重绑为 + // `() => activeEngine`（可变模块变量），dispose() 清空标记后工厂返回 null， + // host 按契约判定 "factory returned null" 并逐回合降级 legacy 直到重启。 + it("rebound factory still returns the engine after dispose clears the module flag", async () => { + const api1 = fullApi(); + graphMemoryProPlugin.register(api1); + const engineA = api1.registerContextEngine.mock.calls[0][1](); + createdEngine = engineA; + + const api2 = fullApi(); + graphMemoryProPlugin.register(api2); // 守卫路径重绑工厂 + const reboundFactory = api2.registerContextEngine.mock.calls[0][1]; + + await engineA.dispose(); // activeEngine → null + + expect(reboundFactory()).toBe(engineA); + expect(reboundFactory()).not.toBeNull(); + }); + + it("skips runtime init for discovery-mode loads (readOnlyDiscovery never serves turns)", () => { + const api1 = fullApi(); + graphMemoryProPlugin.register(api1); + createdEngine = api1.registerContextEngine.mock.calls[0][1](); + + // 配置热重载的 discovery 加载：只重注册 CLI 元数据，不碰运行时， + // 也不与活跃引擎守卫交互（mode 检查在守卫之前） + const api2 = fullApi(); + api2.registrationMode = "discovery"; + graphMemoryProPlugin.register(api2); + expect(api2.registerCli).toHaveBeenCalledOnce(); + expect(api2.registerContextEngine).not.toHaveBeenCalled(); + expect(api2.registerTool).not.toHaveBeenCalled(); + expect(api2.on).not.toHaveBeenCalled(); + }); + + it("creates a fresh engine after dispose() (genuine reload)", async () => { + const api1 = fullApi(); + graphMemoryProPlugin.register(api1); + const engineA = api1.registerContextEngine.mock.calls[0][1](); + createdEngine = engineA; + await engineA.dispose(); + + const api2 = fullApi(); + graphMemoryProPlugin.register(api2); + const engineB = api2.registerContextEngine.mock.calls[0][1](); + expect(engineB).not.toBe(engineA); + expect(api2.registerTool).toHaveBeenCalled(); + // afterEach 清理的是"最后创建"的引擎（engineB 才是当前活跃的） + createdEngine = engineB; + }); + + it("normalizes lowercase llm.baseUrl to baseURL before provider resolution", () => { + // 只有 apiKey 时启发式推断为 anthropic；归一生效后 baseURL 存在 → 推断为 openai + const api = fullApi({ llm: { apiKey: "k", baseUrl: "http://localhost:8080/v1/" } }); + graphMemoryProPlugin.register(api); + createdEngine = api.registerContextEngine.mock.calls[0]?.[1]?.() ?? null; + + expect(api.logger.warn).toHaveBeenCalledWith( + expect.stringContaining('推断为 "openai"'), + ); + }); + + it("explicit baseURL wins over baseUrl spelling", () => { + const api = fullApi({ + llm: { apiKey: "k", baseURL: "http://explicit/v1", baseUrl: "http://lowercase/v1" }, + }); + graphMemoryProPlugin.register(api); + createdEngine = api.registerContextEngine.mock.calls[0]?.[1]?.() ?? null; + + // 显式配置优先 —— 这里只验证 register 未被拼写兼容逻辑破坏（推断告警产生一次） + expect(api.logger.warn).toHaveBeenCalledWith( + expect.stringContaining('推断为 "openai"'), + ); + expect( + api.logger.warn.mock.calls.filter((args: unknown[]) => + String(args[0]).includes("llm.provider 未显式设置"), + ), + ).toHaveLength(1); + }); +}); diff --git a/test/retention-policy.test.ts b/test/retention-policy.test.ts new file mode 100644 index 0000000..77139c4 --- /dev/null +++ b/test/retention-policy.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; + +import { + normalizeMessageRetentionPolicy, + messageRetentionPolicyRevision, +} from "../src/store/retention.ts"; + +describe("normalizeMessageRetentionPolicy", () => { + it("defaults to keep=all with no pruning and bounded batches", () => { + expect(normalizeMessageRetentionPolicy(undefined)).toEqual({ + keep: "all", recentTurns: 0, retentionDays: 0, batchSize: 500, dryRun: false, + }); + expect(normalizeMessageRetentionPolicy({})).toEqual({ + keep: "all", recentTurns: 0, retentionDays: 0, batchSize: 500, dryRun: false, + }); + }); + + it("accepts a valid recent policy", () => { + const policy = normalizeMessageRetentionPolicy({ + keep: "recent", recentTurns: 3, retentionDays: 30, batchSize: 100, dryRun: true, + }); + expect(policy).toEqual({ + keep: "recent", recentTurns: 3, retentionDays: 30, batchSize: 100, dryRun: true, + }); + }); + + it("accepts keep=referenced without window params", () => { + expect(normalizeMessageRetentionPolicy({ keep: "referenced" }).keep).toBe("referenced"); + }); + + it("rejects invalid keep mode and non-object input", () => { + expect(() => normalizeMessageRetentionPolicy({ keep: "aggressive" } as any)).toThrow(TypeError); + expect(() => normalizeMessageRetentionPolicy("recent" as any)).toThrow(TypeError); + expect(() => normalizeMessageRetentionPolicy([] as any)).toThrow(TypeError); + }); + + it("keep=recent requires at least one window parameter", () => { + expect(() => normalizeMessageRetentionPolicy({ keep: "recent" })).toThrow(/requires recentTurns or retentionDays/); + }); + + it("rejects out-of-bounds and non-integer numeric fields", () => { + expect(() => normalizeMessageRetentionPolicy({ batchSize: 0 })).toThrow(/batchSize/); + expect(() => normalizeMessageRetentionPolicy({ batchSize: 10_001 })).toThrow(/batchSize/); + expect(() => normalizeMessageRetentionPolicy({ batchSize: 12.5 })).toThrow(/batchSize/); + expect(() => normalizeMessageRetentionPolicy({ recentTurns: -1 })).toThrow(/recentTurns/); + expect(() => normalizeMessageRetentionPolicy({ retentionDays: 40_000 })).toThrow(/retentionDays/); + }); + + it("rejects non-boolean dryRun", () => { + expect(() => normalizeMessageRetentionPolicy({ dryRun: "yes" as any })).toThrow(/dryRun/); + }); + + it("revision is stable for identical policies and differs across policies", () => { + const a = normalizeMessageRetentionPolicy({ keep: "referenced", batchSize: 100 }); + const b = normalizeMessageRetentionPolicy({ keep: "referenced", batchSize: 100 }); + const c = normalizeMessageRetentionPolicy({ keep: "referenced", batchSize: 200 }); + expect(messageRetentionPolicyRevision(a)).toBe(messageRetentionPolicyRevision(b)); + expect(messageRetentionPolicyRevision(a)).not.toBe(messageRetentionPolicyRevision(c)); + }); +}); diff --git a/test/session-identity.test.ts b/test/session-identity.test.ts index c2ec70c..5da274c 100644 --- a/test/session-identity.test.ts +++ b/test/session-identity.test.ts @@ -1,11 +1,17 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { isCronSessionKey } from "../src/types.ts"; const mocks = vi.hoisted(() => ({ - getBySession: vi.fn(async () => []), + getBySession: vi.fn(async () => [] as unknown[]), + saveMessage: vi.fn(async ( + _driver: unknown, _sid: string, _turn: number, _role: string, _content: unknown, + ): Promise => {}), + getMaxTurnIndex: vi.fn(async () => 0), + getUnextracted: vi.fn(async () => []), + isTurnExtracted: vi.fn(async () => false), recall: vi.fn(async () => ({ nodes: [{ id: "recalled-node" }], edges: [], - tokenEstimate: 1, })), assembleContext: vi.fn(async () => ({ xml: "", systemPrompt: "", tokens: 0 })), runMaintenance: vi.fn(async () => ({ @@ -25,10 +31,11 @@ vi.mock("../src/store/db.ts", () => ({ })); vi.mock("../src/store/store.ts", () => ({ - saveMessage: async () => {}, - getUnextracted: async () => [], + saveMessage: mocks.saveMessage, + getUnextracted: mocks.getUnextracted, + getMaxTurnIndex: mocks.getMaxTurnIndex, markExtracted: async () => {}, - isTurnExtracted: async () => false, + isTurnExtracted: mocks.isTurnExtracted, upsertNode: async () => ({ node: {}, isNew: false }), upsertEdge: async () => {}, findByName: async () => null, @@ -36,7 +43,8 @@ vi.mock("../src/store/store.ts", () => ({ getBySession: mocks.getBySession, edgesFrom: async () => [], edgesTo: async () => [], - deprecate: async () => {}, + edgesTouching: async () => [], + deprecateNodeAndDisconnectById: async () => {}, getStats: async () => ({}), })); @@ -46,12 +54,14 @@ vi.mock("../src/engine/llm.ts", () => ({ })); vi.mock("../src/engine/embed.ts", () => ({ - createEmbedFn: async () => null, + createEmbedder: async () => null, })); vi.mock("../src/recaller/recall.ts", () => ({ Recaller: class { setEmbedFn(): void {} + hasEmbedFn(): boolean { return false; } + get embedFn() { return null; } async recall() { return mocks.recall(); } async syncEmbed(): Promise {} }, @@ -81,21 +91,46 @@ import graphMemoryProPlugin from "../index.ts"; type HookHandler = (event: Record, context: Record) => Promise; type EngineHarness = { readonly bootstrap: (params: { readonly sessionId: string; readonly sessionKey?: string }) => Promise; + readonly ingest: (params: { + readonly sessionId: string; + readonly sessionKey?: string; + readonly message: unknown; + readonly isHeartbeat?: boolean; + }) => Promise<{ readonly ingested: boolean }>; readonly assemble: (params: { readonly sessionId: string; readonly sessionKey?: string; readonly messages: readonly unknown[]; }) => Promise; + readonly compact: (params: { readonly sessionId: string; readonly sessionKey?: string }) => Promise<{ + readonly ok: boolean; + readonly compacted: boolean; + readonly reason?: string; + }>; + readonly afterTurn: (params: { + readonly sessionId: string; + readonly sessionKey?: string; + readonly messages: readonly unknown[]; + readonly prePromptMessageCount: number; + }) => Promise; readonly prepareSubagentSpawn: (params: { readonly parentSessionKey: string; readonly childSessionKey: string; readonly parentSessionId?: string; }) => Promise<{ readonly rollback: () => void }>; + readonly dispose: () => Promise; }; -function registerPlugin(): { readonly hooks: Map; readonly engine: EngineHarness } { +// register() 带防重复注册守卫（模块级 activeEngine）：同一进程内未 dispose 的 +// 二次 register 会被拦截复用。本文件每个用例都注册一个带独立 pluginConfig +// 的隔离引擎 —— 在重复注册前先 dispose 上一个，保持逐用例隔离。 +let previousEngine: { dispose?: () => Promise | void } | null = null; + +function registerPlugin(pluginConfig: Record = {}): { readonly hooks: Map; readonly engine: EngineHarness } { const hooks = new Map(); let engine: EngineHarness | undefined; + previousEngine?.dispose?.(); + previousEngine = null; graphMemoryProPlugin.register({ logger: { debug: () => {}, @@ -104,7 +139,7 @@ function registerPlugin(): { readonly hooks: Map; readonly error: () => {}, }, config: {}, - pluginConfig: {}, + pluginConfig, resolvePath: (path: string) => path, on: (event: string, handler: HookHandler) => { hooks.set(event, handler); }, registerContextEngine: (_id: string, factory: () => EngineHarness) => { engine = factory(); }, @@ -112,9 +147,17 @@ function registerPlugin(): { readonly hooks: Map; readonly registerHttpRoute: () => {}, }); if (!engine) throw new Error("context engine was not registered"); + previousEngine = engine; return { hooks, engine }; } +afterAll(async () => { + // 释放最后一个引擎，清掉模块级 activeEngine —— vitest singleFork 下 + // 所有测试文件共享进程，不能把注册状态泄漏给后续文件 + await previousEngine?.dispose?.(); + previousEngine = null; +}); + describe("session identity", () => { beforeEach(() => { mocks.getBySession.mockClear(); @@ -165,3 +208,243 @@ describe("session identity", () => { ); }); }); + +describe("cron session gating (cron 配置)", () => { + beforeEach(() => { + mocks.getBySession.mockClear(); + mocks.saveMessage.mockClear(); + mocks.getMaxTurnIndex.mockClear(); + mocks.getUnextracted.mockClear(); + mocks.isTurnExtracted.mockClear(); + mocks.recall.mockClear(); + mocks.assembleContext.mockClear(); + mocks.runMaintenance.mockClear(); + }); + + // host 契约：sessionId 是随机 transcript UUID，cron 标记在 sessionKey 上 + const CRON_KEY = "agent:agent-1:cron:daily-report"; + const CRON_SID = "0f1e2d3c-4b5a-6978-8976-543210fedcba"; + + it("isCronSessionKey 按 sessionKey 段匹配 cron 标记", () => { + expect(isCronSessionKey("cron:job-1")).toBe(true); + expect(isCronSessionKey("agent:agent-1:cron:daily")).toBe(true); + expect(isCronSessionKey("agent:agent-1:cron:daily:run:r1")).toBe(true); + expect(isCronSessionKey("agent:main")).toBe(false); + expect(isCronSessionKey("agent:cron-daily:main")).toBe(false); + expect(isCronSessionKey("scheduled-cron:x")).toBe(false); + expect(isCronSessionKey("")).toBe(false); + expect(isCronSessionKey(undefined)).toBe(false); + expect(isCronSessionKey(null)).toBe(false); + }); + + it("默认配置（全 true）下 cron session_end 仍执行 finalize 与图维护（向后兼容）", async () => { + const handler = registerPlugin().hooks.get("session_end"); + if (!handler) throw new Error("session_end hook was not registered"); + + await handler({ sessionId: CRON_SID, sessionKey: CRON_KEY }, {}); + + expect(mocks.getBySession).toHaveBeenCalledWith({}, CRON_SID); + expect(mocks.runMaintenance).toHaveBeenCalledTimes(1); + }); + + it("默认配置下 cron session 正常入库", async () => { + const { engine } = registerPlugin(); + + await expect(engine.ingest({ sessionId: CRON_SID, sessionKey: CRON_KEY, message: { role: "user", content: "hi" } })) + .resolves.toEqual({ ingested: true }); + expect(mocks.saveMessage).toHaveBeenCalledTimes(1); + }); + + it("finalizeAndMaintain=false 时 cron session 跳过 finalize 与图维护", async () => { + const handler = registerPlugin({ cron: { enabled: true, finalizeAndMaintain: false } }).hooks.get("session_end"); + if (!handler) throw new Error("session_end hook was not registered"); + + await handler({ sessionId: CRON_SID, sessionKey: CRON_KEY }, {}); + + expect(mocks.getBySession).not.toHaveBeenCalled(); + expect(mocks.runMaintenance).not.toHaveBeenCalled(); + }); + + it("finalizeAndMaintain=true 时 cron session 执行 finalize 与图维护", async () => { + const handler = registerPlugin({ cron: { enabled: true, finalizeAndMaintain: true } }).hooks.get("session_end"); + if (!handler) throw new Error("session_end hook was not registered"); + + await handler({ sessionId: CRON_SID, sessionKey: CRON_KEY }, {}); + + expect(mocks.getBySession).toHaveBeenCalledWith({}, CRON_SID); + expect(mocks.runMaintenance).toHaveBeenCalledTimes(1); + }); + + it("finalizeAndMaintain=true 不影响普通会话的既有行为", async () => { + const handler = registerPlugin({ cron: { enabled: true, finalizeAndMaintain: false } }).hooks.get("session_end"); + if (!handler) throw new Error("session_end hook was not registered"); + + await handler({ sessionId: "normal-session", sessionKey: "agent:main" }, {}); + + expect(mocks.runMaintenance).toHaveBeenCalledTimes(1); + }); + + it("enabled=false 时 cron session 不召回、不入库、不注入图谱上下文", async () => { + const { hooks, engine } = registerPlugin({ cron: { enabled: false } }); + + const beforeAgentStart = hooks.get("before_agent_start"); + if (!beforeAgentStart) throw new Error("before_agent_start hook was not registered"); + await beforeAgentStart({ prompt: "daily digest" }, { sessionKey: CRON_KEY }); + expect(mocks.recall).not.toHaveBeenCalled(); + + await expect(engine.ingest({ sessionId: CRON_SID, sessionKey: CRON_KEY, message: { role: "user", content: "hi" } })) + .resolves.toEqual({ ingested: false }); + expect(mocks.saveMessage).not.toHaveBeenCalled(); + + await engine.assemble({ sessionId: CRON_SID, sessionKey: CRON_KEY, messages: [] }); + expect(mocks.assembleContext).not.toHaveBeenCalled(); + }); + + it("enabled=false 总开关：cron afterTurn 跳过入库回填，compact 跳过提取", async () => { + const { engine } = registerPlugin({ cron: { enabled: false } }); + + await engine.afterTurn({ + sessionId: CRON_SID, + sessionKey: CRON_KEY, + messages: [{ role: "user", content: "hi" }], + prePromptMessageCount: 0, + }); + expect(mocks.saveMessage).not.toHaveBeenCalled(); + expect(mocks.isTurnExtracted).not.toHaveBeenCalled(); + + const res = await engine.compact({ sessionId: CRON_SID, sessionKey: CRON_KEY }); + expect(res).toEqual({ ok: true, compacted: false, reason: "cron session graph disabled" }); + expect(mocks.getUnextracted).not.toHaveBeenCalled(); + }); + + it("enabled=true 时 cron session 正常入库", async () => { + const { engine } = registerPlugin({ cron: { enabled: true } }); + + await engine.ingest({ sessionId: CRON_SID, sessionKey: CRON_KEY, message: { role: "user", content: "hi" } }); + + expect(mocks.saveMessage).toHaveBeenCalledTimes(1); + }); + + it("extract=false 时 cron session 消息仍入库缓冲但不触发提取", async () => { + const { engine } = registerPlugin({ cron: { enabled: true, extract: false } }); + + await engine.afterTurn({ + sessionId: CRON_SID, + sessionKey: CRON_KEY, + messages: [{ role: "user", content: "hi" }], + prePromptMessageCount: 0, + }); + + expect(mocks.saveMessage).toHaveBeenCalledTimes(1); + expect(mocks.isTurnExtracted).not.toHaveBeenCalled(); + }); + + it("extract=true 时 cron session 触发提取", async () => { + const { engine } = registerPlugin({ cron: { enabled: true, extract: true } }); + + await engine.afterTurn({ + sessionId: CRON_SID, + sessionKey: CRON_KEY, + messages: [{ role: "user", content: "hi" }], + prePromptMessageCount: 0, + }); + // afterTurn 内的 extractTurnKnowledge 是 fire-and-forget，flush 微任务后再断言 + await new Promise(resolve => setTimeout(resolve, 0)); + + expect(mocks.isTurnExtracted).toHaveBeenCalledTimes(1); + }); + + it("extract=false 时 cron session compact 直接跳过提取", async () => { + const { engine } = registerPlugin({ cron: { enabled: true, extract: false } }); + + const res = await engine.compact({ sessionId: CRON_SID, sessionKey: CRON_KEY }); + + expect(res).toEqual({ ok: true, compacted: false, reason: "cron session extraction disabled" }); + expect(mocks.getUnextracted).not.toHaveBeenCalled(); + }); + + it("cron 任务设置自定义 sessionKey（无 cron 段）时按普通会话处理", async () => { + const { engine } = registerPlugin({ cron: { enabled: false } }); + + await expect(engine.ingest({ sessionId: CRON_SID, sessionKey: "agent:my-custom-key", message: { role: "user", content: "hi" } })) + .resolves.toEqual({ ingested: true }); + + expect(mocks.saveMessage).toHaveBeenCalledTimes(1); + }); +}); + +describe("outage buffering (Neo4j 掉线缓冲)", () => { + beforeEach(() => { + // mockReset：清掉上一个测试可能设置的 mockRejectedValue 等持久实现， + // 否则持续拒绝会泄漏到后续测试，把所有消息都打进缓冲 + mocks.saveMessage.mockReset().mockImplementation(async () => {}); + mocks.saveMessage.mockClear(); + mocks.getUnextracted.mockClear(); + mocks.getMaxTurnIndex.mockClear(); + }); + + it("写失败时消息被缓冲且不向 host 抛错", async () => { + const { engine } = registerPlugin(); + mocks.saveMessage.mockRejectedValue(new Error("neo4j down")); + + await expect( + engine.ingest({ sessionId: "outage-1", message: { role: "user", content: "hello" } }), + ).resolves.toEqual({ ingested: true }); + + expect(mocks.saveMessage).toHaveBeenCalledTimes(1); + }); + + it("恢复后缓冲消息经 ingestMessage 补写（seq 由 DB 状态分配，不撞号）", async () => { + const { engine } = registerPlugin(); + mocks.saveMessage.mockRejectedValueOnce(new Error("neo4j down")); + + // 第一条：写失败 → 缓冲（seq 1 被失败尝试消耗） + await engine.ingest({ sessionId: "outage-2", message: { role: "user", content: "first message" } }); + // 第二条：写成功 → 触发 flush → 第一条补写（分配新 seq，绕开撞号） + await engine.ingest({ sessionId: "outage-2", message: { role: "user", content: "second message" } }); + await new Promise(resolve => setTimeout(resolve, 0)); + + expect(mocks.saveMessage).toHaveBeenCalledTimes(3); + const flushed = mocks.saveMessage.mock.calls[2]; + expect(flushed[1]).toBe("outage-2"); + expect(flushed[2]).toBe(3); // seq 3 = max(已写 2) + 1，而非缓冲期的 1 + expect((flushed[4] as { content: string }).content).toContain("first message"); + }); + + it("不可序列化的消息被丢弃，不堵塞后续消息（队头防堵）", async () => { + const { engine } = registerPlugin(); + // 先让写失败一次，把毒消息逼进缓冲路径（真实 saveMessage 会在 stringify 时抛错） + mocks.saveMessage.mockRejectedValueOnce(new Error("neo4j down")); + + const poison: any = { role: "user", content: "ok" }; + poison.self = poison; // 循环引用 → JSON.stringify 抛错 + + await expect( + engine.ingest({ sessionId: "outage-3", message: poison }), + ).resolves.toEqual({ ingested: true }); + await expect( + engine.ingest({ sessionId: "outage-3", message: { role: "user", content: "after poison" } }), + ).resolves.toEqual({ ingested: true }); + await new Promise(resolve => setTimeout(resolve, 0)); + + // 第一次 saveMessage 因序列化失败抛错 → 消息进缓冲即被丢弃； + // 第二条正常直写后 flush 无积压 → 总共 2 次调用，毒消息不重试 + expect(mocks.saveMessage).toHaveBeenCalledTimes(2); + }); + + it("compact 在读取未提取消息前先刷缓冲（恢复补提取顺序）", async () => { + const { engine } = registerPlugin(); + mocks.saveMessage.mockRejectedValueOnce(new Error("neo4j down")); + + await engine.ingest({ sessionId: "outage-4", message: { role: "user", content: "buffered turn" } }); + expect(mocks.saveMessage).toHaveBeenCalledTimes(1); + + const res = await engine.compact({ sessionId: "outage-4" }); + expect(res).toEqual({ ok: true, compacted: false, reason: "no messages" }); + + // flush 先于 getUnextracted：第二条 saveMessage 是缓冲补写，然后才查未提取集 + expect(mocks.saveMessage).toHaveBeenCalledTimes(2); + expect((mocks.saveMessage.mock.calls[1][4] as { content: string }).content).toContain("buffered turn"); + expect(mocks.getUnextracted).toHaveBeenCalledTimes(1); + }); +}); diff --git a/test/turn-filter.test.ts b/test/turn-filter.test.ts new file mode 100644 index 0000000..6cdc338 --- /dev/null +++ b/test/turn-filter.test.ts @@ -0,0 +1,140 @@ +/** + * trivial 轮次本地预筛单测 — LLM 成本控制的第一道闸 + * + * 判定语义（保守取向）： + * 1. 清洗后为空 → 跳过 + * 2. 命中无意义词表（内置 + extra，含标点/大小写容忍）→ 跳过 + * 3. 清洗后 ≤ maxChars（默认 5）且无技术词（连续 ≥3 位字母数字）→ 跳过 + * 4. 其余（正常提问 / 短但含技术词 / 长文本）→ 不跳过 + */ + +import { describe, it, expect } from "vitest"; +import { + normalizeTrivialText, shouldSkipTurnExtraction, turnHasToolWork, BUILTIN_TRIVIAL_PROMPTS, +} from "../src/extractor/turn-filter.ts"; +import { turnUserText } from "../index.ts"; + +describe("normalizeTrivialText", () => { + it("去空白与中西文标点、转小写", () => { + expect(normalizeTrivialText(" 继续。！！ ")).toBe("继续"); + expect(normalizeTrivialText("OK!")).toBe("ok"); + expect(normalizeTrivialText("Resume, please")).toBe("resumeplease"); + }); +}); + +describe("shouldSkipTurnExtraction — 词表命中", () => { + it.each([ + "继续", "请继续", "继续。", "继续吧", "好的！", "ok", "OK", "resume", + "continue", "谢谢", "thanks", "got it", "知道了", + ])("%s → 跳过", (input) => { + expect(shouldSkipTurnExtraction(input)).toBe(true); + }); + + it("额外词表生效且同样容忍标点", () => { + expect(shouldSkipTurnExtraction("下一页。", { extraPrompts: ["下一页"] })).toBe(true); + expect(shouldSkipTurnExtraction("下一页", { extraPrompts: ["下一页"] })).toBe(true); + }); + + it("内置表可被 extra 扩展且不被覆盖", () => { + expect(BUILTIN_TRIVIAL_PROMPTS).toContain("继续"); + expect(shouldSkipTurnExtraction("继续")).toBe(true); + // 不在词表且超长阈值的正常词 → 不跳过 + expect(shouldSkipTurnExtraction("下一页部署 nginx")).toBe(false); + }); +}); + +describe("shouldSkipTurnExtraction — 短文本阈值", () => { + it("≤5 字纯文本（无技术词）→ 跳过", () => { + expect(shouldSkipTurnExtraction("是的呢")).toBe(true); + expect(shouldSkipTurnExtraction("嗯嗯嗯嗯嗯")).toBe(true); // 恰好 5 + expect(shouldSkipTurnExtraction("嗯嗯嗯嗯嗯嗯")).toBe(false); // 6 字，宁可多提取 + }); + + it("短但含技术词（连续 ≥3 位字母数字）→ 不跳过", () => { + expect(shouldSkipTurnExtraction("用pnpm")).toBe(false); + expect(shouldSkipTurnExtraction("试jwt")).toBe(false); + expect(shouldSkipTurnExtraction("k8s呢")).toBe(false); + }); + + it("空输入 → 跳过（无可提取内容）", () => { + expect(shouldSkipTurnExtraction("")).toBe(true); + expect(shouldSkipTurnExtraction(" 。！？ ")).toBe(true); + }); + + it("正常提问不跳过", () => { + expect(shouldSkipTurnExtraction("帮我把 neo4j 的索引重建一下")).toBe(false); + expect(shouldSkipTurnExtraction("这个报错怎么修")).toBe(false); + expect(shouldSkipTurnExtraction("why does the build fail")).toBe(false); + }); + + it("自定义阈值", () => { + expect(shouldSkipTurnExtraction("嗯嗯嗯嗯嗯嗯", { maxChars: 6 })).toBe(true); + expect(shouldSkipTurnExtraction("嗯嗯嗯", { maxChars: 2 })).toBe(false); + }); +}); + +describe("turnUserText — 轮级 user 文本聚合", () => { + it("只取 user 角色，剥离 OpenClaw 元数据", () => { + const messages = [ + { + role: "user", + content: [ + { type: "text", text: "```json\n{\"Sender\":\"meta\"}\n```\n继续" }, + ], + }, + { role: "assistant", content: [{ type: "text", text: "好的，我继续处理部署任务" }] }, + { role: "user", content: [{ type: "text", text: "" }] }, + ]; + const text = turnUserText(messages); + expect(text.trim()).toBe("继续"); + // 元数据剥离后应命中词表 → 该轮会被本地预筛跳过 + expect(shouldSkipTurnExtraction(text)).toBe(true); + }); + + it("string content 同样支持", () => { + expect(turnUserText([{ role: "user", content: "怎么修这个报错" }])).toBe("怎么修这个报错"); + }); + + it("非 user 消息不参与", () => { + expect(turnUserText([{ role: "assistant", content: "继续" }])).toBe(""); + expect(shouldSkipTurnExtraction(turnUserText([{ role: "assistant", content: "继续" }]))).toBe(true); + }); +}); + +describe("turnHasToolWork — 工具劳动守卫（trivial 预筛的误杀保险）", () => { + it("含 tool / toolResult 角色 → true", () => { + expect(turnHasToolWork([ + { role: "user", content: "继续" }, + { role: "assistant", content: [{ type: "toolUse", id: "t1", name: "fix" }] }, + { role: "tool", content: "patched" }, + ])).toBe(true); + expect(turnHasToolWork([ + { role: "user", content: "继续" }, + { role: "toolResult", content: [{ type: "text", text: "done" }] }, + ])).toBe(true); + }); + + it("纯 user/assistant 对话 → false", () => { + expect(turnHasToolWork([ + { role: "user", content: "继续" }, + { role: "assistant", content: "好的" }, + ])).toBe(false); + expect(turnHasToolWork([])).toBe(false); + }); + + it("畸形消息不炸", () => { + expect(turnHasToolWork([null, undefined, "str", 42, { role: 123 }, {}])).toBe(false); + }); + + it("组合语义：user 文本命中词表 + 轮内有工具劳动 → 整轮照常提取", () => { + const messages = [ + { role: "user", content: "继续" }, + { role: "assistant", content: [{ type: "toolUse", id: "t1", name: "bash" }] }, + { role: "toolResult", content: "bug fixed, tests green" }, + ]; + // 单独看 user 文本命中词表…… + expect(shouldSkipTurnExtraction(turnUserText(messages))).toBe(true); + // ……但轮内有工具劳动，预筛必须让路 + expect(turnHasToolWork(messages)).toBe(true); + }); +}); diff --git a/test/update-node.test.ts b/test/update-node.test.ts index 6276c6f..71819cf 100644 --- a/test/update-node.test.ts +++ b/test/update-node.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { applyNodePatch, applyDeprecateMarker } from "../src/store/store.ts"; +import { applyNodePatch, applyDeprecateMarker, stripDeprecateMarker } from "../src/store/store.ts"; import type { GmNode } from "../src/types.ts"; const baseNode: Pick = { @@ -64,3 +64,28 @@ describe("applyDeprecateMarker (gm_update mode=deprecate)", () => { expect(result.endsWith(long)).toBe(true); }); }); + +describe("stripDeprecateMarker (decay 自动弃用节点复活时还原描述)", () => { + it("剥掉 [DEPRECATED] 前缀，还原原文", () => { + expect(stripDeprecateMarker("[DEPRECATED] 处理 PDF 提取")).toBe("处理 PDF 提取"); + }); + + it("空描述的标记 [DEPRECATED]（无尾随空格）还原为空串", () => { + expect(stripDeprecateMarker("[DEPRECATED]")).toBe(""); + }); + + it("无前缀的描述原样返回", () => { + expect(stripDeprecateMarker("普通描述")).toBe("普通描述"); + expect(stripDeprecateMarker("")).toBe(""); + }); + + it("描述中间出现 [DEPRECATED] 子串时只剥前缀", () => { + expect(stripDeprecateMarker("[DEPRECATED] 讨论了 [DEPRECATED] 标记的用法")) + .toBe("讨论了 [DEPRECATED] 标记的用法"); + }); + + it("与 applyDeprecateMarker 互逆（非空描述）", () => { + const desc = "与遗忘曲线联动的生命周期"; + expect(stripDeprecateMarker(applyDeprecateMarker(desc))).toBe(desc); + }); +});